ES时间范围查询案例,并分页查询

 public PageInfo<BTCarGetoutLogs> queryPage(QueryVo queryParams, ProcessForm processForm, String tableName) {
        int pageNum = queryParams.getPageNum().intValue();
        int pageSize = queryParams.getPageSize().intValue();
        List<BTCarGetoutLogs> result = Lists.newArrayList();
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
        //并添加了多个查询条件。must 表示必须匹配的条件,mustNot 表示禁止匹配的条件,should 表示应该匹配的条件。你可以根据需要自由组合这些查询条件。
        for (ConditionVo queryParam : queryParams.getQueryParams()) {
            if (com.jinw.utils.StringUtils.isNotBlank(queryParam.getFieldName()) && queryParam.getFieldName().equals("PLATE_NO") &&
                    ObjectUtil.isNotEmpty(queryParam.getFieldValue())) {
                MatchPhrasePrefixQueryBuilder matchQueryBuilder1 = QueryBuilders.matchPhrasePrefixQuery("plateNo", queryParam.getFieldValue());
                // 添加到布尔查询的 should 子句中
                boolQuery.must(matchQueryBuilder1);
                break;
            }
        }
        for (ConditionVo queryParam : queryParams.getQueryParams()) {
            if (com.jinw.utils.StringUtils.isNotBlank(queryParam.getFieldName()) && queryParam.getFieldName().equals("CROSS_TIME") &&
                    ObjectUtil.isNotEmpty(queryParam.getFieldValue())) {
                WildcardQueryBuilder wildcardQuery = QueryBuilders.wildcardQuery("crossTime", "*" + queryParam.getFieldValue().toString() + "*");                // 添加到布尔查询的 should 子句中
                boolQuery.must(wildcardQuery);
                break;
            }
        }
        sourceBuilder.query(boolQuery);
        Long count = ESUtils.selectByCount("b_t_car_getout_logs", sourceBuilder);

        // 设置排序规则
        sourceBuilder.sort(SortBuilders.fieldSort("crossTime").order(SortOrder.DESC));
        List<Map<String, Object>> list = ESUtils.selectByPage("b_t_car_getout_logs", sourceBuilder, pageNum, pageSize);

        for (Map<String, Object> map : list) {
            Map<String, Object> filteredMap = map.entrySet().stream()
                    .filter(entry -> entry.getValue() != null && StringUtils.isNotBlank(entry.getValue().toString()) && !entry.getValue().equals("null"))
                    .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
            BTCarGetoutLogs peopleInoutLogs = BeanUtil.mapToBean(filteredMap, BTCarGetoutLogs.class, false, CopyOptions.create().ignoreNullValue());
            result.add(peopleInoutLogs);
        }
        int totalPage = (int) Math.ceil((double) count / pageSize);
        PageInfo<BTCarGetoutLogs> pageInfo = new PageInfo<>(pageNum, pageSize, totalPage);
        pageInfo.setRecords(result);
        return pageInfo;
    }

使用DISTINCT关键字后仍然存在相同的ID返回

如果使用DISTINCT关键字后仍然存在相同的ID返回,有几种可能的原因:

数据不同: 即使ID相同,其他列的数据可能不同,因此在结果集中仍然会出现相同的ID。

NULL 值: 如果 ID 列包含 NULL 值,DISTINCT 关键字会将它们视为不同的值,因此可能会导致相同的 ID 出现在结果集中。

数据类型不匹配: 如果ID列是一个字符串类型的列,并且数据在存储时包含了空格或者大小写不同,那么在使用DISTINCT关键字时,这些看似相同的ID会被视为不同的值。

使用联合查询: 如果在查询中使用了联合查询(如UNION),那么DISTINCT将会对整个结果集去重,而不是单独对某一列去重,这可能导致看似相同的ID出现在结果中。

es 字符串时间格式的字符串怎么做小时查询

import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;

// 创建一个脚本,用于解析时间字符串并检查小时是否等于指定的小时
Script script = new Script(
    ScriptType.INLINE,
    "painless",
    "def formatter = DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss');\n" +
    "def dateTime = LocalDateTime.parse(doc['eventTime'].value, formatter);\n" +
    "if (dateTime == null) {\n" +
    "    logger.error('Failed to parse eventTime value: ' + doc['eventTime'].value);\n" +
    "}\n" +
    "return dateTime.getHour() == Integer.valueOf(params.hour);",
    Collections.singletonMap("hour", "00") // 设置参数值为字符串类型的 "00"
);

// 创建一个脚本查询,匹配指定小时的数据
QueryBuilder scriptQuery = QueryBuilders.scriptQuery(script);

// 将脚本查询与其他条件组合
QueryBuilder finalQuery = QueryBuilders.boolQuery()
    .must(QueryBuilders.termQuery("in_out_type", "1")) // 添加其他条件,例如匹配in_out_type为1的数据
    .must(scriptQuery);

// 现在你可以使用finalQuery来执行你的Elasticsearch查询

怎么把达梦数据库文件sql文件导入es中

我是把达梦数据库文件导出到sql文件中
然后再解析sql文件中分批导入到es中

package com.jinw.es;

import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import cn.hutool.core.io.FileUtil;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport;
import com.github.xiaoymin.knife4j.annotations.ApiSort;
import com.google.common.collect.Lists;
import com.jinw.activity.service.IProcessFormService;
import com.jinw.base.model.PageInfo;
import com.jinw.base.model.ProcessForm;
import com.jinw.base.model.QueryVo;
import com.jinw.base.model.RestResult;
import com.jinw.modules.applyCar.domain.ApplyCarParameterVo;
import com.jinw.modules.business.service.IFlowCommonService;
import com.jinw.modules.car.domain.UserCarVo;
import com.jinw.modules.card.domain.UserCardVo;
import com.jinw.modules.excelLogs.domain.IssueVo;
import com.jinw.utils.StringUtils;
import com.jinw.utils.poi.ExcelUtil;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.context.annotation.Lazy;
import com.jinw.base.controller.BaseCommonController;
import com.jinw.modules.excelLogs.domain.BTExcelLogs;
import com.jinw.modules.excelLogs.service.IBTExcelLogsService;
import org.springframework.web.multipart.MultipartFile;

import javax.servlet.http.HttpServletRequest;

/**
 * @author liuxin
 * @Description:
 * @date 2024-04-08
 */

@RestController
@RequestMapping("/es")
@Api(tags = "es")
@Slf4j
public class EsController {

    @Autowired
    private EsDataOperation esDataOperation;
    @Autowired
    private EsIndexOperation esIndexOperation;
    @Autowired
    private EsQueryOperation esQueryOperation;

    @PostMapping("createIndex")
    public void createIndex(String indexName, String filePath) {
  
        boolean b = esIndexOperation.createIndex(indexName, filePath);
        Assert.isTrue(b);
    }

    @PostMapping("insert")
    public void insert(String indexName, String filePath) throws IOException {
      
        parseSqlFile(filePath, indexName);
    }

    public void parseSqlFile(String filePath, String indexName) {
        int batchSize = 100000; // 设置批次大小
        List<Map<String, Object>> dataList = null;
        try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
            int totalLines = FileUtil.getTotalLines(FileUtil.file(filePath)); // 获取总行数
            System.out.println("Total lines in SQL file: " + totalLines);
            int batchCount = (int) Math.ceil((double) totalLines / batchSize); // 计算批次数量
            System.out.println("Total batches to execute: " + batchCount);
            // 循环读取数据
            for (int batchIndex = 0; batchIndex < batchCount; batchIndex++) {
                int linesRead = 0;
                dataList = Lists.newArrayList();
                System.out.println("Batch " + (batchIndex + 1) + "/" + batchCount + ":");

                // 读取每个批次的数据
                String line;
                while ((line = reader.readLine()) != null && linesRead < batchSize) {
                    if (!line.trim().isEmpty()) { // 判断是否为空行
                        linesRead++;
                        // 在这里处理每一行的数据,例如执行 SQL 操作等
                        // System.out.println("line:" + line);
                        Map<String, Object> map = parseInsertStatement(line);
                        dataList.add(map);
                    }
                }
                if (ObjectUtil.isNotEmpty(dataList)) {
                    boolean flag = esDataOperation.batchInsert(indexName, dataList);
                    System.out.println("flag:" + flag);
                }
                System.out.println("Batch " + (batchIndex + 1) + " processed. Lines read: " + linesRead);
            }
            System.out.println("Total lines read: " + totalLines);
        } catch (IOException e) {
            e.printStackTrace();
        }
    }

    private static Map<String, Object> parseInsertStatement(String line) {
        line = line.replaceAll("INSERT INTO \"xxx\".\"xx_INOUT_LOGS\"\\(", "")
                .replaceAll("\\)", "").replaceAll(";", "");
        line = line.replaceAll("INSERT INTO \"xxx\".\"xx_GETOUT_LOGS\"\\(", "")
                .replaceAll("\\)", "").replaceAll(";", "");
        String[] values = line.split("VALUES\\(");
        Map<String, Object> dataMap = new HashMap<>();
        String[] split = values[0].split(",");
        String[] split1 = values[1].split(",");
        for (int i = 0; i < split.length; i++) {
            String stringWithoutQuotes = StrUtil.toCamelCase(split[i].toLowerCase().replaceAll("\"", ""));
            String stringWithoutQuotes2 = split1[i].replaceAll("\"", "").replaceAll("'", "");
            dataMap.put(stringWithoutQuotes, stringWithoutQuotes2);
        }
        return dataMap;
    }

    @PostMapping("update")
    public void update() {
        String indexName = "b_t_people_inout_logs";
        HashMap<String, Object> hashMap = new HashMap<>();
        hashMap.put("id", 5);
        hashMap.put("name", "jack7");
        boolean update = esDataOperation.update(indexName, hashMap, WriteRequest.RefreshPolicy.WAIT_UNTIL);
        Assert.isTrue(update);
    }

    /**
     * 查询总数
     */
    @GetMapping("count")
    public Long count(String indexName) {
        return esQueryOperation.count(indexName);
    }

    @GetMapping("list")
    public List<Map<String, Object>> list(String indexName) {
        // 查询条件
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();

        //精确查询
        //  BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
        //  queryBuilder.must(QueryBuilders.termQuery("personName", "xxx"));

        //模糊查询
        //  MatchPhrasePrefixQueryBuilder matchQueryBuilder = QueryBuilders.matchPhrasePrefixQuery("personName", "杨");


        // 设置模糊查询
        //  sourceBuilder.query(QueryBuilders.fuzzyQuery("personName", "马中明"));

        // 设置排序规则
        //sourceBuilder.sort(SortBuilders.fieldSort("eventTime").order(SortOrder.DESC));
        // 设置超时时间
        sourceBuilder.timeout(TimeValue.timeValueSeconds(10));

        // 分页查询
        sourceBuilder.from(0);
        sourceBuilder.size(10);
        List<Map<String, Object>> list = esQueryOperation.list(indexName, sourceBuilder);
        return list;
    }
}

另外如果要排序的话要把字段类型 设置为keyword
字段映射配置:

{
    "aliases": {
        "btcgl": {}
    },
    "mappings": {
        "properties": {
            "userId": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "quartersId": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "carName": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "quartersName": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "crossTime": {
                "type": "keyword",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "plateNo": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "crossRecordSyscode": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "parkSyscode": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "parkName": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "entranceSyscode": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "entranceName": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "roadwaySyscode": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "roadwayName": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "vehicleOut": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "cardNo": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "releaseMode": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "releaseResult": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "releaseWay": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "releaseReason": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "vehicleColor": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "vehicleType": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "plateColor": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "plateType": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "carCategory": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "carCategoryName": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            },
            "orgId": {
                "type": "text",
                "fields": {
                    "keyword": {
                        "type": "keyword",
                        "ignore_above": 256
                    }
                }
            }
        }
    },
    "settings": {
        "index": {
            "number_of_shards": "1",
            "number_of_replicas": "1"
        }
    }
}
package com.jinw.modules.utils;

import cn.hutool.core.util.ObjectUtil;
import com.google.common.collect.Lists;
import com.jinw.es.EsDataOperation;
import com.jinw.es.EsQueryOperation;
import com.jinw.utils.spring.SpringUtils;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import java.util.List;
import java.util.Map;

/**
 * @author @liuxin
 * @classname ESUtils
 * @date 2024/4/9 8:40
 * @description TODO
 */
@Component
public class ESUtils {
    private static EsQueryOperation esQueryOperation = SpringUtils.getBean(EsQueryOperation.class);
    private static EsDataOperation esDataOperation = SpringUtils.getBean(EsDataOperation.class);

    public static List<Map<String, Object>> selectByPage(String indexName, SearchSourceBuilder sourceBuilder, int pageNum, int pageSize) {
        // 设置超时时间
        sourceBuilder.timeout(TimeValue.timeValueSeconds(10));
        // 分页查询
        sourceBuilder.from((pageNum - 1) * pageSize);
        sourceBuilder.size(pageSize);
        sourceBuilder.trackTotalHits(true);
        List<Map<String, Object>> result = esQueryOperation.list(indexName, sourceBuilder);
        return ObjectUtil.isNotEmpty(result) ? result : Lists.newArrayList();
    }

    public static Long selectByCount(String indexName, SearchSourceBuilder sourceBuilder) {
        return esQueryOperation.countByCondition(indexName, sourceBuilder);
    }


    public static Map<String, Object> selectOneById(
            String indexName, String recordId) {
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
        TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("id", recordId);
        // 添加到布尔查询的 should 子句中
        sourceBuilder.query(termQueryBuilder);
        List<Map<String, Object>> list = esQueryOperation.list(indexName, sourceBuilder);
        Map<String, Object> entity = null;
        if (ObjectUtil.isNotEmpty(list)) {
            entity = list.get(0);
        }
        return entity;
    }

    public static boolean deleteById(
            String indexName, String recordId) {
        return esDataOperation.delete(indexName, recordId);
    }


    public static boolean insert(String indexName, Map<String, Object> dataMap) {
        return esDataOperation.insert(indexName, dataMap);
    }
}
package com.jinw.es.config;

import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;

import java.util.ArrayList;
import java.util.List;

/**
 * restHighLevelClient 客户端配置类
 */
@Slf4j
@Data
@Configuration
@ConfigurationProperties(prefix = "elasticsearch")
public class ElasticsearchConfig {

    /**
     * es host ip 地址(集群)
     */
    private String hosts;
    /**
     * es用户名
     */
    private String userName;
    /**
     * es密码
     */
    private String password;
    /**
     * es 请求方式
     */
    private String scheme;
    /**
     * es 连接超时时间
     */
    private int connectTimeOut;
    /**
     * es socket 连接超时时间
     */
    private int socketTimeOut;
    /**
     * es 请求超时时间
     */
    private int connectionRequestTimeOut;
    /**
     * es 最大连接数
     */
    private int maxConnectNum;
    /**
     * es 每个路由的最大连接数
     */
    private int maxConnectNumPerRoute;


    /**
     * 如果@Bean没有指定bean的名称,那么方法名就是bean的名称
     */
    @Bean(name = "restHighLevelClient")
    public RestHighLevelClient restHighLevelClient() {
        // 构建连接对象
        RestClientBuilder builder = RestClient.builder(getEsHost());

        // 连接延时配置
        builder.setRequestConfigCallback(requestConfigBuilder -> {
            requestConfigBuilder.setConnectTimeout(connectTimeOut);
            requestConfigBuilder.setSocketTimeout(socketTimeOut);
            requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeOut);
            return requestConfigBuilder;
        });

        // 连接数配置
        builder.setHttpClientConfigCallback(httpClientBuilder -> {
            httpClientBuilder.setMaxConnTotal(maxConnectNum);
            httpClientBuilder.setMaxConnPerRoute(maxConnectNumPerRoute);
            httpClientBuilder.setDefaultCredentialsProvider(getCredentialsProvider());
            return httpClientBuilder;
        });

        return new RestHighLevelClient(builder);
    }

    private HttpHost[] getEsHost() {
        // 拆分地址(es为多节点时,不同host以逗号间隔)
        List<HttpHost> hostLists = new ArrayList<>();
        String[] hostList = hosts.split(",");
        for (String addr : hostList) {
            String host = addr.split(":")[0];
            String port = addr.split(":")[1];
            hostLists.add(new HttpHost(host, Integer.parseInt(port), scheme));
        }
        // 转换成 HttpHost 数组
        return hostLists.toArray(new HttpHost[]{});
    }

    private CredentialsProvider getCredentialsProvider() {
        // 设置用户名、密码
        CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
        credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
        return credentialsProvider;
    }
}
package com.jinw.es;

import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.xcontent.XContentType;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.IOException;
import java.util.List;
import java.util.Map;

/**
 * 增删改数据
 */
@Slf4j
@Service
public class EsDataOperation {

    @Resource
    private RestHighLevelClient client;
    private final RequestOptions options = RequestOptions.DEFAULT;

    /**
     * 写入数据
     */
    public boolean insert(String indexName, Map<String, Object> dataMap) {
        try {
            BulkRequest request = new BulkRequest();
            request.add(new IndexRequest(indexName).opType("create")
                    .id(dataMap.get("id").toString())
                    .source(dataMap, XContentType.JSON));
            this.client.bulk(request, options);
            return Boolean.TRUE;
        } catch (Exception e) {
            log.error("EsDataOperation insert error.", e);
        }
        return Boolean.FALSE;
    }

    /**
     * 批量写入数据
     */
    public boolean batchInsert(String indexName, List<Map<String, Object>> userIndexList) {
        try {
            BulkRequest request = new BulkRequest();
            for (Map<String, Object> dataMap : userIndexList) {
                request.add(new IndexRequest(indexName).opType("create")
                        .id(dataMap.get("id").toString())
                        .source(dataMap, XContentType.JSON));
            }
            this.client.bulk(request, options);
            return Boolean.TRUE;
        } catch (Exception e) {
            log.error("EsDataOperation batchInsert error.", e);
        }
        return Boolean.FALSE;
    }

    /**
     * 根据id更新数据,可以直接修改索引结构
     *
     * @param refreshPolicy 数据刷新策略
     */
    public boolean update(String indexName, Map<String, Object> dataMap, WriteRequest.RefreshPolicy refreshPolicy) {
        try {
            UpdateRequest updateRequest = new UpdateRequest(indexName, dataMap.get("id").toString());
            updateRequest.setRefreshPolicy(refreshPolicy);
            updateRequest.doc(dataMap);
            this.client.update(updateRequest, options);
            return Boolean.TRUE;
        } catch (Exception e) {
            log.error("EsDataOperation update error.", e);
        }
        return Boolean.FALSE;
    }

    /**
     * 删除数据
     */
    public boolean delete(String indexName, String id) {
        try {
            DeleteRequest deleteRequest = new DeleteRequest(indexName, id);
            this.client.delete(deleteRequest, options);
            return Boolean.TRUE;
        } catch (Exception e) {
            log.error("EsDataOperation delete error.", e);
        }
        return Boolean.FALSE;
    }


    /**
     * 批量删除数据
     */
    public boolean batchDelete(String indexName, String fieldName, String fieldValue) {
        try {
            // 设置滚动参数
            Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));

            // 构建初始查询请求
            SearchRequest searchRequest = new SearchRequest(indexName);
            searchRequest.scroll(scroll);

            // 构建查询条件
            SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
            sourceBuilder.query(QueryBuilders.rangeQuery(fieldName).lt(fieldValue));
            searchRequest.source(sourceBuilder);

            // 执行初始查询请求
            SearchResponse searchResponse = client.search(searchRequest, options);

            // 获取滚动ID
            String scrollId = searchResponse.getScrollId();

            // 构建批量删除请求
            BulkRequest bulkRequest = new BulkRequest();

            // 循环获取滚动结果并添加到批量删除请求中
            while (true) {
                for (SearchHit hit : searchResponse.getHits().getHits()) {
                    bulkRequest.add(new DeleteRequest(indexName, hit.getId()));
                }

                // 执行批量删除请求
                BulkResponse bulkResponse = client.bulk(bulkRequest, options);

                // 检查是否还有滚动结果
                if (bulkResponse.getItems().length == 0) {
                    break;
                }

                // 继续滚动搜索
                SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
                scrollRequest.scroll(scroll);
                searchResponse = client.scroll(scrollRequest, options);
            }

            // 清除滚动ID
            ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
            clearScrollRequest.addScrollId(scrollId);
            client.clearScroll(clearScrollRequest, options);
            return Boolean.TRUE;
        } catch (Exception e) {
            log.error("EsDataOperation delete error.", e);
        }
        return Boolean.FALSE;
    }
}
package com.jinw.es;

import cn.hutool.core.io.FileUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.xcontent.XContentType;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.io.IOException;

/**
 * 操作ES索引
 */
@Slf4j
@Service
public class EsIndexOperation {

    @Resource
    private RestHighLevelClient restHighLevelClient;
    private final RequestOptions options = RequestOptions.DEFAULT;

    /**
     * 判断索引是否存在
     */
    public boolean checkIndex(String index) {
        try {
            return restHighLevelClient.indices().exists(new GetIndexRequest(index), options);
        } catch (Exception e) {
            log.error("EsIndexOperation checkIndex error.", e);
        }
        return Boolean.FALSE;
    }

    /**
     * 创建索引
     *
     * @param indexName         es索引名
     * @param esSettingFilePath es索引的alias、settings和mapping的配置文件
     */
    public boolean createIndex(String indexName, String esSettingFilePath) {
        String aliases = null;
        String mappings = null;
        String settings = null;
        if (StringUtils.isNotBlank(esSettingFilePath)) {
            try {
                String fileContent = FileUtil.readUtf8String(esSettingFilePath);
                if (StringUtils.isNotBlank(fileContent)) {
                    JSONObject jsonObject = JSON.parseObject(fileContent);
                    aliases = jsonObject.getString("aliases");
                    mappings = jsonObject.getString("mappings");
                    settings = jsonObject.getString("settings");
                }
            } catch (Exception e) {
                log.error("createIndex error.", e);
                return false;
            }
        }

        if (checkIndex(indexName)) {
            log.error("createIndex indexName:[{}]已存在", indexName);
            return false;
        }

        CreateIndexRequest request = new CreateIndexRequest(indexName);
        if ((StringUtils.isNotBlank(aliases))) {
            request.aliases(aliases, XContentType.JSON);
        }

        if (StringUtils.isNotBlank(mappings)) {
            request.mapping(mappings, XContentType.JSON);
        }

        if (StringUtils.isNotBlank(settings)) {
            request.settings(settings, XContentType.JSON);
        }

        try {
            this.restHighLevelClient.indices().create(request, options);
            return true;
        } catch (IOException e) {
            log.error("EsIndexOperation createIndex error.", e);
            return false;
        }
    }

    /**
     * 删除索引
     */
    public boolean deleteIndex(String indexName) {
        try {
            if (checkIndex(indexName)) {
                DeleteIndexRequest request = new DeleteIndexRequest(indexName);
                AcknowledgedResponse response = restHighLevelClient.indices().delete(request, options);
                return response.isAcknowledged();
            }
        } catch (Exception e) {
            log.error("EsIndexOperation deleteIndex error.", e);
        }
        return Boolean.FALSE;
    }
}
package com.jinw.es;

import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.stereotype.Service;

import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;

/**
 * 查询操作
 */
@Slf4j
@Service
public class EsQueryOperation {

    @Resource
    private RestHighLevelClient client;
    private final RequestOptions options = RequestOptions.DEFAULT;

    /**
     * 查询总数
     */
    public Long count(String indexName) {
        CountRequest countRequest = new CountRequest(indexName);
        try {
            CountResponse countResponse = client.count(countRequest, options);
            return countResponse.getCount();
        } catch (Exception e) {
            log.error("EsQueryOperation count error.", e);
        }
        return 0L;
    }


    /**
     * 查询总数
     */
    public Long countByCondition(String indexName, SearchSourceBuilder sourceBuilder) {
        CountRequest countRequest = new CountRequest(indexName);
        countRequest.source(sourceBuilder);
        try {
            CountResponse countResponse = client.count(countRequest, options);
            return countResponse.getCount();
        } catch (Exception e) {
            log.error("EsQueryOperation count error.", e);
        }
        return 0L;
    }

    /**
     * 查询数据集
     */
    public List<Map<String, Object>> list(String indexName, SearchSourceBuilder sourceBuilder) {
        SearchRequest searchRequest = new SearchRequest(indexName);
        searchRequest.source(sourceBuilder);
        try {
            SearchResponse searchResp = client.search(searchRequest, options);
            List<Map<String, Object>> data = new ArrayList<>();
            SearchHit[] searchHitArr = searchResp.getHits().getHits();
            for (SearchHit searchHit : searchHitArr) {
                Map<String, Object> temp = searchHit.getSourceAsMap();
                temp.put("id", searchHit.getId());
                data.add(temp);
            }
            return data;
        } catch (Exception e) {
            log.error("EsQueryOperation list error.", e);
        }
        return null;
    }


}

SearchSourceBuilder的from和size

SearchSourceBuilder是Elasticsearch Java客户端中用于构建搜索请求的类之一。它提供了一种方式来构建查询,指定搜索结果的大小、偏移量等。

在SearchSourceBuilder中,from和size是两个重要的参数,用于控制搜索结果的偏移量和大小。

from:指定搜索结果的偏移量,即从搜索结果中的第几条开始返回结果。默认为0,表示从第一条开始。
size:指定返回的搜索结果的数量。默认情况下,Elasticsearch每次最多返回10条结果。可以通过size参数来修改返回结果的数量。


import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;

public class SearchExample {

    public void searchDocuments(RestHighLevelClient client) throws Exception {
        SearchRequest searchRequest = new SearchRequest("index_name");
        SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();

        // 设置查询条件
        sourceBuilder.query(QueryBuilders.matchAllQuery());

        // 设置排序
        sourceBuilder.sort("field_name", SortOrder.DESC);

        // 设置偏移量和返回结果数量
        sourceBuilder.from(0); // 从第一条开始
        sourceBuilder.size(10); // 返回10条结果

        searchRequest.source(sourceBuilder);

        client.search(searchRequest, RequestOptions.DEFAULT);
    }
}