public PageInfo<BTCarGetoutLogs> queryPage(QueryVo queryParams, ProcessForm processForm, String tableName) {
int pageNum = queryParams.getPageNum().intValue();
int pageSize = queryParams.getPageSize().intValue();
List<BTCarGetoutLogs> result = Lists.newArrayList();
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
//并添加了多个查询条件。must 表示必须匹配的条件，mustNot 表示禁止匹配的条件，should 表示应该匹配的条件。你可以根据需要自由组合这些查询条件。
for (ConditionVo queryParam : queryParams.getQueryParams()) {
if (com.jinw.utils.StringUtils.isNotBlank(queryParam.getFieldName()) && queryParam.getFieldName().equals("PLATE_NO") &&
ObjectUtil.isNotEmpty(queryParam.getFieldValue())) {
MatchPhrasePrefixQueryBuilder matchQueryBuilder1 = QueryBuilders.matchPhrasePrefixQuery("plateNo", queryParam.getFieldValue());
// 添加到布尔查询的 should 子句中
boolQuery.must(matchQueryBuilder1);
break;
}
}
for (ConditionVo queryParam : queryParams.getQueryParams()) {
if (com.jinw.utils.StringUtils.isNotBlank(queryParam.getFieldName()) && queryParam.getFieldName().equals("CROSS_TIME") &&
ObjectUtil.isNotEmpty(queryParam.getFieldValue())) {
WildcardQueryBuilder wildcardQuery = QueryBuilders.wildcardQuery("crossTime", "*" + queryParam.getFieldValue().toString() + "*"); // 添加到布尔查询的 should 子句中
boolQuery.must(wildcardQuery);
break;
}
}
sourceBuilder.query(boolQuery);
Long count = ESUtils.selectByCount("b_t_car_getout_logs", sourceBuilder);
// 设置排序规则
sourceBuilder.sort(SortBuilders.fieldSort("crossTime").order(SortOrder.DESC));
List<Map<String, Object>> list = ESUtils.selectByPage("b_t_car_getout_logs", sourceBuilder, pageNum, pageSize);
for (Map<String, Object> map : list) {
Map<String, Object> filteredMap = map.entrySet().stream()
.filter(entry -> entry.getValue() != null && StringUtils.isNotBlank(entry.getValue().toString()) && !entry.getValue().equals("null"))
.collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
BTCarGetoutLogs peopleInoutLogs = BeanUtil.mapToBean(filteredMap, BTCarGetoutLogs.class, false, CopyOptions.create().ignoreNullValue());
result.add(peopleInoutLogs);
}
int totalPage = (int) Math.ceil((double) count / pageSize);
PageInfo<BTCarGetoutLogs> pageInfo = new PageInfo<>(pageNum, pageSize, totalPage);
pageInfo.setRecords(result);
return pageInfo;
}如果使用DISTINCT关键字后仍然存在相同的ID返回，有几种可能的原因：
数据不同： 即使ID相同，其他列的数据可能不同，因此在结果集中仍然会出现相同的ID。
NULL 值： 如果 ID 列包含 NULL 值，DISTINCT 关键字会将它们视为不同的值，因此可能会导致相同的 ID 出现在结果集中。
数据类型不匹配： 如果ID列是一个字符串类型的列，并且数据在存储时包含了空格或者大小写不同，那么在使用DISTINCT关键字时，这些看似相同的ID会被视为不同的值。
使用联合查询： 如果在查询中使用了联合查询（如UNION），那么DISTINCT将会对整个结果集去重，而不是单独对某一列去重，这可能导致看似相同的ID出现在结果中。
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.script.Script;
import org.elasticsearch.script.ScriptType;
import java.time.LocalDateTime;
import java.time.format.DateTimeFormatter;
import java.util.Collections;
// 创建一个脚本，用于解析时间字符串并检查小时是否等于指定的小时
Script script = new Script(
ScriptType.INLINE,
"painless",
"def formatter = DateTimeFormatter.ofPattern('yyyy-MM-dd HH:mm:ss');\n" +
"def dateTime = LocalDateTime.parse(doc['eventTime'].value, formatter);\n" +
"if (dateTime == null) {\n" +
" logger.error('Failed to parse eventTime value: ' + doc['eventTime'].value);\n" +
"}\n" +
"return dateTime.getHour() == Integer.valueOf(params.hour);",
Collections.singletonMap("hour", "00") // 设置参数值为字符串类型的 "00"
);
// 创建一个脚本查询，匹配指定小时的数据
QueryBuilder scriptQuery = QueryBuilders.scriptQuery(script);
// 将脚本查询与其他条件组合
QueryBuilder finalQuery = QueryBuilders.boolQuery()
.must(QueryBuilders.termQuery("in_out_type", "1")) // 添加其他条件，例如匹配in_out_type为1的数据
.must(scriptQuery);
// 现在你可以使用finalQuery来执行你的Elasticsearch查询
我是把达梦数据库文件导出到sql文件中
然后再解析sql文件中分批导入到es中
package com.jinw.es;
import java.io.BufferedReader;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.lang.Assert;
import cn.hutool.core.util.ObjectUtil;
import cn.hutool.core.util.StrUtil;
import com.github.xiaoymin.knife4j.annotations.ApiOperationSupport;
import com.github.xiaoymin.knife4j.annotations.ApiSort;
import com.google.common.collect.Lists;
import com.jinw.activity.service.IProcessFormService;
import com.jinw.base.model.PageInfo;
import com.jinw.base.model.ProcessForm;
import com.jinw.base.model.QueryVo;
import com.jinw.base.model.RestResult;
import com.jinw.modules.applyCar.domain.ApplyCarParameterVo;
import com.jinw.modules.business.service.IFlowCommonService;
import com.jinw.modules.car.domain.UserCarVo;
import com.jinw.modules.card.domain.UserCardVo;
import com.jinw.modules.excelLogs.domain.IssueVo;
import com.jinw.utils.StringUtils;
import com.jinw.utils.poi.ExcelUtil;
import io.swagger.annotations.Api;
import io.swagger.annotations.ApiImplicitParam;
import io.swagger.annotations.ApiImplicitParams;
import io.swagger.annotations.ApiOperation;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.MatchPhrasePrefixQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.security.access.prepost.PreAuthorize;
import org.springframework.web.bind.annotation.*;
import org.springframework.context.annotation.Lazy;
import com.jinw.base.controller.BaseCommonController;
import com.jinw.modules.excelLogs.domain.BTExcelLogs;
import com.jinw.modules.excelLogs.service.IBTExcelLogsService;
import org.springframework.web.multipart.MultipartFile;
import javax.servlet.http.HttpServletRequest;
/**
* @author liuxin
* @Description:
* @date 2024-04-08
*/
@RestController
@RequestMapping("/es")
@Api(tags = "es")
@Slf4j
public class EsController {
@Autowired
private EsDataOperation esDataOperation;
@Autowired
private EsIndexOperation esIndexOperation;
@Autowired
private EsQueryOperation esQueryOperation;
@PostMapping("createIndex")
public void createIndex(String indexName, String filePath) {
boolean b = esIndexOperation.createIndex(indexName, filePath);
Assert.isTrue(b);
}
@PostMapping("insert")
public void insert(String indexName, String filePath) throws IOException {
parseSqlFile(filePath, indexName);
}
public void parseSqlFile(String filePath, String indexName) {
int batchSize = 100000; // 设置批次大小
List<Map<String, Object>> dataList = null;
try (BufferedReader reader = new BufferedReader(new FileReader(filePath))) {
int totalLines = FileUtil.getTotalLines(FileUtil.file(filePath)); // 获取总行数
System.out.println("Total lines in SQL file: " + totalLines);
int batchCount = (int) Math.ceil((double) totalLines / batchSize); // 计算批次数量
System.out.println("Total batches to execute: " + batchCount);
// 循环读取数据
for (int batchIndex = 0; batchIndex < batchCount; batchIndex++) {
int linesRead = 0;
dataList = Lists.newArrayList();
System.out.println("Batch " + (batchIndex + 1) + "/" + batchCount + ":");
// 读取每个批次的数据
String line;
while ((line = reader.readLine()) != null && linesRead < batchSize) {
if (!line.trim().isEmpty()) { // 判断是否为空行
linesRead++;
// 在这里处理每一行的数据，例如执行 SQL 操作等
// System.out.println("line:" + line);
Map<String, Object> map = parseInsertStatement(line);
dataList.add(map);
}
}
if (ObjectUtil.isNotEmpty(dataList)) {
boolean flag = esDataOperation.batchInsert(indexName, dataList);
System.out.println("flag:" + flag);
}
System.out.println("Batch " + (batchIndex + 1) + " processed. Lines read: " + linesRead);
}
System.out.println("Total lines read: " + totalLines);
} catch (IOException e) {
e.printStackTrace();
}
}
private static Map<String, Object> parseInsertStatement(String line) {
line = line.replaceAll("INSERT INTO \"xxx\".\"xx_INOUT_LOGS\"\\(", "")
.replaceAll("\\)", "").replaceAll(";", "");
line = line.replaceAll("INSERT INTO \"xxx\".\"xx_GETOUT_LOGS\"\\(", "")
.replaceAll("\\)", "").replaceAll(";", "");
String[] values = line.split("VALUES\\(");
Map<String, Object> dataMap = new HashMap<>();
String[] split = values[0].split(",");
String[] split1 = values[1].split(",");
for (int i = 0; i < split.length; i++) {
String stringWithoutQuotes = StrUtil.toCamelCase(split[i].toLowerCase().replaceAll("\"", ""));
String stringWithoutQuotes2 = split1[i].replaceAll("\"", "").replaceAll("'", "");
dataMap.put(stringWithoutQuotes, stringWithoutQuotes2);
}
return dataMap;
}
@PostMapping("update")
public void update() {
String indexName = "b_t_people_inout_logs";
HashMap<String, Object> hashMap = new HashMap<>();
hashMap.put("id", 5);
hashMap.put("name", "jack7");
boolean update = esDataOperation.update(indexName, hashMap, WriteRequest.RefreshPolicy.WAIT_UNTIL);
Assert.isTrue(update);
}
/**
* 查询总数
*/
@GetMapping("count")
public Long count(String indexName) {
return esQueryOperation.count(indexName);
}
@GetMapping("list")
public List<Map<String, Object>> list(String indexName) {
// 查询条件
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
//精确查询
// BoolQueryBuilder queryBuilder = QueryBuilders.boolQuery();
// queryBuilder.must(QueryBuilders.termQuery("personName", "xxx"));
//模糊查询
// MatchPhrasePrefixQueryBuilder matchQueryBuilder = QueryBuilders.matchPhrasePrefixQuery("personName", "杨");
// 设置模糊查询
// sourceBuilder.query(QueryBuilders.fuzzyQuery("personName", "马中明"));
// 设置排序规则
//sourceBuilder.sort(SortBuilders.fieldSort("eventTime").order(SortOrder.DESC));
// 设置超时时间
sourceBuilder.timeout(TimeValue.timeValueSeconds(10));
// 分页查询
sourceBuilder.from(0);
sourceBuilder.size(10);
List<Map<String, Object>> list = esQueryOperation.list(indexName, sourceBuilder);
return list;
}
}
另外如果要排序的话要把字段类型 设置为keyword
字段映射配置：
{
"aliases": {
"btcgl": {}
},
"mappings": {
"properties": {
"userId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"quartersId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"carName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"quartersName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"crossTime": {
"type": "keyword",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"plateNo": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"crossRecordSyscode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"parkSyscode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"parkName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entranceSyscode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"entranceName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"roadwaySyscode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"roadwayName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"vehicleOut": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"cardNo": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"releaseMode": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"releaseResult": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"releaseWay": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"releaseReason": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"vehicleColor": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"vehicleType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"plateColor": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"plateType": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"carCategory": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"carCategoryName": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
},
"orgId": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"settings": {
"index": {
"number_of_shards": "1",
"number_of_replicas": "1"
}
}
}package com.jinw.modules.utils;
import cn.hutool.core.util.ObjectUtil;
import com.google.common.collect.Lists;
import com.jinw.es.EsDataOperation;
import com.jinw.es.EsQueryOperation;
import com.jinw.utils.spring.SpringUtils;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
/**
* @author @liuxin
* @classname ESUtils
* @date 2024/4/9 8:40
* @description TODO
*/
@Component
public class ESUtils {
private static EsQueryOperation esQueryOperation = SpringUtils.getBean(EsQueryOperation.class);
private static EsDataOperation esDataOperation = SpringUtils.getBean(EsDataOperation.class);
public static List<Map<String, Object>> selectByPage(String indexName, SearchSourceBuilder sourceBuilder, int pageNum, int pageSize) {
// 设置超时时间
sourceBuilder.timeout(TimeValue.timeValueSeconds(10));
// 分页查询
sourceBuilder.from((pageNum - 1) * pageSize);
sourceBuilder.size(pageSize);
sourceBuilder.trackTotalHits(true);
List<Map<String, Object>> result = esQueryOperation.list(indexName, sourceBuilder);
return ObjectUtil.isNotEmpty(result) ? result : Lists.newArrayList();
}
public static Long selectByCount(String indexName, SearchSourceBuilder sourceBuilder) {
return esQueryOperation.countByCondition(indexName, sourceBuilder);
}
public static Map<String, Object> selectOneById(
String indexName, String recordId) {
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery("id", recordId);
// 添加到布尔查询的 should 子句中
sourceBuilder.query(termQueryBuilder);
List<Map<String, Object>> list = esQueryOperation.list(indexName, sourceBuilder);
Map<String, Object> entity = null;
if (ObjectUtil.isNotEmpty(list)) {
entity = list.get(0);
}
return entity;
}
public static boolean deleteById(
String indexName, String recordId) {
return esDataOperation.delete(indexName, recordId);
}
public static boolean insert(String indexName, Map<String, Object> dataMap) {
return esDataOperation.insert(indexName, dataMap);
}
}
package com.jinw.es.config;
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider;
import org.apache.http.impl.client.BasicCredentialsProvider;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import java.util.ArrayList;
import java.util.List;
/**
* restHighLevelClient 客户端配置类
*/
@Slf4j
@Data
@Configuration
@ConfigurationProperties(prefix = "elasticsearch")
public class ElasticsearchConfig {
/**
* es host ip 地址（集群）
*/
private String hosts;
/**
* es用户名
*/
private String userName;
/**
* es密码
*/
private String password;
/**
* es 请求方式
*/
private String scheme;
/**
* es 连接超时时间
*/
private int connectTimeOut;
/**
* es socket 连接超时时间
*/
private int socketTimeOut;
/**
* es 请求超时时间
*/
private int connectionRequestTimeOut;
/**
* es 最大连接数
*/
private int maxConnectNum;
/**
* es 每个路由的最大连接数
*/
private int maxConnectNumPerRoute;
/**
* 如果@Bean没有指定bean的名称，那么方法名就是bean的名称
*/
@Bean(name = "restHighLevelClient")
public RestHighLevelClient restHighLevelClient() {
// 构建连接对象
RestClientBuilder builder = RestClient.builder(getEsHost());
// 连接延时配置
builder.setRequestConfigCallback(requestConfigBuilder -> {
requestConfigBuilder.setConnectTimeout(connectTimeOut);
requestConfigBuilder.setSocketTimeout(socketTimeOut);
requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeOut);
return requestConfigBuilder;
});
// 连接数配置
builder.setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setMaxConnTotal(maxConnectNum);
httpClientBuilder.setMaxConnPerRoute(maxConnectNumPerRoute);
httpClientBuilder.setDefaultCredentialsProvider(getCredentialsProvider());
return httpClientBuilder;
});
return new RestHighLevelClient(builder);
}
private HttpHost[] getEsHost() {
// 拆分地址（es为多节点时，不同host以逗号间隔）
List<HttpHost> hostLists = new ArrayList<>();
String[] hostList = hosts.split(",");
for (String addr : hostList) {
String host = addr.split(":")[0];
String port = addr.split(":")[1];
hostLists.add(new HttpHost(host, Integer.parseInt(port), scheme));
}
// 转换成 HttpHost 数组
return hostLists.toArray(new HttpHost[]{});
}
private CredentialsProvider getCredentialsProvider() {
// 设置用户名、密码
CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(userName, password));
return credentialsProvider;
}
}package com.jinw.es;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.search.ClearScrollRequest;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.search.SearchScrollRequest;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.core.TimeValue;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.Scroll;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.xcontent.XContentType;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
import java.util.List;
import java.util.Map;
/**
* 增删改数据
*/
@Slf4j
@Service
public class EsDataOperation {
@Resource
private RestHighLevelClient client;
private final RequestOptions options = RequestOptions.DEFAULT;
/**
* 写入数据
*/
public boolean insert(String indexName, Map<String, Object> dataMap) {
try {
BulkRequest request = new BulkRequest();
request.add(new IndexRequest(indexName).opType("create")
.id(dataMap.get("id").toString())
.source(dataMap, XContentType.JSON));
this.client.bulk(request, options);
return Boolean.TRUE;
} catch (Exception e) {
log.error("EsDataOperation insert error.", e);
}
return Boolean.FALSE;
}
/**
* 批量写入数据
*/
public boolean batchInsert(String indexName, List<Map<String, Object>> userIndexList) {
try {
BulkRequest request = new BulkRequest();
for (Map<String, Object> dataMap : userIndexList) {
request.add(new IndexRequest(indexName).opType("create")
.id(dataMap.get("id").toString())
.source(dataMap, XContentType.JSON));
}
this.client.bulk(request, options);
return Boolean.TRUE;
} catch (Exception e) {
log.error("EsDataOperation batchInsert error.", e);
}
return Boolean.FALSE;
}
/**
* 根据id更新数据，可以直接修改索引结构
*
* @param refreshPolicy 数据刷新策略
*/
public boolean update(String indexName, Map<String, Object> dataMap, WriteRequest.RefreshPolicy refreshPolicy) {
try {
UpdateRequest updateRequest = new UpdateRequest(indexName, dataMap.get("id").toString());
updateRequest.setRefreshPolicy(refreshPolicy);
updateRequest.doc(dataMap);
this.client.update(updateRequest, options);
return Boolean.TRUE;
} catch (Exception e) {
log.error("EsDataOperation update error.", e);
}
return Boolean.FALSE;
}
/**
* 删除数据
*/
public boolean delete(String indexName, String id) {
try {
DeleteRequest deleteRequest = new DeleteRequest(indexName, id);
this.client.delete(deleteRequest, options);
return Boolean.TRUE;
} catch (Exception e) {
log.error("EsDataOperation delete error.", e);
}
return Boolean.FALSE;
}
/**
* 批量删除数据
*/
public boolean batchDelete(String indexName, String fieldName, String fieldValue) {
try {
// 设置滚动参数
Scroll scroll = new Scroll(TimeValue.timeValueMinutes(1L));
// 构建初始查询请求
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.scroll(scroll);
// 构建查询条件
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
sourceBuilder.query(QueryBuilders.rangeQuery(fieldName).lt(fieldValue));
searchRequest.source(sourceBuilder);
// 执行初始查询请求
SearchResponse searchResponse = client.search(searchRequest, options);
// 获取滚动ID
String scrollId = searchResponse.getScrollId();
// 构建批量删除请求
BulkRequest bulkRequest = new BulkRequest();
// 循环获取滚动结果并添加到批量删除请求中
while (true) {
for (SearchHit hit : searchResponse.getHits().getHits()) {
bulkRequest.add(new DeleteRequest(indexName, hit.getId()));
}
// 执行批量删除请求
BulkResponse bulkResponse = client.bulk(bulkRequest, options);
// 检查是否还有滚动结果
if (bulkResponse.getItems().length == 0) {
break;
}
// 继续滚动搜索
SearchScrollRequest scrollRequest = new SearchScrollRequest(scrollId);
scrollRequest.scroll(scroll);
searchResponse = client.scroll(scrollRequest, options);
}
// 清除滚动ID
ClearScrollRequest clearScrollRequest = new ClearScrollRequest();
clearScrollRequest.addScrollId(scrollId);
client.clearScroll(clearScrollRequest, options);
return Boolean.TRUE;
} catch (Exception e) {
log.error("EsDataOperation delete error.", e);
}
return Boolean.FALSE;
}
}package com.jinw.es;
import cn.hutool.core.io.FileUtil;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.xcontent.XContentType;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.io.IOException;
/**
* 操作ES索引
*/
@Slf4j
@Service
public class EsIndexOperation {
@Resource
private RestHighLevelClient restHighLevelClient;
private final RequestOptions options = RequestOptions.DEFAULT;
/**
* 判断索引是否存在
*/
public boolean checkIndex(String index) {
try {
return restHighLevelClient.indices().exists(new GetIndexRequest(index), options);
} catch (Exception e) {
log.error("EsIndexOperation checkIndex error.", e);
}
return Boolean.FALSE;
}
/**
* 创建索引
*
* @param indexName es索引名
* @param esSettingFilePath es索引的alias、settings和mapping的配置文件
*/
public boolean createIndex(String indexName, String esSettingFilePath) {
String aliases = null;
String mappings = null;
String settings = null;
if (StringUtils.isNotBlank(esSettingFilePath)) {
try {
String fileContent = FileUtil.readUtf8String(esSettingFilePath);
if (StringUtils.isNotBlank(fileContent)) {
JSONObject jsonObject = JSON.parseObject(fileContent);
aliases = jsonObject.getString("aliases");
mappings = jsonObject.getString("mappings");
settings = jsonObject.getString("settings");
}
} catch (Exception e) {
log.error("createIndex error.", e);
return false;
}
}
if (checkIndex(indexName)) {
log.error("createIndex indexName:[{}]已存在", indexName);
return false;
}
CreateIndexRequest request = new CreateIndexRequest(indexName);
if ((StringUtils.isNotBlank(aliases))) {
request.aliases(aliases, XContentType.JSON);
}
if (StringUtils.isNotBlank(mappings)) {
request.mapping(mappings, XContentType.JSON);
}
if (StringUtils.isNotBlank(settings)) {
request.settings(settings, XContentType.JSON);
}
try {
this.restHighLevelClient.indices().create(request, options);
return true;
} catch (IOException e) {
log.error("EsIndexOperation createIndex error.", e);
return false;
}
}
/**
* 删除索引
*/
public boolean deleteIndex(String indexName) {
try {
if (checkIndex(indexName)) {
DeleteIndexRequest request = new DeleteIndexRequest(indexName);
AcknowledgedResponse response = restHighLevelClient.indices().delete(request, options);
return response.isAcknowledged();
}
} catch (Exception e) {
log.error("EsIndexOperation deleteIndex error.", e);
}
return Boolean.FALSE;
}
}package com.jinw.es;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.CountRequest;
import org.elasticsearch.client.core.CountResponse;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
/**
* 查询操作
*/
@Slf4j
@Service
public class EsQueryOperation {
@Resource
private RestHighLevelClient client;
private final RequestOptions options = RequestOptions.DEFAULT;
/**
* 查询总数
*/
public Long count(String indexName) {
CountRequest countRequest = new CountRequest(indexName);
try {
CountResponse countResponse = client.count(countRequest, options);
return countResponse.getCount();
} catch (Exception e) {
log.error("EsQueryOperation count error.", e);
}
return 0L;
}
/**
* 查询总数
*/
public Long countByCondition(String indexName, SearchSourceBuilder sourceBuilder) {
CountRequest countRequest = new CountRequest(indexName);
countRequest.source(sourceBuilder);
try {
CountResponse countResponse = client.count(countRequest, options);
return countResponse.getCount();
} catch (Exception e) {
log.error("EsQueryOperation count error.", e);
}
return 0L;
}
/**
* 查询数据集
*/
public List<Map<String, Object>> list(String indexName, SearchSourceBuilder sourceBuilder) {
SearchRequest searchRequest = new SearchRequest(indexName);
searchRequest.source(sourceBuilder);
try {
SearchResponse searchResp = client.search(searchRequest, options);
List<Map<String, Object>> data = new ArrayList<>();
SearchHit[] searchHitArr = searchResp.getHits().getHits();
for (SearchHit searchHit : searchHitArr) {
Map<String, Object> temp = searchHit.getSourceAsMap();
temp.put("id", searchHit.getId());
data.add(temp);
}
return data;
} catch (Exception e) {
log.error("EsQueryOperation list error.", e);
}
return null;
}
}SearchSourceBuilder是Elasticsearch Java客户端中用于构建搜索请求的类之一。它提供了一种方式来构建查询，指定搜索结果的大小、偏移量等。
在SearchSourceBuilder中，from和size是两个重要的参数，用于控制搜索结果的偏移量和大小。
from：指定搜索结果的偏移量，即从搜索结果中的第几条开始返回结果。默认为0，表示从第一条开始。
size：指定返回的搜索结果的数量。默认情况下，Elasticsearch每次最多返回10条结果。可以通过size参数来修改返回结果的数量。
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.elasticsearch.search.sort.SortOrder;
public class SearchExample {
public void searchDocuments(RestHighLevelClient client) throws Exception {
SearchRequest searchRequest = new SearchRequest("index_name");
SearchSourceBuilder sourceBuilder = new SearchSourceBuilder();
// 设置查询条件
sourceBuilder.query(QueryBuilders.matchAllQuery());
// 设置排序
sourceBuilder.sort("field_name", SortOrder.DESC);
// 设置偏移量和返回结果数量
sourceBuilder.from(0); // 从第一条开始
sourceBuilder.size(10); // 返回10条结果
searchRequest.source(sourceBuilder);
client.search(searchRequest, RequestOptions.DEFAULT);
}
}