DSL和java调用
作者：互联网
1 索引
1.1 创建索引
PUT logs_20210210
1.2 添加mapping
POST logs_20210210
{
"properties": {
"msg": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword"
}
}
},
"cost": {
"type": "long"
},
"level": {
"type": "keyword"
},
"logger": {
"type": "keyword"
},
"id": {
"type": "keyword"
},
"threadNo": {
"type": "float"
},
"day": {
"type": "date",
"format": "8uuuu-MM-dd"
},
"timestamp": {
"type": "date"
}
}
}
1.3 插入数据：
POST log_20210128/doc
{
"level": "INFO",
"logger": "c.f.s.controller.IndexController",
"id": "da579fb935524fe387ab6c9483c82bdc",
"threadNo": 36966.33,
"cost": 455,
"msg": "This is INFO message",
"day": "2020-01-15",
"timestamp": "2021-01-15T01:23:23.332Z"
}
POST log_20210128/doc
{
"level": "DEBUG",
"logger": "c.f.s.controller.IndexService",
"id": "0bb205917188487383ac5c310a377d00",
"threadNo": 76966.33,
"cost": 25,
"msg": "这是一条日志",
"day": "2020-01-16",
"timestamp": "2021-01-16T11:23:23.300Z"
}
2 查询
2.1 基本查询
2.1.1 精确查询
据id查询，不进行分词。
POST logs_20210210/_search
{
"query": {
"term": {
"id": {
"value": "da579fb935524fe387ab6c9483c82bdc"
}
}
}
}
多内容精确查询：
POST logs_20210210/_search
{
"query": {
"terms": {
"id": [
"da579fb935524fe387ab6c9483c82bdc",
"0bb205917188487383ac5c310a377d00"
]
}
}
}
代码：
public void termQuery() {
try {
// 构建查询条件（注意：termQuery 支持多种格式查询，如 boolean、int、double、string 等，这里使用的是 string 的查询）
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termQuery("id", "da579fb935524fe387ab6c9483c82bdc"));
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log userInfo = JSON.parseObject(hit.getSourceAsString(), Log.class);
// // 输出查询信息
log.info(userInfo.toString());
}
}
} catch (IOException e) {
System.out.print(e);
}
}
public void termsQuery() {
try {
// 构建查询条件（注意：termsQuery 支持多种格式查询，如 boolean、int、double、string 等，这里使用的是 string
// 的查询）
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.termsQuery("id", "da579fb935524fe387ab6c9483c82bdc",
"0bb205917188487383ac5c310a377d00"));
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log userInfo = JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(userInfo.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
2.1.2 匹配查询
查询所有符合条件的数据，并分页
POST logs_20210210/_search
{
"query": {
"match_all": {}
},
"from": 0,
"size": 10,
"sort": [
{
"cost": {
"order": "asc"
}
}
]
}
匹配查询数据：
POST logs_20210210/_search
{
"query": {
"match": {
"msg": "This is DEBUG message"
}
}
}
词语匹配查询
词语匹配进行查询，匹配 msg中为 君莫笑 的msg信息：
GET logs_20210210/_search
{
"query": {
"match_phrase": {
"msg": "君莫笑"
}
}
}
内容多字段查询
查询在字段 logger，msg中存在 IndexController2 内容的信息：
GET logs_20210210/_search
{
"query": {
"multi_match": {
"query": "IndexController2",
"fields": ["logger","msg"]
}
}
}
代码：
public void matchAllQuery() {
try {
// 构建查询条件
MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery();
// 创建查询源构造器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(matchAllQueryBuilder);
// 设置分页
searchSourceBuilder.from(0);
searchSourceBuilder.size(3);
// 设置排序
searchSourceBuilder.sort("cost", SortOrder.ASC);
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log logEntity = JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(logEntity.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
public void matchQuery() {
try {
// 构建查询条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchQuery("msg", "*连"));
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log logEntity = JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(logEntity.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
public void matchPhraseQuery() {
try {
// 构建查询条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.matchPhraseQuery("msg", "醉里连"));
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log logEntity = JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(logEntity.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
public void matchMultiQuery() {
try {
// 构建查询条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.multiMatchQuery("IndexController2", "logger", "msg"));
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log logEntity= JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(logEntity.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
2.1.3 模糊查询
查询词中有部分错误的情况下也能召回正确的结果。匹配规则为莱温斯坦距离（Levenshtein distance）和莱文斯坦距离扩展版（Damerau–Levenshtein distance），对模糊查询的支持有两种方式match query和fuzzy query。莱温斯坦距离是编辑距离的一种，指两个字符串之间，由一个转成另一个所需的最少编辑操作次数。可以通过查询参数进行配置允许的规则。如下查询所有以level结尾的3 54。
GET logs_20210210/_search
{
"query": {
"fuzzy": {
"level": "\\*3 54"
}
}
}
# 注意转义字符，*匹配多个，?匹配一个
代码：
public void fuzzyQuery() {
try {
// 构建查询条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.fuzzyQuery("level", "*54").fuzziness(Fuzziness.AUTO));
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log l = JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(l.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
2.1.4 范围查询
1> Restful操作实例
查询cost ≥ 900 的数据：
GET log_20210128/_search
{
"query": {
"range": {
"cost": {
"gte": 900
}
}
}
}
代码：
public void rangeQuery() {
try {
// 构建查询条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.rangeQuery("cost").gte(900));
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log l = JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(l.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
2.1.5 通配符查询
查询所有以 “message” 结尾的msg数据：
GET log_20210128/_search
{
"query": {
"wildcard": {
"msg.keyword": {
"value": "*message"
}
}
}
}
代码：
public void wildcardQuery() {
try {
// 构建查询条件
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(QueryBuilders.wildcardQuery("msg.keyword", "*message"));
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log l = JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(l.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
2.1.6 布尔查询
查询在 timestamp在2021-01-10~2021-01-17之间，且logger为c.f.s.controller.IndexController和c.f.s.controller.IndexService的数据：
POST log_20210128/_search
{
"query": {
"bool": {
"filter": {
"range": {
"timestamp": {
"format": "8uuuu-MM-dd",
"gte": "2021-01-10",
"lte": "2021-01-17"
}
}
},
"must": [
{
"terms": {
"logger": [
"c.f.s.controller.IndexController",
"c.f.s.controller.IndexService"
]
}
}
]
}
}
}
-trim，match，match_phrase,boolean,wildcard,
代码：
public void boolQuery() {
try {
// 创建 Bool 查询构建器
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
// 构建查询条件
boolQueryBuilder
.must(QueryBuilders.termsQuery("logger", "c.f.s.controller.IndexController",
"c.f.s.controller.IndexService"))
.filter().add(QueryBuilders.rangeQuery("timestamp").format("8uuuu-MM-dd").gte("2021-01-10")
.lte("2021-01-17"));
// 构建查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(boolQueryBuilder);
// 创建查询请求对象，将查询对象配置到其中
SearchRequest searchRequest = new SearchRequest("logs_20210210");
searchRequest.source(searchSourceBuilder);
// 执行查询，然后处理响应结果
SearchResponse searchResponse = restHighLevelClient.search(searchRequest, RequestOptions.DEFAULT);
// 根据状态和数据条数验证是否返回了数据
if (RestStatus.OK.equals(searchResponse.status()) && searchResponse.getHits().totalHits > 0) {
SearchHits hits = searchResponse.getHits();
for (SearchHit hit : hits) {
// 将 JSON 转换成对象
Log l = JSON.parseObject(hit.getSourceAsString(), Log.class);
// 输出查询信息
log.info(l.toString());
}
}
} catch (IOException e) {
log.error("", e);
}
}
2.2 聚合查询
2.2.1 metrics
满足某个条件的文档集合。分桶后会对文档进行某些有意义的指标计算。
demo:
# 计算平均耗时
{"size":0,"aggs":{"cost_avg":{"avg":{"field":"cost"}}}}
计算指标：stats（统计各项信息）、count（总个数）、sum、max、min、cardinality （求不重复的值有多少）、percentile（百分比）、extended_stats（拓展的统计各项信息）。
统计耗时总数、耗时最高值、耗时最低值、耗时平均、耗时总和：
{
"size": 0,
"aggs": {
"cost_stats": {
"stats": {
"field": "cost"
}
}
}
}
# 响应
{
"took": 2325,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 2,
"max_score": 0,
"hits": []
},
"aggregations": {
"cost_stats": {
"count": 2,
"min": 25,
"max": 455,
"avg": 240,
"sum": 480
}
}
}
拓展的统计各项信息：
GET logs_20210210/_search
{"size":0,"aggs":{"cost_avg":{"extended_stats":{"field":"cost"}}}}
# 注意，size为0，不会返回文档信息，只返回聚合信息。
{
"took": 4,
"timed_out": false,
"_shards": {
"total": 5,
"successful": 5,
"skipped": 0,
"failed": 0
},
"hits": {
"total": 2,
"max_score": 0,
"hits": []
},
"aggregations": {
"cost_stats": {
"count": 2,
"min": 25,
"max": 455,
"avg": 240,
"sum": 480,
"sum_of_squares": 207650,
"variance": 46225,
"std_deviation": 215,
"std_deviation_bounds": {
"upper": 670,
"lower": -190
}
}
}
}
统计cost最低值：
POST log_20210128/_search
{
"size": 0,
"aggs": {
"cost_min": {
"min": {
"field": "cost"
}
}
}
}
统计cost最高值：
POST log_20210128/_search
{
"size": 0,
"aggs": {
"cost_max": {
"max": {
"field": "cost"
}
}
}
}
统计cost平均值：
POST log_20210128/_search
{
"size": 0,
"aggs": {
"cost_avg": {
"avg": {
"field": "cost"
}
}
}
}
统计cost总值：
POST log_20210128/_search
{
"size": 0,
"aggs": {
"cost_sum": {
"sum": {
"field": "cost"
}
}
}
}
统计cost总数：
POST log_20210128/_search
{
"size": 0,
"aggs": {
"log_count": {
"value_count": {
"field": "scostalary"
}
}
}
}
统计cost百分位：
POST log_20210128/_search
{
"size": 0,
"aggs": {
"cost_percentiles": {
"percentiles": {
"field": "cost"
}
}
}
}
代码：
public Object aggregationStats() {
String responseResult = "";
try {
// 设置聚合条件
AggregationBuilder aggr = AggregationBuilders.stats("cost_stats").field("cost");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(aggr);
// 设置查询结果不返回，只返回聚合结果
searchSourceBuilder.size(0);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status()) || aggregations != null) {
// 转换为 Stats 对象
ParsedStats aggregation = aggregations.get("cost_stats");
log.info("-------------------------------------------");
log.info("聚合信息:");
log.info("count：{}", aggregation.getCount());
log.info("avg：{}", aggregation.getAvg());
log.info("max：{}", aggregation.getMax());
log.info("min：{}", aggregation.getMin());
log.info("sum：{}", aggregation.getSum());
log.info("-------------------------------------------");
}
// 根据具体业务逻辑返回不同结果，这里为了方便直接将返回响应对象Json串
responseResult = response.toString();
} catch (IOException e) {
log.error("", e);
}
return responseResult;
}
public Object aggregationMin() {
String responseResult = "";
try {
// 设置聚合条件
AggregationBuilder aggr = AggregationBuilders.min("cost_min").field("cost");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(aggr);
searchSourceBuilder.size(0);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status()) || aggregations != null) {
// 转换为 Min 对象
ParsedMin aggregation = aggregations.get("cost_min");
log.info("-------------------------------------------");
log.info("聚合信息:");
log.info("min：{}", aggregation.getValue());
log.info("-------------------------------------------");
}
// 根据具体业务逻辑返回不同结果，这里为了方便直接将返回响应对象Json串
responseResult = response.toString();
} catch (IOException e) {
log.error("", e);
}
return responseResult;
}
public Object aggregationMax() {
String responseResult = "";
try {
// 设置聚合条件
AggregationBuilder aggr = AggregationBuilders.max("cost_max").field("cost");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(aggr);
searchSourceBuilder.size(0);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status()) || aggregations != null) {
// 转换为 Max 对象
ParsedMax aggregation = aggregations.get("cost_max");
log.info("-------------------------------------------");
log.info("聚合信息:");
log.info("max：{}", aggregation.getValue());
log.info("-------------------------------------------");
}
// 根据具体业务逻辑返回不同结果，这里为了方便直接将返回响应对象Json串
responseResult = response.toString();
} catch (IOException e) {
log.error("", e);
}
return responseResult;
}
public Object aggregationAvg() {
String responseResult = "";
try {
// 设置聚合条件
AggregationBuilder aggr = AggregationBuilders.avg("cost_avg").field("cost");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(aggr);
searchSourceBuilder.size(0);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status()) || aggregations != null) {
// 转换为 Avg 对象
ParsedAvg aggregation = aggregations.get("cost_avg");
log.info("-------------------------------------------");
log.info("聚合信息:");
log.info("avg：{}", aggregation.getValue());
log.info("-------------------------------------------");
}
// 根据具体业务逻辑返回不同结果，这里为了方便直接将返回响应对象Json串
responseResult = response.toString();
} catch (IOException e) {
log.error("", e);
}
return responseResult;
}
public Object aggregationSum() {
String responseResult = "";
try {
// 设置聚合条件
SumAggregationBuilder aggr = AggregationBuilders.sum("cost_sum").field("cost");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(aggr);
searchSourceBuilder.size(0);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status()) || aggregations != null) {
// 转换为 Sum 对象
ParsedSum aggregation = aggregations.get("cost_sum");
log.info("-------------------------------------------");
log.info("聚合信息:");
log.info("sum：{}", String.valueOf((aggregation.getValue())));
log.info("-------------------------------------------");
}
// 根据具体业务逻辑返回不同结果，这里为了方便直接将返回响应对象Json串
responseResult = response.toString();
} catch (IOException e) {
log.error("", e);
}
return responseResult;
}
public Object aggregationCount() {
String responseResult = "";
try {
// 设置聚合条件
AggregationBuilder aggr = AggregationBuilders.count("cost_count").field("cost");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(aggr);
searchSourceBuilder.size(0);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status()) || aggregations != null) {
// 转换为 ValueCount 对象
ParsedValueCount aggregation = aggregations.get("cost_count");
log.info("-------------------------------------------");
log.info("聚合信息:");
log.info("count：{}", aggregation.getValue());
log.info("-------------------------------------------");
}
// 根据具体业务逻辑返回不同结果，这里为了方便直接将返回响应对象Json串
responseResult = response.toString();
} catch (IOException e) {
log.error("", e);
}
return responseResult;
}
public Object aggregationPercentiles() {
String responseResult = "";
try {
// 设置聚合条件
AggregationBuilder aggr = AggregationBuilders.percentiles("cost_percentiles").field("cost");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.aggregation(aggr);
searchSourceBuilder.size(0);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status()) || aggregations != null) {
// 转换为 Percentiles 对象
ParsedPercentiles aggregation = aggregations.get("cost_percentiles");
log.info("-------------------------------------------");
log.info("聚合信息:");
for (Percentile percentile : aggregation) {
log.info("百分位：{}：{}", percentile.getPercent(), percentile.getValue());
}
log.info("-------------------------------------------");
}
// 根据具体业务逻辑返回不同结果，这里为了方便直接将返回响应对象Json串
responseResult = response.toString();
} catch (IOException e) {
log.error("", e);
}
return responseResult;
}
2.2.2 bucket
为某个桶中的文档计算得到的统计信息。类似mysql中的group by，会据条件对文档进行划分。
level共两类DEBUG和INFO，查询每一类的个数：
POST logs_20210210/_search
{"size":0,"aggs":{"terms_test":{"terms":{"field":"level"}}}}
# 返回结果：
{...
"aggregations": {
"terms_test": {
"doc_count_error_upper_bound": 0,
"sum_other_doc_count": 0,
"buckets": [
{
"key": "INFO",
"doc_count": 3774674
}
,
{
"key": "DEBUG",
"doc_count": 3771326
}
]
}
}
}
# doc_count_error_upper_bound：该值表示未进入最终术语列表的术语的最大潜在文档计数
# sum_other_doc_count：所有不属于响应的bucket的文档计数之和
聚合类别：ranges、date_range、histogram和date_histogram等。
按耗时范围进行聚合分桶，统计耗时在 0-500、500-1500 和 1500 以上的数据：
POST logs_20210210/_search
{
"size": 0,
"aggs": {
"cost_range_bucket": {
"range": {
"field": "cost",
"ranges": [
{
"key": "低耗时",
"to": 500
},
{
"key": "中间耗时",
"from": 500,
"to": 1500
},
{
"key": "高耗时",
"from": 1500
}
]
}
}
}
}
按照时间范围进行分桶，统计 2021-01-16~2021-01-18和 2021-01-18~2021-01-20产生的数据：
{
"size": 0,
"aggs": {
"timestamp_range_bucket": {
"date_range": {
"field": "timestamp",
"format": "8uuuu-MM-dd",
"ranges": [
{
"key": "2021-01-18~2021-01-20",
"from": "2021-01-16",
"to": "2021-01-18"
},
{
"key": "2021-01-18~2021-01-20",
"from": "2021-01-18",
"to": "2021-01-20"
}
]
}
}
}
}
按cost多少进行聚合分桶，设置统计的最小值为 0，最大值为 3000，区段间隔为 1000：
{
"size": 0,
"aggs": {
"cost_histogram": {
"histogram": {
"field": "cost",
"extended_bounds": {
"min": 0,
"max": 3000
},
"interval": 1000
}
}
}
}
按day进行分桶：
{
"size": 0,
"aggs": {
"day_histogram": {
"date_histogram": {
"format": "8uuuu-MM-dd",
"field": "day",
"interval": "day"
}
}
}
}
代码：
public void aggrBucketTerms() {
try {
AggregationBuilder aggr = AggregationBuilders.terms("level_bucket").field("level");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(10);
searchSourceBuilder.aggregation(aggr);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status())) {
// 分桶
Terms byCompanyAggregation = aggregations.get("level_bucket");
List<? extends Terms.Bucket> buckets = byCompanyAggregation.getBuckets();
// 输出各个桶的内容
log.info("-------------------------------------------");
log.info("聚合信息:");
for (Terms.Bucket bucket : buckets) {
log.info("桶名：{} | 总数：{}", bucket.getKeyAsString(), bucket.getDocCount());
}
log.info("-------------------------------------------");
}
} catch (IOException e) {
log.error("", e);
}
}
public void aggrBucketRange() {
try {
AggregationBuilder aggr = AggregationBuilders.range("cost_range_bucket").field("cost")
.addUnboundedTo("低耗时", 500).addRange("中间耗时", 500, 1500).addUnboundedFrom("高耗时", 1500);
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
searchSourceBuilder.aggregation(aggr);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status())) {
// 分桶
Range byCompanyAggregation = aggregations.get("cost_range_bucket");
List<? extends Range.Bucket> buckets = byCompanyAggregation.getBuckets();
// 输出各个桶的内容
log.info("-------------------------------------------");
log.info("聚合信息:");
for (Range.Bucket bucket : buckets) {
log.info("桶名：{} | 总数：{}", bucket.getKeyAsString(), bucket.getDocCount());
}
log.info("-------------------------------------------");
}
} catch (IOException e) {
log.error("", e);
}
}
public void aggrBucketDateRange() {
try {
AggregationBuilder aggr = AggregationBuilders.dateRange("timestamp_range_bucket").field("timestamp")
.format("8uuuu-MM-dd").addRange("2021-01-01~2021-01-31", "2021-01-01", "2021-01-31")
.addRange("2021-02-01~2021-02-28", "2021-02-01", "2021-02-28");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
searchSourceBuilder.aggregation(aggr);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status())) {
// 分桶
Range byCompanyAggregation = aggregations.get("timestamp_range_bucket");
List<? extends Range.Bucket> buckets = byCompanyAggregation.getBuckets();
// 输出各个桶的内容
log.info("-------------------------------------------");
log.info("聚合信息:");
for (Range.Bucket bucket : buckets) {
log.info("桶名：{} | 总数：{}", bucket.getKeyAsString(), bucket.getDocCount());
}
log.info("-------------------------------------------");
}
} catch (IOException e) {
log.error("", e);
}
}
public void aggrBucketHistogram() {
try {
AggregationBuilder aggr = AggregationBuilders.histogram("cost_histogram").field("cost")
.extendedBounds(0, 3000).interval(1000);
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
searchSourceBuilder.aggregation(aggr);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status())) {
// 分桶
Histogram byCompanyAggregation = aggregations.get("cost_histogram");
List<? extends Histogram.Bucket> buckets = byCompanyAggregation.getBuckets();
// 输出各个桶的内容
log.info("-------------------------------------------");
log.info("聚合信息:");
for (Histogram.Bucket bucket : buckets) {
log.info("桶名：{} | 总数：{}", bucket.getKeyAsString(), bucket.getDocCount());
}
log.info("-------------------------------------------");
}
} catch (IOException e) {
log.error("", e);
}
}
public void aggrBucketDateHistogram() {
try {
AggregationBuilder aggr = AggregationBuilders.dateHistogram("day_histogram").field("day").interval(1)
.dateHistogramInterval(DateHistogramInterval.DAY).format("8uuuu-MM-dd");
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
searchSourceBuilder.aggregation(aggr);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status())) {
// 分桶
Histogram byCompanyAggregation = aggregations.get("day_histogram");
List<? extends Histogram.Bucket> buckets = byCompanyAggregation.getBuckets();
// 输出各个桶的内容
log.info("-------------------------------------------");
log.info("聚合信息:");
for (Histogram.Bucket bucket : buckets) {
log.info("桶名：{} | 总数：{}", bucket.getKeyAsString(), bucket.getDocCount());
}
log.info("-------------------------------------------");
}
} catch (IOException e) {
log.error("", e);
}
}
2.2.3 metrics和bucket组合
一个聚合就是一些桶和指标的组合。桶中可以有多个嵌套的桶。
按照level分桶、然后统计每个level耗时的最高值：
{
"size": 0,
"aggs": {
"cost_bucket": {
"terms": {
"field": "level",
"size": "10"
},
"aggs": {
"cost_max_log": {
"top_hits": {
"size": 1,
"sort": [
{
"cost": {
"order": "desc"
}
}
]
}
}
}
}
}
}
# top_hits聚合器可以有效地用于通过桶聚合器按特定字段对结果集进行分组。 一个或多个存储桶聚合器确定结果集被切入的属性。
代码：
public void aggregationTopHits() {
try {
AggregationBuilder testTop = AggregationBuilders.topHits("cost_max_user")
.size(1)
.sort("cost", SortOrder.DESC);
AggregationBuilder levelBucket = AggregationBuilders.terms("level_bucket")
.field("level")
.size(10);
levelBucket.subAggregation(testTop);
// 查询源构建器
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.size(0);
searchSourceBuilder.aggregation(levelBucket);
// 创建查询请求对象，将查询条件配置到其中
SearchRequest request = new SearchRequest("logs_20210210");
request.source(searchSourceBuilder);
// 执行请求
SearchResponse response = restHighLevelClient.search(request, RequestOptions.DEFAULT);
// 获取响应中的聚合信息
Aggregations aggregations = response.getAggregations();
// 输出内容
if (RestStatus.OK.equals(response.status())) {
// 分桶
Terms byCompanyAggregation = aggregations.get("level_bucket");
List<? extends Terms.Bucket> buckets = byCompanyAggregation.getBuckets();
// 输出各个桶的内容
log.info("-------------------------------------------");
log.info("聚合信息:");
for (Terms.Bucket bucket : buckets) {
log.info("桶名：{}", bucket.getKeyAsString());
ParsedTopHits topHits = bucket.getAggregations().get("cost_max_user");
for (SearchHit hit:topHits.getHits()){
log.info(hit.getSourceAsString());
}
}
log.info("-------------------------------------------");
}
} catch (IOException e) {
log.error("", e);
}
}
3 删除
3.1 删除数据和表结构
3.1.1 删除索引
DELETE 索引名称
3.1.2 删除多个索引
DELETE 索引名称1，索引名称2
3.1.3 删除以pre开头的索引
DELETE pre*
3.1.4 删除全部索引，配置文件中禁止后将不能使用。
DELETE /*
DELETE _all
3.2 删除数据不删除表结构
3.2.1 根据主键删除
DELETE 索引名称/文档名称/主键编号
3.2.2 根据匹配条件，删除数据
POST 索引名称/文档名称/_delete_by_query
{
"query":{
"term":{
"_id":100000100
}
}
}
3.2.3 删除所有数据，但是不删除表结构
POST /testindex/testtype/_delete_by_query?pretty
{
"query": {
"match_all": {
}
}
}
4 keyword和text区别：
：
text：会分词，然后进行索引，用于全文搜索。支持模糊、精确查询。不支持聚合。
keyword：不进行分词，直接索引。keyword用于关键词搜索、 支持模糊、精确查询。支持聚合。
动态映射：若不指定类型，则es字符串默认被映射成text和keyword类型。当成keyword查询是需要添加”.keyword“，即”level.keyword“。
{
"level": {
"type": "text",
"fields": {
"keyword": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
参考：https://blog.csdn.net/jkjiang123/article/details/106431222
标签：info,调用,java,log,searchSourceBuilder,查询,DSL,cost,new 来源： https://blog.csdn.net/baidu_38225647/article/details/113836760