实时电商数仓（二十四）之实时计算（三）日活处理模块（三）日活数据查询接口
作者：互联网
日活数据查询接口
1 访问路径
|
总数 |
http://publisher:8070/realtime-total?date=2019-02-01 |
|
分时统计 |
http://publisher:8070/realtime-hour?id=dau&date=2019-02-01
|
2 要求数据格式
|
总数 |
[{"id":"dau","name":"新增日活","value":1200}, {"id":"new_mid","name":"新增设备","value":233} ] |
|
分时统计 |
{"yesterday":{"11":383,"12":123,"17":88,"19":200 }, "today":{"12":38,"13":1233,"17":123,"19":688 }} |
3 搭建发布工程
工程目录
4 配置文件
4.1 pom.xml
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.1.15.RELEASE</version>
<relativePath/> <!-- lookup parent from repository -->
</parent>
<groupId>com.atguigu.gmall0105</groupId>
<artifactId>gmall0105-publisher</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>gmall0105-publisher</name>
<description>Demo project for Spring Boot</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.8.1</version>
</dependency>
<dependency>
<groupId>com.alibaba</groupId>
<artifactId>fastjson</artifactId>
<version>1.2.68</version>
</dependency>
<dependency>
<groupId>io.searchbox</groupId>
<artifactId>jest</artifactId>
<version>5.3.3</version>
<exclusions>
<exclusion>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>net.java.dev.jna</groupId>
<artifactId>jna</artifactId>
<version>4.5.2</version>
</dependency>
<dependency>
<groupId>org.codehaus.janino</groupId>
<artifactId>commons-compiler</artifactId>
<version>2.7.8</version>
</dependency>
<!-- https://mvnrepository.com/artifact/org.elasticsearch/elasticsearch -->
<dependency>
<groupId>org.elasticsearch</groupId>
<artifactId>elasticsearch</artifactId>
<version>2.4.6</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
<optional>true</optional>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-jdbc</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>1.3.4</version>
</dependency>
<dependency>
<groupId>ru.yandex.clickhouse</groupId>
<artifactId>clickhouse-jdbc</artifactId>
<version>0.1.55</version>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<version>5.1.47</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
View Code
4.2 application.properties
server.port=8070 spring.elasticsearch.jest.uris=http://hdp1:9200,http://hdp2:9200 spring.datasource.driver-class-name=com.mysql.jdbc.Driver spring.datasource.url=jdbc:mysql://hdp1/gmall0105_rs?characterEncoding=utf-8&useSSL=false spring.datasource.username=root spring.datasource.password=123123 #spring.datasource.driver-class-name=ru.yandex.clickhouse.ClickHouseDriver #spring.datasource.url=jdbc:clickhouse://hdp1:8123/test0105 mybatis.mapperLocations=classpath:mapper/*.xml mybatis.configuration.map-underscore-to-camel-case=true
5 代码部分
5.1 代码结构
|
控制层 |
PublisherController |
实现接口的web发布 |
|
服务层 |
PublisherService |
数据业务查询interface |
|
PublisherServiceImpl |
业务查询的实现类 |
|
|
主程序 |
GmallPublisherApplication |
增加扫描包 |
5.2 controller层
package com.atguigu.gmall0105.publisher.controller;
import com.alibaba.fastjson.JSON;
import com.atguigu.gmall0105.publisher.service.ClickHouseService;
import com.atguigu.gmall0105.publisher.service.EsService;
import org.apache.commons.lang3.time.DateUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.*;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.text.ParseException;
import java.text.SimpleDateFormat;
import java.util.*;
@RestController
public class PublisherController {
@Autowired
EsService esService;
@Autowired
ClickHouseService clickHouseService;
@RequestMapping(value = "realtime-total",method = RequestMethod.GET)
public String realtimeTotal(@RequestParam("date") String dt){
List<Map<String,Object>> rsList=new ArrayList<>();
Map<String,Object> dauMap = new HashMap();
dauMap.put("id","dau");
dauMap.put("name","新增日活");
Long dauTotal=0L;
try {
dauTotal = esService.getDauTotal(dt);
}catch ( Exception e){
e.printStackTrace();
}
if(dauTotal!=null){
dauMap.put("value",dauTotal);
}else {
dauMap.put("value",0L);
}
rsList.add(dauMap);
Map<String,Object> newMidMap = new HashMap();
newMidMap.put("id","new_mid");
newMidMap.put("name","新增设备");
newMidMap.put("value",233);
rsList.add(newMidMap);
Map<String,Object> orderAmountMap = new HashMap();
orderAmountMap.put("id","order_amount");
orderAmountMap.put("name","新增交易额");
BigDecimal orderAmount = clickHouseService.getOrderAmount(dt);
orderAmountMap.put("value",orderAmount);
rsList.add(orderAmountMap);
return JSON.toJSONString(rsList);
}
@GetMapping("realtime-hour")
public String realtimeHour(@RequestParam("id") String id ,@RequestParam("date") String dt){
if(id.equals("dau")){
Map dauHourMapTD = esService.getDauHour(dt);
String yd = getYd(dt);
Map dauHourMapYD = esService.getDauHour(yd);
Map<String,Map<String,Long>> rsMap=new HashMap<>();
rsMap.put("yesterday",dauHourMapYD);
rsMap.put("today",dauHourMapTD);
return JSON.toJSONString(rsMap);
}else if(id.equals("order_amount")){
Map orderAmountHourMapTD = clickHouseService.getOrderAmountHour(dt);
String yd = getYd(dt);
Map orderAmountHourMapYD = clickHouseService.getOrderAmountHour(yd);
Map<String,Map<String,BigDecimal>> rsMap=new HashMap<>();
rsMap.put("yesterday",orderAmountHourMapYD);
rsMap.put("today",orderAmountHourMapTD);
return JSON.toJSONString(rsMap);
}else{
return null;
}
}
private String getYd(String today){
SimpleDateFormat dateFormat = new SimpleDateFormat("yyyy-MM-dd");
try {
Date todayDate = dateFormat.parse(today);
Date ydDate = DateUtils.addDays(todayDate, -1);
return dateFormat.format(ydDate);
} catch (ParseException e) {
e.printStackTrace();
throw new RuntimeException("日期格式不正确");
}
}
}
5.3 service层
package com.atguigu.gmall0105.publisher.service;
import java.math.BigDecimal;
import java.util.List;
import java.util.Map;
public interface EsService {
public Long getDauTotal(String date);
public Map getDauHour(String date);
}
5.4 service层实现类
package com.atguigu.gmall0105.publisher.service.impl;
import com.atguigu.gmall0105.publisher.service.EsService;
import io.searchbox.client.JestClient;
import io.searchbox.core.Search;
import io.searchbox.core.SearchResult;
import io.searchbox.core.search.aggregation.TermsAggregation;
import org.elasticsearch.index.query.MatchAllQueryBuilder;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.terms.TermsBuilder;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.IOException;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Service
public class EsServiceImpl implements EsService {
@Autowired
JestClient jestClient;
@Override
public Long getDauTotal(String date) {
String indexName = "gmall0105_dau_info_"+date+"-query";
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
searchSourceBuilder.query(new MatchAllQueryBuilder());
Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(indexName).addType("_doc").build();
try {
SearchResult searchResult = jestClient.execute(search);
return searchResult.getTotal();
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException("es 查询异常");
}
}
@Override
public Map getDauHour(String date) {
String indexName = "gmall0105_dau_info_"+date+"-query";
//构造查询语句
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
TermsBuilder aggBuilder = AggregationBuilders.terms("groupby_hr").field("hr").size(24);
searchSourceBuilder.aggregation(aggBuilder);
Search search = new Search.Builder(searchSourceBuilder.toString()).addIndex(indexName).addType("_doc").build();
try {
SearchResult searchResult = jestClient.execute(search);
//封装返回结果
Map<String,Long> aggMap=new HashMap<>();
if(searchResult.getAggregations().getTermsAggregation("groupby_hr")!=null){
List<TermsAggregation.Entry> buckets = searchResult.getAggregations().getTermsAggregation("groupby_hr").getBuckets();
for (TermsAggregation.Entry bucket : buckets) {
aggMap.put( bucket.getKey(),bucket.getCount());
}
}
return aggMap;
} catch (IOException e) {
e.printStackTrace();
throw new RuntimeException("es 查询异常");
}
}
}
6 搭建可视化工程进行对接
拷贝前端工程，启动主程序启动。
标签：商数,String,spring,org,实时,日活,new,put,import 来源： https://www.cnblogs.com/qiu-hua/p/13665175.html