Skip to content

谢明辉 / VVAS-Match

Go to a project
Commit fea2fd70 by 李乾广
Options

[add]添加生命周期管理

1 parent 48d1399f
Inline Side-by-side
Showing 9 changed files with 228 additions and 84 deletions
pom.xml
......@@ -4,7 +4,7 @@
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.3.1.RELEASE</version>
<version>2.5.1</version>
<relativePath /> <!-- lookup parent from repository -->
</parent>
<groupId>com.viontech</groupId>
......@@ -34,17 +34,6 @@
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.viontech.keliu</groupId>
<artifactId>keliu-base</artifactId>
<version>6.0.9-SNAPSHOT</version>
<exclusions>
<exclusion>
<artifactId>pagehelper-spring-boot-starter</artifactId>
<groupId>com.github.pagehelper</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
......@@ -59,10 +48,22 @@
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-collections4</artifactId>
<version>4.4</version>
</dependency>
<dependency>
<groupId>com.viontech.keliu</groupId>
<artifactId>AlgApiClient</artifactId>
<version>6.0.9-SNAPSHOT</version>
<version>6.1.0-SNAPSHOT</version>
<exclusions>
<exclusion>
<groupId>javax</groupId>
......
src/main/java/com/viontech/match/config/WebConfiguration.java 0 → 100644
package com.viontech.match.config;
import com.viontech.match.converter.DateConverter;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.convert.converter.Converter;
import org.springframework.web.servlet.config.annotation.CorsRegistry;
import org.springframework.web.servlet.config.annotation.WebMvcConfigurer;
import java.util.Date;
/**
* @author vion
*/
@Configuration
public class WebConfiguration implements WebMvcConfigurer {
@Bean
public Converter<String, Date> addNewConvert() {
return new DateConverter();
}
@Override
public void addCorsMappings(CorsRegistry registry) {
registry.addMapping("/**")
.allowedOriginPatterns("*")
.allowCredentials(true)
.allowedMethods("*")
.allowedHeaders("*")
.maxAge(3600);
}
}
src/main/java/com/viontech/match/converter/DateConverter.java 0 → 100644
package com.viontech.match.converter;
import org.springframework.core.convert.converter.Converter;
import java.text.DateFormat;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
/**
* SpringMVC 自定义 日期转换器
*
* @author :<a href="mailto:zyj@freshz.cn">章英杰</a>
* @date :2016-09-05 14:03:54
*/
public class DateConverter implements Converter<String, Date> {
/**
* 可格式化 的日期 字串
*/
private static final List<String> formarts = new ArrayList();
public DateConverter() {
System.out.println("["+new Date()+"] [INFO] ("+this.getClass().getName()+") - 日期格式化组件创建成功");
}
static{
formarts.add("yyyy-MM");
formarts.add("yyyy-MM-dd");
formarts.add("yyyy-MM-dd HH:mm");
formarts.add("yyyy-MM-dd HH:mm:ss");
formarts.add("HH:mm:ss");
formarts.add("HH:mm");
formarts.add("yyyy-MM-dd'T'HH:mm:ss");
}
@Override
public Date convert(String source) {
String value = source.trim();
if ("".equals(value)) {
return null;
}
if(source.matches("^\\d{4}-\\d{1,2}$")){
return parseDate(source, formarts.get(0));
}else if(source.matches("^\\d{4}-\\d{1,2}-\\d{1,2}$")){
return parseDate(source, formarts.get(1));
}else if(source.matches("^\\d{4}-\\d{1,2}-\\d{1,2} {1}\\d{1,2}:\\d{1,2}$")){
return parseDate(source, formarts.get(2));
}else if(source.matches("^\\d{4}-\\d{1,2}-\\d{1,2} {1}\\d{1,2}:\\d{1,2}:\\d{1,2}$")){
return parseDate(source, formarts.get(3));
}else if(source.matches("([01][0-9]|2[0-3]):[0-5][0-9]:[0-5][0-9]")){
return parseDate(source, formarts.get(4));
}else if(source.matches("([01][0-9]|2[0-3]):[0-5][0-9]")){
return parseDate(source, formarts.get(5));
} else if(source.matches("^\\d{4}-\\d{1,2}-\\d{1,2}T\\d{1,2}:\\d{1,2}:\\d{1,2}[-,+]\\d{1,2}:\\d{1,2}$")){
return parseDate(source,formarts.get(6));
}else {
throw new IllegalArgumentException("Invalid boolean value '" + source + "'");
}
}
/**
* 功能描述：格式化日期
*
* @param dateStr String 字符型日期
* @param format String 格式
* @return Date 日期
*/
public Date parseDate(String dateStr, String format) {
Date date=null;
try {
DateFormat dateFormat = new SimpleDateFormat(format);
date = (Date) dateFormat.parse(dateStr);
} catch (Exception e) {
}
return date;
}
}
\ No newline at end of file
src/main/java/com/viontech/match/entity/vo/RequestVo.java
......@@ -20,31 +20,51 @@ import java.util.List;
@Setter
public class RequestVo {
/** 公用 */
//指令
private CommandEnum command;
//请求唯一id标识
private String rid;
//特征池id
private String poolId;
/**
* 是否使用索引生命周期管理
*/
private boolean useILMPolicy = false;
/** 增加修改特征池时使用 */
//该字段没有使用
private Integer poolType;
//人员池
private List<Person> personPool;
//该字段无实际意义
private Integer updateType;
/** 删除时使用 */
private List<HashMap<String, String>> personIds;
//0:从特征池中删除personIds中的数据，1：删除poolId特征池
private Integer flushPool;
/** 查询时使用 */
//查询特征池列表
private Integer listAll;
/** 人员匹配时用 */
//暂时无用
private Integer personType;
//匹配的信息
private Person person;
//匹配单个特征池id
private String personPoolId;
//匹配多个特征池id，两个都传会加一块
private List<String> unionPersonPoolId;
//需要几条匹配结果
private Integer size;
//是否聚合，相当于关系型数据库group by
private Boolean agg = false;
//暂时无用
private FaceFeature newFaceFeature;
public void setPoolId(String poolId) {
......
src/main/java/com/viontech/match/runner/ILM.java 0 → 100644
package com.viontech.match.runner;
import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.AcknowledgedResponse;
import org.elasticsearch.client.indexlifecycle.DeleteAction;
import org.elasticsearch.client.indexlifecycle.LifecycleAction;
import org.elasticsearch.client.indexlifecycle.LifecyclePolicy;
import org.elasticsearch.client.indexlifecycle.Phase;
import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
import org.elasticsearch.client.indexlifecycle.SetPriorityAction;
import org.elasticsearch.common.unit.TimeValue;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.stereotype.Component;
import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit;
/**
* Author: inggg
* Date: 2022/05/26 16:53
*/
@Component
@Slf4j
public class ILM implements CommandLineRunner {
/**
* 生命周期名字
*/
public final static String LIFECYCLE_NAME = "delete-pool-after-days";
@Value("${delete.after.days:5}")
private Integer deleteAfterDays;
@Autowired
private RestHighLevelClient restHighLevelClient;
@Override
public void run(String... args) throws Exception {
Map<String, Phase> phases = new HashMap<>();
Map<String, LifecycleAction> setPriorityAction = new HashMap<>();
setPriorityAction.put(SetPriorityAction.NAME, new SetPriorityAction(100));
phases.put("hot", new Phase("hot", TimeValue.ZERO, setPriorityAction));
Map<String, LifecycleAction> deleteActions =
Collections.singletonMap(DeleteAction.NAME, new DeleteAction());
phases.put("delete", new Phase("delete",
new TimeValue(deleteAfterDays, TimeUnit.DAYS), deleteActions));
LifecyclePolicy policy = new LifecyclePolicy(LIFECYCLE_NAME, phases);
PutLifecyclePolicyRequest lifecyclePolicyRequest = new PutLifecyclePolicyRequest(policy);
AcknowledgedResponse acknowledgedResponse = restHighLevelClient.indexLifecycle().putLifecyclePolicy(lifecyclePolicyRequest, RequestOptions.DEFAULT);
if (acknowledgedResponse.isAcknowledged()) {
log.info(LIFECYCLE_NAME + "生命周期创建完成");
}
}
}
src/main/java/com/viontech/match/service/PoolService.java
......@@ -6,6 +6,7 @@ import com.viontech.match.config.Constant;
import com.viontech.match.entity.PoolInfo;
import com.viontech.match.entity.vo.RequestVo;
import com.viontech.match.entity.vo.ResponseVo;
import com.viontech.match.runner.ILM;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.http.HttpEntity;
......@@ -73,7 +74,7 @@ public class PoolService {
public ResponseVo createPool(RequestVo requestVo, boolean addPerson) throws Exception {
String rid = requestVo.getRid();
String poolId = requestVo.getPoolId();
log.info("特征池创建操作开始:[{}}", poolId);
log.info("特征池创建操作开始:[{}},IML:{}", poolId, requestVo.isUseILMPolicy());
try {
CreateIndexRequest createIndexRequest = new CreateIndexRequest(poolId);
XContentBuilder builder = getCreateIndexContentBuilder();
......@@ -81,6 +82,9 @@ public class PoolService {
Settings.Builder setting = Settings.builder();
setting.put("index.number_of_shards", shards);
setting.put("index.number_of_replicas", replicas);
if (requestVo.isUseILMPolicy()) {
setting.put("index.lifecycle.name", ILM.LIFECYCLE_NAME);
}
createIndexRequest.settings(setting);
CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
......@@ -94,7 +98,7 @@ public class PoolService {
}
}
log.info("特征池创建操作完成:[{}]", poolId);
log.info("特征池创建操作完成:[{}],ILM:{}", poolId, requestVo.isUseILMPolicy());
return ResponseVo.success(rid);
} catch (ElasticsearchStatusException e) {
if (e.status() == RestStatus.BAD_REQUEST && e.getMessage().contains(Constant.STR_POOL_ID_ALREADY_EXISTS)) {
......
src/main/resources/application-option.properties
# other
#other
server.port=12000
spring.jackson.time-zone=GMT+8
spring.jackson.date-format=yyyy-MM-dd HH:mm:ss
# es
spring.elasticsearch.rest.uris=http://127.0.0.1:9200
#es
spring.elasticsearch.rest.uris=http://192.168.1.106:9200
spring.elasticsearch.rest.username=elastic
spring.elasticsearch.rest.password=vion2021
spring.elasticsearch.rest.password=viontech
#\u5206\u7247\u6570\u91CF
vion.index.number_of_shards=1
#\u526F\u672C\u6570\u91CF
vion.index.number_of_replicas=0
#delete-pool-after-days\u51E0\u5929\u540E\u5220\u9664\u5929\u6570\u914D\u7F6E
delete.after.days=5
\ No newline at end of file
src/main/resources/logback-spring.xml
......@@ -9,8 +9,10 @@
<contextName>logback</contextName>
<!-- name的值是变量的名称，value的值时变量定义的值。通过定义的值会被插入到logger上下文中。定义变量后，可以使“${}”来使用变量。 -->
<property name="log.path" value="logs"/>
<property name="pattern" value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] [%thread] %logger{50} - %msg%n"/>
<property name="pattern" value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] [%-5thread] [%49c{1}:%-4L] %msg%n"/>
<!--
<property name="pattern" value="[%d{yyyy-MM-dd HH:mm:ss.SSS}] [%-5level] [%thread] %logger{50} - %msg%n" />
-->
<!--输出到控制台-->
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<!--此日志appender是为开发使用，只配置最底级别，控制台输出的日志级别是大于或等于此级别的日志信息-->
......
src/test/java/com/viontech/match/Test0.java deleted 100644 → 0
package com.viontech.match;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.viontech.match.entity.PoolInfo;
import org.apache.http.HttpEntity;
import org.elasticsearch.client.Request;
import org.elasticsearch.client.Response;
import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestHighLevelClient;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.Map;
/**
* .
*
* @author 谢明辉
* @date 2020/11/20
*/
@SpringBootTest
@RunWith(SpringRunner.class)
public class Test0 {
@Resource
private RestHighLevelClient client;
@Resource
private ObjectMapper objectMapper;
@Test
public void ttt() throws IOException {
LinkedList<PoolInfo> poolInfos = new LinkedList<>();
RestClient lowLevelClient = client.getLowLevelClient();
Request request = new Request("GET", "/_stats");
Response response = lowLevelClient.performRequest(request);
HttpEntity entity = response.getEntity();
InputStream content = entity.getContent();
HashMap responseMap = objectMapper.readValue(content, HashMap.class);
HashMap<String, HashMap> indices1 = (HashMap<String, HashMap>) responseMap.get("indices");
for (Map.Entry<String, HashMap> entry : indices1.entrySet()) {
String poolId = entry.getKey();
HashMap value = entry.getValue();
HashMap<String, HashMap> primaries = (HashMap<String, HashMap>) value.get("primaries");
HashMap docs = primaries.get("docs");
Integer count = (Integer) docs.get("count");
PoolInfo poolInfo = new PoolInfo();
poolInfo.setPersonCount(count.longValue());
poolInfo.setPoolId(poolId);
poolInfos.add(poolInfo);
}
System.out.println(1);
}
}
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!