Skip to content

谢明辉 / VVAS-Match

Go to a project
Commit 08dcc635 by 姚冰
Options

[chg] 分支合并，提高效率，异步暂时关闭

1 parent 5607af53
Inline Side-by-side
Showing 6 changed files with 100 additions and 33 deletions
pom.xml
...@@ -43,8 +43,64 @@ ...@@ -43,8 +43,64 @@
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<artifactId>tomcat-embed-core</artifactId>
<groupId>org.apache.tomcat.embed</groupId>
</exclusion>
<exclusion>
<artifactId>tomcat-embed-el</artifactId>
<groupId>org.apache.tomcat.embed</groupId>
</exclusion>
<exclusion>
<artifactId>tomcat-embed-websocket</artifactId>
<groupId>org.apache.tomcat.embed</groupId>
</exclusion>
<exclusion>
<artifactId>spring-web</artifactId>
<groupId>org.springframework</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>10.1.43</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-annotations-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-el</artifactId>
<version>10.1.43</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-websocket</artifactId>
<version>10.1.43</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-annotations-api</artifactId>
</exclusion>
<exclusion>
<artifactId>tomcat-embed-core</artifactId>
<groupId>org.apache.tomcat.embed</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<artifactId>spring-web</artifactId>
<groupId>org.springframework</groupId>
<version>6.2.8</version>
</dependency> </dependency>
<!-- <dependency>--> <!-- <dependency>-->
<!-- <groupId>org.elasticsearch.client</groupId>--> <!-- <groupId>org.elasticsearch.client</groupId>-->
<!-- <artifactId>elasticsearch-rest-high-level-client</artifactId>--> <!-- <artifactId>elasticsearch-rest-high-level-client</artifactId>-->
...@@ -74,6 +130,7 @@ ...@@ -74,6 +130,7 @@
<dependency> <dependency>
<groupId>org.apache.commons</groupId> <groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId> <artifactId>commons-lang3</artifactId>
<version>3.18.0</version>
</dependency> </dependency>
<dependency> <dependency>
......
src/main/java/com/viontech/match/controller/MainController.java
...@@ -47,12 +47,12 @@ public class MainController { ...@@ -47,12 +47,12 @@ public class MainController {
case MatchPerson: case MatchPerson:
ResponseVo matchPerson = personService.matchPerson(requestVo); ResponseVo matchPerson = personService.matchPerson(requestVo);
if (matchPerson.getMatchBodies() != null && matchPerson.getMatchBodies().size() > 0) { // if (matchPerson.getMatchBodies() != null && matchPerson.getMatchBodies().size() > 0) {
requestVo.getPerson().setPersonId(matchPerson.getMatchBodies().get(0).getPersonId()); // requestVo.getPerson().setPersonId(matchPerson.getMatchBodies().get(0).getPersonId());
} else if(matchPerson.getMatchPersons() != null && matchPerson.getMatchPersons().size() > 0) { // } else if(matchPerson.getMatchPersons() != null && matchPerson.getMatchPersons().size() > 0) {
requestVo.getPerson().setPersonId(matchPerson.getMatchPersons().get(0).getPersonId()); // requestVo.getPerson().setPersonId(matchPerson.getMatchPersons().get(0).getPersonId());
} // }
poolService.asyncModifyPool(requestVo.getPersonPoolId(), requestVo.getPerson()); // poolService.asyncModifyPool(requestVo.getPersonPoolId(), requestVo.getPerson());
return matchPerson; return matchPerson;
case UpdatePerson: case UpdatePerson:
return personService.updatePerson(requestVo); return personService.updatePerson(requestVo);
......
src/main/java/com/viontech/match/entity/PersonInfo.java
...@@ -10,15 +10,15 @@ import java.util.Date; ...@@ -10,15 +10,15 @@ import java.util.Date;
public class PersonInfo { public class PersonInfo {
private String unid; private String unid;
private String personId; private String personId;
private String _score; // private String _score;
private Double[] body; private Double[] body;
private Double[] data; private Double[] data;
private Integer age; private Integer age;
private String gender; private String gender;
private Integer bodyType; // private Integer bodyType;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8") @JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date counttime; private Date counttime;
private String fid; // private String fid;
private String channelSerialNum; private String channelSerialNum;
private Long gateId; private Long gateId;
private String direction; private String direction;
...@@ -32,13 +32,13 @@ public class PersonInfo { ...@@ -32,13 +32,13 @@ public class PersonInfo {
this.data = data; this.data = data;
this.age = age; this.age = age;
this.gender = gender; this.gender = gender;
this.bodyType = bodyType; // this.bodyType = bodyType;
this.counttime = countTime; this.counttime = countTime;
this.fid = fid; // this.fid = fid;
this.channelSerialNum = channelSerialNum; this.channelSerialNum = channelSerialNum;
this.gateId = gateId; this.gateId = gateId;
this.direction = direction; this.direction = direction;
this._score = "1"; // this._score = "1";
this.mallId = mallId; this.mallId = mallId;
} }
} }
src/main/java/com/viontech/match/service/PersonService.java
...@@ -5,16 +5,9 @@ import co.elastic.clients.elasticsearch._types.*; ...@@ -5,16 +5,9 @@ import co.elastic.clients.elasticsearch._types.*;
import co.elastic.clients.elasticsearch._types.aggregations.*; import co.elastic.clients.elasticsearch._types.aggregations.*;
import co.elastic.clients.elasticsearch._types.query_dsl.*; import co.elastic.clients.elasticsearch._types.query_dsl.*;
import co.elastic.clients.elasticsearch.core.*; import co.elastic.clients.elasticsearch.core.*;
import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
import co.elastic.clients.elasticsearch.core.bulk.IndexOperation;
import co.elastic.clients.elasticsearch.core.knn_search.KnnSearchQuery;
import co.elastic.clients.elasticsearch.core.search.Hit; import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.core.search.HitsMetadata; import co.elastic.clients.elasticsearch.core.search.HitsMetadata;
import co.elastic.clients.elasticsearch.core.search.SourceConfig; import co.elastic.clients.elasticsearch.core.search.SourceConfig;
import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse;
import co.elastic.clients.elasticsearch.ingest.simulate.Document;
import co.elastic.clients.json.JsonData;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.aggregations.Aggregation; import co.elastic.clients.elasticsearch._types.aggregations.Aggregation;
import co.elastic.clients.elasticsearch._types.aggregations.AggregationBuilders; import co.elastic.clients.elasticsearch._types.aggregations.AggregationBuilders;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery; import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
...@@ -27,7 +20,6 @@ import com.viontech.keliu.model.FaceFeature; ...@@ -27,7 +20,6 @@ import com.viontech.keliu.model.FaceFeature;
import com.viontech.keliu.model.Person; import com.viontech.keliu.model.Person;
import com.viontech.keliu.model.Pool; import com.viontech.keliu.model.Pool;
import com.viontech.match.config.Constant; import com.viontech.match.config.Constant;
import com.viontech.match.entity.PersonInfo;
import com.viontech.match.entity.SearchResultHit; import com.viontech.match.entity.SearchResultHit;
import com.viontech.match.entity.vo.RequestVo; import com.viontech.match.entity.vo.RequestVo;
import com.viontech.match.entity.vo.ResponseVo; import com.viontech.match.entity.vo.ResponseVo;
...@@ -35,7 +27,6 @@ import com.viontech.match.util.Utils; ...@@ -35,7 +27,6 @@ import com.viontech.match.util.Utils;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.RequestOptions;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import jakarta.annotation.Resource; import jakarta.annotation.Resource;
......
src/main/java/com/viontech/match/service/PoolService.java
...@@ -18,6 +18,7 @@ import com.viontech.match.entity.PoolInfo; ...@@ -18,6 +18,7 @@ import com.viontech.match.entity.PoolInfo;
import com.viontech.match.entity.vo.RequestVo; import com.viontech.match.entity.vo.RequestVo;
import com.viontech.match.entity.vo.ResponseVo; import com.viontech.match.entity.vo.ResponseVo;
import com.viontech.match.runner.ILM; import com.viontech.match.runner.ILM;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils; import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
...@@ -31,8 +32,7 @@ import org.springframework.scheduling.annotation.Async; ...@@ -31,8 +32,7 @@ import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled; import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import jakarta.annotation.Resource;
import java.io.ByteArrayInputStream;
import java.io.IOException; import java.io.IOException;
import java.io.InputStream; import java.io.InputStream;
import java.util.*; import java.util.*;
...@@ -68,6 +68,15 @@ public class PoolService { ...@@ -68,6 +68,15 @@ public class PoolService {
@Value("${vion.index.merge.scheduler.max_thread_count}") @Value("${vion.index.merge.scheduler.max_thread_count}")
private Integer mergeThreadCount; private Integer mergeThreadCount;
@Value("${vion.index.options.type:int8_hnsw}")
private String indexOptionType;
@Value("${vion.index.options.m:16}")
private Integer indexOptionM;
@Value("${vion.index.options.efConstruction:100}")
private Integer indexOptionEfConstruction;
@Value("${vion.index.options.confidenceInterval:0.99f}")
private Float indexOptionConfidenceInterval;
private final static ConcurrentMap<String, Boolean> poolMap = new ConcurrentHashMap<>(); private final static ConcurrentMap<String, Boolean> poolMap = new ConcurrentHashMap<>();
/** /**
...@@ -255,7 +264,7 @@ public class PoolService { ...@@ -255,7 +264,7 @@ public class PoolService {
return; return;
} }
try { try {
BulkResponse bulkItemResponses = personService.addPerson(poolId, Arrays.asList(person)); BulkResponse bulkItemResponses = personPoolService.addPerson(poolId, Arrays.asList(person));
if (bulkItemResponses != null && bulkItemResponses.errors()) { if (bulkItemResponses != null && bulkItemResponses.errors()) {
log.info(bulkItemResponses.items().toString()); log.info(bulkItemResponses.items().toString());
} else if (bulkItemResponses == null) { } else if (bulkItemResponses == null) {
......
src/main/resources/application-option.properties
...@@ -3,14 +3,16 @@ server.port=12000 ...@@ -3,14 +3,16 @@ server.port=12000
spring.jackson.time-zone=GMT+8 spring.jackson.time-zone=GMT+8
spring.jackson.date-format=yyyy-MM-dd HH:mm:ss spring.jackson.date-format=yyyy-MM-dd HH:mm:ss
#es #es
#spring.elasticsearch.rest.uris=http://182.92.184.40:9200 #spring.elasticsearch.rest.uris=http://182.92.184.40:9205
spring.elasticsearch.uris=http://39.155.171.242:9201 #spring.elasticsearch.rest.uris=http://192.168.1.64:9200
spring.elasticsearch.username=elastic #spring.elasticsearch.rest.uris=http://10.0.14.26:9200
spring.elasticsearch.password=vion2021 spring.elasticsearch.rest.uris=http://127.0.0.1:9200
spring.elasticsearch.socketTimeout=5000 spring.elasticsearch.rest.username=elastic
spring.elasticsearch.ioThreadCount=20 spring.elasticsearch.rest.password=0Qqkw1X2gMR6OZ+GxcUn
spring.elasticsearch.maxConnTotal=100 spring.elasticsearch.rest.socketTimeout=5000
spring.elasticsearch.maxConnPerRoute=20 spring.elasticsearch.rest.ioThreadCount=20
spring.elasticsearch.rest.maxConnTotal=100
spring.elasticsearch.rest.maxConnPerRoute=20
#\u5206\u7247\u6570\u91CF #\u5206\u7247\u6570\u91CF
vion.index.number_of_shards=1 vion.index.number_of_shards=1
#\u526F\u672C\u6570\u91CF #\u526F\u672C\u6570\u91CF
...@@ -25,3 +27,11 @@ vion.index.merge.scheduler.max_thread_count=1 ...@@ -25,3 +27,11 @@ vion.index.merge.scheduler.max_thread_count=1
delete.after.days=5 delete.after.days=5
logging.level.com.viontech.match=info logging.level.com.viontech.match=info
#default int8_hnsw
vion.index.options.type=int8_hnsw
#default 16
vion.index.options.m=16
#default 100
vion.index.options.efConstruction=100
#default 0.99f
vion.index.options.confidenceInterval=1.0f
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!