Skip to content

谢明辉 / VVAS-Match

Go to a project
Commit 08dcc635 by 姚冰
Options

[chg] 分支合并，提高效率，异步暂时关闭

1 parent 5607af53
Inline Side-by-side
Showing 6 changed files with 101 additions and 34 deletions
pom.xml
......@@ -43,8 +43,64 @@
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<artifactId>tomcat-embed-core</artifactId>
<groupId>org.apache.tomcat.embed</groupId>
</exclusion>
<exclusion>
<artifactId>tomcat-embed-el</artifactId>
<groupId>org.apache.tomcat.embed</groupId>
</exclusion>
<exclusion>
<artifactId>tomcat-embed-websocket</artifactId>
<groupId>org.apache.tomcat.embed</groupId>
</exclusion>
<exclusion>
<artifactId>spring-web</artifactId>
<groupId>org.springframework</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-core</artifactId>
<version>10.1.43</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-annotations-api</artifactId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-el</artifactId>
<version>10.1.43</version>
<scope>compile</scope>
</dependency>
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-websocket</artifactId>
<version>10.1.43</version>
<scope>compile</scope>
<exclusions>
<exclusion>
<groupId>org.apache.tomcat</groupId>
<artifactId>tomcat-annotations-api</artifactId>
</exclusion>
<exclusion>
<artifactId>tomcat-embed-core</artifactId>
<groupId>org.apache.tomcat.embed</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<artifactId>spring-web</artifactId>
<groupId>org.springframework</groupId>
<version>6.2.8</version>
</dependency>
<!-- <dependency>-->
<!-- <groupId>org.elasticsearch.client</groupId>-->
<!-- <artifactId>elasticsearch-rest-high-level-client</artifactId>-->
......@@ -74,6 +130,7 @@
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
<version>3.18.0</version>
</dependency>
<dependency>
......
src/main/java/com/viontech/match/controller/MainController.java
......@@ -47,12 +47,12 @@ public class MainController {
case MatchPerson:
ResponseVo matchPerson = personService.matchPerson(requestVo);
if (matchPerson.getMatchBodies() != null && matchPerson.getMatchBodies().size() > 0) {
requestVo.getPerson().setPersonId(matchPerson.getMatchBodies().get(0).getPersonId());
} else if(matchPerson.getMatchPersons() != null && matchPerson.getMatchPersons().size() > 0) {
requestVo.getPerson().setPersonId(matchPerson.getMatchPersons().get(0).getPersonId());
}
poolService.asyncModifyPool(requestVo.getPersonPoolId(), requestVo.getPerson());
// if (matchPerson.getMatchBodies() != null && matchPerson.getMatchBodies().size() > 0) {
// requestVo.getPerson().setPersonId(matchPerson.getMatchBodies().get(0).getPersonId());
// } else if(matchPerson.getMatchPersons() != null && matchPerson.getMatchPersons().size() > 0) {
// requestVo.getPerson().setPersonId(matchPerson.getMatchPersons().get(0).getPersonId());
// }
// poolService.asyncModifyPool(requestVo.getPersonPoolId(), requestVo.getPerson());
return matchPerson;
case UpdatePerson:
return personService.updatePerson(requestVo);
......
src/main/java/com/viontech/match/entity/PersonInfo.java
......@@ -10,15 +10,15 @@ import java.util.Date;
public class PersonInfo {
private String unid;
private String personId;
private String _score;
// private String _score;
private Double[] body;
private Double[] data;
private Integer age;
private String gender;
private Integer bodyType;
// private Integer bodyType;
@JsonFormat(pattern = "yyyy-MM-dd HH:mm:ss", timezone = "GMT+8")
private Date counttime;
private String fid;
// private String fid;
private String channelSerialNum;
private Long gateId;
private String direction;
......@@ -32,13 +32,13 @@ public class PersonInfo {
this.data = data;
this.age = age;
this.gender = gender;
this.bodyType = bodyType;
// this.bodyType = bodyType;
this.counttime = countTime;
this.fid = fid;
// this.fid = fid;
this.channelSerialNum = channelSerialNum;
this.gateId = gateId;
this.direction = direction;
this._score = "1";
// this._score = "1";
this.mallId = mallId;
}
}
src/main/java/com/viontech/match/service/PersonService.java
......@@ -5,16 +5,9 @@ import co.elastic.clients.elasticsearch._types.*;
import co.elastic.clients.elasticsearch._types.aggregations.*;
import co.elastic.clients.elasticsearch._types.query_dsl.*;
import co.elastic.clients.elasticsearch.core.*;
import co.elastic.clients.elasticsearch.core.bulk.BulkOperation;
import co.elastic.clients.elasticsearch.core.bulk.IndexOperation;
import co.elastic.clients.elasticsearch.core.knn_search.KnnSearchQuery;
import co.elastic.clients.elasticsearch.core.search.Hit;
import co.elastic.clients.elasticsearch.core.search.HitsMetadata;
import co.elastic.clients.elasticsearch.core.search.SourceConfig;
import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse;
import co.elastic.clients.elasticsearch.ingest.simulate.Document;
import co.elastic.clients.json.JsonData;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.aggregations.Aggregation;
import co.elastic.clients.elasticsearch._types.aggregations.AggregationBuilders;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
......@@ -27,7 +20,6 @@ import com.viontech.keliu.model.FaceFeature;
import com.viontech.keliu.model.Person;
import com.viontech.keliu.model.Pool;
import com.viontech.match.config.Constant;
import com.viontech.match.entity.PersonInfo;
import com.viontech.match.entity.SearchResultHit;
import com.viontech.match.entity.vo.RequestVo;
import com.viontech.match.entity.vo.ResponseVo;
......@@ -35,7 +27,6 @@ import com.viontech.match.util.Utils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.RequestOptions;
import org.springframework.stereotype.Service;
import jakarta.annotation.Resource;
......
src/main/java/com/viontech/match/service/PoolService.java
......@@ -18,6 +18,7 @@ import com.viontech.match.entity.PoolInfo;
import com.viontech.match.entity.vo.RequestVo;
import com.viontech.match.entity.vo.ResponseVo;
import com.viontech.match.runner.ILM;
import jakarta.annotation.Resource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
......@@ -31,8 +32,7 @@ import org.springframework.scheduling.annotation.Async;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Service;
import jakarta.annotation.Resource;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.util.*;
......@@ -68,6 +68,15 @@ public class PoolService {
@Value("${vion.index.merge.scheduler.max_thread_count}")
private Integer mergeThreadCount;
@Value("${vion.index.options.type:int8_hnsw}")
private String indexOptionType;
@Value("${vion.index.options.m:16}")
private Integer indexOptionM;
@Value("${vion.index.options.efConstruction:100}")
private Integer indexOptionEfConstruction;
@Value("${vion.index.options.confidenceInterval:0.99f}")
private Float indexOptionConfidenceInterval;
private final static ConcurrentMap<String, Boolean> poolMap = new ConcurrentHashMap<>();
/**
......@@ -255,7 +264,7 @@ public class PoolService {
return;
}
try {
BulkResponse bulkItemResponses = personService.addPerson(poolId, Arrays.asList(person));
BulkResponse bulkItemResponses = personPoolService.addPerson(poolId, Arrays.asList(person));
if (bulkItemResponses != null && bulkItemResponses.errors()) {
log.info(bulkItemResponses.items().toString());
} else if (bulkItemResponses == null) {
......
src/main/resources/application-option.properties
......@@ -3,14 +3,16 @@ server.port=12000
spring.jackson.time-zone=GMT+8
spring.jackson.date-format=yyyy-MM-dd HH:mm:ss
#es
#spring.elasticsearch.rest.uris=http://182.92.184.40:9200
spring.elasticsearch.uris=http://39.155.171.242:9201
spring.elasticsearch.username=elastic
spring.elasticsearch.password=vion2021
spring.elasticsearch.socketTimeout=5000
spring.elasticsearch.ioThreadCount=20
spring.elasticsearch.maxConnTotal=100
spring.elasticsearch.maxConnPerRoute=20
#spring.elasticsearch.rest.uris=http://182.92.184.40:9205
#spring.elasticsearch.rest.uris=http://192.168.1.64:9200
#spring.elasticsearch.rest.uris=http://10.0.14.26:9200
spring.elasticsearch.rest.uris=http://127.0.0.1:9200
spring.elasticsearch.rest.username=elastic
spring.elasticsearch.rest.password=0Qqkw1X2gMR6OZ+GxcUn
spring.elasticsearch.rest.socketTimeout=5000
spring.elasticsearch.rest.ioThreadCount=20
spring.elasticsearch.rest.maxConnTotal=100
spring.elasticsearch.rest.maxConnPerRoute=20
#\u5206\u7247\u6570\u91CF
vion.index.number_of_shards=1
#\u526F\u672C\u6570\u91CF
......@@ -24,4 +26,12 @@ vion.index.merge.scheduler.max_thread_count=1
#delete-pool-after-days\u51E0\u5929\u540E\u5220\u9664\u5929\u6570\u914D\u7F6E
delete.after.days=5
logging.level.com.viontech.match=info
\ No newline at end of file
logging.level.com.viontech.match=info
#default int8_hnsw
vion.index.options.type=int8_hnsw
#default 16
vion.index.options.m=16
#default 100
vion.index.options.efConstruction=100
#default 0.99f
vion.index.options.confidenceInterval=1.0f
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!