Skip to content

谢明辉 / VVAS-Match

Go to a project
Commit 3bd562f2 by 姚冰
Options

[chg] es8合并索引

1 parent 1c4ec01c
Inline Side-by-side
Showing 7 changed files with 139 additions and 107 deletions
src/main/java/com/viontech/match/controller/MainController.java
......@@ -49,6 +49,12 @@ public class MainController {
return personService.updatePerson(requestVo);
case CountMatchPerson:
return personService.countMatchPerson(requestVo);
case DelPoolData:
return poolService.deletePoolData(requestVo);
case DelStaffPoolData:
return poolService.deleteStaffPoolData(requestVo);
case UpdateStaff:
return personService.updateStaff(requestVo);
default:
return ResponseVo.commandNotFound(rid);
}
......
src/main/java/com/viontech/match/entity/PersonInfo.java
......@@ -23,7 +23,9 @@ public class PersonInfo {
private Long gateId;
private String direction;
public PersonInfo(String unid, String personId, Double[] body,Double[] data, Integer age, String gender, Integer bodyType, Date countTime, String fid, String channelSerialNum, Long gateId, String direction) {
private Long mallId;
public PersonInfo(String unid, String personId, Double[] body,Double[] data, Integer age, String gender, Integer bodyType, Date countTime, String fid, String channelSerialNum, Long gateId, String direction, Long mallId) {
this.unid = unid;
this.personId = personId;
this.body = body;
......@@ -37,5 +39,6 @@ public class PersonInfo {
this.gateId = gateId;
this.direction = direction;
this._score = "1";
this.mallId = mallId;
}
}
src/main/java/com/viontech/match/entity/SearchResultHit.java
......@@ -16,7 +16,7 @@ public class SearchResultHit {
private String _score;
private Integer age;
private String gender;
private Integer bodyType;
private Integer body_type;
// @DateTimeFormat(pattern = "yyyy-MM-dd HH:mm:ss.SSS")
private String counttime;
private String fid;
......
src/main/java/com/viontech/match/entity/vo/RequestVo.java
......@@ -67,6 +67,10 @@ public class RequestVo {
//暂时无用
private FaceFeature newFaceFeature;
private Long mallId;
private List<String> fidList;
public void setPoolId(String poolId) {
this.poolId = poolId.toLowerCase();
}
......
src/main/java/com/viontech/match/enumeration/CommandEnum.java
......@@ -14,5 +14,8 @@ public enum CommandEnum {
QueryPersonPool,
MatchPerson,
UpdatePerson,
CountMatchPerson
CountMatchPerson,
DelPoolData,
DelStaffPoolData,
UpdateStaff;
}
src/main/java/com/viontech/match/service/PersonService.java
......@@ -14,9 +14,14 @@ import co.elastic.clients.elasticsearch.core.search.SourceConfig;
import co.elastic.clients.elasticsearch.indices.DeleteIndexResponse;
import co.elastic.clients.elasticsearch.ingest.simulate.Document;
import co.elastic.clients.json.JsonData;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.aggregations.Aggregation;
import co.elastic.clients.elasticsearch._types.aggregations.AggregationBuilders;
import co.elastic.clients.elasticsearch._types.query_dsl.BoolQuery;
import co.elastic.clients.elasticsearch._types.query_dsl.Query;
import co.elastic.clients.elasticsearch.core.SearchRequest;
import co.elastic.clients.elasticsearch.core.SearchResponse;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray;
import com.fasterxml.jackson.annotation.JsonValue;
import com.viontech.keliu.model.BodyFeature;
import com.viontech.keliu.model.FaceFeature;
import com.viontech.keliu.model.Person;
......@@ -30,6 +35,7 @@ import com.viontech.match.util.Utils;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.client.RequestOptions;
import org.springframework.stereotype.Service;
import javax.annotation.Resource;
......@@ -52,8 +58,6 @@ public class PersonService {
private static final String[] FETCH_SOURCE = new String[]{"personId", "age", "gender", "fid", "counttime", "channelSerialNum", "body_type","unid"};
@Resource
private ElasticsearchClient client;
@Resource
private PoolService poolService;
......@@ -80,7 +84,13 @@ public class PersonService {
long startTime = System.currentTimeMillis();
try {
String[] indices = poolIds.toArray(new String[poolIds.size()]);
// SearchRequest request = new SearchRequest(indices);
// IndicesOptions defaultIndicesOptions = request.indicesOptions();
// EnumSet<IndicesOptions.Option> options = defaultIndicesOptions.getOptions();
// options.add(IndicesOptions.Option.IGNORE_UNAVAILABLE);
// EnumSet<IndicesOptions.WildcardStates> expandWildcards = defaultIndicesOptions.getExpandWildcards();
// IndicesOptions newIndicesOptions = new IndicesOptions(options, expandWildcards);
// request.indicesOptions(newIndicesOptions);
long count = 0;
List<BodyFeature> bodyFeatures = person.getBodyFeatures();
......@@ -169,8 +179,8 @@ public class PersonService {
pool.setPersonPoolId(id);
if (poolService.existPool(id)) {
pool.setStatus(0);
// List<Person> face = matchPerson(rid, id, requestVo.getPerson(), 0, size, agg);
// matchFaces.addAll(face);
List<Person> face = matchPerson(rid, id, requestVo.getPerson(), 0, size, agg);
matchFaces.addAll(face);
List<Person> body = matchPerson(rid, id, requestVo.getPerson(), 1, size, agg);
matchBodies.addAll(body);
} else {
......@@ -234,6 +244,25 @@ public class PersonService {
return ResponseVo.success(rid);
}
public ResponseVo updateStaff(RequestVo requestVo) {
String rid = requestVo.getRid();
Person person = requestVo.getPerson();
String personId = person.getPersonId();
String poolId = requestVo.getPoolId();
// log.info("店员修改操作开始,poolId:[{}],personId:[{}]", poolId, personId);
try {
if (!poolService.existPool(poolId)) {
poolService.createPool(requestVo, false);
}
BulkResponse bulkItemResponses = addPerson(poolId, Collections.singletonList(person));
} catch (Exception e) {
log.error("店员修改操作异常", e);
return ResponseVo.error(rid, "update failed");
}
log.info("店员修改操作完成,poolId:[{}],personId:[{}]", poolId, personId);
return ResponseVo.success(rid);
}
/**
* 添加人员
*
......@@ -255,6 +284,7 @@ public class PersonService {
String personId = person.getPersonId();
Date personCountTime = person.getCounttime();
String direction = person.getDirection();
Long mallId = person.getMallId();
//Long gateId = person.getGateId();
Integer bodyType = person.getBodyType();
String personChannelSerialNum = person.getChannelSerialNum();
......@@ -266,14 +296,7 @@ public class PersonService {
String fid = faceFeature.getFid();
String unid = faceFeature.getUnid();
Long gateId = faceFeature.getGateId();
// IndexRequest<PersonInfo> indexRequest = new IndexRequest.Builder<PersonInfo>().index(poolId)
// .document(new PersonInfo(0L, unid, personId, feature, age, gender, bodyType, personCountTime, fid, personChannelSerialNum, gateId, direction)).build();
// .source(XContentType.JSON, "personId", personId, "unid", unid,
// "data", feature, "fid", fid, "age", age, "gender", gender, "body_type", bodyType,
// "counttime", personCountTime == null ? null : Constant.DATE_FORMAT.get().format(personCountTime),
// "channelSerialNum", personChannelSerialNum, "gateId", gateId, "direction", direction);
// bulkRequest.add(indexRequest);
operationList.add(new BulkOperation.Builder().index(new IndexOperation.Builder<PersonInfo>().index(poolId).document(new PersonInfo(unid, personId, feature, null, age, gender, bodyType, personCountTime, fid, personChannelSerialNum, gateId, direction)).build()).build());
operationList.add(new BulkOperation.Builder().index(new IndexOperation.Builder<PersonInfo>().index(poolId).document(new PersonInfo(unid, personId, feature, null, age, gender, bodyType, personCountTime, fid, personChannelSerialNum, gateId, direction, mallId)).build()).build());
}
}
}
......@@ -296,14 +319,7 @@ public class PersonService {
Long gateId = bodyFeature.getGateId();
Date counttime = bodyFeature.getCounttime() == null ? personCountTime : bodyFeature.getCounttime();
String channelSerialNum = bodyFeature.getChannelSerialNum() == null ? personChannelSerialNum : person.getChannelSerialNum();
// IndexRequest<PersonInfo> indexRequest = new IndexRequest.Builder<PersonInfo>().index(poolId)
// .document(new PersonInfo(0L, unid, personId, feature, age, gender, bodyType, counttime, fid, channelSerialNum, gateId, direction)).build();
// .source(XContentType.JSON, "personId", personId, "unid", unid,
// "body", feature, "fid", fid, "age", age, "gender", gender, "body_type", bodyType,
// "counttime", counttime == null ? null : Constant.DATE_FORMAT.get().format(counttime)
// , "channelSerialNum", channelSerialNum, "gateId", gateId, "direction", direction);
// bulkRequest.add(indexRequest);
operationList.add(new BulkOperation.Builder().index(new IndexOperation.Builder<PersonInfo>().index(poolId).document(new PersonInfo(unid, personId, feature, null, age, gender, bodyType, personCountTime, fid, personChannelSerialNum, gateId, direction)).build()).build());
operationList.add(new BulkOperation.Builder().index(new IndexOperation.Builder<PersonInfo>().index(poolId).document(new PersonInfo(unid, personId, feature, null, age, gender, bodyType, counttime, fid, personChannelSerialNum, gateId, direction, mallId)).build()).build());
}
}
}
......@@ -332,6 +348,22 @@ public class PersonService {
return client.deleteByQuery(deleteByQueryRequest);
}
public DeleteByQueryResponse deletePersonByMallId(String poolId, Long mallId) throws IOException {
DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest.Builder().index(poolId)
.query(q -> q.match(m -> m.field("mallId").query(mallId)))
.refresh(true)
.build();
return client.deleteByQuery(deleteByQueryRequest);
}
public DeleteByQueryResponse deletePersonByFid(String poolId, List<String> fidList) throws IOException {
DeleteByQueryRequest deleteByQueryRequest = new DeleteByQueryRequest.Builder().index(poolId)
.query(QueryBuilders.terms().field("fid").terms(new TermsQueryField.Builder().value(fidList.stream().map(FieldValue::of).collect(Collectors.toList())).build()).build()._toQuery())
.refresh(true)
.build();
return client.deleteByQuery(deleteByQueryRequest);
}
/**
* 人员匹配入口
*
......@@ -381,7 +413,7 @@ public class PersonService {
matchResult.addAll(match0(searchRequest, agg));
}
} else {
// log.info("no face feature");
log.info("no face feature");
}
}
......@@ -415,15 +447,6 @@ public class PersonService {
private Query getScriptScoreQuery(Double[] feature, Person person, int type) {
return new Query.Builder().knn(getSearchSourceBuilder(feature, person, type)).build();
// return new ScriptScoreQuery.Builder()
// .query(getSearchSourceBuilder(feature, person, type)._toQuery())
// .script(new Script.Builder()
// .lang("painless")
// .source("_score * 100")
// .params("body", JsonData.of(0.0f))
// .build())
// .boost(1.0f)
// .build();
}
private BoolQuery getSearchSourceCountBuilder(Double[] feature, Person person, int type) {
......@@ -450,6 +473,10 @@ public class PersonService {
if (StringUtils.isNotBlank(person.getCompareDirection())) {
queries.add(QueryBuilders.term().field("direction").value(FieldValue.of(person.getCompareDirection())).build()._toQuery());
}
//根据mallId过滤
if (person.getMallId() != null) {
queries.add(QueryBuilders.term().field("mallId").value(FieldValue.of(person.getMallId())).build()._toQuery());
}
Date counttimeGTE = person.getCounttimeGTE();
Date counttimeLTE = person.getCounttimeLTE();
......@@ -499,6 +526,10 @@ public class PersonService {
queries.add(QueryBuilders.term().field("direction").value(FieldValue.of(person.getCompareDirection())).build()._toQuery());
}
if (person.getMallId() != null) {
queries.add(QueryBuilders.term().field("mallId").value(FieldValue.of(person.getMallId())).build()._toQuery());
}
Date counttimeGTE = person.getCounttimeGTE();
Date counttimeLTE = person.getCounttimeLTE();
if (counttimeGTE != null || counttimeLTE != null) {
......@@ -568,7 +599,7 @@ public class PersonService {
p.setAge(hit.getAge());
p.setGender((String) hit.getGender());
p.setChannelSerialNum(hit.getChannelSerialNum());
p.setBodyType(hit.getBodyType());
p.setBodyType(hit.getBody_type());
p.setCaptureUnid(hit.getUnid());
p.setCounttime(Optional.ofNullable((String)hit.getCounttime())
......
src/main/java/com/viontech/match/service/PoolService.java
......@@ -87,19 +87,15 @@ public class PoolService {
.numberOfReplicas(String.valueOf(replicas));
if (StringUtils.isNotEmpty(translogDurability)) {
settings.translog(t -> t.durability(TranslogDurability.Async));
// setting.put("index.translog.durability", translogDurability);
if (StringUtils.isNotEmpty(translogSyncInterval)) {
settings.translog(t -> t.syncInterval(i -> i.time(translogSyncInterval)));
// setting.put("index.translog.sync_interval", translogSyncInterval);
}
}
if (mergeThreadCount != null) {
// setting.put("index.merge.scheduler.max_thread_count", mergeThreadCount);
settings.merge(m -> m.scheduler(s -> s.maxThreadCount(mergeThreadCount)));
}
if (requestVo.isUseILMPolicy()) {
// setting.put("index.lifecycle.name", ILM.LIFECYCLE_NAME);
settings.lifecycle(l -> l.name(ILM.LIFECYCLE_NAME));
}
......@@ -166,6 +162,46 @@ public class PoolService {
}
}
public ResponseVo deletePoolData(RequestVo requestVo) throws Exception {
String rid = requestVo.getRid();
Integer flushPool = requestVo.getFlushPool();
String poolId = requestVo.getPoolId();
Long mallId = requestVo.getMallId();
log.info("特征池删除mallId:{}数据操作开始:[{}]", mallId,poolId);
try {
personService.deletePersonByMallId(poolId, mallId);
log.info("特征池删除mallId:{}操作完成:[{}]", mallId, poolId);
return ResponseVo.success(rid);
} catch (ElasticsearchException e) {
if (e.status() == 404) {
return ResponseVo.poolIdNotExists(rid);
} else {
return ResponseVo.error(rid, e.getMessage());
}
}
}
public ResponseVo deleteStaffPoolData(RequestVo requestVo) throws Exception {
String rid = requestVo.getRid();
Integer flushPool = requestVo.getFlushPool();
String poolId = requestVo.getPoolId();
List<String> fidList = requestVo.getFidList();
log.info("店员特征池删除数据操作开始:[{}]",poolId);
try {
personService.deletePersonByFid(poolId, fidList);
log.info("店员特征池删除操作完成:[{}]",poolId);
return ResponseVo.success(rid);
} catch (ElasticsearchException e) {
if (e.status() == 404) {
return ResponseVo.poolIdNotExists(rid);
} else {
return ResponseVo.error(rid, e.getMessage());
}
}
}
/**
* 修改特征池(添加人员)
*
......@@ -243,73 +279,22 @@ public class PoolService {
public TypeMapping.Builder getCreateIndexContentBuilder() throws IOException {
// JSONObject content = new JSONObject();
// JSONObject properties = new JSONObject();
// JSONObject data = new JSONObject();
// data.put("type", "dense_vector");
// data.put("dims", Constant.FACE_FEATURE_DIMS);
// JSONObject body = new JSONObject();
// body.put("type", "dense_vector");
// body.put("dims", Constant.BODY_FEATURE_DIMS_2048);
// JSONObject personId = new JSONObject();
// personId.put("type", "keyword");
//
// JSONObject unid = new JSONObject();
// unid.put("type", "keyword");
// JSONObject fid = new JSONObject();
// fid.put("type", "text");
// JSONObject age = new JSONObject();
// age.put("type", "integer");
// age.put("doc_values", false);
// age.put("index", false);
// JSONObject gender = new JSONObject();
// gender.put("type", "keyword");
// gender.put("doc_values", false);
// gender.put("index", false);
// JSONObject counttime = new JSONObject();
// counttime.put("type", "date");
// counttime.put("format", "yyyy-MM-dd HH:mm:ss");
// JSONObject channelSerialNum = new JSONObject();
// channelSerialNum.put("type", "keyword");
// JSONObject body_type = new JSONObject();
// body_type.put("type", "integer");
// body_type.put("doc_values", false);
// body_type.put("index", false);
// JSONObject direction = new JSONObject();
// direction.put("type", "keyword");
// JSONObject gateId = new JSONObject();
// gateId.put("type", "long");
// properties.put("data", data);
// properties.put("body", body);
// properties.put("personId", personId);
// properties.put("unid", unid);
// properties.put("fid", fid);
// properties.put("age", age);
// properties.put("gender", gender);
// properties.put("counttime", counttime);
// properties.put("channelSerialNum", channelSerialNum);
// properties.put("body_type", body_type);
// properties.put("gateId", gateId);
// properties.put("direction", direction);
//
// InputStream is = new ByteArrayInputStream(properties.toJSONString().getBytes());
Map<String, Property> properties1 = new HashMap<>();
properties1.put("direction", new Property.Builder().keyword(k -> k.index(true)).build());
properties1.put("fid", new Property.Builder().text(t -> t.index(true)).build());
properties1.put("gateId", new Property.Builder().long_(t -> t.index(true)).build());
properties1.put("personId", new Property.Builder().keyword(k -> k.index(true)).build());
properties1.put("body", new Property.Builder().denseVector(d -> d.dims(Constant.BODY_FEATURE_DIMS_2048).similarity("cosine")).build());
properties1.put("data", new Property.Builder().denseVector(d -> d.dims(Constant.FACE_FEATURE_DIMS).similarity("cosine")).build());
properties1.put("body_type", new Property.Builder().integer(i -> i.index(false).docValues(false)).build());
properties1.put("channelSerialNum", new Property.Builder().keyword(k -> k.index(true)).build());
properties1.put("age", new Property.Builder().integer(i -> i.index(false).docValues(false)).build());
properties1.put("gender", new Property.Builder().keyword(k -> k.index(false).docValues(false)).build());
properties1.put("counttime", new Property.Builder().date(d -> d.format("yyyy-MM-dd HH:mm:ss")).build());
properties1.put("unid", new Property.Builder().keyword(k -> k.index(true)).build());
Map<String, Property> properties = new HashMap<>();
properties.put("direction", new Property.Builder().keyword(k -> k.index(true)).build());
properties.put("fid", new Property.Builder().text(t -> t.index(true)).build());
properties.put("gateId", new Property.Builder().long_(t -> t.index(true)).build());
properties.put("personId", new Property.Builder().keyword(k -> k.index(true)).build());
properties.put("body", new Property.Builder().denseVector(d -> d.dims(Constant.BODY_FEATURE_DIMS_2048).similarity("cosine")).build());
properties.put("data", new Property.Builder().denseVector(d -> d.dims(Constant.FACE_FEATURE_DIMS).similarity("cosine")).build());
properties.put("body_type", new Property.Builder().integer(i -> i.index(false).docValues(false)).build());
properties.put("channelSerialNum", new Property.Builder().keyword(k -> k.index(true)).build());
properties.put("age", new Property.Builder().integer(i -> i.index(false).docValues(false)).build());
properties.put("gender", new Property.Builder().keyword(k -> k.index(false).docValues(false)).build());
properties.put("counttime", new Property.Builder().date(d -> d.format("yyyy-MM-dd HH:mm:ss")).build());
properties.put("unid", new Property.Builder().keyword(k -> k.index(true)).build());
properties.put("mallId", new Property.Builder().keyword(k -> k.index(true)).build());
// builder.properties(properties1);
TypeMapping.Builder builder = new TypeMapping.Builder().properties(properties1);
TypeMapping.Builder builder = new TypeMapping.Builder().properties(properties);
return builder;
}
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!