动态匹配

xmh
Commit ac310f68 authored Apr 26, 2021 by xmh
Showing 5 changed files with 136 additions and 41 deletions
pom.xml
src/main/java/com/viontech/match/config/Constant.java
src/main/java/com/viontech/match/service/PersonService.java
src/main/java/com/viontech/match/service/PoolService.java
src/main/resources/application-option.properties
--- a/pom.xml
+++ b/pom.xml
@@ -37,7 +37,7 @@
        <dependency>
            <groupId>com.viontech.keliu</groupId>
            <artifactId>keliu-base</artifactId>
-            <version>6.0.9</version>
+            <version>6.0.9-SNAPSHOT</version>
            <exclusions>
                <exclusion>
                    <artifactId>pagehelper-spring-boot-starter</artifactId>
@@ -62,7 +62,7 @@
        <dependency>
            <groupId>com.viontech.keliu</groupId>
            <artifactId>AlgApiClient</artifactId>
-            <version>6.0.5</version>
+            <version>6.0.6-SNAPSHOT</version>
            <exclusions>
                <exclusion>
                    <groupId>javax</groupId>

--- a/src/main/java/com/viontech/match/config/Constant.java
+++ b/src/main/java/com/viontech/match/config/Constant.java
 package com.viontech.match.config;
+import java.text.SimpleDateFormat;
 /**
 * 常量
 *
@@ -25,4 +27,6 @@ public class Constant {
    public final static int FACE_MATCH_RESULT_SIZE = 5;
    /** 人体匹配结果的数量 */
    public final static int BODY_MATCH_RESULT_SIZE = 10;
+    public final static ThreadLocal<SimpleDateFormat> DATE_FORMAT = ThreadLocal.withInitial(() -> new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"));
 }
--- a/src/main/java/com/viontech/match/service/PersonService.java
+++ b/src/main/java/com/viontech/match/service/PersonService.java
@@ -19,7 +19,9 @@ import org.elasticsearch.action.search.SearchResponse;
 import org.elasticsearch.client.RequestOptions;
 import org.elasticsearch.client.RestHighLevelClient;
 import org.elasticsearch.common.xcontent.XContentType;
+import org.elasticsearch.index.query.BoolQueryBuilder;
 import org.elasticsearch.index.query.QueryBuilders;
+import org.elasticsearch.index.query.RangeQueryBuilder;
 import org.elasticsearch.index.query.TermQueryBuilder;
 import org.elasticsearch.index.query.functionscore.ScriptScoreQueryBuilder;
 import org.elasticsearch.index.reindex.BulkByScrollResponse;
@@ -48,6 +50,7 @@ import java.util.stream.Stream;
 @Service
 @Slf4j
 public class PersonService {
+    private static final String[] FETCH_SOURCE = new String[]{"personId", "age", "gender", "fid", "counttime", "channelSerialNum", "body_type"};
    @Resource
    private RestHighLevelClient client;
    @Resource
@@ -103,7 +106,7 @@ public class PersonService {
            }
            success.setMatch(1);
            success.setPersonPoolStatus(poolStatus);
-            log.info("人员匹配操作完成,PoolIds:[{}},结果:[{}]", poolIds.toString(), objectMapper.writeValueAsString(success));
+            log.info("人员匹配操作完成,PoolIds:[{}}", poolIds.toString());
            return success;
        } catch (ElasticsearchStatusException e) {
            if (e.status() == RestStatus.BAD_REQUEST && e.getDetailedMessage().contains(Constant.CLASS_CAST_EXCEPTION)) {
@@ -124,7 +127,7 @@ public class PersonService {
    }
    /**
-     * 修改人员特征
+     * 修改人员特征(先删除再添加)
     *
     * @param requestVo rid,person, poolId,
     *
@@ -136,7 +139,6 @@ public class PersonService {
        String personId = person.getPersonId();
        String poolId = requestVo.getPoolId();
        log.info("人员修改操作开始,poolId:[{}],personId:[{}]", poolId, personId);
-        // 先删除
        try {
            BulkByScrollResponse bulkByScrollResponse = deletePerson(poolId, personId);
            BulkResponse bulkItemResponses = addPerson(poolId, Collections.singletonList(person));
@@ -166,6 +168,9 @@ public class PersonService {
            Integer age = person.getAge();
            String gender = person.getGender();
            String personId = person.getPersonId();
+            Date counttime = person.getCounttime();
+            Integer bodyType = person.getBodyType();
+            String channelSerialNum = person.getChannelSerialNum();
            List<FaceFeature> faceFeatures = person.getFaceFeatures();
            if (faceFeatures != null && faceFeatures.size() > 0) {
                for (FaceFeature faceFeature : faceFeatures) {
@@ -174,7 +179,9 @@ public class PersonService {
                        String fid = faceFeature.getFid();
                        IndexRequest indexRequest = new IndexRequest(poolId)
-                                .source(XContentType.JSON, "personId", personId, "data", feature, "fid", fid, "age", age, "gender", gender);
+                                .source(XContentType.JSON, "personId", personId,
+                                        "data", feature, "fid", fid, "age", age, "gender", gender, "body_type", bodyType,
+                                        "counttime", Constant.DATE_FORMAT.get().format(counttime), "channelSerialNum", channelSerialNum);
                        bulkRequest.add(indexRequest);
                    }
                }
@@ -196,7 +203,9 @@ public class PersonService {
                    String fid = bodyFeature.getBid();
                    IndexRequest indexRequest = new IndexRequest(poolId)
-                            .source(XContentType.JSON, "personId", personId, "body", feature, "fid", fid, "age", age, "gender", gender);
+                            .source(XContentType.JSON, "personId", personId,
+                                    "body", feature, "fid", fid, "age", age, "gender", gender, "body_type", bodyType,
+                                    "counttime", Constant.DATE_FORMAT.get().format(counttime), "channelSerialNum", channelSerialNum);
                    bulkRequest.add(indexRequest);
                }
            }
@@ -211,8 +220,10 @@ public class PersonService {
    /**
     * 删除人员
-     * @param poolId 特征池Id
+     *
+     * @param poolId   特征池Id
     * @param personId 人员Id,对应数据库personUnid
+     *
     * @return 删除结果
     * @throws IOException elasticsearch 所产生的异常
     */
@@ -223,17 +234,24 @@ public class PersonService {
        return client.deleteByQuery(deleteByQueryRequest, RequestOptions.DEFAULT);
    }
+    /**
+     * 人员匹配入口
+     *
+     * @param poolId 在哪个特征池中匹配
+     * @param person 用来匹配的人员信息
+     * @param type   匹配类型 0人脸 1人体
+     *
+     * @return 匹配结果
+     * @throws Exception --
+     */
    public List<Person> matchPerson(String poolId, Person person, int type) throws Exception {
        List<Person> matchResult = new ArrayList<>();
-        int matchResultSize = 0;
+        int matchResultSize;
        if (type == 0) {
-            List<FaceFeature> faceFeatures = person.getFaceFeatures();
+            matchFace(poolId, matchResult, person);
-            matchFace(faceFeatures, poolId, matchResult);
            matchResultSize = Constant.FACE_MATCH_RESULT_SIZE;
        } else {
-            List<BodyFeature> bodyFeatures = person.getBodyFeatures();
+            matchBody(poolId, matchResult, person);
-            matchBody(bodyFeatures, poolId, matchResult);
            matchResultSize = Constant.BODY_MATCH_RESULT_SIZE;
        }
@@ -245,10 +263,9 @@ public class PersonService {
        return matchResult;
    }
-    private void matchFace(List<FaceFeature> faceFeatures, String poolId, List<Person> matchResult) throws IOException {
+    private void matchFace(String poolId, List<Person> matchResult, Person person) throws Exception {
+        List<FaceFeature> faceFeatures = person.getFaceFeatures();
        if (faceFeatures != null && faceFeatures.size() > 0) {
            for (FaceFeature faceFeature : faceFeatures) {
                Double[] feature = faceFeature.getFeature();
                if (feature == null || feature.length != Constant.FACE_FEATURE_DIMS) {
@@ -256,19 +273,16 @@ public class PersonService {
                    continue;
                }
-                Script script = new Script(
+                SearchRequest searchRequest = getSearchRequest(poolId, Constant.FACE_MATCH_RESULT_SIZE, feature, person, 0);
-                        ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG,
+                matchResult.addAll(match0(searchRequest));
-                        "(cosineSimilarity(params.face, 'data') + 1) / 2 * 100", Collections.singletonMap("face", feature));
-                ScriptScoreQueryBuilder queryBuilder = QueryBuilders.scriptScoreQuery(QueryBuilders.existsQuery("data"), script);
-                matchResult.addAll(match0(poolId, queryBuilder));
            }
        } else {
            log.info("no face feature");
        }
    }
-    private void matchBody(List<BodyFeature> bodyFeatures, String poolId, List<Person> matchResult) throws IOException {
+    private void matchBody(String poolId, List<Person> matchResult, Person person) throws Exception {
+        List<BodyFeature> bodyFeatures = person.getBodyFeatures();
        if (bodyFeatures != null && bodyFeatures.size() > 0) {
            for (BodyFeature faceFeature : bodyFeatures) {
                Double[] feature = faceFeature.getFeature();
@@ -283,29 +297,60 @@ public class PersonService {
                    feature = Arrays.copyOfRange(feature, 3, Constant.BODY_FEATURE_DIMS_2048 + 3);
                }
-                Script script = new Script(
+                SearchRequest searchRequest = getSearchRequest(poolId, Constant.BODY_MATCH_RESULT_SIZE, feature, person, 1);
-                        ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG,
-                        "(cosineSimilarity(params.body, 'body') + 1) / 2 * 100", Collections.singletonMap("body", feature));
-                ScriptScoreQueryBuilder queryBuilder = QueryBuilders.scriptScoreQuery(QueryBuilders.existsQuery("body"), script);
-                matchResult.addAll(match0(poolId, queryBuilder, Constant.BODY_MATCH_RESULT_SIZE));
+                matchResult.addAll(match0(searchRequest));
            }
        }
    }
-    private List<Person> match0(String poolId, ScriptScoreQueryBuilder scriptScoreQueryBuilder) throws IOException {
+    private SearchRequest getSearchRequest(String poolId, Integer matchResultSize, Double[] feature, Person person, int type) {
-        return match0(poolId, scriptScoreQueryBuilder, Constant.FACE_MATCH_RESULT_SIZE);
+        Script script;
-    }
+        BoolQueryBuilder boolQuery = QueryBuilders.boolQuery();
+        if (type == 0) {
+            script = new Script(
+                    ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG,
+                    "(cosineSimilarity(params.face, 'data') + 1) / 2 * 100", Collections.singletonMap("face", feature));
+            boolQuery.filter(QueryBuilders.existsQuery("data"));
+        } else {
+            script = new Script(
+                    ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG,
+                    "(cosineSimilarity(params.body, 'body') + 1) / 2 * 100", Collections.singletonMap("body", feature));
+            boolQuery.filter(QueryBuilders.existsQuery("body"));
+        }
-    private List<Person> match0(String poolId, ScriptScoreQueryBuilder scriptScoreQueryBuilder, Integer matchResultSize) throws IOException {
+        // 根据通道号过滤
-        List<Person> persons = new ArrayList<>();
+        List<String> channelSerialNums = person.getChannelSerialNums();
+        if (channelSerialNums != null && channelSerialNums.size() > 0) {
+            boolQuery.filter(QueryBuilders.termsQuery("channelSerialNum", channelSerialNums));
+        }
+        // 根据时间过滤
+        RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery("counttime");
+        Date counttimeGTE = person.getCounttimeGTE();
+        Date counttimeLTE = person.getCounttimeLTE();
+        if (counttimeGTE != null) {
+            rangeQueryBuilder.gte(Constant.DATE_FORMAT.get().format(counttimeGTE));
+        }
+        if (counttimeLTE != null) {
+            rangeQueryBuilder.lte(Constant.DATE_FORMAT.get().format(counttimeLTE));
+        }
+        if (counttimeGTE != null || counttimeLTE != null) {
+            boolQuery.filter(rangeQueryBuilder);
+        }
+        ScriptScoreQueryBuilder queryBuilder = QueryBuilders.scriptScoreQuery(boolQuery, script);
        SearchSourceBuilder builder = new SearchSourceBuilder()
                .size(matchResultSize)
-                .query(scriptScoreQueryBuilder)
+                .query(queryBuilder)
-                .fetchSource(new String[]{"personId", "age", "gender", "fid"}, null);
+                .fetchSource(FETCH_SOURCE, null);
-        SearchRequest searchRequest = new SearchRequest(poolId)
-                .source(builder);
+        return new SearchRequest(poolId).source(builder);
+    }
+    private List<Person> match0(SearchRequest searchRequest) throws Exception {
+        List<Person> persons = new ArrayList<>();
        SearchResponse search = client.search(searchRequest, RequestOptions.DEFAULT);
        SearchHits hits = search.getHits();
@@ -316,6 +361,9 @@ public class PersonService {
            p.setPersonId((String) source.get("personId"));
            p.setAge((Integer) source.get("age"));
            p.setGender((String) source.get("gender"));
+            p.setChannelSerialNum((String) source.get("channelSerialNum"));
+            p.setBodyType((Integer) source.get("body_type"));
+            p.setCounttime(Constant.DATE_FORMAT.get().parse((String) source.get("counttime")));
            p.setScore((int) item.getScore());
            p.setPersonPoolId(item.getIndex());
            persons.add(p);

--- a/src/main/java/com/viontech/match/service/PoolService.java
+++ b/src/main/java/com/viontech/match/service/PoolService.java
@@ -22,6 +22,7 @@ import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
 import org.elasticsearch.rest.RestStatus;
+import org.springframework.beans.factory.annotation.Value;
 import org.springframework.stereotype.Service;
 import javax.annotation.Resource;
@@ -47,6 +48,10 @@ public class PoolService {
    private RestHighLevelClient client;
    @Resource
    private ObjectMapper objectMapper;
+    @Value("${vion.index.number_of_shards:1}")
+    private Integer shards;
+    @Value("${vion.index.number_of_replicas:0}")
+    private Integer replicas;
    /**
     * 添加特征池
@@ -65,8 +70,8 @@ public class PoolService {
            XContentBuilder builder = getCreateIndexContentBuilder();
            createIndexRequest.mapping(builder);
            Settings.Builder setting = Settings.builder();
-            setting.put("index.number_of_shards", 1);
+            setting.put("index.number_of_shards", shards);
-            setting.put("index.number_of_replicas", 0);
+            setting.put("index.number_of_replicas", replicas);
            createIndexRequest.settings(setting);
            CreateIndexResponse createIndexResponse = client.indices().create(createIndexRequest, RequestOptions.DEFAULT);
@@ -147,6 +152,7 @@ public class PoolService {
            BulkResponse bulkItemResponses = personService.addPerson(poolId, personPool);
            if (bulkItemResponses != null && bulkItemResponses.hasFailures()) {
                log.info(bulkItemResponses.buildFailureMessage());
+                return ResponseVo.error(rid, bulkItemResponses.buildFailureMessage());
            }
            log.info("特征池修改操作完成:[{}],updateType:[{}]", poolId, updateType);
@@ -159,7 +165,9 @@ public class PoolService {
    /**
     * 查询特征池信息
+     *
     * @param requestVo rid,listAll,poolId
+     *
     * @return ResponseVo
     * @throws Exception --
     */
@@ -210,6 +218,7 @@ public class PoolService {
                    builder.field("dims", Constant.FACE_FEATURE_DIMS);
                }
                builder.endObject();
                // 人体特征
                builder.startObject("body");
                {
@@ -217,12 +226,14 @@ public class PoolService {
                    builder.field("dims", Constant.BODY_FEATURE_DIMS_2048);
                }
                builder.endObject();
                // 人员id
                builder.startObject("personId");
                {
                    builder.field("type", "keyword");
                }
                builder.endObject();
                // 特征id
                builder.startObject("fid");
                {
@@ -230,6 +241,7 @@ public class PoolService {
                }
                builder.endObject();
+                // 年龄
                builder.startObject("age");
                {
                    builder.field("type", "integer");
@@ -238,6 +250,7 @@ public class PoolService {
                }
                builder.endObject();
+                // 性别
                builder.startObject("gender");
                {
                    builder.field("type", "keyword");
@@ -245,6 +258,32 @@ public class PoolService {
                    builder.field("index", false);
                }
                builder.endObject();
+                // 时间
+                builder.startObject("counttime");
+                {
+                    builder.field("type", "date");
+                    builder.field("format", "yyyy-MM-dd HH:mm:ss");
+                }
+                builder.endObject();
+                // 通道序列号
+                builder.startObject("channelSerialNum");
+                {
+                    builder.field("type", "keyword");
+                }
+                builder.endObject();
+                // 身体特征类型类型
+                builder.startObject("body_type");
+                {
+                    builder.field("type", "integer");
+                    builder.field("doc_values", false);
+                    builder.field("index", false);
+                }
+                builder.endObject();
            }
            builder.endObject();
        }

--- a/src/main/resources/application-option.properties
+++ b/src/main/resources/application-option.properties
@@ -3,4 +3,8 @@ server.port=12000
 spring.jackson.time-zone=GMT+8
 spring.jackson.date-format=yyyy-MM-dd HH:mm:ss
 #                   es
-spring.elasticsearch.rest.uris=http://127.0.0.1:9200
\ No newline at end of file
+spring.elasticsearch.rest.uris=http://127.0.0.1:9200
+#\u5206\u7247\u6570\u91CF
+vion.index.number_of_shards=1
+#\u526F\u672C\u6570\u91CF
+vion.index.number_of_replicas=0