基本完成

pom.xml

@@ -17,6 +17,7 @@
 
     <properties>
         <java.version>1.8</java.version>
+        <skipTests>true</skipTests>
     </properties>
 
     <dependencies>

src/main/java/com/xmh/es/match/PoolEnum.java

+package com.xmh.es.match;
+
+/**
+ * .
+ *
+ * @author 谢明辉
+ * @date 2020/11/25
+ */
+public enum PoolEnum {
+    /**  */
+    POOL_2K("test_pool_2k", 2000),
+    POOL_5K("test_pool_5k", 5000),
+    POOL_2W("test_pool_2w", 20000),
+    POOL_10W("test_pool_10w", 100000),
+    POOL_50W("test_pool_50w", 500000),
+    POOL_100W("test_pool_100w", 1000000),
+    POOL_200W("test_pool_200w", 2000000);
+
+    public String name;
+    public int num;
+
+    PoolEnum(String name, int num) {
+        this.name = name;
+        this.num = num;
+    }
+}

src/main/java/com/xmh/es/match/XmhTestApplication.java

@@ -7,10 +7,9 @@ import org.springframework.context.annotation.ComponentScan;
 @SpringBootApplication
 @ComponentScan(basePackages = {"com.viontech.keliu", "com.xmh.es"})
 public class XmhTestApplication {
-    public final static String POOL_NAME = "test_pool";
 
     public static void main(String[] args) {
         SpringApplication.run(XmhTestApplication.class, args);
     }
 
-}
+}
\ No newline at end of file

src/main/java/com/xmh/es/match/controller/MainController.java

 package com.xmh.es.match.controller;
 
-import com.xmh.es.match.entity.MatchResult;
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.viontech.keliu.storage.Storage;
+import com.viontech.keliu.util.DateUtil;
+import com.xmh.es.match.PoolEnum;
+import com.xmh.es.match.entity.*;
 import com.xmh.es.match.entity.vo.RequestData;
 import com.xmh.es.match.repository.FaceRecognitionRepository;
+import com.xmh.es.match.service.AlgService;
 import com.xmh.es.match.service.MatchService;
-import org.springframework.web.bind.annotation.PostMapping;
-import org.springframework.web.bind.annotation.RequestMapping;
-import org.springframework.web.bind.annotation.RestController;
+import org.springframework.web.bind.annotation.*;
+import org.springframework.web.multipart.MultipartFile;
 
 import javax.annotation.Resource;
-import java.util.List;
+import java.io.IOException;
+import java.util.*;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.stream.Collectors;
 
 /**
@@ -22,16 +32,153 @@ import java.util.stream.Collectors;
 @RequestMapping("/alg")
 public class MainController {
 
+    private final ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(20, 30, 1, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadPoolExecutor.CallerRunsPolicy());
+    private final AtomicInteger integer = new AtomicInteger();
+    @Resource
+    ObjectMapper objectMapper;
     @Resource
     private MatchService matchService;
     @Resource
     private FaceRecognitionRepository faceRecognitionRepository;
+    @Resource
+    private AlgService algService;
+    @Resource
+    private Storage featureStorage;
 
     @PostMapping("/match")
     public Object match(RequestData requestData) throws Exception {
-        List<MatchResult> match = matchService.match(requestData);
+        MultipartFile file = requestData.getFile();
+        Double[] data = matchService.extractFeature(file);
+        Person person = new Person().setFeature(data);
+        long begin = System.currentTimeMillis();
+        List<MatchResult> match = algService.matchPerson(person, requestData.getPoolId());
+        long timeConsumingMatching = System.currentTimeMillis() - begin;
         List<String> collect = match.stream().map(MatchResult::getPersonId).collect(Collectors.toList());
-        return faceRecognitionRepository.queryFaceRecognitionByUnids(collect);
+        List<FaceRecognition> faceRecognitions = faceRecognitionRepository.queryFaceRecognitionByUnids(collect);
+        if (faceRecognitions == null) {
+            return null;
+        }
+        Map<String, FaceRecognition> map = faceRecognitions.stream().collect(Collectors.toMap(FaceRecognition::getUnid, x -> x, (x, y) -> y));
+        for (MatchResult item : match) {
+            FaceRecognition faceRecognition = map.get(item.getPersonId());
+            if (faceRecognition != null) {
+                faceRecognition.setMatchScore(item.getScore());
+            }
+        }
+        Collection<FaceRecognition> values = map.values();
+        List<FaceRecognition> collect1 = values.stream().sorted((o1, o2) -> {
+            if (o1.getMatchScore() > o2.getMatchScore()) {
+                return -1;
+            } else if (o1.getMatchScore().equals(o2.getMatchScore())) {
+                return 0;
+            } else {
+                return 1;
+            }
+        }).collect(Collectors.toList());
+
+        HashMap<String, Object> resultMap = new HashMap<>(2);
+        resultMap.put("time", timeConsumingMatching);
+        resultMap.put("result", collect1);
+        return resultMap;
     }
 
+    @GetMapping("/poolIds")
+    public Object getPoolIds() throws IOException {
+        return algService.queryPoolList();
+    }
+
+    @GetMapping("/create/{num}")
+    public Object createPool(@PathVariable("num") Integer num) throws Exception {
+        PoolEnum pool;
+        switch (num) {
+            case 2000:
+                pool = PoolEnum.POOL_2K;
+                break;
+            case 5000:
+                pool = PoolEnum.POOL_5K;
+                break;
+            case 20000:
+                pool = PoolEnum.POOL_2W;
+                break;
+            case 100000:
+                pool = PoolEnum.POOL_10W;
+                break;
+            case 500000:
+                pool = PoolEnum.POOL_50W;
+                break;
+            case 1000000:
+                pool = PoolEnum.POOL_100W;
+                break;
+            case 2000000:
+                pool = PoolEnum.POOL_200W;
+                break;
+            default:
+                return "请输入[2000,5000,20000,100000,500000,1000000,2000000]";
+        }
+
+
+        algService.deletePool(pool.name);
+        algService.createPool(pool.name);
+
+        Date end = DateUtil.addDays(new Date(), -1);
+        Date start = DateUtil.addDays(end, -30);
+        integer.set(0);
+        List<Date> days = DateUtil.getDaysBetweenDates(start, end);
+        for (Date day : days) {
+            List<FaceRecognition> faceRecognitions = faceRecognitionRepository.queryFaceRecognitions(day, 2L);
+            for (FaceRecognition faceRecognition : faceRecognitions) {
+                poolExecutor.execute(() -> {
+                    try {
+                        if (integer.get() >= pool.num) {
+                            return;
+                        }
+                        Double[] feature = getFeature(faceRecognition);
+                        if (feature == null) {
+                            return;
+                        }
+                        Person person = new Person();
+                        person.setPersonId(faceRecognition.getPersonUnid());
+                        person.setFeature(feature);
+                        person.setId(faceRecognition.getUnid());
+                        Object o = algService.addPerson(person, pool.name);
+                        System.out.println(objectMapper.writeValueAsString(o));
+                        integer.addAndGet(1);
+
+                    } catch (Exception e) {
+                        e.printStackTrace();
+                    }
+                });
+                if (integer.get() > pool.num) {
+                    return "success";
+                }
+                while (poolExecutor.getQueue().size() > 1000) {
+                    System.out.println(poolExecutor.getQueue().size());
+                    TimeUnit.SECONDS.sleep(10);
+                }
+                if (integer.get() > pool.num) {
+                    return "success";
+                }
+            }
+        }
+        return "success";
+    }
+
+    private Double[] getFeature(FaceRecognition face) throws JsonProcessingException {
+        Short faceFeatureType = face.getFaceFeatureType();
+        String facePic = face.getFacePic();
+        if (faceFeatureType == 1) {
+            facePic = facePic.replace("face-0.jpg", "face-F.jpg");
+        }
+        String json = (String) featureStorage.getItem(face.getChannelSerialnum() + "/" + facePic);
+        Feature feature = objectMapper.readValue(json, Feature.class);
+        List<Data> datas = feature.getDatas();
+        if (datas != null && datas.size() > 0) {
+            Data data = datas.get(0);
+            Double[] data1 = data.getData();
+            if (data1.length == 512) {
+                return data1;
+            }
+        }
+        return null;
+    }
 }

src/main/java/com/xmh/es/match/entity/FaceRecognition.java

@@ -76,7 +76,7 @@ public class FaceRecognition extends BaseModel {
 
     /** 图像质量分 **/
     private Float faceScore;
-
+    private Float matchScore;
 
     @Override
     public Long getId() {
@@ -384,4 +384,11 @@ public class FaceRecognition extends BaseModel {
         this.trackPath = trackPath;
     }
 
+    public Float getMatchScore() {
+        return matchScore;
+    }
+
+    public void setMatchScore(Float matchScore) {
+        this.matchScore = matchScore;
+    }
 }
\ No newline at end of file

src/main/java/com/xmh/es/match/entity/Person.java

@@ -14,7 +14,7 @@ import lombok.experimental.Accessors;
 @Setter
 @Accessors(chain = true)
 public class Person {
-
+    private String id;
     private Double[] feature;
     private String personId;
 }

src/main/java/com/xmh/es/match/repository/FaceRecognitionRepository.java

@@ -24,7 +24,7 @@ public class FaceRecognitionRepository {
     private static final BeanPropertyRowMapper<FaceRecognition> FACE_RECOGNITION_ROW_MAPPER = new BeanPropertyRowMapper<>(FaceRecognition.class);
 
 
-    private static final String QUERY_FACE = "select * from d_face_recognition where countdate=? and mall_id=? and direction=1 and gender>-1 and face_score>0.7";
+    private static final String QUERY_FACE = "select unid,person_unid,face_feature_type,face_pic,channel_serialnum from d_face_recognition where countdate=? and direction=1 and gender>-1 and face_score>0.7";
     private static final String QUERY_FACE_BY_UNID = "select face_pic,id,channel_serialnum,device_serialnum,person_unid,unid,id,countdate,counttime,age,gender,direction,mall_id,account_id,gate_id,face_score,person_type from d_face_recognition where unid in (:unids)";
 
     @Resource
@@ -34,10 +34,13 @@ public class FaceRecognitionRepository {
 
 
     public List<FaceRecognition> queryFaceRecognitions(Date date, Long mallId) {
-        return jdbcTemplate.query(QUERY_FACE, FACE_RECOGNITION_ROW_MAPPER, date, mallId);
+        return jdbcTemplate.query(QUERY_FACE, FACE_RECOGNITION_ROW_MAPPER, date);
     }
 
     public List<FaceRecognition> queryFaceRecognitionByUnids(List<String> unidList) {
+        if (unidList == null || unidList.size() == 0) {
+            return null;
+        }
         Map<String, List<String>> param = Collections.singletonMap("unids", unidList);
         return namedParameterJdbcTemplate.query(QUERY_FACE_BY_UNID, param, FACE_RECOGNITION_ROW_MAPPER);
     }

src/main/java/com/xmh/es/match/service/AlgService.java

@@ -3,6 +3,7 @@ package com.xmh.es.match.service;
 import com.xmh.es.match.entity.MatchResult;
 import com.xmh.es.match.entity.Person;
 import org.elasticsearch.action.admin.indices.delete.DeleteIndexRequest;
+import org.elasticsearch.action.bulk.BulkRequest;
 import org.elasticsearch.action.delete.DeleteRequest;
 import org.elasticsearch.action.index.IndexRequest;
 import org.elasticsearch.action.search.SearchRequest;
@@ -16,6 +17,7 @@ import org.elasticsearch.client.indices.GetIndexRequest;
 import org.elasticsearch.client.indices.GetIndexResponse;
 import org.elasticsearch.common.xcontent.XContentBuilder;
 import org.elasticsearch.common.xcontent.XContentFactory;
+import org.elasticsearch.common.xcontent.XContentType;
 import org.elasticsearch.index.query.QueryBuilders;
 import org.elasticsearch.index.query.functionscore.ScriptScoreQueryBuilder;
 import org.elasticsearch.script.Script;
@@ -125,12 +127,12 @@ public class AlgService {
         SearchRequest searchRequest = new SearchRequest(poolId);
         SearchSourceBuilder builder = new SearchSourceBuilder();
         builder.fetchSource("personId", null);
-        builder.size(5);
+        builder.size(40);
         Map<String, Object> params = new HashMap<>(1);
         params.put("data", feature);
         Script script = new Script(
                 ScriptType.INLINE, Script.DEFAULT_SCRIPT_LANG,
-                "cosineSimilarity(params.data, 'data')", params);
+                "cosineSimilarity(params.data, 'data') + 1", params);
 
         ScriptScoreQueryBuilder queryBuilder = QueryBuilders.scriptScoreQuery(QueryBuilders.matchAllQuery(), script);
         builder.query(queryBuilder);
@@ -142,8 +144,7 @@ public class AlgService {
         SearchHit[] hits1 = hits.getHits();
         ArrayList<MatchResult> matchResults = new ArrayList<>();
         for (SearchHit item : hits1) {
-            Map<String, Object> sourceAsMap = item.getSourceAsMap();
-            String personId = (String) sourceAsMap.get("personId");
+            String personId = item.getId();
             float score = item.getScore();
             String index = item.getIndex();
             MatchResult matchResult = new MatchResult().setPersonId(personId).setScore(score).setPoolId(index);
@@ -159,17 +160,22 @@ public class AlgService {
      * @param poolId 特征库名称
      */
     public Object addPerson(Person person, String poolId) throws IOException {
-        IndexRequest indexRequest = new IndexRequest(poolId);
-        indexRequest.id(person.getPersonId());
-        XContentBuilder xContentBuilder = XContentFactory.jsonBuilder();
-        xContentBuilder.startObject();
-        xContentBuilder.field("personId", person.getPersonId());
-        xContentBuilder.field("data", person.getFeature());
-        xContentBuilder.endObject();
-        indexRequest.source(xContentBuilder);
+        IndexRequest indexRequest = new IndexRequest(poolId)
+                .id(person.getId())
+                .source(XContentType.JSON, "personId", person.getPersonId(), "data", person.getFeature());
         return client.index(indexRequest, RequestOptions.DEFAULT);
     }
 
+    public Object addAllPerson(List<Person> persons, String poolId) throws Exception {
+        BulkRequest bulkRequest = new BulkRequest();
+        for (Person person : persons) {
+            bulkRequest.add(new IndexRequest(poolId)
+                    .id(person.getId())
+                    .source(XContentType.JSON, "personId", person.getPersonId(), "data", person.getFeature()));
+        }
+        return client.bulk(bulkRequest, RequestOptions.DEFAULT);
+    }
+
     /**
      * 删除人员
      *

src/main/java/com/xmh/es/match/service/MatchService.java

@@ -38,6 +38,12 @@ public class MatchService {
 
     public List<MatchResult> match(RequestData requestData) throws Exception {
         MultipartFile file = requestData.getFile();
+        Double[] data = extractFeature(file);
+        Person person = new Person().setFeature(data);
+        return algService.matchPerson(person, requestData.getPoolId());
+    }
+
+    public Double[] extractFeature(MultipartFile file) throws Exception {
         BufferedImage originalImage = ImageIO.read(file.getInputStream());
         BufferedImage jpgImage = Utils.convertImg2Jpg(originalImage);
 
@@ -65,9 +71,7 @@ public class MatchService {
         JSONArray feature = faceFeatureJson.getJSONArray("feature");
         List<Object> objects = feature.toList();
 
-        Double[] data = objects.toArray(new Double[]{});
-        Person person = new Person().setFeature(data);
-        return algService.matchPerson(person, requestData.getPoolId());
+        return objects.toArray(new Double[]{});
     }
 
 }

src/main/resources/application-option.properties

@@ -11,4 +11,8 @@ oss.config.bucket=vion-retail
 #                   ws
 ws.featureUrl=http://101.200.130.13:10000
 #                   other
-server.port=12000
\ No newline at end of file
+server.port=12000
+spring.jackson.time-zone=GMT+8
+spring.jackson.date-format=yyyy-MM-dd HH:mm:ss
+#                   es
+spring.elasticsearch.rest.uris=http://139.217.100.35:9200
\ No newline at end of file

src/main/resources/application.yml

 spring:
-  elasticsearch:
-    rest:
-      uris: http://127.0.0.1:9200
   profiles:
     active: option
 

src/main/resources/static/index.html

+<!DOCTYPE html><html><head><meta charset=utf-8><meta name=viewport content="width=device-width,initial-scale=1"><title>hello</title><link href=/static/css/app.e7c2112962ef9f4a39d379467a4bcf3c.css rel=stylesheet></head><body><div id=app></div><script type=text/javascript src=/static/js/manifest.2ae2e69a05c33dfc65f8.js></script><script type=text/javascript src=/static/js/vendor.8d9ad72abf44221abb05.js></script><script type=text/javascript src=/static/js/app.975f2cc40df4e4cd82a5.js></script></body></html>
\ No newline at end of file

src/main/resources/static/static/js/app.975f2cc40df4e4cd82a5.js

+webpackJsonp([1],{NHnr:function(t,e,a){"use strict";Object.defineProperty(e,"__esModule",{value:!0});var o=a("7+uW"),l={name:"HelloWorld",data:function(){return{formData:new FormData,imageUrl:null,form:{poolId:null,file:null},poolIds:[],poolIdSelected:null,input:null,faceList:null,matchTime:null}},methods:{getSrc:function(t){return"https://vion-retail.oss-cn-beijing.aliyuncs.com/picture/"+t.facePath+t.facePic},beforeupload:function(t){return this.form.file=t,!1},onChange:function(t){this.imageUrl=URL.createObjectURL(t.raw)},onSubmit:function(){var t=this;this.$refs.upload.submit(),this.formData.set("poolId",this.form.poolId),this.formData.set("file",this.form.file),console.log(this.formData),this.axios.post("http://127.0.0.1:12000/alg/match",this.formData,{headers:{"Content-Type":"multipart/form-data"}}).then(function(e){var a=e.data.result;if(null!=a){t.faceList=[];for(var o=a.length/5,l=0;l<o;l++){var n=a.slice(5*l,5*l+5);t.faceList.push(n)}}t.matchTime=e.data.time}).catch(function(t){console.error(t)}),this.formData=new FormData},getPoolIds:function(){var t=this;this.axios({method:"get",url:"http://127.0.0.1:12000/alg/poolIds"}).then(function(e){t.poolIds=e.data,t.poolIds.length>0&&(t.form.poolId=t.poolIds[0])}).catch(function(t){console.log(t)})}},mounted:function(){this.getPoolIds()}},n={render:function(){var t=this,e=t.$createElement,a=t._self._c||e;return a("div",{staticClass:"hello",staticStyle:{margin:"auto auto"}},[a("el-form",{ref:"form",staticStyle:{width:"400px",margin:"auto auto"},attrs:{model:t.form}},[a("el-form-item",{attrs:{label:"特征库"}},[a("el-select",{attrs:{placeholder:"请选择特征库"},model:{value:t.form.poolId,callback:function(e){t.$set(t.form,"poolId",e)},expression:"form.poolId"}},t._l(t.poolIds,function(t){return a("el-option",{key:t,attrs:{label:t,value:t}})}),1)],1),t._v(" "),a("el-form-item",{attrs:{label:"人脸图片"}},[a("el-upload",{ref:"upload",staticClass:"avatar-uploader",attrs:{action:"http://127.0.0.1:12000/","show-file-list":!1,"auto-upload":!1,"on-change":t.onChange,"before-upload":t.beforeupload}},[t.imageUrl?a("img",{staticClass:"avatar",attrs:{src:t.imageUrl}}):a("i",{staticClass:"el-icon-plus avatar-uploader-icon"})])],1),t._v(" "),a("el-form-item",[a("el-button",{staticStyle:{width:"100%"},attrs:{type:"primary"},on:{click:t.onSubmit}},[t._v("比对")])],1)],1),t._v(" "),null!=t.matchTime?a("p",[t._v("匹配耗时:"+t._s(t.matchTime))]):t._e(),t._v(" "),t._l(t.faceList,function(e,o){return a("el-row",{key:o,staticClass:"row-bg",staticStyle:{width:"80%",margin:"auto"},attrs:{type:"flex",justify:"space-around"}},t._l(t.faceList[o],function(e){return a("el-col",{key:e.unid,attrs:{span:4}},[a("el-card",{staticStyle:{margin:"20px",width:"200px"},attrs:{"body-style":{padding:"0px"}}},[a("img",{staticClass:"image",attrs:{src:t.getSrc(e)}}),t._v(" "),a("div",{staticStyle:{padding:"15px"}},[a("p",{staticStyle:{"font-size":"12px"}},[t._v("匹配分数:"+t._s(e.matchScore))]),t._v(" "),a("p",{staticStyle:{"font-size":"12px"}},[t._v("性别:"+t._s(0==e.gender?"女":1==e.gender?"男":"未知"))]),t._v(" "),a("p",{staticStyle:{"font-size":"12px"}},[t._v("年龄:"+t._s(e.age))]),t._v(" "),a("p",{staticStyle:{"font-size":"12px"}},[t._v("时间:"+t._s(e.counttime))])])])],1)}),1)})],2)},staticRenderFns:[]};var i={name:"App",components:{HelloWorld:a("VU/8")(l,n,!1,function(t){a("fKca")},null,null).exports}},s={render:function(){var t=this.$createElement,e=this._self._c||t;return e("div",{attrs:{id:"app"}},[e("HelloWorld")],1)},staticRenderFns:[]};var r=a("VU/8")(i,s,!1,function(t){a("m71c")},null,null).exports,c=a("mtWM"),u=a.n(c),f=a("zL8q"),p=a.n(f);a("tvR6");o.default.use(p.a),o.default.config.productionTip=!1,o.default.prototype.axios=u.a,new o.default({el:"#app",components:{App:r},template:"<App/>"})},fKca:function(t,e){},m71c:function(t,e){},tvR6:function(t,e){}},["NHnr"]);
+//# sourceMappingURL=app.975f2cc40df4e4cd82a5.js.map
\ No newline at end of file

src/main/resources/static/static/js/manifest.2ae2e69a05c33dfc65f8.js

+!function(r){var n=window.webpackJsonp;window.webpackJsonp=function(e,u,c){for(var f,i,p,a=0,l=[];a<e.length;a++)i=e[a],o[i]&&l.push(o[i][0]),o[i]=0;for(f in u)Object.prototype.hasOwnProperty.call(u,f)&&(r[f]=u[f]);for(n&&n(e,u,c);l.length;)l.shift()();if(c)for(a=0;a<c.length;a++)p=t(t.s=c[a]);return p};var e={},o={2:0};function t(n){if(e[n])return e[n].exports;var o=e[n]={i:n,l:!1,exports:{}};return r[n].call(o.exports,o,o.exports,t),o.l=!0,o.exports}t.m=r,t.c=e,t.d=function(r,n,e){t.o(r,n)||Object.defineProperty(r,n,{configurable:!1,enumerable:!0,get:e})},t.n=function(r){var n=r&&r.__esModule?function(){return r.default}:function(){return r};return t.d(n,"a",n),n},t.o=function(r,n){return Object.prototype.hasOwnProperty.call(r,n)},t.p="/",t.oe=function(r){throw console.error(r),r}}([]);
+//# sourceMappingURL=manifest.2ae2e69a05c33dfc65f8.js.map
\ No newline at end of file

src/main/resources/static/static/js/manifest.2ae2e69a05c33dfc65f8.js.map

+{"version":3,"sources":["webpack:///webpack/bootstrap d2d68e467296ba9c478b"],"names":["parentJsonpFunction","window","chunkIds","moreModules","executeModules","moduleId","chunkId","result","i","resolves","length","installedChunks","push","Object","prototype","hasOwnProperty","call","modules","shift","__webpack_require__","s","installedModules","2","exports","module","l","m","c","d","name","getter","o","defineProperty","configurable","enumerable","get","n","__esModule","object","property","p","oe","err","console","error"],"mappings":"aACA,IAAAA,EAAAC,OAAA,aACAA,OAAA,sBAAAC,EAAAC,EAAAC,GAIA,IADA,IAAAC,EAAAC,EAAAC,EAAAC,EAAA,EAAAC,KACQD,EAAAN,EAAAQ,OAAoBF,IAC5BF,EAAAJ,EAAAM,GACAG,EAAAL,IACAG,EAAAG,KAAAD,EAAAL,GAAA,IAEAK,EAAAL,GAAA,EAEA,IAAAD,KAAAF,EACAU,OAAAC,UAAAC,eAAAC,KAAAb,EAAAE,KACAY,EAAAZ,GAAAF,EAAAE,IAIA,IADAL,KAAAE,EAAAC,EAAAC,GACAK,EAAAC,QACAD,EAAAS,OAAAT,GAEA,GAAAL,EACA,IAAAI,EAAA,EAAYA,EAAAJ,EAAAM,OAA2BF,IACvCD,EAAAY,IAAAC,EAAAhB,EAAAI,IAGA,OAAAD,GAIA,IAAAc,KAGAV,GACAW,EAAA,GAIA,SAAAH,EAAAd,GAGA,GAAAgB,EAAAhB,GACA,OAAAgB,EAAAhB,GAAAkB,QAGA,IAAAC,EAAAH,EAAAhB,IACAG,EAAAH,EACAoB,GAAA,EACAF,YAUA,OANAN,EAAAZ,GAAAW,KAAAQ,EAAAD,QAAAC,IAAAD,QAAAJ,GAGAK,EAAAC,GAAA,EAGAD,EAAAD,QAKAJ,EAAAO,EAAAT,EAGAE,EAAAQ,EAAAN,EAGAF,EAAAS,EAAA,SAAAL,EAAAM,EAAAC,GACAX,EAAAY,EAAAR,EAAAM,IACAhB,OAAAmB,eAAAT,EAAAM,GACAI,cAAA,EACAC,YAAA,EACAC,IAAAL,KAMAX,EAAAiB,EAAA,SAAAZ,GACA,IAAAM,EAAAN,KAAAa,WACA,WAA2B,OAAAb,EAAA,SAC3B,WAAiC,OAAAA,GAEjC,OADAL,EAAAS,EAAAE,EAAA,IAAAA,GACAA,GAIAX,EAAAY,EAAA,SAAAO,EAAAC,GAAsD,OAAA1B,OAAAC,UAAAC,eAAAC,KAAAsB,EAAAC,IAGtDpB,EAAAqB,EAAA,IAGArB,EAAAsB,GAAA,SAAAC,GAA8D,MAApBC,QAAAC,MAAAF,GAAoBA","file":"static/js/manifest.2ae2e69a05c33dfc65f8.js","sourcesContent":[" \t// install a JSONP callback for chunk loading\n \tvar parentJsonpFunction = window[\"webpackJsonp\"];\n \twindow[\"webpackJsonp\"] = function webpackJsonpCallback(chunkIds, moreModules, executeModules) {\n \t\t// add \"moreModules\" to the modules object,\n \t\t// then flag all \"chunkIds\" as loaded and fire callback\n \t\tvar moduleId, chunkId, i = 0, resolves = [], result;\n \t\tfor(;i < chunkIds.length; i++) {\n \t\t\tchunkId = chunkIds[i];\n \t\t\tif(installedChunks[chunkId]) {\n \t\t\t\tresolves.push(installedChunks[chunkId][0]);\n \t\t\t}\n \t\t\tinstalledChunks[chunkId] = 0;\n \t\t}\n \t\tfor(moduleId in moreModules) {\n \t\t\tif(Object.prototype.hasOwnProperty.call(moreModules, moduleId)) {\n \t\t\t\tmodules[moduleId] = moreModules[moduleId];\n \t\t\t}\n \t\t}\n \t\tif(parentJsonpFunction) parentJsonpFunction(chunkIds, moreModules, executeModules);\n \t\twhile(resolves.length) {\n \t\t\tresolves.shift()();\n \t\t}\n \t\tif(executeModules) {\n \t\t\tfor(i=0; i < executeModules.length; i++) {\n \t\t\t\tresult = __webpack_require__(__webpack_require__.s = executeModules[i]);\n \t\t\t}\n \t\t}\n \t\treturn result;\n \t};\n\n \t// The module cache\n \tvar installedModules = {};\n\n \t// objects to store loaded and loading chunks\n \tvar installedChunks = {\n \t\t2: 0\n \t};\n\n \t// The require function\n \tfunction __webpack_require__(moduleId) {\n\n \t\t// Check if module is in cache\n \t\tif(installedModules[moduleId]) {\n \t\t\treturn installedModules[moduleId].exports;\n \t\t}\n \t\t// Create a new module (and put it into the cache)\n \t\tvar module = installedModules[moduleId] = {\n \t\t\ti: moduleId,\n \t\t\tl: false,\n \t\t\texports: {}\n \t\t};\n\n \t\t// Execute the module function\n \t\tmodules[moduleId].call(module.exports, module, module.exports, __webpack_require__);\n\n \t\t// Flag the module as loaded\n \t\tmodule.l = true;\n\n \t\t// Return the exports of the module\n \t\treturn module.exports;\n \t}\n\n\n \t// expose the modules object (__webpack_modules__)\n \t__webpack_require__.m = modules;\n\n \t// expose the module cache\n \t__webpack_require__.c = installedModules;\n\n \t// define getter function for harmony exports\n \t__webpack_require__.d = function(exports, name, getter) {\n \t\tif(!__webpack_require__.o(exports, name)) {\n \t\t\tObject.defineProperty(exports, name, {\n \t\t\t\tconfigurable: false,\n \t\t\t\tenumerable: true,\n \t\t\t\tget: getter\n \t\t\t});\n \t\t}\n \t};\n\n \t// getDefaultExport function for compatibility with non-harmony modules\n \t__webpack_require__.n = function(module) {\n \t\tvar getter = module && module.__esModule ?\n \t\t\tfunction getDefault() { return module['default']; } :\n \t\t\tfunction getModuleExports() { return module; };\n \t\t__webpack_require__.d(getter, 'a', getter);\n \t\treturn getter;\n \t};\n\n \t// Object.prototype.hasOwnProperty.call\n \t__webpack_require__.o = function(object, property) { return Object.prototype.hasOwnProperty.call(object, property); };\n\n \t// __webpack_public_path__\n \t__webpack_require__.p = \"/\";\n\n \t// on error function for async loading\n \t__webpack_require__.oe = function(err) { console.error(err); throw err; };\n\n\n\n// WEBPACK FOOTER //\n// webpack/bootstrap d2d68e467296ba9c478b"],"sourceRoot":""}
\ No newline at end of file

src/test/java/com/xmh/es/match/Test0.java

@@ -21,6 +21,10 @@ import java.text.ParseException;
 import java.util.Date;
 import java.util.List;
 import java.util.UUID;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
 
 /**
  * .
@@ -33,6 +37,9 @@ import java.util.UUID;
 @RunWith(SpringRunner.class)
 public class Test0 {
 
+    private final ThreadPoolExecutor poolExecutor = new ThreadPoolExecutor(20, 30, 1, TimeUnit.MINUTES, new LinkedBlockingQueue<>(), new ThreadPoolExecutor.CallerRunsPolicy());
+    private final AtomicInteger integer = new AtomicInteger();
+    private final PoolEnum pool = PoolEnum.POOL_50W;
     @Resource
     ObjectMapper objectMapper;
     @Resource
@@ -44,14 +51,14 @@ public class Test0 {
 
     @Test
     public void createPoolTest() throws IOException {
-        Object test2 = algService.createPool(XmhTestApplication.POOL_NAME);
+        Object test2 = algService.createPool(pool.name);
         System.out.println(test2);
 
     }
 
     @Test
     public void deleteIndexTest() throws IOException {
-        algService.deletePool(XmhTestApplication.POOL_NAME);
+        algService.deletePool(pool.name);
     }
 
     @Test
@@ -65,41 +72,61 @@ public class Test0 {
     @Test
     public void addPersonTest() throws IOException {
         Person person1 = getPerson("3");
-        Object test2 = algService.addPerson(person1, XmhTestApplication.POOL_NAME);
+        Object test2 = algService.addPerson(person1, pool.name);
         System.out.println(objectMapper.writeValueAsString(test2));
     }
 
     @Test
     public void matchPersonTest() throws Exception {
         Person person = getPerson("3");
-        List<MatchResult> test2 = algService.matchPerson(person, XmhTestApplication.POOL_NAME);
+        List<MatchResult> test2 = algService.matchPerson(person, pool.name);
         System.out.println(objectMapper.writeValueAsString(test2));
 
     }
 
     @Test
-    public void batchAdd() throws ParseException {
-        Date start = DateUtil.parse(DateUtil.FORMAT_SHORT, "2020-10-01");
-        Date end = DateUtil.parse(DateUtil.FORMAT_SHORT, "2020-11-19");
+    public void batchAdd() throws ParseException, IOException, InterruptedException {
+        algService.deletePool(pool.name);
+        algService.createPool(pool.name);
 
+        Date start = DateUtil.parse(DateUtil.FORMAT_SHORT, "2020-11-01");
+        Date end = DateUtil.parse(DateUtil.FORMAT_SHORT, "2020-11-24");
+        int i = 0;
 
         List<Date> days = DateUtil.getDaysBetweenDates(start, end);
         for (Date day : days) {
             List<FaceRecognition> faceRecognitions = faceRecognitionRepository.queryFaceRecognitions(day, 2L);
             for (FaceRecognition faceRecognition : faceRecognitions) {
-                try {
-                    Double[] feature = getFeature(faceRecognition);
-                    if (feature == null) {
-                        continue;
+                poolExecutor.execute(() -> {
+                    try {
+                        if (integer.get() >= pool.num) {
+                            return;
+                        }
+                        Double[] feature = getFeature(faceRecognition);
+                        if (feature == null) {
+                            return;
+                        }
+                        Person person = new Person();
+                        person.setPersonId(faceRecognition.getPersonUnid());
+                        person.setFeature(feature);
+                        person.setId(faceRecognition.getUnid());
+                        Object o = algService.addPerson(person, pool.name);
+                        System.out.println(objectMapper.writeValueAsString(o));
+                        integer.addAndGet(1);
+
+                    } catch (Exception e) {
+                        e.printStackTrace();
                     }
-                    Person person = new Person();
-                    person.setPersonId(faceRecognition.getUnid());
-                    person.setFeature(feature);
-                    Object o = algService.addPerson(person, XmhTestApplication.POOL_NAME);
-                    System.out.println(objectMapper.writeValueAsString(o));
-
-                } catch (Exception e) {
-                    e.printStackTrace();
+                });
+                if (integer.get() > pool.num) {
+                    return;
+                }
+                while (poolExecutor.getQueue().size() > 1000) {
+                    System.out.println(poolExecutor.getQueue().size());
+                    TimeUnit.SECONDS.sleep(10);
+                }
+                if (integer.get() > pool.num) {
+                    return;
                 }
             }
         }