Skip to content

谢明辉 / vion-label

Go to a project
Commit 124ae36c by xmh
Options

1. 添加人员池重构接口 

2. 修改日志下载查询语句
3. 添加reid获取人员列表和上一个下一个人时根据数量进行筛选
4. 图片信息和图片数据分开进行缓存
1 parent 51ced3cc
Inline Side-by-side
Showing 7 changed files with 109 additions and 36 deletions
label-core/src/main/java/com/viontech/label/core/constant/Constants.java
......@@ -17,11 +17,16 @@ public class Constants {
public static final Integer USER_TYPE_OUT_INSPECTOR = 4;
public static final Integer USER_TYPE_IN_INSPECTOR = 3;
private static final String REDIS_KEY_PIC_PREFIX = "pic";
private static final String REDIS_KEY_PIC_IMAGE_PREFIX = "picImage";
public static String getRedisKeyPic(Long picId) {
return REDIS_KEY_PIC_PREFIX + ":" + picId;
}
public static String getRedisKeyPicImage(Long picId) {
return REDIS_KEY_PIC_IMAGE_PREFIX + ":" + picId;
}
public static String getReidPoolName(Long packId) {
return "reid_pool_" + packId;
}
......
label-platform/src/main/java/com/viontech/label/platform/controller/web/PicController.java
......@@ -54,7 +54,7 @@ public class PicController extends PicBaseController {
@GetMapping("/image/{picId}")
@ResponseBody
public void getImage(@PathVariable("picId") Long picId, HttpServletResponse response) throws Exception {
PicVo pic = storageUtils.getPic(picId);
PicVo pic = storageUtils.getPicWithImg(picId);
if (pic == null) {
return;
}
......
label-platform/src/main/java/com/viontech/label/platform/controller/web/ReidController.java
......@@ -51,10 +51,11 @@ public class ReidController {
*/
@GetMapping("/getPeople")
public Object getPeople(@RequestParam Long packId, @RequestParam(required = false) Integer status
, @RequestParam(required = false) Long page, @RequestParam(required = false) Long size) {
, @RequestParam(required = false) Long page, @RequestParam(required = false) Long size
, @RequestParam(defaultValue = "1") Integer countGTE, @RequestParam(defaultValue = "10000") Integer countLTE) {
SubTaskMapper mapper = (SubTaskMapper) subTaskService.getMapper();
List<SubTask> people = mapper.getPeople(packId, status, page == null ? null : (page - 1) * size, size);
List<SubTask> people = mapper.getPeople(packId, status, page == null ? null : (page - 1) * size, size, countGTE, countLTE);
LinkedHashMap<String, List<SubTask>> temp = new LinkedHashMap<>();
for (SubTask item : people) {
......@@ -64,7 +65,7 @@ public class ReidController {
if (page != null) {
PageInfo<Object> pageInfo = new PageInfo<>(new ArrayList<>(temp.entrySet()), page.intValue());
pageInfo.setTotal(mapper.countPeople(packId, status));
pageInfo.setTotal(mapper.countPeople(packId, status, countGTE, countLTE));
return JsonMessageUtil.getSuccessJsonMsg(pageInfo);
} else {
return JsonMessageUtil.getSuccessJsonMsg(temp);
......@@ -72,9 +73,11 @@ public class ReidController {
}
@GetMapping("/getOtherPeople")
public Object getNextPeoPle(@RequestParam String personUnid, @RequestParam Long packId, @RequestParam(required = false) Integer status, @RequestParam Integer type) {
public Object getNextPeoPle(@RequestParam String personUnid, @RequestParam Long packId
, @RequestParam(required = false) Integer status, @RequestParam Integer type
, @RequestParam(defaultValue = "1") Integer countGTE, @RequestParam(defaultValue = "10000") Integer countLTE) {
SubTaskMapper mapper = (SubTaskMapper) subTaskService.getMapper();
List<SubTask> pics = mapper.getOtherPeople(packId, status, personUnid, type == 0 ? ">" : "<", type == 0 ? "" : "desc");
List<SubTask> pics = mapper.getOtherPeople(packId, status, personUnid, type == 0 ? ">" : "<", type == 0 ? "" : "desc", countGTE, countLTE);
if (pics.size() == 0) {
return JsonMessageUtil.getSuccessJsonMsg("没有数据了");
}
......@@ -200,8 +203,8 @@ public class ReidController {
}
@GetMapping("recovery")
public Object recovery(@RequestParam(required = false) String personUnid,@RequestParam(required = false) Long[] subTaskIdArr, @RequestParam Long packId) throws Exception {
reidService.recovery(personUnid,subTaskIdArr, packId);
public Object recovery(@RequestParam(required = false) String personUnid, @RequestParam(required = false) Long[] subTaskIdArr, @RequestParam Long packId) throws Exception {
reidService.recovery(personUnid, subTaskIdArr, packId);
return JsonMessageUtil.getSuccessJsonMsg("success");
}
......@@ -233,7 +236,11 @@ public class ReidController {
response.setHeader("Content-Disposition",
"attachment;filename=" + packId + "-" + DateUtil.format("yyyyMMdd", start) + "-" + DateUtil.format("yyyyMMdd", end) + ".xlsx");
EasyExcel.write(response.getOutputStream(), LogVo.class).includeColumnFiledNames(includeColumn).sheet().doWrite(logs);
}
@GetMapping("reconstruction")
public Object featurePoolReconstruction(@RequestParam Long packId) {
reidService.reConstruction(packId);
return JsonMessageUtil.getSuccessJsonMsg("success");
}
}
label-platform/src/main/java/com/viontech/label/platform/mapper/LogMapper.java
......@@ -33,8 +33,6 @@ public interface LogMapper extends BaseMapper {
int updateByPrimaryKey(Log record);
@Select("select operate as personUnid,count as picNum,u.name as username,log.create_time as finishTime from " +
"(select person_unid,count(*) from d_pic where pack_id=#{packId} and status=3 group by person_unid) as pic left join " +
"(select * from (select *,row_number() over (partition by operate order by create_time desc) row_id from s_log where operate_type=5 and operate_date between #{start} and #{end} ) t where t.row_id=1 ) as log on pic.person_unid=log.operate left join s_user u on log.operate_user=u.id where operate_user is not null")
@Select("select \"max\"(s_user.name) as username ,\"max\"(annotate_time) as finishTime ,person_unid as personUnid,count(*) as picNum from d_sub_task left join s_user on d_sub_task.annotator_id=s_user.id where pack_id=#{packId} and status=3 and annotate_time BETWEEN #{start} and #{end} group by person_unid ")
List<LogVo> exportLogByDateAndPack(Long packId, Date start, Date end);
}
\ No newline at end of file
label-platform/src/main/java/com/viontech/label/platform/mapper/SubTaskMapper.java
......@@ -59,12 +59,12 @@ public interface SubTaskMapper extends BaseMapper {
" <if test = 'status != null'> and status=#{status}</if> " +
" <if test = 'status == null'> and status != -1</if> " +
"group by person_unid " +
// " having count(*) > 1 " +
" having count(*) between #{countGTE} and #{countLTE} " +
" order by person_unid " +
"<if test='offset != null'> offset #{offset} limit #{limit}</if>) " +
" order by person_unid " +
"</script>")
List<SubTask> getPeople(Long packId, Integer status, Long offset, Long limit);
List<SubTask> getPeople(Long packId, Integer status, Long offset, Long limit, Integer countGTE, Integer countLTE);
@Select("<script>" +
"select count(*) from " +
......@@ -72,10 +72,10 @@ public interface SubTaskMapper extends BaseMapper {
" <if test = 'status != null'> and status=#{status}</if> " +
" <if test = 'status == null'> and status != -1</if> " +
" group by person_unid " +
// " having count(*) > 1 " +
" having count(*) between #{countGTE} and #{countLTE} " +
") as t " +
"</script>")
int countPeople(Long packId, Integer status);
int countPeople(Long packId, Integer status, Integer countGTE, Integer countLTE);
@Select("<script>" +
"select id,unid,create_time as createTime,pack_id as packId,pic_id as picId,task_id as taskId,person_unid as personUnid,status from d_sub_task where pack_id=#{packId} and person_unid=" +
......@@ -83,8 +83,8 @@ public interface SubTaskMapper extends BaseMapper {
"<if test = 'status != null'> and status=#{status}</if>" +
" <if test = 'status == null'> and status != -1</if> " +
" and person_unid ${type} #{personUnid} group by person_unid " +
// " having count(*) > 1 " +
" having count(*) between #{countGTE} and #{countLTE} " +
" order by person_unid ${sort} limit 1) order by id" +
"</script>")
List<SubTask> getOtherPeople(Long packId, Integer status, String personUnid, String type, String sort);
List<SubTask> getOtherPeople(Long packId, Integer status, String personUnid, String type, String sort, Integer countGTE, Integer countLTE);
}
\ No newline at end of file
label-platform/src/main/java/com/viontech/label/platform/service/main/ReidService.java
......@@ -102,6 +102,7 @@ public class ReidService {
subTask.setPackId(packId);
subTask.setCreateTime(data.getCountTime());
subTask.setTaskId(data.getTaskId());
subTask.setStatus(SubTaskStatus.TO_BE_LABELED.val);
subTaskService.insertSelective(subTask);
......@@ -331,6 +332,8 @@ public class ReidService {
subTaskService.updateByExampleSelective(subTask, subTaskExample);
return;
}
} else if (status.contains(SubTaskStatus.recycled.val)) {
return;
}
subTask.setAnnotatorId(StpUtil.getLoginIdAsLong());
......@@ -477,9 +480,12 @@ public class ReidService {
List<SubTask> subTasks = subTaskService.selectByExample(subTaskExample);
if (subTasks.size() > 0 && subTasks.get(0).getStatus().equals(SubTaskStatus.FINISH_LABELING.val)) {
if (subTasks.size() > 0) {
Integer status = subTasks.get(0).getStatus();
if (status.equals(SubTaskStatus.FINISH_LABELING.val) || status.equals(SubTaskStatus.recycled.val)) {
return true;
}
}
if (subTasks.size() == 0) {
return true;
}
......@@ -539,23 +545,22 @@ public class ReidService {
public void exitLabeling(String personUnid, Long packId) {
// 数据库更新状态
SubTask subTask = new SubTask();
subTask.setStatus(SubTaskStatus.TO_BE_LABELED.val);
SubTaskExample subTaskExample = new SubTaskExample();
subTaskExample.createCriteria().andPersonUnidEqualTo(personUnid).andPackIdEqualTo(packId);
// redis 删除缓存
RMap<String, User> labelingSet = redissonClient.getMap("labeling:" + packId);
labelingSet.remove(personUnid);
List<SubTask> subTasks = subTaskService.selectByExample(subTaskExample);
if (subTasks.size() > 0 && subTasks.get(0).getStatus() != SubTaskStatus.LABELING.val) {
return;
}
SubTask subTask = new SubTask();
subTask.setStatus(SubTaskStatus.TO_BE_LABELED.val);
subTaskService.updateByExampleSelective(subTask, subTaskExample);
// redis 删除缓存
RMap<String, User> labelingSet = redissonClient.getMap("labeling:" + packId);
labelingSet.remove(personUnid);
//websocket
ReidWebsocket.sendInfo(packId, new ReidWebsocket.Message().setCommand("exitLabeling").setData(personUnid));
}
......@@ -667,13 +672,12 @@ public class ReidService {
List<SubTask> allSubTask = subTaskService.selectByExample(subTaskExample);
Map<String, List<SubTask>> collect = allSubTask.stream().collect(Collectors.groupingBy(SubTask::getPersonUnid, Collectors.toList()));
byte[] bytes;
for (Map.Entry<String, List<SubTask>> entry : collect.entrySet()) {
String personUnid = entry.getKey();
List<SubTask> subtasks = entry.getValue();
for (SubTask subTask : subtasks) {
PicVo pic = storageUtils.getPic(subTask.getPicId());
PicVo pic = storageUtils.getPicWithImg(subTask.getPicId());
if (pic == null) {
continue;
}
......@@ -689,6 +693,47 @@ public class ReidService {
return tempFile;
}
public void reConstruction(Long packId) {
SubTaskExample subTaskExample = new SubTaskExample();
subTaskExample.createCriteria().andPackIdEqualTo(packId);
String poolName = Constants.getReidPoolName(packId);
try {
CompletableFuture<AlgResult> algResultCompletableFuture = matchClient.EASY_POOL_API.deletePool(poolName);
AlgResult algResult = algResultCompletableFuture.get();
} catch (Exception ignore) {
}
List<SubTask> subTasks = subTaskService.selectByExample(subTaskExample);
if (subTasks.size() == 0) {
return;
}
Map<String, List<SubTask>> collect = subTasks.stream().collect(Collectors.groupingBy(SubTask::getPersonUnid, Collectors.toList()));
for (Map.Entry<String, List<SubTask>> entry : collect.entrySet()) {
String personUnid = entry.getKey();
List<SubTask> value = entry.getValue();
ArrayList<Person> people = new ArrayList<>();
for (SubTask item : value) {
Feature featureByPic = storageUtils.getFeatureByPic(item.getPicId());
if (featureByPic == null) {
continue;
}
Person person = new Person().setPersonId(personUnid).setCounttime(item.getCreateTime()).setBodyFeatures(getBodyFeatures(featureByPic));
people.add(person);
}
try {
if (people.size() > 0) {
CompletableFuture<AlgResult> future = matchClient.EASY_PERSON_API.addPerson(poolName, people);
AlgResult result1 = future.get(20, TimeUnit.SECONDS);
log.info(result1.toString());
}
} catch (Exception e) {
log.error("添加特征失败", e);
}
}
}
public LogService getLogService() {
return logService;
}
......
label-platform/src/main/java/com/viontech/label/platform/utils/StorageUtils.java
......@@ -51,26 +51,44 @@ public class StorageUtils {
public PicVo getPic(Long picId) {
String redisKey = Constants.getRedisKeyPic(picId);
PicVo picVo = redissonClient.<PicVo>getBucket(redisKey).get();
RBucket<PicVo> picInfoBucket = redissonClient.getBucket(redisKey);
PicVo picVo = picInfoBucket.get();
if (picVo == null) {
// 获取并缓存 picInfo
Pic pic = picService.selectByPrimaryKey(picId);
if (pic == null) {
return null;
}
picVo = new PicVo(pic);
Long packId = pic.getPackId();
byte[] bytes;
picInfoBucket.set(picVo);
}
picInfoBucket.expire(12, TimeUnit.HOURS);
return picVo;
}
public PicVo getPicWithImg(Long picId) {
PicVo picVo = getPic(picId);
if (picVo == null) {
return null;
}
// 获取并缓存 image
Long packId = picVo.getPackId();
String picImageKey = Constants.getRedisKeyPicImage(picId);
RBucket<byte[]> picImageBucket = redissonClient.<byte[]>getBucket(picImageKey);
byte[] bytes = picImageBucket.get();
if (bytes == null) {
try {
bytes = getFile(packId, pic.getName());
bytes = getFile(packId, picVo.getName());
} catch (Exception e) {
log.info("", e);
return null;
}
picVo.setImage(bytes);
RBucket<PicVo> bucket = redissonClient.getBucket(redisKey);
bucket.set(picVo);
bucket.expire(12, TimeUnit.HOURS);
picImageBucket.set(bytes);
}
picImageBucket.expire(12, TimeUnit.HOURS);
picVo.setImage(bytes);
return picVo;
}
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!