Commit ed98d597 by xmh

<feat> 完善校时配置下发功能

<refactor> 重构导出代码
<fix> 数据概览添加数据量默认值 0
1 parent 884f56a0
...@@ -72,7 +72,6 @@ public class AuthorizationFilter implements GlobalFilter { ...@@ -72,7 +72,6 @@ public class AuthorizationFilter implements GlobalFilter {
} }
private ImmutablePair<Boolean, String> checkToken(String token) { private ImmutablePair<Boolean, String> checkToken(String token) {
// todo authorize
String username; String username;
try { try {
Map<String, String> api = new HashMap<>(2); Map<String, String> api = new HashMap<>(2);
......
...@@ -3,10 +3,8 @@ package com.viontech.fanxing.ops.controller.main; ...@@ -3,10 +3,8 @@ package com.viontech.fanxing.ops.controller.main;
import com.viontech.fanxing.commons.base.BaseController; import com.viontech.fanxing.commons.base.BaseController;
import com.viontech.fanxing.ops.model.OpsServer; import com.viontech.fanxing.ops.model.OpsServer;
import com.viontech.fanxing.ops.service.main.OpsServerService; import com.viontech.fanxing.ops.service.main.OpsServerService;
import org.springframework.web.bind.annotation.PostMapping; import org.springframework.web.bind.annotation.*;
import org.springframework.web.bind.annotation.RequestBody; import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
import javax.annotation.Resource; import javax.annotation.Resource;
...@@ -30,4 +28,10 @@ public class OpsController { ...@@ -30,4 +28,10 @@ public class OpsController {
return BaseController.success(); return BaseController.success();
} }
@PostMapping("updateVaServer")
public Object updateVaServer(@RequestParam String[] devIdArr, @RequestParam MultipartFile file) {
opsServerService.updateVaServer(devIdArr, file);
return BaseController.success();
}
} }
...@@ -8,7 +8,9 @@ import com.viontech.fanxing.commons.model.Content; ...@@ -8,7 +8,9 @@ import com.viontech.fanxing.commons.model.Content;
import com.viontech.fanxing.commons.model.ContentExample; import com.viontech.fanxing.commons.model.ContentExample;
import com.viontech.fanxing.commons.model.main.ImageKeepConfig; import com.viontech.fanxing.commons.model.main.ImageKeepConfig;
import com.viontech.fanxing.ops.mapper.ContentMapper; import com.viontech.fanxing.ops.mapper.ContentMapper;
import com.viontech.fanxing.ops.model.OpsServer;
import com.viontech.fanxing.ops.service.adapter.ContentService; import com.viontech.fanxing.ops.service.adapter.ContentService;
import com.viontech.fanxing.ops.service.main.OpsServerService;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import javax.annotation.Resource; import javax.annotation.Resource;
...@@ -21,6 +23,8 @@ public class ContentServiceImpl extends BaseServiceImpl<Content> implements Cont ...@@ -21,6 +23,8 @@ public class ContentServiceImpl extends BaseServiceImpl<Content> implements Cont
private static final String NAME_IMAGE_KEEP_CONFIG = "imageKeepConfig"; private static final String NAME_IMAGE_KEEP_CONFIG = "imageKeepConfig";
@Resource @Resource
private ContentMapper contentMapper; private ContentMapper contentMapper;
@Resource
private OpsServerService opsServerService;
@Override @Override
...@@ -57,7 +61,10 @@ public class ContentServiceImpl extends BaseServiceImpl<Content> implements Cont ...@@ -57,7 +61,10 @@ public class ContentServiceImpl extends BaseServiceImpl<Content> implements Cont
contentExample.createCriteria().andTypeEqualTo(TYPE_PLATFORM_CONFIG).andNameEqualTo(NAME_TIMING_CONFIG); contentExample.createCriteria().andTypeEqualTo(TYPE_PLATFORM_CONFIG).andNameEqualTo(NAME_TIMING_CONFIG);
addOrUpdate(TYPE_PLATFORM_CONFIG, NAME_TIMING_CONFIG, jsonObject.toJSONString()); addOrUpdate(TYPE_PLATFORM_CONFIG, NAME_TIMING_CONFIG, jsonObject.toJSONString());
// todo 发给运维服务 // todo 发给运维服务
List<OpsServer> opsServers = opsServerService.listAll();
for (OpsServer opsServer : opsServers) {
opsServerService.distributeTimingConfig(opsServer, jsonObject);
}
} }
......
package com.viontech.fanxing.ops.service.main; package com.viontech.fanxing.ops.service.main;
import com.alibaba.fastjson.JSONObject;
import com.viontech.fanxing.ops.model.OpsServer; import com.viontech.fanxing.ops.model.OpsServer;
import lombok.extern.slf4j.Slf4j;
import org.redisson.api.RBucket; import org.redisson.api.RBucket;
import org.redisson.api.RKeys; import org.redisson.api.RKeys;
import org.redisson.api.RedissonClient; import org.redisson.api.RedissonClient;
import org.springframework.stereotype.Service; import org.springframework.stereotype.Service;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.reactive.function.client.WebClient;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.time.Duration; import java.time.Duration;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.List; import java.util.List;
/** /**
...@@ -19,6 +24,7 @@ import java.util.List; ...@@ -19,6 +24,7 @@ import java.util.List;
*/ */
@Service @Service
@Slf4j
public class OpsServerService { public class OpsServerService {
@Resource @Resource
...@@ -29,7 +35,7 @@ public class OpsServerService { ...@@ -29,7 +35,7 @@ public class OpsServerService {
addOrUpdateOpsServer(opsServer); addOrUpdateOpsServer(opsServer);
} }
private List<OpsServer> listAll() { public List<OpsServer> listAll() {
ArrayList<OpsServer> opsServers = new ArrayList<>(); ArrayList<OpsServer> opsServers = new ArrayList<>();
RKeys keys = redissonClient.getKeys(); RKeys keys = redissonClient.getKeys();
Iterable<String> pattern = keys.getKeysByPattern("ops:server:*"); Iterable<String> pattern = keys.getKeysByPattern("ops:server:*");
...@@ -39,7 +45,7 @@ public class OpsServerService { ...@@ -39,7 +45,7 @@ public class OpsServerService {
return opsServers; return opsServers;
} }
private OpsServer getOpsServerFromRedisByIp(String ip) { public OpsServer getOpsServerFromRedisByIp(String ip) {
RBucket<OpsServer> bucket = redissonClient.getBucket("ops:server:" + ip); RBucket<OpsServer> bucket = redissonClient.getBucket("ops:server:" + ip);
if (bucket.isExists()) { if (bucket.isExists()) {
OpsServer opsServer = bucket.get(); OpsServer opsServer = bucket.get();
...@@ -54,6 +60,29 @@ public class OpsServerService { ...@@ -54,6 +60,29 @@ public class OpsServerService {
} }
} }
public void distributeTimingConfig(OpsServer opsServer, JSONObject jsonObject) {
try {
String ip = opsServer.getIp();
Integer port = opsServer.getPort();
JSONObject nvsResponse = WebClient.create()
.post()
.uri(uriBuilder -> uriBuilder.scheme("http").host(ip).port(port).path("/api/v1/isg/timing").build())
.bodyValue(jsonObject.toJSONString())
.retrieve()
.bodyToMono(JSONObject.class)
.block(Duration.ofSeconds(20));
} catch (Exception e) {
log.error("下发校时配置失败", e);
}
}
public void updateVaServer(String[] devIdArr, MultipartFile file) {
// todo 保存文件到本地,根据 devIdArr 获取对应分析服务,再根据分析服务的 ip 获取对应的运维服务,下发文件地址和升级信息
log.info("收到升级请求,目标服务:{},文件:{}", Arrays.toString(devIdArr), file.getName());
}
private void addOrUpdateOpsServer(OpsServer opsServer) { private void addOrUpdateOpsServer(OpsServer opsServer) {
RBucket<OpsServer> bucket = redissonClient.getBucket("ops:server:" + opsServer.getIp()); RBucket<OpsServer> bucket = redissonClient.getBucket("ops:server:" + opsServer.getIp());
bucket.set(opsServer); bucket.set(opsServer);
......
...@@ -14,9 +14,9 @@ import lombok.Setter; ...@@ -14,9 +14,9 @@ import lombok.Setter;
@Setter @Setter
public class DataOverViewModel { public class DataOverViewModel {
private Long taskId; private Long taskId;
private Long traffic; private Long traffic = 0L;
private Long flow; private Long flow = 0L;
private Long behavior; private Long behavior = 0L;
private Integer hour; private Integer hour;
private String taskName; private String taskName;
private Integer effectiveAnalysisTime; private Integer effectiveAnalysisTime;
......
...@@ -8,6 +8,7 @@ import com.alibaba.fastjson.JSON; ...@@ -8,6 +8,7 @@ import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONArray; import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject; import com.alibaba.fastjson.JSONObject;
import com.github.pagehelper.PageInfo; import com.github.pagehelper.PageInfo;
import com.google.common.base.Function;
import com.viontech.fanxing.commons.config.VionConfig; import com.viontech.fanxing.commons.config.VionConfig;
import com.viontech.fanxing.commons.exception.FanXingException; import com.viontech.fanxing.commons.exception.FanXingException;
import com.viontech.fanxing.commons.model.*; import com.viontech.fanxing.commons.model.*;
...@@ -27,6 +28,7 @@ import com.viontech.keliu.util.DateUtil; ...@@ -27,6 +28,7 @@ import com.viontech.keliu.util.DateUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.apache.commons.io.FileUtils; import org.apache.commons.io.FileUtils;
import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.StringUtils;
import org.apache.commons.lang3.tuple.ImmutableTriple;
import org.apache.poi.ss.usermodel.CellStyle; import org.apache.poi.ss.usermodel.CellStyle;
import org.redisson.api.RLock; import org.redisson.api.RLock;
import org.springframework.context.annotation.Profile; import org.springframework.context.annotation.Profile;
...@@ -126,9 +128,15 @@ public class ExportDataJob { ...@@ -126,9 +128,15 @@ public class ExportDataJob {
TrafficFlowRequestVo trafficFlowRequestVo = JSON.parseObject(param, TrafficFlowRequestVo.class); TrafficFlowRequestVo trafficFlowRequestVo = JSON.parseObject(param, TrafficFlowRequestVo.class);
String statisticType = trafficFlowRequestVo.getStatistic_type(); String statisticType = trafficFlowRequestVo.getStatistic_type();
int nextPage = 1; int nextPage = 1;
boolean interrupted = false;
List<String> pathList = new ArrayList<>(); List<String> pathList = new ArrayList<>();
while (nextPage != 0) { while (nextPage != 0) {
// 如果被删除了则需要打断
if (needInterrupt(item.getId())) {
log.info("任务已被删除,打断任务");
interrupted = true;
break;
}
trafficFlowRequestVo.setOffset((nextPage - 1) * pageSize); trafficFlowRequestVo.setOffset((nextPage - 1) * pageSize);
trafficFlowRequestVo.setLimit(pageSize); trafficFlowRequestVo.setLimit(pageSize);
Map<String, Object> stringObjectMap = flowEventController.statisticsResult(trafficFlowRequestVo); Map<String, Object> stringObjectMap = flowEventController.statisticsResult(trafficFlowRequestVo);
...@@ -171,9 +179,29 @@ public class ExportDataJob { ...@@ -171,9 +179,29 @@ public class ExportDataJob {
int totalpage = (totalNum / pageSize) + ((totalNum % pageSize) > 0 ? 1 : 0); int totalpage = (totalNum / pageSize) + ((totalNum % pageSize) > 0 ? 1 : 0);
nextPage = nextPage < totalpage ? nextPage + 1 : 0; nextPage = nextPage < totalpage ? nextPage + 1 : 0;
} }
// 如果被删除了则需要打断
if (needInterrupt(item.getId())) {
log.info("任务已被删除,打断任务");
interrupted = true;
}
if (interrupted) {
new Thread(() -> {
try {
File file = new File(vionConfig.getImage().getPath() + "/export/" + item.getId());
while (file.exists()) {
FileUtils.deleteDirectory(file);
Thread.sleep(10L);
log.info("删除目录:{}", file.getPath());
}
} catch (Exception e) {
log.error("中断了,删除失败", e);
}
}).start();
} else {
item.setPath(JSON.toJSONString(pathList)); item.setPath(JSON.toJSONString(pathList));
exportDataService.updateByPrimaryKeySelective(item); exportDataService.updateByPrimaryKeySelective(item);
} }
}
private ExcelWriter getWriter() { private ExcelWriter getWriter() {
ExcelWriter writer = ExcelUtil.getWriter(true); ExcelWriter writer = ExcelUtil.getWriter(true);
...@@ -184,37 +212,32 @@ public class ExportDataJob { ...@@ -184,37 +212,32 @@ public class ExportDataJob {
return writer; return writer;
} }
private void exportTraffic(ExportData item, int type) {
item.setStatus(1); private <K> ImmutableTriple<Boolean, String, Integer> export(ExportData item, Function<Integer, PageInfo<K>> getJsonData, Function<K, ExportBaseModel> build, boolean withPic, boolean withVideo, Integer nextPage,
boolean withVideo = item.getWithVideo() == 1; Function<K, String> getPic, Function<K, String> getVideo) {
boolean withPic = item.getWithPic() == 1; if (needInterrupt(item.getId())) {
int pageSize = withVideo ? LIMIT_WITH_VIDEO : (withPic ? LIMIT_WITH_PIC : LIMIT); log.info("任务已被删除,打断任务");
String param = item.getParam(); return ImmutableTriple.of(false, null, null);
TrafficVo trafficVo = JSON.parseObject(param, TrafficVo.class); }
TrafficExample example = trafficController.getExample(trafficVo); PageInfo<K> jsonData = getJsonData.apply(nextPage);
int nextPage = 1;
List<String> pathList = new ArrayList<>();
while (nextPage != 0) {
PageInfo<TrafficVo> jsonData = trafficService.getJsonData(example, nextPage, pageSize);
log.info("数据总量:{}", jsonData.getTotal()); log.info("数据总量:{}", jsonData.getTotal());
item.setCount(jsonData.getTotal()); item.setCount(jsonData.getTotal());
List<TrafficVo> list = jsonData.getList(); List<K> list = jsonData.getList();
if (list.size() == 0) { if (list.size() == 0) {
break; return ImmutableTriple.of(false, null, null);
} }
String path = vionConfig.getImage().getPath() + "/export/" + item.getId() + "/" + item.getName() + "_" + nextPage + ".zip"; String path = vionConfig.getImage().getPath() + "/export/" + item.getId() + "/" + item.getName() + "_" + nextPage + ".zip";
pathList.add(path);
File zipFile = new File(path); File zipFile = new File(path);
zipFile.getParentFile().mkdirs(); zipFile.getParentFile().mkdirs();
try (ZipOutputStream zipO = new ZipOutputStream(new FileOutputStream(zipFile))) { try (ZipOutputStream zipO = new ZipOutputStream(new FileOutputStream(zipFile))) {
List<ExportBaseModel> excelData = new ArrayList<>(); List<ExportBaseModel> excelData = new ArrayList<>();
for (TrafficVo vo : list) { for (K vo : list) {
// 添加数据 // 添加数据
ExportBaseModel o; ExportBaseModel o;
try { try {
o = buildTrafficData(type, vo); o = build.apply(vo);
} catch (Exception e) { } catch (Exception e) {
log.error("", e); log.error("", e);
continue; continue;
...@@ -222,8 +245,8 @@ public class ExportDataJob { ...@@ -222,8 +245,8 @@ public class ExportDataJob {
excelData.add(o); excelData.add(o);
// 写入图片文件 // 写入图片文件
if (withPic && StringUtils.isNotEmpty(vo.getPics())) { if (withPic && StringUtils.isNotEmpty(getPic.apply(vo))) {
File file = new File(vo.getPics()); File file = new File(getPic.apply(vo));
o.setPicName(new FormulaCellValue(getHyperLink(file.getName()))); o.setPicName(new FormulaCellValue(getHyperLink(file.getName())));
if (file.exists()) { if (file.exists()) {
try { try {
...@@ -237,8 +260,8 @@ public class ExportDataJob { ...@@ -237,8 +260,8 @@ public class ExportDataJob {
} }
} }
// 写入视频文件 // 写入视频文件
if (withVideo && StringUtils.isNotEmpty(vo.getVideoName())) { if (withVideo && StringUtils.isNotEmpty(getVideo.apply(vo))) {
File file = new File(vo.getVideoName()); File file = new File(getVideo.apply(vo));
o.setVideoName(new FormulaCellValue(getHyperLink(file.getName()))); o.setVideoName(new FormulaCellValue(getHyperLink(file.getName())));
if (file.exists()) { if (file.exists()) {
try { try {
...@@ -264,10 +287,41 @@ public class ExportDataJob { ...@@ -264,10 +287,41 @@ public class ExportDataJob {
log.error("", e); log.error("", e);
} }
nextPage = jsonData.getNextPage(); nextPage = jsonData.getNextPage();
return ImmutableTriple.of(true, path, nextPage);
}
private void exportTraffic(ExportData item, int type) {
item.setStatus(1);
boolean withVideo = item.getWithVideo() == 1;
boolean withPic = item.getWithPic() == 1;
int pageSize = withVideo ? LIMIT_WITH_VIDEO : (withPic ? LIMIT_WITH_PIC : LIMIT);
String param = item.getParam();
TrafficVo trafficVo = JSON.parseObject(param, TrafficVo.class);
TrafficExample example = trafficController.getExample(trafficVo);
int nextPage = 1;
List<String> pathList = new ArrayList<>();
while (nextPage != 0) {
ImmutableTriple<Boolean, String, Integer> export = export(item,
(page) -> trafficService.getJsonData(example, page, pageSize),
x -> buildTrafficData(type, x), withPic, withVideo, nextPage,
TrafficVo::getPics, TrafficVo::getVideoName);
if (export.left) {
String path = export.middle;
pathList.add(path);
nextPage = export.right;
} else {
break;
}
} }
// 如果被删除了则需要打断
if (needInterrupt(item.getId())) {
log.info("任务已被删除,打断任务");
deleteAsync(vionConfig.getImage().getPath() + "/export/" + item.getId());
} else {
item.setPath(JSON.toJSONString(pathList)); item.setPath(JSON.toJSONString(pathList));
exportDataService.updateByPrimaryKeySelective(item); exportDataService.updateByPrimaryKeySelective(item);
} }
}
private void exportBehavior(ExportData item) { private void exportBehavior(ExportData item) {
item.setStatus(1); item.setStatus(1);
...@@ -278,80 +332,34 @@ public class ExportDataJob { ...@@ -278,80 +332,34 @@ public class ExportDataJob {
BehaviorVo behaviorVo = JSON.parseObject(param, BehaviorVo.class); BehaviorVo behaviorVo = JSON.parseObject(param, BehaviorVo.class);
BehaviorExample example = behaviorController.getExample(behaviorVo); BehaviorExample example = behaviorController.getExample(behaviorVo);
int nextPage = 1; int nextPage = 1;
boolean interrupted = false;
List<String> pathList = new ArrayList<>(); List<String> pathList = new ArrayList<>();
while (nextPage != 0) { while (nextPage != 0) {
ImmutableTriple<Boolean, String, Integer> export = export(item,
(page) -> behaviorService.getJsonData(example, page, pageSize),
this::buildBehaviorModel, withPic, withVideo, nextPage,
BehaviorVo::getPics, BehaviorVo::getVideo);
if (export.left) {
String path = export.middle;
pathList.add(path);
nextPage = export.right;
} else {
break;
}
}
// 如果被删除了则需要打断 // 如果被删除了则需要打断
if (needInterrupt(item.getId())) { if (needInterrupt(item.getId())) {
log.info("任务已被删除,打断任务"); log.info("任务已被删除,打断任务");
interrupted = true; deleteAsync(vionConfig.getImage().getPath() + "/export/" + item.getId());
break; } else {
item.setPath(JSON.toJSONString(pathList));
exportDataService.updateByPrimaryKeySelective(item);
} }
PageInfo<BehaviorVo> jsonData = behaviorService.getJsonData(example, nextPage, pageSize);
log.info("数据总量:{}", jsonData.getTotal());
item.setCount(jsonData.getTotal());
List<BehaviorVo> list = jsonData.getList();
if (list.size() == 0) {
break;
} }
String path = vionConfig.getImage().getPath() + "/export/" + item.getId() + "/" + item.getName() + "_" + nextPage + ".zip";
pathList.add(path);
File zipFile = new File(path);
zipFile.getParentFile().mkdirs();
try (ZipOutputStream zipO = new ZipOutputStream(new FileOutputStream(zipFile))) {
List<ExportBaseModel> excelData = new ArrayList<>();
for (BehaviorVo vo : list) {
ExportBaseModel o = buildBehaviorModel(vo); private void deleteAsync(String path) {
excelData.add(o);
// 写入图片文件
if (withPic && StringUtils.isNotEmpty(vo.getPics())) {
File file = new File(vo.getPics());
if (file.exists()) {
try {
byte[] bytes = FileUtils.readFileToByteArray(file);
zipO.putNextEntry(new ZipEntry(file.getName()));
zipO.write(bytes);
o.setPicName(new FormulaCellValue(getHyperLink(file.getName())));
} catch (Exception e) {
log.error("", e);
}
}
}
// 写入视频文件
if (withVideo && StringUtils.isNotEmpty(vo.getVideo())) {
File file = new File(vo.getVideo());
if (file.exists()) {
try {
byte[] bytes = FileUtils.readFileToByteArray(file);
zipO.putNextEntry(new ZipEntry(file.getName()));
zipO.write(bytes);
o.setVideoName(new FormulaCellValue(getHyperLink(file.getName())));
} catch (Exception e) {
log.error("", e);
}
}
}
}
// 写入 excel
ByteArrayOutputStream outputStream = new ByteArrayOutputStream();
ExcelWriter writer = getExcelWriter();
writer.write(excelData, true);
writer.flush(outputStream);
writer.close();
zipO.putNextEntry(new ZipEntry("data.xlsx"));
zipO.write(outputStream.toByteArray());
} catch (Exception e) {
log.error("", e);
}
nextPage = jsonData.getNextPage();
}
if (interrupted) {
new Thread(() -> { new Thread(() -> {
try { try {
File file = new File(vionConfig.getImage().getPath() + "/export/" + item.getId()); File file = new File(path);
while (file.exists()) { while (file.exists()) {
FileUtils.deleteDirectory(file); FileUtils.deleteDirectory(file);
Thread.sleep(10L); Thread.sleep(10L);
...@@ -361,10 +369,6 @@ public class ExportDataJob { ...@@ -361,10 +369,6 @@ public class ExportDataJob {
log.error("中断了,删除失败", e); log.error("中断了,删除失败", e);
} }
}).start(); }).start();
} else {
item.setPath(JSON.toJSONString(pathList));
exportDataService.updateByPrimaryKeySelective(item);
}
} }
private boolean needInterrupt(Long id) { private boolean needInterrupt(Long id) {
......
...@@ -36,7 +36,7 @@ public class RandomRuntimeConfig implements RuntimeConfig { ...@@ -36,7 +36,7 @@ public class RandomRuntimeConfig implements RuntimeConfig {
@Override @Override
public ImmutablePair<Long, Long> getNextTimeOfExecutionAndTerminal() { public ImmutablePair<Long, Long> getNextTimeOfExecutionAndTerminal() {
// todo 目前逻辑还未定 // todo 目前逻辑还未定,只有手动启动才会执行一次
long running = TimeUnit.MINUTES.toMillis(runningTime); long running = TimeUnit.MINUTES.toMillis(runningTime);
Date date = DateUtil.setDayMinTime(new Date()); Date date = DateUtil.setDayMinTime(new Date());
long time = date.getTime() - running; long time = date.getTime() - running;
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!