Skip to content

谢明辉 / oss-update-application

Go to a project
Commit f61c8909 by 谢明辉
Options

改为下载

1 parent 9cca5a8b
Inline Side-by-side
Showing 5 changed files with 161 additions and 121 deletions
src/main/java/com/vion/Application.java
......@@ -2,30 +2,26 @@ package com.vion;
import com.aliyun.oss.model.GetObjectRequest;
import com.aliyun.oss.model.ListObjectsRequest;
import com.aliyun.oss.model.ObjectListing;
import com.vion.configuration.OssConfigration;
import com.vion.dao.IDao;
import com.vion.entity.OssClientEntity;
import com.vion.utils.OssFileUtil;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
import javax.annotation.Resource;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.ArrayList;
import java.util.List;
@ComponentScan(value = "com.vion")
@SpringBootApplication
@SpringBootApplication(exclude = {DataSourceAutoConfiguration.class})
@Slf4j
public class Application implements CommandLineRunner {
......@@ -35,12 +31,16 @@ public class Application implements CommandLineRunner {
public static void main(String[] args) {
try {
SpringApplication.run(Application.class, args);
} catch (Exception e) {
e.printStackTrace();
}
}
@Autowired
private IDao dao;
// @Autowired
// private IDao dao;
@Resource
private OssClientEntity ossClientEntity;
......@@ -52,48 +52,82 @@ public class Application implements CommandLineRunner {
private String begin;
@Value("${date.end}")
private String end;
@Value("${prefix}")
private String prefix;
@Value(("${serialnumStr:}"))
private String serialnumStr;
@Override
public void run(String... args) throws Exception {
// File file = new File(facePath);
// if (!file.exists()) {
// log.error("face路径不存在!!!!!:{}", facePath);
// System.exit(1);
// }
// File file1 = new File(featurePath);
// if (!file1.exists()) {
// log.error("feature路径不存在!!!!!!!!!!{}", featurePath);
// System.exit(2);
// }
// File file = new File(facePath);
// if (!file.exists()) {
// log.error("face路径不存在!!!!!:{}", facePath);
// System.exit(1);
// }
// File file1 = new File(featurePath);
// if (!file1.exists()) {
// log.error("feature路径不存在!!!!!!!!!!{}", featurePath);
// System.exit(2);
// }
List<String> filePathList = new ArrayList<>();
if (!serialnumStr.isEmpty()) {
String[] serialnums = serialnumStr.split(",");
for (String serialnum : serialnums) {
filePathList.add(prefix + serialnum + "/");
}
} else {
filePathList.add(prefix);
}
for (String filePath : filePathList) {
/*下载文件*/
OssFileUtil ossFileUtil = new OssFileUtil();
ObjectListing objectListing = ossFileUtil.getFilesInPrefix(ossClientEntity.getBucket(), "picture/face/20190322/08A0-F93D-CEFE-9881-01/", ossClientEntity.getOssClient(), 1000, false);
ObjectListing objectListing = null;
do {
ListObjectsRequest request = ossFileUtil.getFilesInPrefix(ossClientEntity.getBucket(), filePath, ossClientEntity.getOssClient(), 500, false);
if (objectListing != null) {
request.setMarker(objectListing.getNextMarker());
}
objectListing = ossClientEntity.getOssClient().listObjects(request);
objectListing.getObjectSummaries().forEach(summery -> {
File f = new File("D:\\" + summery.getKey().replace("\\", "/"));
if (!f.getParentFile().exists()) {
f.getParentFile().mkdirs();
}
ossClientEntity.getOssClient().getObject(new GetObjectRequest(ossClientEntity.getBucket(), summery.getKey()),f );
if (!f.exists()) {
System.out.println("正在下载" + summery.getKey());
ossClientEntity.getOssClient().getObject(new GetObjectRequest(ossClientEntity.getBucket(), summery.getKey()), f);
} else {
System.out.println(summery.getKey() + "已存在，跳过-------------");
}
}
);
} while (objectListing.isTruncated());
}
//上传文件
// 建立线程池
//
// ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
// Thread t = new Thread(r);
// t.setName("线程" + OssConfigration.COUNT);
// OssConfigration.COUNT++;
// return t;
// });
//
//
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// Date beginDate = format.parse(begin);
// Date endDate = format.parse(end);
//
// dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool);
//
// ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
// Thread t = new Thread(r);
// t.setName("线程" + OssConfigration.COUNT);
// OssConfigration.COUNT++;
// return t;
// });
//
//
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// Date beginDate = format.parse(begin);
// Date endDate = format.parse(end);
//
// dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool);
}
}
src/main/java/com/vion/dao/MyFaceDao.java
......@@ -22,9 +22,9 @@ import java.util.concurrent.ThreadPoolExecutor;
*/
@Slf4j
public class MyFaceDao implements IDao {
@Autowired
private JdbcTemplate jdbcTemplate;
//
// @Autowired
// private JdbcTemplate jdbcTemplate;
public static long DAY = 3600L * 24L * 1000L;
......@@ -33,41 +33,41 @@ public class MyFaceDao implements IDao {
@Override
public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) {
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// 按天循环
for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
log.info(count + "================================================");
// 查询一天的数据
List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
FaceEntity entity = new FaceEntity();
entity.setBodyPic(r.getString("frr_arrival_showbody_picture"));
entity.setFacePic(r.getString("frr_arrival_face_picture1"));
entity.setChannelSerialNum(r.getString("chanel_id"));
return entity;
});
// 平均分成50份，每个线程执行其中的一份，共50个线程
List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
Integer pathDate = Integer.parseInt(format.format(new Date(count)));
lists.forEach(faceEntities -> threadPool.execute(new MyUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
// 等待线程池执行完成，执行下个循环
while (threadPool.getActiveCount() != 0) {
}
log.info("==============================={}结束=========================================", new Date(count));
System.gc();
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
//
// // 按天循环
// for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
// log.info(count + "================================================");
//
// // 查询一天的数据
// List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
// FaceEntity entity = new FaceEntity();
// entity.setBodyPic(r.getString("frr_arrival_showbody_picture"));
// entity.setFacePic(r.getString("frr_arrival_face_picture1"));
// entity.setChannelSerialNum(r.getString("chanel_id"));
// return entity;
// });
//
// // 平均分成50份，每个线程执行其中的一份，共50个线程
// List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
//
// Integer pathDate = Integer.parseInt(format.format(new Date(count)));
// lists.forEach(faceEntities -> threadPool.execute(new MyUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
//
// // 等待线程池执行完成，执行下个循环
// while (threadPool.getActiveCount() != 0) {
//
// }
//
// log.info("==============================={}结束=========================================", new Date(count));
//
// System.gc();
//
// try {
// Thread.sleep(1000L);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
// }
}
}
src/main/java/com/vion/dao/PostFaceDao.java
......@@ -27,46 +27,46 @@ public class PostFaceDao implements IDao {
public static long DAY = 3600L * 24L * 1000L;
public static final String FIND_ALL = "SELECT channel_serialnum,face_pic,body_pic FROM d_face_recognition WHERE countdate=?";
@Autowired
private JdbcTemplate jdbcTemplate;
// @Autowired
// private JdbcTemplate jdbcTemplate;
@Override
public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) {
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// 按天循环
for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
log.info(count + "================================================");
// 查询一天的数据
List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
FaceEntity entity = new FaceEntity();
entity.setBodyPic(r.getString("body_pic"));
entity.setFacePic(r.getString("face_pic"));
entity.setChannelSerialNum(r.getString("channel_serialnum"));
return entity;
});
// 平均分成50份，每个线程执行其中的一份，共50个线程
List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
Integer pathDate = Integer.parseInt(format.format(new Date(count)));
lists.forEach(faceEntities -> threadPool.execute(new PostUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
// 等待线程池执行完成，执行下个循环
while (threadPool.getActiveCount() != 0) {
}
log.info("==============================={}结束=========================================", new Date(count));
System.gc();
try {
Thread.sleep(1000L);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
//
// // 按天循环
// for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
// log.info(count + "================================================");
//
// // 查询一天的数据
// List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
// FaceEntity entity = new FaceEntity();
// entity.setBodyPic(r.getString("body_pic"));
// entity.setFacePic(r.getString("face_pic"));
// entity.setChannelSerialNum(r.getString("channel_serialnum"));
// return entity;
// });
//
// // 平均分成50份，每个线程执行其中的一份，共50个线程
// List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
//
// Integer pathDate = Integer.parseInt(format.format(new Date(count)));
// lists.forEach(faceEntities -> threadPool.execute(new PostUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
//
// // 等待线程池执行完成，执行下个循环
// while (threadPool.getActiveCount() != 0) {
//
// }
//
// log.info("==============================={}结束=========================================", new Date(count));
//
// System.gc();
//
// try {
// Thread.sleep(1000L);
// } catch (InterruptedException e) {
// e.printStackTrace();
// }
// }
}
}
src/main/java/com/vion/utils/OssFileUtil.java
package com.vion.utils;
import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.ListObjectsRequest;
import com.aliyun.oss.model.ObjectListing;
/**
* @author 谢明辉
* @createDate 2018-11-6
......@@ -21,16 +23,15 @@ public class OssFileUtil {
* @createDate 2018-11-6
* @description 列举指定目录下的所有文件
*/
public ObjectListing getFilesInPrefix(String bucketName, String prefix, OSSClient ossClient, Integer maxKeys, boolean delimiter) {
public ListObjectsRequest getFilesInPrefix(String bucketName, String prefix, OSSClient ossClient, Integer maxKeys, boolean delimiter) {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName);
if (maxKeys != null) {
maxKeys = maxKeys > 1000 ? 1000 : (maxKeys < 1 ? 1 : maxKeys);
listObjectsRequest.setMaxKeys(maxKeys);
}
if (delimiter) {
listObjectsRequest.setDelimiter("/");
}
listObjectsRequest.setPrefix(prefix);
return ossClient.listObjects(listObjectsRequest);
return listObjectsRequest;
}
}
src/main/resources/application-pro.properties
oss.config.bucket=vion-mall
oss.config.endPoint=oss-cn-beijing.aliyuncs.com
oss.config.accessKeyId=LTAI9WfUr3GDne4c
......@@ -18,9 +19,13 @@ date.end=20180825
#spring.datasource.password=vion
#flag=postgre
########################Path format\uFF1Aface/date/channel_serialNum ,Database connection using mysql flag=mysql
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc\:mysql\://47.94.47.137\:3306/vioncountcloud?useUnicode\=true&characterEncoding\=UTF-8
spring.datasource.username=root
spring.datasource.password=IH5c3t0di14Ay7y5
#spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#spring.datasource.url=jdbc\:mysql\://47.94.47.137\:3306/vioncountcloud?useUnicode\=true&characterEncoding\=UTF-8
#spring.datasource.username=root
#spring.datasource.password=IH5c3t0di14Ay7y5
flag=mysql
server.port=8888
prefix=picture/body/20190402/
serialnumStr=0269-F060-E72E-52FE-01,00FE-63D0-0345-CA16-01
\ No newline at end of file
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!