Skip to content

谢明辉 / oss-update-application

Go to a project
Commit f61c8909 by 谢明辉
Options

改为下载

1 parent 9cca5a8b
Inline Side-by-side
Showing 5 changed files with 161 additions and 121 deletions
src/main/java/com/vion/Application.java
...@@ -2,30 +2,26 @@ package com.vion; ...@@ -2,30 +2,26 @@ package com.vion;
import com.aliyun.oss.model.GetObjectRequest; import com.aliyun.oss.model.GetObjectRequest;
import com.aliyun.oss.model.ListObjectsRequest;
import com.aliyun.oss.model.ObjectListing; import com.aliyun.oss.model.ObjectListing;
import com.vion.configuration.OssConfigration;
import com.vion.dao.IDao;
import com.vion.entity.OssClientEntity; import com.vion.entity.OssClientEntity;
import com.vion.utils.OssFileUtil; import com.vion.utils.OssFileUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner; import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication; import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.File; import java.io.File;
import java.text.SimpleDateFormat; import java.util.ArrayList;
import java.util.Date; import java.util.List;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ComponentScan(value = "com.vion") @ComponentScan(value = "com.vion")
@SpringBootApplication @SpringBootApplication(exclude = {DataSourceAutoConfiguration.class})
@Slf4j @Slf4j
public class Application implements CommandLineRunner { public class Application implements CommandLineRunner {
...@@ -35,12 +31,16 @@ public class Application implements CommandLineRunner { ...@@ -35,12 +31,16 @@ public class Application implements CommandLineRunner {
public static void main(String[] args) { public static void main(String[] args) {
try {
SpringApplication.run(Application.class, args); SpringApplication.run(Application.class, args);
} catch (Exception e) {
e.printStackTrace();
}
} }
@Autowired // @Autowired
private IDao dao; // private IDao dao;
@Resource @Resource
private OssClientEntity ossClientEntity; private OssClientEntity ossClientEntity;
...@@ -52,48 +52,82 @@ public class Application implements CommandLineRunner { ...@@ -52,48 +52,82 @@ public class Application implements CommandLineRunner {
private String begin; private String begin;
@Value("${date.end}") @Value("${date.end}")
private String end; private String end;
@Value("${prefix}")
private String prefix;
@Value(("${serialnumStr:}"))
private String serialnumStr;
@Override @Override
public void run(String... args) throws Exception { public void run(String... args) throws Exception {
// File file = new File(facePath); // File file = new File(facePath);
// if (!file.exists()) { // if (!file.exists()) {
// log.error("face路径不存在!!!!!:{}", facePath); // log.error("face路径不存在!!!!!:{}", facePath);
// System.exit(1); // System.exit(1);
// } // }
// File file1 = new File(featurePath); // File file1 = new File(featurePath);
// if (!file1.exists()) { // if (!file1.exists()) {
// log.error("feature路径不存在!!!!!!!!!!{}", featurePath); // log.error("feature路径不存在!!!!!!!!!!{}", featurePath);
// System.exit(2); // System.exit(2);
// } // }
List<String> filePathList = new ArrayList<>();
if (!serialnumStr.isEmpty()) {
String[] serialnums = serialnumStr.split(",");
for (String serialnum : serialnums) {
filePathList.add(prefix + serialnum + "/");
}
} else {
filePathList.add(prefix);
}
for (String filePath : filePathList) {
/*下载文件*/ /*下载文件*/
OssFileUtil ossFileUtil = new OssFileUtil(); OssFileUtil ossFileUtil = new OssFileUtil();
ObjectListing objectListing = ossFileUtil.getFilesInPrefix(ossClientEntity.getBucket(), "picture/face/20190322/08A0-F93D-CEFE-9881-01/", ossClientEntity.getOssClient(), 1000, false); ObjectListing objectListing = null;
do {
ListObjectsRequest request = ossFileUtil.getFilesInPrefix(ossClientEntity.getBucket(), filePath, ossClientEntity.getOssClient(), 500, false);
if (objectListing != null) {
request.setMarker(objectListing.getNextMarker());
}
objectListing = ossClientEntity.getOssClient().listObjects(request);
objectListing.getObjectSummaries().forEach(summery -> { objectListing.getObjectSummaries().forEach(summery -> {
File f = new File("D:\\" + summery.getKey().replace("\\", "/")); File f = new File("D:\\" + summery.getKey().replace("\\", "/"));
if (!f.getParentFile().exists()) { if (!f.getParentFile().exists()) {
f.getParentFile().mkdirs(); f.getParentFile().mkdirs();
} }
ossClientEntity.getOssClient().getObject(new GetObjectRequest(ossClientEntity.getBucket(), summery.getKey()),f ); if (!f.exists()) {
System.out.println("正在下载" + summery.getKey());
ossClientEntity.getOssClient().getObject(new GetObjectRequest(ossClientEntity.getBucket(), summery.getKey()), f);
} else {
System.out.println(summery.getKey() + "已存在，跳过-------------");
}
} }
); );
} while (objectListing.isTruncated());
}
//上传文件 //上传文件
// 建立线程池 // 建立线程池
// //
// ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> { // ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
// Thread t = new Thread(r); // Thread t = new Thread(r);
// t.setName("线程" + OssConfigration.COUNT); // t.setName("线程" + OssConfigration.COUNT);
// OssConfigration.COUNT++; // OssConfigration.COUNT++;
// return t; // return t;
// }); // });
// //
// //
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); // SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// Date beginDate = format.parse(begin); // Date beginDate = format.parse(begin);
// Date endDate = format.parse(end); // Date endDate = format.parse(end);
// //
// dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool); // dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool);
} }
} }
src/main/java/com/vion/dao/MyFaceDao.java
...@@ -22,9 +22,9 @@ import java.util.concurrent.ThreadPoolExecutor; ...@@ -22,9 +22,9 @@ import java.util.concurrent.ThreadPoolExecutor;
*/ */
@Slf4j @Slf4j
public class MyFaceDao implements IDao { public class MyFaceDao implements IDao {
//
@Autowired // @Autowired
private JdbcTemplate jdbcTemplate; // private JdbcTemplate jdbcTemplate;
public static long DAY = 3600L * 24L * 1000L; public static long DAY = 3600L * 24L * 1000L;
...@@ -33,41 +33,41 @@ public class MyFaceDao implements IDao { ...@@ -33,41 +33,41 @@ public class MyFaceDao implements IDao {
@Override @Override
public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) { public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) {
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); // SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
//
// 按天循环 // // 按天循环
for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) { // for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
log.info(count + "================================================"); // log.info(count + "================================================");
//
// 查询一天的数据 // // 查询一天的数据
List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> { // List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
FaceEntity entity = new FaceEntity(); // FaceEntity entity = new FaceEntity();
entity.setBodyPic(r.getString("frr_arrival_showbody_picture")); // entity.setBodyPic(r.getString("frr_arrival_showbody_picture"));
entity.setFacePic(r.getString("frr_arrival_face_picture1")); // entity.setFacePic(r.getString("frr_arrival_face_picture1"));
entity.setChannelSerialNum(r.getString("chanel_id")); // entity.setChannelSerialNum(r.getString("chanel_id"));
return entity; // return entity;
}); // });
//
// 平均分成50份，每个线程执行其中的一份，共50个线程 // // 平均分成50份，每个线程执行其中的一份，共50个线程
List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50); // List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
//
Integer pathDate = Integer.parseInt(format.format(new Date(count))); // Integer pathDate = Integer.parseInt(format.format(new Date(count)));
lists.forEach(faceEntities -> threadPool.execute(new MyUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath))); // lists.forEach(faceEntities -> threadPool.execute(new MyUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
//
// 等待线程池执行完成，执行下个循环 // // 等待线程池执行完成，执行下个循环
while (threadPool.getActiveCount() != 0) { // while (threadPool.getActiveCount() != 0) {
//
} // }
//
log.info("==============================={}结束=========================================", new Date(count)); // log.info("==============================={}结束=========================================", new Date(count));
//
System.gc(); // System.gc();
//
try { // try {
Thread.sleep(1000L); // Thread.sleep(1000L);
} catch (InterruptedException e) { // } catch (InterruptedException e) {
e.printStackTrace(); // e.printStackTrace();
} // }
} // }
} }
} }
src/main/java/com/vion/dao/PostFaceDao.java
...@@ -27,46 +27,46 @@ public class PostFaceDao implements IDao { ...@@ -27,46 +27,46 @@ public class PostFaceDao implements IDao {
public static long DAY = 3600L * 24L * 1000L; public static long DAY = 3600L * 24L * 1000L;
public static final String FIND_ALL = "SELECT channel_serialnum,face_pic,body_pic FROM d_face_recognition WHERE countdate=?"; public static final String FIND_ALL = "SELECT channel_serialnum,face_pic,body_pic FROM d_face_recognition WHERE countdate=?";
@Autowired // @Autowired
private JdbcTemplate jdbcTemplate; // private JdbcTemplate jdbcTemplate;
@Override @Override
public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) { public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) {
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); // SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
//
// 按天循环 // // 按天循环
for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) { // for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
log.info(count + "================================================"); // log.info(count + "================================================");
//
// 查询一天的数据 // // 查询一天的数据
List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> { // List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
FaceEntity entity = new FaceEntity(); // FaceEntity entity = new FaceEntity();
entity.setBodyPic(r.getString("body_pic")); // entity.setBodyPic(r.getString("body_pic"));
entity.setFacePic(r.getString("face_pic")); // entity.setFacePic(r.getString("face_pic"));
entity.setChannelSerialNum(r.getString("channel_serialnum")); // entity.setChannelSerialNum(r.getString("channel_serialnum"));
return entity; // return entity;
}); // });
//
// 平均分成50份，每个线程执行其中的一份，共50个线程 // // 平均分成50份，每个线程执行其中的一份，共50个线程
List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50); // List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
//
Integer pathDate = Integer.parseInt(format.format(new Date(count))); // Integer pathDate = Integer.parseInt(format.format(new Date(count)));
lists.forEach(faceEntities -> threadPool.execute(new PostUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath))); // lists.forEach(faceEntities -> threadPool.execute(new PostUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
//
// 等待线程池执行完成，执行下个循环 // // 等待线程池执行完成，执行下个循环
while (threadPool.getActiveCount() != 0) { // while (threadPool.getActiveCount() != 0) {
//
} // }
//
log.info("==============================={}结束=========================================", new Date(count)); // log.info("==============================={}结束=========================================", new Date(count));
//
System.gc(); // System.gc();
//
try { // try {
Thread.sleep(1000L); // Thread.sleep(1000L);
} catch (InterruptedException e) { // } catch (InterruptedException e) {
e.printStackTrace(); // e.printStackTrace();
} // }
} // }
} }
} }
src/main/java/com/vion/utils/OssFileUtil.java
package com.vion.utils; package com.vion.utils;
import com.aliyun.oss.OSSClient; import com.aliyun.oss.OSSClient;
import com.aliyun.oss.model.ListObjectsRequest; import com.aliyun.oss.model.ListObjectsRequest;
import com.aliyun.oss.model.ObjectListing; import com.aliyun.oss.model.ObjectListing;
/** /**
* @author 谢明辉 * @author 谢明辉
* @createDate 2018-11-6 * @createDate 2018-11-6
...@@ -21,16 +23,15 @@ public class OssFileUtil { ...@@ -21,16 +23,15 @@ public class OssFileUtil {
* @createDate 2018-11-6 * @createDate 2018-11-6
* @description 列举指定目录下的所有文件 * @description 列举指定目录下的所有文件
*/ */
public ObjectListing getFilesInPrefix(String bucketName, String prefix, OSSClient ossClient, Integer maxKeys, boolean delimiter) { public ListObjectsRequest getFilesInPrefix(String bucketName, String prefix, OSSClient ossClient, Integer maxKeys, boolean delimiter) {
ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName); ListObjectsRequest listObjectsRequest = new ListObjectsRequest(bucketName);
if (maxKeys != null) { if (maxKeys != null) {
maxKeys = maxKeys > 1000 ? 1000 : (maxKeys < 1 ? 1 : maxKeys);
listObjectsRequest.setMaxKeys(maxKeys); listObjectsRequest.setMaxKeys(maxKeys);
} }
if (delimiter) { if (delimiter) {
listObjectsRequest.setDelimiter("/"); listObjectsRequest.setDelimiter("/");
} }
listObjectsRequest.setPrefix(prefix); listObjectsRequest.setPrefix(prefix);
return ossClient.listObjects(listObjectsRequest); return listObjectsRequest;
} }
} }
src/main/resources/application-pro.properties
oss.config.bucket=vion-mall oss.config.bucket=vion-mall
oss.config.endPoint=oss-cn-beijing.aliyuncs.com oss.config.endPoint=oss-cn-beijing.aliyuncs.com
oss.config.accessKeyId=LTAI9WfUr3GDne4c oss.config.accessKeyId=LTAI9WfUr3GDne4c
...@@ -18,9 +19,13 @@ date.end=20180825 ...@@ -18,9 +19,13 @@ date.end=20180825
#spring.datasource.password=vion #spring.datasource.password=vion
#flag=postgre #flag=postgre
########################Path format\uFF1Aface/date/channel_serialNum ,Database connection using mysql flag=mysql ########################Path format\uFF1Aface/date/channel_serialNum ,Database connection using mysql flag=mysql
spring.datasource.driver-class-name=com.mysql.jdbc.Driver #spring.datasource.driver-class-name=com.mysql.jdbc.Driver
spring.datasource.url=jdbc\:mysql\://47.94.47.137\:3306/vioncountcloud?useUnicode\=true&characterEncoding\=UTF-8 #spring.datasource.url=jdbc\:mysql\://47.94.47.137\:3306/vioncountcloud?useUnicode\=true&characterEncoding\=UTF-8
spring.datasource.username=root #spring.datasource.username=root
spring.datasource.password=IH5c3t0di14Ay7y5 #spring.datasource.password=IH5c3t0di14Ay7y5
flag=mysql flag=mysql
server.port=8888 server.port=8888
prefix=picture/body/20190402/
serialnumStr=0269-F060-E72E-52FE-01,00FE-63D0-0345-CA16-01
\ No newline at end of file \ No newline at end of file
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!