Skip to content

谢明辉 / oss-update-application

Go to a project
Commit 71c105bd by 谢明辉
Options

1

1 parent e3309767
Inline Side-by-side
Showing 5 changed files with 186 additions and 130 deletions
src/main/java/com/vion/Application.java
...@@ -3,6 +3,7 @@ package com.vion; ...@@ -3,6 +3,7 @@ package com.vion;
import com.aliyun.oss.model.GetObjectRequest; import com.aliyun.oss.model.GetObjectRequest;
import com.aliyun.oss.model.ListObjectsRequest; import com.aliyun.oss.model.ListObjectsRequest;
import com.aliyun.oss.model.OSSObjectSummary;
import com.aliyun.oss.model.ObjectListing; import com.aliyun.oss.model.ObjectListing;
import com.vion.entity.OssClientEntity; import com.vion.entity.OssClientEntity;
import com.vion.utils.OssFileUtil; import com.vion.utils.OssFileUtil;
...@@ -13,11 +14,16 @@ import org.springframework.boot.SpringApplication; ...@@ -13,11 +14,16 @@ import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration; import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan;
import org.springframework.scheduling.concurrent.DefaultManagedAwareThreadFactory;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.File; import java.io.File;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ComponentScan(value = "com.vion") @ComponentScan(value = "com.vion")
...@@ -39,19 +45,8 @@ public class Application implements CommandLineRunner { ...@@ -39,19 +45,8 @@ public class Application implements CommandLineRunner {
} }
// @Autowired
// private IDao dao;
@Resource @Resource
private OssClientEntity ossClientEntity; private OssClientEntity ossClientEntity;
@Value("${path.picture}")
private String facePath;
@Value("${path.feature}")
private String featurePath;
@Value("${date.begin}")
private String begin;
@Value("${date.end}")
private String end;
@Value("${prefix}") @Value("${prefix}")
private String prefix; private String prefix;
@Value(("${serialnumStr:}")) @Value(("${serialnumStr:}"))
...@@ -62,17 +57,8 @@ public class Application implements CommandLineRunner { ...@@ -62,17 +57,8 @@ public class Application implements CommandLineRunner {
@Override @Override
public void run(String... args) throws Exception { public void run(String... args) throws Exception {
// File file = new File(facePath);
// if (!file.exists()) {
// log.error("face路径不存在!!!!!:{}", facePath);
// System.exit(1);
// }
// File file1 = new File(featurePath);
// if (!file1.exists()) {
// log.error("feature路径不存在!!!!!!!!!!{}", featurePath);
// System.exit(2);
// }
ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new LinkedBlockingQueue<>(),new DefaultManagedAwareThreadFactory());
if (!"".equals(filePath)) { if (!"".equals(filePath)) {
try { try {
...@@ -110,39 +96,33 @@ public class Application implements CommandLineRunner { ...@@ -110,39 +96,33 @@ public class Application implements CommandLineRunner {
objectListing = ossClientEntity.getOssClient().listObjects(request); objectListing = ossClientEntity.getOssClient().listObjects(request);
objectListing.getObjectSummaries().forEach(summery -> { objectListing.getObjectSummaries().forEach(summery -> threadPool.execute(() -> {download(summery);})
File f = new File("D:\\" + summery.getKey().replace("\\", "/")); );
} while (objectListing.isTruncated());
}
}
private void download(OSSObjectSummary summery) {
try {
File f = new File("D:\\" + summery.getKey().replace("\\", "/"));
if (!f.getParentFile().exists()) {
synchronized (this) {
if (!f.getParentFile().exists()) { if (!f.getParentFile().exists()) {
f.getParentFile().mkdirs(); f.getParentFile().mkdirs();
} }
if (!f.exists()) {
System.out.println("正在下载" + summery.getKey());
ossClientEntity.getOssClient().getObject(new GetObjectRequest(ossClientEntity.getBucket(), summery.getKey()), f);
} else {
System.out.println(summery.getKey() + "已存在，跳过-------------");
}
} }
); }
} while (objectListing.isTruncated()); if (!f.exists()) {
System.out.println("正在下载" + summery.getKey());
ossClientEntity.getOssClient().getObject(new GetObjectRequest(ossClientEntity.getBucket(), summery.getKey()), f);
} else {
System.out.println(summery.getKey() + "已存在，跳过-------------");
}
} catch (Exception e) {
e.printStackTrace();
} }
//上传文件
// 建立线程池
//
// ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
// Thread t = new Thread(r);
// t.setName("线程" + OssConfigration.COUNT);
// OssConfigration.COUNT++;
// return t;
// });
//
//
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// Date beginDate = format.parse(begin);
// Date endDate = format.parse(end);
//
// dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool);
} }
} }
src/main/java/com/vion/UpdateRunner.java 0 → 100644
package com.vion;
import com.vion.configuration.OssConfigration;
import com.vion.dao.IDao;
import com.vion.entity.OssClientEntity;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner;
import javax.annotation.Resource;
import java.io.File;
import java.text.SimpleDateFormat;
import java.util.Date;
import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
/**
* .
*
* @author 谢明辉
* @date 2019-7-29 14:44
*/
//@Component
//@Order(1)
@Slf4j
public class UpdateRunner implements CommandLineRunner {
@Value("${date.begin}")
private String begin;
@Value("${date.end}")
private String end;
@Autowired
private IDao dao;
@Resource
private OssClientEntity ossClientEntity;
@Value("${path.picture}")
private String facePath;
@Value("${path.feature}")
private String featurePath;
@Override
public void run(String... args) throws Exception {
File file = new File(facePath);
if (!file.exists()) {
log.error("face路径不存在!!!!!:{}", facePath);
System.exit(1);
}
File file1 = new File(featurePath);
if (!file1.exists()) {
log.error("feature路径不存在!!!!!!!!!!{}", featurePath);
System.exit(2);
}
ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
Thread t = new Thread(r);
t.setName("线程" + OssConfigration.COUNT);
OssConfigration.COUNT++;
return t;
});
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
Date beginDate = format.parse(begin);
Date endDate = format.parse(end);
dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool);
}
}
src/main/java/com/vion/dao/MyFaceDao.java
...@@ -22,9 +22,9 @@ import java.util.concurrent.ThreadPoolExecutor; ...@@ -22,9 +22,9 @@ import java.util.concurrent.ThreadPoolExecutor;
*/ */
@Slf4j @Slf4j
public class MyFaceDao implements IDao { public class MyFaceDao implements IDao {
//
// @Autowired @Autowired
// private JdbcTemplate jdbcTemplate; private JdbcTemplate jdbcTemplate;
public static long DAY = 3600L * 24L * 1000L; public static long DAY = 3600L * 24L * 1000L;
...@@ -33,41 +33,41 @@ public class MyFaceDao implements IDao { ...@@ -33,41 +33,41 @@ public class MyFaceDao implements IDao {
@Override @Override
public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) { public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) {
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
//
// // 按天循环 // 按天循环
// for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) { for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
// log.info(count + "================================================"); log.info(count + "================================================");
//
// // 查询一天的数据 // 查询一天的数据
// List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> { List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
// FaceEntity entity = new FaceEntity(); FaceEntity entity = new FaceEntity();
// entity.setBodyPic(r.getString("frr_arrival_showbody_picture")); entity.setBodyPic(r.getString("frr_arrival_showbody_picture"));
// entity.setFacePic(r.getString("frr_arrival_face_picture1")); entity.setFacePic(r.getString("frr_arrival_face_picture1"));
// entity.setChannelSerialNum(r.getString("chanel_id")); entity.setChannelSerialNum(r.getString("chanel_id"));
// return entity; return entity;
// }); });
//
// // 平均分成50份，每个线程执行其中的一份，共50个线程 // 平均分成50份，每个线程执行其中的一份，共50个线程
// List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50); List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
//
// Integer pathDate = Integer.parseInt(format.format(new Date(count))); Integer pathDate = Integer.parseInt(format.format(new Date(count)));
// lists.forEach(faceEntities -> threadPool.execute(new MyUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath))); lists.forEach(faceEntities -> threadPool.execute(new MyUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
//
// // 等待线程池执行完成，执行下个循环 // 等待线程池执行完成，执行下个循环
// while (threadPool.getActiveCount() != 0) { while (threadPool.getActiveCount() != 0) {
//
// } }
//
// log.info("==============================={}结束=========================================", new Date(count)); log.info("==============================={}结束=========================================", new Date(count));
//
// System.gc(); System.gc();
//
// try { try {
// Thread.sleep(1000L); Thread.sleep(1000L);
// } catch (InterruptedException e) { } catch (InterruptedException e) {
// e.printStackTrace(); e.printStackTrace();
// } }
// } }
} }
} }
src/main/java/com/vion/dao/PostFaceDao.java
...@@ -27,46 +27,46 @@ public class PostFaceDao implements IDao { ...@@ -27,46 +27,46 @@ public class PostFaceDao implements IDao {
public static long DAY = 3600L * 24L * 1000L; public static long DAY = 3600L * 24L * 1000L;
public static final String FIND_ALL = "SELECT channel_serialnum,face_pic,body_pic FROM d_face_recognition WHERE countdate=?"; public static final String FIND_ALL = "SELECT channel_serialnum,face_pic,body_pic FROM d_face_recognition WHERE countdate=?";
// @Autowired @Autowired
// private JdbcTemplate jdbcTemplate; private JdbcTemplate jdbcTemplate;
@Override @Override
public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) { public void getAll(Date beginDate, Date endDate, OssClientEntity ossClientEntity, String facePath, String featurePath, ThreadPoolExecutor threadPool) {
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
//
// // 按天循环 // 按天循环
// for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) { for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
// log.info(count + "================================================"); log.info(count + "================================================");
//
// // 查询一天的数据 // 查询一天的数据
// List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> { List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
// FaceEntity entity = new FaceEntity(); FaceEntity entity = new FaceEntity();
// entity.setBodyPic(r.getString("body_pic")); entity.setBodyPic(r.getString("body_pic"));
// entity.setFacePic(r.getString("face_pic")); entity.setFacePic(r.getString("face_pic"));
// entity.setChannelSerialNum(r.getString("channel_serialnum")); entity.setChannelSerialNum(r.getString("channel_serialnum"));
// return entity; return entity;
// }); });
//
// // 平均分成50份，每个线程执行其中的一份，共50个线程 // 平均分成50份，每个线程执行其中的一份，共50个线程
// List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50); List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
//
// Integer pathDate = Integer.parseInt(format.format(new Date(count))); Integer pathDate = Integer.parseInt(format.format(new Date(count)));
// lists.forEach(faceEntities -> threadPool.execute(new PostUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath))); lists.forEach(faceEntities -> threadPool.execute(new PostUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
//
// // 等待线程池执行完成，执行下个循环 // 等待线程池执行完成，执行下个循环
// while (threadPool.getActiveCount() != 0) { while (threadPool.getActiveCount() != 0) {
//
// } }
//
// log.info("==============================={}结束=========================================", new Date(count)); log.info("==============================={}结束=========================================", new Date(count));
//
// System.gc(); System.gc();
//
// try { try {
// Thread.sleep(1000L); Thread.sleep(1000L);
// } catch (InterruptedException e) { } catch (InterruptedException e) {
// e.printStackTrace(); e.printStackTrace();
// } }
// } }
} }
} }
src/main/resources/application-pro.properties
oss.config.bucket=vion-mall oss.config.bucket=vion-retail
oss.config.endPoint=oss-cn-beijing.aliyuncs.com oss.config.endPoint=oss-cn-beijing.aliyuncs.com
oss.config.accessKeyId=LTAI9WfUr3GDne4c oss.config.accessKeyId=LTAI9WfUr3GDne4c
oss.config.accessKeySecret=SDzMs26kNyiDIIRJIY2qUBxBOeUEZ8 oss.config.accessKeySecret=SDzMs26kNyiDIIRJIY2qUBxBOeUEZ8
...@@ -27,9 +27,9 @@ flag=mysql ...@@ -27,9 +27,9 @@ flag=mysql
server.port=8888 server.port=8888
########## ##########
prefix=picture/body/20190402/ prefix=picture/face/20190725/
serialnumStr=0269-F060-E72E-52FE-01,00FE-63D0-0345-CA16-01 serialnumStr=0D1B-2680-A196-F936-01,3E55-F0AE-1FDF-B42B-01,D8C0-833D-1F08-FE70-01,C63D-32E2-2582-2BE1-01,7F6A-C70D-D580-D06B-01,E7EB-ECB4-B73B-B4C4-01,E7EB-ECB4-B73B-B4C4-02
#### assigned to a specific file\u2018s path #### assigned to a specific file\u2018s path
file_path=staff/face/upload/6450f60a-5f6f-408f-a75a-b417cdc6302d.jpg
\ No newline at end of file \ No newline at end of file
#file_path=staff/face/upload/6450f60a-5f6f-408f-a75a-b417cdc6302d.jpg
\ No newline at end of file \ No newline at end of file
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!