Skip to content

谢明辉 / oss-update-application

Go to a project
Commit 9cca5a8b by 谢明辉
Options

123

1 parent da6dec3a
Inline Side-by-side
Showing 3 changed files with 45 additions and 30 deletions
src/main/java/com/vion/Application.java
package com.vion; package com.vion;
import com.aliyun.oss.model.GetObjectRequest;
import com.aliyun.oss.model.ObjectListing;
import com.vion.configuration.OssConfigration; import com.vion.configuration.OssConfigration;
import com.vion.dao.IDao; import com.vion.dao.IDao;
import com.vion.entity.OssClientEntity; import com.vion.entity.OssClientEntity;
import com.vion.utils.OssFileUtil;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
...@@ -13,10 +16,10 @@ import org.springframework.boot.autoconfigure.SpringBootApplication; ...@@ -13,10 +16,10 @@ import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.ComponentScan; import org.springframework.context.annotation.ComponentScan;
import javax.annotation.Resource; import javax.annotation.Resource;
import java.io.File;
import java.text.SimpleDateFormat; import java.text.SimpleDateFormat;
import java.util.Date; import java.util.Date;
import java.util.concurrent.ArrayBlockingQueue; import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
...@@ -52,36 +55,45 @@ public class Application implements CommandLineRunner { ...@@ -52,36 +55,45 @@ public class Application implements CommandLineRunner {
@Override @Override
public void run(String... args) throws Exception { public void run(String... args) throws Exception {
/*下载文件*/ // File file = new File(facePath);
// OssFileUtil ossFileUtil = new OssFileUtil();
// ObjectListing objectListing = ossFileUtil.getFilesInPrefix(ossClientEntity.getBucket(), "picture/face/20180808/", ossClientEntity.getOssClient(), 1000, false);
// objectListing.getObjectSummaries().forEach(summery -> {
// String[] strings = summery.getKey().split("/");
// File file = new File("D:/picture/face/20180808/" + strings[3]);
// if (!file.exists()) { // if (!file.exists()) {
// if (!file.mkdir()) { // log.error("face路径不存在!!!!!:{}", facePath);
// System.out.println("创建文件夹失败:" + file.getName()); // System.exit(1);
// }
// } // }
// ossClientEntity.getOssClient().getObject(new GetObjectRequest(ossClientEntity.getBucket(), summery.getKey()), new File("D:\\" + summery.getKey().replace("\\", "/"))); // File file1 = new File(featurePath);
// if (!file1.exists()) {
// log.error("feature路径不存在!!!!!!!!!!{}", featurePath);
// System.exit(2);
// } // }
// );
//上传文件
// 建立线程池
ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
Thread t = new Thread(r);
t.setName("线程" + OssConfigration.COUNT);
OssConfigration.COUNT++;
return t;
});
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd"); /*下载文件*/
Date beginDate = format.parse(begin); OssFileUtil ossFileUtil = new OssFileUtil();
Date endDate = format.parse(end); ObjectListing objectListing = ossFileUtil.getFilesInPrefix(ossClientEntity.getBucket(), "picture/face/20190322/08A0-F93D-CEFE-9881-01/", ossClientEntity.getOssClient(), 1000, false);
objectListing.getObjectSummaries().forEach(summery -> {
File f = new File("D:\\" + summery.getKey().replace("\\", "/"));
if (!f.getParentFile().exists()) {
f.getParentFile().mkdirs();
}
ossClientEntity.getOssClient().getObject(new GetObjectRequest(ossClientEntity.getBucket(), summery.getKey()),f );
}
);
dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool); //上传文件
// 建立线程池
//
// ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
// Thread t = new Thread(r);
// t.setName("线程" + OssConfigration.COUNT);
// OssConfigration.COUNT++;
// return t;
// });
//
//
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// Date beginDate = format.parse(begin);
// Date endDate = format.parse(end);
//
// dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool);
} }
} }
src/main/java/com/vion/utils/FileUtils.java
...@@ -6,6 +6,7 @@ import java.util.ArrayList; ...@@ -6,6 +6,7 @@ import java.util.ArrayList;
import java.util.List; import java.util.List;
import java.util.regex.Pattern; import java.util.regex.Pattern;
/** /**
* @author 谢明辉 * @author 谢明辉
* @createDate 2018-10-30 * @createDate 2018-10-30
...@@ -13,6 +14,7 @@ import java.util.regex.Pattern; ...@@ -13,6 +14,7 @@ import java.util.regex.Pattern;
*/ */
public class FileUtils { public class FileUtils {
public static Pattern pattern = Pattern.compile("[0-9]{8}"); public static Pattern pattern = Pattern.compile("[0-9]{8}");
public static void getFiles(List<File> fileList, String basePath, Integer begin, Integer end) { public static void getFiles(List<File> fileList, String basePath, Integer begin, Integer end) {
...@@ -104,14 +106,15 @@ public class FileUtils { ...@@ -104,14 +106,15 @@ public class FileUtils {
public static String getPathName(String basePath, Integer dateNumber, String channelSerialNum, String fileName, boolean isFeature) { public static String getPathName(String basePath, Integer dateNumber, String channelSerialNum, String fileName, boolean isFeature) {
StringBuilder sb = new StringBuilder(); StringBuilder sb = new StringBuilder();
sb.append(basePath).append(dateNumber).append("\\"); sb.append(basePath).append(dateNumber).append(File.separator);
if (isNotNull(channelSerialNum)) { if (isNotNull(channelSerialNum)) {
sb.append(channelSerialNum).append("\\"); sb.append(channelSerialNum).append(File.separator);
} }
sb.append(fileName); sb.append(fileName);
if (isFeature) { if (isFeature) {
sb.append(".feature"); sb.append(".feature");
} }
System.out.println(sb.toString());
return sb.toString(); return sb.toString();
} }
} }
src/main/resources/application-pro.properties
oss.config.bucket=vion-retail oss.config.bucket=vion-mall
oss.config.endPoint=oss-cn-beijing.aliyuncs.com oss.config.endPoint=oss-cn-beijing.aliyuncs.com
oss.config.accessKeyId=LTAI9WfUr3GDne4c oss.config.accessKeyId=LTAI9WfUr3GDne4c
oss.config.accessKeySecret=SDzMs26kNyiDIIRJIY2qUBxBOeUEZ8 oss.config.accessKeySecret=SDzMs26kNyiDIIRJIY2qUBxBOeUEZ8
#######################Face root directory####################### #######################Face root directory#######################
path.picture=D:\\face\\s path.picture=D:\\face\\
##################Feature root directory###################### ##################Feature root directory######################
path.feature=D:\\feature\\ path.feature=D:\\feature\\
#########################the application will upload the data in [beginDate,endDate] #########################the application will upload the data in [beginDate,endDate]
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!