Skip to content
Toggle navigation
Projects
Groups
Snippets
Help
Toggle navigation
This project
Loading...
Sign in
谢明辉
/
oss-update-application
Go to a project
Project
Repository
Issues
0
Merge Requests
0
Pipelines
Wiki
Settings
Activity
Graph
Charts
Create a new issue
Jobs
Commits
Issue Boards
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Charts
Commit f61c8909
authored
May 14, 2019
by
谢明辉
Browse Files
Options
Browse Files
Tag
Download
Email Patches
Plain Diff
改为下载
1 parent
9cca5a8b
Show whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
161 additions
and
121 deletions
src/main/java/com/vion/Application.java
src/main/java/com/vion/dao/MyFaceDao.java
src/main/java/com/vion/dao/PostFaceDao.java
src/main/java/com/vion/utils/OssFileUtil.java
src/main/resources/application-pro.properties
src/main/java/com/vion/Application.java
View file @
f61c890
...
...
@@ -2,30 +2,26 @@ package com.vion;
import
com.aliyun.oss.model.GetObjectRequest
;
import
com.aliyun.oss.model.ListObjectsRequest
;
import
com.aliyun.oss.model.ObjectListing
;
import
com.vion.configuration.OssConfigration
;
import
com.vion.dao.IDao
;
import
com.vion.entity.OssClientEntity
;
import
com.vion.utils.OssFileUtil
;
import
lombok.extern.slf4j.Slf4j
;
import
org.springframework.beans.factory.annotation.Autowired
;
import
org.springframework.beans.factory.annotation.Value
;
import
org.springframework.boot.CommandLineRunner
;
import
org.springframework.boot.SpringApplication
;
import
org.springframework.boot.autoconfigure.SpringBootApplication
;
import
org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration
;
import
org.springframework.context.annotation.ComponentScan
;
import
javax.annotation.Resource
;
import
java.io.File
;
import
java.text.SimpleDateFormat
;
import
java.util.Date
;
import
java.util.concurrent.ArrayBlockingQueue
;
import
java.util.concurrent.ThreadPoolExecutor
;
import
java.util.concurrent.TimeUnit
;
import
java.util.ArrayList
;
import
java.util.List
;
@ComponentScan
(
value
=
"com.vion"
)
@SpringBootApplication
@SpringBootApplication
(
exclude
=
{
DataSourceAutoConfiguration
.
class
})
@Slf4j
public
class
Application
implements
CommandLineRunner
{
...
...
@@ -35,12 +31,16 @@ public class Application implements CommandLineRunner {
public
static
void
main
(
String
[]
args
)
{
try
{
SpringApplication
.
run
(
Application
.
class
,
args
);
}
catch
(
Exception
e
)
{
e
.
printStackTrace
();
}
}
@Autowired
private
IDao
dao
;
//
@Autowired
//
private IDao dao;
@Resource
private
OssClientEntity
ossClientEntity
;
...
...
@@ -52,48 +52,82 @@ public class Application implements CommandLineRunner {
private
String
begin
;
@Value
(
"${date.end}"
)
private
String
end
;
@Value
(
"${prefix}"
)
private
String
prefix
;
@Value
((
"${serialnumStr:}"
))
private
String
serialnumStr
;
@Override
public
void
run
(
String
...
args
)
throws
Exception
{
// File file = new File(facePath);
// if (!file.exists()) {
// log.error("face路径不存在!!!!!:{}", facePath);
// System.exit(1);
// }
// File file1 = new File(featurePath);
// if (!file1.exists()) {
// log.error("feature路径不存在!!!!!!!!!!{}", featurePath);
// System.exit(2);
// }
// File file = new File(facePath);
// if (!file.exists()) {
// log.error("face路径不存在!!!!!:{}", facePath);
// System.exit(1);
// }
// File file1 = new File(featurePath);
// if (!file1.exists()) {
// log.error("feature路径不存在!!!!!!!!!!{}", featurePath);
// System.exit(2);
// }
List
<
String
>
filePathList
=
new
ArrayList
<>();
if
(!
serialnumStr
.
isEmpty
())
{
String
[]
serialnums
=
serialnumStr
.
split
(
","
);
for
(
String
serialnum
:
serialnums
)
{
filePathList
.
add
(
prefix
+
serialnum
+
"/"
);
}
}
else
{
filePathList
.
add
(
prefix
);
}
for
(
String
filePath
:
filePathList
)
{
/*下载文件*/
OssFileUtil
ossFileUtil
=
new
OssFileUtil
();
ObjectListing
objectListing
=
ossFileUtil
.
getFilesInPrefix
(
ossClientEntity
.
getBucket
(),
"picture/face/20190322/08A0-F93D-CEFE-9881-01/"
,
ossClientEntity
.
getOssClient
(),
1000
,
false
);
ObjectListing
objectListing
=
null
;
do
{
ListObjectsRequest
request
=
ossFileUtil
.
getFilesInPrefix
(
ossClientEntity
.
getBucket
(),
filePath
,
ossClientEntity
.
getOssClient
(),
500
,
false
);
if
(
objectListing
!=
null
)
{
request
.
setMarker
(
objectListing
.
getNextMarker
());
}
objectListing
=
ossClientEntity
.
getOssClient
().
listObjects
(
request
);
objectListing
.
getObjectSummaries
().
forEach
(
summery
->
{
File
f
=
new
File
(
"D:\\"
+
summery
.
getKey
().
replace
(
"\\"
,
"/"
));
if
(!
f
.
getParentFile
().
exists
())
{
f
.
getParentFile
().
mkdirs
();
}
ossClientEntity
.
getOssClient
().
getObject
(
new
GetObjectRequest
(
ossClientEntity
.
getBucket
(),
summery
.
getKey
()),
f
);
if
(!
f
.
exists
())
{
System
.
out
.
println
(
"正在下载"
+
summery
.
getKey
());
ossClientEntity
.
getOssClient
().
getObject
(
new
GetObjectRequest
(
ossClientEntity
.
getBucket
(),
summery
.
getKey
()),
f
);
}
else
{
System
.
out
.
println
(
summery
.
getKey
()
+
"已存在,跳过-------------"
);
}
}
);
}
while
(
objectListing
.
isTruncated
());
}
//上传文件
// 建立线程池
//
// ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
// Thread t = new Thread(r);
// t.setName("线程" + OssConfigration.COUNT);
// OssConfigration.COUNT++;
// return t;
// });
//
//
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// Date beginDate = format.parse(begin);
// Date endDate = format.parse(end);
//
// dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool);
//
// ThreadPoolExecutor threadPool = new ThreadPoolExecutor(50, 100, 10, TimeUnit.MINUTES, new ArrayBlockingQueue<Runnable>(250), r -> {
// Thread t = new Thread(r);
// t.setName("线程" + OssConfigration.COUNT);
// OssConfigration.COUNT++;
// return t;
// });
//
//
// SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
// Date beginDate = format.parse(begin);
// Date endDate = format.parse(end);
//
// dao.getAll(beginDate, endDate, ossClientEntity, facePath, featurePath, threadPool);
}
}
src/main/java/com/vion/dao/MyFaceDao.java
View file @
f61c890
...
...
@@ -22,9 +22,9 @@ import java.util.concurrent.ThreadPoolExecutor;
*/
@Slf4j
public
class
MyFaceDao
implements
IDao
{
@Autowired
private
JdbcTemplate
jdbcTemplate
;
//
//
@Autowired
//
private JdbcTemplate jdbcTemplate;
public
static
long
DAY
=
3600L
*
24L
*
1000L
;
...
...
@@ -33,41 +33,41 @@ public class MyFaceDao implements IDao {
@Override
public
void
getAll
(
Date
beginDate
,
Date
endDate
,
OssClientEntity
ossClientEntity
,
String
facePath
,
String
featurePath
,
ThreadPoolExecutor
threadPool
)
{
SimpleDateFormat
format
=
new
SimpleDateFormat
(
"yyyyMMdd"
);
// 按天循环
for
(
long
count
=
beginDate
.
getTime
();
count
<=
endDate
.
getTime
();
count
+=
DAY
)
{
log
.
info
(
count
+
"================================================"
);
// 查询一天的数据
List
<
FaceEntity
>
list
=
jdbcTemplate
.
query
(
FIND_ALL
,
new
Object
[]{
new
Date
(
count
)},
(
r
,
i
)
->
{
FaceEntity
entity
=
new
FaceEntity
();
entity
.
setBodyPic
(
r
.
getString
(
"frr_arrival_showbody_picture"
));
entity
.
setFacePic
(
r
.
getString
(
"frr_arrival_face_picture1"
));
entity
.
setChannelSerialNum
(
r
.
getString
(
"chanel_id"
));
return
entity
;
});
// 平均分成50份,每个线程执行其中的一份,共50个线程
List
<
List
<
FaceEntity
>>
lists
=
FileUtils
.
averageAssign
(
list
,
50
);
Integer
pathDate
=
Integer
.
parseInt
(
format
.
format
(
new
Date
(
count
)));
lists
.
forEach
(
faceEntities
->
threadPool
.
execute
(
new
MyUpdateJob
(
faceEntities
,
ossClientEntity
,
pathDate
,
facePath
,
featurePath
)));
// 等待线程池执行完成,执行下个循环
while
(
threadPool
.
getActiveCount
()
!=
0
)
{
}
log
.
info
(
"==============================={}结束========================================="
,
new
Date
(
count
));
System
.
gc
();
try
{
Thread
.
sleep
(
1000L
);
}
catch
(
InterruptedException
e
)
{
e
.
printStackTrace
();
}
}
//
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
//
//
// 按天循环
//
for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
//
log.info(count + "================================================");
//
//
// 查询一天的数据
//
List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
//
FaceEntity entity = new FaceEntity();
//
entity.setBodyPic(r.getString("frr_arrival_showbody_picture"));
//
entity.setFacePic(r.getString("frr_arrival_face_picture1"));
//
entity.setChannelSerialNum(r.getString("chanel_id"));
//
return entity;
//
});
//
//
// 平均分成50份,每个线程执行其中的一份,共50个线程
//
List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
//
//
Integer pathDate = Integer.parseInt(format.format(new Date(count)));
//
lists.forEach(faceEntities -> threadPool.execute(new MyUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
//
//
// 等待线程池执行完成,执行下个循环
//
while (threadPool.getActiveCount() != 0) {
//
//
}
//
//
log.info("==============================={}结束=========================================", new Date(count));
//
//
System.gc();
//
//
try {
//
Thread.sleep(1000L);
//
} catch (InterruptedException e) {
//
e.printStackTrace();
//
}
//
}
}
}
src/main/java/com/vion/dao/PostFaceDao.java
View file @
f61c890
...
...
@@ -27,46 +27,46 @@ public class PostFaceDao implements IDao {
public
static
long
DAY
=
3600L
*
24L
*
1000L
;
public
static
final
String
FIND_ALL
=
"SELECT channel_serialnum,face_pic,body_pic FROM d_face_recognition WHERE countdate=?"
;
@Autowired
private
JdbcTemplate
jdbcTemplate
;
//
@Autowired
//
private JdbcTemplate jdbcTemplate;
@Override
public
void
getAll
(
Date
beginDate
,
Date
endDate
,
OssClientEntity
ossClientEntity
,
String
facePath
,
String
featurePath
,
ThreadPoolExecutor
threadPool
)
{
SimpleDateFormat
format
=
new
SimpleDateFormat
(
"yyyyMMdd"
);
// 按天循环
for
(
long
count
=
beginDate
.
getTime
();
count
<=
endDate
.
getTime
();
count
+=
DAY
)
{
log
.
info
(
count
+
"================================================"
);
// 查询一天的数据
List
<
FaceEntity
>
list
=
jdbcTemplate
.
query
(
FIND_ALL
,
new
Object
[]{
new
Date
(
count
)},
(
r
,
i
)
->
{
FaceEntity
entity
=
new
FaceEntity
();
entity
.
setBodyPic
(
r
.
getString
(
"body_pic"
));
entity
.
setFacePic
(
r
.
getString
(
"face_pic"
));
entity
.
setChannelSerialNum
(
r
.
getString
(
"channel_serialnum"
));
return
entity
;
});
// 平均分成50份,每个线程执行其中的一份,共50个线程
List
<
List
<
FaceEntity
>>
lists
=
FileUtils
.
averageAssign
(
list
,
50
);
Integer
pathDate
=
Integer
.
parseInt
(
format
.
format
(
new
Date
(
count
)));
lists
.
forEach
(
faceEntities
->
threadPool
.
execute
(
new
PostUpdateJob
(
faceEntities
,
ossClientEntity
,
pathDate
,
facePath
,
featurePath
)));
// 等待线程池执行完成,执行下个循环
while
(
threadPool
.
getActiveCount
()
!=
0
)
{
}
log
.
info
(
"==============================={}结束========================================="
,
new
Date
(
count
));
System
.
gc
();
try
{
Thread
.
sleep
(
1000L
);
}
catch
(
InterruptedException
e
)
{
e
.
printStackTrace
();
}
}
//
SimpleDateFormat format = new SimpleDateFormat("yyyyMMdd");
//
//
// 按天循环
//
for (long count = beginDate.getTime(); count <= endDate.getTime(); count += DAY) {
//
log.info(count + "================================================");
//
//
// 查询一天的数据
//
List<FaceEntity> list = jdbcTemplate.query(FIND_ALL, new Object[]{new Date(count)}, (r, i) -> {
//
FaceEntity entity = new FaceEntity();
//
entity.setBodyPic(r.getString("body_pic"));
//
entity.setFacePic(r.getString("face_pic"));
//
entity.setChannelSerialNum(r.getString("channel_serialnum"));
//
return entity;
//
});
//
//
// 平均分成50份,每个线程执行其中的一份,共50个线程
//
List<List<FaceEntity>> lists = FileUtils.averageAssign(list, 50);
//
//
Integer pathDate = Integer.parseInt(format.format(new Date(count)));
//
lists.forEach(faceEntities -> threadPool.execute(new PostUpdateJob(faceEntities, ossClientEntity, pathDate, facePath, featurePath)));
//
//
// 等待线程池执行完成,执行下个循环
//
while (threadPool.getActiveCount() != 0) {
//
//
}
//
//
log.info("==============================={}结束=========================================", new Date(count));
//
//
System.gc();
//
//
try {
//
Thread.sleep(1000L);
//
} catch (InterruptedException e) {
//
e.printStackTrace();
//
}
//
}
}
}
src/main/java/com/vion/utils/OssFileUtil.java
View file @
f61c890
package
com
.
vion
.
utils
;
import
com.aliyun.oss.OSSClient
;
import
com.aliyun.oss.model.ListObjectsRequest
;
import
com.aliyun.oss.model.ObjectListing
;
/**
* @author 谢明辉
* @createDate 2018-11-6
...
...
@@ -21,16 +23,15 @@ public class OssFileUtil {
* @createDate 2018-11-6
* @description 列举指定目录下的所有文件
*/
public
ObjectListing
getFilesInPrefix
(
String
bucketName
,
String
prefix
,
OSSClient
ossClient
,
Integer
maxKeys
,
boolean
delimiter
)
{
public
ListObjectsRequest
getFilesInPrefix
(
String
bucketName
,
String
prefix
,
OSSClient
ossClient
,
Integer
maxKeys
,
boolean
delimiter
)
{
ListObjectsRequest
listObjectsRequest
=
new
ListObjectsRequest
(
bucketName
);
if
(
maxKeys
!=
null
)
{
maxKeys
=
maxKeys
>
1000
?
1000
:
(
maxKeys
<
1
?
1
:
maxKeys
);
listObjectsRequest
.
setMaxKeys
(
maxKeys
);
}
if
(
delimiter
)
{
listObjectsRequest
.
setDelimiter
(
"/"
);
}
listObjectsRequest
.
setPrefix
(
prefix
);
return
ossClient
.
listObjects
(
listObjectsRequest
)
;
return
listObjectsRequest
;
}
}
src/main/resources/application-pro.properties
View file @
f61c890
oss.config.bucket
=
vion-mall
oss.config.endPoint
=
oss-cn-beijing.aliyuncs.com
oss.config.accessKeyId
=
LTAI9WfUr3GDne4c
...
...
@@ -18,9 +19,13 @@ date.end=20180825
#spring.datasource.password=vion
#flag=postgre
########################Path format\uFF1Aface/date/channel_serialNum ,Database connection using mysql flag=mysql
spring.datasource.driver-class-
name
=
com.mysql.jdbc.Driver
spring.datasource.url
=
jdbc
\:
mysql
\:
//47.94.47.137
\:
3306/vioncountcloud?useUnicode
\=
true&characterEncoding
\=
UTF-8
spring.datasource.username
=
root
spring.datasource.password
=
IH5c3t0di14Ay7y5
#
spring.datasource.driver-class-name=com.mysql.jdbc.Driver
#
spring.datasource.url=jdbc\:mysql\://47.94.47.137\:3306/vioncountcloud?useUnicode\=true&characterEncoding\=UTF-8
#
spring.datasource.username=root
#
spring.datasource.password=IH5c3t0di14Ay7y5
flag
=
mysql
server.port
=
8888
prefix
=
picture/body/20190402/
serialnumStr
=
0269-F060-E72E-52FE-01,00FE-63D0-0345-CA16-01
\ No newline at end of file
Write
Preview
Markdown
is supported
Attach a file
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to post a comment