Skip to content

谢明辉 / vion-label

Go to a project
Commit 82dec15e by 李乾广
Options

添加tk.mybatis，添加ES，用户违规判断，保留访问日志到ES

1 parent 847cd936
Inline Side-by-side
Showing 26 changed files with 1336 additions and 18 deletions
label-platform/pom.xml
......@@ -29,13 +29,13 @@
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.mybatis.spring.boot</groupId>
<artifactId>mybatis-spring-boot-starter</artifactId>
<version>2.1.4</version>
<groupId>tk.mybatis</groupId>
<artifactId>mapper-spring-boot-starter</artifactId>
<version>2.1.5</version>
<exclusions>
<exclusion>
<artifactId>mybatis</artifactId>
<groupId>org.mybatis</groupId>
<artifactId>mybatis</artifactId>
</exclusion>
</exclusions>
</dependency>
......@@ -103,9 +103,22 @@
<scope>test</scope>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
</dependency>
<dependency>
<groupId>cn.hutool</groupId>
<artifactId>hutool-all</artifactId>
<version>5.3.8</version>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-data-elasticsearch</artifactId>
</dependency>
<dependency>
<groupId>com.viontech.keliu</groupId>
<artifactId>AlgApiClient</artifactId>
<version>6.0.6-SNAPSHOT</version>
<version>6.0.6</version>
<exclusions>
<exclusion>
<artifactId>tomcat-websocket</artifactId>
......@@ -144,6 +157,31 @@
</excludes>
</configuration>
</plugin>
<!-- mybatis代码生成插件 -->
<plugin>
<groupId>org.mybatis.generator</groupId>
<artifactId>mybatis-generator-maven-plugin</artifactId>
<version>1.3.5</version>
<configuration>
<!--配置文件的位置-->
<configurationFile>src/main/resources/generatorConfig.xml</configurationFile>
<verbose>true</verbose>
<overwrite>true</overwrite>
</configuration>
<dependencies>
<dependency>
<groupId>org.postgresql</groupId>
<artifactId>postgresql</artifactId>
<version>42.2.20</version>
</dependency>
<!-- mapper 依赖库 -->
<dependency>
<groupId>tk.mybatis</groupId>
<artifactId>mapper</artifactId>
<version>4.0.3</version>
</dependency>
</dependencies>
</plugin>
</plugins>
<resources>
<resource>
......
label-platform/src/main/java/com/viontech/label/platform/LabelPlatformApp.java
package com.viontech.label.platform;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.EnableAspectJAutoProxy;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.web.socket.config.annotation.EnableWebSocket;
import org.springframework.web.socket.config.annotation.EnableWebSocketMessageBroker;
import tk.mybatis.spring.annotation.MapperScan;
/**
* @author 谢明辉
......
label-platform/src/main/java/com/viontech/label/platform/config/elasticsearch/ESClient.java 0 → 100644
package com.viontech.label.platform.config.elasticsearch;
import com.alibaba.fastjson.JSON;
import org.apache.commons.lang3.ClassUtils;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.DocWriteResponse;
import org.elasticsearch.action.bulk.BulkRequest;
import org.elasticsearch.action.bulk.BulkResponse;
import org.elasticsearch.action.delete.DeleteRequest;
import org.elasticsearch.action.delete.DeleteResponse;
import org.elasticsearch.action.index.IndexRequest;
import org.elasticsearch.action.index.IndexResponse;
import org.elasticsearch.action.search.SearchRequest;
import org.elasticsearch.action.search.SearchResponse;
import org.elasticsearch.action.support.WriteRequest;
import org.elasticsearch.action.support.master.AcknowledgedResponse;
import org.elasticsearch.action.update.UpdateRequest;
import org.elasticsearch.action.update.UpdateResponse;
import org.elasticsearch.client.IndicesClient;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.MainResponse;
import org.elasticsearch.client.indices.CreateIndexRequest;
import org.elasticsearch.client.indices.GetIndexRequest;
import org.elasticsearch.client.indices.PutMappingRequest;
import org.elasticsearch.common.xcontent.XContentType;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.IdsQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.TermQueryBuilder;
import org.elasticsearch.search.SearchHit;
import org.elasticsearch.search.SearchHits;
import org.elasticsearch.search.builder.SearchSourceBuilder;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.Document;
import java.io.Closeable;
import java.io.IOException;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Date;
import java.util.List;
import java.util.Objects;
/**
* Author: inggg
* Date: 2019/8/12 9:42
*/
public class ESClient implements Closeable {
private RestHighLevelClient client;
private RequestOptions options;
public ESClient(RestHighLevelClient client, RequestOptions options) {
this.client = client;
this.options = options;
}
public final MainResponse info() throws IOException {
return client.info(options);
}
public final <T> List<T> select(T t) throws Exception {
Class<T> clazz = (Class<T>) t.getClass();
SearchRequest searchRequest = ESRequestFactory.getSearchRequest(clazz);
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
Field[] fields = clazz.getDeclaredFields();
for (Field field : fields) {
String name = field.getName();
Class<?> type = field.getType();
//获取对应属性的属性值
field.setAccessible(true);
Object value = field.get(t);
if (!Objects.isNull(value) && !type.isAssignableFrom(Date.class) && !type.isAssignableFrom(List.class)) {
TermQueryBuilder termQueryBuilder = QueryBuilders.termQuery(name, value);
boolQueryBuilder.must(termQueryBuilder);
}
}
searchSourceBuilder.query(boolQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse response = client.search(searchRequest, options);
SearchHits hits = response.getHits();
SearchHit[] hits1 = hits.getHits();
List<T> results = new ArrayList<>();
if (0 == hits1.length) {
return results;
}
for (SearchHit hit : hits1) {
String sourceAsString = hit.getSourceAsString();
T t1 = JSON.parseObject(sourceAsString, clazz);
results.add(t1);
}
return results;
}
public final <T> T findById(T t) throws Exception {
Class<T> clazz = (Class<T>) t.getClass();
SearchRequest searchRequest = ESRequestFactory.getSearchRequest(t.getClass());
Field[] fields = clazz.getDeclaredFields();
Object id = null;
for (Field field : fields) {
Id idAnnotation = field.getAnnotation(Id.class);
if (null != idAnnotation) {
field.setAccessible(true);
id = field.get(t);
if (Objects.isNull(id)) {
throw new RuntimeException("id不能为空");
}
break;
}
}
SearchSourceBuilder searchSourceBuilder = new SearchSourceBuilder();
IdsQueryBuilder idsQueryBuilder = QueryBuilders.idsQuery().addIds((String) id);
searchSourceBuilder.query(idsQueryBuilder);
searchRequest.source(searchSourceBuilder);
SearchResponse response = client.search(searchRequest, options);
SearchHits hits = response.getHits();
SearchHit[] hits1 = hits.getHits();
if (0 == hits1.length) {
return null;
}
SearchHit searchHit = hits1[0];
String sourceAsString = searchHit.getSourceAsString();
return JSON.parseObject(sourceAsString, clazz);
}
/**
* 存储数据，默认不只用自增主键，使用自定义主键
* 默认立即刷新
*
* @param t 数据
* @param <T>
* @return
* @throws Exception
*/
public final <T> T saveOrUpdate(T t) throws Exception {
return saveOrUpdate(t, false);
}
/**
* 存储数据
*
* @param t 数据
* @param delayRefresh 是否延迟刷新
* @param <T>
* @return
* @throws Exception
*/
public final <T> T saveOrUpdate(T t, boolean delayRefresh) throws Exception {
Class clazz = t.getClass();
Document annotation = (Document) clazz.getAnnotation(Document.class);
if (null == annotation) {
throw new RuntimeException("无法使用ES save方法");
}
String index = annotation.indexName();
if (StringUtils.isEmpty(index)) {
throw new RuntimeException("索引不能为空");
}
Field[] fields = clazz.getDeclaredFields();
int idCount = 0;
Field idField = null;
for (Field field : fields) {
Id idAnnotation = field.getAnnotation(Id.class);
if (null != idAnnotation) {
idCount++;
idField = field;
}
}
/*if (idCount == 0) {
throw new RuntimeException(clazz.getName() + "没有id字段");
}*/
if (idCount > 1) {
throw new RuntimeException(clazz.getName() + "有两个id");
}
String s = JSON.toJSONString(t);
IndexRequest indexRequest = new IndexRequest(index);
if (delayRefresh) {
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
} else {
indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
}
//是否有id，没有id使用默认
if (null != idField) {
idField.setAccessible(true);
Object id = idField.get(t);
if (null != id) {
indexRequest.id(id.toString());
}
}
indexRequest.source(s, XContentType.JSON);
IndexResponse indexResponse = this.index(indexRequest);
if (indexResponse.getResult() == DocWriteResponse.Result.CREATED
|| indexResponse.getResult() == DocWriteResponse.Result.UPDATED) {
return t;
}
return null;
}
public final <T> BulkResponse batchUpsert(List<T> tList) throws Exception {
return batchUpsert(tList, false);
}
public final <T> BulkResponse batchUpsert(List<T> tList, boolean delayRefresh) throws Exception {
BulkRequest bulkRequest = null;
int count = 0;
for (T t : tList) {
if (null == bulkRequest) {
bulkRequest = new BulkRequest();
if (delayRefresh) {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
} else {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
}
}
Class clazz = t.getClass();
Document annotation = (Document) clazz.getAnnotation(Document.class);
if (null == annotation) {
throw new RuntimeException("无法使用ES upsert方法");
}
String index = annotation.indexName();
if (StringUtils.isEmpty(index)) {
throw new RuntimeException("索引不能为空");
}
Field[] fields = clazz.getDeclaredFields();
Field idField = null;
for (Field field : fields) {
Id idAnnotation = field.getAnnotation(Id.class);
if (null != idAnnotation) {
idField = field;
break;
}
}
if (Objects.isNull(idField)) {
throw new NullPointerException("没有id字段");
}
idField.setAccessible(true);
Object id = idField.get(t);
if (Objects.isNull(id)) {
throw new IllegalArgumentException("id不能为空");
}
UpdateRequest request = new UpdateRequest(index, (String)id);
String json = JSON.toJSONString(t);
request.doc(json, XContentType.JSON);
request.docAsUpsert(true);
bulkRequest.add(request);
count++;
if (count > 10000) {
BulkResponse bulk = this.bulk(bulkRequest);
boolean hasFailures = bulk.hasFailures();
if (hasFailures) {
return bulk;
} else {
count = 0;
bulkRequest = null;
}
}
}
BulkResponse bulk = this.bulk(bulkRequest);
return bulk;
}
/**
* 有则更新，无则插入，默认不适用延迟刷新
* @param t
* @param <T>
* @return
* @throws Exception
*/
public final <T> String upsert(T t) throws Exception {
return upsert(t, false);
}
/**
* 有则更新，无则插入
* @param t 数据
* @param delayRefresh 是否延迟刷新
* @param <T>
* @return
*/
public final <T> String upsert(T t, boolean delayRefresh) throws Exception {
if (Objects.isNull(t)) {
throw new NullPointerException("参数不能为空");
}
Class clazz = t.getClass();
Document annotation = (Document) clazz.getAnnotation(Document.class);
if (null == annotation) {
throw new RuntimeException("无法使用ES upsert方法");
}
String index = annotation.indexName();
if (StringUtils.isEmpty(index)) {
throw new RuntimeException("索引不能为空");
}
Field[] fields = clazz.getDeclaredFields();
Field idField = null;
for (Field field : fields) {
Id idAnnotation = field.getAnnotation(Id.class);
if (null != idAnnotation) {
idField = field;
break;
}
}
if (Objects.isNull(idField)) {
throw new NullPointerException("没有id字段");
}
idField.setAccessible(true);
Object id = idField.get(t);
if (Objects.isNull(id)) {
throw new IllegalArgumentException("id不能为空");
}
UpdateRequest request = new UpdateRequest(index, (String)id);
String json = JSON.toJSONString(t);
request.doc(json, XContentType.JSON);
request.docAsUpsert(true);
if (delayRefresh) {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
} else {
request.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
}
UpdateResponse response = this.update(request);
return response.getResult().getLowercase();
}
public final UpdateResponse update(UpdateRequest request) throws IOException {
return client.update(request, options);
}
public final <T> BulkResponse batchSaveOrUpdate(List<T> tList) throws Exception {
return batchSaveOrUpdate(tList, false);
}
public final <T> BulkResponse batchSaveOrUpdate(List<T> tList, boolean delayRefresh) throws Exception {
BulkRequest bulkRequest = null;
int count = 0;
for (T t : tList) {
if (null == bulkRequest) {
bulkRequest = new BulkRequest();
if (delayRefresh) {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.NONE);
} else {
bulkRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.WAIT_UNTIL);
}
}
Class clazz = t.getClass();
Document annotation = (Document) clazz.getAnnotation(Document.class);
if (null == annotation) {
throw new RuntimeException("无妨使用ES save方法");
}
String index = annotation.indexName();
if (StringUtils.isEmpty(index)) {
throw new RuntimeException("索引不能为空");
}
Field[] fields = clazz.getDeclaredFields();
int idCount = 0;
Field idField = null;
for (Field field : fields) {
Id idAnnotation = field.getAnnotation(Id.class);
if (null != idAnnotation) {
idCount++;
idField = field;
}
}
/*if (idCount == 0) {
throw new RuntimeException(clazz.getName() + "没有id字段");
}*/
if (idCount > 1) {
throw new RuntimeException(clazz.getName() + "有两个id");
}
String s = JSON.toJSONString(t);
IndexRequest indexRequest = new IndexRequest(index);
//是否有id，没有id使用默认
if (null != idField) {
idField.setAccessible(true);
Object id = idField.get(t);
if (null != id) {
indexRequest.id(id.toString());
}
}
indexRequest.source(s, XContentType.JSON);
bulkRequest.add(indexRequest);
count++;
if (count > 10000) {
BulkResponse bulk = this.bulk(bulkRequest);
boolean hasFailures = bulk.hasFailures();
if (hasFailures) {
return bulk;
} else {
count = 0;
bulkRequest = null;
}
}
}
BulkResponse bulk = this.bulk(bulkRequest);
return bulk;
}
public final IndexResponse index(IndexRequest indexRequest) throws IOException {
return client.index(indexRequest, options);
}
public final BulkResponse bulk(BulkRequest bulkRequest) throws IOException {
return client.bulk(bulkRequest, options);
}
public final DeleteResponse delete(DeleteRequest deleteRequest) throws IOException {
return client.delete(deleteRequest, options);
}
public final SearchResponse search(SearchRequest searchRequest) throws IOException {
return client.search(searchRequest, options);
}
public final IndicesClient indices() {
return client.indices();
}
public final boolean exists(GetIndexRequest getIndexRequest) throws IOException {
return client.indices().exists(getIndexRequest, options);
}
public final AcknowledgedResponse create(CreateIndexRequest createIndexRequest) throws IOException {
return client.indices().create(createIndexRequest, options);
}
public final AcknowledgedResponse putMapping(PutMappingRequest putMappingRequest) throws IOException {
return client.indices().putMapping(putMappingRequest, options);
}
@Override
public void close() throws IOException {
client.close();
}
}
label-platform/src/main/java/com/viontech/label/platform/config/elasticsearch/ESConfig.java 0 → 100644
package com.viontech.label.platform.config.elasticsearch;
import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
/**
* Author: inggg
* Date: 2019/8/12 9:42
*/
@Configuration
public class ESConfig {
private static final Logger logger = LoggerFactory.getLogger(ESConfig.class);
//ES查询结果最大数量
public static final Integer MAX_SIZE = 200000;
@Bean
ESClient esClient(RestHighLevelClient restHighLevelClient) {
RequestOptions options = RequestOptions.DEFAULT;
ESClient client = new ESClient(restHighLevelClient, options);
return client;
}
}
label-platform/src/main/java/com/viontech/label/platform/config/elasticsearch/ESRequestFactory.java 0 → 100644
package com.viontech.label.platform.config.elasticsearch;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.action.search.SearchRequest;
import org.springframework.data.elasticsearch.annotations.Document;
/**
* Author: inggg
* Date: 2019/8/22 14:58
*/
public class ESRequestFactory {
public static SearchRequest getSearchRequest(Class clazz) {
Document annotation = (Document) clazz.getAnnotation(Document.class);
if (null == annotation) {
throw new IllegalArgumentException(clazz.getName() + "没有Document注解");
}
String index = annotation.indexName();
if (StringUtils.isEmpty(index)) {
throw new RuntimeException("索引不能为空");
}
SearchRequest searchRequest = new SearchRequest(annotation.indexName());
return searchRequest;
}
}
label-platform/src/main/java/com/viontech/label/platform/config/elasticsearch/util/AndQueryBuilder.java 0 → 100644
package com.viontech.label.platform.config.elasticsearch.util;
import org.apache.commons.lang3.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.index.query.RangeQueryBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.util.CollectionUtils;
import java.util.Collection;
/**
* Author: inggg
* Date: 2020/5/25 9:34
*/
public class AndQueryBuilder {
private final static Logger logger = LoggerFactory.getLogger(AndQueryBuilder.class);
private BoolQueryBuilder boolQueryBuilder;
public AndQueryBuilder() {
boolQueryBuilder = QueryBuilders.boolQuery();
}
/**
* where and field = value
* @param field
* @param value
* @return
*/
public AndQueryBuilder eq(String field, String value) {
if (StringUtils.isEmpty(value)) {
return this;
}
boolQueryBuilder.filter(QueryBuilders.termQuery(field, value));
return this;
}
/**
* where and field = value
* @param field
* @param value
* @return
*/
public AndQueryBuilder eq(String field, Integer value) {
if (null == value) {
return this;
}
boolQueryBuilder.filter(QueryBuilders.termQuery(field, value));
return this;
}
/**
* where and field in values
* @param field
* @param values
* @return
*/
public AndQueryBuilder in(String field, String... values) {
if (null == values || values.length == 0) {
return this;
}
boolQueryBuilder.filter(QueryBuilders.termsQuery(field, values));
return this;
}
/**
* where and field in values
* @param field
* @param values
* @return
*/
public AndQueryBuilder in(String field, Integer... values) {
if (null == values || values.length == 0) {
return this;
}
boolQueryBuilder.filter(QueryBuilders.termsQuery(field, values));
return this;
}
public AndQueryBuilder in(String field, Collection<?> values) {
if (CollectionUtils.isEmpty(values)) {
return this;
}
boolQueryBuilder.filter(QueryBuilders.termsQuery(field, values));
return this;
}
/**
* where field like *value*
* @param field
* @param value
* @return
*/
public AndQueryBuilder like(String field, String value) {
if (StringUtils.isEmpty(value)) {
return this;
}
boolQueryBuilder.filter(QueryBuilders.wildcardQuery(field, "*" + value + "*"));
return this;
}
/**
* where filed > from and field < to
* @param field
* @param from
* @param to
* @param includeLower
* @param includeUpper
* @return
*/
public AndQueryBuilder between(String field, Object from, boolean includeLower, Object to, boolean includeUpper) {
RangeQueryBuilder rangeQueryBuilder = QueryBuilders.rangeQuery(field);
if (null == from && null == to) {
return this;
}
if (null != from) {
rangeQueryBuilder.from(from, includeLower);
}
if (null != to) {
rangeQueryBuilder.to(to, includeUpper);
}
boolQueryBuilder.filter(rangeQueryBuilder);
return this;
}
/**
* where field is not null and field.length != ""
* @param field
* @param includeNullString 是否包含空字符串，""，如果为true，值为""也是存在，如果为false，value长度必须大于0
* @return
*/
public AndQueryBuilder exists(String field, boolean includeNullString) {
boolQueryBuilder.filter(QueryBuilders.existsQuery(field));
if (!includeNullString) {
BoolQueryBuilder subBoolQueryBuilder = QueryBuilders.boolQuery();
subBoolQueryBuilder.mustNot(QueryBuilders.termQuery(field, ""));
boolQueryBuilder.filter(subBoolQueryBuilder);
}
return this;
}
/**
* where (field is null or field = '')
* @param field
* @param includeNullString 空字符串''是否也算不存在，如果为true，''也表示不存在，如果为false，''为存在
* @return
*/
public AndQueryBuilder notExists(String field, boolean includeNullString) {
if (includeNullString) {
BoolQueryBuilder subBoolQueryBuilder = QueryBuilders.boolQuery();
subBoolQueryBuilder.should(QueryBuilders.termQuery(field, ""));
subBoolQueryBuilder.should(QueryBuilders.boolQuery().mustNot(QueryBuilders.existsQuery(field)));
boolQueryBuilder.filter(subBoolQueryBuilder);
} else {
BoolQueryBuilder subBoolQueryBuilder = QueryBuilders.boolQuery();
subBoolQueryBuilder.mustNot(QueryBuilders.existsQuery(field));
boolQueryBuilder.filter(subBoolQueryBuilder);
}
return this;
}
public BoolQueryBuilder build() {
return boolQueryBuilder;
}
}
label-platform/src/main/java/com/viontech/label/platform/controller/web/ForbiddenController.java 0 → 100644
package com.viontech.label.platform.controller.web;
import com.viontech.label.platform.mapper.SForbiddenUserMapper;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* Author: inggg
* Date: 2021/10/25 09:56
*/
@RestController
public class ForbiddenController {
private final static Logger logger = LoggerFactory.getLogger(ForbiddenController.class);
@Autowired
private SForbiddenUserMapper sForbiddenUserMapper;
@RequestMapping("/test")
public Object test() {
return sForbiddenUserMapper.selectAll();
}
}
label-platform/src/main/java/com/viontech/label/platform/interceptor/AuthInterceptor.java
......@@ -2,15 +2,30 @@ package com.viontech.label.platform.interceptor;
import cn.dev33.satoken.exception.NotLoginException;
import cn.dev33.satoken.stp.StpUtil;
import cn.hutool.core.collection.CollectionUtil;
import cn.hutool.extra.servlet.ServletUtil;
import com.viontech.label.platform.mapper.SForbiddenUserMapper;
import com.viontech.label.platform.model.SForbiddenUser;
import com.viontech.label.platform.model.User;
import com.viontech.label.platform.model.log.AccessLog;
import com.viontech.label.platform.schedule.IllegalProperties;
import com.viontech.label.platform.service.adapter.AccessLogService;
import com.viontech.label.platform.service.adapter.UserService;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.StringUtils;
import org.redisson.api.RBoundedBlockingQueue;
import org.redisson.api.RLock;
import org.redisson.api.RedissonClient;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import org.springframework.web.servlet.handler.HandlerInterceptorAdapter;
import javax.annotation.Resource;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
import java.util.Date;
import java.util.List;
import java.util.concurrent.TimeUnit;
/**
* .
......@@ -23,8 +38,16 @@ import javax.servlet.http.HttpServletResponse;
@Slf4j
public class AuthInterceptor extends HandlerInterceptorAdapter {
@Autowired
private RedissonClient redissonClient;
@Resource
private UserService userService;
@Autowired
private AccessLogService accessLogService;
@Autowired
private IllegalProperties illegalProperties;
@Autowired
private SForbiddenUserMapper sForbiddenUserMapper;
@Override
public boolean preHandle(HttpServletRequest request, HttpServletResponse response, Object handler) {
......@@ -35,9 +58,74 @@ public class AuthInterceptor extends HandlerInterceptorAdapter {
StpUtil.checkLogin();
long loginId = StpUtil.getLoginIdAsLong();
User user = userService.selectByPrimaryKey(loginId);
request.setAttribute("user", user);
request.setAttribute("userId", user.getId());
request.setAttribute("accountId", user.getAccountId());
String servletPath = request.getServletPath();
if (0 == user.getType()) {
//如果用户是管理员，则不限制
return true;
}
if (StringUtils.equalsAny(servletPath, "/accounts")) {
return true;
}
//获取违规规则
String limitKey = servletPath.replaceAll("/", "") + ".limit";
String durationKey = servletPath.replaceAll("/", "") + ".duration";
Integer limit = illegalProperties.getMap().get(limitKey);
Integer duration = illegalProperties.getMap().get(durationKey);
//没有配置违规规则直接通过
if (null == limit || null == duration) {
return true;
}
//使用分布式锁
String key = user.getUnid() + servletPath;
RLock lock = redissonClient.getLock("label:lock:" + key);
try {
if (lock.tryLock(0, 10, TimeUnit.SECONDS)) {
//查看用户是否已经被锁定
SForbiddenUser param = new SForbiddenUser();
param.setUserId(user.getUnid());
param.setDeleted(new Short("0"));
List<SForbiddenUser> select = sForbiddenUserMapper.select(param);
if (CollectionUtil.isNotEmpty(select)) {
log.error("用户{}已被锁定", user.getUnid());
throw NotLoginException.newInstance("", null);
}
RBoundedBlockingQueue<Object> rateQueue = redissonClient.getBoundedBlockingQueue("label:queue:" + key);
rateQueue.trySetCapacity(limit);
rateQueue.expire(10, TimeUnit.MINUTES);
long now = System.currentTimeMillis();
if (rateQueue.size() < limit) {
rateQueue.offer(now);
} else {
long index0Time = (long)rateQueue.poll();
if (now - index0Time < duration) {
SForbiddenUser forbiddenUser = new SForbiddenUser();
forbiddenUser.setUserId(user.getUnid());
forbiddenUser.setName(user.getName());
forbiddenUser.setStatus(new Short("0"));
forbiddenUser.setDeleted(new Short("0"));
forbiddenUser.setUrl(servletPath);
forbiddenUser.setRemark(now-index0Time + "内调用" + limit + "次");
forbiddenUser.setCreateTime(new Date());
sForbiddenUserMapper.insertUseGeneratedKeys(forbiddenUser);
log.error("调用太频繁:用户{}，{}接口{}内调用{}次", user.getUnid(), servletPath, now - index0Time, limit);
return false;
} else {
rateQueue.offer(now);
}
}
} else {
throw new RuntimeException("请求接口太快");
}
} catch (Exception e) {
throw new RuntimeException("你违规请求，我们已经记住你，请注意！");
} finally {
lock.unlock();
}
return true;
}
......@@ -51,6 +139,26 @@ public class AuthInterceptor extends HandlerInterceptorAdapter {
String uri = request.getRequestURL().toString();
String path = queryString == null ? uri : uri + "?" + queryString;
User user = (User) request.getAttribute("user");
//记录访问日志
String servletPath = request.getServletPath();
//String header = request.getHeader("User-Agent");
String method = request.getMethod();
String clientIP = ServletUtil.getClientIP(request);
if (!"/accounts".equalsIgnoreCase(servletPath)) {
AccessLog accessLog = AccessLog.builder()
.userId(user.getUnid())
.name(user.getName())
.clientIP(clientIP)
.method(method)
.servletPath(servletPath)
.accessTime(new Date())
//header user agent暂时不存
//.header(header)
.build();
accessLogService.saveAccessLogs(accessLog);
}
log.info("\n用户名:[{}]\n请求信息:[{}]\n请求方式:[{}]\n处理时长:[{}]\n", user.getUsername(), path, request.getMethod(), System.currentTimeMillis() - requestStart);
}
super.afterCompletion(request, response, handler, ex);
......
label-platform/src/main/java/com/viontech/label/platform/mapper/SForbiddenUserMapper.java 0 → 100644
package com.viontech.label.platform.mapper;
import com.viontech.label.platform.model.SForbiddenUser;
import com.viontech.label.platform.util.MyMapper;
public interface SForbiddenUserMapper extends MyMapper<SForbiddenUser> {
}
\ No newline at end of file
label-platform/src/main/java/com/viontech/label/platform/mapping/SForbiddenUserMapper.xml 0 → 100644
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
<mapper namespace="com.viontech.label.platform.mapper.SForbiddenUserMapper">
<resultMap id="BaseResultMap" type="com.viontech.label.platform.model.SForbiddenUser">
<!--
WARNING - @mbg.generated
-->
<result column="id" jdbcType="BIGINT" property="id" />
<result column="user_id" jdbcType="VARCHAR" property="userId" />
<result column="status" jdbcType="SMALLINT" property="status" />
<result column="deleted" jdbcType="SMALLINT" property="deleted" />
<result column="name" jdbcType="VARCHAR" property="name" />
<result column="remark" jdbcType="VARCHAR" property="remark" />
<result column="url" jdbcType="VARCHAR" property="url" />
<result column="create_time" jdbcType="TIMESTAMP" property="createTime" />
<result column="update_time" jdbcType="TIMESTAMP" property="updateTime" />
</resultMap>
</mapper>
\ No newline at end of file
label-platform/src/main/java/com/viontech/label/platform/model/SForbiddenUser.java 0 → 100644
package com.viontech.label.platform.model;
import lombok.Data;
import javax.persistence.Column;
import javax.persistence.Id;
import javax.persistence.Table;
import java.util.Date;
@Data
@Table(name = "s_forbidden_user")
public class SForbiddenUser {
/**
* id
*/
@Id
@Column(name = "id")
private Long id;
/**
* 用户id
*/
@Column(name = "user_id")
private String userId;
/**
* 状态，0：禁止登陆，查询，等所有操作
*/
@Column(name = "status")
private Short status;
/**
* 是否删除：0：未删除，1：已删除
*/
@Column(name = "deleted")
private Short deleted;
/**
* 用户名字
*/
@Column(name = "name")
private String name;
/**
* 备注
*/
@Column(name = "remark")
private String remark;
/**
* 请求接口
*/
@Column(name = "url")
private String url;
/**
* 创建时间
*/
@Column(name = "create_time")
private Date createTime;
/**
* 修改时间
*/
@Column(name = "update_time")
private Date updateTime;
}
\ No newline at end of file
label-platform/src/main/java/com/viontech/label/platform/model/log/AccessLog.java 0 → 100644
package com.viontech.label.platform.model.log;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.DateFormat;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import java.util.Date;
/**
* Author: inggg
* Date: 2021/10/22 10:59
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Document(indexName = "index_reid_label_access_log", createIndex = true, shards = 1, replicas = 0)
public class AccessLog {
/**
* id
*/
@Id
@Field(type = FieldType.Keyword)
private String id;
/**
* 用户id
*/
@Field(type = FieldType.Keyword)
private String userId;
/**
* 用户名字
*/
@Field(type = FieldType.Keyword)
private String name;
/**
* 客户端IP
*/
@Field(type = FieldType.Keyword)
private String clientIP;
/**
* 请求方法， get post put delete
*/
@Field(type = FieldType.Keyword)
private String method;
/**
* 访问地址
*/
@Field(type = FieldType.Keyword)
private String servletPath;
/**
* 访问时间
*/
@Field(type = FieldType.Date_Nanos, format = DateFormat.basic_date_time)
private Date accessTime;
/**
* 请求头
*/
@Field(type = FieldType.Text)
private String header;
}
label-platform/src/main/java/com/viontech/label/platform/model/log/UserIllegalRecord.java 0 → 100644
package com.viontech.label.platform.model.log;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import org.springframework.data.annotation.Id;
import org.springframework.data.elasticsearch.annotations.DateFormat;
import org.springframework.data.elasticsearch.annotations.Document;
import org.springframework.data.elasticsearch.annotations.Field;
import org.springframework.data.elasticsearch.annotations.FieldType;
import java.util.Date;
/**
* Author: inggg
* Date: 2021/10/22 11:56
*/
@Data
@Builder
@NoArgsConstructor
@AllArgsConstructor
@Document(indexName = "index_reid_label_user_illegal_record", createIndex = true, shards = 1, replicas = 0)
public class UserIllegalRecord {
/**
* id
*/
@Id
@Field(type = FieldType.Keyword)
private String id;
/**
* 用户id
*/
@Field(type = FieldType.Keyword)
private String userId;
/**
* 用户名字
*/
@Field(type = FieldType.Keyword)
private String name;
/**
* 查询次数
*/
@Field(type = FieldType.Long)
private Long searchTimes;
/**
* 提交次数
*/
@Field(type = FieldType.Long)
private Long submitTimes;
/**
* 开始时间
*/
@Field(type = FieldType.Date_Nanos, format = DateFormat.basic_date_time)
private Date startTime;
/**
* 结束时间
*/
@Field(type = FieldType.Date_Nanos, format = DateFormat.basic_date_time)
private Date endTime;
/**
* 创建时间
*/
@Field(type = FieldType.Date_Nanos, format = DateFormat.basic_date_time)
private Date createTime;
}
label-platform/src/main/java/com/viontech/label/platform/repository/AccessLogRepository.java 0 → 100644
package com.viontech.label.platform.repository;
import com.viontech.label.platform.model.log.AccessLog;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
/**
* Author: inggg
* Date: 2021/10/25 16:50
*/
public interface AccessLogRepository extends ElasticsearchRepository<AccessLog, String> {
}
label-platform/src/main/java/com/viontech/label/platform/repository/UserIllegalRecordRepository.java 0 → 100644
package com.viontech.label.platform.repository;
import com.viontech.label.platform.model.log.UserIllegalRecord;
import org.springframework.data.elasticsearch.repository.ElasticsearchRepository;
/**
* Author: inggg
* Date: 2021/10/25 16:51
*/
public interface UserIllegalRecordRepository extends ElasticsearchRepository<UserIllegalRecord, String> {
}
label-platform/src/main/java/com/viontech/label/platform/schedule/AccessLogsAggsTask.java 0 → 100644
package com.viontech.label.platform.schedule;
import com.viontech.label.platform.config.elasticsearch.ESClient;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.scheduling.annotation.Scheduled;
/**
* 访问日志聚合统计任务
* Author: inggg
* Date: 2021/10/22 15:55
*/
//@Component
public class AccessLogsAggsTask {
private final static Logger logger = LoggerFactory.getLogger(AccessLogsAggsTask.class);
@Autowired
private ESClient esClient;
@Autowired
private IllegalProperties illegalProperties;
@Scheduled(cron = "${access.log.aggs.cron}")
public void run() throws Exception {
/*SearchRequest searchRequest = ESRequestFactory.getSearchRequest(AccessLog.class);
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.filter(QueryBuilders.matchAllQuery());
SearchSourceBuilder builder = new SearchSourceBuilder().query(boolQueryBuilder);
builder.aggregation(AggregationBuilders
.terms("userId")
.field("userId")
.size(100000)
.minDocCount(1)
.subAggregation(AggregationBuilders
.terms("servletPath")
.field("servletPath")
.size(100)
.minDocCount(1)));
builder.size(0);
searchRequest.source(builder);
try {
SearchResponse response = esClient.search(searchRequest);
ParsedStringTerms userIdTerms = response.getAggregations().get("userId");
List<? extends Terms.Bucket> buckets = userIdTerms.getBuckets();
for (Terms.Bucket bucket : buckets) {
String userId = bucket.getKeyAsString();
ParsedStringTerms servletPathTerms = bucket.getAggregations().get("servletPath");
long searchTimes = 0;
long submitTimes = 0;
for (String url : illegalProperties.getSearchUrls()) {
Terms.Bucket bucketByKey = servletPathTerms.getBucketByKey(url);
searchTimes += bucketByKey == null ? 0 : bucketByKey.getDocCount();
}
for (String url : illegalProperties.getSubmitUrls()) {
Terms.Bucket bucketByKey = servletPathTerms.getBucketByKey(url);
submitTimes += bucketByKey == null ? 0 : bucketByKey.getDocCount();
}
logger.info("{},{},{}", userId, searchTimes, submitTimes);
}
} catch (IOException e) {
e.printStackTrace();
}*/
}
}
label-platform/src/main/java/com/viontech/label/platform/schedule/IllegalProperties.java 0 → 100644
package com.viontech.label.platform.schedule;
import lombok.Getter;
import lombok.Setter;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.stereotype.Component;
import java.util.Map;
/**
* Author: inggg
* Date: 2021/10/22 16:31
*/
@Getter
@Setter
@Component
@ConfigurationProperties("illegal")
public class IllegalProperties {
/**
* 请求接口集合
*/
private Map<String, Integer> map;
}
label-platform/src/main/java/com/viontech/label/platform/service/adapter/AccessLogService.java 0 → 100644
package com.viontech.label.platform.service.adapter;
import com.viontech.label.platform.model.log.AccessLog;
/**
* Author: inggg
* Date: 2021/10/22 11:12
*/
public interface AccessLogService {
/**
* 保存访问日志
* @param accessLog
*/
void saveAccessLogs(AccessLog accessLog);
}
label-platform/src/main/java/com/viontech/label/platform/service/impl/AccessLogServiceImpl.java 0 → 100644
package com.viontech.label.platform.service.impl;
import com.viontech.label.platform.config.elasticsearch.ESClient;
import com.viontech.label.platform.model.log.AccessLog;
import com.viontech.label.platform.service.adapter.AccessLogService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
/**
* Author: inggg
* Date: 2021/10/22 11:14
*/
@Service
public class AccessLogServiceImpl implements AccessLogService {
private final static Logger logger = LoggerFactory.getLogger(AccessLogServiceImpl.class);
@Autowired
private ESClient esClient;
@Override
public void saveAccessLogs(AccessLog accessLog) {
try {
esClient.saveOrUpdate(accessLog, true);
} catch (Exception e) {
logger.error("保存访问日志失败:{}", accessLog);
}
}
}
label-platform/src/main/java/com/viontech/label/platform/util/MyMapper.java 0 → 100644
package com.viontech.label.platform.util;
import tk.mybatis.mapper.common.*;
/**
* Author: inggg
* Date: 2021/07/15 11:10
*/
public interface MyMapper<T> extends BaseMapper<T>, MySqlMapper<T>
, ExampleMapper<T>, IdsMapper<T>
, ConditionMapper<T> {
}
label-platform/src/main/resources/application-dev.properties
server.port=12100
# database
spring.datasource.url=jdbc:postgresql://127.0.0.1:5432/vion_label
spring.datasource.url=jdbc:postgresql://192.168.1.106:5432/vion_label
spring.datasource.username=postgres
spring.datasource.password=vion
# redis
spring.redis.host=127.0.0.1
spring.redis.host=192.168.1.106
spring.redis.port=6379
spring.redis.password=vionredis
# temporary
logging.level.com.viontech.label.platform.mapper=error
logging.level.com.viontech.label.platform.mapper=debug
debug=false
# vion
vion.match-url=http://127.0.0.1:12000/alg
\ No newline at end of file
vion.match-url=http://127.0.0.1:12000/alg
spring.elasticsearch.rest.uris=192.168.1.106:9200
spring.elasticsearch.rest.username=elastic
spring.elasticsearch.rest.password=viontech
access.log.aggs.cron=0/5 * * * * ?
illegal.map./reid/getPeople.duration=3000
illegal.map./reid/getPeople.limit=6
illegal.map./subTasks.duration=3000
illegal.map./subTasks.limit=4
illegal.map./reid/getSimilarPerson.duration=3000
illegal.map./reid/getSimilarPerson.limit=4
illegal.map./packs.duration=10000
illegal.map./packs.limit=2
illegal.map./packs/info.duration=10000
illegal.map./packs/info.limit=2
label-platform/src/main/resources/application-pro.properties
......@@ -17,4 +17,19 @@ debug=false
# vion
#vion.match-url=http://36.112.68.214:12000/alg
vion.match-url=http://101.201.36.180:12001/alg
#vion.match-url=http://127.0.0.1:12000/alg
\ No newline at end of file
#vion.match-url=http://127.0.0.1:12000/alg
spring.elasticsearch.rest.uris=101.201.36.180:9200
spring.elasticsearch.rest.username=
spring.elasticsearch.rest.password=
access.log.aggs.cron=0/5 * * * * ?
illegal.map./reid/getPeople.duration=3000
illegal.map./reid/getPeople.limit=6
illegal.map./subTasks.duration=3000
illegal.map./subTasks.limit=4
illegal.map./reid/getSimilarPerson.duration=3000
illegal.map./reid/getSimilarPerson.limit=4
\ No newline at end of file
label-platform/src/main/resources/generatorConfig.xml 0 → 100644
<!DOCTYPE generatorConfiguration
PUBLIC "-//mybatis.org//DTD MyBatis Generator Configuration 1.0//EN"
"http://mybatis.org/dtd/mybatis-generator-config_1_0.dtd">
<generatorConfiguration>
<context id="Mysql" targetRuntime="MyBatis3Simple" defaultModelType="flat">
<property name="javaFileEncoding" value="UTF-8"/>
<property name="useMapperCommentGenerator" value="true"/>
<plugin type="tk.mybatis.mapper.generator.MapperPlugin">
<!-- 配置后生成的 Mapper 接口都会自动继承接口 value: 需要继承的接口, 该接口不可以在 MapperScan 扫描范围中-->
<property name="mappers" value="com.viontech.label.platform.util.MyMapper"/>
<!-- caseSensitive默认false，当数据库表名区分大小写时，可以将该属性设置为true -->
<property name="caseSensitive" value="true"/>
<property name="forceAnnotation" value="true"/>
</plugin>
<!-- 数据库连接属性: -->
<jdbcConnection
driverClass="org.postgresql.Driver"
connectionURL="jdbc:postgresql://192.168.1.106:5432/vion_label"
userId="postgres"
password="vion"/>
<!-- MyBatis 生成器生成 Model -->
<javaModelGenerator targetPackage="com.viontech.label.platform.model"
targetProject="src/main/java"/>
<!-- MyBatis 生成器生成 Mapper XML -->
<sqlMapGenerator targetPackage="com.viontech.label.platform.mapping"
targetProject="src/main/java"/>
<!-- MyBatis 生成器生成 Mapper class -->
<javaClientGenerator targetPackage="com.viontech.label.platform.mapper"
targetProject="src/main/java"
type="XMLMAPPER"/>
<table tableName="s_forbidden_user"></table>
</context>
</generatorConfiguration>
\ No newline at end of file
label-platform/src/main/resources/logback-spring.xml
......@@ -15,7 +15,7 @@
<appender name="CONSOLE" class="ch.qos.logback.core.ConsoleAppender">
<!--此日志appender是为开发使用，只配置最底级别，控制台输出的日志级别是大于或等于此级别的日志信息-->
<filter class="ch.qos.logback.classic.filter.ThresholdFilter">
<level>info</level>
<level>debug</level>
</filter>
<encoder>
<Pattern>${pattern}</Pattern>
......
label-platform/src/test/java/com/viontech/label/platform/LabelApplicationTests.java
......@@ -15,9 +15,7 @@ import com.viontech.label.platform.utils.StorageUtils;
import com.viontech.label.platform.vo.PicVo;
import org.apache.commons.io.FileUtils;
import org.junit.jupiter.api.Test;
import org.junit.runner.RunWith;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.junit4.SpringRunner;
import javax.annotation.Resource;
import java.io.File;
......@@ -27,7 +25,6 @@ import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
@SpringBootTest
@RunWith(SpringRunner.class)
class LabelApplicationTests {
@Resource
......
label-tool-keliu/pom.xml
......@@ -44,7 +44,7 @@
<dependency>
<groupId>com.viontech.keliu</groupId>
<artifactId>AlgApiClient</artifactId>
<version>6.0.6-SNAPSHOT</version>
<version>6.0.6</version>
<exclusions>
<exclusion>
<artifactId>tomcat-websocket</artifactId>
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!