Skip to content

谢明辉 / VVAS-Match

Go to a project
Commit 113515d4 by 姚冰
Options

[chg] 对接es8.15，使用elasticsearch-java

1 parent a19efee4
Expand all
Inline Side-by-side
Showing 7 changed files with 231 additions and 47 deletions
pom.xml
...@@ -22,6 +22,7 @@ ...@@ -22,6 +22,7 @@
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter</artifactId> <artifactId>spring-boot-starter</artifactId>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
...@@ -32,18 +33,45 @@ ...@@ -32,18 +33,45 @@
<artifactId>android-json</artifactId> <artifactId>android-json</artifactId>
<groupId>com.vaadin.external.google</groupId> <groupId>com.vaadin.external.google</groupId>
</exclusion> </exclusion>
<exclusion>
<groupId>org.elasticsearch.client</groupId>
<artifactId>elasticsearch-rest-client</artifactId>
</exclusion>
</exclusions> </exclusions>
</dependency> </dependency>
<dependency> <dependency>
<groupId>org.springframework.boot</groupId> <groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId> <artifactId>spring-boot-starter-web</artifactId>
<exclusions>
<exclusion>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
</exclusion>
</exclusions>
</dependency> </dependency>
<dependency> <!-- <dependency>-->
<groupId>org.springframework.boot</groupId> <!-- <groupId>org.elasticsearch.client</groupId>-->
<artifactId>spring-boot-starter-data-elasticsearch</artifactId> <!-- <artifactId>elasticsearch-rest-high-level-client</artifactId>-->
</dependency> <!-- <version>8.15.0</version>-->
<!-- </dependency>-->
<!-- &lt;!&ndash; Elasticsearch Rest Hight Level Client 的依赖 &ndash;&gt;-->
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-starter-data-elasticsearch</artifactId>-->
<!-- </dependency>-->
<!-- <dependency>-->
<!-- <groupId>org.springframework.boot</groupId>-->
<!-- <artifactId>spring-boot-starter-data-elasticsearch</artifactId>-->
<!-- <version></version>-->
<!-- <exclusions>-->
<!-- <exclusion>-->
<!-- <groupId>org.elasticsearch.client</groupId>-->
<!-- <artifactId>elasticsearch-rest-client</artifactId>-->
<!-- </exclusion>-->
<!-- </exclusions>-->
<!-- </dependency>-->
<dependency> <dependency>
<groupId>org.projectlombok</groupId> <groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId> <artifactId>lombok</artifactId>
...@@ -60,6 +88,43 @@ ...@@ -60,6 +88,43 @@
<version>4.4</version> <version>4.4</version>
</dependency> </dependency>
<!-- https://mvnrepository.com/artifact/co.elastic.clients/elasticsearch-java -->
<dependency>
<groupId>co.elastic.clients</groupId>
<artifactId>elasticsearch-java</artifactId>
<version>8.15.0</version>
<exclusions>
<exclusion>
<groupId>jakarta.json</groupId>
<artifactId>jakarta.json-api</artifactId>
</exclusion>
<exclusion>
<artifactId>elasticsearch-rest-client</artifactId>
<groupId>org.elasticsearch.client</groupId>
</exclusion>
</exclusions>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.17.0</version>
</dependency>
<dependency>
<groupId>com.fasterxml.jackson.core</groupId>
<artifactId>jackson-databind</artifactId>
<version>2.12.3</version>
</dependency>
<dependency>
<artifactId>elasticsearch-rest-client</artifactId>
<groupId>org.elasticsearch.client</groupId>
<version>8.15.0</version>
</dependency>
<dependency>
<groupId>jakarta.json</groupId>
<artifactId>jakarta.json-api</artifactId>
<version>2.0.1</version>
</dependency>
<dependency> <dependency>
<groupId>com.viontech.keliu</groupId> <groupId>com.viontech.keliu</groupId>
<artifactId>AlgApiClient</artifactId> <artifactId>AlgApiClient</artifactId>
......
src/main/java/com/viontech/match/config/ElasticsearchConfiguration.java
package com.viontech.match.config; package com.viontech.match.config;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.json.jackson.JacksonJsonpMapper;
import co.elastic.clients.transport.ElasticsearchTransport;
import co.elastic.clients.transport.rest_client.RestClientTransport;
import org.apache.http.HttpHost; import org.apache.http.HttpHost;
import org.apache.http.auth.AuthScope; import org.apache.http.auth.AuthScope;
import org.apache.http.auth.UsernamePasswordCredentials; import org.apache.http.auth.UsernamePasswordCredentials;
import org.apache.http.client.CredentialsProvider; import org.apache.http.client.CredentialsProvider;
import org.apache.http.conn.ssl.NoopHostnameVerifier;
import org.apache.http.conn.ssl.TrustStrategy;
import org.apache.http.impl.client.BasicCredentialsProvider; import org.apache.http.impl.client.BasicCredentialsProvider;
import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager;
import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor;
import org.apache.http.impl.nio.reactor.IOReactorConfig; import org.apache.http.impl.nio.reactor.IOReactorConfig;
import org.apache.http.nio.reactor.IOReactorException; import org.apache.http.nio.reactor.IOReactorException;
import org.apache.http.ssl.SSLContextBuilder;
import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClient;
import org.elasticsearch.client.RestClientBuilder;
import org.elasticsearch.client.RestHighLevelClient;
import org.springframework.boot.context.properties.ConfigurationProperties; import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Bean;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils; import org.springframework.util.StringUtils;
import javax.net.ssl.SSLContext;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import java.security.KeyManagementException;
import java.security.KeyStoreException;
import java.security.NoSuchAlgorithmException;
import java.security.cert.CertificateException;
import java.security.cert.X509Certificate;
@Component @Component
public class ElasticsearchConfiguration { public class ElasticsearchConfiguration {
...@@ -27,8 +41,7 @@ public class ElasticsearchConfiguration { ...@@ -27,8 +41,7 @@ public class ElasticsearchConfiguration {
} }
@Bean @Bean
public RestHighLevelClient createClient(CustomElasticsearchProperties properties) { public ElasticsearchClient createElasticsearchClient(CustomElasticsearchProperties properties) {
HttpHost[] esHosts = properties.getUris().stream().filter(StringUtils::hasLength).map(HttpHost::create).toArray(HttpHost[]::new); HttpHost[] esHosts = properties.getUris().stream().filter(StringUtils::hasLength).map(HttpHost::create).toArray(HttpHost[]::new);
final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); final CredentialsProvider credentialsProvider = new BasicCredentialsProvider();
if (properties.getUsername() != null) { if (properties.getUsername() != null) {
...@@ -48,17 +61,66 @@ public class ElasticsearchConfiguration { ...@@ -48,17 +61,66 @@ public class ElasticsearchConfiguration {
} }
PoolingNHttpClientConnectionManager finalConnectionManager = connectionManager; PoolingNHttpClientConnectionManager finalConnectionManager = connectionManager;
RestClientBuilder builder = RestClient.builder(esHosts).setRequestConfigCallback(requestConfigBuilder -> { RestClient restClient = RestClient.builder(esHosts)
requestConfigBuilder.setConnectTimeout(properties.getConnectionTimeout()); .setRequestConfigCallback(requestConfigBuilder -> {
requestConfigBuilder.setSocketTimeout(properties.getSocketTimeout()); requestConfigBuilder.setConnectTimeout(properties.getConnectionTimeout());
requestConfigBuilder.setConnectionRequestTimeout(-1); requestConfigBuilder.setSocketTimeout(properties.getSocketTimeout());
return requestConfigBuilder; requestConfigBuilder.setConnectionRequestTimeout(-1);
}).setHttpClientConfigCallback(httpClientBuilder -> { return requestConfigBuilder;
httpClientBuilder.disableAuthCaching(); })
httpClientBuilder.setMaxConnTotal(properties.getMaxConnTotal()); .setHttpClientConfigCallback(httpClientBuilder -> {
httpClientBuilder.setMaxConnPerRoute(properties.getMaxConnPerRoute()); httpClientBuilder.setMaxConnTotal(properties.getMaxConnTotal());
return httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider).setConnectionManager(finalConnectionManager); httpClientBuilder.setMaxConnPerRoute(properties.getMaxConnPerRoute());
}); httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider);
return new RestHighLevelClient(builder); httpClientBuilder.setConnectionManager(finalConnectionManager);
// try {
// // 创建一个信任所有证书的 TrustStrategy 策略
// TrustStrategy acceptTrustStrategy = new TrustStrategy() {
// @Override
// public boolean isTrusted(X509Certificate[] x509Certificates, String s) throws CertificateException {
// return true;
// }
// };
// // 使用 SSLContextBuilder 创建 SSLContext
// SSLContext sslContext = SSLContextBuilder.create().loadTrustMaterial(null, acceptTrustStrategy).build();
// httpClientBuilder.setSSLContext(sslContext);
// } catch (NoSuchAlgorithmException | KeyStoreException | KeyManagementException e) {
// e.printStackTrace();
// }
try {
SSLContext sslContext = SSLContextBuilder.create().loadTrustMaterial(null, (TrustStrategy) (chain, authType) -> true).build();
// sslContext.init(null, new TrustManager[] {
// new X509TrustManager() {
// @Override
// public void checkClientTrusted(X509Certificate[] x509Certificates, String s)
// throws CertificateException {
// // 忽略证书错误 信任任何客户端证书
// }
//
// @Override
// public void checkServerTrusted(X509Certificate[] x509Certificates, String s)
// throws CertificateException {
// // 忽略证书错误 信任任何客户端证书
// }
//
// @Override
// public X509Certificate[] getAcceptedIssuers() {
// return new X509Certificate[0];
// }
// }
// },
// null);
httpClientBuilder.setSSLContext(sslContext);
} catch (NoSuchAlgorithmException | KeyManagementException | KeyStoreException e) {
System.out.println("忽略证书错误");
}
httpClientBuilder.setSSLHostnameVerifier(new NoopHostnameVerifier());
return httpClientBuilder;
})
.build();
ElasticsearchTransport transport = new RestClientTransport(restClient, new JacksonJsonpMapper());
return new ElasticsearchClient(transport);
} }
} }
src/main/java/com/viontech/match/entity/PersonInfo.java 0 → 100644
package com.viontech.match.entity;
import lombok.Data;
import java.util.Date;
@Data
public class PersonInfo {
private String unid;
private String personId;
private String _score;
private Double[] body;
private Double[] data;
private Integer age;
private String gender;
private Integer bodyType;
private Date countTime;
private String fid;
private String channelSerialNum;
private Long gateId;
private String direction;
public PersonInfo(String unid, String personId, Double[] body,Double[] data, Integer age, String gender, Integer bodyType, Date countTime, String fid, String channelSerialNum, Long gateId, String direction) {
this.unid = unid;
this.personId = personId;
this.body = body;
this.data = data;
this.age = age;
this.gender = gender;
this.bodyType = bodyType;
this.countTime = countTime;
this.fid = fid;
this.channelSerialNum = channelSerialNum;
this.gateId = gateId;
this.direction = direction;
this._score = "1";
}
}
src/main/java/com/viontech/match/entity/SearchResultHit.java 0 → 100644
package com.viontech.match.entity;
import lombok.Data;
import java.util.Date;
@Data
public class SearchResultHit {
private String index;
private String id;
private Double score;
// private PersonInfo source;
private String personId;
private String unid;
private String _score;
private Integer age;
private String gender;
private Integer bodyType;
private Date countTime;
private String fid;
private String channelSerialNum;
private Long gateId;
private String direction;
}
src/main/java/com/viontech/match/runner/ILM.java
package com.viontech.match.runner; package com.viontech.match.runner;
import co.elastic.clients.elasticsearch.ElasticsearchClient;
import co.elastic.clients.elasticsearch._types.Time;
import co.elastic.clients.elasticsearch.ilm.*;
import co.elastic.clients.elasticsearch.ilm.get_lifecycle.Lifecycle;
import lombok.extern.slf4j.Slf4j; import lombok.extern.slf4j.Slf4j;
import org.elasticsearch.client.RequestOptions; import org.elasticsearch.client.RequestOptions;
import org.elasticsearch.client.RestHighLevelClient;
import org.elasticsearch.client.core.AcknowledgedResponse;
import org.elasticsearch.client.indexlifecycle.DeleteAction;
import org.elasticsearch.client.indexlifecycle.LifecycleAction;
import org.elasticsearch.client.indexlifecycle.LifecyclePolicy;
import org.elasticsearch.client.indexlifecycle.Phase;
import org.elasticsearch.client.indexlifecycle.PutLifecyclePolicyRequest;
import org.elasticsearch.client.indexlifecycle.SetPriorityAction;
import org.elasticsearch.common.unit.TimeValue;
import org.springframework.beans.factory.annotation.Autowired; import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value; import org.springframework.beans.factory.annotation.Value;
import org.springframework.boot.CommandLineRunner; import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.autoconfigure.context.LifecycleAutoConfiguration;
import org.springframework.boot.autoconfigure.context.LifecycleProperties;
import org.springframework.stereotype.Component; import org.springframework.stereotype.Component;
import java.io.IOException; import java.io.IOException;
import java.util.Collections; import java.util.*;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeUnit;
/** /**
...@@ -39,25 +34,25 @@ public class ILM implements CommandLineRunner { ...@@ -39,25 +34,25 @@ public class ILM implements CommandLineRunner {
private Integer deleteAfterDays; private Integer deleteAfterDays;
@Autowired @Autowired
private RestHighLevelClient restHighLevelClient; private ElasticsearchClient client;
@Override @Override
public void run(String... args) throws Exception { public void run(String... args) throws Exception {
try { try {
Map<String, Phase> phases = new HashMap<>(); Actions hotAction = new Actions.Builder()
Map<String, LifecycleAction> setPriorityAction = new HashMap<>(); .setPriority(p -> p.priority(100))
setPriorityAction.put(SetPriorityAction.NAME, new SetPriorityAction(100)); .build();
phases.put("hot", new Phase("hot", TimeValue.ZERO, setPriorityAction)); Actions deleteAction = new Actions.Builder().delete(s -> s.deleteSearchableSnapshot(true)).build();
IlmPolicy ilmPolicy = new IlmPolicy.Builder()
Map<String, LifecycleAction> deleteActions = .phases(new Phases.Builder()
Collections.singletonMap(DeleteAction.NAME, new DeleteAction()); .hot(new Phase.Builder().actions(hotAction).minAge(Time.of(t -> t.time("0ms"))).build())
phases.put("delete", new Phase("delete", .delete(new Phase.Builder().actions(deleteAction).minAge(Time.of(t -> t.time(deleteAfterDays + "d"))).build())
new TimeValue(deleteAfterDays, TimeUnit.DAYS), deleteActions)); .build())
.build();
LifecyclePolicy policy = new LifecyclePolicy(LIFECYCLE_NAME, phases); PutLifecycleRequest request = new PutLifecycleRequest.Builder().name(LIFECYCLE_NAME).policy(ilmPolicy).build();
PutLifecyclePolicyRequest lifecyclePolicyRequest = new PutLifecyclePolicyRequest(policy); PutLifecycleResponse response = client.ilm().putLifecycle(request);
AcknowledgedResponse acknowledgedResponse = restHighLevelClient.indexLifecycle().putLifecyclePolicy(lifecyclePolicyRequest, RequestOptions.DEFAULT); GetLifecycleResponse getResponse = client.ilm().getLifecycle(new GetLifecycleRequest.Builder().name(LIFECYCLE_NAME).build());
if (acknowledgedResponse.isAcknowledged()) { if (getResponse.get(LIFECYCLE_NAME) != null) {
log.info(LIFECYCLE_NAME + "生命周期创建完成"); log.info(LIFECYCLE_NAME + "生命周期创建完成");
} }
} catch (Exception e) { } catch (Exception e) {
......
Markdown is supported
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!