chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,98 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-e2e</artifactId>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hertzbeat-collector-basic-e2e</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<postgresql.version>42.5.5</postgresql.version>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-common-e2e</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-basic</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-common</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Testcontainers Jupiter / JUnit 5 -->
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- Testcontainers JDBC Support -->
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-jdbc</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<!-- Testcontainers Database Modules -->
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-mysql</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-postgresql</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- JDBC Drivers -->
|
||||
<dependency>
|
||||
<groupId>com.mysql</groupId>
|
||||
<artifactId>mysql-connector-j</artifactId>
|
||||
<scope>test</scope>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.postgresql</groupId>
|
||||
<artifactId>postgresql</artifactId>
|
||||
<version>${postgresql.version}</version>
|
||||
<scope>test</scope>
|
||||
<optional>true</optional>
|
||||
</dependency>
|
||||
|
||||
</dependencies>
|
||||
</project>
|
||||
+72
@@ -0,0 +1,72 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.basic.database;
|
||||
|
||||
/**
|
||||
* Database type (image mame) and image tag enumeration class
|
||||
*/
|
||||
public enum DatabaseImagesEnum {
|
||||
MYSQL("mysql", "8.0.36"),
|
||||
POSTGRESQL("postgresql", "15");
|
||||
|
||||
private final String imageName;
|
||||
private final String defaultTag;
|
||||
|
||||
DatabaseImagesEnum(String imageName, String defaultTag) {
|
||||
this.imageName = imageName;
|
||||
this.defaultTag = defaultTag;
|
||||
}
|
||||
|
||||
public String getImageName() {
|
||||
return imageName;
|
||||
}
|
||||
|
||||
public String getDefaultTag() {
|
||||
return defaultTag;
|
||||
}
|
||||
|
||||
public String getFullImageName() {
|
||||
return imageName + ":" + defaultTag;
|
||||
}
|
||||
|
||||
public static DatabaseImagesEnum fromImageName(String imageName) {
|
||||
for (DatabaseImagesEnum value : values()) {
|
||||
if (value.getImageName().equalsIgnoreCase(imageName)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unknown database image name: " + imageName);
|
||||
}
|
||||
|
||||
public static DatabaseImagesEnum fromFullImageName(String fullImageName) {
|
||||
for (DatabaseImagesEnum value : values()) {
|
||||
if (value.getFullImageName().equalsIgnoreCase(fullImageName)) {
|
||||
return value;
|
||||
}
|
||||
}
|
||||
throw new IllegalArgumentException("Unknown full database image name: " + fullImageName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return "DatabaseImagesEnum{"
|
||||
+ "imageName='" + imageName + '\''
|
||||
+ ", defaultTag='" + defaultTag + '\''
|
||||
+ '}';
|
||||
}
|
||||
}
|
||||
+99
@@ -0,0 +1,99 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.basic.database;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
|
||||
import org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect;
|
||||
import org.apache.hertzbeat.collector.util.CollectUtil;
|
||||
import org.apache.hertzbeat.common.entity.job.Configmap;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
|
||||
|
||||
/**
|
||||
* E2e test monitored by Jdbc Common
|
||||
*/
|
||||
@Slf4j
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
public class JdbcCommonCollectE2eTest extends AbstractCollectE2eTest {
|
||||
private static final DatabaseImagesEnum databaseImage = DatabaseImagesEnum.fromImageName("postgresql");
|
||||
|
||||
private static final String DATABASE_IMAGE_NAME = databaseImage.getImageName();
|
||||
|
||||
private static final String DATABASE_IMAGE_TAG = databaseImage.getDefaultTag();
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
collect = new JdbcCommonCollect();
|
||||
metrics = new Metrics();
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
|
||||
JdbcProtocol jdbcProtocol = (JdbcProtocol) buildProtocol(metricsDef);
|
||||
metrics.setJdbc(jdbcProtocol);
|
||||
CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder();
|
||||
metricsData.setApp(DATABASE_IMAGE_NAME);
|
||||
metrics.setAliasFields(metricsDef.getAliasFields());
|
||||
return collectMetricsData(metrics, metricsDef, metricsData);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Protocol buildProtocol(Metrics metricsDef) {
|
||||
JdbcProtocol jdbcProtocol = metricsDef.getJdbc();
|
||||
jdbcProtocol.setHost(DATABASE_IMAGE_NAME);
|
||||
jdbcProtocol.setPort(DATABASE_IMAGE_TAG);
|
||||
jdbcProtocol.setDatabase("test");
|
||||
jdbcProtocol.setPlatform("testcontainers");
|
||||
return jdbcProtocol;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWithJdbcTcUrl() {
|
||||
Job dockerJob = appService.getAppDefine(DATABASE_IMAGE_NAME);
|
||||
List<Map<String, Configmap>> configmapFromPreCollectData = new LinkedList<>();
|
||||
for (Metrics metricsDef : dockerJob.getMetrics()) {
|
||||
metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, configmapFromPreCollectData.size() > 0 ? configmapFromPreCollectData.get(0) : new HashMap<>());
|
||||
String metricName = metricsDef.getName();
|
||||
if ("slow_sql".equals(metricName)) {
|
||||
Metrics finalMetricsDef = metricsDef;
|
||||
assertDoesNotThrow(() -> collectMetrics(finalMetricsDef),
|
||||
String.format("%s failed to collect metrics)", metricName));
|
||||
log.info("{} metrics validation passed", metricName);
|
||||
continue; // skip slow_sql, extra extensions
|
||||
}
|
||||
CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricName, true);
|
||||
CollectUtil.getConfigmapFromPreCollectData(metricsData);
|
||||
}
|
||||
}
|
||||
}
|
||||
+144
@@ -0,0 +1,144 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.basic.http;
|
||||
|
||||
import com.sun.net.httpserver.HttpExchange;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
|
||||
import org.apache.hertzbeat.collector.collect.http.HttpCollectImpl;
|
||||
import org.apache.hertzbeat.collector.util.CollectUtil;
|
||||
import org.apache.hertzbeat.common.entity.job.Configmap;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.util.ResourceUtils;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.nio.file.Files;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Integration test for Docker monitoring functionality
|
||||
*/
|
||||
@Slf4j
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
public class DockerMonitorE2eTest extends AbstractCollectE2eTest {
|
||||
|
||||
private static final int MOCK_SERVER_PORT = 52375;
|
||||
private static final String LOCALHOST = "127.0.0.1";
|
||||
private static HttpServer mockServer;
|
||||
|
||||
@AfterAll
|
||||
public static void tearDown() {
|
||||
if (mockServer != null) {
|
||||
mockServer.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() throws Exception {
|
||||
|
||||
super.setUp();
|
||||
// Setup collect instance
|
||||
collect = new HttpCollectImpl();
|
||||
|
||||
// Setup mock server and endpoints
|
||||
mockServer = HttpServer.create(new InetSocketAddress(MOCK_SERVER_PORT), 0);
|
||||
mockServer.setExecutor(null);
|
||||
mockServer.start();
|
||||
|
||||
// Setup Docker API endpoints
|
||||
String containerResponse = loadResponseFromFile("classpath:http/docker/containers_result.txt");
|
||||
String infoResponse = loadResponseFromFile("classpath:http/docker/system_result.txt");
|
||||
String containerStatsResponse = loadResponseFromFile("classpath:http/docker/containers_stats.txt");
|
||||
|
||||
mockServer.createContext("/containers/json", exchange -> {
|
||||
String query = exchange.getRequestURI().getQuery();
|
||||
if (query != null && query.contains("all=true")) {
|
||||
sendJsonResponse(exchange, containerResponse);
|
||||
} else {
|
||||
exchange.sendResponseHeaders(404, 0);
|
||||
exchange.close();
|
||||
}
|
||||
});
|
||||
|
||||
mockServer.createContext("/info", exchange -> sendJsonResponse(exchange, infoResponse));
|
||||
mockServer.createContext("/containers/34174a918eb2e38cdb097c910f74af845e7383b04765d26ad52f940f86342a64/stats", exchange -> sendJsonResponse(exchange, containerStatsResponse));
|
||||
|
||||
}
|
||||
|
||||
private String loadResponseFromFile(String resourcePath) throws Exception {
|
||||
return new String(Files.readAllBytes(ResourceUtils.getFile(resourcePath).toPath()));
|
||||
}
|
||||
|
||||
private void sendJsonResponse(HttpExchange exchange, String response) throws IOException {
|
||||
exchange.getResponseHeaders().set("Content-Type", "application/json");
|
||||
final byte[] array = response.getBytes(StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(200, array.length);
|
||||
try (OutputStream os = exchange.getResponseBody()) {
|
||||
os.write(array);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDockerMonitor() {
|
||||
Job dockerJob = appService.getAppDefine("docker");
|
||||
List<Map<String, Configmap>> configmapFromPreCollectData = new LinkedList<>();
|
||||
for (Metrics metricsDef : dockerJob.getMetrics()) {
|
||||
metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, configmapFromPreCollectData.size() > 0 ? configmapFromPreCollectData.get(0) : new HashMap<>());
|
||||
CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricsDef.getName());
|
||||
configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Protocol buildProtocol(Metrics metricsDef) {
|
||||
// Setup HTTP protocol
|
||||
HttpProtocol protocol = new HttpProtocol();
|
||||
protocol.setHost(LOCALHOST);
|
||||
protocol.setPort(String.valueOf(MOCK_SERVER_PORT));
|
||||
protocol.setMethod(metricsDef.getHttp().getMethod());
|
||||
protocol.setParseType(metricsDef.getHttp().getParseType());
|
||||
protocol.setParseScript(metricsDef.getHttp().getParseScript());
|
||||
protocol.setUrl(metricsDef.getHttp().getUrl());
|
||||
return protocol;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
|
||||
HttpProtocol protocol = (HttpProtocol) buildProtocol(metricsDef);
|
||||
metrics.setHttp(protocol);
|
||||
return collectMetricsData(metrics, metricsDef);
|
||||
}
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.basic.http;
|
||||
|
||||
import com.sun.net.httpserver.HttpExchange;
|
||||
import com.sun.net.httpserver.HttpServer;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
|
||||
import org.apache.hertzbeat.collector.collect.http.HttpCollectImpl;
|
||||
import org.apache.hertzbeat.collector.util.CollectUtil;
|
||||
import org.apache.hertzbeat.common.entity.job.Configmap;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.HttpProtocol;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.io.OutputStream;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* E2E test for HTTP monitor
|
||||
*/
|
||||
@Slf4j
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@SpringBootTest(webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
public class HttpMonitorE2eTest extends AbstractCollectE2eTest {
|
||||
|
||||
private static final int MOCK_SERVER_PORT = 52376;
|
||||
private static final String LOCALHOST = "127.0.0.1";
|
||||
private static final String RELATIVE_PATH = "/";
|
||||
private static final List<String> ALLOW_EMPTY_WHITE_LIST = List.of("header");
|
||||
private static HttpServer mockServer;
|
||||
|
||||
@AfterAll
|
||||
public static void tearDown() {
|
||||
if (mockServer != null) {
|
||||
mockServer.stop(0);
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() throws Exception {
|
||||
|
||||
super.setUp();
|
||||
// Setup collect instance
|
||||
collect = new HttpCollectImpl();
|
||||
|
||||
// Setup mock server and endpoints
|
||||
mockServer = HttpServer.create(new InetSocketAddress(MOCK_SERVER_PORT), 0);
|
||||
mockServer.setExecutor(null);
|
||||
mockServer.start();
|
||||
|
||||
mockServer.createContext(RELATIVE_PATH, exchange -> sendJsonResponse(exchange, ""));
|
||||
|
||||
}
|
||||
|
||||
private void sendJsonResponse(HttpExchange exchange, String response) throws IOException {
|
||||
exchange.getResponseHeaders().set("Content-Type", "application/json");
|
||||
final byte[] array = response.getBytes(StandardCharsets.UTF_8);
|
||||
exchange.sendResponseHeaders(200, array.length);
|
||||
try (OutputStream os = exchange.getResponseBody()) {
|
||||
os.write(array);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testHttpMonitor() {
|
||||
Job dockerJob = appService.getAppDefine("api");
|
||||
List<Map<String, Configmap>> configmapFromPreCollectData = new LinkedList<>();
|
||||
for (Metrics metricsDef : dockerJob.getMetrics()) {
|
||||
metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, configmapFromPreCollectData.size() > 0 ? configmapFromPreCollectData.get(0) : new HashMap<>());
|
||||
CollectRep.MetricsData metricsData;
|
||||
if (ALLOW_EMPTY_WHITE_LIST.contains(metricsDef.getName())) {
|
||||
metricsData = validateMetricsCollection(metricsDef, metricsDef.getName(), true);
|
||||
} else {
|
||||
metricsData = validateMetricsCollection(metricsDef, metricsDef.getName());
|
||||
}
|
||||
configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Protocol buildProtocol(Metrics metricsDef) {
|
||||
// Setup HTTP protocol
|
||||
HttpProtocol protocol = new HttpProtocol();
|
||||
protocol.setHost(LOCALHOST);
|
||||
protocol.setUrl(RELATIVE_PATH);
|
||||
protocol.setMethod("GET");
|
||||
protocol.setPort(String.valueOf(MOCK_SERVER_PORT));
|
||||
protocol.setParseType(metricsDef.getHttp().getParseType());
|
||||
protocol.setParseScript(metricsDef.getHttp().getParseScript());
|
||||
return protocol;
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
|
||||
HttpProtocol protocol = (HttpProtocol) buildProtocol(metricsDef);
|
||||
metrics.setHttp(protocol);
|
||||
return collectMetricsData(metrics, metricsDef);
|
||||
}
|
||||
}
|
||||
+126
@@ -0,0 +1,126 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.basic.redis;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
|
||||
import org.apache.hertzbeat.collector.collect.redis.RedisCommonCollectImpl;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.RedisProtocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.Network;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.lifecycle.Startables;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* End-to-end test class for redis collector
|
||||
*/
|
||||
@Slf4j
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
public class RedisCommonCollectE2eTest extends AbstractCollectE2eTest {
|
||||
private static final String REDIS_IMAGE = "redis:7.4.2";
|
||||
private static final String HOST = "127.0.0.1";
|
||||
private static final int REDIS_PORT = 6379;
|
||||
private static final String REDIS_PATTERN = "1";
|
||||
private static final List<String> ALLOW_EMPTY_WHITE_LIST = Arrays.asList("server", "errorstats", "commandstats", "keyspace");
|
||||
|
||||
private static GenericContainer<?> redisContainer;
|
||||
|
||||
private Integer mappedPort;
|
||||
|
||||
@AfterAll
|
||||
public static void tearDown() throws InterruptedException {
|
||||
if (redisContainer != null) {
|
||||
redisContainer.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
collect = new RedisCommonCollectImpl();
|
||||
|
||||
// Set up and start container
|
||||
setupAndStartContainer();
|
||||
|
||||
mappedPort = redisContainer.getMappedPort(REDIS_PORT);
|
||||
log.info("Container started successfully with mapped port: {}", mappedPort);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRedisCollect() throws ExecutionException, InterruptedException, TimeoutException {
|
||||
Assertions.assertTrue(redisContainer.isRunning(), "redis container should be running");
|
||||
|
||||
Job redisJob = appService.getAppDefine("redis");
|
||||
redisJob.getMetrics().forEach(metricsDef -> {
|
||||
if (ALLOW_EMPTY_WHITE_LIST.contains(metricsDef.getName())) {
|
||||
validateMetricsCollection(metricsDef, metricsDef.getName(), true);
|
||||
} else {
|
||||
validateMetricsCollection(metricsDef, metricsDef.getName());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
|
||||
//Build redis protocol configuration
|
||||
RedisProtocol redisProtocol = (RedisProtocol) buildProtocol(metricsDef);
|
||||
metrics.setRedis(redisProtocol);
|
||||
metrics.setName(metricsDef.getName());
|
||||
metrics.setFields(metricsDef.getFields());
|
||||
return collectMetricsData(metrics, metricsDef);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Protocol buildProtocol(Metrics metricsDef) {
|
||||
RedisProtocol redisProtocol = new RedisProtocol();
|
||||
redisProtocol.setHost(HOST);
|
||||
redisProtocol.setPort(mappedPort.toString());
|
||||
redisProtocol.setPattern(REDIS_PATTERN);
|
||||
return redisProtocol;
|
||||
}
|
||||
|
||||
public void setupAndStartContainer() {
|
||||
Network network = Network.builder().build();
|
||||
redisContainer = new GenericContainer<>(DockerImageName.parse(REDIS_IMAGE))
|
||||
.withExposedPorts(REDIS_PORT)
|
||||
.withNetwork(network)
|
||||
.withNetworkAliases("redis")
|
||||
.waitingFor(Wait.forListeningPort())
|
||||
.withStartupTimeout(Duration.ofSeconds(30));
|
||||
|
||||
Startables.deepStart(Stream.of(redisContainer)).join();
|
||||
}
|
||||
}
|
||||
+150
@@ -0,0 +1,150 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.basic.ssh;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
|
||||
import org.apache.hertzbeat.collector.collect.ssh.SshCollectImpl;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.SshProtocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.Network;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.lifecycle.Startables;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Random;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* End-to-end test class for SSH collector
|
||||
*/
|
||||
@Slf4j
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
public class SshCollectE2eTest extends AbstractCollectE2eTest {
|
||||
private static final String UBUNTU_IMAGE = "rastasheep/ubuntu-sshd:18.04";
|
||||
private static final String HOST = "127.0.0.1";
|
||||
private static final String ROOT_USER = "root";
|
||||
private static final int SSH_PORT = 22;
|
||||
private static final int PASSWORD_LENGTH = 12;
|
||||
private static final List<String> ALLOW_EMPTY_WHITE_LIST = Arrays.asList("top_mem_process", "top_cpu_process");
|
||||
|
||||
private static GenericContainer<?> linuxContainer;
|
||||
|
||||
private Integer mappedPort;
|
||||
private String password;
|
||||
|
||||
@AfterAll
|
||||
public static void tearDown() {
|
||||
if (linuxContainer != null) {
|
||||
linuxContainer.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
collect = new SshCollectImpl();
|
||||
password = generateRandomPassword();
|
||||
|
||||
// Set up and start container
|
||||
setupAndStartContainer();
|
||||
|
||||
// Set root password
|
||||
setContainerRootPassword();
|
||||
|
||||
mappedPort = linuxContainer.getMappedPort(SSH_PORT);
|
||||
log.info("Container started successfully with mapped port: {}", mappedPort);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSshCollect() throws ExecutionException, InterruptedException, TimeoutException {
|
||||
Assertions.assertTrue(linuxContainer.isRunning(), "Ubuntu container should be running");
|
||||
|
||||
Job ubuntuJob = appService.getAppDefine("ubuntu");
|
||||
ubuntuJob.getMetrics().forEach(metricsDef -> {
|
||||
if (ALLOW_EMPTY_WHITE_LIST.contains(metricsDef.getName())) {
|
||||
validateMetricsCollection(metricsDef, metricsDef.getName(), true);
|
||||
} else {
|
||||
validateMetricsCollection(metricsDef, metricsDef.getName());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
|
||||
// Build SSH protocol configuration
|
||||
SshProtocol sshProtocol = (SshProtocol) buildProtocol(metricsDef);
|
||||
metrics.setSsh(sshProtocol);
|
||||
return collectMetricsData(metrics, metricsDef);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Protocol buildProtocol(Metrics metricsDef) {
|
||||
SshProtocol sshProtocol = new SshProtocol();
|
||||
sshProtocol.setHost(HOST);
|
||||
sshProtocol.setPort(mappedPort.toString());
|
||||
sshProtocol.setUsername(ROOT_USER);
|
||||
sshProtocol.setPassword(password);
|
||||
sshProtocol.setScript(metricsDef.getSsh().getScript());
|
||||
sshProtocol.setParseType(metricsDef.getSsh().getParseType());
|
||||
return sshProtocol;
|
||||
}
|
||||
|
||||
private void setupAndStartContainer() {
|
||||
Network network = Network.builder().build();
|
||||
linuxContainer = new GenericContainer<>(DockerImageName.parse(UBUNTU_IMAGE))
|
||||
.withExposedPorts(SSH_PORT)
|
||||
.withNetwork(network)
|
||||
.withNetworkAliases("ubuntu")
|
||||
.waitingFor(Wait.forListeningPort())
|
||||
.withCommand("/usr/sbin/sshd", "-D")
|
||||
.withStartupTimeout(Duration.ofSeconds(30));
|
||||
|
||||
Startables.deepStart(Stream.of(linuxContainer)).join();
|
||||
}
|
||||
|
||||
private void setContainerRootPassword() throws Exception {
|
||||
linuxContainer.execInContainer("bash", "-c",
|
||||
String.format("echo 'root:%s' | chpasswd", password));
|
||||
}
|
||||
|
||||
private String generateRandomPassword() {
|
||||
String characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789";
|
||||
StringBuilder password = new StringBuilder();
|
||||
Random random = new Random();
|
||||
for (int i = 0; i < PASSWORD_LENGTH; i++) {
|
||||
password.append(characters.charAt(random.nextInt(characters.length())));
|
||||
}
|
||||
return password.toString();
|
||||
}
|
||||
}
|
||||
+128
@@ -0,0 +1,128 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.basic.telnet;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
|
||||
import org.apache.hertzbeat.collector.collect.telnet.TelnetCollectImpl;
|
||||
import org.apache.hertzbeat.collector.util.CollectUtil;
|
||||
import org.apache.hertzbeat.common.entity.job.Configmap;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.TelnetProtocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* Integration test for Zookeeper monitoring functionality
|
||||
*/
|
||||
@Slf4j
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
public class ZookeeperMonitorE2eTest extends AbstractCollectE2eTest {
|
||||
|
||||
private static final String ZOOKEEPER_IMAGE_NAME = "zookeeper:3.8.4";
|
||||
private static final String ZOOKEEPER_NAME = "zookeeper";
|
||||
private static final Integer ZOOKEEPER_PORT = 2181;
|
||||
private static GenericContainer<?> zookeeperContainer;
|
||||
|
||||
@AfterAll
|
||||
public static void tearDown() {
|
||||
if (zookeeperContainer != null) {
|
||||
zookeeperContainer.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() throws Exception {
|
||||
super.setUp();
|
||||
collect = new TelnetCollectImpl();
|
||||
metrics = new Metrics();
|
||||
|
||||
try {
|
||||
// Start Zookeeper container with custom configuration
|
||||
zookeeperContainer = new GenericContainer<>(DockerImageName.parse(ZOOKEEPER_IMAGE_NAME))
|
||||
.withExposedPorts(ZOOKEEPER_PORT)
|
||||
.withEnv("ZOO_4LW_COMMANDS_WHITELIST", "*")
|
||||
.withNetworkAliases(ZOOKEEPER_NAME)
|
||||
.waitingFor(
|
||||
Wait.forLogMessage(".*Started AdminServer on address.*\\n", 1)
|
||||
.withStartupTimeout(Duration.ofSeconds(60))
|
||||
)
|
||||
.withLogConsumer(outputFrame -> {
|
||||
log.info(outputFrame.getUtf8String());
|
||||
});
|
||||
|
||||
zookeeperContainer.start();
|
||||
log.info("Zookeeper container started at {}:{}",
|
||||
zookeeperContainer.getHost(),
|
||||
zookeeperContainer.getMappedPort(ZOOKEEPER_PORT));
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
log.error("Failed to start Zookeeper container", e);
|
||||
throw e;
|
||||
}
|
||||
Thread.sleep(30000);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
|
||||
TelnetProtocol telnetProtocol = (TelnetProtocol) buildProtocol(metricsDef);
|
||||
metrics.setTelnet(telnetProtocol);
|
||||
CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder();
|
||||
metricsData.setApp(ZOOKEEPER_NAME);
|
||||
metrics.setAliasFields(metricsDef.getAliasFields());
|
||||
return collectMetricsData(metrics, metricsDef, metricsData);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Protocol buildProtocol(Metrics metricsDef) {
|
||||
TelnetProtocol protocol = new TelnetProtocol();
|
||||
protocol.setHost(zookeeperContainer.getHost());
|
||||
protocol.setPort(String.valueOf(zookeeperContainer.getMappedPort(ZOOKEEPER_PORT)));
|
||||
protocol.setCmd(metricsDef.getTelnet().getCmd());
|
||||
return protocol;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testZookeeperMonitor() {
|
||||
Assertions.assertTrue(zookeeperContainer.isRunning(), "Zookeeper container should be running");
|
||||
|
||||
Job dockerJob = appService.getAppDefine("zookeeper");
|
||||
List<Map<String, Configmap>> configmapFromPreCollectData = new LinkedList<>();
|
||||
for (Metrics metricsDef : dockerJob.getMetrics()) {
|
||||
metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef, configmapFromPreCollectData.size() > 0 ? configmapFromPreCollectData.get(0) : new HashMap<>());
|
||||
CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricsDef.getName());
|
||||
configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData);
|
||||
}
|
||||
}
|
||||
}
|
||||
+1
@@ -0,0 +1 @@
|
||||
[{"Id":"34174a918eb2e38cdb097c910f74af845e7383b04765d26ad52f940f86342a64","Names":["/mysql"],"Image":"mysql:8.0","ImageID":"sha256:6c55ddbef96911f9f36d1330ffe3f7557c019d49434e738cafabd1a3dd6b4bac","Command":"docker-entrypoint.sh mysqld","Created":1735286602,"Ports":[{"IP":"0.0.0.0","PrivatePort":3306,"PublicPort":3306,"Type":"tcp"},{"IP":"::","PrivatePort":3306,"PublicPort":3306,"Type":"tcp"},{"PrivatePort":33060,"Type":"tcp"}],"Labels":{},"State":"running","Status":"Up 42 hours","HostConfig":{"NetworkMode":"bridge"},"NetworkSettings":{"Networks":{"bridge":{"IPAMConfig":null,"Links":null,"Aliases":null,"MacAddress":"02:42:ac:11:00:04","NetworkID":"305ba4c311d55570760456274cf1653e22bf5383873498e6a159d76baaba0e4e","EndpointID":"6925c9ece99d84ed57c4b08a41c38d009cd8becc9cc2e941eb2b572478a22747","Gateway":"172.17.0.1","IPAddress":"172.17.0.4","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DriverOpts":null,"DNSNames":null}}},"Mounts":[{"Type":"bind","Source":"/docker/mysql/conf/my.cnf","Destination":"/etc/mysql/my.cnf","Mode":"","RW":true,"Propagation":"rprivate"},{"Type":"bind","Source":"/docker/mysql/data","Destination":"/var/lib/mysql","Mode":"","RW":true,"Propagation":"rprivate"}]},{"Id":"d1f070438d28de7059113df7e79423f5ba1fdc4968cee4914b661cec222ddb06","Names":["/dbgate-instance"],"Image":"dbgate:latest","ImageID":"sha256:107efa35665b98c97cb9698cae2daf3f80aa88854ac90d81bba24b8de02c830d","Command":"/home/dbgate-docker/entrypoint.sh","Created":1734751223,"Ports":[{"IP":"0.0.0.0","PrivatePort":3000,"PublicPort":3000,"Type":"tcp"},{"IP":"::","PrivatePort":3000,"PublicPort":3000,"Type":"tcp"}],"Labels":{"org.opencontainers.image.ref.name":"ubuntu","org.opencontainers.image.version":"22.04"},"State":"running","Status":"Up 7 days","HostConfig":{"NetworkMode":"bridge"},"NetworkSettings":{"Networks":{"bridge":{"IPAMConfig":null,"Links":null,"Aliases":null,"MacAddress":"02:42:ac:11:00:03","NetworkID":"305ba4c311d55570760456274cf1653e22bf5383873498e6a159d76baaba0e4e","EndpointID":"a8f31d5d2ec0100137dced7550b3d662b0804bf8f767a457cbaebda863fb4504","Gateway":"172.17.0.1","IPAddress":"172.17.0.3","IPPrefixLen":16,"IPv6Gateway":"","GlobalIPv6Address":"","GlobalIPv6PrefixLen":0,"DriverOpts":null,"DNSNames":null}}},"Mounts":[{"Type":"volume","Name":"d0d9b8e8edce3dc092b2b50a1f9c6bb754fd556cffe95be8ae00e016daaaa3fd","Source":"","Destination":"/root/.dbgate","Driver":"local","Mode":"","RW":true,"Propagation":""}]}]
|
||||
+1
@@ -0,0 +1 @@
|
||||
{"read":"2000-01-01T00:00:15.864368408Z","preread":"2000-01-01T00:00:14.86209639Z","pids_stats":{"current":41},"blkio_stats":{"io_service_bytes_recursive":[{"major":8,"minor":0,"op":"Read","value":118784},{"major":8,"minor":0,"op":"Write","value":267395072},{"major":8,"minor":0,"op":"Sync","value":267395072},{"major":8,"minor":0,"op":"Async","value":118784},{"major":8,"minor":0,"op":"Total","value":267513856},{"major":253,"minor":0,"op":"Read","value":118784},{"major":253,"minor":0,"op":"Write","value":267395072},{"major":253,"minor":0,"op":"Sync","value":267395072},{"major":253,"minor":0,"op":"Async","value":118784},{"major":253,"minor":0,"op":"Total","value":267513856}],"io_serviced_recursive":[{"major":8,"minor":0,"op":"Read","value":8},{"major":8,"minor":0,"op":"Write","value":7892},{"major":8,"minor":0,"op":"Sync","value":7892},{"major":8,"minor":0,"op":"Async","value":8},{"major":8,"minor":0,"op":"Total","value":7900},{"major":253,"minor":0,"op":"Read","value":8},{"major":253,"minor":0,"op":"Write","value":7892},{"major":253,"minor":0,"op":"Sync","value":7892},{"major":253,"minor":0,"op":"Async","value":8},{"major":253,"minor":0,"op":"Total","value":7900}],"io_queue_recursive":[],"io_service_time_recursive":[],"io_wait_time_recursive":[],"io_merged_recursive":[],"io_time_recursive":[],"sectors_recursive":[]},"num_procs":0,"storage_stats":{},"cpu_stats":{"cpu_usage":{"total_usage":4810174080874,"percpu_usage":[733251885463,588352569755,649761861064,692942680516,618939038384,513206148155,431772846470,581947051067],"usage_in_kernelmode":1191390000000,"usage_in_usermode":1463160000000},"system_cpu_usage":21196078922904483,"online_cpus":8,"throttling_data":{"periods":0,"throttled_periods":0,"throttled_time":0}},"precpu_stats":{"cpu_usage":{"total_usage":4810165603627,"percpu_usage":[733250739914,588352569755,649761643572,692942567027,618937059553,513205711609,431769258762,581946053435],"usage_in_kernelmode":1191390000000,"usage_in_usermode":1463160000000},"system_cpu_usage":21196070922904483,"online_cpus":8,"throttling_data":{"periods":0,"throttled_periods":0,"throttled_time":0}},"memory_stats":{"usage":505090048,"max_usage":519450624,"stats":{"active_anon":396529664,"active_file":52400128,"cache":108560384,"dirty":0,"hierarchical_memory_limit":9223372036854771712,"hierarchical_memsw_limit":9223372036854771712,"inactive_anon":0,"inactive_file":56160256,"mapped_file":40960,"pgfault":409148,"pgmajfault":0,"pgpgin":343733,"pgpgout":220420,"rss":396529664,"rss_huge":0,"total_active_anon":396529664,"total_active_file":52400128,"total_cache":108560384,"total_dirty":0,"total_inactive_anon":0,"total_inactive_file":56160256,"total_mapped_file":40960,"total_pgfault":0,"total_pgmajfault":0,"total_pgpgin":0,"total_pgpgout":0,"total_rss":396529664,"total_rss_huge":0,"total_unevictable":0,"total_writeback":0,"unevictable":0,"writeback":0},"limit":33566269440},"name":"/mysql","id":"34174a918eb2e38cdb097c910f74af845e7383b04765d26ad52f940f86342a64","networks":{"eth0":{"rx_bytes":68182,"rx_packets":533,"rx_errors":0,"rx_dropped":0,"tx_bytes":211850,"tx_packets":361,"tx_errors":0,"tx_dropped":0}}}
|
||||
+1
File diff suppressed because one or more lines are too long
@@ -0,0 +1,70 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-e2e</artifactId>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hertzbeat-collector-common-e2e</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-basic</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-common</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-jar-plugin</artifactId>
|
||||
<version>${maven-jar-plugin.version}</version>
|
||||
<configuration>
|
||||
<skip>false</skip>
|
||||
</configuration>
|
||||
<executions>
|
||||
<execution>
|
||||
<goals>
|
||||
<goal>test-jar</goal>
|
||||
</goals>
|
||||
</execution>
|
||||
</executions>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
+158
@@ -0,0 +1,158 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.collector.dispatch.CollectDataDispatch;
|
||||
import org.apache.hertzbeat.collector.dispatch.MetricsCollect;
|
||||
import org.apache.hertzbeat.common.timer.Timeout;
|
||||
import org.apache.hertzbeat.collector.timer.WheelTimerTask;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.manager.dao.DefineDao;
|
||||
import org.apache.hertzbeat.manager.service.impl.AppServiceImpl;
|
||||
import org.apache.hertzbeat.manager.service.impl.ObjectStoreConfigServiceImpl;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* AbstractCollectE2eTest
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class AbstractCollectE2eTest {
|
||||
|
||||
@InjectMocks
|
||||
protected AppServiceImpl appService;
|
||||
|
||||
protected AbstractCollect collect;
|
||||
|
||||
protected MetricsCollect metricsCollect;
|
||||
|
||||
protected Metrics metrics;
|
||||
|
||||
@Mock
|
||||
protected ObjectStoreConfigServiceImpl objectStoreConfigService;
|
||||
@Mock
|
||||
private WheelTimerTask timerJob;
|
||||
@Mock
|
||||
private Timeout timeout;
|
||||
@Mock
|
||||
private Job job;
|
||||
@Mock
|
||||
private DefineDao defineDao;
|
||||
|
||||
public void setUp() throws Exception {
|
||||
// Initialize mocks
|
||||
MockitoAnnotations.openMocks(this);
|
||||
lenient().when(defineDao.findAll()).thenReturn(new ArrayList<>());
|
||||
when(timeout.task()).thenReturn(timerJob);
|
||||
when(timerJob.getJob()).thenReturn(job);
|
||||
metricsCollect = new MetricsCollect(mock(Metrics.class), timeout, mock(CollectDataDispatch.class), null, List.of());
|
||||
|
||||
// Initialize services and components
|
||||
appService.afterPropertiesSet();
|
||||
metrics = new Metrics();
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate metrics collection, check if the metrics values are not empty <br/>
|
||||
*
|
||||
* @param metricsDef metrics definition
|
||||
* @param metricName metric name
|
||||
* @return metrics data
|
||||
*/
|
||||
protected CollectRep.MetricsData validateMetricsCollection(Metrics metricsDef, String metricName) {
|
||||
// By default, we do not allow empty values
|
||||
return validateMetricsCollection(metricsDef, metricName, false);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate metrics collection, check if the metrics values are not empty <br/>
|
||||
* We believe that all monitoring metrics should have data
|
||||
*
|
||||
* @param metricsDef metrics definition
|
||||
* @param metricName metric name
|
||||
* @param allowEmpty In some special scenarios, it is not necessary to check if the value is ` `
|
||||
*/
|
||||
protected CollectRep.MetricsData validateMetricsCollection(Metrics metricsDef, String metricName, boolean allowEmpty) {
|
||||
CollectRep.MetricsData.Builder metricsData = collectMetrics(metricsDef);
|
||||
|
||||
metricsCollect.calculateFields(metricsDef, metricsData);
|
||||
|
||||
Assertions.assertTrue(metricsData.getValuesList().size() > 0,
|
||||
String.format("%s metrics values should not be empty, detail: %s", metricName, metricsData.getMsg()));
|
||||
|
||||
for (CollectRep.ValueRow valueRow : metricsData.getValuesList()) {
|
||||
for (int i = 0; i < valueRow.getColumnsCount(); i++) {
|
||||
Assertions.assertFalse(valueRow.getColumns(i).isEmpty(),
|
||||
String.format("%s metric column %d should not be empty", metricName, i));
|
||||
if (!allowEmpty) {
|
||||
// Check if the value is not null
|
||||
Assertions.assertNotEquals(CommonConstants.NULL_VALUE, valueRow.getColumns(i), String.format("%s metric column %d should not be null", metricName, i));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
log.info("{} metrics validation passed", metricName);
|
||||
return metricsData.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Set alias fields for metrics
|
||||
*
|
||||
* @param metrics metrics
|
||||
* @param metricsDef metrics definition
|
||||
*/
|
||||
protected void setMetricsAliasFields(Metrics metrics, Metrics metricsDef) {
|
||||
List<String> aliasFields = metricsDef.getAliasFields() == null
|
||||
? metricsDef.getFields().stream().map(Metrics.Field::getField).collect(Collectors.toList())
|
||||
: metricsDef.getAliasFields();
|
||||
metrics.setAliasFields(aliasFields);
|
||||
metricsDef.setAliasFields(aliasFields);
|
||||
}
|
||||
|
||||
protected abstract CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef);
|
||||
|
||||
protected CollectRep.MetricsData.Builder collectMetricsData(Metrics metrics, Metrics metricsDef) {
|
||||
CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder();
|
||||
return this.collectMetricsData(metrics, metricsDef, metricsData);
|
||||
}
|
||||
|
||||
protected CollectRep.MetricsData.Builder collectMetricsData(Metrics metrics, Metrics metricsDef, CollectRep.MetricsData.Builder metricsData) {
|
||||
setMetricsAliasFields(metrics, metricsDef);
|
||||
|
||||
// Collect metrics
|
||||
collect.collect(metricsData, metrics);
|
||||
return metricsData;
|
||||
}
|
||||
|
||||
protected abstract Protocol buildProtocol(Metrics metricsDef);
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-e2e</artifactId>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hertzbeat-collector-kafka-e2e</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-kafka</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.kafka</groupId>
|
||||
<artifactId>kafka-clients</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-kafka</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-common</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
+145
@@ -0,0 +1,145 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.kafka;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.KafkaProtocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.kafka.clients.admin.AdminClient;
|
||||
import org.apache.kafka.clients.admin.KafkaAdminClient;
|
||||
import org.apache.kafka.clients.admin.NewTopic;
|
||||
import org.junit.jupiter.api.AfterAll;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.KafkaContainer;
|
||||
import org.testcontainers.containers.Network;
|
||||
import org.testcontainers.containers.output.Slf4jLogConsumer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.lifecycle.Startables;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
import org.testcontainers.utility.DockerLoggerFactory;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.ExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.TimeoutException;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
/**
|
||||
* KafkaCollectE2E
|
||||
*/
|
||||
@Slf4j
|
||||
public class KafkaCollectE2eTest {
|
||||
|
||||
private static final String ZOOKEEPER_IMAGE_NAME = "zookeeper:3.8.4";
|
||||
private static final String ZOOKEEPER_NAME = "zookeeper";
|
||||
private static final Integer ZOOKEEPER_PORT = 2181;
|
||||
private static final String KAFKA_IMAGE_NAME = "confluentinc/cp-kafka:7.4.7";
|
||||
private static final String KAFKA_NAME = "kafka";
|
||||
private static GenericContainer<?> zookeeperContainer;
|
||||
private static KafkaContainer kafkaContainer;
|
||||
private Metrics metrics;
|
||||
private KafkaCollectImpl kafkaCollect;
|
||||
private CollectRep.MetricsData.Builder builder;
|
||||
|
||||
@AfterAll
|
||||
public static void tearDown() {
|
||||
kafkaContainer.stop();
|
||||
zookeeperContainer.stop();
|
||||
}
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() {
|
||||
kafkaCollect = new KafkaCollectImpl();
|
||||
metrics = new Metrics();
|
||||
Network.NetworkImpl network = Network.builder().build();
|
||||
zookeeperContainer = new GenericContainer<>(DockerImageName.parse(ZOOKEEPER_IMAGE_NAME))
|
||||
.withExposedPorts(ZOOKEEPER_PORT)
|
||||
.withNetwork(network)
|
||||
.withNetworkAliases(ZOOKEEPER_NAME)
|
||||
.waitingFor(Wait.forListeningPort())
|
||||
.withStartupTimeout(Duration.ofSeconds(120));
|
||||
zookeeperContainer.setPortBindings(Collections.singletonList(ZOOKEEPER_PORT + ":" + ZOOKEEPER_PORT));
|
||||
|
||||
Startables.deepStart(Stream.of(zookeeperContainer)).join();
|
||||
|
||||
kafkaContainer = new KafkaContainer(DockerImageName.parse(KAFKA_IMAGE_NAME))
|
||||
.withExternalZookeeper(ZOOKEEPER_NAME + ":2181")
|
||||
.withNetwork(network)
|
||||
.withNetworkAliases(KAFKA_NAME)
|
||||
.withLogConsumer(
|
||||
new Slf4jLogConsumer(
|
||||
DockerLoggerFactory.getLogger(KAFKA_IMAGE_NAME)))
|
||||
.withStartupTimeout(Duration.ofSeconds(120));
|
||||
Startables.deepStart(Stream.of(kafkaContainer)).join();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testKafkaCollect() throws ExecutionException, InterruptedException, TimeoutException {
|
||||
|
||||
Assertions.assertTrue(zookeeperContainer.isRunning(), "Zookeeper container should be running");
|
||||
Assertions.assertTrue(kafkaContainer.isRunning(), "Kafka container should be running");
|
||||
|
||||
String topicName = "test-topic";
|
||||
|
||||
String bootstrapServers = kafkaContainer.getBootstrapServers().replace("PLAINTEXT://", "");
|
||||
KafkaProtocol kafkaProtocol = new KafkaProtocol();
|
||||
kafkaProtocol.setHost(bootstrapServers.split(":")[0]);
|
||||
kafkaProtocol.setPort(bootstrapServers.split(":")[1]);
|
||||
kafkaProtocol.setCommand("topic-list");
|
||||
metrics.setKclient(kafkaProtocol);
|
||||
log.info("bootstrapServers: {}", bootstrapServers);
|
||||
|
||||
// Create Topic
|
||||
Properties properties = new Properties();
|
||||
properties.put("bootstrap.servers", bootstrapServers);
|
||||
int numPartitions = 1;
|
||||
short replicationFactor = 1;
|
||||
try (AdminClient adminClient = KafkaAdminClient.create(properties)) {
|
||||
NewTopic newTopic = new NewTopic(topicName, numPartitions, replicationFactor);
|
||||
adminClient.createTopics(Collections.singletonList(newTopic)).all().get(60, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
// Verify the information of topic list monitoring
|
||||
builder = CollectRep.MetricsData.newBuilder();
|
||||
kafkaCollect.collect(builder, metrics);
|
||||
Assertions.assertTrue(builder.getValuesList().stream()
|
||||
.anyMatch(valueRow -> valueRow.getColumns(0).equals(topicName)));
|
||||
|
||||
// Verify the information monitored by topic description
|
||||
builder = CollectRep.MetricsData.newBuilder();
|
||||
kafkaProtocol.setCommand("topic-describe");
|
||||
kafkaCollect.collect(builder, metrics);
|
||||
List<CollectRep.ValueRow> topicDescribeList = builder.getValuesList();
|
||||
CollectRep.ValueRow firstRow = topicDescribeList.get(0);
|
||||
Assertions.assertAll(
|
||||
() -> Assertions.assertEquals(topicName, firstRow.getColumns(0)),
|
||||
() -> Assertions.assertEquals(String.valueOf(numPartitions), firstRow.getColumns(1)),
|
||||
() -> Assertions.assertEquals("0", firstRow.getColumns(2)),
|
||||
() -> Assertions.assertEquals(kafkaProtocol.getHost(), firstRow.getColumns(3)),
|
||||
() -> Assertions.assertEquals(kafkaProtocol.getPort(), firstRow.getColumns(4)),
|
||||
() -> Assertions.assertEquals(String.valueOf(replicationFactor), firstRow.getColumns(5))
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-e2e</artifactId>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hertzbeat-collector-mysql-r2dbc-e2e</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<mysql.r2dbc.reactor.bom.version>2024.0.3</mysql.r2dbc.reactor.bom.version>
|
||||
</properties>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>io.projectreactor</groupId>
|
||||
<artifactId>reactor-bom</artifactId>
|
||||
<version>${mysql.r2dbc.reactor.bom.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-startup</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
<exclusions>
|
||||
<exclusion>
|
||||
<groupId>com.mysql</groupId>
|
||||
<artifactId>mysql-connector-j</artifactId>
|
||||
</exclusion>
|
||||
<exclusion>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-webflux</artifactId>
|
||||
</exclusion>
|
||||
</exclusions>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-common-e2e</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
<type>test-jar</type>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-basic</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-common</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-collector-collector</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>io.projectreactor.netty</groupId>
|
||||
<artifactId>reactor-netty-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-mysql</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<build>
|
||||
<plugins>
|
||||
<plugin>
|
||||
<groupId>org.apache.maven.plugins</groupId>
|
||||
<artifactId>maven-surefire-plugin</artifactId>
|
||||
<configuration>
|
||||
<systemPropertyVariables>
|
||||
<!-- Force Reactor Netty onto the portable NIO path in this E2E so startup's WebFlux stack
|
||||
cannot pull the test onto a macOS-native transport variant. -->
|
||||
<reactor.netty.native>false</reactor.netty.native>
|
||||
</systemPropertyVariables>
|
||||
</configuration>
|
||||
</plugin>
|
||||
</plugins>
|
||||
</build>
|
||||
</project>
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.mysql;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import org.apache.hertzbeat.collector.collect.AbstractCollectE2eTest;
|
||||
import org.apache.hertzbeat.collector.collect.database.JdbcCommonCollect;
|
||||
import org.apache.hertzbeat.collector.collect.database.mysql.MysqlCollectorProperties;
|
||||
import org.apache.hertzbeat.collector.collect.database.mysql.MysqlJdbcDriverAvailability;
|
||||
import org.apache.hertzbeat.collector.collect.database.mysql.MysqlR2dbcJdbcQueryExecutor;
|
||||
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcConnectionFactoryProvider;
|
||||
import org.apache.hertzbeat.collector.mysql.r2dbc.MysqlR2dbcQueryExecutor;
|
||||
import org.apache.hertzbeat.collector.mysql.r2dbc.ResultSetMapper;
|
||||
import org.apache.hertzbeat.collector.mysql.r2dbc.SqlGuard;
|
||||
import org.apache.hertzbeat.collector.util.CollectUtil;
|
||||
import org.apache.hertzbeat.common.entity.job.Configmap;
|
||||
import org.apache.hertzbeat.common.entity.job.Job;
|
||||
import org.apache.hertzbeat.common.entity.job.Metrics;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.JdbcProtocol;
|
||||
import org.apache.hertzbeat.common.entity.job.protocol.Protocol;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.junit.jupiter.api.Assertions;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
/**
|
||||
* Shared MySQL-compatible E2E support for the collector-side R2DBC adapter.
|
||||
*/
|
||||
abstract class AbstractMysqlR2dbcCollectE2eTest extends AbstractCollectE2eTest {
|
||||
|
||||
protected static final String TEST_DATABASE = "hzb";
|
||||
protected static final String TEST_USERNAME = "test";
|
||||
protected static final String TEST_PASSWORD = "test123";
|
||||
protected static final String ROOT_PASSWORD = "root123";
|
||||
|
||||
protected GenericContainer<?> container;
|
||||
private MysqlR2dbcJdbcQueryExecutor jdbcQueryExecutor;
|
||||
|
||||
protected void setUpTarget(DatabaseTarget target) throws Exception {
|
||||
super.setUp();
|
||||
collect = new JdbcCommonCollect();
|
||||
metrics = new Metrics();
|
||||
|
||||
container = createContainer(target);
|
||||
container.start();
|
||||
awaitTcpLoginReady();
|
||||
initMonitoringData();
|
||||
|
||||
MysqlCollectorProperties properties = new MysqlCollectorProperties();
|
||||
properties.setQueryEngine(MysqlCollectorProperties.QueryEngine.R2DBC);
|
||||
jdbcQueryExecutor = new MysqlR2dbcJdbcQueryExecutor(
|
||||
properties,
|
||||
new MysqlR2dbcQueryExecutor(
|
||||
new MysqlR2dbcConnectionFactoryProvider(),
|
||||
new ResultSetMapper(),
|
||||
new SqlGuard()),
|
||||
new MysqlJdbcDriverAvailability());
|
||||
jdbcQueryExecutor.afterPropertiesSet();
|
||||
}
|
||||
|
||||
protected void tearDownTarget() throws Exception {
|
||||
if (jdbcQueryExecutor != null) {
|
||||
jdbcQueryExecutor.destroy();
|
||||
jdbcQueryExecutor = null;
|
||||
}
|
||||
if (container != null) {
|
||||
container.stop();
|
||||
container = null;
|
||||
}
|
||||
}
|
||||
|
||||
protected void assertMysqlJdbcDriverAbsent() {
|
||||
Assertions.assertThrows(ClassNotFoundException.class, () -> Class.forName("com.mysql.cj.jdbc.Driver"));
|
||||
}
|
||||
|
||||
protected void collectMysqlTemplate(Set<String> metricFilter) throws Exception {
|
||||
Job mysqlJob = appService.getAppDefine("mysql");
|
||||
List<Map<String, Configmap>> configmapFromPreCollectData = new LinkedList<>();
|
||||
for (Metrics metricsDef : mysqlJob.getMetrics()) {
|
||||
if (metricFilter != null && !metricFilter.contains(metricsDef.getName())) {
|
||||
continue;
|
||||
}
|
||||
metricsDef = CollectUtil.replaceCryPlaceholderToMetrics(metricsDef,
|
||||
configmapFromPreCollectData.isEmpty() ? new HashMap<>() : configmapFromPreCollectData.getFirst());
|
||||
String metricName = metricsDef.getName();
|
||||
if ("process_state".equals(metricName)) {
|
||||
startBackgroundSleepQuery();
|
||||
}
|
||||
if ("slow_sql".equals(metricName)) {
|
||||
generateSlowQuery();
|
||||
}
|
||||
CollectRep.MetricsData metricsData = validateMetricsCollection(metricsDef, metricName, true);
|
||||
configmapFromPreCollectData = CollectUtil.getConfigmapFromPreCollectData(metricsData);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
protected CollectRep.MetricsData.Builder collectMetrics(Metrics metricsDef) {
|
||||
JdbcProtocol jdbcProtocol = (JdbcProtocol) buildProtocol(metricsDef);
|
||||
metrics.setJdbc(jdbcProtocol);
|
||||
CollectRep.MetricsData.Builder metricsData = CollectRep.MetricsData.newBuilder();
|
||||
metricsData.setApp("mysql");
|
||||
return collectMetricsData(metrics, metricsDef, metricsData);
|
||||
}
|
||||
|
||||
@Override
|
||||
protected Protocol buildProtocol(Metrics metricsDef) {
|
||||
JdbcProtocol jdbcProtocol = metricsDef.getJdbc();
|
||||
jdbcProtocol.setHost(container.getHost());
|
||||
jdbcProtocol.setPort(String.valueOf(container.getMappedPort(3306)));
|
||||
jdbcProtocol.setUsername(TEST_USERNAME);
|
||||
jdbcProtocol.setPassword(TEST_PASSWORD);
|
||||
jdbcProtocol.setDatabase(TEST_DATABASE);
|
||||
jdbcProtocol.setTimeout("8000");
|
||||
jdbcProtocol.setReuseConnection("false");
|
||||
jdbcProtocol.setUrl(null);
|
||||
jdbcProtocol.setSshTunnel(null);
|
||||
return jdbcProtocol;
|
||||
}
|
||||
|
||||
private GenericContainer<?> createContainer(DatabaseTarget target) {
|
||||
GenericContainer<?> mysql = new GenericContainer<>(target.image())
|
||||
.withExposedPorts(3306)
|
||||
.waitingFor(Wait.forListeningPort());
|
||||
if (target.mariaDb()) {
|
||||
return mysql.withEnv("MARIADB_DATABASE", TEST_DATABASE)
|
||||
.withEnv("MARIADB_USER", TEST_USERNAME)
|
||||
.withEnv("MARIADB_PASSWORD", TEST_PASSWORD)
|
||||
.withEnv("MARIADB_ROOT_PASSWORD", ROOT_PASSWORD);
|
||||
}
|
||||
return mysql.withEnv("MYSQL_DATABASE", TEST_DATABASE)
|
||||
.withEnv("MYSQL_USER", TEST_USERNAME)
|
||||
.withEnv("MYSQL_PASSWORD", TEST_PASSWORD)
|
||||
.withEnv("MYSQL_ROOT_PASSWORD", ROOT_PASSWORD);
|
||||
}
|
||||
|
||||
private void initMonitoringData() throws Exception {
|
||||
execRoot("GRANT SELECT ON mysql.* TO '" + TEST_USERNAME + "'@'%';"
|
||||
+ " GRANT PROCESS ON *.* TO '" + TEST_USERNAME + "'@'%';"
|
||||
+ " SET GLOBAL log_output='TABLE';"
|
||||
+ " SET GLOBAL slow_query_log='ON';"
|
||||
+ " SET GLOBAL long_query_time=0;"
|
||||
+ " FLUSH PRIVILEGES;");
|
||||
generateSlowQuery();
|
||||
}
|
||||
|
||||
private void generateSlowQuery() throws Exception {
|
||||
execUser(TEST_DATABASE, "SELECT SLEEP(0.2);");
|
||||
Thread.sleep(300);
|
||||
}
|
||||
|
||||
private void startBackgroundSleepQuery() throws Exception {
|
||||
String command = String.join(" ",
|
||||
"CLIENT=$(command -v mysql || command -v mariadb)",
|
||||
"&&",
|
||||
"nohup sh -lc",
|
||||
"'$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
|
||||
"-u" + TEST_USERNAME,
|
||||
"-p" + TEST_PASSWORD,
|
||||
TEST_DATABASE,
|
||||
"-e",
|
||||
"\"SELECT SLEEP(15)\" >/tmp/process-state.log 2>&1'",
|
||||
">/dev/null 2>&1 &");
|
||||
container.execInContainer("sh", "-lc", command);
|
||||
Thread.sleep(500);
|
||||
}
|
||||
|
||||
private void awaitTcpLoginReady() throws Exception {
|
||||
long deadline = System.currentTimeMillis() + 30_000L;
|
||||
while (System.currentTimeMillis() < deadline) {
|
||||
try {
|
||||
var result = container.execInContainer("sh", "-lc",
|
||||
mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, TEST_DATABASE, "SELECT 1"));
|
||||
if (result.getExitCode() == 0) {
|
||||
return;
|
||||
}
|
||||
} catch (Exception ignored) {
|
||||
// Wait for the MySQL entrypoint to finish bootstrapping and switch to the final TCP listener.
|
||||
}
|
||||
Thread.sleep(1000);
|
||||
}
|
||||
throw new IllegalStateException("Timed out waiting for MySQL-compatible TCP login to become ready");
|
||||
}
|
||||
|
||||
private void execRoot(String sql) throws Exception {
|
||||
var result = container.execInContainer("sh", "-lc", mysqlCliCommand("root", ROOT_PASSWORD, "mysql", sql));
|
||||
if (result.getExitCode() != 0) {
|
||||
throw new IllegalStateException("root mysql command failed: " + result.getStderr());
|
||||
}
|
||||
}
|
||||
|
||||
private void execUser(String database, String sql) throws Exception {
|
||||
var result = container.execInContainer("sh", "-lc", mysqlCliCommand(TEST_USERNAME, TEST_PASSWORD, database, sql));
|
||||
if (result.getExitCode() != 0) {
|
||||
throw new IllegalStateException("user mysql command failed: " + result.getStderr());
|
||||
}
|
||||
}
|
||||
|
||||
private String mysqlCliCommand(String username, String password, String database, String sql) {
|
||||
return String.join(" ",
|
||||
"CLIENT=$(command -v mysql || command -v mariadb)",
|
||||
"&&",
|
||||
"$CLIENT --protocol=TCP -h127.0.0.1 -P3306",
|
||||
"-u" + username,
|
||||
"-p" + password,
|
||||
database,
|
||||
"-e",
|
||||
"\"" + sql.replace("\"", "\\\"") + "\"");
|
||||
}
|
||||
|
||||
protected record DatabaseTarget(String name, DockerImageName image, boolean mariaDb) {
|
||||
}
|
||||
}
|
||||
+61
@@ -0,0 +1,61 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.mysql;
|
||||
|
||||
import java.util.Set;
|
||||
import java.util.stream.Stream;
|
||||
import org.junit.jupiter.api.DynamicTest;
|
||||
import org.junit.jupiter.api.TestFactory;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
/**
|
||||
* Compatibility E2E coverage for the collector-side MySQL R2DBC adapter.
|
||||
*/
|
||||
class MysqlR2dbcCollectCompatibilityE2eTest extends AbstractMysqlR2dbcCollectE2eTest {
|
||||
|
||||
private static final Set<String> MARIADB_REPRESENTATIVE_METRICS =
|
||||
Set.of("basic", "process_state", "slow_sql");
|
||||
|
||||
@TestFactory
|
||||
Stream<DynamicTest> shouldCollectMysqlTemplateAcrossCompatibilityMatrixWithoutMysqlJdbcDriver() {
|
||||
return Stream.of(
|
||||
new MatrixTarget(
|
||||
new DatabaseTarget("mysql-5.7.44", DockerImageName.parse("mysql:5.7.44"), false),
|
||||
null),
|
||||
new MatrixTarget(
|
||||
new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false),
|
||||
null),
|
||||
new MatrixTarget(
|
||||
new DatabaseTarget("mariadb-11.4", DockerImageName.parse("mariadb:11.4"), true),
|
||||
MARIADB_REPRESENTATIVE_METRICS))
|
||||
.map(target -> DynamicTest.dynamicTest(target.databaseTarget().name(), () -> verifyTarget(target)));
|
||||
}
|
||||
|
||||
private void verifyTarget(MatrixTarget target) throws Exception {
|
||||
setUpTarget(target.databaseTarget());
|
||||
try {
|
||||
assertMysqlJdbcDriverAbsent();
|
||||
collectMysqlTemplate(target.metricFilter());
|
||||
} finally {
|
||||
tearDownTarget();
|
||||
}
|
||||
}
|
||||
|
||||
private record MatrixTarget(DatabaseTarget databaseTarget, Set<String> metricFilter) {
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.collector.collect.mysql;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
|
||||
/**
|
||||
* E2E test for collector-side MySQL monitoring through the R2DBC query adapter.
|
||||
*/
|
||||
@Slf4j
|
||||
class MysqlR2dbcCollectE2eTest extends AbstractMysqlR2dbcCollectE2eTest {
|
||||
|
||||
@Test
|
||||
void shouldCollectMysqlTemplateWithoutMysqlJdbcDriver() throws Exception {
|
||||
DatabaseTarget target = new DatabaseTarget("mysql-8.0.36", DockerImageName.parse("mysql:8.0.36"), false);
|
||||
setUpTarget(target);
|
||||
try {
|
||||
assertMysqlJdbcDriverAbsent();
|
||||
collectMysqlTemplate(null);
|
||||
} finally {
|
||||
tearDownTarget();
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<parent>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-e2e</artifactId>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
|
||||
<artifactId>hertzbeat-log-e2e</artifactId>
|
||||
|
||||
<properties>
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<maven.resources.plugin.version>3.3.1</maven.resources.plugin.version>
|
||||
<maven.dependency.plugin.version>3.6.1</maven.dependency.plugin.version>
|
||||
|
||||
</properties>
|
||||
|
||||
<dependencies>
|
||||
<!-- HertzBeat modules -->
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-startup</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-manager</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-log</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-alerter</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-warehouse</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- TestContainers -->
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Spring Boot Test -->
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-starter-test</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.springframework.boot</groupId>
|
||||
<artifactId>spring-boot-test-autoconfigure</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- JUnit 5 -->
|
||||
<dependency>
|
||||
<groupId>org.junit.jupiter</groupId>
|
||||
<artifactId>junit-jupiter</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- AssertJ -->
|
||||
<dependency>
|
||||
<groupId>org.assertj</groupId>
|
||||
<artifactId>assertj-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
|
||||
<!-- Awaitility -->
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
<artifactId>awaitility</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</project>
|
||||
+242
@@ -0,0 +1,242 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.log.alert;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.alert.calculate.periodic.PeriodicAlertRuleScheduler;
|
||||
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
|
||||
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.web.server.LocalServerPort;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoSpyBean;
|
||||
import org.springframework.test.context.DynamicPropertyRegistry;
|
||||
import org.springframework.test.context.DynamicPropertySource;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.Network;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.Testcontainers;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
import org.testcontainers.utility.MountableFile;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyMap;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
|
||||
/**
|
||||
* E2E tests for periodic log alert processing.
|
||||
*/
|
||||
@SpringBootTest(classes = org.apache.hertzbeat.startup.HertzBeatApplication.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@TestPropertySource(properties = {
|
||||
"warehouse.store.duckdb.enabled=false",
|
||||
"warehouse.store.greptime.enabled=true"
|
||||
})
|
||||
@Slf4j
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public class LogPeriodicAlertE2eTest {
|
||||
|
||||
private static final String VECTOR_IMAGE = "timberio/vector:latest-alpine";
|
||||
private static final int VECTOR_PORT = 8686;
|
||||
private static final String VECTOR_CONFIG_PATH = "/etc/vector/vector.yml";
|
||||
private static final String ENV_HERTZBEAT_PORT = "HERTZBEAT_PORT";
|
||||
private static final String GREPTIME_IMAGE = "greptime/greptimedb:latest";
|
||||
private static final int GREPTIME_HTTP_PORT = 4000;
|
||||
private static final int GREPTIME_GRPC_PORT = 4001;
|
||||
private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(120);
|
||||
|
||||
@LocalServerPort
|
||||
private int port;
|
||||
|
||||
@Autowired
|
||||
PeriodicAlertRuleScheduler periodicAlertRuleScheduler;
|
||||
|
||||
AlertDefine errorCountAlertByGroup;
|
||||
AlertDefine errorCountAlertByIndividual;
|
||||
|
||||
@MockitoSpyBean
|
||||
private AlarmCommonReduce alarmCommonReduce;
|
||||
|
||||
static GenericContainer<?> vector;
|
||||
|
||||
static GenericContainer<?> greptimedb;
|
||||
|
||||
static {
|
||||
greptimedb = new GenericContainer<>(DockerImageName.parse(GREPTIME_IMAGE))
|
||||
.withExposedPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT)
|
||||
.withCommand("standalone", "start",
|
||||
"--http-addr", "0.0.0.0:" + GREPTIME_HTTP_PORT,
|
||||
"--rpc-bind-addr", "0.0.0.0:" + GREPTIME_GRPC_PORT)
|
||||
.waitingFor(Wait.forListeningPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT))
|
||||
.withStartupTimeout(CONTAINER_STARTUP_TIMEOUT);
|
||||
greptimedb.start();
|
||||
}
|
||||
|
||||
@DynamicPropertySource
|
||||
static void greptimeProps(DynamicPropertyRegistry r) {
|
||||
// Configure GreptimeDB storage endpoints (dynamic ports)
|
||||
r.add("warehouse.store.greptime.http-endpoint", () -> "http://localhost:" + greptimedb.getMappedPort(GREPTIME_HTTP_PORT));
|
||||
r.add("warehouse.store.greptime.grpc-endpoints", () -> "localhost:" + greptimedb.getMappedPort(GREPTIME_GRPC_PORT));
|
||||
r.add("warehouse.store.greptime.username", () -> "");
|
||||
r.add("warehouse.store.greptime.password", () -> "");
|
||||
}
|
||||
|
||||
@BeforeAll
|
||||
void setUpAll() throws InterruptedException {
|
||||
// Setup test alert definitions
|
||||
setupTestAlertDefines();
|
||||
Testcontainers.exposeHostPorts(port);
|
||||
|
||||
// Wait for HertzBeat to be fully ready before starting Vector
|
||||
log.info("Waiting for HertzBeat to be fully ready on port {}...", port);
|
||||
Thread.sleep(5000); // Give HertzBeat time to fully initialize
|
||||
|
||||
vector = new GenericContainer<>(DockerImageName.parse(VECTOR_IMAGE))
|
||||
.withExposedPorts(VECTOR_PORT)
|
||||
.withCopyFileToContainer(MountableFile.forClasspathResource("vector.yml"), VECTOR_CONFIG_PATH)
|
||||
.withCommand("--config", "/etc/vector/vector.yml", "--verbose")
|
||||
.withLogConsumer(outputFrame -> log.info("Vector: {}", outputFrame.getUtf8String()))
|
||||
.withNetwork(Network.newNetwork())
|
||||
.withEnv(ENV_HERTZBEAT_PORT, String.valueOf(port))
|
||||
.waitingFor(Wait.forListeningPort())
|
||||
.withStartupTimeout(CONTAINER_STARTUP_TIMEOUT);
|
||||
vector.start();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testPeriodicLogAlertWithIndividualAlert() {
|
||||
|
||||
List<SingleAlert> capturedAlerts = new ArrayList<>();
|
||||
|
||||
doAnswer(invocation -> {
|
||||
SingleAlert alert = invocation.getArgument(0);
|
||||
capturedAlerts.add(alert);
|
||||
return null;
|
||||
}).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class));
|
||||
|
||||
periodicAlertRuleScheduler.updateSchedule(errorCountAlertByIndividual);
|
||||
|
||||
// Wait for periodic individual alert to be generated through AlarmCommonReduce
|
||||
await().atMost(Duration.ofSeconds(60))
|
||||
.pollInterval(Duration.ofSeconds(3))
|
||||
.untilAsserted(() -> assertFalse(capturedAlerts.isEmpty(),
|
||||
"Should have generated at least one periodic individual alert"));
|
||||
|
||||
// Verify alert properties
|
||||
SingleAlert firstAlert = capturedAlerts.get(0);
|
||||
assertNotNull(firstAlert, "First periodic alert should not be null");
|
||||
assertEquals(CommonConstants.ALERT_STATUS_FIRING, firstAlert.getStatus(), "Alert should be in firing status");
|
||||
assertNotNull(firstAlert.getLabels(), "Alert should have labels");
|
||||
assertTrue(firstAlert.getLabels().containsKey(CommonConstants.LABEL_ALERT_SEVERITY), "Alert should have severity label");
|
||||
assertEquals(CommonConstants.ALERT_SEVERITY_CRITICAL, firstAlert.getLabels().get(CommonConstants.LABEL_ALERT_SEVERITY), "Alert should have critical severity");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testPeriodicLogAlertWithGroupAlert() {
|
||||
|
||||
List<List<SingleAlert>> capturedGroupAlerts = new ArrayList<>();
|
||||
|
||||
doAnswer(invocation -> {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<SingleAlert> alerts = invocation.getArgument(1);
|
||||
capturedGroupAlerts.add(alerts);
|
||||
return null;
|
||||
}).when(alarmCommonReduce).reduceAndSendAlarmGroup(anyMap(), anyList());
|
||||
|
||||
periodicAlertRuleScheduler.updateSchedule(errorCountAlertByGroup);
|
||||
|
||||
await().atMost(Duration.ofSeconds(60))
|
||||
.pollInterval(Duration.ofSeconds(3))
|
||||
.untilAsserted(() -> {
|
||||
Optional<SingleAlert> matchedAlert = capturedGroupAlerts.stream()
|
||||
.flatMap(List::stream)
|
||||
.filter(alert -> alert.getLabels() != null)
|
||||
.filter(alert -> String.valueOf(errorCountAlertByGroup.getId())
|
||||
.equals(alert.getLabels().get(CommonConstants.LABEL_DEFINE_ID)))
|
||||
.findFirst();
|
||||
|
||||
assertTrue(matchedAlert.isPresent(), "Should have captured group alert from target alert define");
|
||||
SingleAlert anyAlert = matchedAlert.get();
|
||||
assertEquals(CommonConstants.ALERT_STATUS_FIRING, anyAlert.getStatus(), "Alert should be in firing status");
|
||||
assertEquals(CommonConstants.ALERT_SEVERITY_CRITICAL,
|
||||
anyAlert.getLabels().get(CommonConstants.LABEL_ALERT_SEVERITY),
|
||||
"Alert should have critical severity");
|
||||
assertTrue(anyAlert.getTriggerTimes() >= 1, "Alert should indicate aggregated trigger times");
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup test alert definitions for periodic log processing
|
||||
*/
|
||||
private void setupTestAlertDefines() {
|
||||
// group
|
||||
errorCountAlertByGroup = AlertDefine.builder()
|
||||
.id(10L)
|
||||
.name("periodic_error_count_alert_group")
|
||||
.type(CommonConstants.LOG_ALERT_THRESHOLD_TYPE_PERIODIC)
|
||||
.expr("SELECT COUNT(*) as error_count FROM hertzbeat_logs WHERE time_unix_nano > NOW() - INTERVAL '10 minute'")
|
||||
.period(10) // Faster schedule for tests
|
||||
.template("High error count detected: {{ error_count }} errors in last period")
|
||||
.datasource("sql")
|
||||
.enable(true)
|
||||
.build();
|
||||
|
||||
Map<String, String> errorLabelsByGroup = new HashMap<>();
|
||||
errorLabelsByGroup.put(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_CRITICAL);
|
||||
errorLabelsByGroup.put(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_GROUP);
|
||||
errorLabelsByGroup.put("type", "error_count");
|
||||
errorCountAlertByGroup.setLabels(errorLabelsByGroup);
|
||||
|
||||
// individual
|
||||
errorCountAlertByIndividual = AlertDefine.builder()
|
||||
.id(11L)
|
||||
.name("periodic_error_count_alert_individual")
|
||||
.type(CommonConstants.LOG_ALERT_THRESHOLD_TYPE_PERIODIC)
|
||||
.expr("SELECT COUNT(*) as error_count, severity_text FROM hertzbeat_logs "
|
||||
+ "WHERE severity_text = 'ERROR' AND time_unix_nano > NOW() - INTERVAL '5 minute' GROUP BY severity_text HAVING COUNT(*) > 2")
|
||||
.period(10) // Faster schedule for tests
|
||||
.template("High error count detected: {{ error_count }} errors in last period")
|
||||
.datasource("sql")
|
||||
.enable(true)
|
||||
.build();
|
||||
|
||||
Map<String, String> errorLabelsByIndividual = new HashMap<>();
|
||||
errorLabelsByIndividual.put(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_CRITICAL);
|
||||
errorLabelsByIndividual.put(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL);
|
||||
errorLabelsByIndividual.put("type", "error_count");
|
||||
errorCountAlertByIndividual.setLabels(errorLabelsByIndividual);
|
||||
}
|
||||
}
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.log.alert;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.alert.reduce.AlarmCommonReduce;
|
||||
import org.apache.hertzbeat.common.cache.CacheFactory;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.alerter.AlertDefine;
|
||||
import org.apache.hertzbeat.common.entity.alerter.SingleAlert;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.web.server.LocalServerPort;
|
||||
import org.springframework.test.context.bean.override.mockito.MockitoSpyBean;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.Network;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.Testcontainers;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
import org.testcontainers.utility.MountableFile;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyList;
|
||||
import static org.mockito.ArgumentMatchers.anyMap;
|
||||
import static org.mockito.Mockito.doAnswer;
|
||||
|
||||
/**
|
||||
* E2E tests for real-time log alert processing.
|
||||
*/
|
||||
@SpringBootTest(classes = org.apache.hertzbeat.startup.HertzBeatApplication.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@Slf4j
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public class LogRealTimeAlertE2eTest {
|
||||
|
||||
private static final String VECTOR_IMAGE = "timberio/vector:latest-alpine";
|
||||
private static final int VECTOR_PORT = 8686;
|
||||
private static final String VECTOR_CONFIG_PATH = "/etc/vector/vector.yml";
|
||||
private static final String ENV_HERTZBEAT_PORT = "HERTZBEAT_PORT";
|
||||
private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(180);
|
||||
private static final Duration TEST_WAIT_TIMEOUT = Duration.ofSeconds(120);
|
||||
|
||||
@LocalServerPort
|
||||
private int port;
|
||||
|
||||
private final List<SingleAlert> capturedAlerts = new ArrayList<>();
|
||||
private final ArrayList<List<SingleAlert>> capturedGroupAlerts = new ArrayList<>();
|
||||
|
||||
@MockitoSpyBean
|
||||
private AlarmCommonReduce alarmCommonReduce;
|
||||
|
||||
static GenericContainer<?> vector;
|
||||
|
||||
@BeforeAll
|
||||
void setUpAll() throws InterruptedException {
|
||||
// Setup test alert definitions
|
||||
setupTestAlertDefines();
|
||||
|
||||
// Expose host ports for testcontainers
|
||||
Testcontainers.exposeHostPorts(port);
|
||||
|
||||
// Wait for HertzBeat to be fully ready before starting Vector
|
||||
log.info("Waiting for HertzBeat to be fully ready on port {}...", port);
|
||||
Thread.sleep(5000); // Give HertzBeat time to fully initialize
|
||||
|
||||
vector = new GenericContainer<>(DockerImageName.parse(VECTOR_IMAGE))
|
||||
.withExposedPorts(VECTOR_PORT)
|
||||
.withCopyFileToContainer(MountableFile.forClasspathResource("vector.yml"), VECTOR_CONFIG_PATH)
|
||||
.withCommand("--config", "/etc/vector/vector.yml", "--verbose")
|
||||
.withLogConsumer(outputFrame -> log.info("Vector: {}", outputFrame.getUtf8String()))
|
||||
.withNetwork(Network.newNetwork())
|
||||
.withEnv(ENV_HERTZBEAT_PORT, String.valueOf(port))
|
||||
.waitingFor(Wait.forListeningPort())
|
||||
.withStartupTimeout(CONTAINER_STARTUP_TIMEOUT);
|
||||
vector.start();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRealTimeLogAlertWithIndividualAlert() {
|
||||
doAnswer(invocation -> {
|
||||
SingleAlert alert = invocation.getArgument(0);
|
||||
capturedAlerts.add(alert);
|
||||
return null;
|
||||
}).when(alarmCommonReduce).reduceAndSendAlarm(any(SingleAlert.class));
|
||||
// Clear any existing captured alerts
|
||||
capturedAlerts.clear();
|
||||
|
||||
// Wait for real alert to be generated through AlarmCommonReduce
|
||||
await().atMost(TEST_WAIT_TIMEOUT)
|
||||
.pollInterval(Duration.ofSeconds(3))
|
||||
.untilAsserted(() -> assertFalse(capturedAlerts.isEmpty(),
|
||||
"Should have generated at least one alert for error logs"));
|
||||
|
||||
// Verify alert properties
|
||||
SingleAlert firstAlert = capturedAlerts.get(0);
|
||||
assertNotNull(firstAlert, "First alert should not be null");
|
||||
assertEquals(CommonConstants.ALERT_STATUS_FIRING, firstAlert.getStatus(), "Alert should be in firing status");
|
||||
assertNotNull(firstAlert.getLabels(), "Alert should have labels");
|
||||
assertTrue(firstAlert.getLabels().containsKey(CommonConstants.LABEL_ALERT_SEVERITY), "Alert should have severity label");
|
||||
assertEquals(CommonConstants.ALERT_SEVERITY_CRITICAL, firstAlert.getLabels().get(CommonConstants.LABEL_ALERT_SEVERITY), "Alert should have critical severity");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testRealTimeLogAlertWithGroupAlert() {
|
||||
doAnswer(invocation -> {
|
||||
@SuppressWarnings("unchecked")
|
||||
List<SingleAlert> alerts = invocation.getArgument(1);
|
||||
capturedGroupAlerts.add(alerts);
|
||||
return null;
|
||||
}).when(alarmCommonReduce).reduceAndSendAlarmGroup(anyMap(), anyList());
|
||||
|
||||
// Clear any existing captured alerts
|
||||
capturedGroupAlerts.clear();
|
||||
|
||||
// Wait for group alert to be generated via AlarmCommonReduce
|
||||
await().atMost(TEST_WAIT_TIMEOUT)
|
||||
.pollInterval(Duration.ofSeconds(3))
|
||||
.untilAsserted(() -> assertFalse(capturedGroupAlerts.isEmpty(),
|
||||
"Should have generated high frequency warning group alert"));
|
||||
|
||||
List<SingleAlert> groupAlerts = capturedGroupAlerts.get(0);
|
||||
assertNotNull(groupAlerts, "Group alerts should not be null");
|
||||
assertTrue(groupAlerts.size() >= 2, "Group should contain at least 5 alerts");
|
||||
|
||||
SingleAlert anyAlert = groupAlerts.get(0);
|
||||
assertEquals(CommonConstants.ALERT_STATUS_FIRING, anyAlert.getStatus(), "Alert should be in firing status");
|
||||
assertNotNull(anyAlert.getLabels(), "Alert should have labels");
|
||||
assertEquals(CommonConstants.ALERT_SEVERITY_WARNING, anyAlert.getLabels().get(CommonConstants.LABEL_ALERT_SEVERITY), "Alert should have warning severity");
|
||||
assertTrue(anyAlert.getTriggerTimes() >= 2, "Alert should indicate high frequency trigger");
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup test alert definitions for real-time log processing
|
||||
*/
|
||||
private void setupTestAlertDefines() {
|
||||
List<AlertDefine> alertDefines = new ArrayList<>();
|
||||
|
||||
// Create error log alert definition
|
||||
AlertDefine errorLogAlert = AlertDefine.builder()
|
||||
.id(1L)
|
||||
.name("error_log_alert")
|
||||
.type(CommonConstants.LOG_ALERT_THRESHOLD_TYPE_REALTIME)
|
||||
.expr("log.severityText == 'ERROR'")
|
||||
.period(10) // 10 seconds window
|
||||
.times(1) // Trigger on first occurrence
|
||||
.template("Error detected: {{ body }}")
|
||||
.enable(true)
|
||||
.build();
|
||||
|
||||
Map<String, String> errorLabels = new HashMap<>();
|
||||
errorLabels.put(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_CRITICAL);
|
||||
errorLabels.put(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_INDIVIDUAL);
|
||||
errorLogAlert.setLabels(errorLabels);
|
||||
|
||||
alertDefines.add(errorLogAlert);
|
||||
|
||||
// Create high frequency warning alert definition
|
||||
AlertDefine highFrequencyWarning = AlertDefine.builder()
|
||||
.id(2L)
|
||||
.name("high_frequency_warning")
|
||||
.type(CommonConstants.LOG_ALERT_THRESHOLD_TYPE_REALTIME)
|
||||
.expr("log.severityText == 'ERROR'")
|
||||
.period(10)
|
||||
.times(2)
|
||||
.template("High frequency warnings detected: {{ __rows__ }} warnings in 30 seconds")
|
||||
.enable(true)
|
||||
.build();
|
||||
|
||||
Map<String, String> warningLabels = new HashMap<>();
|
||||
warningLabels.put(CommonConstants.LABEL_ALERT_SEVERITY, CommonConstants.ALERT_SEVERITY_WARNING);
|
||||
warningLabels.put(CommonConstants.ALERT_MODE_LABEL, CommonConstants.ALERT_MODE_GROUP);
|
||||
highFrequencyWarning.setLabels(warningLabels);
|
||||
|
||||
alertDefines.add(highFrequencyWarning);
|
||||
|
||||
// Set alert definitions in cache
|
||||
CacheFactory.setLogAlertDefineCache(alertDefines);
|
||||
}
|
||||
|
||||
}
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.log.ingestion;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.queue.CommonDataQueue;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.web.server.LocalServerPort;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.Network;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.Testcontainers;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
import org.testcontainers.utility.MountableFile;
|
||||
|
||||
import java.time.Duration;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
/**
|
||||
* E2E tests for log ingestion.
|
||||
*/
|
||||
@SpringBootTest(classes = org.apache.hertzbeat.startup.HertzBeatApplication.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@Slf4j
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public class LogIngestionE2eTest {
|
||||
|
||||
private static final String VECTOR_IMAGE = "timberio/vector:latest-alpine";
|
||||
private static final int VECTOR_PORT = 8686;
|
||||
private static final String VECTOR_CONFIG_PATH = "/etc/vector/vector.yml";
|
||||
private static final String ENV_HERTZBEAT_PORT = "HERTZBEAT_PORT";
|
||||
private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(120);
|
||||
|
||||
@LocalServerPort
|
||||
private int port;
|
||||
|
||||
@Autowired
|
||||
private CommonDataQueue commonDataQueue;
|
||||
|
||||
static GenericContainer<?> vector;
|
||||
|
||||
@BeforeAll
|
||||
void setUpAll() throws InterruptedException {
|
||||
Testcontainers.exposeHostPorts(port);
|
||||
|
||||
// Wait for HertzBeat to be fully ready before starting Vector
|
||||
log.info("Waiting for HertzBeat to be fully ready on port {}...", port);
|
||||
Thread.sleep(5000); // Give HertzBeat time to fully initialize
|
||||
|
||||
vector = new GenericContainer<>(DockerImageName.parse(VECTOR_IMAGE))
|
||||
.withExposedPorts(VECTOR_PORT)
|
||||
.withCopyFileToContainer(MountableFile.forClasspathResource("vector.yml"), VECTOR_CONFIG_PATH)
|
||||
.withCommand("--config", "/etc/vector/vector.yml", "--verbose")
|
||||
.withLogConsumer(outputFrame -> log.info("Vector: {}", outputFrame.getUtf8String()))
|
||||
.withNetwork(Network.newNetwork())
|
||||
.withEnv(ENV_HERTZBEAT_PORT, String.valueOf(port))
|
||||
.waitingFor(Wait.forListeningPort())
|
||||
.withStartupTimeout(CONTAINER_STARTUP_TIMEOUT);
|
||||
vector.start();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLogIngestion() {
|
||||
|
||||
// Start polling for log entries from the queue
|
||||
List<LogEntry> capturedLogs = new ArrayList<>();
|
||||
|
||||
// Wait for Vector to generate and send demo logs to HertzBeat
|
||||
await().atMost(Duration.ofSeconds(60))
|
||||
.pollInterval(Duration.ofSeconds(3))
|
||||
.untilAsserted(() -> {
|
||||
try {
|
||||
LogEntry logEntry = commonDataQueue.pollLogEntry();
|
||||
if (logEntry != null) {
|
||||
capturedLogs.add(logEntry);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new RuntimeException("Test interrupted", e);
|
||||
}
|
||||
});
|
||||
|
||||
// Verify the captured logs
|
||||
assertFalse(capturedLogs.isEmpty(), "No logs were captured from Vector");
|
||||
LogEntry firstLog = capturedLogs.get(0);
|
||||
assertNotNull(firstLog, "First log should not be null");
|
||||
assertNotNull(firstLog.getBody(), "Log body should not be null");
|
||||
assertNotNull(firstLog.getSeverityText(), "Severity text should not be null");
|
||||
}
|
||||
}
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.log.storage;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.queue.CommonDataQueue;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeDbDataStorage;
|
||||
import org.junit.jupiter.api.BeforeAll;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.TestInstance;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.boot.test.web.server.LocalServerPort;
|
||||
import org.springframework.test.context.DynamicPropertyRegistry;
|
||||
import org.springframework.test.context.DynamicPropertySource;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import org.testcontainers.containers.GenericContainer;
|
||||
import org.testcontainers.containers.Network;
|
||||
import org.testcontainers.containers.wait.strategy.Wait;
|
||||
import org.testcontainers.Testcontainers;
|
||||
import org.testcontainers.utility.DockerImageName;
|
||||
import org.testcontainers.utility.MountableFile;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
|
||||
/**
|
||||
* E2E tests for GreptimeDB log storage.
|
||||
*/
|
||||
@SpringBootTest(classes = org.apache.hertzbeat.startup.HertzBeatApplication.class, webEnvironment = SpringBootTest.WebEnvironment.RANDOM_PORT)
|
||||
@TestPropertySource(properties = {
|
||||
"warehouse.store.duckdb.enabled=false",
|
||||
"warehouse.store.greptime.enabled=true"
|
||||
})
|
||||
@Slf4j
|
||||
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
|
||||
public class GreptimeLogStorageE2eTest {
|
||||
|
||||
private static final String VECTOR_IMAGE = "timberio/vector:latest-alpine";
|
||||
private static final int VECTOR_PORT = 8686;
|
||||
private static final String VECTOR_CONFIG_PATH = "/etc/vector/vector.yml";
|
||||
private static final String ENV_HERTZBEAT_PORT = "HERTZBEAT_PORT";
|
||||
private static final String GREPTIME_IMAGE = "greptime/greptimedb:latest";
|
||||
private static final int GREPTIME_HTTP_PORT = 4000;
|
||||
private static final int GREPTIME_GRPC_PORT = 4001;
|
||||
private static final Duration CONTAINER_STARTUP_TIMEOUT = Duration.ofSeconds(120);
|
||||
|
||||
@LocalServerPort
|
||||
private int port;
|
||||
|
||||
@Autowired
|
||||
private CommonDataQueue commonDataQueue;
|
||||
|
||||
@Autowired
|
||||
private GreptimeDbDataStorage greptimeDbDataStorage;
|
||||
|
||||
static GenericContainer<?> vector;
|
||||
static GenericContainer<?> greptimedb;
|
||||
|
||||
static {
|
||||
greptimedb = new GenericContainer<>(DockerImageName.parse(GREPTIME_IMAGE))
|
||||
.withExposedPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT)
|
||||
.withCommand("standalone", "start",
|
||||
"--http-addr", "0.0.0.0:" + GREPTIME_HTTP_PORT,
|
||||
"--rpc-bind-addr", "0.0.0.0:" + GREPTIME_GRPC_PORT)
|
||||
.waitingFor(Wait.forListeningPorts(GREPTIME_HTTP_PORT, GREPTIME_GRPC_PORT))
|
||||
.withStartupTimeout(CONTAINER_STARTUP_TIMEOUT);
|
||||
greptimedb.start();
|
||||
}
|
||||
|
||||
@DynamicPropertySource
|
||||
static void greptimeProps(DynamicPropertyRegistry r) {
|
||||
r.add("warehouse.store.greptime.http-endpoint", () -> "http://localhost:" + greptimedb.getMappedPort(GREPTIME_HTTP_PORT));
|
||||
r.add("warehouse.store.greptime.grpc-endpoints", () -> "localhost:" + greptimedb.getMappedPort(GREPTIME_GRPC_PORT));
|
||||
r.add("warehouse.store.greptime.username", () -> "");
|
||||
r.add("warehouse.store.greptime.password", () -> "");
|
||||
}
|
||||
|
||||
|
||||
@BeforeAll
|
||||
void setUpAll() throws InterruptedException {
|
||||
// Expose host ports for testcontainers
|
||||
Testcontainers.exposeHostPorts(port);
|
||||
|
||||
// Wait for HertzBeat to be fully ready before starting Vector
|
||||
log.info("Waiting for HertzBeat to be fully ready on port {}...", port);
|
||||
Thread.sleep(5000); // Give HertzBeat time to fully initialize
|
||||
|
||||
vector = new GenericContainer<>(DockerImageName.parse(VECTOR_IMAGE))
|
||||
.withExposedPorts(VECTOR_PORT)
|
||||
.withCopyFileToContainer(MountableFile.forClasspathResource("vector.yml"), VECTOR_CONFIG_PATH)
|
||||
.withCommand("--config", "/etc/vector/vector.yml", "--verbose")
|
||||
.withLogConsumer(outputFrame -> log.info("Vector: {}", outputFrame.getUtf8String()))
|
||||
.withNetwork(Network.newNetwork())
|
||||
.withEnv(ENV_HERTZBEAT_PORT, String.valueOf(port))
|
||||
.waitingFor(Wait.forListeningPort())
|
||||
.withStartupTimeout(CONTAINER_STARTUP_TIMEOUT);
|
||||
vector.start();
|
||||
}
|
||||
|
||||
@Test
|
||||
void testLogStorageToGreptimeDb() {
|
||||
log.info("GreptimeDbDataStorage serverAvailable: {}", greptimeDbDataStorage.isServerAvailable());
|
||||
|
||||
List<LogEntry> capturedLogs = new ArrayList<>();
|
||||
|
||||
// Wait for Vector to generate and send logs to HertzBeat
|
||||
await().atMost(Duration.ofSeconds(30))
|
||||
.pollInterval(Duration.ofSeconds(3))
|
||||
.untilAsserted(() -> {
|
||||
// Poll log entries from the queue (non-blocking)
|
||||
try {
|
||||
LogEntry logEntry = commonDataQueue.pollLogEntry();
|
||||
if (logEntry != null) {
|
||||
capturedLogs.add(logEntry);
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
throw new RuntimeException("Test interrupted", e);
|
||||
}
|
||||
|
||||
// Assert that we have captured at least some logs
|
||||
assertFalse(capturedLogs.isEmpty(), "Should have captured at least one log entry");
|
||||
});
|
||||
|
||||
// Verify the captured logs
|
||||
assertFalse(capturedLogs.isEmpty(), "No logs were captured from Vector");
|
||||
LogEntry firstLog = capturedLogs.get(0);
|
||||
assertNotNull(firstLog, "First log should not be null");
|
||||
assertNotNull(firstLog.getBody(), "Log body should not be null");
|
||||
assertNotNull(firstLog.getSeverityText(), "Severity text should not be null");
|
||||
|
||||
// Directly write logs to GreptimeDB to test storage functionality
|
||||
log.info("Directly writing {} captured logs to GreptimeDB", capturedLogs.size());
|
||||
greptimeDbDataStorage.saveLogDataBatch(capturedLogs);
|
||||
|
||||
// Give some time for the write to complete
|
||||
try {
|
||||
Thread.sleep(2000);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
|
||||
// Additional wait to ensure logs are persisted to GreptimeDB
|
||||
await().atMost(Duration.ofSeconds(30))
|
||||
.pollInterval(Duration.ofSeconds(2))
|
||||
.untilAsserted(() -> {
|
||||
// Query GreptimeDB directly to verify data persistence
|
||||
List<LogEntry> storedLogs = queryStoredLogs();
|
||||
log.info("Queried {} logs from GreptimeDB", storedLogs.size());
|
||||
assertFalse(storedLogs.isEmpty(), "Should have logs stored in GreptimeDB");
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to query stored logs directly from GreptimeDB
|
||||
*/
|
||||
private List<LogEntry> queryStoredLogs() {
|
||||
// Query without time condition to verify data exists
|
||||
List<LogEntry> result = greptimeDbDataStorage.queryLogsByMultipleConditions(
|
||||
null, null, null, null, null, null, null);
|
||||
log.info("queryLogsByMultipleConditions returned {} entries", result.size());
|
||||
return result;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
## -- sureness.yml account source -- ##
|
||||
|
||||
# config the resource restful api that need auth protection, base rbac
|
||||
# rule: api===method===role
|
||||
# eg: /api/v1/source1===get===[admin] means /api/v2/host===post support role[admin] access.
|
||||
# eg: /api/v1/source2===get===[] means /api/v1/source2===get can not access by any role.
|
||||
resourceRole:
|
||||
- /api/account/auth/refresh===post===[admin,user,guest]
|
||||
- /api/apps/**===get===[admin,user,guest]
|
||||
- /api/monitor/**===get===[admin,user,guest]
|
||||
- /api/monitor/**===post===[admin,user]
|
||||
- /api/monitor/**===put===[admin,user]
|
||||
- /api/monitor/**===delete==[admin]
|
||||
- /api/monitors/**===get===[admin,user,guest]
|
||||
- /api/monitors/**===post===[admin,user]
|
||||
- /api/monitors/**===put===[admin,user]
|
||||
- /api/monitors/**===delete===[admin]
|
||||
- /api/alert/**===get===[admin,user,guest]
|
||||
- /api/alert/**===post===[admin,user]
|
||||
- /api/alert/**===put===[admin,user]
|
||||
- /api/alert/**===delete===[admin]
|
||||
- /api/alerts/**===get===[admin,user,guest]
|
||||
- /api/alerts/**===post===[admin,user]
|
||||
- /api/alerts/**===put===[admin,user]
|
||||
- /api/alerts/**===delete===[admin]
|
||||
- /api/notice/**===get===[admin,user,guest]
|
||||
- /api/notice/**===post===[admin,user]
|
||||
- /api/notice/**===put===[admin,user]
|
||||
- /api/notice/**===delete===[admin]
|
||||
- /api/tag/**===get===[admin,user,guest]
|
||||
- /api/tag/**===post===[admin,user]
|
||||
- /api/tag/**===put===[admin,user]
|
||||
- /api/tag/**===delete===[admin]
|
||||
- /api/summary/**===get===[admin,user,guest]
|
||||
- /api/summary/**===post===[admin,user]
|
||||
- /api/summary/**===put===[admin,user]
|
||||
- /api/summary/**===delete===[admin]
|
||||
- /api/collector/**===get===[admin,user,guest]
|
||||
- /api/collector/**===post===[admin,user]
|
||||
- /api/collector/**===put===[admin,user]
|
||||
- /api/collector/**===delete===[admin]
|
||||
- /api/status/page/**===get===[admin,user,guest]
|
||||
- /api/status/page/**===post===[admin,user]
|
||||
- /api/status/page/**===put===[admin,user]
|
||||
- /api/status/page/**===delete===[admin]
|
||||
- /api/grafana/**===get===[admin,user,guest]
|
||||
- /api/grafana/**===post===[admin,user]
|
||||
- /api/grafana/**===put===[admin,user]
|
||||
- /api/grafana/**===delete===[admin]
|
||||
- /api/bulletin/**===get===[admin,user,guest]
|
||||
- /api/bulletin/**===post===[admin,user]
|
||||
- /api/bulletin/**===put===[admin,user]
|
||||
- /api/bulletin/**===delete===[admin]
|
||||
- /api/sse/**===get===[admin,user]
|
||||
- /api/sse/**===post===[admin,user]
|
||||
- /api/chat/**===get===[admin,user]
|
||||
- /api/chat/**===post===[admin,user]
|
||||
- /api/logs/ingest/**===post===[admin,user]
|
||||
|
||||
# config the resource restful api that need bypass auth protection
|
||||
# rule: api===method
|
||||
# eg: /api/v1/source3===get means /api/v1/source3===get can be access by anyone, no need auth.
|
||||
excludedResource:
|
||||
- /api/alert/sse/**===*
|
||||
- /api/logs/sse/**===*
|
||||
- /api/account/auth/**===*
|
||||
- /api/i18n/**===get
|
||||
- /api/apps/hierarchy===get
|
||||
- /api/push/**===*
|
||||
- /api/status/page/public/**===*
|
||||
- /api/manager/sse/**===*
|
||||
# web ui resource
|
||||
- /===get
|
||||
- /assets/**===get
|
||||
- /dashboard/**===get
|
||||
- /monitors/**===get
|
||||
- /alert/**===get
|
||||
- /account/**===get
|
||||
- /setting/**===get
|
||||
- /passport/**===get
|
||||
- /status/**===get
|
||||
- /**/*.html===get
|
||||
- /**/*.js===get
|
||||
- /**/*.css===get
|
||||
- /**/*.ico===get
|
||||
- /**/*.ttf===get
|
||||
- /**/*.png===get
|
||||
- /**/*.gif===get
|
||||
- /**/*.jpg===get
|
||||
- /**/*.svg===get
|
||||
- /**/*.json===get
|
||||
- /**/*.woff===get
|
||||
- /**/*.eot===get
|
||||
# swagger ui resource
|
||||
- /swagger-resources/**===get
|
||||
- /v2/api-docs===get
|
||||
- /v3/api-docs===get
|
||||
# h2 database
|
||||
- /h2-console/**===*
|
||||
|
||||
# account info config
|
||||
# eg: admin has role [admin,user], password is hertzbeat
|
||||
# eg: tom has role [user], password is hertzbeat
|
||||
# eg: lili has role [guest], plain password is lili, salt is 123, salted password is 1A676730B0C7F54654B0E09184448289
|
||||
account:
|
||||
- appId: admin
|
||||
credential: hertzbeat
|
||||
role: [admin]
|
||||
- appId: tom
|
||||
credential: hertzbeat
|
||||
role: [user]
|
||||
- appId: guest
|
||||
credential: hertzbeat
|
||||
role: [guest]
|
||||
- appId: lili
|
||||
# credential = MD5(password + salt)
|
||||
# plain password: hertzbeat
|
||||
# attention: digest authentication does not support salted encrypted password accounts
|
||||
credential: 94C6B34E7A199A9F9D4E1F208093B489
|
||||
salt: 123
|
||||
role: [user]
|
||||
@@ -0,0 +1,126 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
# Vector configuration for HertzBeat E2E testing
|
||||
|
||||
api:
|
||||
enabled: true
|
||||
address: "0.0.0.0:8686"
|
||||
|
||||
sources:
|
||||
generate_syslog:
|
||||
type: "demo_logs"
|
||||
format: "syslog"
|
||||
count: 100000
|
||||
interval: 1
|
||||
|
||||
transforms:
|
||||
remap_syslog:
|
||||
inputs: ["generate_syslog"]
|
||||
type: "remap"
|
||||
source: |
|
||||
syslog = parse_syslog!(.message)
|
||||
|
||||
severity_text = if includes(["emerg", "err", "crit", "alert"], syslog.severity) {
|
||||
"ERROR"
|
||||
} else if syslog.severity == "warning" {
|
||||
"WARN"
|
||||
} else if syslog.severity == "debug" {
|
||||
"DEBUG"
|
||||
} else if includes(["info", "notice"], syslog.severity) {
|
||||
"INFO"
|
||||
} else {
|
||||
syslog.severity
|
||||
}
|
||||
|
||||
severity_number = if includes(["emerg", "err", "crit", "alert"], syslog.severity) {
|
||||
17
|
||||
} else if syslog.severity == "warning" {
|
||||
13
|
||||
} else if syslog.severity == "debug" {
|
||||
5
|
||||
} else if includes(["info", "notice"], syslog.severity) {
|
||||
9
|
||||
} else {
|
||||
9
|
||||
}
|
||||
|
||||
.resourceLogs = [{
|
||||
"resource": {
|
||||
"attributes": [
|
||||
{ "key": "source_type", "value": { "stringValue": .source_type } },
|
||||
{ "key": "service.name", "value": { "stringValue": syslog.appname } },
|
||||
{ "key": "host.hostname", "value": { "stringValue": syslog.hostname } }
|
||||
]
|
||||
},
|
||||
"scopeLogs": [{
|
||||
"scope": {
|
||||
"name": syslog.msgid
|
||||
},
|
||||
"logRecords": [{
|
||||
"timeUnixNano": to_unix_timestamp!(syslog.timestamp, unit: "nanoseconds"),
|
||||
"body": { "stringValue": syslog.message },
|
||||
"severityText": severity_text,
|
||||
"severityNumber": severity_number,
|
||||
"attributes": [
|
||||
{ "key": "syslog.procid", "value": { "stringValue": to_string(syslog.procid) } },
|
||||
{ "key": "syslog.facility", "value": { "stringValue": syslog.facility } },
|
||||
{ "key": "syslog.version", "value": { "stringValue": to_string(syslog.version) } }
|
||||
]
|
||||
}]
|
||||
}]
|
||||
}]
|
||||
|
||||
del(.message)
|
||||
del(.timestamp)
|
||||
del(.service)
|
||||
del(.source_type)
|
||||
|
||||
sinks:
|
||||
# Debug sink to log all processed data
|
||||
debug_console:
|
||||
inputs: [ "remap_syslog" ]
|
||||
type: console
|
||||
encoding:
|
||||
codec: json
|
||||
|
||||
# Send to HertzBeat with increased timeout and retry
|
||||
emit_syslog:
|
||||
inputs: [ "remap_syslog" ]
|
||||
type: opentelemetry
|
||||
protocol:
|
||||
type: http
|
||||
uri: "http://host.testcontainers.internal:${HERTZBEAT_PORT:-1157}/api/logs/ingest/otlp"
|
||||
method: post
|
||||
encoding:
|
||||
codec: json
|
||||
framing:
|
||||
method: newline_delimited
|
||||
auth:
|
||||
strategy: basic
|
||||
user: admin
|
||||
password: hertzbeat
|
||||
# Increase timeout and retry settings for stability in CI environments
|
||||
request:
|
||||
headers:
|
||||
content-type: application/json
|
||||
timeout_secs: 60
|
||||
retry_attempts: 10
|
||||
retry_initial_backoff_secs: 2
|
||||
retry_max_duration_secs: 120
|
||||
# Batch settings for better throughput
|
||||
batch:
|
||||
max_bytes: 524288
|
||||
timeout_secs: 1
|
||||
@@ -0,0 +1,92 @@
|
||||
<?xml version="1.0" encoding="UTF-8"?>
|
||||
<!--
|
||||
~ Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
~ contributor license agreements. See the NOTICE file distributed with
|
||||
~ this work for additional information regarding copyright ownership.
|
||||
~ The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
~ (the "License"); you may not use this file except in compliance with
|
||||
~ the License. You may obtain a copy of the License at
|
||||
~
|
||||
~ http://www.apache.org/licenses/LICENSE-2.0
|
||||
~
|
||||
~ Unless required by applicable law or agreed to in writing, software
|
||||
~ distributed under the License is distributed on an "AS IS" BASIS,
|
||||
~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
~ See the License for the specific language governing permissions and
|
||||
~ limitations under the License.
|
||||
-->
|
||||
<project xmlns="http://maven.apache.org/POM/4.0.0"
|
||||
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
|
||||
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
|
||||
<modelVersion>4.0.0</modelVersion>
|
||||
<parent>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat</artifactId>
|
||||
<version>2.0-SNAPSHOT</version>
|
||||
</parent>
|
||||
|
||||
<artifactId>hertzbeat-e2e</artifactId>
|
||||
<packaging>pom</packaging>
|
||||
<modules>
|
||||
<module>hertzbeat-collector-common-e2e</module>
|
||||
<module>hertzbeat-collector-kafka-e2e</module>
|
||||
<module>hertzbeat-collector-basic-e2e</module>
|
||||
<module>hertzbeat-collector-mysql-r2dbc-e2e</module>
|
||||
<module>hertzbeat-log-e2e</module>
|
||||
</modules>
|
||||
|
||||
<properties>
|
||||
<maven.test.skip>true</maven.test.skip>
|
||||
<maven.compiler.source>${java.version}</maven.compiler.source>
|
||||
<maven.compiler.target>${java.version}</maven.compiler.target>
|
||||
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
|
||||
<testcontainers.version>2.0.3</testcontainers.version>
|
||||
<junit.version>4.13.2</junit.version>
|
||||
<awaitility.version>4.2.0</awaitility.version>
|
||||
</properties>
|
||||
|
||||
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.mockito</groupId>
|
||||
<artifactId>mockito-core</artifactId>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>junit</groupId>
|
||||
<artifactId>junit</artifactId>
|
||||
<version>${junit.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.awaitility</groupId>
|
||||
<artifactId>awaitility</artifactId>
|
||||
<version>${awaitility.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
<dependency>
|
||||
<groupId>org.apache.hertzbeat</groupId>
|
||||
<artifactId>hertzbeat-startup</artifactId>
|
||||
<version>${hertzbeat.version}</version>
|
||||
<scope>test</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
|
||||
<dependencyManagement>
|
||||
<dependencies>
|
||||
<dependency>
|
||||
<groupId>org.testcontainers</groupId>
|
||||
<artifactId>testcontainers-bom</artifactId>
|
||||
<version>${testcontainers.version}</version>
|
||||
<type>pom</type>
|
||||
<scope>import</scope>
|
||||
</dependency>
|
||||
</dependencies>
|
||||
</dependencyManagement>
|
||||
|
||||
</project>
|
||||
Reference in New Issue
Block a user