chore: import upstream snapshot with attribution
MCP Bash Server CI / Test MCP Bash Server (dev) (push) Has been cancelled
MCP Bash Server CI / Test MCP Bash Server (release) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 13:11:39 +08:00
commit c8cebdfeee
4654 changed files with 626756 additions and 0 deletions
@@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.SynchronousQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
import org.apache.hertzbeat.common.concurrent.ManagedExecutors;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.springframework.beans.factory.DisposableBean;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
/**
* warehouse worker thread pool
*/
@Component
@Slf4j
public class WarehouseWorkerPool implements DisposableBean {
private final ManagedExecutor workerExecutor;
private final ManagedExecutor longRunningExecutor;
public WarehouseWorkerPool() {
this(VirtualThreadProperties.defaults());
}
@Autowired
public WarehouseWorkerPool(VirtualThreadProperties virtualThreadProperties) {
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
this.workerExecutor = createWorkerExecutor(properties);
this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor);
}
private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) {
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
log.error("Warehouse workerExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
};
if (properties.enabled()) {
VirtualThreadProperties.PoolProperties poolProperties = properties.warehouse();
return ManagedExecutors.newVirtualExecutor("warehouse-worker", "warehouse-worker-",
poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler);
}
return ManagedExecutors.wrap("warehouse-worker", createLegacyExecutor(handler));
}
private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) {
if (!properties.enabled()) {
return fallback;
}
return ManagedExecutors.newPlatformExecutor("warehouse-long-running", "warehouse-long-running-",
(thread, throwable) -> {
log.error("Warehouse longRunningExecutor has uncaughtException.");
log.error(throwable.getMessage(), throwable);
});
}
private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) {
ThreadFactory threadFactory = new ThreadFactoryBuilder()
.setUncaughtExceptionHandler(handler)
.setDaemon(true)
.setNameFormat("warehouse-worker-%d")
.build();
return new ThreadPoolExecutor(2,
Integer.MAX_VALUE,
10,
TimeUnit.SECONDS,
new SynchronousQueue<>(),
threadFactory,
new ThreadPoolExecutor.AbortPolicy());
}
/**
* Run warehouse task
* @param runnable task
* @throws RejectedExecutionException when THREAD POOL FULL
*/
public void executeJob(Runnable runnable) throws RejectedExecutionException {
workerExecutor.execute(runnable);
}
/**
* Run a long-lived warehouse consumer outside of the per-task executor.
*
* @param runnable consumer task
*/
public void executeLongRunning(Runnable runnable) {
longRunningExecutor.execute(runnable);
}
@Override
public void destroy() throws Exception {
workerExecutor.close();
if (longRunningExecutor != workerExecutor) {
longRunningExecutor.close();
}
}
}
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.config;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.springframework.boot.autoconfigure.AutoConfiguration;
import org.springframework.context.annotation.ComponentScan;
/**
* WarehouseAutoConfiguration class
* @version 2.1
*/
@AutoConfiguration
@ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG
+ SignConstants.DOT
+ ConfigConstants.FunctionModuleConstants.WAREHOUSE)
public class WarehouseAutoConfiguration {
}
@@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.constants;
/**
* Warehouse configuration constants.
*/
public interface WarehouseConstants {
String STORE = "store";
String REAL_TIME = "real-time";
/**
* History database name.
*/
interface HistoryName {
String GREPTIME = "greptime";
String INFLUXDB = "influxdb";
String IOT_DB = "iot-db";
String JPA = "jpa";
String TD_ENGINE = "td-engine";
String VM = "victoria-metrics";
String VM_CLUSTER = "victoria-metrics.cluster";
String QUEST_DB = "questdb";
String DUCKDB = "duckdb";
String DORIS = "doris";
}
/**
* Real-time database name.
*/
interface RealTimeName {
String REDIS = "redis";
String MEMORY = "memory";
}
String PROMQL = "promql";
String SQL = "sql";
String RANGE = "range";
String INSTANT = "instant";
String LOG_TABLE_NAME = "hertzbeat_logs";
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.controller;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import java.util.List;
import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
import org.apache.hertzbeat.warehouse.service.DatasourceQueryService;
import org.springframework.beans.factory.annotation.Autowired;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.PostMapping;
import org.springframework.web.bind.annotation.RequestBody;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;
/**
* Metrics Data Query API
*/
@RestController
@RequestMapping(produces = {APPLICATION_JSON_VALUE})
@Tag(name = "Metrics Data Query API")
public class DataQueryController {
@Autowired(required = false)
private DatasourceQueryService datasourceQueryService;
@PostMapping("/api/warehouse/query")
@Operation(summary = "Warehouse Query")
public ResponseEntity<Message<List<DatasourceQueryData>>> query(
@Parameter(description = "Query Expr") @RequestBody List<DatasourceQuery> queries) {
return ResponseEntity.ok(Message.success(datasourceQueryService.query(queries)));
}
}
@@ -0,0 +1,103 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.controller;
import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE;
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
import io.swagger.v3.oas.annotations.Operation;
import io.swagger.v3.oas.annotations.Parameter;
import io.swagger.v3.oas.annotations.tags.Tag;
import org.apache.hertzbeat.common.entity.dto.Message;
import org.apache.hertzbeat.common.entity.dto.MetricsData;
import org.apache.hertzbeat.common.entity.dto.MetricsHistoryData;
import org.apache.hertzbeat.warehouse.service.MetricsDataService;
import org.springframework.http.ResponseEntity;
import org.springframework.web.bind.annotation.GetMapping;
import org.springframework.web.bind.annotation.PathVariable;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RequestParam;
import org.springframework.web.bind.annotation.RestController;
/**
* Indicator data query interface
*/
@RestController
@RequestMapping(produces = {APPLICATION_JSON_VALUE})
@Tag(name = "Metrics Data API | The API for monitoring metric data")
public class MetricsDataController {
private static final Integer METRIC_FULL_LENGTH = 3;
private final MetricsDataService metricsDataService;
public MetricsDataController(MetricsDataService metricsDataService) {
this.metricsDataService = metricsDataService;
}
@GetMapping("/api/warehouse/storage/status")
@Operation(summary = "Query Warehouse Storage Server Status", description = "Query the availability status of the storage service under the warehouse")
public ResponseEntity<Message<Void>> getWarehouseStorageServerStatus() {
Boolean status = metricsDataService.getWarehouseStorageServerStatus();
if (Boolean.TRUE.equals(status)) {
return ResponseEntity.ok(Message.success());
}
// historyDataReader does not exist or is not available
return ResponseEntity.ok(Message.fail(FAIL_CODE, "Service not available!"));
}
@GetMapping("/api/monitor/{monitorId}/metrics/{metrics}")
@Operation(summary = "Query Real Time Metrics Data", description = "Query real-time metrics data of monitoring indicators")
public ResponseEntity<Message<MetricsData>> getMetricsData(
@Parameter(description = "Monitor Id", example = "343254354")
@PathVariable Long monitorId,
@Parameter(description = "Metrics Name", example = "cpu")
@PathVariable String metrics) {
MetricsData metricsData = metricsDataService.getMetricsData(monitorId, metrics);
if (metricsData == null){
return ResponseEntity.ok(Message.success("query metrics data is empty"));
}
return ResponseEntity.ok(Message.success(metricsData));
}
@GetMapping("/api/monitor/{instance}/metric/{metricFull}")
@Operation(summary = "Queries historical data for a specified metric for monitoring", description = "Queries historical data for a specified metric under monitoring")
public ResponseEntity<Message<MetricsHistoryData>> getMetricHistoryData(
@Parameter(description = "monitor instance", example = "127.0.0.1:8080")
@PathVariable String instance,
@Parameter(description = "monitor metric full path", example = "linux.cpu.usage")
@PathVariable() String metricFull,
@Parameter(description = "query historical time period, default 6h-6 hours: s-seconds, M-minutes, h-hours, d-days, w-weeks", example = "6h")
@RequestParam(required = false) String history,
@Parameter(description = "aggregate data calc. off by default; 4-hour window, query limit >1 week", example = "false")
@RequestParam(required = false) Boolean interval
) {
if (!metricsDataService.getWarehouseStorageServerStatus()) {
return ResponseEntity.ok(Message.fail(FAIL_CODE, "time series database not available"));
}
String[] names = metricFull.split("\\.");
if (names.length != METRIC_FULL_LENGTH) {
throw new IllegalArgumentException("metrics full name: " + metricFull + " is illegal.");
}
String app = names[0];
String metrics = names[1];
String metric = names[2];
MetricsHistoryData historyData = metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, interval);
return ResponseEntity.ok(Message.success(historyData));
}
}
@@ -0,0 +1,59 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.dao;
import org.apache.hertzbeat.common.entity.warehouse.History;
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
import org.springframework.data.repository.query.Param;
import org.springframework.transaction.annotation.Transactional;
/**
* history entity dao
*/
public interface HistoryDao extends JpaRepository<History, Long>, JpaSpecificationExecutor<History> {
/**
* delete history before expireTime
* @param expireTime expireTime
* @return rows deleted
*/
@Modifying
@Transactional(rollbackFor = Exception.class)
int deleteHistoriesByTimeBefore(Long expireTime);
/**
* delete older history record
* @param delNum number to be deleted
* @return rows deleted
*/
@Modifying
@Transactional(rollbackFor = Exception.class)
@Query(value = "DELETE FROM hzb_history WHERE id IN ( SELECT t2.id from (SELECT t1.id FROM hzb_history t1 LIMIT ?1) as t2)", nativeQuery = true)
int deleteOlderHistoriesRecord(@Param(value = "delNum") int delNum);
/**
* truncateTable
*/
@Modifying
@Transactional(rollbackFor = Exception.class)
@Query(value = "TRUNCATE TABLE hzb_history", nativeQuery = true)
void truncateTable();
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
/**
* query executor for victor metrics
*/
@Component("greptimePromqlQueryExecutor")
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
@Slf4j
public class GreptimePromqlQueryExecutor extends PromqlQueryExecutor {
private static final String QUERY_PATH = "/v1/prometheus";
private static final String Datasource = "Greptime-promql";
private final GreptimeProperties greptimeProperties;
public GreptimePromqlQueryExecutor(GreptimeProperties greptimeProperties, RestTemplate restTemplate) {
super(restTemplate, new HttpPromqlProperties(greptimeProperties.httpEndpoint() + QUERY_PATH,
greptimeProperties.username(), greptimeProperties.password()));
this.greptimeProperties = greptimeProperties;
}
@Override
public String getDatasource() {
return Datasource;
}
}
@@ -0,0 +1,133 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeSqlQueryContent;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpEntity;
import org.springframework.http.MediaType;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
* query executor for GreptimeDB SQL
*/
@Slf4j
@Component("greptimeSqlQueryExecutor")
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
public class GreptimeSqlQueryExecutor extends SqlQueryExecutor {
private static final String QUERY_PATH = "/v1/sql";
private static final String DATASOURCE = "Greptime-sql";
private final GreptimeProperties greptimeProperties;
public GreptimeSqlQueryExecutor(GreptimeProperties greptimeProperties, RestTemplate restTemplate) {
super(restTemplate, new SqlQueryExecutor.HttpSqlProperties(greptimeProperties.httpEndpoint() + QUERY_PATH,
greptimeProperties.username(), greptimeProperties.password()));
this.greptimeProperties = greptimeProperties;
}
@Override
public List<Map<String, Object>> execute(String queryString) {
List<Map<String, Object>> results = new LinkedList<>();
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
if (StringUtils.hasText(greptimeProperties.username())
&& StringUtils.hasText(greptimeProperties.password())) {
String authStr = greptimeProperties.username() + ":" + greptimeProperties.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
String requestBody = "sql=" + queryString;
HttpEntity<String> httpEntity = new HttpEntity<>(requestBody, headers);
String url = greptimeProperties.httpEndpoint() + QUERY_PATH;
if (StringUtils.hasText(greptimeProperties.database())) {
url += "?db=" + greptimeProperties.database();
}
ResponseEntity<GreptimeSqlQueryContent> responseEntity;
try {
responseEntity = restTemplate.exchange(url,
HttpMethod.POST, httpEntity, GreptimeSqlQueryContent.class);
} catch (Exception e) {
log.error("Exception occurred while querying GreptimeDB SQL: {}", e.getMessage(), e);
throw new RuntimeException("Failed to execute GreptimeDB SQL query", e);
}
if (responseEntity.getStatusCode().is2xxSuccessful()) {
GreptimeSqlQueryContent responseBody = responseEntity.getBody();
// GreptimeDB SQL HTTP API may not return 'code' field in successful response
// Check if output exists and is not empty
if (responseBody != null && responseBody.getOutput() != null && !responseBody.getOutput().isEmpty()) {
for (GreptimeSqlQueryContent.Output output : responseBody.getOutput()) {
if (output.getRecords() != null && output.getRecords().getRows() != null) {
GreptimeSqlQueryContent.Output.Records.Schema schema = output.getRecords().getSchema();
List<List<Object>> rows = output.getRecords().getRows();
for (List<Object> row : rows) {
Map<String, Object> rowMap = new HashMap<>();
if (schema != null && schema.getColumnSchemas() != null) {
for (int i = 0; i < Math.min(schema.getColumnSchemas().size(), row.size()); i++) {
String columnName = schema.getColumnSchemas().get(i).getName();
Object value = row.get(i);
rowMap.put(columnName, value);
}
} else {
for (int i = 0; i < row.size(); i++) {
rowMap.put("col_" + i, row.get(i));
}
}
results.add(rowMap);
}
}
}
}
} else {
log.error("query metrics data from greptime failed. {}", responseEntity);
}
return results;
}
@Override
public String getDatasource() {
return DATASOURCE;
}
}
@@ -0,0 +1,217 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.binary.Base64;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.TimePeriodUtil;
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.INSTANT;
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.PROMQL;
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.RANGE;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.PromQlQueryContent;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpEntity;
import org.springframework.http.MediaType;
import org.springframework.http.HttpMethod;
import org.springframework.http.ResponseEntity;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
/**
* abstract class for promql query executor
*/
@Slf4j
public abstract class PromqlQueryExecutor implements QueryExecutor {
private static final String supportQueryLanguage = PROMQL;
private static final String QUERY_RANGE_PATH = "/api/v1/query_range";
private static final String QUERY_PATH = "/api/v1/query";
protected static final String HTTP_QUERY_PARAM = "query";
protected static final String HTTP_TIME_PARAM = "time";
protected static final String HTTP_START_PARAM = "start";
protected static final String HTTP_END_PARAM = "end";
protected static final String HTTP_STEP_PARAM = "step";
private static final String INNER_KEY_TIME = "__ts__";
private static final String INNER_KEY_VALUE = "__value__";
private final RestTemplate restTemplate;
private final HttpPromqlProperties httpPromqlProperties;
PromqlQueryExecutor(RestTemplate restTemplate, HttpPromqlProperties httpPromqlProperties) {
this.restTemplate = restTemplate;
this.httpPromqlProperties = httpPromqlProperties;
}
/**
* record class for promql http connection
*/
protected record HttpPromqlProperties(
String url,
String username,
String password
) {
}
@Override
public List<Map<String, Object>> execute(String queryString) {
List<Map<String, Object>> results = new LinkedList<>();
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
if (StringUtils.hasText(httpPromqlProperties.username())
&& StringUtils.hasText(httpPromqlProperties.password())) {
String authStr = httpPromqlProperties.username() + ":" + httpPromqlProperties.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromUriString(httpPromqlProperties.url + QUERY_PATH);
uriComponentsBuilder.queryParam(HTTP_QUERY_PARAM, queryString);
URI uri = uriComponentsBuilder.build().toUri();
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri,
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
Map<String, Object> queryResult = new HashMap<>(8);
queryResult.putAll(labels);
if (content.getValue() != null && content.getValue().length == 2) {
queryResult.put("__timestamp__", content.getValue()[0]);
queryResult.put("__value__", content.getValue()[1]);
} else if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Object> values = new LinkedList<>();
for (Object[] valueArr : content.getValues()) {
values.add(valueArr[1]);
}
queryResult.put("__value__", values);
}
results.add(queryResult);
}
}
} else {
log.error("query metrics data from greptime failed. {}", responseEntity);
}
} catch (Exception e) {
log.error(e.toString(), e);
}
return results;
}
@Override
public DatasourceQueryData query(DatasourceQuery datasourceQuery) {
DatasourceQueryData.DatasourceQueryDataBuilder queryDataBuilder = DatasourceQueryData.builder()
.refId(datasourceQuery.getRefId()).status(200);
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
if (StringUtils.hasText(httpPromqlProperties.username())
&& StringUtils.hasText(httpPromqlProperties.password())) {
String authStr = httpPromqlProperties.username() + ":" + httpPromqlProperties.password();
String encodedAuth = new String(Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + " " + encodedAuth);
}
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
URI uri;
if (datasourceQuery.getTimeType().equals(RANGE)) {
uri = UriComponentsBuilder.fromUriString(httpPromqlProperties.url() + QUERY_RANGE_PATH)
.queryParam(HTTP_QUERY_PARAM, datasourceQuery.getExpr())
.queryParam(HTTP_START_PARAM, datasourceQuery.getStart())
.queryParam(HTTP_END_PARAM, datasourceQuery.getEnd())
.queryParam(HTTP_STEP_PARAM, datasourceQuery.getStep())
.build().toUri();
} else if (datasourceQuery.getTimeType().equals(INSTANT)) {
uri = UriComponentsBuilder.fromUriString(httpPromqlProperties.url() + QUERY_PATH)
.queryParam(HTTP_QUERY_PARAM, datasourceQuery.getExpr())
.build().toUri();
} else {
throw new IllegalArgumentException(String.format("no such time type for query id %s.", datasourceQuery.getRefId()));
}
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity,
PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("query metrics data from promql http api success. {}", uri);
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
List<DatasourceQueryData.SchemaData> schemaDataList = new LinkedList<>();
for (PromQlQueryContent.ContentData.Content content : contents) {
DatasourceQueryData.MetricSchema.MetricSchemaBuilder schemaBuilder = DatasourceQueryData.MetricSchema
.builder().fields(List.of(
// todo: unit?
DatasourceQueryData.MetricField.builder().name(INNER_KEY_TIME)
.type("time").build(),
DatasourceQueryData.MetricField.builder().name(INNER_KEY_VALUE)
.type("number").build()
)).labels(content.getMetric());
List<Object[]> values;
if (datasourceQuery.getTimeType().equals(RANGE)) {
values = content.getValues();
}
else {
values = List.<Object[]>of(content.getValue());
}
values.forEach(objects -> {
objects[0] = TimePeriodUtil.normalizeToMilliseconds(objects[0]);
});
DatasourceQueryData.SchemaData.SchemaDataBuilder schemaData = DatasourceQueryData.SchemaData.builder()
.schema(schemaBuilder.build()).data(values);
schemaDataList.add(schemaData.build());
}
queryDataBuilder.frames(schemaDataList);
}
} else {
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
queryDataBuilder.msg("query metrics data from victoria-metrics failed. ");
queryDataBuilder.status(responseEntity.getStatusCode().value());
}
} catch (Exception e) {
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
queryDataBuilder.msg("query metrics data from victoria-metrics error: " + e.getMessage());
queryDataBuilder.status(400);
}
return queryDataBuilder.build();
}
@Override
public boolean support(String queryLanguage) {
return StringUtils.hasText(queryLanguage) && queryLanguage.equalsIgnoreCase(supportQueryLanguage);
}
}
@@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
import java.util.List;
import java.util.Map;
/**
* query executor interface
*/
public interface QueryExecutor {
List<Map<String, Object>> execute(String query);
DatasourceQueryData query(DatasourceQuery datasourceQuery);
String getDatasource();
boolean support(String queryLanguage);
}
@@ -0,0 +1,73 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.SQL;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import java.util.List;
import java.util.Map;
/**
* abstract class for sql query executor
*/
@Slf4j
public abstract class SqlQueryExecutor implements QueryExecutor {
private static final String supportQueryLanguage = SQL;
protected final RestTemplate restTemplate;
protected final SqlQueryExecutor.HttpSqlProperties httpSqlProperties;
SqlQueryExecutor(RestTemplate restTemplate, SqlQueryExecutor.HttpSqlProperties httpSqlProperties) {
this.restTemplate = restTemplate;
this.httpSqlProperties = httpSqlProperties;
}
/**
* record class for sql http connection
*/
protected record HttpSqlProperties(
String url,
String username,
String password
) {
}
@Override
public List<Map<String, Object>> execute(String query) {
return null;
}
@Override
public DatasourceQueryData query(DatasourceQuery datasourceQuery) {
return null;
}
@Override
public boolean support(String queryLanguage) {
return StringUtils.hasText(queryLanguage) && queryLanguage.equalsIgnoreCase(supportQueryLanguage);
}
}
@@ -0,0 +1,56 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsClusterProperties;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsSelectProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
/**
* query executor for victoria metrics cluster
*/
@Component
@ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics.cluster", name = "enabled", havingValue = "true")
@Slf4j
public class VictoriaMetricsClusterQueryExecutor extends PromqlQueryExecutor {
private static final String QUERY_PATH = "/select/%s/prometheus";
private static final String DATASOURCE = "VictoriaMetricsCluster";
public VictoriaMetricsClusterQueryExecutor(VictoriaMetricsClusterProperties victoriaMetricsClusterProps,
RestTemplate restTemplate) {
super(restTemplate, buildHttpPromqlProperties(victoriaMetricsClusterProps));
}
private static HttpPromqlProperties buildHttpPromqlProperties(VictoriaMetricsClusterProperties props) {
VictoriaMetricsSelectProperties selectProperties = props.select();
return new HttpPromqlProperties(
selectProperties.url() + QUERY_PATH.formatted(props.accountID()),
selectProperties.username(),
selectProperties.password()
);
}
@Override
public String getDatasource() {
return DATASOURCE;
}
}
@@ -0,0 +1,49 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsProperties;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import org.springframework.web.client.RestTemplate;
/**
* query executor for victor metrics
*/
@Component
@ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics", name = "enabled", havingValue = "true")
@Slf4j
public class VictoriaMetricsQueryExecutor extends PromqlQueryExecutor {
private static final String Datasource = "VictoriaMetrics";
private final VictoriaMetricsProperties victoriaMetricsProp;
public VictoriaMetricsQueryExecutor(VictoriaMetricsProperties victoriaMetricsProp, RestTemplate restTemplate) {
super(restTemplate, new HttpPromqlProperties(victoriaMetricsProp.url(),
victoriaMetricsProp.username(), victoriaMetricsProp.password()));
this.victoriaMetricsProp = victoriaMetricsProp;
}
@Override
public String getDatasource() {
return Datasource;
}
}
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.listener;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.springframework.boot.context.event.ApplicationReadyEvent;
import org.springframework.context.event.EventListener;
import org.springframework.stereotype.Component;
/**
* WareHouseApplicationReadyListener
*/
@Slf4j
@Component
public class WareHouseApplicationReadyListener {
private final Optional<AbstractHistoryDataStorage> historyDataStorage;
public WareHouseApplicationReadyListener(Optional<AbstractHistoryDataStorage> historyDataStorage) {
this.historyDataStorage = historyDataStorage;
}
@EventListener(classes = {ApplicationReadyEvent.class})
public void listen() {
if (historyDataStorage.isEmpty()) {
log.warn("The historical data repository is not configured");
}
}
}
@@ -0,0 +1,35 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.service;
import java.util.List;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
/**
* metrics data query service
*/
public interface DatasourceQueryService {
/**
* Query metrics data
* @param queries query expr
* @return data
*/
List<DatasourceQueryData> query(List<DatasourceQuery> queries);
}
@@ -0,0 +1,54 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.service;
import org.apache.hertzbeat.common.entity.dto.MetricsData;
import org.apache.hertzbeat.common.entity.dto.MetricsHistoryData;
/**
* service for metrics data
*/
public interface MetricsDataService {
/**
* warehouse storage server availability or not
* @return true or false
*/
Boolean getWarehouseStorageServerStatus();
/**
* Query Real Time Metrics Data
* @param monitorId Monitor Id
* @param metrics Metrics Name
* @return metrics data
*/
MetricsData getMetricsData(Long monitorId, String metrics);
/**
* Queries historical data for a specified metric for monitoring
*
* @param instance Instance e.g. ip:port or ip or domain
* @param app Monitor Type
* @param metrics Metrics Name
* @param metric Metrics Field Name
* @param history Query Historical Time Period
* @param interval aggregate data calc
* @return metrics history data
*/
MetricsHistoryData getMetricHistoryData(String instance, String app, String metrics, String metric, String history, Boolean interval);
}
@@ -0,0 +1,34 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.service;
import java.util.List;
import org.apache.hertzbeat.common.entity.message.CollectRep;
/**
* service for warehouse
*/
public interface WarehouseService {
/**
* query monitor real time metrics data by monitor id
* @param monitorId monitor id
* @return metrics data
*/
List<CollectRep.MetricsData> queryMonitorMetricsData(Long monitorId);
}
@@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.warehouse.service.impl;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
import org.apache.hertzbeat.warehouse.db.QueryExecutor;
import org.apache.hertzbeat.warehouse.service.DatasourceQueryService;
import org.springframework.stereotype.Service;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* datasource query service impl
*/
@Service
public class DatasourceQueryServiceImpl implements DatasourceQueryService {
Map<String, QueryExecutor> executorMap;
DatasourceQueryServiceImpl(List<QueryExecutor> executors) {
executorMap = executors.stream().collect(Collectors.toMap(QueryExecutor::getDatasource, executor -> executor));
}
@Override
public List<DatasourceQueryData> query(List<DatasourceQuery> queries) {
if (queries == null) {
throw new IllegalArgumentException("No query found");
}
List<DatasourceQueryData> datasourceQueryDataList = new ArrayList<>();
for (DatasourceQuery datasourceQuery : queries) {
QueryExecutor executor = executorMap.get(datasourceQuery.getDatasource());
if (executor == null) {
throw new IllegalArgumentException("Unsupported datasource: " + datasourceQuery.getDatasource());
}
datasourceQueryDataList.add(executor.query(datasourceQuery));
}
return datasourceQueryDataList;
}
}
@@ -0,0 +1,129 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.service.impl;
import com.google.common.collect.Maps;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Field;
import org.apache.hertzbeat.common.entity.dto.MetricsData;
import org.apache.hertzbeat.common.entity.dto.MetricsHistoryData;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.dto.ValueRow;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.support.exception.CommonException;
import org.apache.hertzbeat.warehouse.service.MetricsDataService;
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader;
import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataReader;
import org.springframework.stereotype.Service;
/**
* Metrics Data Service impl
*/
@Slf4j
@Service
public class MetricsDataServiceImpl implements MetricsDataService {
private final RealTimeDataReader realTimeDataReader;
private final Optional<HistoryDataReader> historyDataReader;
public MetricsDataServiceImpl(RealTimeDataReader realTimeDataReader, Optional<HistoryDataReader> historyDataReader) {
this.realTimeDataReader = realTimeDataReader;
this.historyDataReader = historyDataReader;
}
@Override
public Boolean getWarehouseStorageServerStatus() {
return historyDataReader.isPresent() && historyDataReader.get().isServerAvailable();
}
@Override
public MetricsData getMetricsData(Long monitorId, String metrics) {
boolean available = realTimeDataReader.isServerAvailable();
if (!available) {
throw new CommonException("real time store not available");
}
CollectRep.MetricsData storageData = realTimeDataReader.getCurrentMetricsData(monitorId, metrics);
if (storageData == null) {
return null;
}
MetricsData.MetricsDataBuilder dataBuilder = MetricsData.builder();
dataBuilder.id(storageData.getId()).app(storageData.getApp()).metrics(storageData.getMetrics())
.time(storageData.getTime());
dataBuilder.fields(storageData.getFields().stream()
.map(field -> Field.builder().name(field.getName())
.type((byte) field.getType())
.label(field.getLabel())
.unit(field.getUnit())
.build())
.toList());
List<ValueRow> valueRows = new ArrayList<>();
if (storageData.rowCount() > 0) {
RowWrapper rowWrapper = storageData.readRow();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
List<Value> values = new ArrayList<>();
rowWrapper.cellStream().forEach(cell -> {
String origin = cell.getValue();
if (CommonConstants.NULL_VALUE.equals(origin)) {
values.add(new Value());
} else {
values.add(new Value(origin));
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL)) {
labels.put(cell.getField().getName(), origin);
}
}
});
valueRows.add(ValueRow.builder().labels(labels).values(values).build());
}
dataBuilder.valueRows(valueRows);
}
return dataBuilder.build();
}
@Override
public MetricsHistoryData getMetricHistoryData(String instance, String app, String metrics, String metric, String history, Boolean interval) {
if (history == null) {
history = "6h";
}
Map<String, List<Value>> instanceValuesMap;
if (interval == null || !interval) {
instanceValuesMap = historyDataReader.get().getHistoryMetricData(instance, app, metrics, metric, history);
} else {
instanceValuesMap = historyDataReader.get().getHistoryIntervalMetricData(instance, app, metrics, metric, history);
}
if (instanceValuesMap.containsKey("{}")) {
instanceValuesMap.put("", instanceValuesMap.get("{}"));
instanceValuesMap.remove("{}");
}
return MetricsHistoryData.builder()
.instance(instance).metrics(metrics).values(instanceValuesMap)
.field(Field.builder().name(metric).type(CommonConstants.TYPE_NUMBER).build())
.build();
}
}
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.service.impl;
import java.util.Collections;
import java.util.List;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.service.WarehouseService;
import org.apache.hertzbeat.warehouse.store.realtime.AbstractRealTimeDataStorage;
import org.springframework.stereotype.Service;
/**
* warehouse service impl
*/
@Service
@Slf4j
public class WarehouseServiceImpl implements WarehouseService {
private final AbstractRealTimeDataStorage realTimeDataStorage;
public WarehouseServiceImpl(AbstractRealTimeDataStorage realTimeDataStorage) {
this.realTimeDataStorage = realTimeDataStorage;
}
@Override
public List<CollectRep.MetricsData> queryMonitorMetricsData(Long monitorId) {
boolean available = realTimeDataStorage.isServerAvailable();
if (!available) {
log.error("real time store not available");
return Collections.emptyList();
}
return realTimeDataStorage.getCurrentMetricsData(monitorId);
}
}
@@ -0,0 +1,155 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store;
import java.util.List;
import java.util.Optional;
import jakarta.persistence.EntityManager;
import jakarta.persistence.PersistenceContext;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.entity.manager.Monitor;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.queue.CommonDataQueue;
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
import org.apache.hertzbeat.common.util.BackoffUtils;
import org.apache.hertzbeat.common.util.ExponentialBackoff;
import org.apache.hertzbeat.plugin.PostCollectPlugin;
import org.apache.hertzbeat.plugin.runner.PluginRunner;
import org.apache.hertzbeat.warehouse.WarehouseWorkerPool;
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataWriter;
import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataWriter;
import org.springframework.jdbc.core.JdbcTemplate;
import org.springframework.stereotype.Component;
/**
* dispatch storage metrics data
*/
@Slf4j
@Component
public class DataStorageDispatch {
private final CommonDataQueue commonDataQueue;
private final WarehouseWorkerPool workerPool;
private final JdbcTemplate jdbcTemplate;
private final RealTimeDataWriter realTimeDataWriter;
private final Optional<HistoryDataWriter> historyDataWriter;
private final PluginRunner pluginRunner;
private static final int LOG_BATCH_SIZE = 1000;
@PersistenceContext
private EntityManager entityManager;
public DataStorageDispatch(CommonDataQueue commonDataQueue,
WarehouseWorkerPool workerPool,
JdbcTemplate jdbcTemplate,
Optional<HistoryDataWriter> historyDataWriter,
RealTimeDataWriter realTimeDataWriter,
PluginRunner pluginRunner) {
this.commonDataQueue = commonDataQueue;
this.workerPool = workerPool;
this.jdbcTemplate = jdbcTemplate;
this.realTimeDataWriter = realTimeDataWriter;
this.historyDataWriter = historyDataWriter;
this.pluginRunner = pluginRunner;
startPersistentDataStorage();
startLogDataStorage();
}
protected void startPersistentDataStorage() {
Runnable runnable = () -> {
Thread.currentThread().setName("warehouse-persistent-data-storage");
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
while (!Thread.currentThread().isInterrupted()) {
try {
CollectRep.MetricsData metricsData = commonDataQueue.pollMetricsDataToStorage();
if (metricsData == null) {
continue;
}
backoff.reset();
try {
calculateMonitorStatus(metricsData);
historyDataWriter.ifPresent(dataWriter -> dataWriter.saveData(metricsData));
pluginRunner.pluginExecute(PostCollectPlugin.class, ((postCollectPlugin, pluginContext) -> postCollectPlugin.execute(metricsData, pluginContext)));
} finally {
realTimeDataWriter.saveData(metricsData);
}
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
} catch (CommonDataQueueUnknownException ue) {
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
break;
}
} catch (Exception e) {
log.error(e.getMessage(), e);
}
}
};
workerPool.executeLongRunning(runnable);
}
protected void startLogDataStorage() {
Runnable runnable = () -> {
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
Thread.currentThread().setName("warehouse-log-data-storage");
while (!Thread.currentThread().isInterrupted()) {
try {
List<LogEntry> logEntries = commonDataQueue.pollLogEntryToStorageBatch(LOG_BATCH_SIZE);
if (logEntries == null || logEntries.isEmpty()) {
continue;
}
backoff.reset();
historyDataWriter.ifPresent(dataWriter -> {
try {
dataWriter.saveLogDataBatch(logEntries);
} catch (Exception e) {
log.error("Failed to save log entries batch: {}", e.getMessage(), e);
}
});
} catch (InterruptedException interruptedException) {
Thread.currentThread().interrupt();
} catch (CommonDataQueueUnknownException ue) {
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
break;
}
} catch (Exception e) {
log.error("Error in log data storage thread: {}", e.getMessage(), e);
}
}
};
workerPool.executeLongRunning(runnable);
}
protected void calculateMonitorStatus(CollectRep.MetricsData metricsData) {
if (metricsData.getPriority() == 0) {
long id = metricsData.getId();
CollectRep.Code code = metricsData.getCode();
try {
String sql = "UPDATE hzb_monitor SET status = ? WHERE id = ? AND status = ?";
int status = code == CollectRep.Code.SUCCESS ? CommonConstants.MONITOR_UP_CODE : CommonConstants.MONITOR_DOWN_CODE;
int preStatus = code == CollectRep.Code.SUCCESS ? CommonConstants.MONITOR_DOWN_CODE : CommonConstants.MONITOR_UP_CODE;
int matchedRows = jdbcTemplate.update(sql, status, id, preStatus);
if (matchedRows > 0) {
entityManager.getEntityManagerFactory().getCache().evict(Monitor.class, id);
}
} catch (Exception e) {
log.error("Update monitor status failed for monitor id: {}", id, e);
}
}
}
}
@@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
/**
* data storage abstract class
*/
@Slf4j
public abstract class AbstractHistoryDataStorage implements HistoryDataReader, HistoryDataWriter, DisposableBean {
protected boolean serverAvailable;
/**
* @return data storage available
*/
@Override
public boolean isServerAvailable() {
return serverAvailable;
}
}
@@ -0,0 +1,131 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb;
import java.util.List;
import java.util.Map;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.log.LogEntry;
/**
* history data reader
*/
public interface HistoryDataReader {
/**
* @return data storage available
*/
boolean isServerAvailable();
/**
* query history range metrics data from tsdb
*
* @param instance instance e.g. ip:port or ip or domain
* @param app monitor type
* @param metrics metrics
* @param metric metric
* @param history range
* @return metrics data
*/
Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history);
/**
* query history range interval metrics data from tsdb
* max min mean metrics value
*
* @param instance instance e.g. ip:port or ip or domain
* @param app monitor type
* @param metrics metrics
* @param metric metric
* @param history history range
* @return metrics data
*/
Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history);
/**
* Query logs with multiple filter conditions
* @param startTime start time in milliseconds
* @param endTime end time in milliseconds
* @param traceId trace ID filter
* @param spanId span ID filter
* @param severityNumber severity number filter
* @param severityText severity text filter
* @param searchContent search content in log body
* @return filtered log entries
*/
default List<LogEntry> queryLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber,
String severityText, String searchContent) {
throw new UnsupportedOperationException("query logs by multiple conditions is not supported");
}
/**
* Query logs with multiple filter conditions and pagination (Legacy)
*/
default List<LogEntry> queryLogsByMultipleConditionsWithPagination(Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber,
String severityText, Integer offset, Integer limit) {
return queryLogsByMultipleConditionsWithPagination(startTime, endTime, traceId, spanId, severityNumber, severityText, null, offset, limit);
}
/**
* Query logs with multiple filter conditions and pagination including search content
* @param startTime start time in milliseconds
* @param endTime end time in milliseconds
* @param traceId trace ID filter
* @param spanId span ID filter
* @param severityNumber severity number filter
* @param severityText severity text filter
* @param searchContent search content in log body
* @param offset pagination offset
* @param limit pagination limit
* @return filtered log entries with pagination
*/
default List<LogEntry> queryLogsByMultipleConditionsWithPagination(Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber,
String severityText, String searchContent,
Integer offset, Integer limit) {
throw new UnsupportedOperationException("query logs by multiple conditions with pagination is not supported");
}
/**
* Count logs with multiple filter conditions (Legacy)
*/
default long countLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber,
String severityText) {
return countLogsByMultipleConditions(startTime, endTime, traceId, spanId, severityNumber, severityText, null);
}
/**
* Count logs with multiple filter conditions including search content
* @param startTime start time in milliseconds
* @param endTime end time in milliseconds
* @param traceId trace ID filter
* @param spanId span ID filter
* @param severityNumber severity number filter
* @param severityText severity text filter
* @param searchContent search content in log body
* @return count of matching log entries
*/
default long countLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber,
String severityText, String searchContent) {
throw new UnsupportedOperationException("count logs by multiple conditions is not supported");
}
}
@@ -0,0 +1,70 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import java.util.List;
/**
* history data writer
*/
public interface HistoryDataWriter {
/**
* @return data storage available
*/
boolean isServerAvailable();
/**
* save metrics data
* @param metricsData metrics data
*/
void saveData(CollectRep.MetricsData metricsData);
/**
* default save log data
* @param logEntry log entry
*/
default void saveLogData(LogEntry logEntry) {
throw new UnsupportedOperationException("save log data is not supported");
}
/**
* Batch delete logs by time timestamps
* @param timeUnixNanos list of time timestamps to delete
* @return true if deletion is successful, false otherwise
*/
default boolean batchDeleteLogs(List<Long> timeUnixNanos) {
throw new UnsupportedOperationException("batch delete logs is not supported");
}
/**
* Batch save log data
* @param logEntries list of log entries
*/
default void saveLogDataBatch(List<LogEntry> logEntries) {
if (logEntries == null || logEntries.isEmpty()) {
return;
}
for (LogEntry logEntry : logEntries) {
saveLogData(logEntry);
}
}
}
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.doris;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.sql.Timestamp;
/**
* Internal class to represent a metric row for Doris storage.
* This class is used by both DorisDataStorage and DorisStreamLoadWriter.
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class DorisMetricRow {
public String instance;
public String app;
public String metrics;
public String metric;
public byte metricType;
public Integer int32Value;
public Double doubleValue;
public String strValue;
public Timestamp recordTime;
public String labels;
}
@@ -0,0 +1,235 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.doris;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* Apache Doris configuration properties
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.DORIS)
public record DorisProperties(
@DefaultValue("false") boolean enabled,
@DefaultValue("jdbc:mysql://127.0.0.1:9030/hertzbeat") String url,
String username,
String password,
TableConfig tableConfig,
PoolConfig poolConfig,
WriteConfig writeConfig) {
/**
* Table structure configuration
*/
public record TableConfig(
// Whether to enable dynamic partitioning (default enabled)
@DefaultValue("false") boolean enablePartition,
// Partition time unit: DAY, HOUR, MONTH
@DefaultValue("DAY") String partitionTimeUnit,
// Dynamic partition retention days
@DefaultValue("7") int partitionRetentionDays,
// Dynamic partition creation range (future partitions to create)
@DefaultValue("3") int partitionFutureDays,
// Number of buckets
@DefaultValue("8") int buckets,
// Number of replicas (recommended 3 for production)
@DefaultValue("1") int replicationNum,
// Maximum length of string columns
@DefaultValue("4096") int strColumnMaxLength) {
public TableConfig {
if (partitionRetentionDays <= 0) {
partitionRetentionDays = 7;
}
if (partitionFutureDays <= 0) {
partitionFutureDays = 3;
}
if (buckets <= 0) {
buckets = 8;
}
if (replicationNum <= 0) {
replicationNum = 1;
}
if (strColumnMaxLength <= 0 || strColumnMaxLength > 65533) {
strColumnMaxLength = 4096;
}
}
}
/**
* Connection pool configuration (based on HikariCP)
*/
public record PoolConfig(
// Minimum idle connections
@DefaultValue("5") int minimumIdle,
// Maximum pool size
@DefaultValue("20") int maximumPoolSize,
// Connection timeout in milliseconds
@DefaultValue("30000") int connectionTimeout,
// Maximum connection lifetime in milliseconds (0 means no limit)
@DefaultValue("0") long maxLifetime,
// Idle connection timeout in milliseconds (0 means never recycle)
@DefaultValue("600000") long idleTimeout) {
public PoolConfig {
if (minimumIdle <= 0) {
minimumIdle = 5;
}
if (maximumPoolSize <= 0) {
maximumPoolSize = 20;
}
if (minimumIdle > maximumPoolSize) {
minimumIdle = maximumPoolSize;
}
if (connectionTimeout <= 0) {
connectionTimeout = 30000;
}
}
}
/**
* Write configuration
*/
public record WriteConfig(
// Write mode: jdbc (batch insert) or stream (HTTP stream load)
@DefaultValue("jdbc") String writeMode,
// Batch write size (for jdbc mode)
@DefaultValue("1000") int batchSize,
// Batch write flush interval in seconds (for jdbc mode)
@DefaultValue("5") int flushInterval,
// Fallback to JDBC when stream load fails (may introduce duplicate data in ambiguous cases)
@DefaultValue("false") boolean fallbackToJdbcOnFailure,
// Stream load configuration (for stream mode)
StreamLoadConfig streamLoadConfig) {
public WriteConfig {
String normalizedWriteMode = writeMode == null ? "" : writeMode.trim().toLowerCase();
if (!"jdbc".equals(normalizedWriteMode) && !"stream".equals(normalizedWriteMode)) {
writeMode = "jdbc";
} else {
writeMode = normalizedWriteMode;
}
if (batchSize <= 0) {
batchSize = 1000;
}
if (flushInterval <= 0) {
flushInterval = 5;
}
if (streamLoadConfig == null) {
streamLoadConfig = StreamLoadConfig.createDefault();
}
}
}
/**
* Stream Load configuration for HTTP-based streaming writes
*/
public record StreamLoadConfig(
// Doris FE HTTP port for Stream Load API
@DefaultValue(":8030") String httpPort,
// Stream load timeout in seconds
@DefaultValue("60") int timeout,
// Max batch size in bytes for stream load
@DefaultValue("10485760") int maxBytesPerBatch,
// Maximum allowed filter ratio in [0,1]
@DefaultValue("0.1") double maxFilterRatio,
// Enable strict mode
@DefaultValue("false") boolean strictMode,
// Import timezone, empty means Doris default
@DefaultValue("") String timezone,
// Redirect policy: direct/public/private
@DefaultValue("") String redirectPolicy,
// Group commit mode: async_mode/sync_mode/off_mode
@DefaultValue("") String groupCommit,
// Send batch parallelism, 0 means Doris default
@DefaultValue("0") int sendBatchParallelism,
// Retry times for one label when stream load is retryable
@DefaultValue("2") int retryTimes) {
public StreamLoadConfig {
if (httpPort == null || httpPort.isBlank()) {
httpPort = ":8030";
} else {
httpPort = httpPort.trim();
}
if (timeout <= 0) {
timeout = 60;
}
if (maxBytesPerBatch <= 0) {
maxBytesPerBatch = 10485760; // 10MB
}
if (maxFilterRatio < 0 || maxFilterRatio > 1) {
maxFilterRatio = 0.1;
}
if (sendBatchParallelism < 0) {
sendBatchParallelism = 0;
}
if (retryTimes < 0) {
retryTimes = 2;
}
if (!isValidRedirectPolicy(redirectPolicy)) {
redirectPolicy = "";
}
if (!isValidGroupCommit(groupCommit)) {
groupCommit = "";
}
timezone = timezone == null ? "" : timezone.trim();
}
/**
* Factory method to create default StreamLoadConfig
*/
public static StreamLoadConfig createDefault() {
return new StreamLoadConfig(":8030", 60, 10485760,
0.1, false, "", "", "", 0, 2);
}
private static boolean isValidRedirectPolicy(String value) {
if (value == null || value.isBlank()) {
return true;
}
return "direct".equalsIgnoreCase(value)
|| "public".equalsIgnoreCase(value)
|| "private".equalsIgnoreCase(value);
}
private static boolean isValidGroupCommit(String value) {
if (value == null || value.isBlank()) {
return true;
}
return "async_mode".equalsIgnoreCase(value)
|| "sync_mode".equalsIgnoreCase(value)
|| "off_mode".equalsIgnoreCase(value);
}
}
// Provide default values for nested configs if null
public DorisProperties {
if (tableConfig == null) {
tableConfig = new TableConfig(false, "DAY", 7, 3, 8, 1, 4096);
}
if (poolConfig == null) {
poolConfig = new PoolConfig(5, 20, 30000, 0, 600000);
}
if (writeConfig == null) {
writeConfig = new WriteConfig("jdbc", 1000, 5, false, StreamLoadConfig.createDefault());
}
}
}
@@ -0,0 +1,603 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.doris;
import tools.jackson.databind.JsonNode;
import lombok.Getter;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.codec.binary.Base64;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.http.Header;
import org.apache.http.HttpEntityEnclosingRequest;
import org.apache.http.HttpRequest;
import org.apache.http.HttpResponse;
import org.apache.http.HttpHeaders;
import org.apache.http.protocol.HttpContext;
import org.apache.http.ProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpPut;
import org.apache.http.entity.StringEntity;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.DefaultRedirectStrategy;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
import org.apache.http.util.EntityUtils;
import org.apache.http.client.methods.HttpUriRequest;
import org.apache.http.client.utils.URIBuilder;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import java.nio.charset.StandardCharsets;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.ZoneId;
import java.time.format.DateTimeFormatter;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.atomic.AtomicLong;
/**
* Doris Stream Load writer using Apache HttpClient for high-throughput writes.
* Based on official Apache Doris Stream Load Java example.
* <p>
* Reference: <a href="https://github.com/apache/doris/blob/master/samples/doris-stream-load-demo/src/main/java/DorisStreamLoad.java">...</a>
*/
@Slf4j
public class DorisStreamLoadWriter {
private static final int CONNECT_TIMEOUT = 30000;
private static final int SOCKET_TIMEOUT = 60000;
private static final int CONNECTION_REQUEST_TIMEOUT = 5000;
private static final int MAX_TOTAL_CONNECTIONS = 20;
private static final int MAX_PER_ROUTE_CONNECTIONS = 10;
private static final long RETRY_BACKOFF_MS = 200L;
private static final DateTimeFormatter DORIS_DATETIME_FORMATTER =
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
private static final String METRIC_JSON_PATHS =
"[\"$.instance\",\"$.app\",\"$.metrics\",\"$.metric\",\"$.recordTime\",\"$.metricType\",\"$.int32Value\",\"$.doubleValue\",\"$.strValue\",\"$.labels\"]";
private static final String METRIC_COLUMNS =
"instance,app,metrics,metric,record_time,metric_type,int32_value,double_value,str_value,labels";
private static final String LOG_JSON_PATHS =
"[\"$.timeUnixNano\",\"$.observedTimeUnixNano\",\"$.eventTime\",\"$.severityNumber\",\"$.severityText\""
+ ",\"$.body\",\"$.traceId\",\"$.spanId\",\"$.traceFlags\",\"$.attributes\",\"$.resource\""
+ ",\"$.instrumentationScope\",\"$.droppedAttributesCount\"]";
private static final String LOG_COLUMNS =
"time_unix_nano,observed_time_unix_nano,event_time,severity_number,severity_text,body,trace_id,span_id,trace_flags,attributes,resource,instrumentation_scope,dropped_attributes_count";
public static final String STATUS_SUCCESS = "Success";
public static final String STATUS_PUBLISH_TIMEOUT = "Publish Timeout";
public static final String STATUS_FAIL = "Fail";
public static final String STATUS_LABEL_ALREADY_EXISTS = "Label Already Exists";
private final String databaseName;
private final String tableName;
private final String feHost;
private final int feHttpPort;
private final String username;
private final String password;
private final DorisProperties.StreamLoadConfig config;
private final AtomicLong transactionId = new AtomicLong(0);
private final CloseableHttpClient httpClient;
private enum LoadResult {
SUCCESS,
RETRYABLE_FAILURE,
NON_RETRYABLE_FAILURE
}
private static final class StreamLoadMetricRow {
public String instance;
public String app;
public String metrics;
public String metric;
public String recordTime;
public Byte metricType;
public Integer int32Value;
public Double doubleValue;
public String strValue;
public String labels;
}
private static final class StreamLoadLogRow {
public Long timeUnixNano;
public Long observedTimeUnixNano;
public String eventTime;
public Integer severityNumber;
public String severityText;
public String body;
public String traceId;
public String spanId;
public Integer traceFlags;
public String attributes;
public String resource;
public String instrumentationScope;
public Integer droppedAttributesCount;
}
/**
* -- GETTER --
* Check if writer is available
*/
@Getter
private volatile boolean available = true;
public DorisStreamLoadWriter(String databaseName, String tableName,
String jdbcUrl, String username, String password,
DorisProperties.StreamLoadConfig config) {
this.databaseName = databaseName;
this.tableName = tableName;
this.username = username;
this.password = password;
this.config = config;
this.feHost = parseHostFromJdbcUrl(jdbcUrl);
this.feHttpPort = parseHttpPort(config.httpPort());
// Create HTTP client with connection pool and redirect support
this.httpClient = createHttpClient();
log.info("[Doris StreamLoad] Writer initialized for {}.{}", databaseName, tableName);
}
private String parseHostFromJdbcUrl(String jdbcUrl) {
if (jdbcUrl == null || jdbcUrl.isBlank()) {
return "127.0.0.1";
}
String hostPart = jdbcUrl;
if (hostPart.startsWith("jdbc:mysql://")) {
hostPart = hostPart.substring("jdbc:mysql://".length());
}
int slashIndex = hostPart.indexOf('/');
if (slashIndex > 0) {
hostPart = hostPart.substring(0, slashIndex);
}
int queryIndex = hostPart.indexOf('?');
if (queryIndex > 0) {
hostPart = hostPart.substring(0, queryIndex);
}
// Multi-host URL: use first host
int commaIndex = hostPart.indexOf(',');
if (commaIndex > 0) {
hostPart = hostPart.substring(0, commaIndex);
}
int colonIndex = hostPart.lastIndexOf(':');
if (colonIndex > 0) {
hostPart = hostPart.substring(0, colonIndex);
}
return hostPart;
}
private int parseHttpPort(String portStr) {
if (portStr == null || portStr.isBlank()) {
return 8030;
}
if (portStr.startsWith(":")) {
portStr = portStr.substring(1);
}
try {
return Integer.parseInt(portStr);
} catch (NumberFormatException e) {
return 8030; // default
}
}
/**
* Create HTTP client with connection pool and automatic redirect handling
* Replaces internal IPs in redirect URLs with external FE host for cloud deployments
*/
private CloseableHttpClient createHttpClient() {
// Connection pool configuration
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
connectionManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
connectionManager.setDefaultMaxPerRoute(MAX_PER_ROUTE_CONNECTIONS);
// Request configuration
RequestConfig requestConfig = RequestConfig.custom()
.setConnectTimeout(CONNECT_TIMEOUT)
.setSocketTimeout(SOCKET_TIMEOUT)
.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT)
.build();
// Custom redirect strategy to handle PUT requests and replace internal IPs
DefaultRedirectStrategy redirectStrategy = new DefaultRedirectStrategy() {
@Override
protected boolean isRedirectable(String method) {
// Enable redirect for PUT method (required for Doris Stream Load)
return true;
}
@Override
public HttpUriRequest getRedirect(
HttpRequest request,
HttpResponse response,
HttpContext context) throws ProtocolException {
URI uri = getLocationURI(request, response, context);
String redirectLocation = uri.toString();
// Check if redirect points to localhost/internal IP
String host = uri.getHost();
if (host != null && ("127.0.0.1".equals(host) || "localhost".equals(host)
|| host.startsWith("192.168.") || host.startsWith("10.")
|| "0.0.0.0".equals(host))) {
// Replace internal IP with external FE host, keep BE port and path
int port = uri.getPort() > 0 ? uri.getPort() : 8040;
String path = uri.getPath();
String query = uri.getQuery();
redirectLocation = "http://" + feHost + ":" + port + path
+ (query != null ? "?" + query : "");
log.debug("[Doris StreamLoad] Replaced internal IP {} with external {}:{}",
host, feHost, port);
try {
uri = new URI(redirectLocation);
} catch (URISyntaxException e) {
log.warn("[Doris StreamLoad] Failed to build URI for redirect location", e);
}
}
// Remove any embedded credentials like http://root:@
if (redirectLocation.contains("@")) {
try {
URIBuilder uriBuilder = new URIBuilder(redirectLocation);
uriBuilder.setUserInfo(null);
uri = uriBuilder.build();
} catch (URISyntaxException e) {
log.warn("[Doris StreamLoad] Failed to remove credentials from redirect URL", e);
}
}
// Create new redirect request
HttpPut redirect = new HttpPut(uri);
if (request instanceof HttpEntityEnclosingRequest sourceRequest
&& sourceRequest.getEntity() != null) {
redirect.setEntity(sourceRequest.getEntity());
}
// Copy headers from original request
copyHeaders(request, redirect);
return redirect;
}
private void copyHeaders(HttpRequest source, HttpUriRequest target) {
// Keep all business headers for redirected PUT request.
for (Header header : source.getAllHeaders()) {
String name = header.getName();
if (HttpHeaders.CONTENT_LENGTH.equalsIgnoreCase(name)
|| HttpHeaders.HOST.equalsIgnoreCase(name)
|| HttpHeaders.TRANSFER_ENCODING.equalsIgnoreCase(name)) {
continue;
}
target.setHeader(name, header.getValue());
}
}
};
return HttpClients.custom()
.setConnectionManager(connectionManager)
.setDefaultRequestConfig(requestConfig)
.setRedirectStrategy(redirectStrategy)
.build();
}
/**
* Build Basic Authentication header
*/
private String basicAuthHeader(String username, String password) {
String toBeEncode = username + ":" + (password != null ? password : "");
byte[] encoded = Base64.encodeBase64(toBeEncode.getBytes(StandardCharsets.UTF_8));
return "Basic " + new String(encoded, StandardCharsets.UTF_8);
}
/**
* Build unique label for idempotency
*/
private String buildLabel() {
return "hzb_" + System.currentTimeMillis() + "_" + transactionId.incrementAndGet();
}
/**
* Build Stream Load URL
*/
private String buildLoadUrl() {
return String.format("http://%s:%d/api/%s/%s/_stream_load",
feHost, feHttpPort, databaseName, tableName);
}
/**
* Write metrics data using Stream Load API
*
* @param rows list of metric rows to write
* @return true if write succeeded, false otherwise
*/
public boolean write(List<DorisMetricRow> rows) {
if (!available || rows == null || rows.isEmpty()) {
return false;
}
return writeMetricWithAutoSplit(new ArrayList<>(rows));
}
public boolean writeLogs(List<LogEntry> logEntries) {
if (!available || logEntries == null || logEntries.isEmpty()) {
return false;
}
return writeLogWithAutoSplit(new ArrayList<>(logEntries));
}
private boolean writeMetricWithAutoSplit(List<DorisMetricRow> rows) {
List<StreamLoadMetricRow> streamLoadRows = toStreamLoadRows(rows);
String jsonData = JsonUtil.toJson(streamLoadRows);
if (jsonData == null) {
log.error("[Doris StreamLoad] Failed to serialize {} rows to JSON.", rows.size());
return false;
}
int payloadBytes = jsonData.getBytes(StandardCharsets.UTF_8).length;
int maxBytesPerBatch = config.maxBytesPerBatch();
if (payloadBytes > maxBytesPerBatch && rows.size() > 1) {
int mid = rows.size() / 2;
List<DorisMetricRow> left = new ArrayList<>(rows.subList(0, mid));
List<DorisMetricRow> right = new ArrayList<>(rows.subList(mid, rows.size()));
log.debug("[Doris StreamLoad] Split batch: rows={}, bytes={}, maxBytes={}",
rows.size(), payloadBytes, maxBytesPerBatch);
return writeMetricWithAutoSplit(left) && writeMetricWithAutoSplit(right);
}
String label = buildLabel();
return writeSingleBatch(rows.size(), jsonData, label, METRIC_JSON_PATHS, METRIC_COLUMNS);
}
private boolean writeLogWithAutoSplit(List<LogEntry> logEntries) {
List<StreamLoadLogRow> streamLoadRows = toStreamLoadLogRows(logEntries);
String jsonData = JsonUtil.toJson(streamLoadRows);
if (jsonData == null) {
log.error("[Doris StreamLoad] Failed to serialize {} log entries to JSON.", logEntries.size());
return false;
}
int payloadBytes = jsonData.getBytes(StandardCharsets.UTF_8).length;
int maxBytesPerBatch = config.maxBytesPerBatch();
if (payloadBytes > maxBytesPerBatch && logEntries.size() > 1) {
int mid = logEntries.size() / 2;
List<LogEntry> left = new ArrayList<>(logEntries.subList(0, mid));
List<LogEntry> right = new ArrayList<>(logEntries.subList(mid, logEntries.size()));
log.debug("[Doris StreamLoad] Split log batch: rows={}, bytes={}, maxBytes={}",
logEntries.size(), payloadBytes, maxBytesPerBatch);
return writeLogWithAutoSplit(left) && writeLogWithAutoSplit(right);
}
String label = buildLabel();
return writeSingleBatch(logEntries.size(), jsonData, label, LOG_JSON_PATHS, LOG_COLUMNS);
}
private List<StreamLoadMetricRow> toStreamLoadRows(List<DorisMetricRow> rows) {
List<StreamLoadMetricRow> result = new ArrayList<>(rows.size());
for (DorisMetricRow row : rows) {
StreamLoadMetricRow item = new StreamLoadMetricRow();
item.instance = row.instance;
item.app = row.app;
item.metrics = row.metrics;
item.metric = row.metric;
item.recordTime = formatRecordTime(row.recordTime);
item.metricType = row.metricType;
item.int32Value = row.int32Value;
item.doubleValue = row.doubleValue;
item.strValue = row.strValue;
item.labels = row.labels;
result.add(item);
}
return result;
}
private String formatRecordTime(Timestamp timestamp) {
if (timestamp == null) {
return null;
}
return DORIS_DATETIME_FORMATTER.format(timestamp.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime());
}
private List<StreamLoadLogRow> toStreamLoadLogRows(List<LogEntry> logEntries) {
List<StreamLoadLogRow> result = new ArrayList<>(logEntries.size());
for (LogEntry logEntry : logEntries) {
long timeUnixNano = logEntry.getTimeUnixNano() != null
? logEntry.getTimeUnixNano()
: System.currentTimeMillis() * 1_000_000L;
long observedTimeUnixNano = logEntry.getObservedTimeUnixNano() != null
? logEntry.getObservedTimeUnixNano()
: timeUnixNano;
StreamLoadLogRow row = new StreamLoadLogRow();
row.timeUnixNano = timeUnixNano;
row.observedTimeUnixNano = observedTimeUnixNano;
row.eventTime = formatEpochNanos(timeUnixNano);
row.severityNumber = logEntry.getSeverityNumber();
row.severityText = logEntry.getSeverityText();
row.body = JsonUtil.toJson(logEntry.getBody());
row.traceId = logEntry.getTraceId();
row.spanId = logEntry.getSpanId();
row.traceFlags = logEntry.getTraceFlags();
row.attributes = JsonUtil.toJson(logEntry.getAttributes());
row.resource = JsonUtil.toJson(logEntry.getResource());
row.instrumentationScope = JsonUtil.toJson(logEntry.getInstrumentationScope());
row.droppedAttributesCount = logEntry.getDroppedAttributesCount();
result.add(row);
}
return result;
}
private String formatEpochNanos(long epochNanos) {
long millis = epochNanos / 1_000_000L;
return DORIS_DATETIME_FORMATTER.format(
Instant.ofEpochMilli(millis).atZone(ZoneId.systemDefault()).toLocalDateTime());
}
private boolean writeSingleBatch(int rowCount, String jsonData, String label,
String jsonPaths, String columns) {
String loadUrl = buildLoadUrl();
int maxAttempts = Math.max(1, config.retryTimes() + 1);
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
LoadResult loadResult;
String responseBody = "";
int statusCode = -1;
String action = String.format("label=%s attempt=%d/%d rows=%d",
label, attempt, maxAttempts, rowCount);
HttpPut httpPut = new HttpPut(loadUrl);
httpPut.setHeader(HttpHeaders.EXPECT, "100-continue");
httpPut.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(username, password));
httpPut.setHeader("label", label);
httpPut.setHeader("format", "json");
httpPut.setHeader("strip_outer_array", "true");
httpPut.setHeader("timeout", String.valueOf(config.timeout()));
httpPut.setHeader("max_filter_ratio", String.valueOf(config.maxFilterRatio()));
httpPut.setHeader("strict_mode", String.valueOf(config.strictMode()));
httpPut.setHeader("jsonpaths", jsonPaths);
httpPut.setHeader("columns", columns);
setHeaderIfHasText(httpPut, "timezone", config.timezone());
setHeaderIfHasText(httpPut, "redirect-policy", config.redirectPolicy());
setHeaderIfHasText(httpPut, "group_commit", config.groupCommit());
if (config.sendBatchParallelism() > 0) {
httpPut.setHeader("send_batch_parallelism", String.valueOf(config.sendBatchParallelism()));
}
httpPut.setEntity(new StringEntity(jsonData, StandardCharsets.UTF_8));
try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
statusCode = response.getStatusLine().getStatusCode();
if (response.getEntity() != null) {
responseBody = EntityUtils.toString(response.getEntity());
}
loadResult = handleResponse(statusCode, responseBody, rowCount);
} catch (Exception e) {
log.warn("[Doris StreamLoad] Request error, {}: {}", action, e.getMessage());
loadResult = LoadResult.RETRYABLE_FAILURE;
}
if (loadResult == LoadResult.SUCCESS) {
return true;
}
if (loadResult == LoadResult.NON_RETRYABLE_FAILURE || attempt >= maxAttempts) {
log.error("[Doris StreamLoad] Batch failed, {} statusCode={} response={}",
action, statusCode, responseBody);
return false;
}
try {
Thread.sleep(RETRY_BACKOFF_MS * attempt);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.warn("[Doris StreamLoad] Retry interrupted, {}", action);
return false;
}
}
return false;
}
private void setHeaderIfHasText(HttpPut httpPut, String name, String value) {
if (value != null && !value.isBlank()) {
httpPut.setHeader(name, value.trim());
}
}
/**
* Handle Stream Load response
*
* Note: statusCode 200 only indicates the BE service is OK, not that stream load succeeded.
* We must parse the response JSON to check the actual status.
*/
private LoadResult handleResponse(int statusCode, String body, int rowCount) {
if (statusCode != 200) {
log.warn("[Doris StreamLoad] HTTP request returned status {}, response: {}",
statusCode, body);
return statusCode >= 500 ? LoadResult.RETRYABLE_FAILURE : LoadResult.NON_RETRYABLE_FAILURE;
}
JsonNode jsonNode = JsonUtil.fromJson(body);
if (jsonNode == null || !jsonNode.has("Status")) {
log.error("[Doris StreamLoad] Invalid response body: {}", body);
return LoadResult.NON_RETRYABLE_FAILURE;
}
String status = jsonNode.get("Status").asText("");
if (STATUS_SUCCESS.equalsIgnoreCase(status)) {
log.info("[Doris StreamLoad] Successfully loaded {} rows", rowCount);
return LoadResult.SUCCESS;
}
if (STATUS_PUBLISH_TIMEOUT.equalsIgnoreCase(status)) {
log.warn("[Doris StreamLoad] Publish Timeout for {} rows, treated as success. Response: {}",
rowCount, body);
return LoadResult.SUCCESS;
}
if (STATUS_LABEL_ALREADY_EXISTS.equalsIgnoreCase(status)) {
String existingStatus = jsonNode.has("ExistingJobStatus")
? jsonNode.get("ExistingJobStatus").asText("")
: "";
if ("FINISHED".equalsIgnoreCase(existingStatus) || "VISIBLE".equalsIgnoreCase(existingStatus)) {
log.info("[Doris StreamLoad] Label exists and already finished for {} rows", rowCount);
return LoadResult.SUCCESS;
}
return LoadResult.RETRYABLE_FAILURE;
}
if (STATUS_FAIL.equalsIgnoreCase(status)) {
log.error("[Doris StreamLoad] Failed to load {} rows, status={}, response={}",
rowCount, status, body);
return LoadResult.NON_RETRYABLE_FAILURE;
}
log.warn("[Doris StreamLoad] Unexpected status={} for {} rows, response={}",
status, rowCount, body);
return LoadResult.RETRYABLE_FAILURE;
}
/**
* Get current statistics
*/
public String getStats() {
return String.format("transactions=%d, available=%s", transactionId.get(), available);
}
/**
* Close the HTTP client and release resources
*/
public void close() {
available = false;
try {
if (httpClient != null) {
httpClient.close();
log.info("[Doris StreamLoad] HTTP client closed");
}
} catch (IOException e) {
log.error("[Doris StreamLoad] Failed to close HTTP client", e);
}
}
}
@@ -0,0 +1,567 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.duckdb;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.common.util.TimePeriodUtil;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.time.temporal.TemporalAmount;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
/**
* data storage by duckdb
*/
@Component
@ConditionalOnProperty(prefix = "warehouse.store.duckdb", name = "enabled", havingValue = "true")
@Slf4j
public class DuckdbDatabaseDataStorage extends AbstractHistoryDataStorage {
private static final String DRIVER_NAME = "org.duckdb.DuckDBDriver";
private static final String URL_PREFIX = "jdbc:duckdb:";
// Ideal number of data points for charting (avoids frontend lag)
private static final int TARGET_CHART_POINTS = 800;
// Regex to strictly match days format, e.g., "90d", "7D". Group 1 captures the digits.
private static final Pattern DAY_PATTERN = Pattern.compile("^(\\d+)[dD]$");
private final String expireTimeStr;
private final String dbPath;
private final ScheduledExecutorService cleanerScheduler;
private final ExecutorService cleanerExecutor;
private final ScheduledDispatchTask cleanerTask;
private HikariDataSource dataSource;
public DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties) {
this(duckdbProperties, VirtualThreadProperties.defaults(), true);
}
@Autowired
public DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties,
VirtualThreadProperties virtualThreadProperties) {
this(duckdbProperties, virtualThreadProperties, true);
}
DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties,
VirtualThreadProperties virtualThreadProperties,
boolean autoStartCleaner) {
this.dbPath = duckdbProperties.storePath();
this.expireTimeStr = duckdbProperties.expireTime();
this.cleanerScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
Thread thread = new Thread(r, "duckdb-cleaner");
thread.setDaemon(true);
return thread;
});
this.cleanerExecutor = createCleanerExecutor(virtualThreadProperties);
this.cleanerTask = new ScheduledDispatchTask(cleanerExecutor, this::runExpiredDataCleaner);
this.serverAvailable = initDuckDb();
if (this.serverAvailable && autoStartCleaner) {
startExpiredDataCleaner();
}
}
private boolean initDuckDb() {
try {
Class.forName(DRIVER_NAME);
// Initialize HikariCP
HikariConfig config = new HikariConfig();
config.setJdbcUrl(URL_PREFIX + dbPath);
config.setDriverClassName(DRIVER_NAME);
config.setPoolName("DuckDB-Pool");
// Important: Maintain at least one connection to keep the DB file lock held
// and avoid frequent open/close of the embedded DB file which causes lock errors.
config.setMinimumIdle(1);
config.setMaximumPoolSize(10);
config.setConnectionTimeout(30000);
config.setConnectionTestQuery("SELECT 1");
this.dataSource = new HikariDataSource(config);
try (Connection connection = this.dataSource.getConnection();
Statement statement = connection.createStatement()) {
String createTableSql = """
CREATE TABLE IF NOT EXISTS hzb_history (
instance VARCHAR,
app VARCHAR,
metrics VARCHAR,
metric VARCHAR,
metric_type SMALLINT,
int32_value INTEGER,
double_value DOUBLE,
str_value VARCHAR,
record_time BIGINT,
labels VARCHAR)""";
statement.execute(createTableSql);
// Re-add indexes for performance on queries and cleanup
statement.execute("CREATE INDEX IF NOT EXISTS idx_hzb_history_composite ON hzb_history (instance, app, metrics, metric, record_time)");
statement.execute("CREATE INDEX IF NOT EXISTS idx_hzb_history_record_time ON hzb_history (record_time)");
return true;
}
} catch (Exception e) {
log.error("Failed to init duckdb: {}", e.getMessage(), e);
if (this.dataSource != null) {
this.dataSource.close();
}
return false;
}
}
private void startExpiredDataCleaner() {
// Run every 1 hour
cleanerScheduler.scheduleAtFixedRate(this::dispatchExpiredDataCleaner, 5, 60, TimeUnit.MINUTES);
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS || metricsData.getValues().isEmpty()) {
return;
}
String monitorType = metricsData.getApp();
String metrics = metricsData.getMetrics();
String insertSql = "INSERT INTO hzb_history VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
try (Connection connection = this.dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(insertSql)) {
RowWrapper rowWrapper = metricsData.readRow();
Map<String, String> labels = new HashMap<>();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
long time = metricsData.getTime();
// First pass: collect labels
rowWrapper.cellStream().forEach(cell -> {
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL)) {
labels.put(cell.getField().getName(), cell.getValue());
}
});
String labelsJson = JsonUtil.toJson(labels);
// Second pass: insert data
rowWrapper.cellStream().forEach(cell -> {
try {
String metric = cell.getField().getName();
String columnValue = cell.getValue();
int fieldType = cell.getMetadataAsInteger(MetricDataConstants.TYPE);
preparedStatement.setString(1, metricsData.getInstance());
preparedStatement.setString(2, monitorType);
preparedStatement.setString(3, metrics);
preparedStatement.setString(4, metric);
if (CommonConstants.NULL_VALUE.equals(columnValue)) {
preparedStatement.setShort(5, (short) CommonConstants.TYPE_NUMBER);
preparedStatement.setObject(6, null);
preparedStatement.setObject(7, null);
preparedStatement.setObject(8, null);
} else {
switch (fieldType) {
case CommonConstants.TYPE_STRING -> {
preparedStatement.setShort(5, (short) CommonConstants.TYPE_STRING);
preparedStatement.setObject(6, null);
preparedStatement.setObject(7, null);
preparedStatement.setString(8, columnValue);
}
case CommonConstants.TYPE_TIME -> {
preparedStatement.setShort(5, (short) CommonConstants.TYPE_TIME);
preparedStatement.setInt(6, Integer.parseInt(columnValue));
preparedStatement.setObject(7, null);
preparedStatement.setObject(8, null);
}
default -> {
preparedStatement.setShort(5, (short) CommonConstants.TYPE_NUMBER);
preparedStatement.setObject(6, null);
double v = Double.parseDouble(columnValue);
preparedStatement.setDouble(7, v);
preparedStatement.setObject(8, null);
}
}
}
preparedStatement.setLong(9, time);
preparedStatement.setString(10, labelsJson);
preparedStatement.addBatch();
} catch (SQLException e) {
log.error("error setting prepared statement", e);
}
});
labels.clear();
}
preparedStatement.executeBatch();
} catch (Exception e) {
log.error("[duckdb] save data error: {}", e.getMessage(), e);
}
}
@Override
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
if (!isServerAvailable()) {
return instanceValuesMap;
}
StringBuilder sqlBuilder = new StringBuilder("""
SELECT record_time,
metric_type,
int32_value,
double_value,
str_value,
labels FROM hzb_history
WHERE instance = ?
AND app = ?
AND metrics = ?
AND metric = ?
""");
long timeBefore = 0;
if (history != null) {
try {
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
timeBefore = dateTime.toEpochSecond() * 1000L;
sqlBuilder.append(" AND record_time >= ?");
} catch (Exception e) {
log.error("parse history time error: {}", e.getMessage());
}
}
sqlBuilder.append(" ORDER BY record_time DESC LIMIT 20000"); // Add safety limit for raw data
try (Connection connection = this.dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(sqlBuilder.toString())) {
preparedStatement.setString(1, instance);
preparedStatement.setString(2, app);
preparedStatement.setString(3, metrics);
preparedStatement.setString(4, metric);
if (timeBefore > 0) {
preparedStatement.setLong(5, timeBefore);
}
try (ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
long time = resultSet.getLong("record_time");
int type = resultSet.getShort("metric_type");
String labels = resultSet.getString("labels");
String value = formatValue(type, resultSet);
List<Value> valueList = instanceValuesMap.computeIfAbsent(labels == null ? "" : labels, k -> new LinkedList<>());
valueList.add(new Value(value, time));
}
}
} catch (SQLException e) {
log.error("[duckdb] query data error: {}", e.getMessage(), e);
}
return instanceValuesMap;
}
@Override
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) {
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
if (!isServerAvailable()) {
return instanceValuesMap;
}
long timeBefore = 0;
long endTime = System.currentTimeMillis();
long interval = 0;
if (history != null) {
try {
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
timeBefore = dateTime.toEpochSecond() * 1000L;
// Calculate bucket interval based on desired target points (e.g. 800 points on chart)
interval = (endTime - timeBefore) / TARGET_CHART_POINTS;
// Minimum interval 1 minute
if (interval < 60000) {
interval = 60000;
}
} catch (Exception e) {
log.error("parse history time error: {}", e.getMessage());
// Fallback defaults
timeBefore = System.currentTimeMillis() - 24 * 60 * 60 * 1000L;
interval = 60 * 1000L;
}
}
// DuckDB Aggregation Query: Group by time bucket
// We use casting to integer division for bucketing: (time / interval) * interval
String sql = """
SELECT
CAST(record_time / ? AS BIGINT) * ? AS ts_bucket,
AVG(double_value) AS avg_val,
MIN(double_value) AS min_val,
MAX(double_value) AS max_val,
FIRST(str_value) AS str_val,
metric_type, labels
FROM hzb_history
WHERE instance = ? AND app = ? AND metrics = ? AND metric = ? AND record_time >= ?
GROUP BY ts_bucket, metric_type, labels
ORDER BY ts_bucket""";
try (Connection connection = this.dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(sql)) {
preparedStatement.setLong(1, interval);
preparedStatement.setLong(2, interval);
preparedStatement.setString(3, instance);
preparedStatement.setString(4, app);
preparedStatement.setString(5, metrics);
preparedStatement.setString(6, metric);
preparedStatement.setLong(7, timeBefore);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
long time = resultSet.getLong("ts_bucket");
int type = resultSet.getShort("metric_type");
String labels = resultSet.getString("labels");
Value valueObj;
if (type == CommonConstants.TYPE_NUMBER) {
double avg = resultSet.getDouble("avg_val");
double min = resultSet.getDouble("min_val");
double max = resultSet.getDouble("max_val");
if (resultSet.wasNull()) {
valueObj = new Value(null, time);
} else {
String avgStr = BigDecimal.valueOf(avg).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
String minStr = BigDecimal.valueOf(min).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
String maxStr = BigDecimal.valueOf(max).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
valueObj = new Value(avgStr, time);
valueObj.setMean(avgStr);
valueObj.setMin(minStr);
valueObj.setMax(maxStr);
}
} else {
// For non-numeric, we just took the FIRST value in the bucket
String strVal = resultSet.getString("str_val");
valueObj = new Value(strVal, time);
}
List<Value> valueList = instanceValuesMap.computeIfAbsent(labels == null ? "" : labels, k -> new LinkedList<>());
valueList.add(valueObj);
}
}
} catch (SQLException e) {
log.error("[duckdb] query interval data error: {}", e.getMessage(), e);
}
return instanceValuesMap;
}
private String formatValue(int type, ResultSet resultSet) throws SQLException {
if (type == CommonConstants.TYPE_NUMBER) {
double v = resultSet.getDouble("double_value");
if (!resultSet.wasNull()) {
return BigDecimal.valueOf(v).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
}
} else if (type == CommonConstants.TYPE_TIME) {
int v = resultSet.getInt("int32_value");
if (!resultSet.wasNull()) {
return String.valueOf(v);
}
} else {
return resultSet.getString("str_value");
}
return "";
}
@Override
public void destroy() throws Exception {
cleanerScheduler.shutdownNow();
if (cleanerExecutor != null) {
cleanerExecutor.shutdownNow();
}
if (this.dataSource != null) {
this.dataSource.close();
}
}
void dispatchExpiredDataCleaner() {
cleanerTask.dispatch();
}
void beforeExpiredDataCleanerRun() {
// Test hook.
}
private ExecutorService createCleanerExecutor(VirtualThreadProperties virtualThreadProperties) {
VirtualThreadProperties properties =
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
if (!properties.enabled()) {
return null;
}
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
.name("duckdb-cleaner-vt-", 0)
.uncaughtExceptionHandler((thread, throwable) -> {
log.error("[duckdb] cleaner worker has uncaughtException.");
log.error(throwable.getMessage(), throwable);
})
.factory());
}
private void runExpiredDataCleaner() {
log.info("[duckdb] start data cleaner and checkpoint...");
beforeExpiredDataCleanerRun();
long expireTime;
try {
// Ensure no whitespace issues
String cleanExpireStr = expireTimeStr == null ? "" : expireTimeStr.trim();
Matcher dayMatcher = DAY_PATTERN.matcher(cleanExpireStr);
if (NumberUtils.isParsable(cleanExpireStr)) {
expireTime = NumberUtils.toLong(cleanExpireStr);
expireTime = (ZonedDateTime.now().toEpochSecond() - expireTime) * 1000L;
} else if (dayMatcher.matches()) {
// Strictly matched "90d" or "90D" format
long days = Long.parseLong(dayMatcher.group(1));
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(days));
expireTime = dateTime.toEpochSecond() * 1000L;
} else {
// Fallback to existing utility for other units (h, m, s, etc.)
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(cleanExpireStr);
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
expireTime = dateTime.toEpochSecond() * 1000L;
}
} catch (Exception e) {
log.error("expiredDataCleaner time error: {}. use default expire time to clean: 30d", e.getMessage());
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(30));
expireTime = dateTime.toEpochSecond() * 1000L;
}
try (Connection connection = this.dataSource.getConnection()) {
// 1. Delete expired data
try (PreparedStatement statement = connection.prepareStatement("DELETE FROM hzb_history WHERE record_time < ?")) {
statement.setLong(1, expireTime);
int rows = statement.executeUpdate();
if (rows > 0) {
log.info("[duckdb] delete {} expired records.", rows);
}
}
// 2. Force Checkpoint to compress data and flush WAL
// This is crucial for keeping file size small and moving data from WAL to column store
try (Statement statement = connection.createStatement()) {
statement.execute("CHECKPOINT");
}
} catch (Exception e) {
log.error("[duckdb] clean expired data error: {}", e.getMessage(), e);
}
}
private static final class ScheduledDispatchTask {
private final ExecutorService executorService;
private final Runnable task;
private final Object lock = new Object();
private boolean running;
private int pendingRuns;
private ScheduledDispatchTask(ExecutorService executorService, Runnable task) {
this.executorService = executorService;
this.task = task;
}
private void dispatch() {
if (executorService == null) {
task.run();
return;
}
synchronized (lock) {
if (running) {
pendingRuns++;
return;
}
running = true;
}
submit();
}
private void submit() {
boolean submitted = false;
try {
executorService.execute(() -> {
try {
task.run();
} finally {
onComplete();
}
});
submitted = true;
} finally {
if (!submitted) {
synchronized (lock) {
running = false;
pendingRuns = 0;
}
}
}
}
private void onComplete() {
boolean shouldRunAgain;
synchronized (lock) {
if (pendingRuns > 0) {
pendingRuns--;
shouldRunAgain = true;
} else {
running = false;
shouldRunAgain = false;
}
}
if (shouldRunAgain) {
submit();
}
}
}
}
@@ -0,0 +1,40 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.duckdb;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* Duckdb configuration information
* @param enabled use duckdb store metrics history data
* @param expireTime save data expire time (supports formats like '30d', '90d', or milliseconds)
* @param storePath duckdb database file path
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.DUCKDB)
public record DuckdbProperties(@DefaultValue("true") boolean enabled,
@DefaultValue("90d") String expireTime,
@DefaultValue("data/history.duckdb") String storePath) {
}
@@ -0,0 +1,864 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import io.greptime.GreptimeDB;
import io.greptime.models.AuthInfo;
import io.greptime.models.DataType;
import io.greptime.models.Err;
import io.greptime.models.Result;
import io.greptime.models.Table;
import io.greptime.models.TableSchema;
import io.greptime.models.WriteOk;
import io.greptime.options.GreptimeOptions;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URI;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.Instant;
import java.time.ZonedDateTime;
import java.time.temporal.ChronoUnit;
import java.time.temporal.TemporalAmount;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.BiConsumer;
import java.util.function.Function;
import java.util.stream.Collectors;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.common.util.TimePeriodUtil;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.PromQlQueryContent;
import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.util.MultiValueMap;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponents;
import org.springframework.web.util.UriComponentsBuilder;
/**
* GreptimeDB data storage, only supports GreptimeDB version >= v0.5
*/
@Component
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
@Slf4j
public class GreptimeDbDataStorage extends AbstractHistoryDataStorage {
private static final String BASIC = "Basic";
private static final String QUERY_RANGE_PATH = "/v1/prometheus/api/v1/query_range";
private static final String LABEL_KEY_NAME = "__name__";
private static final String LABEL_KEY_FIELD = "__field__";
private static final String LABEL_KEY_INSTANCE = "instance";
private static final String LOG_TABLE_NAME = WarehouseConstants.LOG_TABLE_NAME;
private static final String LABEL_KEY_START_TIME = "start";
private static final String LABEL_KEY_END_TIME = "end";
private static final String LABEL_KEY_TS = "ts";
private static final int LOG_BATCH_SIZE = 500;
private GreptimeDB greptimeDb;
private final GreptimeProperties greptimeProperties;
private final RestTemplate restTemplate;
private final GreptimeSqlQueryExecutor greptimeSqlQueryExecutor;
public GreptimeDbDataStorage(GreptimeProperties greptimeProperties, RestTemplate restTemplate,
GreptimeSqlQueryExecutor greptimeSqlQueryExecutor) {
if (greptimeProperties == null) {
log.error("init error, please config Warehouse GreptimeDB props in application.yml");
throw new IllegalArgumentException("please config Warehouse GreptimeDB props");
}
this.restTemplate = restTemplate;
this.greptimeProperties = greptimeProperties;
this.greptimeSqlQueryExecutor = greptimeSqlQueryExecutor;
serverAvailable = initGreptimeDbClient(greptimeProperties);
}
private boolean initGreptimeDbClient(GreptimeProperties greptimeProperties) {
String endpoints = greptimeProperties.grpcEndpoints();
try {
GreptimeOptions opts = GreptimeOptions.newBuilder(endpoints.split(","), greptimeProperties.database())
.writeMaxRetries(3)
.authInfo(new AuthInfo(greptimeProperties.username(), greptimeProperties.password()))
.routeTableRefreshPeriodSeconds(30)
.build();
this.greptimeDb = GreptimeDB.create(opts);
} catch (Exception e) {
log.error("[warehouse greptime] Fail to start GreptimeDB client");
return false;
}
return true;
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
return;
}
if (metricsData.getValues().isEmpty()) {
log.info("[warehouse greptime] flush metrics data {} {}is null, ignore.", metricsData.getId(), metricsData.getMetrics());
return;
}
String instance = metricsData.getInstance();
String app = metricsData.getApp();
String tableName = getTableName(app, metricsData.getMetrics());
TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(tableName);
tableSchemaBuilder.addTag("instance", DataType.String)
.addTimestamp("ts", DataType.TimestampMillisecond);
List<CollectRep.Field> fields = metricsData.getFields();
Map<String, String> customLabels = metricsData.getLabels();
List<String> fieldNames = fields.stream().map(CollectRep.Field::getName).collect(Collectors.toList());
fields.forEach(field -> {
if (field.getLabel()) {
tableSchemaBuilder.addTag(field.getName(), DataType.String);
} else {
if (field.getType() == CommonConstants.TYPE_NUMBER) {
tableSchemaBuilder.addField(field.getName(), DataType.Float64);
} else if (field.getType() == CommonConstants.TYPE_STRING) {
tableSchemaBuilder.addField(field.getName(), DataType.String);
}
}
});
List<String> labelKeys = new LinkedList<>();
if (!Objects.isNull(customLabels) && !customLabels.isEmpty()) {
for (Map.Entry<String, String> label : customLabels.entrySet()) {
String key = label.getKey();
if (!LABEL_KEY_INSTANCE.equals(key) && !LABEL_KEY_TS.equals(key) && !fieldNames.contains(key)) {
tableSchemaBuilder.addTag(key, DataType.String);
labelKeys.add(key);
}
}
}
Table table = Table.from(tableSchemaBuilder.build());
long now = System.currentTimeMillis();
Object[] values = new Object[2 + fields.size() + labelKeys.size()];
values[0] = instance;
values[1] = now;
RowWrapper rowWrapper = metricsData.readRow();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
AtomicInteger index = new AtomicInteger(-1);
rowWrapper.cellStream().forEach(cell -> {
index.getAndIncrement();
if (CommonConstants.NULL_VALUE.equals(cell.getValue())) {
values[2 + index.get()] = null;
return;
}
Boolean label = cell.getMetadataAsBoolean(MetricDataConstants.LABEL);
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
if (label) {
values[2 + index.get()] = cell.getValue();
} else {
if (type == CommonConstants.TYPE_NUMBER) {
values[2 + index.get()] = Double.parseDouble(cell.getValue());
} else if (type == CommonConstants.TYPE_STRING) {
values[2 + index.get()] = cell.getValue();
}
}
});
for (int i = 0; i < labelKeys.size(); i++) {
values[2 + fields.size() + i] = customLabels.get(labelKeys.get(i));
}
table.addRow(values);
}
CompletableFuture<Result<WriteOk, Err>> writeFuture = greptimeDb.write(table);
try {
Result<WriteOk, Err> result = writeFuture.get(10, TimeUnit.SECONDS);
if (result.isOk()) {
log.debug("[warehouse greptime]-Write successful");
} else {
log.warn("[warehouse greptime]--Write failed: {}", result.getErr());
}
} catch (Throwable throwable) {
log.error("[warehouse greptime]--Error occurred: {}", throwable.getMessage());
}
}
@Override
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric,
String history) {
Map<String, Long> timeRange = getTimeRange(history);
Long start = timeRange.get(LABEL_KEY_START_TIME);
Long end = timeRange.get(LABEL_KEY_END_TIME);
String step = getTimeStep(start, end);
return getHistoryData(start, end, step, instance, app, metrics, metric);
}
private String getTableName(String app, String metrics) {
return app + "_" + metrics;
}
@Override
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
String metric, String history) {
Map<String, Long> timeRange = getTimeRange(history);
Long start = timeRange.get(LABEL_KEY_START_TIME);
Long end = timeRange.get(LABEL_KEY_END_TIME);
String step = getTimeStep(start, end);
Map<String, List<Value>> instanceValuesMap = getHistoryData(start, end, step, instance, app, metrics, metric);
if (instanceValuesMap.isEmpty()) {
return Collections.emptyMap();
}
// Queries below this point may yield inconsistent results due to exceeding the valid data range.
// Therefore, we restrict the valid range by obtaining the post-query timeframe.
// Since `gretime`'s `end` excludes the specified time, we add 4 hours.
List<Value> values = instanceValuesMap.get(instanceValuesMap.keySet().iterator().next());
// effective time
long effectiveStart = values.get(0).getTime() / 1000;
long effectiveEnd = values.get(values.size() - 1).getTime() / 1000 + Duration.ofHours(4).getSeconds();
String name = getTableName(app, metrics);
String timeSeriesSelector = name + "{" + LABEL_KEY_INSTANCE + "=\"" + instance + "\"";
if (!CommonConstants.PROMETHEUS.equals(app)) {
timeSeriesSelector = timeSeriesSelector + "," + LABEL_KEY_FIELD + "=\"" + metric + "\"";
}
timeSeriesSelector = timeSeriesSelector + "}";
try {
// max
String finalTimeSeriesSelector = timeSeriesSelector;
URI uri = getUri(effectiveStart, effectiveEnd, step, uriComponents -> "max_over_time(" + finalTimeSeriesSelector + "[" + step + "])");
requestIntervalMetricAndPutValue(uri, instanceValuesMap, Value::setMax);
// min
uri = getUri(effectiveStart, effectiveEnd, step, uriComponents -> "min_over_time(" + finalTimeSeriesSelector + "[" + step + "])");
requestIntervalMetricAndPutValue(uri, instanceValuesMap, Value::setMin);
// avg
uri = getUri(effectiveStart, effectiveEnd, step, uriComponents -> "avg_over_time(" + finalTimeSeriesSelector + "[" + step + "])");
requestIntervalMetricAndPutValue(uri, instanceValuesMap, Value::setMean);
} catch (Exception e) {
log.error("query interval metrics data from greptime error. {}", e.getMessage(), e);
}
return instanceValuesMap;
}
/**
* Get time range
*
* @param history history range
* @return time range
*/
private Map<String, Long> getTimeRange(String history) {
// Build start and end times
Instant now = Instant.now();
long start;
try {
if (NumberUtils.isParsable(history)) {
start = NumberUtils.toLong(history);
start = (ZonedDateTime.now().toEpochSecond() - start);
} else {
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
assert temporalAmount != null;
Instant dateTime = now.minus(temporalAmount);
start = dateTime.getEpochSecond();
}
} catch (Exception e) {
log.error("history time error: {}. use default: 6h", e.getMessage());
start = now.minus(6, ChronoUnit.HOURS).getEpochSecond();
}
long end = now.getEpochSecond();
return Map.of("start", start, "end", end);
}
/**
* Get time step
*
* @param start start time
* @param end end time
* @return step
*/
private String getTimeStep(long start, long end) {
// get step
String step = "60s";
if (end - start < Duration.ofDays(7).getSeconds() && end - start > Duration.ofDays(1).getSeconds()) {
step = "1h";
} else if (end - start >= Duration.ofDays(7).getSeconds()) {
step = "4h";
}
return step;
}
/**
* Get history metric data
*
* @param start start time
* @param end end time
* @param step step
* @param instance monitor instance
* @param app monitor type
* @param metrics metrics
* @param metric metric
* @return history metric data
*/
private Map<String, List<Value>> getHistoryData(long start, long end, String step, String instance, String app, String metrics, String metric) {
String name = getTableName(app, metrics);
String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + name + "\""
+ "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\"";
if (!CommonConstants.PROMETHEUS.equals(app)) {
timeSeriesSelector = timeSeriesSelector + "," + LABEL_KEY_FIELD + "=\"" + metric + "\"";
}
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
try {
HttpEntity<Void> httpEntity = getHttpEntity();
String finalTimeSeriesSelector = timeSeriesSelector;
URI uri = getUri(start, end, step, uriComponents -> {
MultiValueMap<String, String> queryParams = uriComponents.getQueryParams();
if (!queryParams.isEmpty()) {
return "{" + finalTimeSeriesSelector + "}";
}
return null;
});
ResponseEntity<PromQlQueryContent> responseEntity = null;
if (uri != null) {
responseEntity = restTemplate.exchange(uri,
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
}
if (responseEntity != null && responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("query metrics data from greptime success. {}", uri);
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_INSTANCE);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
for (Object[] valueArr : content.getValues()) {
long timestamp = ((Double) valueArr[0]).longValue();
String value = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
valueList.add(new Value(value, timestamp * 1000));
}
}
}
}
} else {
log.error("query metrics data from greptime failed. {}", responseEntity);
}
} catch (Exception e) {
log.error("query metrics data from greptime error. {}", e.getMessage(), e);
}
return instanceValuesMap;
}
/**
* Get HTTP instance
*
* @return HTTP instance
*/
private HttpEntity<Void> getHttpEntity() {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
if (StringUtils.hasText(greptimeProperties.username())
&& StringUtils.hasText(greptimeProperties.password())) {
String authStr = greptimeProperties.username() + ":" + greptimeProperties.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, BASIC + " " + encodedAuth);
}
return new HttpEntity<>(headers);
}
/**
* Get Request URI
*
* @param start start time
* @param end end time
* @param step interval
* @param queryFunction request parameters
* @return URI
*/
private URI getUri(long start, long end, String step, Function<UriComponents, String> queryFunction) {
UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromUriString(greptimeProperties.httpEndpoint() + QUERY_RANGE_PATH)
.queryParam("start", start)
.queryParam("end", end)
.queryParam("step", step)
.queryParam("db", greptimeProperties.database());
UriComponents cloneUriComponents = uriComponentsBuilder.cloneBuilder().build(true);
String queryValue = queryFunction.apply(cloneUriComponents);
if (!StringUtils.hasText(queryValue)) {
return null;
}
UriComponents uriComponents = uriComponentsBuilder
.queryParam(
URLEncoder.encode("query", StandardCharsets.UTF_8),
URLEncoder.encode(queryValue, StandardCharsets.UTF_8)
).build(true);
return uriComponents.toUri();
}
/**
* Request greptime and assign a value
*
* @param uri request URI
* @param instanceValuesMap metrics data
* @param valueConsumer consumer used for assigning values
*/
private void requestIntervalMetricAndPutValue(URI uri, Map<String, List<Value>> instanceValuesMap, BiConsumer<Value, String> valueConsumer) {
if (uri == null) {
return;
}
HttpEntity<Void> httpEntity = getHttpEntity();
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri,
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (!responseEntity.getStatusCode().is2xxSuccessful()) {
log.error("query interval metrics data from greptime failed. {}", responseEntity);
return;
}
log.debug("query interval metrics data from greptime success. {}", uri);
PromQlQueryContent body = responseEntity.getBody();
if (body == null || body.getData() == null || body.getData().getResult() == null) {
return;
}
List<PromQlQueryContent.ContentData.Content> contents = body.getData().getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_INSTANCE);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() == null || content.getValues().isEmpty()) {
continue;
}
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
if (valueList.size() == content.getValues().size()) {
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
Value value = valueList.get(timestampIndex);
Object[] valueArr = content.getValues().get(timestampIndex);
String avgValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
valueConsumer.accept(value, avgValue);
}
}
}
}
@Override
public void destroy() {
if (this.greptimeDb != null) {
this.greptimeDb.shutdownGracefully();
this.greptimeDb = null;
}
}
@Override
public void saveLogData(LogEntry logEntry) {
if (!isServerAvailable()) {
return;
}
try {
// Create table schema
TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(LOG_TABLE_NAME);
tableSchemaBuilder.addTimestamp("time_unix_nano", DataType.TimestampNanosecond)
.addField("observed_time_unix_nano", DataType.TimestampNanosecond)
.addField("severity_number", DataType.Int32)
.addField("severity_text", DataType.String)
.addField("body", DataType.Json)
.addField("trace_id", DataType.String)
.addField("span_id", DataType.String)
.addField("trace_flags", DataType.Int32)
.addField("attributes", DataType.Json)
.addField("resource", DataType.Json)
.addField("instrumentation_scope", DataType.Json)
.addField("dropped_attributes_count", DataType.Int32);
Table table = Table.from(tableSchemaBuilder.build());
// Convert LogEntry to table row
Object[] values = new Object[] {
logEntry.getTimeUnixNano() != null ? logEntry.getTimeUnixNano() : System.nanoTime(),
logEntry.getObservedTimeUnixNano() != null ? logEntry.getObservedTimeUnixNano() : System.nanoTime(),
logEntry.getSeverityNumber(),
logEntry.getSeverityText(),
JsonUtil.toJson(logEntry.getBody()),
logEntry.getTraceId(),
logEntry.getSpanId(),
logEntry.getTraceFlags(),
JsonUtil.toJson(logEntry.getAttributes()),
JsonUtil.toJson(logEntry.getResource()),
JsonUtil.toJson(logEntry.getInstrumentationScope()),
logEntry.getDroppedAttributesCount()
};
table.addRow(values);
// Write to GreptimeDB
CompletableFuture<Result<WriteOk, Err>> writeFuture = greptimeDb.write(table);
Result<WriteOk, Err> result = writeFuture.get(10, TimeUnit.SECONDS);
if (result.isOk()) {
log.debug("[warehouse greptime-log] Write successful");
} else {
log.warn("[warehouse greptime-log] Write failed: {}", result.getErr());
}
} catch (Exception e) {
log.error("[warehouse greptime-log] Error saving log entry", e);
}
}
@Override
public List<LogEntry> queryLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber,
String severityText, String searchContent) {
try {
StringBuilder sql = new StringBuilder("SELECT * FROM ").append(LOG_TABLE_NAME);
buildWhereConditions(sql, startTime, endTime, traceId, spanId, severityNumber, severityText, searchContent);
sql.append(" ORDER BY time_unix_nano DESC");
List<Map<String, Object>> rows = greptimeSqlQueryExecutor.execute(sql.toString());
return mapRowsToLogEntries(rows);
} catch (Exception e) {
log.error("[warehouse greptime-log] queryLogsByMultipleConditions error: {}", e.getMessage(), e);
return List.of();
}
}
@Override
public List<LogEntry> queryLogsByMultipleConditionsWithPagination(Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber,
String severityText, String searchContent,
Integer offset, Integer limit) {
try {
StringBuilder sql = new StringBuilder("SELECT * FROM ").append(LOG_TABLE_NAME);
buildWhereConditions(sql, startTime, endTime, traceId, spanId, severityNumber, severityText, searchContent);
sql.append(" ORDER BY time_unix_nano DESC");
// Add pagination
if (limit != null && limit > 0) {
sql.append(" LIMIT ").append(limit);
if (offset != null && offset > 0) {
sql.append(" OFFSET ").append(offset);
}
}
List<Map<String, Object>> rows = greptimeSqlQueryExecutor.execute(sql.toString());
return mapRowsToLogEntries(rows);
} catch (Exception e) {
log.error("[warehouse greptime-log] queryLogsByMultipleConditionsWithPagination error: {}", e.getMessage(), e);
return List.of();
}
}
@Override
public long countLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber,
String severityText, String searchContent) {
try {
StringBuilder sql = new StringBuilder("SELECT COUNT(*) as count FROM ").append(LOG_TABLE_NAME);
buildWhereConditions(sql, startTime, endTime, traceId, spanId, severityNumber, severityText, searchContent);
List<Map<String, Object>> rows = greptimeSqlQueryExecutor.execute(sql.toString());
if (rows != null && !rows.isEmpty()) {
Object countObj = rows.get(0).get("count");
if (countObj instanceof Number) {
return ((Number) countObj).longValue();
}
}
return 0;
} catch (Exception e) {
log.error("[warehouse greptime-log] countLogsByMultipleConditions error: {}", e.getMessage(), e);
return 0;
}
}
private static long msToNs(Long ms) {
return ms * 1_000_000L;
}
private static String safeString(String input) {
if (input == null) {
return "";
}
return input.replace("'", "''");
}
/**
* build WHERE conditions
* @param sql SQL builder
* @param startTime start time
* @param endTime end time
* @param traceId trace id
* @param spanId span id
* @param severityNumber severity number
*/
private void buildWhereConditions(StringBuilder sql, Long startTime, Long endTime, String traceId,
String spanId, Integer severityNumber, String severityText, String searchContent) {
List<String> conditions = new ArrayList<>();
// Time range condition
if (startTime != null && endTime != null) {
conditions.add("time_unix_nano >= " + msToNs(startTime) + " AND time_unix_nano <= " + msToNs(endTime));
}
// TraceId condition
if (StringUtils.hasText(traceId)) {
conditions.add("trace_id = '" + safeString(traceId) + "'");
}
// SpanId condition
if (StringUtils.hasText(spanId)) {
conditions.add("span_id = '" + safeString(spanId) + "'");
}
// Severity condition
if (severityNumber != null) {
conditions.add("severity_number = " + severityNumber);
}
// SeverityText condition
if (StringUtils.hasText(severityText)) {
conditions.add("severity_text = '" + safeString(severityText) + "'");
}
// Search content condition - search in body field
if (StringUtils.hasText(searchContent)) {
conditions.add("body LIKE '%" + safeString(searchContent) + "%'");
}
// Add WHERE clause if there are conditions
if (!conditions.isEmpty()) {
sql.append(" WHERE ").append(String.join(" AND ", conditions));
}
}
private List<LogEntry> mapRowsToLogEntries(List<Map<String, Object>> rows) {
List<LogEntry> list = new LinkedList<>();
if (rows == null || rows.isEmpty()) {
return list;
}
for (Map<String, Object> row : rows) {
try {
LogEntry.InstrumentationScope scope = null;
Object scopeObj = row.get("instrumentation_scope");
if (scopeObj instanceof String scopeStr && StringUtils.hasText(scopeStr)) {
try {
scope = JsonUtil.fromJson(scopeStr, LogEntry.InstrumentationScope.class);
} catch (Exception ignore) {
scope = null;
}
}
Object bodyObj = parseJsonMaybe(row.get("body"));
Map<String, Object> attributes = castToMap(parseJsonMaybe(row.get("attributes")));
Map<String, Object> resource = castToMap(parseJsonMaybe(row.get("resource")));
LogEntry entry = LogEntry.builder()
.timeUnixNano(castToLong(row.get("time_unix_nano")))
.observedTimeUnixNano(castToLong(row.get("observed_time_unix_nano")))
.severityNumber(castToInteger(row.get("severity_number")))
.severityText(castToString(row.get("severity_text")))
.body(bodyObj)
.traceId(castToString(row.get("trace_id")))
.spanId(castToString(row.get("span_id")))
.traceFlags(castToInteger(row.get("trace_flags")))
.attributes(attributes)
.resource(resource)
.instrumentationScope(scope)
.droppedAttributesCount(castToInteger(row.get("dropped_attributes_count")))
.build();
list.add(entry);
} catch (Exception e) {
log.warn("[warehouse greptime-log] map row to LogEntry error: {}", e.getMessage());
}
}
return list;
}
private static Object parseJsonMaybe(Object value) {
if (value == null) return null;
if (value instanceof Map) return value;
if (value instanceof String str) {
String s = str.trim();
if ((s.startsWith("{") && s.endsWith("}")) || (s.startsWith("[") && s.endsWith("]"))) {
try {
return JsonUtil.fromJson(s, Object.class);
} catch (Exception e) {
return s;
}
}
return s;
}
return value;
}
@SuppressWarnings("unchecked")
private static Map<String, Object> castToMap(Object obj) {
if (obj instanceof Map) {
return (Map<String, Object>) obj;
}
return null;
}
private static Long castToLong(Object obj) {
if (obj == null) return null;
if (obj instanceof Number n) return n.longValue();
try {
return Long.parseLong(String.valueOf(obj));
} catch (Exception e) {
return null;
}
}
private static Integer castToInteger(Object obj) {
if (obj == null) return null;
if (obj instanceof Number n) return n.intValue();
try {
return Integer.parseInt(String.valueOf(obj));
} catch (Exception e) {
return null;
}
}
private static String castToString(Object obj) {
return obj == null ? null : String.valueOf(obj);
}
@Override
public boolean batchDeleteLogs(List<Long> timeUnixNanos) {
if (!isServerAvailable() || timeUnixNanos == null || timeUnixNanos.isEmpty()) {
return false;
}
try {
StringBuilder sql = new StringBuilder("DELETE FROM ").append(LOG_TABLE_NAME).append(" WHERE time_unix_nano IN (");
sql.append(timeUnixNanos.stream()
.filter(Objects::nonNull)
.map(String::valueOf)
.collect(Collectors.joining(", ")));
sql.append(")");
greptimeSqlQueryExecutor.execute(sql.toString());
log.info("[warehouse greptime-log] Batch delete executed successfully for {} logs", timeUnixNanos.size());
return true;
} catch (Exception e) {
log.error("[warehouse greptime-log] batchDeleteLogs error: {}", e.getMessage(), e);
return false;
}
}
@Override
public void saveLogDataBatch(List<LogEntry> logEntries) {
if (!isServerAvailable() || logEntries == null || logEntries.isEmpty()) {
return;
}
int total = logEntries.size();
for (int i = 0; i < total; i += LOG_BATCH_SIZE) {
int end = Math.min(i + LOG_BATCH_SIZE, total);
List<LogEntry> batch = logEntries.subList(i, end);
doSaveLogBatch(batch);
}
}
private void doSaveLogBatch(List<LogEntry> logEntries) {
try {
TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(LOG_TABLE_NAME);
tableSchemaBuilder.addTimestamp("time_unix_nano", DataType.TimestampNanosecond)
.addField("observed_time_unix_nano", DataType.TimestampNanosecond)
.addField("severity_number", DataType.Int32)
.addField("severity_text", DataType.String)
.addField("body", DataType.Json)
.addField("trace_id", DataType.String)
.addField("span_id", DataType.String)
.addField("trace_flags", DataType.Int32)
.addField("attributes", DataType.Json)
.addField("resource", DataType.Json)
.addField("instrumentation_scope", DataType.Json)
.addField("dropped_attributes_count", DataType.Int32);
Table table = Table.from(tableSchemaBuilder.build());
for (LogEntry logEntry : logEntries) {
Object[] values = new Object[] {
logEntry.getTimeUnixNano() != null ? logEntry.getTimeUnixNano() : System.nanoTime(),
logEntry.getObservedTimeUnixNano() != null ? logEntry.getObservedTimeUnixNano() : System.nanoTime(),
logEntry.getSeverityNumber(),
logEntry.getSeverityText(),
JsonUtil.toJson(logEntry.getBody()),
logEntry.getTraceId(),
logEntry.getSpanId(),
logEntry.getTraceFlags(),
JsonUtil.toJson(logEntry.getAttributes()),
JsonUtil.toJson(logEntry.getResource()),
JsonUtil.toJson(logEntry.getInstrumentationScope()),
logEntry.getDroppedAttributesCount()
};
table.addRow(values);
}
CompletableFuture<Result<WriteOk, Err>> writeFuture = greptimeDb.write(table);
Result<WriteOk, Err> result = writeFuture.get(10, TimeUnit.SECONDS);
if (result.isOk()) {
log.debug("[warehouse greptime-log] Batch write {} logs successful", logEntries.size());
} else {
log.warn("[warehouse greptime-log] Batch write failed: {}", result.getErr());
}
} catch (Exception e) {
log.error("[warehouse greptime-log] Error saving log entries batch", e);
}
}
}
@@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* GrepTimeDB configuration information
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.GREPTIME)
public record GreptimeProperties(@DefaultValue("false") boolean enabled,
@DefaultValue("127.0.0.1:4001") String grpcEndpoints, @DefaultValue("http://127.0.0.1:4000") String httpEndpoint,
@DefaultValue("public") String database, String username, String password) {
}
@@ -0,0 +1,93 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import com.fasterxml.jackson.annotation.JsonProperty;
import lombok.AllArgsConstructor;
import lombok.Data;
import lombok.NoArgsConstructor;
import java.util.List;
/**
* GreptimeDB SQL query content
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public class GreptimeSqlQueryContent {
private Integer code;
private List<Output> output;
@JsonProperty("execution_time_ms")
private Long executionTimeMs;
/**
* Represents the output of a GreptimeDB SQL query.
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class Output {
private Records records;
/**
* Represents the records returned by a GreptimeDB SQL query.
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class Records {
private Schema schema;
private List<List<Object>> rows;
/**
* Represents the schema of the records returned by a GreptimeDB SQL query.
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class Schema {
@JsonProperty("column_schemas")
private List<ColumnSchema> columnSchemas;
/**
* Represents a column schema in the records returned by a GreptimeDB SQL query.
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public static class ColumnSchema {
private String name;
@JsonProperty("data_type")
private String dataType;
}
}
}
}
}
@@ -0,0 +1,364 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.influxdb;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import okhttp3.ConnectionPool;
import okhttp3.OkHttpClient;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.apache.http.ssl.SSLContexts;
import org.influxdb.InfluxDB;
import org.influxdb.InfluxDBFactory;
import org.influxdb.dto.BatchPoints;
import org.influxdb.dto.Point;
import org.influxdb.dto.Query;
import org.influxdb.dto.QueryResult;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
/**
* HistoryInfluxdbDataStorage class
*/
@Component
@ConditionalOnProperty(prefix = "warehouse.store.influxdb", name = "enabled", havingValue = "true")
@Slf4j
public class InfluxdbDataStorage extends AbstractHistoryDataStorage {
private static final String DATABASE = "hertzbeat";
private static final String SHOW_DATABASE = "SHOW DATABASES";
private static final String CREATE_DATABASE = "CREATE DATABASE %s";
private static final String QUERY_HISTORY_SQL = "SELECT metric_labels, \"%s\" FROM \"%s\" WHERE time >= now() - %s order by time desc";
private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL =
"SELECT FIRST(\"%s\"), MEAN(\"%s\"), MAX(\"%s\"), MIN(\"%s\") FROM \"%s\" WHERE metric_labels = '%s' and time >= now() - %s GROUP BY time(4h)";
private static final String QUERY_INSTANCE_SQL = "show tag values from \"%s\" with key = \"metric_labels\"";
private static final String CREATE_RETENTION_POLICY = "CREATE RETENTION POLICY \"%s_retention\" ON \"%s\" DURATION %s REPLICATION %d DEFAULT";
private InfluxDB influxDb;
public InfluxdbDataStorage(InfluxdbProperties influxdbProperties) {
this.initInfluxDb(influxdbProperties);
}
public void initInfluxDb(InfluxdbProperties influxdbProperties) {
var client = new OkHttpClient.Builder()
.readTimeout(NetworkConstants.HttpClientConstants.READ_TIME_OUT, TimeUnit.SECONDS)
.writeTimeout(NetworkConstants.HttpClientConstants.WRITE_TIME_OUT, TimeUnit.SECONDS)
.connectTimeout(NetworkConstants.HttpClientConstants.CONNECT_TIME_OUT, TimeUnit.SECONDS)
.connectionPool(new ConnectionPool(
NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS,
NetworkConstants.HttpClientConstants.KEEP_ALIVE_TIMEOUT,
TimeUnit.SECONDS)
).sslSocketFactory(defaultSslSocketFactory(), defaultTrustManager())
.hostnameVerifier(noopHostnameVerifier())
.retryOnConnectionFailure(true);
this.influxDb = InfluxDBFactory.connect(influxdbProperties.serverUrl(), influxdbProperties.username(), influxdbProperties.password(), client);
// Close it if your application is terminating, or you are not using it anymore.
Runtime.getRuntime().addShutdownHook(new Thread(influxDb::close));
this.serverAvailable = this.createDatabase(influxdbProperties);
}
private boolean createDatabase(InfluxdbProperties influxdbProperties) {
QueryResult queryResult = this.influxDb.query(new Query(SHOW_DATABASE));
if (queryResult.hasError()) {
log.error("show databases in influxdb error, msg: {}", queryResult.getError());
return false;
}
for (QueryResult.Result result : queryResult.getResults()) {
for (QueryResult.Series series : result.getSeries()) {
for (List<Object> values : series.getValues()) {
if (values.contains(DATABASE)) {
// database exists
return true;
}
}
}
}
// create the database
String createDatabaseSql = String.format(CREATE_DATABASE, DATABASE);
QueryResult createDatabaseResult = this.influxDb.query(new Query(createDatabaseSql));
if (createDatabaseResult.hasError()) {
log.error("create database {} in influxdb error, msg: {}", DATABASE, createDatabaseResult.getError());
return false;
}
// set the expiration time
String createRetentionPolicySql = String.format(CREATE_RETENTION_POLICY, DATABASE, DATABASE,
influxdbProperties.expireTime(), influxdbProperties.replication());
QueryResult createRetentionPolicySqlResult = this.influxDb.query(new Query(createRetentionPolicySql));
if (createRetentionPolicySqlResult.hasError()) {
log.error("create retention policy in influxdb error, msg: {}", createDatabaseResult.getError());
return false;
}
return true;
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
return;
}
if (metricsData.getValues().isEmpty()) {
log.info("[warehouse influxdb] flush metrics data {} is null, ignore.", metricsData.getInstance());
return;
}
String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance());
List<Point> points = new ArrayList<>();
try {
RowWrapper rowWrapper = metricsData.readRow();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
Point.Builder builder = Point.measurement(table);
builder.time(metricsData.getTime(), TimeUnit.MILLISECONDS);
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
rowWrapper.cellStream().forEach(cell -> {
if (CommonConstants.NULL_VALUE.equals(cell.getValue())) {
builder.addField(cell.getField().getName(), "");
return;
}
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
if (type == CommonConstants.TYPE_NUMBER) {
builder.addField(cell.getField().getName(), Double.parseDouble(cell.getValue()));
} else if (type == CommonConstants.TYPE_STRING) {
builder.addField(cell.getField().getName(), cell.getValue());
}
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL)) {
labels.put(cell.getField().getName(), cell.getValue());
}
});
builder.tag("metric_labels", JsonUtil.toJson(labels));
points.add(builder.build());
}
BatchPoints.Builder builder = BatchPoints.database(DATABASE);
builder.points(points);
this.influxDb.write(builder.build());
} catch (Exception e) {
log.error("[warehouse influxdb]--Error: {}", e.getMessage(), e);
}
}
@Override
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
String table = this.generateTable(app, metrics, instance);
String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history);
Map<String, List<Value>> instanceValueMap = new HashMap<>(8);
try {
QueryResult selectResult = this.influxDb.query(new Query(selectSql, DATABASE), TimeUnit.MILLISECONDS);
for (QueryResult.Result result : selectResult.getResults()) {
if (result.getSeries() == null) {
continue;
}
for (QueryResult.Series series : result.getSeries()) {
for (List<Object> value : series.getValues()) {
long time = this.parseTimeToMillis(value.get(0));
String instanceValue = value.get(1) == null ? "" : String.valueOf(value.get(1));
String strValue = value.get(2) == null ? null : this.parseDoubleValue(value.get(2).toString());
if (strValue == null) {
continue;
}
List<Value> valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
valueList.add(new Value(strValue, time));
}
}
}
} catch (Exception e) {
log.error("select history metric data in influxdb error, sql:{}, msg: {}", selectSql, e.getMessage());
}
return instanceValueMap;
}
@Override
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) {
String table = this.generateTable(app, metrics, instance);
Map<String, List<Value>> instanceValueMap = new HashMap<>(8);
Set<String> instances = new HashSet<>(8);
// query all metric_labels
String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table);
QueryResult instanceQueryResult = this.influxDb.query(new Query(queryInstanceSql, DATABASE), TimeUnit.MILLISECONDS);
for (QueryResult.Result result : instanceQueryResult.getResults()) {
if (result.getSeries() == null) {
continue;
}
for (QueryResult.Series series : result.getSeries()) {
for (List<Object> value : series.getValues()) {
if (value != null && value.get(1) != null) {
instances.add(value.get(1).toString());
}
}
}
}
try {
history = history.toLowerCase();
if (instances.isEmpty()) {
instances.add("");
}
for (String instanceValue : instances) {
String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, metric, metric, metric, table, instanceValue, history);
QueryResult selectResult = this.influxDb.query(new Query(selectSql, DATABASE), TimeUnit.MILLISECONDS);
for (QueryResult.Result result : selectResult.getResults()) {
if (result.getSeries() == null) {
continue;
}
for (QueryResult.Series series : result.getSeries()) {
for (List<Object> value : series.getValues()) {
Value.ValueBuilder valueBuilder = Value.builder();
long time = this.parseTimeToMillis(value.get(0));
valueBuilder.time(time);
if (value.get(1) != null) {
valueBuilder.origin(this.parseDoubleValue(value.get(1).toString()));
} else {
continue;
}
if (value.get(2) != null) {
valueBuilder.mean(this.parseDoubleValue(value.get(2).toString()));
} else {
continue;
}
if (value.get(3) != null) {
valueBuilder.max(this.parseDoubleValue(value.get(3).toString()));
} else {
continue;
}
if (value.get(4) != null) {
valueBuilder.min(this.parseDoubleValue(value.get(4).toString()));
} else {
continue;
}
List<Value> valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
valueList.add(valueBuilder.build());
}
}
}
List<Value> instanceValueList = instanceValueMap.get(instanceValue);
if (instanceValueList == null || instanceValueList.isEmpty()) {
instanceValueMap.remove(instanceValue);
}
}
} catch (Exception e) {
log.error("select history interval metric data in influxdb error, msg: {}", e.getMessage());
}
return instanceValueMap;
}
private String generateTable(String app, String metrics, String instance) {
if (instance.contains(".") || instance.contains(":") || instance.contains("[")) {
instance = instance.replace(".", "_")
.replace(":", "_")
.replace("[", "_")
.replace("]", "_");
}
return app + "_" + metrics + "_" + instance;
}
private long parseTimeToMillis(Object time) {
if (time == null) {
return 0;
}
Double doubleTime = (Double) time;
return doubleTime.longValue();
}
private String parseDoubleValue(String value) {
return (new BigDecimal(value)).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
}
private static X509TrustManager defaultTrustManager() {
return new X509TrustManager() {
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
@Override
public void checkClientTrusted(X509Certificate[] certs, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
};
}
private static SSLSocketFactory defaultSslSocketFactory() {
try {
SSLContext sslContext = SSLContexts.createDefault();
sslContext.init(null, new TrustManager[]{
defaultTrustManager()
}, new SecureRandom());
return sslContext.getSocketFactory();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static HostnameVerifier noopHostnameVerifier() {
return (s, sslSession) -> true;
}
@Override
public void destroy() throws Exception {
if (this.influxDb != null) {
this.influxDb.close();
}
}
}
@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.influxdb;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* Influxdb configuration information
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.INFLUXDB)
public record InfluxdbProperties(@DefaultValue("false") boolean enabled,
String serverUrl,
String username,
String password,
@DefaultValue("30d") String expireTime,
@DefaultValue("1") int replication) {
}
@@ -0,0 +1,469 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.iotdb;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import com.google.common.collect.Maps;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.apache.iotdb.rpc.IoTDBConnectionException;
import org.apache.iotdb.rpc.StatementExecutionException;
import org.apache.iotdb.session.pool.SessionDataSetWrapper;
import org.apache.iotdb.session.pool.SessionPool;
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
import org.apache.iotdb.tsfile.read.common.RowRecord;
import org.apache.iotdb.tsfile.write.record.Tablet;
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
/**
* IoTDB data storage
*/
@Component
@ConditionalOnProperty(prefix = "warehouse.store.iot-db", name = "enabled", havingValue = "true")
@Slf4j
public class IotDbDataStorage extends AbstractHistoryDataStorage {
private static final String BACK_QUOTE = "`";
/**
* set ttl never expire
*/
private static final String NEVER_EXPIRE = "-1";
/**
* storage group
*/
private static final String STORAGE_GROUP = "root.hertzbeat";
private static final String SHOW_DATABASE = "show databases %s";
private static final String CREATE_DATABASE = "create database %s";
private static final String SET_TTL = "set ttl to %s %s";
private static final String CANCEL_TTL = "unset ttl to %s";
private static final String SHOW_DEVICES = "SHOW DEVICES %s";
private static final String SHOW_STORAGE_GROUP = "show storage group";
private static final String QUERY_HISTORY_SQL =
"SELECT %s FROM %s WHERE Time >= now() - %s order by Time desc";
private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL =
"SELECT FIRST_VALUE(%s), AVG(%s), MIN_VALUE(%s), MAX_VALUE(%s) FROM %s GROUP BY ([now() - %s, now()), 4h)";
private SessionPool sessionPool;
private long queryTimeoutInMs;
public IotDbDataStorage(IotDbProperties iotDbProperties) {
this.serverAvailable = this.initIotDbSession(iotDbProperties);
}
private boolean initIotDbSession(IotDbProperties properties) {
SessionPool.Builder builder = new SessionPool.Builder();
builder.host(properties.host());
if (properties.rpcPort() != null) {
builder.port(properties.rpcPort());
}
if (properties.username() != null) {
builder.user(properties.username());
}
if (properties.password() != null) {
builder.password(properties.password());
}
if (properties.nodeUrls() != null && !properties.nodeUrls().isEmpty()) {
builder.nodeUrls(properties.nodeUrls());
}
if (properties.zoneId() != null) {
builder.zoneId(properties.zoneId());
}
this.queryTimeoutInMs = properties.queryTimeoutInMs();
this.sessionPool = builder.build();
boolean available = checkConnection();
if (!available) {
log.error("IotDB session pool init error with check connection");
return false;
}
available = this.createDatabase();
if (!available) {
log.error("IotDB session pool init error with create database");
return false;
}
this.initTtl(properties.expireTime());
log.info("IotDB session pool init success");
return true;
}
private boolean checkConnection() {
try {
this.sessionPool.executeNonQueryStatement(SHOW_STORAGE_GROUP);
return true;
} catch (Exception e) {
log.error(e.getMessage(), e);
return false;
}
}
private boolean createDatabase() {
SessionDataSetWrapper dataSet = null;
try {
// if root.hertzbeat database not exist, create database
String showDatabaseSql = String.format(SHOW_DATABASE, STORAGE_GROUP);
dataSet = this.sessionPool.executeQueryStatement(showDatabaseSql);
if (!dataSet.hasNext()) {
String createDatabaseSql = String.format(CREATE_DATABASE, STORAGE_GROUP);
this.sessionPool.executeNonQueryStatement(createDatabaseSql);
}
} catch (IoTDBConnectionException | StatementExecutionException e) {
log.error("create database error, error: {}", e.getMessage());
return false;
} finally {
if (dataSet != null) {
this.sessionPool.closeResultSet(dataSet);
}
}
return true;
}
private void initTtl(String expireTime) {
if (expireTime == null || expireTime.isEmpty()) {
return;
}
try {
if (NEVER_EXPIRE.equals(expireTime)) {
// DELETE TTL that might already exist
String cancelTtlSql = String.format(CANCEL_TTL, STORAGE_GROUP);
this.sessionPool.executeNonQueryStatement(cancelTtlSql);
} else {
String sstTtlSql = String.format(SET_TTL, STORAGE_GROUP, expireTime);
this.sessionPool.executeNonQueryStatement(sstTtlSql);
}
} catch (IoTDBConnectionException | StatementExecutionException e) {
// Failure does not affect the primary business
log.error("IoTDB init ttl error, expireTime: {}, error: {}", expireTime, e.getMessage());
}
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
return;
}
if (metricsData.getValues().isEmpty()) {
log.info("[warehouse iotdb] flush metrics data {} is null, ignore.", metricsData.getInstance());
return;
}
List<MeasurementSchema> schemaList = new ArrayList<>();
Map<String, Tablet> tabletMap = Maps.newHashMapWithExpectedSize(8);
// todo Measurement schema is a data structure that is generated on the client side, and encoding and compression have no effect
try {
metricsData.getFields().forEach(field -> {
MeasurementSchema schema = new MeasurementSchema();
schema.setMeasurementId(field.getName());
byte type = (byte) field.getType();
// handle field type
if (type == CommonConstants.TYPE_NUMBER) {
schema.setType(TSDataType.DOUBLE);
} else if (type == CommonConstants.TYPE_STRING) {
schema.setType(TSDataType.TEXT);
}
schemaList.add(schema);
});
long now = System.currentTimeMillis();
RowWrapper rowWrapper = metricsData.readRow();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
rowWrapper.cellStream().forEach(cell -> {
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL) && !CommonConstants.NULL_VALUE.equals(cell.getValue())) {
labels.put(cell.getField().getName(), cell.getValue());
}
});
String label = JsonUtil.toJson(labels);
String deviceId = getDeviceId(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance(), label, true);
if (tabletMap.containsKey(label)) {
// Avoid Time repeats
now++;
} else {
tabletMap.put(label, new Tablet(deviceId, schemaList));
}
Tablet tablet = tabletMap.get(label);
int rowIndex = tablet.rowSize++;
tablet.addTimestamp(rowIndex, now);
rowWrapper.cellStream().forEach(cell -> {
if (CommonConstants.NULL_VALUE.equals(cell.getValue())) {
tablet.addValue(cell.getField().getName(), rowIndex, null);
return;
}
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
if (type == CommonConstants.TYPE_NUMBER) {
tablet.addValue(cell.getField().getName(), rowIndex, Double.parseDouble(cell.getValue()));
} else if (type == CommonConstants.TYPE_STRING) {
tablet.addValue(cell.getField().getName(), rowIndex, cell.getValue());
}
});
}
for (Tablet tablet : tabletMap.values()) {
this.sessionPool.insertTablet(tablet, true);
}
} catch (Exception e) {
log.error(e.getMessage(), e);
} finally {
for (Tablet tablet : tabletMap.values()) {
tablet.reset();
}
tabletMap.clear();
}
}
@Override
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric,
String history) {
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
if (!isServerAvailable()) {
log.error("""
\t---------------IotDb Init Failed---------------
\t--------------Please Config IotDb--------------
\t----------Can Not Use Metric History Now----------
""");
return instanceValuesMap;
}
String deviceId = getDeviceId(app, metrics, instance, null, true);
String selectSql = "";
try {
// query all devices
List<String> devices = queryAllDevices(deviceId);
if (devices.isEmpty()) {
log.warn("no iot device found for deviceId: {}", deviceId);
return instanceValuesMap;
}
for (String device : devices) {
selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), device, history);
String labels = extractLabelsFromDevice(device, deviceId);
handleHistorySelect(selectSql, labels, instanceValuesMap);
}
} catch (StatementExecutionException | IoTDBConnectionException e) {
log.error("select error history sql: {}", selectSql);
log.error(e.getMessage(), e);
}
return instanceValuesMap;
}
private void handleHistorySelect(String selectSql, String labels, Map<String, List<Value>> instanceValuesMap)
throws IoTDBConnectionException, StatementExecutionException {
SessionDataSetWrapper dataSet = null;
try {
dataSet = this.sessionPool.executeQueryStatement(selectSql, this.queryTimeoutInMs);
log.debug("iot select sql: {}", selectSql);
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
long timestamp = rowRecord.getTimestamp();
double value = rowRecord.getFields().get(0).getDoubleV();
String strValue = BigDecimal.valueOf(value).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
List<Value> valueList = instanceValuesMap.computeIfAbsent(labels, k -> new LinkedList<>());
valueList.add(new Value(strValue, timestamp));
}
} finally {
if (dataSet != null) {
// need to close the result set! ! ! otherwise it will cause server-side heap
this.sessionPool.closeResultSet(dataSet);
}
}
}
@Override
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
String metric, String history) {
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
if (!isServerAvailable()) {
log.error("""
\t---------------IotDb Init Failed---------------
\t--------------Please Config IotDb--------------
\t----------Can Not Use Metric History Now----------
""");
return instanceValuesMap;
}
String deviceId = getDeviceId(app, metrics, instance, null, true);
String selectSql;
List<String> devices = queryAllDevices(deviceId);
if (devices.isEmpty()) {
log.warn("no iot device found for deviceId: {}", deviceId);
return instanceValuesMap;
} else {
for (String device : devices) {
selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL,
addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), device, history);
String labels = extractLabelsFromDevice(device, deviceId);
handleHistoryIntervalSelect(selectSql, labels, instanceValuesMap);
}
}
return instanceValuesMap;
}
private void handleHistoryIntervalSelect(String selectSql, String instance,
Map<String, List<Value>> instanceValuesMap) {
SessionDataSetWrapper dataSet = null;
try {
dataSet = this.sessionPool.executeQueryStatement(selectSql, this.queryTimeoutInMs);
log.debug("iot select sql: {}", selectSql);
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
if (rowRecord.hasNullField()) {
continue;
}
long timestamp = rowRecord.getTimestamp();
double origin = rowRecord.getFields().get(0).getDoubleV();
String originStr = BigDecimal.valueOf(origin).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
double avg = rowRecord.getFields().get(1).getDoubleV();
String avgStr = BigDecimal.valueOf(avg).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
double min = rowRecord.getFields().get(2).getDoubleV();
String minStr = BigDecimal.valueOf(min).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
double max = rowRecord.getFields().get(3).getDoubleV();
String maxStr = BigDecimal.valueOf(max).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
Value value = Value.builder()
.origin(originStr).mean(avgStr)
.min(minStr).max(maxStr)
.time(timestamp)
.build();
List<Value> valueList = instanceValuesMap.computeIfAbsent(instance, k -> new LinkedList<>());
valueList.add(value);
}
} catch (StatementExecutionException | IoTDBConnectionException e) {
log.error("select error history interval sql: {}", selectSql);
log.error(e.getMessage(), e);
} finally {
if (dataSet != null) {
// need to close the result set! ! ! otherwise it will cause server-side heap
this.sessionPool.closeResultSet(dataSet);
}
}
}
/**
* Query all devices by deviceId
*
* @param deviceId deviceId
*/
private List<String> queryAllDevices(String deviceId) {
List<String> devices = new ArrayList<>();
List<String> sqls = new ArrayList<>();
sqls.add(String.format(SHOW_DEVICES, deviceId));
sqls.add(String.format(SHOW_DEVICES, deviceId + ".*"));
for (String sql : sqls) {
SessionDataSetWrapper dataSet = null;
try {
dataSet = this.sessionPool.executeQueryStatement(sql, this.queryTimeoutInMs);
while (dataSet.hasNext()) {
RowRecord rowRecord = dataSet.next();
devices.add(rowRecord.getFields().get(0).getStringValue());
}
} catch (StatementExecutionException | IoTDBConnectionException e) {
log.error("query show devices sql error. sql: {}", sql);
log.error(e.getMessage(), e);
} finally {
if (dataSet != null) {
// need to close the result set! ! ! otherwise it will cause server-side heap
this.sessionPool.closeResultSet(dataSet);
}
}
}
return devices;
}
/**
* use ${group}.${app}.${metrics}.${monitor}.${metric_labels} to get device id if labels exist
* otherwise use ${group}.${app}.${metrics}.${monitor}
* Use ${group}.${app}.${metrics}.${monitor}.* to get all instance data when you tend to query
*/
private String getDeviceId(String app, String metrics, String instance, String labels, boolean useQuote) {
if (instance.contains(".") || instance.contains(":") || instance.contains("[")) {
instance = instance.replace(".", "_")
.replace(":", "_")
.replace("[", "_")
.replace("]", "_");
}
String deviceId = STORAGE_GROUP + "."
+ (useQuote ? addQuote(app) : app) + "."
+ (useQuote ? addQuote(metrics) : metrics) + "."
+ addQuote(instance);
if (labels != null && !labels.isEmpty() && !labels.equals(CommonConstants.NULL_VALUE) && !"{}".equals(labels)) {
deviceId += "." + addQuote(labels);
}
return deviceId;
}
/**
* Extract labels from device path
*/
private String extractLabelsFromDevice(String device, String baseDeviceId) {
if (device.length() > baseDeviceId.length() + 1) {
String labelsPart = device.substring(baseDeviceId.length() + 1);
return labelsPart.replace("`", "");
}
return "";
}
/**
* add quote,prevents keyword errors during queries(eg: nodes)
*/
private String addQuote(String text) {
if (text == null || text.isEmpty() || (text.startsWith(BACK_QUOTE) && text.endsWith(BACK_QUOTE))) {
return text;
}
text = text.replace("*", "-");
text = String.format("`%s`", text);
return text;
}
@Override
public void destroy() {
if (this.sessionPool != null) {
this.sessionPool.close();
}
}
}
@@ -0,0 +1,53 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.iotdb;
import java.time.ZoneId;
import java.util.List;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* IotDB configuration information
* @param enabled Whether the iotDB data store is enabled
* @param host iotDB host
* @param expireTime save data expire time(ms)-1 means it never expires Data storage time (unit: ms,-1 means never expire)
* Note: Why is String used here instead of Long? At present, the set ttl of IoTDB only supports milliseconds as a unit,
* and other units may be added later, so the String type is used for compatibility Data storage time (unit: ms, -1 means never expires)
* Note: Why use String instead of Long here? Currently, IoTDB's set ttl only supports milliseconds as the unit.
* Other units may be added later. In order to be compatible with the future, the String type is used.
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.IOT_DB)
public record IotDbProperties(@DefaultValue("false") boolean enabled,
@DefaultValue("127.0.0.1") String host,
@DefaultValue("6667") Integer rpcPort,
String username,
String password,
List<String> nodeUrls,
ZoneId zoneId,
long queryTimeoutInMs,
String expireTime) {
}
@@ -0,0 +1,414 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.questdb;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URLEncoder;
import java.nio.charset.StandardCharsets;
import java.security.SecureRandom;
import java.security.cert.X509Certificate;
import java.time.Instant;
import java.util.Base64;
import java.util.HashMap;
import java.util.HashSet;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.concurrent.TimeUnit;
import javax.net.ssl.HostnameVerifier;
import javax.net.ssl.SSLContext;
import javax.net.ssl.SSLSocketFactory;
import javax.net.ssl.TrustManager;
import javax.net.ssl.X509TrustManager;
import com.google.common.collect.Maps;
import io.questdb.client.Sender;
import lombok.extern.slf4j.Slf4j;
import okhttp3.ConnectionPool;
import okhttp3.OkHttpClient;
import okhttp3.Request;
import okhttp3.Response;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.apache.http.ssl.SSLContexts;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.stereotype.Component;
/**
* HistoryQuestdbDataStorage class
*/
@Component
@ConditionalOnProperty(prefix = "warehouse.store.questdb", name = "enabled", havingValue = "true")
@Slf4j
public class QuestdbDataStorage extends AbstractHistoryDataStorage {
private static final String QUERY_HISTORY_SQL = "SELECT timestamp AS ts, metric_labels, \"%s\" AS value FROM \"%s\" WHERE timestamp >= %s ORDER BY timestamp DESC";
private static final String QUERY_HISTORY_SQL_WITH_INSTANCE =
"SELECT timestamp AS ts, metric_labels, \"%s\" AS value FROM \"%s\" WHERE metric_labels = '%s' AND timestamp >= %s ORDER BY timestamp DESC";
private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL =
"SELECT timestamp AS ts, first(\"%s\") AS origin, avg(\"%s\") AS mean, max(\"%s\") AS max, min(\"%s\") AS min FROM \"%s\" WHERE metric_labels = '%s' AND timestamp >= %s SAMPLE BY 4h";
private static final String QUERY_INSTANCE_SQL = "SELECT DISTINCT metric_labels FROM \"%s\"";
private Sender sender;
private OkHttpClient client;
private String queryBaseUrl;
private QuestdbProperties questdbProperties;
public QuestdbDataStorage(QuestdbProperties questdbProperties) {
this.questdbProperties = questdbProperties;
this.initQuestDb(questdbProperties);
}
public void initQuestDb(QuestdbProperties questdbProperties) {
String ilpAddress = questdbProperties.url(); // e.g., "localhost:9009"
this.sender = Sender.builder(Sender.Transport.HTTP)
.address(ilpAddress)
.httpUsernamePassword(questdbProperties.username(), questdbProperties.password())
.build();
// Parse host and set query base URL (assuming HTTP port is 9000)
String[] parts = ilpAddress.split(":");
String host = parts[0];
String queryPort = "9000"; // Default QuestDB HTTP port
this.queryBaseUrl = "http://" + host + ":" + queryPort + "/exec?query=";
this.client = new OkHttpClient.Builder()
.readTimeout(NetworkConstants.HttpClientConstants.READ_TIME_OUT, TimeUnit.SECONDS)
.writeTimeout(NetworkConstants.HttpClientConstants.WRITE_TIME_OUT, TimeUnit.SECONDS)
.connectTimeout(NetworkConstants.HttpClientConstants.CONNECT_TIME_OUT, TimeUnit.SECONDS)
.connectionPool(new ConnectionPool(
NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS,
NetworkConstants.HttpClientConstants.KEEP_ALIVE_TIMEOUT,
TimeUnit.SECONDS)
).sslSocketFactory(defaultSslSocketFactory(), defaultTrustManager())
.hostnameVerifier(noopHostnameVerifier())
.retryOnConnectionFailure(true)
.build();
this.serverAvailable = this.checkConnection();
}
private boolean checkConnection() {
// Test query to check if server is available
Map<String, Object> result = executeQuery("SELECT 1");
return result != null && result.containsKey("dataset");
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS || metricsData.getValues().isEmpty()) {
return;
}
String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance());
try {
RowWrapper rowWrapper = metricsData.readRow();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
// Wrap the construction of each row in a try-catch block
// to prevent one bad row from corrupting the sender's state.
try {
// 1. Set the table for the new row.
sender.table(table);
// 2. Process and write all symbols FIRST.
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
rowWrapper.cellStream()
.filter(cell -> cell.getMetadataAsBoolean(MetricDataConstants.LABEL))
.forEach(cell -> labels.put(cell.getField().getName(), cell.getValue()));
if (!labels.isEmpty()) {
sender.symbol("metric_labels", JsonUtil.toJson(labels));
} else {
sender.symbol("metric_labels", metricsData.getApp()
+ "_" + metricsData.getMetrics());
}
// 3. Now, process and write all other columns (fields).
rowWrapper.cellStream().forEach(cell -> {
if (CommonConstants.NULL_VALUE.equals(cell.getValue())) {
return;
}
String fieldName = cell.getField().getName();
String fieldValue = cell.getValue();
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
if (type == CommonConstants.TYPE_NUMBER) {
sender.doubleColumn(fieldName, Double.parseDouble(fieldValue));
} else if (type == CommonConstants.TYPE_STRING) {
sender.stringColumn(fieldName, fieldValue);
}
});
// 4. Finally, set the timestamp to commit the row.
sender.atNow();
} catch (Exception e) {
log.error("[warehouse questdb]--Could not process a row, cancelling it. Error: {}", e.getMessage());
// IMPORTANT: Cancel the partially built row to reset the sender's state.
sender.cancelRow();
}
}
sender.flush();
} catch (Exception e) {
log.error("[warehouse questdb]--Error during batch save: {}", e.getMessage(), e);
}
}
@Override
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
String table = this.generateTable(app, metrics, instance);
String dateAdd = getDateAdd(history);
String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, dateAdd);
Map<String, List<Value>> instanceValueMap = new HashMap<>(8);
try {
Map<String, Object> selectResult = executeQuery(selectSql);
if (selectResult == null || !selectResult.containsKey("dataset")) {
return instanceValueMap;
}
List<Map<String, String>> columns = (List<Map<String, String>>) selectResult.get("columns");
List<List<Object>> dataset = (List<List<Object>>) selectResult.get("dataset");
Map<String, Integer> colMap = new HashMap<>();
for (int i = 0; i < columns.size(); i++) {
colMap.put(columns.get(i).get("name"), i);
}
int tsIdx = colMap.get("ts");
int metricLabelsIdx = colMap.get("metric_labels");
int valueIdx = colMap.get("value");
for (List<Object> row : dataset) {
String tsStr = (String) row.get(tsIdx);
long time = Instant.parse(tsStr).toEpochMilli();
String instanceValue = row.get(metricLabelsIdx) == null ? "" : (String) row.get(metricLabelsIdx);
Object valObj = row.get(valueIdx);
String strValue = valObj == null ? null : this.parseDoubleValue(valObj.toString());
if (strValue == null) {
continue;
}
List<Value> valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
valueList.add(new Value(strValue, time));
}
} catch (Exception e) {
log.error("select history metric data in questdb error, sql:{}, msg: {}", selectSql, e.getMessage());
}
return instanceValueMap;
}
@Override
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) {
String table = this.generateTable(app, metrics, instance);
String dateAdd = getDateAdd(history);
Map<String, List<Value>> instanceValueMap = new HashMap<>(8);
Set<String> instances = new HashSet<>(8);
// query all metric_labels
String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table);
Map<String, Object> instanceQueryResult = executeQuery(queryInstanceSql);
if (instanceQueryResult != null && instanceQueryResult.containsKey("dataset")) {
List<List<Object>> dataset = (List<List<Object>>) instanceQueryResult.get("dataset");
for (List<Object> row : dataset) {
if (!row.isEmpty() && row.get(0) != null) {
instances.add(row.get(0).toString());
}
}
}
try {
if (instances.isEmpty()) {
instances.add("");
}
for (String instanceValue : instances) {
String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, metric, metric, metric, table, instanceValue.replace("'", "\\'"), dateAdd);
Map<String, Object> selectResult = executeQuery(selectSql);
if (selectResult == null || !selectResult.containsKey("dataset")) {
continue;
}
List<Map<String, String>> columns = (List<Map<String, String>>) selectResult.get("columns");
List<List<Object>> dataset = (List<List<Object>>) selectResult.get("dataset");
Map<String, Integer> colMap = new HashMap<>();
for (int i = 0; i < columns.size(); i++) {
colMap.put(columns.get(i).get("name"), i);
}
int tsIdx = colMap.get("ts");
int originIdx = colMap.get("origin");
int meanIdx = colMap.get("mean");
int maxIdx = colMap.get("max");
int minIdx = colMap.get("min");
for (List<Object> row : dataset) {
String tsStr = (String) row.get(tsIdx);
long time = Instant.parse(tsStr).toEpochMilli();
Value.ValueBuilder valueBuilder = Value.builder();
valueBuilder.time(time);
Object originObj = row.get(originIdx);
if (originObj == null) {
continue;
}
valueBuilder.origin(this.parseDoubleValue(originObj.toString()));
Object meanObj = row.get(meanIdx);
if (meanObj == null) {
continue;
}
valueBuilder.mean(this.parseDoubleValue(meanObj.toString()));
Object maxObj = row.get(maxIdx);
if (maxObj == null) {
continue;
}
valueBuilder.max(this.parseDoubleValue(maxObj.toString()));
Object minObj = row.get(minIdx);
if (minObj == null) {
continue;
}
valueBuilder.min(this.parseDoubleValue(minObj.toString()));
List<Value> valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
valueList.add(valueBuilder.build());
}
}
} catch (Exception e) {
log.error("select history interval metric data in questdb error, msg: {}", e.getMessage());
}
return instanceValueMap;
}
private Map<String, Object> executeQuery(String sql) {
try {
String encodedSql = URLEncoder.encode(sql, StandardCharsets.UTF_8);
String url = queryBaseUrl + encodedSql + "&timestamptype=rfc3339";
String authHeader = "Basic " + Base64.getEncoder().encodeToString(
(questdbProperties.username() + ":" + questdbProperties.password())
.getBytes(StandardCharsets.UTF_8)
);
Request request = new Request.Builder()
.url(url)
.addHeader("Authorization", authHeader)
.get()
.build();
try (Response response = client.newCall(request).execute()) {
if (!response.isSuccessful()) {
log.error("QuestDB query failed: {} - {}", response.code(), response.message());
return null;
}
String body = response.body().string();
return JsonUtil.fromJson(body, Map.class);
}
} catch (Exception e) {
log.error("Error executing QuestDB query: {} - {}", sql, e.getMessage());
return null;
}
}
private String getDateAdd(String history) {
history = history.toLowerCase();
char unitChar = history.charAt(history.length() - 1);
int count = Integer.parseInt(history.substring(0, history.length() - 1));
String unit;
switch (unitChar) {
case 'd':
unit = "d";
break;
case 'h':
unit = "h";
break;
case 'm': // minute
unit = "m";
break;
case 's':
unit = "s";
break;
default: throw new IllegalArgumentException("Invalid history unit: " + unitChar);
}
return String.format("dateadd('%s', %d, now())", unit, -count);
}
private String generateTable(String app, String metrics, String instance) {
if (instance.contains(".") || instance.contains(":") || instance.contains("[")) {
instance = instance.replace(".", "_")
.replace(":", "_")
.replace("[", "_")
.replace("]", "_");
}
return app + "_" + metrics + "_" + instance;
}
private String parseDoubleValue(String value) {
return (new BigDecimal(value)).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
}
private static X509TrustManager defaultTrustManager() {
return new X509TrustManager() {
@Override
public X509Certificate[] getAcceptedIssuers() {
return new X509Certificate[0];
}
@Override
public void checkClientTrusted(X509Certificate[] certs, String authType) {
}
@Override
public void checkServerTrusted(X509Certificate[] certs, String authType) {
}
};
}
private static SSLSocketFactory defaultSslSocketFactory() {
try {
SSLContext sslContext = SSLContexts.createDefault();
sslContext.init(null, new TrustManager[]{
defaultTrustManager()
}, new SecureRandom());
return sslContext.getSocketFactory();
} catch (Exception e) {
throw new RuntimeException(e);
}
}
private static HostnameVerifier noopHostnameVerifier() {
return (s, sslSession) -> true;
}
@Override
public void destroy() throws Exception {
if (this.sender != null) {
this.sender.close();
}
if (this.client != null) {
this.client.dispatcher().executorService().shutdown();
}
}
}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.questdb;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* QuestDB configuration information
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.QUEST_DB)
public record QuestdbProperties(@DefaultValue("false") boolean enabled,
String url,
String username,
String password) {
}
@@ -0,0 +1,509 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.tdengine;
import com.google.common.collect.Maps;
import com.taosdata.jdbc.TSDBDriver;
import com.zaxxer.hikari.HikariConfig;
import com.zaxxer.hikari.HikariDataSource;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Statement;
import java.sql.Timestamp;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Properties;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.regex.Pattern;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.ObjectUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.common.util.StrBuffer;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.jetbrains.annotations.NotNull;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Primary;
import org.springframework.stereotype.Component;
/**
* tdengine data storage
*/
@Primary
@Component
@ConditionalOnProperty(prefix = "warehouse.store.td-engine",
name = "enabled", havingValue = "true")
@Slf4j
public class TdEngineDataStorage extends AbstractHistoryDataStorage {
private static final String CONSTANTS_URL_PREFIX = "jdbc:TAOS-RS://";
private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')");
private static final String INSTANCE_NULL = "''";
private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS %s";
private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO `%s` USING `%s` TAGS (%s) VALUES %s";
private static final String CREATE_SUPER_TABLE_SQL = "CREATE STABLE IF NOT EXISTS `%s` %s TAGS (instance NCHAR(128))";
private static final String NO_SUPER_TABLE_ERROR = "Table does not exist";
/**
* schema version suffix - increment when TAG definition changes.
* v1: instance BIGINT (original monitorId like 123456789)
* v2: instance NCHAR(128) (supports string instance values like 127.0.0.1:8080)
*/
private static final String SUPER_TABLE_VERSION = "_v2";
private static final String QUERY_HISTORY_WITH_INSTANCE_SQL =
"SELECT ts, metric_labels, `%s` FROM `%s` WHERE metric_labels = '%s' AND ts >= now - %s order by ts desc";
private static final String QUERY_HISTORY_SQL =
"SELECT ts, metric_labels, `%s` FROM `%s` WHERE ts >= now - %s order by ts desc";
private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL =
"SELECT first(ts), first(`%s`), avg(`%s`), min(`%s`), max(`%s`) FROM `%s` WHERE metric_labels = '%s' AND ts >= now - %s interval(4h)";
private static final String QUERY_INSTANCE_SQL =
"SELECT DISTINCT metric_labels FROM `%s` WHERE ts >= now - 1w";
private static final String TABLE_NOT_EXIST = "Table does not exist";
private static final Runnable INSTANCE_EXCEPTION_PRINT = () -> {
if (log.isErrorEnabled()) {
log.error(
"""
\t---------------TdEngine Init Failed---------------
\t--------------Please Config Tdengine--------------
\t----------Can Not Use Metric History Now----------
"""
);
}
};
private HikariDataSource hikariDataSource;
private final int tableStrColumnDefineMaxLength;
public TdEngineDataStorage(TdEngineProperties tdEngineProperties) {
if (tdEngineProperties == null) {
log.error("init error, please config Warehouse TdEngine props in application.yml");
throw new IllegalArgumentException("please config Warehouse TdEngine props");
}
tableStrColumnDefineMaxLength = tdEngineProperties.tableStrColumnDefineMaxLength();
serverAvailable = initTdEngineDatasource(tdEngineProperties);
}
/**
* {@code TdEngine} init TdEngineDatabase
*
* @param tdEngineProperties {@link TdEngineProperties}
*/
private void initTdEngineDatabase(final TdEngineProperties tdEngineProperties) throws SQLException {
final Properties parseResultProperties = com.taosdata.jdbc.utils.StringUtils.parseUrl(tdEngineProperties.url(), null);
final String host = ObjectUtils.requireNonEmpty(parseResultProperties.getProperty(TSDBDriver.PROPERTY_KEY_HOST));
final String port = ObjectUtils.requireNonEmpty(parseResultProperties.getProperty(TSDBDriver.PROPERTY_KEY_PORT));
final String dbName = ObjectUtils.requireNonEmpty(parseResultProperties.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME));
try (
final Connection tempConnection = DriverManager.getConnection(
CONSTANTS_URL_PREFIX + host + ":" + port,
tdEngineProperties.username(),
tdEngineProperties.password()
)
) {
tempConnection.prepareStatement(String.format(CONSTANTS_CREATE_DATABASE, dbName))
.execute();
}
}
private boolean initTdEngineDatasource(final TdEngineProperties tdEngineProperties) {
try {
initTdEngineDatabase(tdEngineProperties);
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error(e.getMessage(), e);
}
INSTANCE_EXCEPTION_PRINT.run();
return false;
}
final HikariConfig config = new HikariConfig();
// jdbc properties
config.setJdbcUrl(tdEngineProperties.url());
config.setUsername(tdEngineProperties.username());
config.setPassword(tdEngineProperties.password());
config.setDriverClassName(tdEngineProperties.driverClassName());
//minimum number of idle connection
config.setMinimumIdle(10);
//maximum number of connection in the pool
config.setMaximumPoolSize(10);
//maximum wait milliseconds for get connection from pool
config.setConnectionTimeout(30000);
// maximum lifetime for each connection
config.setMaxLifetime(0);
// max idle time for recycle idle connection
config.setIdleTimeout(0);
//validation query
config.setConnectionTestQuery("select server_status()");
try {
this.hikariDataSource = new HikariDataSource(config);
} catch (Exception e) {
INSTANCE_EXCEPTION_PRINT.run();
return false;
}
return true;
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
return;
}
if (metricsData.getValues().isEmpty()) {
if (log.isInfoEnabled()) {
log.info("[warehouse tdengine] flush metrics data {} is null, ignore.", metricsData.getInstance());
}
return;
}
String instance = metricsData.getInstance();
String app = metricsData.getApp();
String metrics = metricsData.getMetrics();
String superTable = getSuperTable(app, metrics);
String table = getTable(app, metrics, instance);
StringBuilder sqlBuffer = new StringBuilder();
int i = 0;
try {
RowWrapper rowWrapper = metricsData.readRow();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
StringBuilder sqlRowBuffer = new StringBuilder("(");
sqlRowBuffer.append(metricsData.getTime() + i++).append(", ");
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
sqlRowBuffer.append("'").append("%s").append("', ");
AtomicInteger index = new AtomicInteger(-1);
int fieldMaxSize = rowWrapper.getFieldList().size();
rowWrapper.cellStream().forEach(cell -> {
index.getAndIncrement();
String value = cell.getValue();
final int fieldType;
if ((fieldType = cell.getMetadataAsByte(MetricDataConstants.TYPE)) == CommonConstants.TYPE_NUMBER || fieldType == CommonConstants.TYPE_TIME) {
// number data
if (CommonConstants.NULL_VALUE.equals(value)) {
sqlRowBuffer.append("NULL");
} else {
try {
double number = Double.parseDouble(value);
// TDengine doesn't support NaN or Infinity literals, convert to NULL
if (Double.isNaN(number) || Double.isInfinite(number)) {
sqlRowBuffer.append("NULL");
} else {
sqlRowBuffer.append(number);
}
} catch (Exception e) {
if (log.isWarnEnabled()) {
log.warn(e.getMessage());
}
sqlRowBuffer.append("NULL");
}
}
} else {
// string
if (CommonConstants.NULL_VALUE.equals(value)) {
sqlRowBuffer.append("NULL");
} else {
sqlRowBuffer.append("'").append(StrBuffer.escapeForFormat(formatStringValue(value))).append("'");
}
}
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL) && !CommonConstants.NULL_VALUE.equals(value)) {
labels.put(cell.getField().getName(), formatStringValue(value));
}
if (index.get() != fieldMaxSize - 1) {
sqlRowBuffer.append(", ");
}
});
sqlRowBuffer.append(")");
sqlBuffer.append(" ").append(String.format(sqlRowBuffer.toString(), formatStringValue(JsonUtil.toJson(labels))));
}
String insertDataSql = String.format(INSERT_TABLE_DATA_SQL, table, superTable, "'" + formatStringValue(instance) + "'", sqlBuffer);
if (log.isDebugEnabled()) {
log.debug(insertDataSql);
}
Connection connection = null;
Statement statement = null;
try {
connection = hikariDataSource.getConnection();
statement = connection.createStatement();
statement.execute(insertDataSql);
connection.close();
} catch (Exception e) {
if (e.getMessage().contains(NO_SUPER_TABLE_ERROR)) {
// stable not exists, create it
StringBuilder fieldSqlBuilder = new StringBuilder("(");
fieldSqlBuilder.append("ts TIMESTAMP, ");
fieldSqlBuilder.append("metric_labels NCHAR(").append(tableStrColumnDefineMaxLength).append("), ");
for (int index = 0; index < metricsData.getFields().size(); index++) {
CollectRep.Field field = metricsData.getFields().get(index);
String fieldName = field.getName();
final int fieldType = field.getType();
if (fieldType == CommonConstants.TYPE_NUMBER || fieldType == CommonConstants.TYPE_TIME) {
fieldSqlBuilder.append("`").append(fieldName).append("` ").append("DOUBLE");
} else {
fieldSqlBuilder.append("`").append(fieldName).append("` ").append("NCHAR(")
.append(tableStrColumnDefineMaxLength).append(")");
}
if (index != metricsData.getFields().size() - 1) {
fieldSqlBuilder.append(", ");
}
}
fieldSqlBuilder.append(")");
String createTableSql = String.format(CREATE_SUPER_TABLE_SQL, superTable, fieldSqlBuilder);
try {
assert statement != null;
if (log.isInfoEnabled()) {
log.info("[tdengine-data]: create {} use sql: {}.", superTable, createTableSql);
}
statement.execute(createTableSql);
statement.execute(insertDataSql);
} catch (Exception createTableException) {
if (log.isErrorEnabled()) {
log.error(e.getMessage(), createTableException);
}
}
} else {
if (log.isErrorEnabled()) {
log.error(e.getMessage());
}
}
} finally {
try {
assert connection != null;
connection.close();
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error(e.getMessage());
}
}
}
} catch (Exception e) {
log.error("[warehouse tdEngine]--Error: {}", e.getMessage(), e);
}
}
@NotNull
private static String getTable(String app, String metrics, String instance) {
if (instance.contains(".") || instance.contains(":") || instance.contains("[")) {
instance = instance.replace(".", "_")
.replace(":", "_")
.replace("[", "_")
.replace("]", "_");
}
return app + "_" + metrics + "_" + instance + SUPER_TABLE_VERSION;
}
@NotNull
private static String getSuperTable(String app, String metrics) {
return app + "_" + metrics + "_super" + SUPER_TABLE_VERSION;
}
private String formatStringValue(String value) {
String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0");
// bugfix Argument list too long
if (formatValue != null && formatValue.length() > tableStrColumnDefineMaxLength) {
formatValue = formatValue.substring(0, tableStrColumnDefineMaxLength);
}
return formatValue;
}
@Override
public void destroy() {
if (hikariDataSource != null) {
hikariDataSource.close();
}
}
@Override
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
String table = getTable(app, metrics, instance);
String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history);
if (log.isDebugEnabled()) {
log.debug(selectSql);
}
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
if (!serverAvailable) {
INSTANCE_EXCEPTION_PRINT.run();
return instanceValuesMap;
}
try (Connection connection = hikariDataSource.getConnection();
Statement statement = connection.createStatement();
ResultSet resultSet = statement.executeQuery(selectSql)) {
while (resultSet.next()) {
Timestamp ts = resultSet.getTimestamp(1);
if (ts == null) {
if (log.isErrorEnabled()) {
log.error("warehouse tdengine query result timestamp is null, ignore. {}.", selectSql);
}
continue;
}
String instanceValue = resultSet.getString(2);
if (instanceValue == null || StringUtils.isBlank(instanceValue)) {
instanceValue = "";
}
double value = resultSet.getDouble(3);
String strValue = new BigDecimal(value).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
List<Value> valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
valueList.add(new Value(strValue, ts.getTime() / 100 * 100));
}
return instanceValuesMap;
} catch (SQLException sqlException) {
String msg = sqlException.getMessage();
if (msg != null && !msg.contains(TABLE_NOT_EXIST)) {
if (log.isWarnEnabled()) {
log.warn(sqlException.getMessage());
}
}
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error(e.getMessage(), e);
}
}
return instanceValuesMap;
}
@Override
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
String metric, String history) {
if (!serverAvailable) {
INSTANCE_EXCEPTION_PRINT.run();
return Collections.emptyMap();
}
String table = getTable(app, metrics, instance);
List<String> instances = new LinkedList<>();
// query all metric_labels from the table
String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table);
Connection connection = null;
try {
connection = hikariDataSource.getConnection();
Statement statement = connection.createStatement();
ResultSet resultSet = statement.executeQuery(queryInstanceSql);
while (resultSet.next()) {
String instanceValue = resultSet.getString(1);
if (instanceValue == null || StringUtils.isBlank(instanceValue)) {
instances.add("''");
} else {
instances.add(instanceValue);
}
}
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error(e.getMessage());
}
} finally {
try {
assert connection != null;
connection.close();
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error(e.getMessage());
}
}
}
Map<String, List<Value>> instanceValuesMap = new HashMap<>(instances.size());
for (String instanceValue : instances) {
if (INSTANCE_NULL.equals(instanceValue)) {
instanceValue = "";
}
String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL,
metric, metric, metric, metric, table, instanceValue, history);
if (log.isDebugEnabled()) {
log.debug(selectSql);
}
List<Value> values = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
Connection conn = null;
try {
conn = hikariDataSource.getConnection();
Statement statement = conn.createStatement();
ResultSet resultSet = statement.executeQuery(selectSql);
while (resultSet.next()) {
Timestamp ts = resultSet.getTimestamp(1);
double origin = resultSet.getDouble(2);
String originStr = BigDecimal.valueOf(origin).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
double avg = resultSet.getDouble(3);
String avgStr = BigDecimal.valueOf(avg).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
double min = resultSet.getDouble(4);
String minStr = BigDecimal.valueOf(min).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
double max = resultSet.getDouble(5);
String maxStr = BigDecimal.valueOf(max).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
Value value = Value.builder()
.origin(originStr).mean(avgStr)
.min(minStr).max(maxStr)
.time(ts.getTime() / 100 * 100)
.build();
values.add(value);
}
resultSet.close();
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error(e.getMessage(), e);
}
} finally {
try {
assert conn != null;
conn.close();
} catch (Exception e) {
if (log.isErrorEnabled()) {
log.error(e.getMessage());
}
}
}
}
return instanceValuesMap;
}
}
@@ -0,0 +1,46 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.tdengine;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
*
* @param enabled Whether the TdEngine data store is enabled
* @param url TdEngine connect url
* @param driverClassName tdengine driver, default restful driver
* @param username tdengine username
* @param password tdengine password
* @param tableStrColumnDefineMaxLength auto create table's string column define max length : NCHAR(200)
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.TD_ENGINE)
public record TdEngineProperties(@DefaultValue("false") boolean enabled,
@DefaultValue("jdbc:TAOS-RS://localhost:6041/demo") String url,
@DefaultValue("com.taosdata.jdbc.rs.RestfulDriver") String driverClassName,
String username, String password,
@DefaultValue("200") int tableStrColumnDefineMaxLength) {
}
@@ -0,0 +1,100 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
import java.util.List;
import java.util.Map;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
/**
* promql query content
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public class PromQlQueryContent {
private String status;
private ContentData data;
/**
* content data
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public static final class ContentData {
private String resultType;
private List<Content> result;
/**
* content
*/
@Data
@AllArgsConstructor
@NoArgsConstructor
public static final class Content {
/**
* metric contains metric name plus labels for a particular time series
*/
private Map<String, String> metric;
/**
* values contains raw sample values for the given time series
* value-timestamp
* [1700993195,"436960986"]
*/
private Object[] value;
/**
* values contains raw sample values for the given time series
* value-timestamp list
* [[1700993195,"436960986"],[1700993195,"436960986"]...]
*/
private List<Object[]> values;
}
}
}
@@ -0,0 +1,707 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
import tools.jackson.databind.JsonNode;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URI;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.time.temporal.TemporalAmount;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Objects;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import java.util.stream.Stream;
import com.google.common.collect.Maps;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.HashedWheelTimer;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.timer.TimerTask;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.common.util.TimePeriodUtil;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Primary;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
import static org.apache.hertzbeat.common.constants.ConfigConstants.FunctionModuleConstants.STATUS;
/**
* VictoriaMetrics data storage
*/
@Primary
@Component
@ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics.cluster", name = "enabled", havingValue = "true")
@Slf4j
public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorage {
private static final String VM_INSERT_BASE_PATH = "/insert/%s/%s";
private static final String VM_SELECT_BASE_PATH = "/select/%s/%s";
private static final String IMPORT_PATH = "prometheus/api/v1/import";
private static final String EXPORT_PATH = "prometheus/api/v1/export";
private static final String STATUS_PATH = "prometheus/api/v1/status/tsdb";
private static final String STATUS_SUCCESS = "success";
private static final String QUERY_RANGE_PATH = "prometheus/api/v1/query_range";
private static final String LABEL_KEY_NAME = "__name__";
private static final String LABEL_KEY_JOB = "job";
private static final String LABEL_KEY_INSTANCE = "instance";
private static final String LABEL_KEY_MONITOR_ID = "__monitor_id__";
private static final String SPILT = "_";
private static final String MONITOR_METRICS_KEY = "__metrics__";
private static final String MONITOR_METRIC_KEY = "__metric__";
private static final long MAX_WAIT_MS = 500L;
private static final int MAX_RETRIES = 3;
private final VictoriaMetricsClusterProperties vmClusterProps;
private final VictoriaMetricsInsertProperties vmInsertProps;
private final VictoriaMetricsSelectProperties vmSelectProps;
private final RestTemplate restTemplate;
private final BlockingQueue<VictoriaMetricsDataStorage.VictoriaMetricsContent> metricsBufferQueue;
private HashedWheelTimer metricsFlushTimer = null;
private MetricsFlushTask metricsFlushtask = null;
private boolean isBatchImportEnabled = false;
public VictoriaMetricsClusterDataStorage(VictoriaMetricsClusterProperties vmClusterProps,
RestTemplate restTemplate) {
if (vmClusterProps == null) {
log.error("init error, please config Warehouse victoriaMetrics cluster props in application.yml");
throw new IllegalArgumentException("please config Warehouse victoriaMetrics cluster props");
}
this.restTemplate = restTemplate;
this.vmClusterProps = vmClusterProps;
this.vmInsertProps = vmClusterProps.insert();
this.vmSelectProps = vmClusterProps.select();
serverAvailable = checkVictoriaMetricsDatasourceAvailable();
metricsBufferQueue = new LinkedBlockingQueue<>(vmInsertProps.bufferSize());
isBatchImportEnabled = vmInsertProps.flushInterval() != 0 && vmInsertProps.bufferSize() != 0;
if (isBatchImportEnabled){
initializeFlushTimer();
}
}
private void initializeFlushTimer() {
this.metricsFlushTimer = new HashedWheelTimer(r -> {
Thread thread = new Thread(r, "victoria-metrics-flush-timer");
thread.setDaemon(true);
return thread;
}, 1, TimeUnit.SECONDS, 512);
metricsFlushtask = new MetricsFlushTask();
this.metricsFlushTimer.newTimeout(metricsFlushtask, 0, TimeUnit.SECONDS);
}
private boolean checkVictoriaMetricsDatasourceAvailable() {
// check server status
try {
String selectNodeStatusUrl = vmClusterProps.select().url() + VM_SELECT_BASE_PATH.formatted(vmClusterProps.accountID(), STATUS_PATH);
HttpHeaders headers = new HttpHeaders();
if (StringUtils.hasText(vmInsertProps.username())
&& StringUtils.hasText(vmInsertProps.password())) {
String authStr = vmInsertProps.username() + ":" + vmInsertProps.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
HttpEntity<Void> requestEntity = new HttpEntity<>(headers);
ResponseEntity<String> responseEntity = restTemplate.exchange(
selectNodeStatusUrl,
HttpMethod.GET,
requestEntity,
String.class
);
String result = responseEntity.getBody();
JsonNode jsonNode = JsonUtil.fromJson(result);
if (jsonNode != null && STATUS_SUCCESS.equalsIgnoreCase(jsonNode.get(STATUS).asText())) {
return true;
}
log.error("check victoria metrics cluster server status not success: {}.", result);
} catch (Exception e) {
log.error("check victoria metrics cluster server status error: {}.", e.getMessage());
}
return false;
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
if (!isServerAvailable()) {
serverAvailable = checkVictoriaMetricsDatasourceAvailable();
}
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
return;
}
if (metricsData.getValues().isEmpty()) {
log.info("[warehouse victoria-metrics] flush metrics data {} {} {} is null, ignore.",
metricsData.getId(), metricsData.getApp(), metricsData.getMetrics());
return;
}
Map<String, String> defaultLabels = Maps.newHashMapWithExpectedSize(8);
defaultLabels.put(MONITOR_METRICS_KEY, metricsData.getMetrics());
boolean isPrometheusAuto;
if (metricsData.getApp().startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
isPrometheusAuto = true;
defaultLabels.remove(MONITOR_METRICS_KEY);
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()
.substring(CommonConstants.PROMETHEUS_APP_PREFIX.length()));
} else {
isPrometheusAuto = false;
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp());
}
defaultLabels.put(LABEL_KEY_INSTANCE, metricsData.getInstance());
try {
List<CollectRep.Field> fieldList = metricsData.getFields();
Long[] timestamp = new Long[]{metricsData.getTime()};
Map<String, Double> fieldsValue = Maps.newHashMapWithExpectedSize(fieldList.size());
Map<String, String> labels = Maps.newHashMapWithExpectedSize(fieldList.size());
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> contentList = new LinkedList<>();
RowWrapper rowWrapper = metricsData.readRow();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
fieldsValue.clear();
labels.clear();
rowWrapper.cellStream().forEach(cell -> {
String value = cell.getValue();
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
Boolean label = cell.getMetadataAsBoolean(MetricDataConstants.LABEL);
if (type == CommonConstants.TYPE_NUMBER && !label) {
// number metrics data
if (!CommonConstants.NULL_VALUE.equals(value)) {
fieldsValue.put(cell.getField().getName(), CommonUtil.parseStrDouble(value));
}
}
// label
if (label && !CommonConstants.NULL_VALUE.equals(value)) {
labels.put(cell.getField().getName(), value);
}
for (Map.Entry<String, Double> entry : fieldsValue.entrySet()) {
if (entry.getKey() != null && entry.getValue() != null) {
try {
labels.putAll(defaultLabels);
String labelName = isPrometheusAuto ? metricsData.getMetrics()
: metricsData.getMetrics() + SPILT + entry.getKey();
labels.put(LABEL_KEY_NAME, labelName);
if (!isPrometheusAuto) {
labels.put(MONITOR_METRIC_KEY, entry.getKey());
}
labels.put(LABEL_KEY_MONITOR_ID, String.valueOf(metricsData.getId()));
// add customized labels as identifier
var customizedLabels = metricsData.getLabels();
if (!ObjectUtils.isEmpty(customizedLabels)) {
labels.putAll(customizedLabels);
}
VictoriaMetricsDataStorage.VictoriaMetricsContent content = VictoriaMetricsDataStorage.VictoriaMetricsContent.builder()
.metric(new HashMap<>(labels))
.values(new Double[]{entry.getValue()})
.timestamps(timestamp)
.build();
contentList.add(content);
} catch (Exception e) {
log.error("combine metrics data error: {}.", e.getMessage(), e);
}
}
}
});
}
if (contentList.isEmpty()) {
log.info("[warehouse victoria-metrics] flush metrics data {} is empty, ignore.", metricsData.getId());
return;
}
if (!isBatchImportEnabled){
doSaveData(contentList);
return;
}
sendVictoriaMetrics(contentList);
} catch (Exception e) {
log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e);
}
}
@Override
public void destroy() {
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
metricsFlushTimer.stop();
}
}
@Override
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric,
String history) {
String labelName = metrics + SPILT + metric;
if (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
labelName = metrics;
}
String timeSeriesSelector = Stream.of(
LABEL_KEY_NAME + "=\"" + labelName + "\"",
LABEL_KEY_INSTANCE + "=\"" + instance + "\"",
app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX) ? null : MONITOR_METRIC_KEY + "=\"" + metric + "\""
).filter(Objects::nonNull).collect(Collectors.joining(","));
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
if (StringUtils.hasText(vmSelectProps.username()) && StringUtils.hasText(vmSelectProps.password())) {
String authStr = vmSelectProps.username() + ":" + vmSelectProps.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC
+ SignConstants.BLANK + encodedAuth);
}
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
String exportUrl = vmClusterProps.select().url() + VM_SELECT_BASE_PATH.formatted(vmClusterProps.accountID(), EXPORT_PATH);
URI uri = UriComponentsBuilder.fromUriString(exportUrl)
.queryParam("match[]", "{" + timeSeriesSelector + "}")
.queryParam("start", "now-" + history)
.queryParam("end", "now")
.build()
.encode()
.toUri();
ResponseEntity<String> responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity,
String.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("query metrics data from victoria-metrics success. {}", uri);
if (StringUtils.hasText(responseEntity.getBody())) {
String[] contentJsonArr = responseEntity.getBody().split("\n");
List<VictoriaMetricsContent> contents = Arrays.stream(contentJsonArr)
.map(item -> JsonUtil.fromJson(item, VictoriaMetricsContent.class)).toList();
for (VictoriaMetricsContent content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && content.getTimestamps() != null) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
k -> new LinkedList<>());
if (content.getValues().length != content.getTimestamps().length) {
log.error("content.getValues().length != content.getTimestamps().length");
continue;
}
Double[] values = content.getValues();
Long[] timestamps = content.getTimestamps();
for (int index = 0; index < content.getValues().length; index++) {
String strValue = BigDecimal.valueOf(values[index]).setScale(4, RoundingMode.HALF_UP)
.stripTrailingZeros().toPlainString();
// read timestamp here is ms unit
valueList.add(new Value(strValue, timestamps[index]));
}
}
}
}
} else {
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
}
} catch (Exception e) {
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
}
return instanceValuesMap;
}
@Override
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
String metric, String history) {
if (!serverAvailable) {
log.error("""
\t---------------VictoriaMetrics Init Failed---------------
\t--------------Please Config VictoriaMetrics--------------
\t----------Can Not Use Metric History Now----------
""");
return Collections.emptyMap();
}
long endTime = ZonedDateTime.now().toEpochSecond();
long startTime;
try {
if (NumberUtils.isParsable(history)) {
startTime = NumberUtils.toLong(history);
startTime = (ZonedDateTime.now().toEpochSecond() - startTime);
} else {
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
startTime = dateTime.toEpochSecond();
}
} catch (Exception e) {
log.error("history time error: {}. use default: 6h", e.getMessage());
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6));
startTime = dateTime.toEpochSecond();
}
String labelName = metrics + SPILT + metric;
if (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
labelName = metrics;
}
String timeSeriesSelector = Stream.of(
LABEL_KEY_NAME + "=\"" + labelName + "\"",
LABEL_KEY_INSTANCE + "=\"" + instance + "\"",
app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX) ? null : MONITOR_METRIC_KEY + "=\"" + metric + "\""
).filter(Objects::nonNull).collect(Collectors.joining(","));
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
if (StringUtils.hasText(vmSelectProps.username()) && StringUtils.hasText(vmSelectProps.password())) {
String authStr = vmSelectProps.username() + ":" + vmSelectProps.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC
+ SignConstants.BLANK + encodedAuth);
}
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
String rangeUrl = vmClusterProps.select().url() + VM_SELECT_BASE_PATH.formatted(vmClusterProps.accountID(), QUERY_RANGE_PATH);
URI uri = UriComponentsBuilder.fromUriString(rangeUrl)
.queryParam("query", "{" + timeSeriesSelector + "}")
.queryParam("step", "4h")
.queryParam("start", startTime)
.queryParam("end", endTime)
.build()
.encode()
.toUri();
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri, HttpMethod.GET,
httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("query metrics data from victoria-metrics success. {}", uri);
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData()
.getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
k -> new LinkedList<>());
for (Object[] valueArr : content.getValues()) {
long timestamp = Long.parseLong(String.valueOf(valueArr[0]));
String value = new BigDecimal(String.valueOf(valueArr[1])).setScale(4,
RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
// read timestamp here is s unit
valueList.add(new Value(value, timestamp * 1000));
}
}
}
}
} else {
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
}
// max
uri = UriComponentsBuilder.fromUriString(rangeUrl)
.queryParam("query", "max_over_time({" + timeSeriesSelector + "})")
.queryParam("step", "4h")
.queryParam("start", startTime)
.queryParam("end", endTime)
.build()
.encode()
.toUri();
responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData()
.getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
k -> new LinkedList<>());
if (valueList.size() == content.getValues().size()) {
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
Value value = valueList.get(timestampIndex);
Object[] valueArr = content.getValues().get(timestampIndex);
String maxValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4,
RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
value.setMax(maxValue);
}
}
}
}
}
}
// min
uri = UriComponentsBuilder.fromUriString(rangeUrl)
.queryParam("query", "min_over_time({" + timeSeriesSelector + "})")
.queryParam("step", "4h")
.queryParam("start", startTime)
.queryParam("end", endTime)
.build()
.encode()
.toUri();
responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData()
.getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
k -> new LinkedList<>());
if (valueList.size() == content.getValues().size()) {
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
Value value = valueList.get(timestampIndex);
Object[] valueArr = content.getValues().get(timestampIndex);
String minValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4,
RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
value.setMin(minValue);
}
}
}
}
}
}
// avg
uri = UriComponentsBuilder.fromUriString(rangeUrl)
.queryParam("query", "avg_over_time({" + timeSeriesSelector + "})")
.queryParam("step", "4h")
.queryParam("start", startTime)
.queryParam("end", endTime)
.build()
.encode()
.toUri();
responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData()
.getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
k -> new LinkedList<>());
if (valueList.size() == content.getValues().size()) {
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
Value value = valueList.get(timestampIndex);
Object[] valueArr = content.getValues().get(timestampIndex);
String avgValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4,
RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
value.setMean(avgValue);
}
}
}
}
}
}
} catch (Exception e) {
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
}
return instanceValuesMap;
}
/**
* Save metric data to victoria-metric via HTTP call
*/
public void doSaveData(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> contentList){
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
if (StringUtils.hasText(vmInsertProps.username())
&& StringUtils.hasText(vmInsertProps.password())) {
String authStr = vmInsertProps.username() + ":" + vmInsertProps.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
StringBuilder stringBuilder = new StringBuilder();
for (VictoriaMetricsDataStorage.VictoriaMetricsContent content : contentList) {
stringBuilder.append(JsonUtil.toJson(content)).append("\n");
}
HttpEntity<String> httpEntity = new HttpEntity<>(stringBuilder.toString(), headers);
String importUrl = vmClusterProps.insert().url() + VM_INSERT_BASE_PATH.formatted(vmClusterProps.accountID(), IMPORT_PATH);
ResponseEntity<String> responseEntity = restTemplate.postForEntity(importUrl,
httpEntity, String.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("insert metrics data to victoria-metrics success.");
} else {
log.error("insert metrics data to victoria-metrics failed. {}", responseEntity.getBody());
}
} catch (Exception e){
log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e);
}
}
/**
* add victoriaMetricsContent to buffer
* @param contentList victoriaMetricsContent List
*/
private void sendVictoriaMetrics(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> contentList) {
for (VictoriaMetricsDataStorage.VictoriaMetricsContent content : contentList) {
boolean offered = false;
int retryCount = 0;
while (!offered && retryCount < MAX_RETRIES) {
try {
// Attempt to add to the queue for a limited time
offered = metricsBufferQueue.offer(content, MAX_WAIT_MS, TimeUnit.MILLISECONDS);
if (!offered) {
// If the queue is still full, trigger an immediate refresh to free up space
if (retryCount == 0) {
log.debug("victoria metrics buffer queue is full, triggering immediate flush");
triggerImmediateFlush();
}
retryCount++;
// The short sleep allows the queue to clear out
if (retryCount < MAX_RETRIES) {
Thread.sleep(100L * retryCount);
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.error("[Victoria Metrics] Interrupted while offering metrics to buffer queue", e);
break;
}
}
// When the maximum number of retries is reached, if it still cannot be added to the queue, the data is saved directly
if (!offered) {
log.warn("[Victoria Metrics] Failed to add metrics to buffer after {} retries, saving directly", MAX_RETRIES);
try {
doSaveData(contentList);
} catch (Exception e) {
log.error("[Victoria Metrics] Failed to save metrics directly: {}", e.getMessage(), e);
}
}
// Refresh in advance to avoid waiting
if (metricsBufferQueue.size() >= vmInsertProps.bufferSize() * 0.8) {
triggerImmediateFlush();
}
}
}
private void triggerImmediateFlush() {
metricsFlushTimer.newTimeout(metricsFlushtask, 0, TimeUnit.MILLISECONDS);
}
/**
* Regularly refresh the buffer queue to the vm
*/
private class MetricsFlushTask implements TimerTask {
@Override
public void run(Timeout timeout) {
try {
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch = new ArrayList<>(vmInsertProps.bufferSize());
metricsBufferQueue.drainTo(batch, vmInsertProps.bufferSize());
if (!batch.isEmpty()) {
doSaveData(batch);
log.debug("[Victoria Metrics] Flushed {} metrics items", batch.size());
}
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
metricsFlushTimer.newTimeout(this, vmInsertProps.flushInterval(), TimeUnit.SECONDS);
}
} catch (Exception e) {
log.error("[VictoriaMetrics] flush task error: {}", e.getMessage(), e);
}
}
}
/**
* victoria metrics content
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public static final class VictoriaMetricsContent {
/**
* metric contains metric name plus labels for a particular time series
*/
private Map<String, String> metric;
/**
* values contains raw sample values for the given time series
*/
private Double[] values;
/**
* timestamps contains raw sample UNIX timestamps in milliseconds for the given time series
* every timestamp is associated with the value at the corresponding position
*/
private Long[] timestamps;
}
}
@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* Victoriametrics configuration information
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.VM_CLUSTER)
public record VictoriaMetricsClusterProperties(
@DefaultValue("false") boolean enabled,
@DefaultValue("0") String accountID,
VictoriaMetricsInsertProperties insert,
VictoriaMetricsSelectProperties select
) {
}
@@ -0,0 +1,716 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
import java.io.ByteArrayOutputStream;
import java.math.BigDecimal;
import java.math.RoundingMode;
import java.net.URI;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.time.ZonedDateTime;
import java.time.temporal.TemporalAmount;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.zip.GZIPOutputStream;
import com.google.common.collect.Maps;
import lombok.AllArgsConstructor;
import lombok.Builder;
import lombok.Data;
import lombok.NoArgsConstructor;
import lombok.extern.slf4j.Slf4j;
import org.apache.commons.lang3.math.NumberUtils;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.constants.NetworkConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.timer.HashedWheelTimer;
import org.apache.hertzbeat.common.timer.Timeout;
import org.apache.hertzbeat.common.timer.TimerTask;
import org.apache.hertzbeat.common.util.Base64Util;
import org.apache.hertzbeat.common.util.CommonUtil;
import org.apache.hertzbeat.common.util.JsonUtil;
import org.apache.hertzbeat.common.util.TimePeriodUtil;
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Primary;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpHeaders;
import org.springframework.http.HttpMethod;
import org.springframework.http.MediaType;
import org.springframework.http.ResponseEntity;
import org.springframework.stereotype.Component;
import org.springframework.util.ObjectUtils;
import org.springframework.util.StringUtils;
import org.springframework.web.client.RestTemplate;
import org.springframework.web.util.UriComponentsBuilder;
/**
* VictoriaMetrics data storage
*/
@Primary
@Component
@ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics", name = "enabled", havingValue = "true")
@Slf4j
public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage {
private static final String IMPORT_PATH = "/api/v1/import";
/**
* <a href="https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-export-data-in-json-line-format">
* https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-export-data-in-json-line-format
* </a>
*/
private static final String EXPORT_PATH = "/api/v1/export";
private static final String QUERY_RANGE_PATH = "/api/v1/query_range";
private static final String STATUS_PATH = "/api/v1/status/tsdb";
private static final String LABEL_KEY_NAME = "__name__";
private static final String LABEL_KEY_JOB = "job";
private static final String LABEL_KEY_INSTANCE = "instance";
private static final String LABEL_KEY_MONITOR_ID = "__monitor_id__";
private static final String SPILT = "_";
private static final String MONITOR_METRICS_KEY = "__metrics__";
private static final String MONITOR_METRIC_KEY = "__metric__";
private static final long MAX_WAIT_MS = 500L;
private static final int MAX_RETRIES = 3;
private final VictoriaMetricsProperties victoriaMetricsProp;
private final RestTemplate restTemplate;
private final BlockingQueue<VictoriaMetricsDataStorage.VictoriaMetricsContent> metricsBufferQueue;
private HashedWheelTimer metricsFlushTimer = null;
private final VictoriaMetricsProperties.InsertConfig insertConfig;
private final AtomicBoolean draining = new AtomicBoolean(false);
public VictoriaMetricsDataStorage(VictoriaMetricsProperties victoriaMetricsProperties, RestTemplate restTemplate) {
if (victoriaMetricsProperties == null) {
log.error("init error, please config Warehouse victoriaMetrics props in application.yml");
throw new IllegalArgumentException("please config Warehouse victoriaMetrics props");
}
this.restTemplate = restTemplate;
victoriaMetricsProp = victoriaMetricsProperties;
serverAvailable = checkVictoriaMetricsDatasourceAvailable();
insertConfig = victoriaMetricsProperties.insert() == null ? new VictoriaMetricsProperties.InsertConfig(100, 3,
new VictoriaMetricsProperties.Compression(false)) : victoriaMetricsProperties.insert();
metricsBufferQueue = new LinkedBlockingQueue<>(insertConfig.bufferSize());
initializeFlushTimer();
}
private void initializeFlushTimer() {
this.metricsFlushTimer = new HashedWheelTimer(r -> {
Thread thread = new Thread(r, "victoria-metrics-cluster-flush-timer");
thread.setDaemon(true);
return thread;
}, 1, TimeUnit.SECONDS, 512);
// start flush interval timer
this.metricsFlushTimer.newTimeout(new MetricsFlushTask(null), insertConfig.flushInterval(), TimeUnit.SECONDS);
}
private boolean checkVictoriaMetricsDatasourceAvailable() {
// check server status
try {
HttpHeaders headers = new HttpHeaders();
if (StringUtils.hasText(victoriaMetricsProp.username())
&& StringUtils.hasText(victoriaMetricsProp.password())) {
String authStr = victoriaMetricsProp.username() + SignConstants.DOUBLE_MARK + victoriaMetricsProp.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + " " + encodedAuth);
}
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
ResponseEntity<String> responseEntity = restTemplate.exchange(victoriaMetricsProp.url() + STATUS_PATH,
HttpMethod.GET, httpEntity, String.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.info("check victoria metrics server status success.");
return true;
}
log.error("check victoria metrics server status not success: {}.", responseEntity.getBody());
} catch (Exception e) {
log.error("check victoria metrics server status error: {}.", e.getMessage());
}
return false;
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
if (!isServerAvailable()) {
serverAvailable = checkVictoriaMetricsDatasourceAvailable();
}
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
return;
}
if (metricsData.getValues().isEmpty()) {
log.info("[warehouse victoria-metrics] flush metrics data {} {} {} is null, ignore.",
metricsData.getId(), metricsData.getApp(), metricsData.getMetrics());
return;
}
Map<String, String> defaultLabels = Maps.newHashMapWithExpectedSize(8);
defaultLabels.put(MONITOR_METRICS_KEY, metricsData.getMetrics());
boolean isPrometheusAuto = false;
if (metricsData.getApp().startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
isPrometheusAuto = true;
defaultLabels.remove(MONITOR_METRICS_KEY);
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()
.substring(CommonConstants.PROMETHEUS_APP_PREFIX.length()));
} else {
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp());
}
defaultLabels.put(LABEL_KEY_INSTANCE, metricsData.getInstance());
List<VictoriaMetricsContent> contentList = new LinkedList<>();
try {
final int fieldSize = metricsData.getFields().size();
Long[] timestamp = new Long[]{metricsData.getTime()};
Map<String, Double> fieldsValue = Maps.newHashMapWithExpectedSize(fieldSize);
Map<String, String> labels = Maps.newHashMapWithExpectedSize(fieldSize);
RowWrapper rowWrapper = metricsData.readRow();
while (rowWrapper.hasNextRow()) {
rowWrapper = rowWrapper.nextRow();
fieldsValue.clear();
labels.clear();
rowWrapper.cellStream().forEach(cell -> {
String value = cell.getValue();
boolean isLabel = cell.getMetadataAsBoolean(MetricDataConstants.LABEL);
byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
if (type == CommonConstants.TYPE_NUMBER && !isLabel) {
// number metrics data
if (!CommonConstants.NULL_VALUE.equals(value)) {
fieldsValue.put(cell.getField().getName(), CommonUtil.parseStrDouble(value));
}
}
// label
if (isLabel && !CommonConstants.NULL_VALUE.equals(value)) {
labels.put(cell.getField().getName(), value);
}
});
for (Map.Entry<String, Double> entry : fieldsValue.entrySet()) {
if (entry.getKey() != null && entry.getValue() != null) {
try {
labels.putAll(defaultLabels);
String labelName = isPrometheusAuto ? metricsData.getMetrics()
: metricsData.getMetrics() + SPILT + entry.getKey();
labels.put(LABEL_KEY_NAME, labelName);
if (!isPrometheusAuto) {
labels.put(MONITOR_METRIC_KEY, entry.getKey());
}
labels.put(LABEL_KEY_MONITOR_ID, String.valueOf(metricsData.getId()));
// add customized labels as identifier
var customizedLabels = metricsData.getLabels();
if (!ObjectUtils.isEmpty(customizedLabels)) {
labels.putAll(customizedLabels);
}
VictoriaMetricsContent content = VictoriaMetricsContent.builder()
.metric(new HashMap<>(labels))
.values(new Double[]{entry.getValue()})
.timestamps(timestamp)
.build();
contentList.add(content);
} catch (Exception e) {
log.error("combine metrics data error: {}.", e.getMessage(), e);
}
}
}
}
} catch (Exception e) {
log.error("save metrics data to victoria error: {}.", e.getMessage(), e);
}
if (contentList.isEmpty()) {
log.info("[warehouse victoria-metrics] flush metrics data {} is empty, ignore.", metricsData.getId());
return;
}
sendVictoriaMetrics(contentList);
}
@Override
public void destroy() {
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
metricsFlushTimer.stop();
}
}
@Override
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
String labelName = metrics + SPILT + metric;
if (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
labelName = metrics;
}
String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + labelName + "\""
+ "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\""
+ (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX) ? "" : "," + MONITOR_METRIC_KEY + "=\"" + metric + "\"");
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
if (StringUtils.hasText(victoriaMetricsProp.username())
&& StringUtils.hasText(victoriaMetricsProp.password())) {
String authStr = victoriaMetricsProp.username() + ":" + victoriaMetricsProp.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
URI uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + EXPORT_PATH)
.queryParam("match[]", "{" + timeSeriesSelector + "}")
.queryParam("start", "now-" + history)
.queryParam("end", "now")
.build()
.encode()
.toUri();
ResponseEntity<String> responseEntity = restTemplate.exchange(uri,
HttpMethod.GET, httpEntity, String.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("query metrics data from victoria-metrics success. {}", uri);
if (StringUtils.hasText(responseEntity.getBody())) {
String[] contentJsonArr = responseEntity.getBody().split("\n");
List<VictoriaMetricsContent> contents = Arrays.stream(contentJsonArr).map(
item -> JsonUtil.fromJson(item, VictoriaMetricsContent.class)
).toList();
for (VictoriaMetricsContent content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && content.getTimestamps() != null) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
if (content.getValues().length != content.getTimestamps().length) {
log.error("content.getValues().length != content.getTimestamps().length");
continue;
}
Double[] values = content.getValues();
Long[] timestamps = content.getTimestamps();
for (int index = 0; index < content.getValues().length; index++) {
String strValue = BigDecimal.valueOf(values[index]).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
// read timestamp here is ms unit
valueList.add(new Value(strValue, timestamps[index]));
}
}
}
}
} else {
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
}
} catch (Exception e) {
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
}
return instanceValuesMap;
}
@Override
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
String metric, String history) {
if (!serverAvailable) {
log.error("""
\t---------------VictoriaMetrics Init Failed---------------
\t--------------Please Config VictoriaMetrics--------------
\t----------Can Not Use Metric History Now----------
""");
return Collections.emptyMap();
}
long endTime = ZonedDateTime.now().toEpochSecond();
long startTime;
try {
if (NumberUtils.isParsable(history)) {
startTime = NumberUtils.toLong(history);
startTime = (ZonedDateTime.now().toEpochSecond() - startTime);
} else {
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
startTime = dateTime.toEpochSecond();
}
} catch (Exception e) {
log.error("history time error: {}. use default: 6h", e.getMessage());
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6));
startTime = dateTime.toEpochSecond();
}
String labelName = metrics + SPILT + metric;
if (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
labelName = metrics;
}
String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + labelName + "\""
+ "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\""
+ (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX) ? "" : "," + MONITOR_METRIC_KEY + "=\"" + metric + "\"");
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
if (StringUtils.hasText(victoriaMetricsProp.username())
&& StringUtils.hasText(victoriaMetricsProp.password())) {
String authStr = victoriaMetricsProp.username() + ":" + victoriaMetricsProp.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC
+ SignConstants.BLANK + encodedAuth);
}
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
URI uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + QUERY_RANGE_PATH)
.queryParam("query", "{" + timeSeriesSelector + "}")
.queryParam("step", "4h")
.queryParam("start", startTime)
.queryParam("end", endTime)
.build()
.encode()
.toUri();
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri,
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("query metrics data from victoria-metrics success. {}", uri);
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
for (Object[] valueArr : content.getValues()) {
long timestamp = Long.parseLong(String.valueOf(valueArr[0]));
String value = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
// read timestamp here is s unit
valueList.add(new Value(value, timestamp * 1000));
}
}
}
}
} else {
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
}
// max
uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + QUERY_RANGE_PATH)
.queryParam("query", "max_over_time({" + timeSeriesSelector + "})")
.queryParam("step", "4h")
.queryParam("start", startTime)
.queryParam("end", endTime)
.build()
.encode()
.toUri();
responseEntity = restTemplate.exchange(uri,
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
if (valueList.size() == content.getValues().size()) {
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
Value value = valueList.get(timestampIndex);
Object[] valueArr = content.getValues().get(timestampIndex);
String maxValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
value.setMax(maxValue);
}
}
}
}
}
}
// min
uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + QUERY_RANGE_PATH)
.queryParam("query", "min_over_time({" + timeSeriesSelector + "})")
.queryParam("step", "4h")
.queryParam("start", startTime)
.queryParam("end", endTime)
.build()
.encode()
.toUri();
responseEntity = restTemplate.exchange(uri,
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
if (valueList.size() == content.getValues().size()) {
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
Value value = valueList.get(timestampIndex);
Object[] valueArr = content.getValues().get(timestampIndex);
String minValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
value.setMin(minValue);
}
}
}
}
}
}
// avg
uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + QUERY_RANGE_PATH)
.queryParam("query", "avg_over_time({" + timeSeriesSelector + "})")
.queryParam("step", "4h")
.queryParam("start", startTime)
.queryParam("end", endTime)
.build()
.encode()
.toUri();
responseEntity = restTemplate.exchange(uri,
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
&& responseEntity.getBody().getData().getResult() != null) {
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
for (PromQlQueryContent.ContentData.Content content : contents) {
Map<String, String> labels = content.getMetric();
labels.remove(LABEL_KEY_NAME);
labels.remove(LABEL_KEY_JOB);
labels.remove(LABEL_KEY_INSTANCE);
labels.remove(LABEL_KEY_MONITOR_ID);
labels.remove(MONITOR_METRICS_KEY);
labels.remove(MONITOR_METRIC_KEY);
String labelStr = JsonUtil.toJson(labels);
if (content.getValues() != null && !content.getValues().isEmpty()) {
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
if (valueList.size() == content.getValues().size()) {
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
Value value = valueList.get(timestampIndex);
Object[] valueArr = content.getValues().get(timestampIndex);
String avgValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
value.setMean(avgValue);
}
}
}
}
}
}
} catch (Exception e) {
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
}
return instanceValuesMap;
}
/**
* victoria metrics content
*/
@Data
@Builder
@AllArgsConstructor
@NoArgsConstructor
public static final class VictoriaMetricsContent {
/**
* metric contains metric name plus labels for a particular time series
*/
private Map<String, String> metric;
/**
* values contains raw sample values for the given time series
*/
private Double[] values;
/**
* timestamps contains raw sample UNIX timestamps in milliseconds for the given time series
* every timestamp is associated with the value at the corresponding position
*/
private Long[] timestamps;
}
/**
* add victoriaMetricsContent to buffer
* @param contentList victoriaMetricsContent List
*/
private void sendVictoriaMetrics(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> contentList) {
for (VictoriaMetricsDataStorage.VictoriaMetricsContent content : contentList) {
boolean offered = false;
int retryCount = 0;
while (!offered && retryCount < MAX_RETRIES) {
try {
// Attempt to add to the queue for a limited time
offered = metricsBufferQueue.offer(content, MAX_WAIT_MS, TimeUnit.MILLISECONDS);
if (!offered) {
// If the queue is still full, trigger an immediate refresh to free up space
if (retryCount == 0) {
log.debug("victoria metrics buffer queue is full, triggering immediate flush");
triggerImmediateFlush();
}
retryCount++;
// The short sleep allows the queue to clear out
if (retryCount < MAX_RETRIES) {
Thread.sleep(100L * retryCount);
}
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
log.error("[Victoria Metrics] Interrupted while offering metrics to buffer queue", e);
break;
}
}
// When the maximum number of retries is reached, if it still cannot be added to the queue, the data is saved directly
if (!offered) {
log.warn("[Victoria Metrics] Failed to add metrics to buffer after {} retries, saving directly", MAX_RETRIES);
try {
doSaveData(contentList);
} catch (Exception e) {
log.error("[Victoria Metrics] Failed to save metrics directly: {}", e.getMessage(), e);
}
}
}
// Refresh in advance to avoid waiting
if (metricsBufferQueue.size() >= insertConfig.bufferSize() * 0.8
&& draining.compareAndSet(false, true)) {
triggerImmediateFlush();
}
}
private void triggerImmediateFlush() {
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch = new ArrayList<>(insertConfig.bufferSize());
metricsBufferQueue.drainTo(batch, insertConfig.bufferSize());
draining.set(false);
if (!batch.isEmpty()) {
metricsFlushTimer.newTimeout(new MetricsFlushTask(batch), 0, TimeUnit.MILLISECONDS);
}
}
/**
* Regularly refresh the buffer queue to the vm
*/
private class MetricsFlushTask implements TimerTask {
private final List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch;
public MetricsFlushTask(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch) {
this.batch = batch;
}
@Override
public void run(Timeout timeout) {
try {
if (batch == null) {
// If the batch is null, it means that the timer is triggered by flush interval timer
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batchT = new ArrayList<>(insertConfig.bufferSize());
metricsBufferQueue.drainTo(batchT, insertConfig.bufferSize());
triggerDoSaveData(batchT);
// Reschedule the next flush task
triggerIntervalFlushTimer();
} else {
// If the batch is not null, it means that the timer is triggered by the immediate flush
triggerDoSaveData(batch);
}
} catch (Exception e) {
log.error("[VictoriaMetrics] flush task error: {}", e.getMessage(), e);
}
}
private void triggerDoSaveData(List<VictoriaMetricsContent> batch) {
if (!batch.isEmpty()) {
doSaveData(batch);
log.debug("[Victoria Metrics] Flushed {} metrics items", batch.size());
}
}
private void triggerIntervalFlushTimer() {
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
metricsFlushTimer.newTimeout(new MetricsFlushTask(null), insertConfig.flushInterval(), TimeUnit.SECONDS);
log.debug("[Victoria Metrics] Rescheduled next flush task in {} seconds.", insertConfig.flushInterval());
}
}
}
/**
* Save metric data to victoria-metric via HTTP call
*/
private void doSaveData(List<VictoriaMetricsContent> contentList) {
try {
HttpHeaders headers = new HttpHeaders();
headers.setContentType(MediaType.APPLICATION_JSON);
if (StringUtils.hasText(victoriaMetricsProp.username())
&& StringUtils.hasText(victoriaMetricsProp.password())) {
String authStr = victoriaMetricsProp.username() + ":" + victoriaMetricsProp.password();
String encodedAuth = Base64Util.encode(authStr);
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
}
StringBuilder stringBuilder = new StringBuilder();
for (VictoriaMetricsContent content : contentList) {
stringBuilder.append(JsonUtil.toJson(content)).append("\n");
}
String payload = stringBuilder.toString();
Object httpEntity;
if (insertConfig.compression().enabled()) {
// enable compression
headers.set(HttpHeaders.CONTENT_ENCODING, "gzip");
// compress the payload using gzip
try (ByteArrayOutputStream bos = new ByteArrayOutputStream(payload.length());
GZIPOutputStream gzip = new GZIPOutputStream(bos)) {
gzip.write(payload.getBytes(StandardCharsets.UTF_8));
// finishes writing compressed data before the gzip stream is closed.
gzip.finish();
httpEntity = new HttpEntity<>(bos.toByteArray(), headers);
}
} else {
httpEntity = new HttpEntity<>(payload, headers);
}
ResponseEntity<String> responseEntity = restTemplate.postForEntity(victoriaMetricsProp.url() + IMPORT_PATH,
httpEntity, String.class);
if (responseEntity.getStatusCode().is2xxSuccessful()) {
log.debug("insert metrics data to victoria-metrics success.");
} else {
log.error("insert metrics data to victoria-metrics failed. {}", responseEntity.getBody());
}
} catch (Exception e){
log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e);
}
}
}
@@ -0,0 +1,32 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* vminsert configuration information
*/
public record VictoriaMetricsInsertProperties(
String url,
String username,
String password,
@DefaultValue("1000") int bufferSize,
@DefaultValue("3") int flushInterval
) {
}
@@ -0,0 +1,47 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* Victoria metrics configuration information.
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.STORE
+ SignConstants.DOT
+ WarehouseConstants.HistoryName.VM)
public record VictoriaMetricsProperties(@DefaultValue("false") boolean enabled,
@DefaultValue("http://localhost:8428") String url,
String username,
String password,
InsertConfig insert) {
record InsertConfig(@DefaultValue("100") int bufferSize,
@DefaultValue("3") int flushInterval,
@DefaultValue Compression compression) {
}
record Compression(@DefaultValue("false") boolean enabled) {
}
}
@@ -0,0 +1,28 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
/**
* vmselect configuration information
*/
public record VictoriaMetricsSelectProperties(
String url,
String username,
String password
) {
}
@@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.DisposableBean;
/**
* Real-time data storage abstract class
*/
@Slf4j
public abstract class AbstractRealTimeDataStorage implements RealTimeDataReader, RealTimeDataWriter, DisposableBean {
protected volatile boolean serverAvailable;
/**
* @return data Whether the storage is available
*/
@Override
public boolean isServerAvailable() {
return serverAvailable;
}
}
@@ -0,0 +1,50 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime;
import java.util.List;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.springframework.lang.NonNull;
/**
* Real-time data reading class
*/
public interface RealTimeDataReader {
/**
* @return data storage available
*/
boolean isServerAvailable();
/**
* query real-time last metrics data
* @param monitorId monitorId
* @param metric metric name
* @return metrics data
*/
CollectRep.MetricsData getCurrentMetricsData(@NonNull Long monitorId, @NonNull String metric);
/**
* query real-time last metrics data
* @param monitorId monitor id
* @return metrics data
*/
List<CollectRep.MetricsData> getCurrentMetricsData(@NonNull Long monitorId);
}
@@ -0,0 +1,38 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime;
import org.apache.hertzbeat.common.entity.message.CollectRep;
/**
* Real-time data writing class
*/
public interface RealTimeDataWriter {
/**
* @return data storage available
*/
boolean isServerAvailable();
/**
* save metrics data
* @param metricsData metrics data
*/
void saveData(CollectRep.MetricsData metricsData);
}
@@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.memory;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.store.realtime.AbstractRealTimeDataStorage;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
/**
* Store and collect real-time data - memory
*/
@Component
@ConditionalOnProperty(prefix = "warehouse.real-time.memory", name = "enabled", havingValue = "true", matchIfMissing = true)
@Slf4j
public class MemoryDataStorage extends AbstractRealTimeDataStorage {
/**
* monitorId -> metricsName -> data
*/
private final Map<Long, Map<String, CollectRep.MetricsData>> monitorMetricsDataMap;
private static final Integer DEFAULT_INIT_SIZE = 16;
private static final Integer METRICS_SIZE = 8;
public MemoryDataStorage(MemoryProperties memoryProperties) {
int initSize = DEFAULT_INIT_SIZE;
if (memoryProperties != null
&& memoryProperties.initSize() != null) {
initSize = memoryProperties.initSize();
}
monitorMetricsDataMap = new ConcurrentHashMap<>(initSize);
this.serverAvailable = true;
}
@Override
public CollectRep.MetricsData getCurrentMetricsData(@NonNull Long monitorId, @NonNull String metric) {
Map<String, CollectRep.MetricsData> metricsDataMap = monitorMetricsDataMap.computeIfAbsent(monitorId, key -> new ConcurrentHashMap<>(METRICS_SIZE));
return metricsDataMap.get(metric);
}
@Override
public List<CollectRep.MetricsData> getCurrentMetricsData(@NonNull Long monitorId) {
Map<String, CollectRep.MetricsData> metricsDataMap = monitorMetricsDataMap.computeIfAbsent(monitorId, key -> new ConcurrentHashMap<>(METRICS_SIZE));
return new ArrayList<>(metricsDataMap.values());
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
Long monitorId = metricsData.getId();
String metrics = metricsData.getMetrics();
if (metricsData.getCode() != CollectRep.Code.SUCCESS) {
metricsData.close();
return;
}
Map<String, CollectRep.MetricsData> metricsDataMap =
monitorMetricsDataMap.computeIfAbsent(monitorId, key -> new ConcurrentHashMap<>(METRICS_SIZE));
CollectRep.MetricsData oldMetricsData = metricsDataMap.get(metrics);
if (oldMetricsData != null) {
oldMetricsData.close();
}
metricsDataMap.put(metrics, metricsData);
}
@Override
public void destroy() {
if (monitorMetricsDataMap != null) {
monitorMetricsDataMap.clear();
}
}
}
@@ -0,0 +1,39 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.memory;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* Memory storage configuration information
* @param enabled Whether memory data storage is enabled
* @param initSize Memory storage map initialization size
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.REAL_TIME
+ SignConstants.DOT
+ WarehouseConstants.RealTimeName.MEMORY)
public record MemoryProperties(@DefaultValue("true") boolean enabled,
@DefaultValue("1024") Integer initSize) {
}
@@ -0,0 +1,82 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.redis;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.store.realtime.AbstractRealTimeDataStorage;
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.RedisCommandDelegate;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.context.annotation.Primary;
import org.springframework.lang.NonNull;
import org.springframework.stereotype.Component;
/**
* redis storage collects real-time data
*/
@Primary
@Component
@ConditionalOnProperty(prefix = "warehouse.real-time.redis", name = "enabled", havingValue = "true")
@Slf4j
public class RedisDataStorage extends AbstractRealTimeDataStorage {
private final RedisCommandDelegate redisCommandDelegate;
public RedisDataStorage(RedisProperties redisProperties) {
final RedisCommandDelegate delegate = RedisCommandDelegate.getInstance();
this.serverAvailable = delegate.initRedisClient(redisProperties);
this.redisCommandDelegate = delegate;
}
@Override
public CollectRep.MetricsData getCurrentMetricsData(@NonNull Long monitorId, @NonNull String metric) {
return redisCommandDelegate.operate().hget(String.valueOf(monitorId), metric);
}
@Override
public List<CollectRep.MetricsData> getCurrentMetricsData(@NonNull Long monitorId) {
Map<String, CollectRep.MetricsData> metricsDataMap = redisCommandDelegate.operate().hgetAll(String.valueOf(monitorId));
return new ArrayList<>(metricsDataMap.values());
}
@Override
public void saveData(CollectRep.MetricsData metricsData) {
String key = String.valueOf(metricsData.getId());
String hashKey = metricsData.getMetrics();
if (metricsData.getCode() == CollectRep.Code.SUCCESS) {
redisCommandDelegate.operate().hset(key, hashKey, metricsData, future -> future.thenAccept(response -> {
metricsData.close();
if (response) {
log.debug("[warehouse] redis add new data {}:{}.", key, hashKey);
} else {
log.debug("[warehouse] redis replace data {}:{}.", key, hashKey);
}
}));
} else {
metricsData.close();
}
}
@Override
public void destroy() throws Exception {
redisCommandDelegate.destroy();
}
}
@@ -0,0 +1,41 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.redis;
import org.apache.hertzbeat.common.constants.ConfigConstants;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.bind.DefaultValue;
/**
* Redis configuration information
*/
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
+ SignConstants.DOT
+ WarehouseConstants.REAL_TIME
+ SignConstants.DOT
+ WarehouseConstants.RealTimeName.REDIS)
public record RedisProperties(@DefaultValue("false") boolean enabled,
@DefaultValue("single") String mode,
@DefaultValue("127.0.0.1:6379") String address,
String masterName,
String password,
@DefaultValue("0") Integer db) {
}
@@ -0,0 +1,37 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.redis.client;
import io.lettuce.core.RedisFuture;
import io.lettuce.core.codec.RedisCodec;
import java.util.Map;
import java.util.function.Consumer;
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
/**
* Redis Client Operation
*/
public interface RedisClientOperation<K, V> extends AutoCloseable {
RedisClientOperation<K, V> connect(RedisProperties redisProperties, RedisCodec<K, V> redisCodec);
V hget(K key, K field);
Map<K, V> hgetAll(K key);
void hset(K key, K field, V value, Consumer<RedisFuture<Boolean>> redisFutureConsumer);
}
@@ -0,0 +1,75 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.redis.client;
import lombok.extern.slf4j.Slf4j;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.serialize.RedisMetricsDataCodec;
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl.RedisClusterClientImpl;
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl.RedisSentinelClientImpl;
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl.RedisSimpleClientImpl;
/**
* Redis command delegate
*/
@Slf4j
public class RedisCommandDelegate {
private static final String SINGLE_MODE = "single";
private static final String SENTINEL_MODE = "sentinel";
private static final String CLUSTER_MODE = "cluster";
private static final RedisCommandDelegate INSTANCE = new RedisCommandDelegate();
private RedisClientOperation<String, CollectRep.MetricsData> operation;
public static RedisCommandDelegate getInstance() {
return INSTANCE;
}
public RedisClientOperation<String, CollectRep.MetricsData> operate() {
return operation;
}
public void destroy() throws Exception {
operation.close();
}
public boolean initRedisClient(RedisProperties redisProperties) {
if (redisProperties == null) {
log.error("init error, please config Warehouse redis props in application.yml");
return false;
}
try {
operation = switch (redisProperties.mode()) {
case SINGLE_MODE -> new RedisSimpleClientImpl().connect(redisProperties, new RedisMetricsDataCodec());
case SENTINEL_MODE -> new RedisSentinelClientImpl().connect(redisProperties, new RedisMetricsDataCodec());
case CLUSTER_MODE -> new RedisClusterClientImpl().connect(redisProperties, new RedisMetricsDataCodec());
default -> throw new UnsupportedOperationException("Incorrect redis mode: " + redisProperties.mode());
};
return true;
} catch (Exception e) {
log.error("init redis error {}", e.getMessage(), e);
}
return false;
}
private RedisCommandDelegate() {
}
}
@@ -0,0 +1,95 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl;
import io.lettuce.core.RedisFuture;
import io.lettuce.core.RedisURI;
import io.lettuce.core.cluster.RedisClusterClient;
import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
import io.lettuce.core.codec.RedisCodec;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.HashSet;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.RedisClientOperation;
/**
* Redis Client for cluster mode
*/
public class RedisClusterClientImpl implements RedisClientOperation<String, CollectRep.MetricsData> {
private RedisClusterClient redisClusterClient;
private StatefulRedisClusterConnection<String, CollectRep.MetricsData> connection;
@Override
public RedisClientOperation<String, CollectRep.MetricsData> connect(RedisProperties redisProperties,
RedisCodec<String, CollectRep.MetricsData> redisCodec) {
final String[] clusterAddress = redisProperties.address().split(SignConstants.COMMA);
Set<RedisURI> clusterUri = new HashSet<>();
for (String address : clusterAddress) {
final String[] split = address.split(SignConstants.DOUBLE_MARK);
RedisURI.Builder uriBuilder = RedisURI.builder()
.withHost(split[0])
.withPort(Integer.parseInt(split[1]))
.withTimeout(Duration.of(10, ChronoUnit.SECONDS));
if (StringUtils.isNotBlank(redisProperties.password())) {
uriBuilder.withPassword(redisProperties.password().toCharArray());
}
clusterUri.add(uriBuilder.build());
}
redisClusterClient = RedisClusterClient.create(clusterUri);
connection = redisClusterClient.connect(redisCodec);
return this;
}
@Override
public CollectRep.MetricsData hget(String key, String field) {
return connection.sync().hget(key, field);
}
@Override
public Map<String, CollectRep.MetricsData> hgetAll(String key) {
return connection.sync().hgetall(key);
}
@Override
public void hset(String key, String field, CollectRep.MetricsData value, Consumer<RedisFuture<Boolean>> redisFutureConsumer) {
final RedisFuture<Boolean> redisFuture = connection.async().hset(key, field, value);
if (Objects.nonNull(redisFutureConsumer)) {
redisFutureConsumer.accept(redisFuture);
}
}
@Override
public void close() throws Exception {
if (Objects.nonNull(connection)) {
connection.close();
}
if (Objects.nonNull(redisClusterClient)) {
redisClusterClient.close();
}
}
}
@@ -0,0 +1,117 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisFuture;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.codec.RedisCodec;
import io.lettuce.core.sentinel.api.StatefulRedisSentinelConnection;
import java.net.InetSocketAddress;
import java.net.SocketAddress;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.serialize.RedisMetricsDataCodec;
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.RedisClientOperation;
/**
* Redis Client for sentinel mode
*/
public class RedisSentinelClientImpl implements RedisClientOperation<String, CollectRep.MetricsData> {
private RedisClient sentinelRedisClient;
private StatefulRedisSentinelConnection<String, String> sentinelConnection;
private RedisClient masterRedisClient;
private StatefulRedisConnection<String, CollectRep.MetricsData> masterConnection;
@Override
public RedisClientOperation<String, CollectRep.MetricsData> connect(RedisProperties redisProperties,
RedisCodec<String, CollectRep.MetricsData> redisCodec) {
final String[] address = redisProperties.address().split(SignConstants.DOUBLE_MARK);
RedisURI.Builder uriBuilder = RedisURI.builder()
.withHost(address[0])
.withPort(Integer.parseInt(address[1]))
.withTimeout(Duration.of(10, ChronoUnit.SECONDS));
if (StringUtils.isNotBlank(redisProperties.password())) {
uriBuilder.withPassword(redisProperties.password().toCharArray());
}
sentinelRedisClient = RedisClient.create(uriBuilder.build());
sentinelConnection = sentinelRedisClient.connectSentinel();
final SocketAddress masterSocketAddress = sentinelConnection.sync().getMasterAddrByName(redisProperties.masterName());
if (masterSocketAddress instanceof InetSocketAddress masterAddress) {
RedisURI.Builder masterUriBuilder = RedisURI.builder()
.withHost(masterAddress.getHostName())
.withPort(masterAddress.getPort())
.withTimeout(Duration.of(10, ChronoUnit.SECONDS))
.withDatabase(redisProperties.db());
if (StringUtils.isNotBlank(redisProperties.password())) {
masterUriBuilder.withPassword(redisProperties.password().toCharArray());
}
masterRedisClient = RedisClient.create(masterUriBuilder.build());
masterConnection = masterRedisClient.connect(new RedisMetricsDataCodec());
} else {
throw new UnsupportedOperationException("Incorrect type of SocketAddress, connect redis sentinel failed...");
}
return this;
}
@Override
public CollectRep.MetricsData hget(String key, String field) {
return masterConnection.sync().hget(key, field);
}
@Override
public Map<String, CollectRep.MetricsData> hgetAll(String key) {
return masterConnection.sync().hgetall(key);
}
@Override
public void hset(String key, String field, CollectRep.MetricsData value, Consumer<RedisFuture<Boolean>> redisFutureConsumer) {
final RedisFuture<Boolean> redisFuture = masterConnection.async().hset(key, field, value);
if (Objects.nonNull(redisFutureConsumer)) {
redisFutureConsumer.accept(redisFuture);
}
}
@Override
public void close() throws Exception {
if (Objects.nonNull(masterConnection)) {
masterConnection.close();
}
if (Objects.nonNull(masterRedisClient)) {
masterRedisClient.close();
}
if (Objects.nonNull(sentinelConnection)) {
sentinelConnection.close();
}
if (Objects.nonNull(sentinelRedisClient)) {
sentinelRedisClient.close();
}
}
}
@@ -0,0 +1,90 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl;
import io.lettuce.core.RedisClient;
import io.lettuce.core.RedisFuture;
import io.lettuce.core.RedisURI;
import io.lettuce.core.api.StatefulRedisConnection;
import io.lettuce.core.codec.RedisCodec;
import java.time.Duration;
import java.time.temporal.ChronoUnit;
import java.util.Map;
import java.util.Objects;
import java.util.function.Consumer;
import org.apache.commons.lang3.StringUtils;
import org.apache.hertzbeat.common.constants.SignConstants;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.serialize.RedisMetricsDataCodec;
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.RedisClientOperation;
/**
* Redis Client
*/
public class RedisSimpleClientImpl implements RedisClientOperation<String, CollectRep.MetricsData> {
private RedisClient redisClient;
private StatefulRedisConnection<String, CollectRep.MetricsData> connection;
@Override
public RedisClientOperation<String, CollectRep.MetricsData> connect(RedisProperties redisProperties,
RedisCodec<String, CollectRep.MetricsData> redisCodec) {
final String[] address = redisProperties.address().split(SignConstants.DOUBLE_MARK);
RedisURI.Builder uriBuilder = RedisURI.builder()
.withHost(address[0])
.withPort(Integer.parseInt(address[1]))
.withTimeout(Duration.of(10, ChronoUnit.SECONDS))
.withDatabase(redisProperties.db());
if (StringUtils.isNotBlank(redisProperties.password())) {
uriBuilder.withPassword(redisProperties.password().toCharArray());
}
redisClient = RedisClient.create(uriBuilder.build());
connection = redisClient.connect(new RedisMetricsDataCodec());
return this;
}
@Override
public CollectRep.MetricsData hget(String key, String field) {
return connection.sync().hget(key, field);
}
@Override
public Map<String, CollectRep.MetricsData> hgetAll(String key) {
return connection.sync().hgetall(key);
}
@Override
public void hset(String key, String field, CollectRep.MetricsData value, Consumer<RedisFuture<Boolean>> redisFutureConsumer) {
final RedisFuture<Boolean> redisFuture = connection.async().hset(key, field, value);
if (Objects.nonNull(redisFutureConsumer)) {
redisFutureConsumer.accept(redisFuture);
}
}
@Override
public void close() throws Exception {
if (Objects.nonNull(connection)) {
connection.close();
}
if (Objects.nonNull(redisClient)) {
redisClient.close();
}
}
}
@@ -0,0 +1,16 @@
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
# The ASF licenses this file to You under the Apache License, Version 2.0
# (the "License"); you may not use this file except in compliance with
# the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
org.apache.hertzbeat.warehouse.config.WarehouseAutoConfiguration
@@ -0,0 +1,110 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicBoolean;
import org.apache.hertzbeat.common.concurrent.AdmissionMode;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
/**
* Test case for {@link WarehouseWorkerPool}
*/
class WarehouseWorkerPoolTest {
private static final int NUMBER_OF_THREADS = 10;
private WarehouseWorkerPool pool;
@AfterEach
void tearDown() throws Exception {
if (pool != null) {
pool.destroy();
}
}
@Test
void executeJob() throws InterruptedException {
pool = new WarehouseWorkerPool();
AtomicInteger counter = new AtomicInteger();
CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS);
for (int i = 0; i < NUMBER_OF_THREADS; i++) {
pool.executeJob(() -> {
counter.incrementAndGet();
latch.countDown();
});
}
latch.await();
assertEquals(NUMBER_OF_THREADS, counter.get());
}
@Test
void executeJobRunsOnVirtualThread() throws InterruptedException {
pool = new WarehouseWorkerPool();
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
pool.executeJob(() -> {
virtualThread.set(Thread.currentThread().isVirtual());
latch.countDown();
});
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void executeJobRejectsWhenConcurrencyLimitReached() throws InterruptedException {
VirtualThreadProperties properties = new VirtualThreadProperties(
true,
VirtualThreadProperties.PoolProperties.collectorDefaults(),
VirtualThreadProperties.PoolProperties.commonDefaults(),
VirtualThreadProperties.PoolProperties.managerDefaults(),
VirtualThreadProperties.AlerterProperties.defaults(),
new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1),
VirtualThreadProperties.AsyncProperties.defaults());
pool = new WarehouseWorkerPool(properties);
CountDownLatch started = new CountDownLatch(1);
CountDownLatch release = new CountDownLatch(1);
pool.executeJob(() -> {
started.countDown();
try {
release.await(5, TimeUnit.SECONDS);
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
}
});
assertTrue(started.await(5, TimeUnit.SECONDS));
try {
assertThrows(RejectedExecutionException.class, () -> pool.executeJob(() -> {
}));
} finally {
release.countDown();
}
}
}
@@ -0,0 +1,171 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.controller;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.when;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
import jakarta.servlet.ServletException;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.dto.Field;
import org.apache.hertzbeat.common.entity.dto.MetricsData;
import org.apache.hertzbeat.common.entity.dto.MetricsHistoryData;
import org.apache.hertzbeat.warehouse.service.MetricsDataService;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.test.web.servlet.MockMvc;
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
import org.springframework.util.LinkedMultiValueMap;
import org.springframework.util.MultiValueMap;
/**
* Test case for {@link MetricsDataController}
*/
@ExtendWith(MockitoExtension.class)
class MetricsDataControllerTest {
private MockMvc mockMvc;
@InjectMocks
MetricsDataController metricsDataController;
@Mock
MetricsDataService metricsDataService;
@BeforeEach
void setUp() {
metricsDataController = new MetricsDataController(metricsDataService);
this.mockMvc = MockMvcBuilders.standaloneSetup(metricsDataController).build();
}
@Test
void getWarehouseStorageServerStatus() throws Exception {
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(true);
this.mockMvc.perform(MockMvcRequestBuilders.get("/api/warehouse/storage/status"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
.andExpect(jsonPath("$.data").isEmpty())
.andExpect(jsonPath("$.msg").isEmpty())
.andReturn();
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(false);
this.mockMvc.perform(MockMvcRequestBuilders.get("/api/warehouse/storage/status"))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
.andExpect(jsonPath("$.data").isEmpty())
.andReturn();
}
@Test
void getMetricsData() throws Exception {
final long monitorId = 343254354;
final String metric = "cpu";
final String app = "testapp";
final long time = System.currentTimeMillis();
final String getUrl = "/api/monitor/" + monitorId + "/metrics/" + metric;
when(metricsDataService.getMetricsData(eq(monitorId), eq(metric))).thenReturn(null);
this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
.andExpect(jsonPath("$.msg").value("query metrics data is empty"))
.andExpect(jsonPath("$.data").isEmpty())
.andReturn();
MetricsData metricsData = MetricsData.builder()
.id(monitorId)
.app(app)
.metrics(metric)
.time(time)
.build();
when(metricsDataService.getMetricsData(eq(monitorId), eq(metric))).thenReturn(metricsData);
this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
.andExpect(jsonPath("$.msg").isEmpty())
.andExpect(jsonPath("$.data.id").value(monitorId))
.andExpect(jsonPath("$.data.metrics").value(metric))
.andExpect(jsonPath("$.data.app").value(app))
.andExpect(jsonPath("$.data.time").value(time))
.andReturn();
}
@Test
void getMetricHistoryData() throws Exception {
final String instance = "127.0.0.1:8081";
final String app = "linux";
final String metrics = "cpu";
final String metric = "usage";
final String metricFull = "linux.cpu.usage";
final String metricFullFail = "linux.usage";
final String label = "disk2";
final String history = "6h";
final Boolean interval = false;
final String getUrl = "/api/monitor/" + instance + "/metric/" + metricFull;
final String getUrlFail = "/api/monitor/" + instance + "/metric/" + metricFullFail;
MultiValueMap<String, String> params = new LinkedMultiValueMap<>();
params.add("instance", instance);
params.add("label", label);
params.add("history", history);
params.add("interval", String.valueOf(interval));
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(false);
this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl).params(params))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
.andExpect(jsonPath("$.msg").value("time series database not available"))
.andExpect(jsonPath("$.data").isEmpty())
.andReturn();
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(true);
ServletException exception = assertThrows(ServletException.class, () -> this.mockMvc.perform(MockMvcRequestBuilders.get(getUrlFail).params(params))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
.andExpect(jsonPath("$.data").isEmpty())
.andReturn());
assertTrue(exception.getMessage().contains("IllegalArgumentException"));
MetricsHistoryData metricsHistoryData = MetricsHistoryData.builder()
.instance(instance)
.metrics(metrics)
.field(Field.builder().name(metric).type(CommonConstants.TYPE_NUMBER).build())
.build();
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(true);
lenient().when(metricsDataService.getMetricHistoryData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history), eq(interval)))
.thenReturn(metricsHistoryData);
this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl).params(params))
.andExpect(status().isOk())
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
.andExpect(jsonPath("$.data.instance").value(instance))
.andExpect(jsonPath("$.data.metrics").value(metrics))
.andExpect(jsonPath("$.data.field.name").value(metric))
.andExpect(jsonPath("$.data.field.type").value(String.valueOf(CommonConstants.TYPE_NUMBER)))
.andExpect(jsonPath("$.msg").isEmpty())
.andReturn();
}
}
@@ -0,0 +1,136 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeSqlQueryContent;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
/**
* Test case for {@link GreptimeSqlQueryExecutor}
*/
@ExtendWith(MockitoExtension.class)
class GreptimeSqlQueryExecutorTest {
@Mock
private GreptimeProperties greptimeProperties;
@Mock
private RestTemplate restTemplate;
private GreptimeSqlQueryExecutor greptimeSqlQueryExecutor;
@BeforeEach
void setUp() {
when(greptimeProperties.httpEndpoint()).thenReturn("http://127.0.0.1:4000");
when(greptimeProperties.database()).thenReturn("hertzbeat");
when(greptimeProperties.username()).thenReturn("username");
when(greptimeProperties.password()).thenReturn("password");
greptimeSqlQueryExecutor = new GreptimeSqlQueryExecutor(greptimeProperties, restTemplate);
}
@Test
void testExecuteSuccess() {
// Mock successful response
GreptimeSqlQueryContent mockResponse = createMockResponse();
ResponseEntity<GreptimeSqlQueryContent> responseEntity =
new ResponseEntity<>(mockResponse, HttpStatus.OK);
when(restTemplate.exchange(
any(String.class),
eq(HttpMethod.POST),
any(HttpEntity.class),
eq(GreptimeSqlQueryContent.class)
)).thenReturn(responseEntity);
// Execute
List<Map<String, Object>> result = greptimeSqlQueryExecutor.execute("SELECT * FROM metrics");
// Verify
assertNotNull(result);
assertEquals(1, result.size());
assertEquals("cpu", result.get(0).get("metric_name"));
assertEquals(85.5, result.get(0).get("value"));
}
@Test
void testExecuteError() {
// Mock error response
when(restTemplate.exchange(
any(String.class),
eq(HttpMethod.POST),
any(HttpEntity.class),
eq(GreptimeSqlQueryContent.class)
)).thenThrow(new RuntimeException("Connection error"));
// Execute
assertThrows(RuntimeException.class, () -> greptimeSqlQueryExecutor.execute("SELECT * FROM metrics"));
}
private GreptimeSqlQueryContent createMockResponse() {
GreptimeSqlQueryContent response = new GreptimeSqlQueryContent();
response.setCode(0);
// Create simple schema
List<GreptimeSqlQueryContent.Output.Records.Schema.ColumnSchema> columnSchemas = new ArrayList<>();
columnSchemas.add(new GreptimeSqlQueryContent.Output.Records.Schema.ColumnSchema("metric_name", "String"));
columnSchemas.add(new GreptimeSqlQueryContent.Output.Records.Schema.ColumnSchema("value", "Float64"));
GreptimeSqlQueryContent.Output.Records.Schema schema =
new GreptimeSqlQueryContent.Output.Records.Schema();
schema.setColumnSchemas(columnSchemas);
// Create simple row
List<List<Object>> rows = new ArrayList<>();
rows.add(List.of("cpu", 85.5));
// Build response structure
GreptimeSqlQueryContent.Output.Records records =
new GreptimeSqlQueryContent.Output.Records();
records.setSchema(schema);
records.setRows(rows);
GreptimeSqlQueryContent.Output output = new GreptimeSqlQueryContent.Output();
output.setRecords(records);
response.setOutput(List.of(output));
return response;
}
}
@@ -0,0 +1,207 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*/
package org.apache.hertzbeat.warehouse.db;
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.PROMQL;
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.RANGE;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.net.URI;
import java.util.List;
import java.util.Map;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.PromQlQueryContent;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsClusterProperties;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsInsertProperties;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsSelectProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
/**
* Test case for {@link VictoriaMetricsClusterQueryExecutor}
*/
@ExtendWith(MockitoExtension.class)
class VictoriaMetricsClusterQueryExecutorTest {
@Mock
private RestTemplate restTemplate;
private VictoriaMetricsClusterQueryExecutor queryExecutor;
@BeforeEach
void setUp() {
VictoriaMetricsClusterProperties properties = new VictoriaMetricsClusterProperties(
true,
"42",
new VictoriaMetricsInsertProperties("http://vm-insert:8480", "insert-user", "insert-pass", 1000, 3),
new VictoriaMetricsSelectProperties("http://vm-select:8481", "select-user", "select-pass")
);
queryExecutor = new VictoriaMetricsClusterQueryExecutor(properties, restTemplate);
}
@Test
void testGetDatasource() {
assertEquals("VictoriaMetricsCluster", queryExecutor.getDatasource());
}
@Test
void testExecuteSuccess() {
PromQlQueryContent mockResponse = createInstantResponse();
ResponseEntity<PromQlQueryContent> responseEntity = new ResponseEntity<>(mockResponse, HttpStatus.OK);
when(restTemplate.exchange(
any(URI.class),
eq(HttpMethod.GET),
any(HttpEntity.class),
eq(PromQlQueryContent.class)
)).thenReturn(responseEntity);
List<Map<String, Object>> result = queryExecutor.execute("up");
assertNotNull(result);
assertEquals(1, result.size());
assertEquals("cpu_usage", result.getFirst().get("__name__"));
assertEquals("server1", result.getFirst().get("instance"));
assertEquals(1712300000.0d, result.getFirst().get("__timestamp__"));
assertEquals("85.5", result.getFirst().get("__value__"));
ArgumentCaptor<URI> uriCaptor = ArgumentCaptor.forClass(URI.class);
verify(restTemplate).exchange(
uriCaptor.capture(),
eq(HttpMethod.GET),
any(HttpEntity.class),
eq(PromQlQueryContent.class)
);
assertEquals("http", uriCaptor.getValue().getScheme());
assertEquals("vm-select", uriCaptor.getValue().getHost());
assertEquals(8481, uriCaptor.getValue().getPort());
assertEquals("/select/42/prometheus/api/v1/query", uriCaptor.getValue().getPath());
assertTrue(uriCaptor.getValue().getQuery().contains("query=up"));
}
@Test
void testExecuteErrorReturnsEmptyResult() {
when(restTemplate.exchange(
any(URI.class),
eq(HttpMethod.GET),
any(HttpEntity.class),
eq(PromQlQueryContent.class)
)).thenThrow(new RuntimeException("Connection error"));
List<Map<String, Object>> result = queryExecutor.execute("up");
assertNotNull(result);
assertEquals(0, result.size());
}
@Test
void testQueryRangeSuccess() {
PromQlQueryContent mockResponse = createRangeResponse();
ResponseEntity<PromQlQueryContent> responseEntity = new ResponseEntity<>(mockResponse, HttpStatus.OK);
when(restTemplate.exchange(
any(URI.class),
eq(HttpMethod.GET),
any(HttpEntity.class),
eq(PromQlQueryContent.class)
)).thenReturn(responseEntity);
DatasourceQuery datasourceQuery = DatasourceQuery.builder()
.refId("A")
.datasource(queryExecutor.getDatasource())
.expr("avg_over_time(up[5m])")
.exprType(PROMQL)
.timeType(RANGE)
.start(1712300000L)
.end(1712300300L)
.step("60s")
.build();
DatasourceQueryData result = queryExecutor.query(datasourceQuery);
assertNotNull(result);
assertEquals("A", result.getRefId());
assertEquals(200, result.getStatus());
assertNotNull(result.getFrames());
assertEquals(1, result.getFrames().size());
assertEquals("cpu_usage", result.getFrames().getFirst().getSchema().getLabels().get("__name__"));
assertEquals("server1", result.getFrames().getFirst().getSchema().getLabels().get("instance"));
assertEquals(1712300000000L, result.getFrames().getFirst().getData().get(0)[0]);
assertEquals("85.5", result.getFrames().getFirst().getData().get(0)[1]);
assertEquals(1712300060000L, result.getFrames().getFirst().getData().get(1)[0]);
assertEquals("86.1", result.getFrames().getFirst().getData().get(1)[1]);
ArgumentCaptor<URI> uriCaptor = ArgumentCaptor.forClass(URI.class);
verify(restTemplate).exchange(
uriCaptor.capture(),
eq(HttpMethod.GET),
any(HttpEntity.class),
eq(PromQlQueryContent.class)
);
assertEquals("/select/42/prometheus/api/v1/query_range", uriCaptor.getValue().getPath());
assertTrue(uriCaptor.getValue().getQuery().contains("query=avg_over_time"));
assertTrue(uriCaptor.getValue().getQuery().contains("start=1712300000"));
assertTrue(uriCaptor.getValue().getQuery().contains("end=1712300300"));
assertTrue(uriCaptor.getValue().getQuery().contains("step=60s"));
}
private PromQlQueryContent createInstantResponse() {
PromQlQueryContent response = new PromQlQueryContent();
PromQlQueryContent.ContentData data = new PromQlQueryContent.ContentData();
PromQlQueryContent.ContentData.Content content = new PromQlQueryContent.ContentData.Content();
content.setMetric(Map.of("__name__", "cpu_usage", "instance", "server1"));
content.setValue(new Object[] {1712300000.0d, "85.5"});
data.setResult(List.of(content));
response.setData(data);
return response;
}
private PromQlQueryContent createRangeResponse() {
PromQlQueryContent response = new PromQlQueryContent();
PromQlQueryContent.ContentData data = new PromQlQueryContent.ContentData();
PromQlQueryContent.ContentData.Content content = new PromQlQueryContent.ContentData.Content();
content.setMetric(Map.of("__name__", "cpu_usage", "instance", "server1"));
content.setValues(List.of(
new Object[] {1712300000L, "85.5"},
new Object[] {1712300060L, "86.1"}
));
data.setResult(List.of(content));
response.setData(data);
return response;
}
}
@@ -0,0 +1,115 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.service;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.eq;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.HashMap;
import java.util.Optional;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.common.support.exception.CommonException;
import org.apache.hertzbeat.warehouse.service.impl.MetricsDataServiceImpl;
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader;
import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataReader;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Test case for {@link MetricsDataService}
*/
@ExtendWith(MockitoExtension.class)
public class MetricsDataServiceTest {
@InjectMocks
private MetricsDataServiceImpl metricsDataService;
@Mock
private RealTimeDataReader realTimeDataReader;
@Mock
private HistoryDataReader historyDataReader;
@BeforeEach
public void setUp() {
metricsDataService = new MetricsDataServiceImpl(realTimeDataReader, Optional.of(historyDataReader));
}
@Test
public void testGetWarehouseStorageServerStatus() {
when(historyDataReader.isServerAvailable()).thenReturn(false);
assertFalse(metricsDataService.getWarehouseStorageServerStatus());
when(historyDataReader.isServerAvailable()).thenReturn(true);
assertTrue(metricsDataService.getWarehouseStorageServerStatus());
}
@Test
public void testGetMetricsData() {
Long monitorId = 1L;
String metrics = "disk";
when(realTimeDataReader.isServerAvailable()).thenReturn(false);
assertThrows(CommonException.class, () -> metricsDataService.getMetricsData(monitorId, metrics), "real time store not available");
when(realTimeDataReader.isServerAvailable()).thenReturn(true);
when(realTimeDataReader.getCurrentMetricsData(eq(monitorId), eq(metrics))).thenReturn(null);
assertNull(metricsDataService.getMetricsData(monitorId, metrics));
CollectRep.MetricsData storageData = CollectRep.MetricsData.newBuilder()
.setId(monitorId)
.setMetrics(metrics)
.build();
when(realTimeDataReader.isServerAvailable()).thenReturn(true);
when(realTimeDataReader.getCurrentMetricsData(eq(monitorId), eq(metrics))).thenReturn(storageData);
assertNotNull(metricsDataService.getMetricsData(monitorId, metrics));
}
@Test
public void testGetMetricHistoryData() {
String instance = "127.0.0.1:8080";
String app = "linux";
String metrics = "disk";
String metric = "used";
String label = "label";
String history = "6h";
Boolean intervalFalse = false;
Boolean intervalTrue = true;
when(historyDataReader.getHistoryMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history))).thenReturn(new HashMap<>());
assertNotNull(metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, intervalFalse));
verify(historyDataReader, times(1)).getHistoryMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history));
when(historyDataReader.getHistoryIntervalMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history))).thenReturn(new HashMap<>());
assertNotNull(metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, intervalTrue));
verify(historyDataReader, times(1)).getHistoryIntervalMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history));
}
}
@@ -0,0 +1,88 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.service;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.anyLong;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.Collections;
import java.util.List;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.service.impl.WarehouseServiceImpl;
import org.apache.hertzbeat.warehouse.store.realtime.AbstractRealTimeDataStorage;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.springframework.test.context.junit.jupiter.SpringExtension;
/**
* test case for {@link WarehouseServiceImpl}
*/
@ExtendWith(SpringExtension.class)
class WarehouseServiceTest {
@Mock
private AbstractRealTimeDataStorage realTimeDataStorage;
@InjectMocks
private WarehouseServiceImpl warehouseService;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
}
@Test
void testQueryMonitorMetricsData() {
Long monitorId = 1L;
List<CollectRep.MetricsData> expectedData = Collections.emptyList();
when(realTimeDataStorage.isServerAvailable()).thenReturn(true);
when(realTimeDataStorage.getCurrentMetricsData(monitorId)).thenReturn(expectedData);
List<CollectRep.MetricsData> result = warehouseService.queryMonitorMetricsData(monitorId);
assertEquals(expectedData, result);
verify(realTimeDataStorage, times(1)).isServerAvailable();
verify(realTimeDataStorage, times(1)).getCurrentMetricsData(monitorId);
}
@Test
void testQueryMonitorMetricsDataNotAvailable() {
Long monitorId = 1L;
when(realTimeDataStorage.isServerAvailable()).thenReturn(false);
List<CollectRep.MetricsData> result = warehouseService.queryMonitorMetricsData(monitorId);
assertTrue(result.isEmpty());
verify(realTimeDataStorage, times(1)).isServerAvailable();
verify(realTimeDataStorage, never()).getCurrentMetricsData(anyLong());
}
}
@@ -0,0 +1,122 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
import java.util.List;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.store.realtime.memory.MemoryDataStorage;
import org.apache.hertzbeat.warehouse.store.realtime.memory.MemoryProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
/**
* Test case for {@link MemoryDataStorage}
*/
class MemoryDataStorageTest {
@Mock
private MemoryProperties memoryProperties;
@InjectMocks
private MemoryDataStorage memoryDataStorage;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
when(memoryProperties.initSize()).thenReturn(null);
memoryDataStorage = new MemoryDataStorage(memoryProperties);
}
@Test
void testGetCurrentMetricsDataByMetric() {
Long monitorId = 1L;
String metric = "cpuUsage";
CollectRep.MetricsData metricsData = mock(CollectRep.MetricsData.class);
memoryDataStorage.saveData(metricsData);
CollectRep.MetricsData result = memoryDataStorage.getCurrentMetricsData(monitorId, metric);
assertNull(result);
}
@Test
void testGetCurrentMetricsData() {
Long monitorId = 1L;
CollectRep.MetricsData metricsData1 = mock(CollectRep.MetricsData.class);
CollectRep.MetricsData metricsData2 = mock(CollectRep.MetricsData.class);
when(metricsData1.getId()).thenReturn(monitorId);
when(metricsData1.getMetrics()).thenReturn("cpuUsage");
when(metricsData1.getCode()).thenReturn(CollectRep.Code.SUCCESS);
when(metricsData2.getId()).thenReturn(monitorId);
when(metricsData2.getMetrics()).thenReturn("memoryUsage");
when(metricsData2.getCode()).thenReturn(CollectRep.Code.SUCCESS);
memoryDataStorage.saveData(metricsData1);
memoryDataStorage.saveData(metricsData2);
List<CollectRep.MetricsData> result = memoryDataStorage.getCurrentMetricsData(monitorId);
assertEquals(2, result.size());
assertTrue(result.contains(metricsData1));
assertTrue(result.contains(metricsData2));
}
@Test
void testSaveDataFailure() {
CollectRep.MetricsData metricsData = mock(CollectRep.MetricsData.class);
when(metricsData.getCode()).thenReturn(CollectRep.Code.FAIL);
memoryDataStorage.saveData(metricsData);
List<CollectRep.MetricsData> result = memoryDataStorage.getCurrentMetricsData(metricsData.getId());
assertTrue(result.isEmpty());
}
@Test
void testDestroy() {
CollectRep.MetricsData metricsData = mock(CollectRep.MetricsData.class);
when(metricsData.getId()).thenReturn(1L);
when(metricsData.getMetrics()).thenReturn("cpuUsage");
when(metricsData.getCode()).thenReturn(CollectRep.Code.SUCCESS);
memoryDataStorage.saveData(metricsData);
memoryDataStorage.destroy();
List<CollectRep.MetricsData> result = memoryDataStorage.getCurrentMetricsData(1L);
assertTrue(result.isEmpty());
}
}
@@ -0,0 +1,85 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import com.google.common.collect.Lists;
import java.util.List;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisDataStorage;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
/**
* Test case for {@link RedisDataStorage}
*/
class RedisDataStorageTest {
@Mock
private RedisDataStorage redisDataStorage;
@BeforeEach
void setUp() {
MockitoAnnotations.openMocks(this);
}
@Test
void testGetCurrentMetricsData() {
Long monitorId = 1L;
String metric = "testMetric";
CollectRep.MetricsData expectedMetricsData = CollectRep.MetricsData.newBuilder().setMetrics(metric).setId(monitorId).build();
Mockito.doReturn(expectedMetricsData).when(redisDataStorage).getCurrentMetricsData(monitorId, metric);
CollectRep.MetricsData actualMetricsData = redisDataStorage.getCurrentMetricsData(monitorId, metric);
assertNotNull(actualMetricsData);
assertEquals(expectedMetricsData, actualMetricsData);
verify(redisDataStorage, times(1)).getCurrentMetricsData(monitorId, metric);
}
@Test
void testGetCurrentMetricsDataByMonitorId() {
Long monitorId = 1L;
String metric = "testMetric";
CollectRep.MetricsData expectedMetricsData = CollectRep.MetricsData.newBuilder().setMetrics(metric).setId(monitorId).build();
Mockito.doReturn(Lists.newArrayList(expectedMetricsData)).when(redisDataStorage).getCurrentMetricsData(monitorId);
List<CollectRep.MetricsData> actualMetricsData = redisDataStorage.getCurrentMetricsData(monitorId);
assertNotNull(actualMetricsData);
assertEquals(expectedMetricsData, actualMetricsData.get(0));
verify(redisDataStorage, times(1)).getCurrentMetricsData(monitorId);
}
@Test
void testSaveData() {
long monitorId = 1L;
String metric = "testMetric";
CollectRep.MetricsData expectedMetricsData = CollectRep.MetricsData.newBuilder().setMetrics(metric).setId(monitorId).build();
redisDataStorage.saveData(expectedMetricsData);
verify(redisDataStorage, times(1)).saveData(expectedMetricsData);
}
}
@@ -0,0 +1,206 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store;
import com.zaxxer.hikari.HikariDataSource;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.arrow.ArrowCell;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.store.history.tsdb.tdengine.TdEngineDataStorage;
import org.apache.hertzbeat.warehouse.store.history.tsdb.tdengine.TdEngineProperties;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import java.sql.Connection;
import java.sql.Statement;
import java.util.List;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
/**
* Test case for {@link TdEngineDataStorage}
*/
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
class TdEngineDataStorageTest {
@Mock
private TdEngineProperties tdEngineProperties;
@Mock
private HikariDataSource mockHikariDataSource;
@Mock
private Connection mockConnection;
@Mock
private Statement mockStatement;
private TdEngineDataStorage tdEngineDataStorage;
@BeforeEach
void setUp() throws Exception {
when(tdEngineProperties.enabled()).thenReturn(true);
when(tdEngineProperties.url()).thenReturn("jdbc:TAOS-RS://localhost:6041/demo");
when(tdEngineProperties.username()).thenReturn("root");
when(tdEngineProperties.password()).thenReturn("root");
when(tdEngineProperties.tableStrColumnDefineMaxLength()).thenReturn(200);
when(mockHikariDataSource.getConnection()).thenReturn(mockConnection);
when(mockConnection.createStatement()).thenReturn(mockStatement);
}
@Test
void isServerAvailable() {
}
@Test
void testSaveData() throws Exception {
tdEngineDataStorage = new TdEngineDataStorage(tdEngineProperties);
setPrivateField(tdEngineDataStorage, "hikariDataSource", mockHikariDataSource);
setParentPrivateField(tdEngineDataStorage, "serverAvailable", true);
CollectRep.MetricsData metricsData = generateMockedMetricsData();
tdEngineDataStorage.saveData(metricsData);
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
verify(mockStatement, atLeastOnce()).execute(sqlCaptor.capture());
String executedSql = sqlCaptor.getValue();
// Verify SQL structure
assertNotNull(executedSql);
assertTrue(executedSql.startsWith("INSERT INTO"), "SQL should start with INSERT INTO");
assertTrue(executedSql.contains("USING"), "SQL should contain USING clause");
assertTrue(executedSql.contains("TAGS"), "SQL should contain TAGS clause");
assertTrue(executedSql.contains("VALUES"), "SQL should contain VALUES clause");
// Verify table name format: app_metrics_instance_v2
assertTrue(executedSql.contains("app_cpu_test-%server-01_v2"), "Should contain correct table name");
// Verify super table name format: app_metrics_super_v2
assertTrue(executedSql.contains("app_cpu_super_v2"), "Should contain correct super table name");
// Verify tags format: test-%server-01
assertTrue(executedSql.contains("TAGS ('test-%server-01')"), "Should contain correct super table name");
// Verify VALUES clause structure (timestamp + data values)
assertTrue(executedSql.matches(".*VALUES\\s+\\(\\d+.*68\\.7\\)"), "Should contain timestamp and value 68.7");
}
@Test
void destroy() {
}
@Test
void getHistoryMetricData() {
}
@Test
void getHistoryIntervalMetricData() {
}
/**
* Helper method to set private field using reflection
*/
private void setPrivateField(Object target, String fieldName, Object value) throws Exception {
java.lang.reflect.Field field = target.getClass().getDeclaredField(fieldName);
field.setAccessible(true);
field.set(target, value);
}
/**
* Helper method to set private field from parent class using reflection
*/
private void setParentPrivateField(Object target, String fieldName, Object value) throws Exception {
java.lang.reflect.Field field = target.getClass().getSuperclass().getDeclaredField(fieldName);
field.setAccessible(true);
field.set(target, value);
}
public static CollectRep.MetricsData generateMockedMetricsData() {
CollectRep.MetricsData mockMetricsData = Mockito.mock(CollectRep.MetricsData.class);
when(mockMetricsData.getId()).thenReturn(0L);
when(mockMetricsData.getMetrics()).thenReturn("cpu");
when(mockMetricsData.getTime()).thenReturn(System.currentTimeMillis());
when(mockMetricsData.getCode()).thenReturn(CollectRep.Code.SUCCESS);
when(mockMetricsData.getApp()).thenReturn("app");
when(mockMetricsData.getInstance()).thenReturn("test-%server-01");
CollectRep.ValueRow mockValueRow = Mockito.mock(CollectRep.ValueRow.class);
List<String> columnValues = List.of("test-%server-01", "68.7");
when(mockValueRow.getColumnsList()).thenReturn(columnValues);
when(mockValueRow.getColumns(0)).thenReturn("test-%server-01");
when(mockValueRow.getColumns(1)).thenReturn("68.7");
List<CollectRep.ValueRow> mockValueRowsList = List.of(mockValueRow);
when(mockMetricsData.getValues()).thenReturn(mockValueRowsList);
CollectRep.Field instanceField = Mockito.mock(CollectRep.Field.class);
when(instanceField.getName()).thenReturn("instance");
CollectRep.Field usageField = Mockito.mock(CollectRep.Field.class);
when(usageField.getName()).thenReturn("usage");
CollectRep.Field systemField = Mockito.mock(CollectRep.Field.class);
when(systemField.getName()).thenReturn("system");
List<CollectRep.Field> mockFields = List.of(instanceField, usageField, systemField);
when(mockMetricsData.getFields()).thenReturn(mockFields);
ArrowType instanceArrowType = new ArrowType.Utf8();
FieldType instanceFieldType = new FieldType(true, instanceArrowType, null, null);
Field instanceArrowField = new Field("instance", instanceFieldType, null);
ArrowCell instanceCell = Mockito.mock(ArrowCell.class);
when(instanceCell.getField()).thenReturn(instanceArrowField);
when(instanceCell.getValue()).thenReturn("test-%server-01");
when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true);
when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);
ArrowType usageArrowType = new ArrowType.Utf8();
FieldType usageFieldType = new FieldType(true, usageArrowType, null, null);
Field usageArrowField = new Field("usage", usageFieldType, null);
ArrowCell usageCell = Mockito.mock(ArrowCell.class);
when(usageCell.getField()).thenReturn(usageArrowField);
when(usageCell.getValue()).thenReturn("68.7");
when(usageCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(false);
when(usageCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_NUMBER);
List<ArrowCell> mockCells = List.of(instanceCell, usageCell);
// Create Arrow Field list for RowWrapper
List<org.apache.arrow.vector.types.pojo.Field> arrowFields = List.of(instanceArrowField, usageArrowField);
RowWrapper mockRowWrapper = Mockito.mock(RowWrapper.class);
when(mockRowWrapper.hasNextRow()).thenReturn(true).thenReturn(false);
when(mockRowWrapper.nextRow()).thenReturn(mockRowWrapper);
when(mockRowWrapper.cellStream()).thenAnswer(invocation -> mockCells.stream());
when(mockRowWrapper.getFieldList()).thenReturn(arrowFields);
when(mockMetricsData.readRow()).thenReturn(mockRowWrapper);
return mockMetricsData;
}
}
@@ -0,0 +1,314 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.doris;
import static org.assertj.core.api.Assertions.assertThat;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockConstruction;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import com.zaxxer.hikari.HikariDataSource;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.Statement;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.warehouse.WarehouseWorkerPool;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.MockedConstruction;
import org.mockito.MockedStatic;
import org.mockito.junit.jupiter.MockitoExtension;
/**
* Tests for {@link DorisDataStorage}.
*/
@ExtendWith(MockitoExtension.class)
class DorisDataStorageTest {
private static final long NANOS_PER_MILLISECOND = 1_000_000L;
@Mock
private WarehouseWorkerPool workerPool;
@Test
void queryLogsWithPaginationShouldContainLimitAndOffsetClauses() throws Exception {
QueryStorageContext context = createQueryStorageContext(createProperties(false));
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
when(context.resultSet().next()).thenReturn(false);
context.storage().queryLogsByMultipleConditionsWithPagination(
1000L, 2000L, null, null, null, null, null, 5, 20
);
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
String sql = sqlCaptor.getValue();
assertThat(sql).contains("ORDER BY time_unix_nano DESC");
assertThat(sql).contains("LIMIT ?");
assertThat(sql).contains("OFFSET ?");
}
@Test
void queryLogsShouldBindAllConditionsWithConvertedTimeInOrder() throws Exception {
QueryStorageContext context = createQueryStorageContext(createProperties(false));
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
when(context.resultSet().next()).thenReturn(false);
context.storage().queryLogsByMultipleConditions(
1000L, 2000L, "trace-1", "span-1", 9, "INFO", "error"
);
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
String sql = sqlCaptor.getValue();
assertThat(sql).contains("WHERE");
assertThat(sql).contains("time_unix_nano >= ?");
assertThat(sql).contains("time_unix_nano <= ?");
assertThat(sql).contains("trace_id = ?");
assertThat(sql).contains("span_id = ?");
assertThat(sql).contains("severity_number = ?");
assertThat(sql).contains("severity_text = ?");
assertThat(sql).contains("body LIKE ?");
assertThat(sql).contains("ORDER BY time_unix_nano DESC");
verify(context.queryPreparedStatement()).setObject(1, 1000L * NANOS_PER_MILLISECOND);
verify(context.queryPreparedStatement()).setObject(2, 2000L * NANOS_PER_MILLISECOND);
verify(context.queryPreparedStatement()).setObject(3, "trace-1");
verify(context.queryPreparedStatement()).setObject(4, "span-1");
verify(context.queryPreparedStatement()).setObject(5, 9);
verify(context.queryPreparedStatement()).setObject(6, "INFO");
verify(context.queryPreparedStatement()).setObject(7, "%error%");
}
@Test
void queryLogsWithPaginationShouldNotAppendOffsetWhenOffsetIsZero() throws Exception {
QueryStorageContext context = createQueryStorageContext(createProperties(false));
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
when(context.resultSet().next()).thenReturn(false);
context.storage().queryLogsByMultipleConditionsWithPagination(
null, null, null, null, null, null, null, 0, 20
);
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
String sql = sqlCaptor.getValue();
assertThat(sql).contains("LIMIT ?");
assertThat(sql).doesNotContain("OFFSET ?");
verify(context.queryPreparedStatement()).setObject(1, 20);
}
@Test
void countLogsShouldReturnCountAndBindConditions() throws Exception {
QueryStorageContext context = createQueryStorageContext(createProperties(false));
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
when(context.resultSet().next()).thenReturn(true);
when(context.resultSet().getLong("count")).thenReturn(5L);
long count = context.storage().countLogsByMultipleConditions(
1000L, null, "trace-1", null, null, null, null
);
assertThat(count).isEqualTo(5L);
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
String sql = sqlCaptor.getValue();
assertThat(sql).contains("SELECT COUNT(*) AS count");
assertThat(sql).contains("time_unix_nano >= ?");
assertThat(sql).contains("trace_id = ?");
verify(context.queryPreparedStatement()).setObject(1, 1000L * NANOS_PER_MILLISECOND);
verify(context.queryPreparedStatement()).setObject(2, "trace-1");
}
@Test
void batchDeleteLogsShouldFilterNullValuesAndExecuteDelete() throws Exception {
QueryStorageContext context = createQueryStorageContext(createProperties(false));
when(context.queryPreparedStatement().executeUpdate()).thenReturn(2);
List<Long> timeUnixNanos = new ArrayList<>();
timeUnixNanos.add(111L);
timeUnixNanos.add(null);
timeUnixNanos.add(333L);
boolean deleted = context.storage().batchDeleteLogs(timeUnixNanos);
assertThat(deleted).isTrue();
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
assertThat(sqlCaptor.getValue()).contains("time_unix_nano IN (?,?)");
verify(context.queryPreparedStatement()).setLong(1, 111L);
verify(context.queryPreparedStatement()).setLong(2, 333L);
}
@Test
void batchDeleteLogsShouldReturnFalseWhenAllValuesAreNull() throws Exception {
DorisDataStorage storage = createStorageContext(createProperties(false));
List<Long> timeUnixNanos = new ArrayList<>();
timeUnixNanos.add(null);
timeUnixNanos.add(null);
boolean deleted = storage.batchDeleteLogs(timeUnixNanos);
assertThat(deleted).isFalse();
}
@Test
void queryLogsShouldMapJsonColumnsToLogEntryFields() throws Exception {
QueryStorageContext context = createQueryStorageContext(createProperties(false));
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
ResultSet resultSet = context.resultSet();
when(resultSet.next()).thenReturn(true, false);
when(resultSet.getLong("time_unix_nano")).thenReturn(111L);
when(resultSet.getLong("observed_time_unix_nano")).thenReturn(222L);
when(resultSet.getInt("severity_number")).thenReturn(9);
when(resultSet.getString("severity_text")).thenReturn("INFO");
when(resultSet.getString("body")).thenReturn("{\"message\":\"ok\"}");
when(resultSet.getString("trace_id")).thenReturn("trace-1");
when(resultSet.getString("span_id")).thenReturn("span-1");
when(resultSet.getInt("trace_flags")).thenReturn(1);
when(resultSet.getString("attributes")).thenReturn("{\"k\":\"v\"}");
when(resultSet.getString("resource")).thenReturn("{\"service\":\"warehouse\"}");
when(resultSet.getString("instrumentation_scope")).thenReturn("{\"name\":\"scope\",\"version\":\"1.0.0\"}");
when(resultSet.getInt("dropped_attributes_count")).thenReturn(3);
when(resultSet.wasNull()).thenReturn(false, false, false);
List<LogEntry> entries = context.storage().queryLogsByMultipleConditions(
null, null, null, null, null, null, null
);
assertThat(entries).hasSize(1);
LogEntry entry = entries.get(0);
assertThat(entry.getTimeUnixNano()).isEqualTo(111L);
assertThat(entry.getObservedTimeUnixNano()).isEqualTo(222L);
assertThat(entry.getSeverityNumber()).isEqualTo(9);
assertThat(entry.getSeverityText()).isEqualTo("INFO");
assertThat(entry.getTraceId()).isEqualTo("trace-1");
assertThat(entry.getSpanId()).isEqualTo("span-1");
assertThat(entry.getBody()).isInstanceOf(Map.class);
@SuppressWarnings("unchecked")
Map<String, Object> body = (Map<String, Object>) entry.getBody();
assertThat(body).containsEntry("message", "ok");
assertThat(entry.getAttributes()).containsEntry("k", "v");
assertThat(entry.getResource()).containsEntry("service", "warehouse");
assertThat(entry.getInstrumentationScope()).isNotNull();
assertThat(entry.getInstrumentationScope().getName()).isEqualTo("scope");
assertThat(entry.getInstrumentationScope().getVersion()).isEqualTo("1.0.0");
}
private QueryStorageContext createQueryStorageContext(DorisProperties properties) {
Connection initConnection = mock(Connection.class);
Statement initStatement = mock(Statement.class);
Connection tableConnection = mock(Connection.class);
Statement tableStatement = mock(Statement.class);
Connection queryConnection = mock(Connection.class);
PreparedStatement queryPreparedStatement = mock(PreparedStatement.class);
ResultSet resultSet = mock(ResultSet.class);
AtomicInteger dataSourceConnectionCalls = new AtomicInteger(0);
String baseUrl = properties.url();
String username = properties.username();
String password = properties.password();
try {
when(initConnection.createStatement()).thenReturn(initStatement);
when(tableConnection.createStatement()).thenReturn(tableStatement);
when(queryConnection.prepareStatement(anyString())).thenReturn(queryPreparedStatement);
} catch (Exception e) {
throw new RuntimeException(e);
}
try (MockedStatic<DriverManager> driverManagerMock = mockStatic(DriverManager.class);
MockedConstruction<HikariDataSource> hikariDataSourceMockedConstruction = mockConstruction(
HikariDataSource.class,
(mock, context) -> when(mock.getConnection()).thenAnswer(invocation -> {
int call = dataSourceConnectionCalls.getAndIncrement();
return call == 0 ? tableConnection : queryConnection;
}))) {
driverManagerMock.when(() -> DriverManager.getConnection(baseUrl, username, password))
.thenReturn(initConnection);
DorisDataStorage storage = new DorisDataStorage(properties, workerPool);
return new QueryStorageContext(storage, queryConnection, queryPreparedStatement, resultSet);
}
}
private DorisDataStorage createStorageContext(DorisProperties properties) {
Connection initConnection = mock(Connection.class);
Statement initStatement = mock(Statement.class);
Connection tableConnection = mock(Connection.class);
Statement tableStatement = mock(Statement.class);
String baseUrl = properties.url();
String username = properties.username();
String password = properties.password();
try {
when(initConnection.createStatement()).thenReturn(initStatement);
when(tableConnection.createStatement()).thenReturn(tableStatement);
} catch (Exception e) {
throw new RuntimeException(e);
}
try (MockedStatic<DriverManager> driverManagerMock = mockStatic(DriverManager.class);
MockedConstruction<HikariDataSource> hikariDataSourceMockedConstruction = mockConstruction(
HikariDataSource.class, (mock, context) -> when(mock.getConnection()).thenReturn(tableConnection))) {
driverManagerMock.when(() -> DriverManager.getConnection(baseUrl, username, password))
.thenReturn(initConnection);
return new DorisDataStorage(properties, workerPool);
}
}
private DorisProperties createProperties(boolean enablePartition) {
DorisProperties.TableConfig tableConfig = new DorisProperties.TableConfig(
enablePartition, "HOUR", 2, 1, 12, 1, 4096
);
DorisProperties.PoolConfig poolConfig = new DorisProperties.PoolConfig(
1, 2, 500, 0, 60_000
);
DorisProperties.WriteConfig writeConfig = new DorisProperties.WriteConfig(
"jdbc", 1000, 5, false, DorisProperties.StreamLoadConfig.createDefault()
);
return new DorisProperties(
true,
"jdbc:mysql://127.0.0.1:9030/hertzbeat",
"root",
"123456",
tableConfig,
poolConfig,
writeConfig
);
}
private record QueryStorageContext(DorisDataStorage storage, Connection queryConnection,
PreparedStatement queryPreparedStatement, ResultSet resultSet) {
}
}
@@ -0,0 +1,148 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.duckdb;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.nio.file.Path;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
/**
* Test case for {@link DuckdbDatabaseDataStorage}.
*/
class DuckdbDatabaseDataStorageTest {
private TestDuckdbDatabaseDataStorage dataStorage;
@TempDir
Path tempDir;
@AfterEach
void tearDown() throws Exception {
if (dataStorage != null) {
dataStorage.destroy();
}
}
@Test
void dispatchExpiredDataCleanerRunsOnVirtualThread() throws Exception {
CountDownLatch latch = new CountDownLatch(1);
AtomicBoolean virtualThread = new AtomicBoolean(false);
dataStorage = new TestDuckdbDatabaseDataStorage(properties(), latch, virtualThread,
null, null, null, null, null, null);
dataStorage.dispatchExpiredDataCleaner();
assertTrue(latch.await(5, TimeUnit.SECONDS));
assertTrue(virtualThread.get());
}
@Test
void dispatchExpiredDataCleanerDoesNotRunConcurrently() throws Exception {
CountDownLatch firstStarted = new CountDownLatch(1);
CountDownLatch releaseFirst = new CountDownLatch(1);
CountDownLatch secondStarted = new CountDownLatch(1);
AtomicInteger concurrent = new AtomicInteger();
AtomicInteger maxConcurrent = new AtomicInteger();
AtomicInteger invocations = new AtomicInteger();
dataStorage = new TestDuckdbDatabaseDataStorage(properties(), null, null,
firstStarted, releaseFirst, secondStarted, concurrent, maxConcurrent, invocations);
dataStorage.dispatchExpiredDataCleaner();
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
dataStorage.dispatchExpiredDataCleaner();
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
releaseFirst.countDown();
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
assertEquals(1, maxConcurrent.get());
}
private DuckdbProperties properties() {
return new DuckdbProperties(true, "1d", tempDir.resolve("history.duckdb").toString());
}
private static final class TestDuckdbDatabaseDataStorage extends DuckdbDatabaseDataStorage {
private final CountDownLatch virtualThreadLatch;
private final AtomicBoolean virtualThread;
private final CountDownLatch firstStarted;
private final CountDownLatch releaseFirst;
private final CountDownLatch secondStarted;
private final AtomicInteger concurrent;
private final AtomicInteger maxConcurrent;
private final AtomicInteger invocations;
private TestDuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties,
CountDownLatch virtualThreadLatch,
AtomicBoolean virtualThread,
CountDownLatch firstStarted,
CountDownLatch releaseFirst,
CountDownLatch secondStarted,
AtomicInteger concurrent,
AtomicInteger maxConcurrent,
AtomicInteger invocations) {
super(duckdbProperties, new VirtualThreadProperties(), false);
this.virtualThreadLatch = virtualThreadLatch;
this.virtualThread = virtualThread;
this.firstStarted = firstStarted;
this.releaseFirst = releaseFirst;
this.secondStarted = secondStarted;
this.concurrent = concurrent;
this.maxConcurrent = maxConcurrent;
this.invocations = invocations;
}
@Override
void beforeExpiredDataCleanerRun() {
if (virtualThreadLatch != null && virtualThread != null) {
virtualThread.set(Thread.currentThread().isVirtual());
virtualThreadLatch.countDown();
}
if (firstStarted == null || releaseFirst == null || secondStarted == null
|| concurrent == null || maxConcurrent == null || invocations == null) {
return;
}
int running = concurrent.incrementAndGet();
maxConcurrent.accumulateAndGet(running, Math::max);
int currentInvocation = invocations.incrementAndGet();
try {
if (currentInvocation == 1) {
firstStarted.countDown();
releaseFirst.await(5, TimeUnit.SECONDS);
} else if (currentInvocation == 2) {
secondStarted.countDown();
}
} catch (InterruptedException e) {
Thread.currentThread().interrupt();
} finally {
concurrent.decrementAndGet();
}
}
}
}
@@ -0,0 +1,460 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.mockStatic;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import io.greptime.GreptimeDB;
import io.greptime.models.Err;
import io.greptime.models.Result;
import io.greptime.models.Table;
import io.greptime.models.WriteOk;
import io.greptime.v1.Common;
import io.greptime.v1.RowData;
import java.lang.reflect.Field;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.stream.Collectors;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.entity.arrow.ArrowCell;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.dto.Value;
import org.apache.hertzbeat.common.entity.log.LogEntry;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.PromQlQueryContent;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.ArgumentCaptor;
import org.mockito.Mock;
import org.mockito.MockedStatic;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
/**
* Test case for {@link GreptimeDbDataStorage}
*/
@ExtendWith(MockitoExtension.class)
class GreptimeDbDataStorageTest {
@Mock
private GreptimeProperties greptimeProperties;
@Mock
private RestTemplate restTemplate;
@Mock
private GreptimeSqlQueryExecutor greptimeSqlQueryExecutor;
@Mock
private GreptimeDB greptimeDb;
private GreptimeDbDataStorage greptimeDbDataStorage;
@BeforeEach
void setUp() {
lenient().when(greptimeProperties.grpcEndpoints()).thenReturn("127.0.0.1:4001");
lenient().when(greptimeProperties.database()).thenReturn("hertzbeat");
lenient().when(greptimeProperties.username()).thenReturn("username");
lenient().when(greptimeProperties.password()).thenReturn("password");
lenient().when(greptimeProperties.httpEndpoint()).thenReturn("http://127.0.0.1:4000");
}
@Test
void testConstructor() {
// Test constructor with null properties
assertThrows(IllegalArgumentException.class, () -> new GreptimeDbDataStorage(null, restTemplate, greptimeSqlQueryExecutor));
// Test successful constructor initialization
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
GreptimeDbDataStorage storage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
assertNotNull(storage);
assertTrue(storage.isServerAvailable());
}
// Test constructor when GreptimeDB.create throws an exception
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenThrow(new RuntimeException("Connection failed"));
GreptimeDbDataStorage storage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
assertNotNull(storage);
assertFalse(storage.isServerAvailable());
}
}
@Test
void testSaveData() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
// Mock the write result
@SuppressWarnings("unchecked")
Result<WriteOk, Err> mockResult = mock(Result.class);
when(mockResult.isOk()).thenReturn(true);
CompletableFuture<Result<WriteOk, Err>> mockFuture = CompletableFuture.completedFuture(mockResult);
when(greptimeDb.write(any(Table.class))).thenReturn(mockFuture);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
// Test with valid metrics data
CollectRep.MetricsData metricsData = createMockMetricsData(true);
greptimeDbDataStorage.saveData(metricsData);
verify(greptimeDb, times(1)).write(any(Table.class));
// Test with failure code
CollectRep.MetricsData failMetricsData = mock(CollectRep.MetricsData.class);
when(failMetricsData.getCode()).thenReturn(CollectRep.Code.FAIL);
greptimeDbDataStorage.saveData(failMetricsData);
// Verify write was not called again
verify(greptimeDb, times(1)).write(any(Table.class));
// Test with empty values
CollectRep.MetricsData emptyMetricsData = createMockMetricsData(false);
greptimeDbDataStorage.saveData(emptyMetricsData);
// Verify write was not called again
verify(greptimeDb, times(1)).write(any(Table.class));
}
}
@Test
void testSaveDataWithCustomLabels() throws Exception {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
Result<WriteOk, Err> mockResult = mock(Result.class);
when(mockResult.isOk()).thenReturn(true);
CompletableFuture<Result<WriteOk, Err>> mockFuture = CompletableFuture.completedFuture(mockResult);
when(greptimeDb.write(any(Table.class))).thenReturn(mockFuture);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
CollectRep.MetricsData metricsData = createMockMetricsData(true);
Map<String, String> customLabels = new LinkedHashMap<>();
customLabels.put("instance", "bad_instance");
customLabels.put("ts", "bad_ts");
customLabels.put("usage", "bad_usage");
customLabels.put("env", "prod");
when(metricsData.getLabels()).thenReturn(customLabels);
when(metricsData.getInstance()).thenReturn("server1");
ArgumentCaptor<Table> tableCaptor = ArgumentCaptor.forClass(Table.class);
greptimeDbDataStorage.saveData(metricsData);
verify(greptimeDb).write(tableCaptor.capture());
Table capturedTable = tableCaptor.getValue();
List<RowData.ColumnSchema> columnSchemas = getColumnSchemas(capturedTable);
List<String> columnNames = columnSchemas.stream()
.map(RowData.ColumnSchema::getColumnName)
.collect(Collectors.toList());
assertEquals(5, columnNames.size());
assertEquals(2, Collections.frequency(columnNames, "instance"));
assertEquals(1, Collections.frequency(columnNames, "ts"));
assertEquals(1, Collections.frequency(columnNames, "usage"));
List<RowData.ColumnSchema> envColumnSchemas = columnSchemas.stream()
.filter(columnSchema -> "env".equals(columnSchema.getColumnName()))
.collect(Collectors.toList());
assertEquals(1, envColumnSchemas.size());
assertEquals(Common.SemanticType.TAG, envColumnSchemas.get(0).getSemanticType());
List<RowData.Row> rows = getRows(capturedTable);
assertEquals(1, rows.size());
RowData.Row row = rows.get(0);
assertEquals("prod", row.getValuesList().get(4).getStringValue());
}
}
@Test
void testGetHistoryMetricData() {
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
PromQlQueryContent content = createMockPromQlQueryContent();
ResponseEntity<PromQlQueryContent> responseEntity = new ResponseEntity<>(content, HttpStatus.OK);
when(restTemplate.exchange(any(), eq(HttpMethod.GET), any(HttpEntity.class), eq(PromQlQueryContent.class)))
.thenReturn(responseEntity);
Map<String, List<Value>> result = greptimeDbDataStorage.getHistoryMetricData("127.0.0.1:8080", "test_app", "test_metrics", "test_metric", "6h");
assertNotNull(result);
assertFalse(result.isEmpty());
// Verify that the mapping logic correctly extracted the value
assertEquals("85.5", result.values().iterator().next().get(0).getOrigin());
}
@Test
void testSaveLogData() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
// Mock the write result
@SuppressWarnings("unchecked")
Result<WriteOk, Err> mockResult = mock(Result.class);
when(mockResult.isOk()).thenReturn(true);
CompletableFuture<Result<WriteOk, Err>> mockFuture = CompletableFuture.completedFuture(mockResult);
when(greptimeDb.write(any(Table.class))).thenReturn(mockFuture);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
LogEntry logEntry = createMockLogEntry();
greptimeDbDataStorage.saveLogData(logEntry);
verify(greptimeDb, times(1)).write(any(Table.class));
}
}
@Test
void testQueryAndCountLogs() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
List<Map<String, Object>> mockLogRows = createMockLogRows();
when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(mockLogRows);
// Test basic query
List<LogEntry> result = greptimeDbDataStorage.queryLogsByMultipleConditions(
System.currentTimeMillis() - 3600000, System.currentTimeMillis(), "trace123", "span456", 1, "INFO", "content"
);
assertNotNull(result);
assertEquals(1, result.size());
assertEquals("trace123", result.get(0).getTraceId());
// Test count query
List<Map<String, Object>> mockCountResult = List.of(Map.of("count", 5L));
when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(mockCountResult);
long count = greptimeDbDataStorage.countLogsByMultipleConditions(
System.currentTimeMillis() - 3600000, System.currentTimeMillis(), null, null, null, null, null
);
assertEquals(5L, count);
// Test count query with executor error
when(greptimeSqlQueryExecutor.execute(anyString())).thenThrow(new RuntimeException("Database error"));
long errorCount = greptimeDbDataStorage.countLogsByMultipleConditions(System.currentTimeMillis() - 3600000, System.currentTimeMillis(), null, null, null, null, null);
assertEquals(0L, errorCount);
}
}
@Test
void testQueryLogsWithPagination() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(createMockLogRows());
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
greptimeDbDataStorage.queryLogsByMultipleConditionsWithPagination(
System.currentTimeMillis() - 3600000, System.currentTimeMillis(),
null, null, null, null, null, 1, 10
);
verify(greptimeSqlQueryExecutor).execute(sqlCaptor.capture());
String capturedSql = sqlCaptor.getValue();
assertTrue(capturedSql.toLowerCase().contains("limit 10"));
assertTrue(capturedSql.toLowerCase().contains("offset 1"));
}
}
@Test
void testBatchDeleteLogsWithValidList() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
// Test with valid list
boolean result = greptimeDbDataStorage.batchDeleteLogs(List.of(1L, 2L));
assertTrue(result);
verify(greptimeSqlQueryExecutor, times(1)).execute(anyString());
}
}
@Test
void testBatchDeleteLogsWithEmptyList() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
// Test with empty list
boolean emptyResult = greptimeDbDataStorage.batchDeleteLogs(Collections.emptyList());
assertFalse(emptyResult);
verify(greptimeSqlQueryExecutor, never()).execute(anyString());
}
}
@Test
void testBatchDeleteLogsWithNullList() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
// Test with null list
boolean nullResult = greptimeDbDataStorage.batchDeleteLogs(null);
assertFalse(nullResult);
verify(greptimeSqlQueryExecutor, never()).execute(anyString());
}
}
@Test
void testDestroy() {
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
greptimeDbDataStorage.destroy();
verify(greptimeDb, times(1)).shutdownGracefully();
}
}
private CollectRep.MetricsData createMockMetricsData(boolean hasValues) {
CollectRep.MetricsData mockMetricsData = mock(CollectRep.MetricsData.class);
lenient().when(mockMetricsData.getCode()).thenReturn(CollectRep.Code.SUCCESS);
lenient().when(mockMetricsData.getMetrics()).thenReturn("cpu");
lenient().when(mockMetricsData.getId()).thenReturn(1L);
if (!hasValues) {
lenient().when(mockMetricsData.getValues()).thenReturn(Collections.emptyList());
return mockMetricsData;
}
// Only create detailed mocks when hasValues = true
// Mock fields
CollectRep.Field mockField1 = mock(CollectRep.Field.class);
lenient().when(mockField1.getName()).thenReturn("usage");
lenient().when(mockField1.getLabel()).thenReturn(false);
lenient().when(mockField1.getType()).thenReturn((int) CommonConstants.TYPE_NUMBER);
CollectRep.Field mockField2 = mock(CollectRep.Field.class);
lenient().when(mockField2.getName()).thenReturn("instance");
lenient().when(mockField2.getLabel()).thenReturn(true);
lenient().when(mockField2.getType()).thenReturn((int) CommonConstants.TYPE_STRING);
lenient().when(mockMetricsData.getFields()).thenReturn(List.of(mockField1, mockField2));
// Create ValueRow mock
CollectRep.ValueRow mockValueRow = mock(CollectRep.ValueRow.class);
lenient().when(mockValueRow.getColumnsList()).thenReturn(List.of("server1", "85.5"));
lenient().when(mockMetricsData.getValues()).thenReturn(List.of(mockValueRow));
// Mock RowWrapper for readRow()
RowWrapper mockRowWrapper = mock(RowWrapper.class);
lenient().when(mockRowWrapper.hasNextRow()).thenReturn(true, false);
lenient().when(mockRowWrapper.nextRow()).thenReturn(mockRowWrapper);
// Mock cell stream
ArrowCell mockCell1 = mock(ArrowCell.class);
lenient().when(mockCell1.getValue()).thenReturn("85.5");
lenient().when(mockCell1.getMetadataAsBoolean(any())).thenReturn(false);
lenient().when(mockCell1.getMetadataAsByte(any())).thenReturn(CommonConstants.TYPE_NUMBER);
ArrowCell mockCell2 = mock(ArrowCell.class);
lenient().when(mockCell2.getValue()).thenReturn("server1");
lenient().when(mockCell2.getMetadataAsBoolean(any())).thenReturn(true);
lenient().when(mockCell2.getMetadataAsByte(any())).thenReturn(CommonConstants.TYPE_STRING);
lenient().when(mockRowWrapper.cellStream()).thenReturn(java.util.stream.Stream.of(mockCell1, mockCell2));
lenient().when(mockMetricsData.readRow()).thenReturn(mockRowWrapper);
return mockMetricsData;
}
private PromQlQueryContent createMockPromQlQueryContent() {
PromQlQueryContent content = new PromQlQueryContent();
PromQlQueryContent.ContentData data = new PromQlQueryContent.ContentData();
PromQlQueryContent.ContentData.Content result = new PromQlQueryContent.ContentData.Content();
Map<String, String> metric = new HashMap<>();
metric.put("__name__", "cpu");
metric.put("instance", "1");
result.setMetric(metric);
List<Object[]> values = new ArrayList<>();
values.add(new Object[]{System.currentTimeMillis() / 1000.0, "85.5"});
result.setValues(values);
data.setResult(List.of(result));
content.setData(data);
return content;
}
private LogEntry createMockLogEntry() {
return LogEntry.builder()
.timeUnixNano(System.nanoTime())
.severityText("INFO")
.body("Test log message")
.traceId("trace123")
.spanId("span456")
.build();
}
private List<Map<String, Object>> createMockLogRows() {
Map<String, Object> row = new HashMap<>();
row.put("time_unix_nano", System.nanoTime());
row.put("severity_text", "INFO");
row.put("body", "\"Test log message\"");
row.put("trace_id", "trace123");
row.put("span_id", "span456");
return List.of(row);
}
private List<RowData.ColumnSchema> getColumnSchemas(Table table) throws Exception {
Field columnSchemasField = table.getClass().getDeclaredField("columnSchemas");
columnSchemasField.setAccessible(true);
@SuppressWarnings("unchecked")
List<RowData.ColumnSchema> columnSchemas = (List<RowData.ColumnSchema>) columnSchemasField.get(table);
return columnSchemas;
}
private List<RowData.Row> getRows(Table table) throws Exception {
Field rowsField = table.getClass().getDeclaredField("rows");
rowsField.setAccessible(true);
@SuppressWarnings("unchecked")
List<RowData.Row> rows = (List<RowData.Row>) rowsField.get(table);
return rows;
}
}
@@ -0,0 +1,245 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.startsWith;
import static org.mockito.Mockito.when;
import static org.assertj.core.api.Assertions.assertThat;
import org.apache.arrow.vector.types.pojo.ArrowType;
import org.apache.arrow.vector.types.pojo.Field;
import org.apache.arrow.vector.types.pojo.FieldType;
import org.apache.hertzbeat.common.constants.CommonConstants;
import org.apache.hertzbeat.common.constants.MetricDataConstants;
import org.apache.hertzbeat.common.entity.arrow.ArrowCell;
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
import org.apache.hertzbeat.common.entity.message.CollectRep;
import org.awaitility.Awaitility;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
import org.mockito.junit.jupiter.MockitoSettings;
import org.mockito.quality.Strictness;
import org.springframework.http.HttpEntity;
import org.springframework.http.HttpMethod;
import org.springframework.http.HttpStatus;
import org.springframework.http.ResponseEntity;
import org.springframework.web.client.RestTemplate;
import java.util.List;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
/**
* Test case for {@link VictoriaMetricsDataStorage}
*/
@ExtendWith(MockitoExtension.class)
@MockitoSettings(strictness = Strictness.LENIENT)
class VictoriaMetricsDataStorageTest {
@Mock
private VictoriaMetricsProperties victoriaMetricsProperties;
@Mock
private RestTemplate restTemplate;
@Mock
private ResponseEntity<String> responseEntity;
private VictoriaMetricsDataStorage victoriaMetricsDataStorage;
private final AtomicInteger postForEntityCount = new AtomicInteger(0);
@BeforeEach
void setUp() {
when(victoriaMetricsProperties.enabled()).thenReturn(true);
when(victoriaMetricsProperties.url()).thenReturn("http://localhost:8428");
when(victoriaMetricsProperties.username()).thenReturn("root");
when(victoriaMetricsProperties.password()).thenReturn("root");
// on successful write, VictoriaMetrics returns HTTP 204 (No Content)
when(responseEntity.getStatusCode()).thenReturn(HttpStatus.NO_CONTENT);
when(restTemplate.exchange(
anyString(),
eq(HttpMethod.GET),
any(HttpEntity.class),
eq(String.class)
)).thenReturn(responseEntity);
when(restTemplate.postForEntity(
startsWith(victoriaMetricsProperties.url()),
any(HttpEntity.class),
eq(String.class)
)).thenAnswer(invocation -> {
postForEntityCount.incrementAndGet();
return responseEntity;
});
}
@Test
void testSaveDataInitialization() {
// using the default insertion policy, schedule a task once upon object instantiation (with second-level time granularity)
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
// execute one-time data insertion
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
// wait for the timer's first insertion task execution and verify if it was called once (default 3 seconds)
Awaitility.await()
.pollInterval(2, TimeUnit.SECONDS)
.atMost(7, TimeUnit.SECONDS)
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
}
@Test
void testSaveDataBySize() {
// verify insert process for buffer size, with the flush interval defined as an unreachable state
when(victoriaMetricsProperties.insert()).thenReturn(new VictoriaMetricsProperties.InsertConfig(
10, Integer.MAX_VALUE, new VictoriaMetricsProperties.Compression(false)));
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
// triggers the buffer size insertion condition
for (int i = 0; i < 10 * 0.8; i++) {
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
}
// wait for the timer to execute the task
Awaitility.await()
.pollInterval(1, TimeUnit.SECONDS)
.atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
}
@Test
void testSaveDataByTime() {
// verify insert process for flush interval and set the buffer size to an unreachable state
when(victoriaMetricsProperties.insert()).thenReturn(new VictoriaMetricsProperties.InsertConfig(
10000, 2, new VictoriaMetricsProperties.Compression(false)));
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
// wait for the timer to execute its first insertion task
Awaitility.await()
.pollInterval(500, TimeUnit.MILLISECONDS)
.atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
// wait for the flush interval to be triggered again
Awaitility.await()
.pollInterval(500, TimeUnit.MILLISECONDS)
.atMost(5, TimeUnit.SECONDS)
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(2));
}
@Test
void testMultiThreadSaveDataBySize() {
int threadCount = 100;
int bufferSize = 10;
int writeSize = (int) (bufferSize * 0.8);
// verify insert process for buffer size, with the flush interval defined as an unreachable state
when(victoriaMetricsProperties.insert()).thenReturn(new VictoriaMetricsProperties.InsertConfig(
bufferSize, Integer.MAX_VALUE, new VictoriaMetricsProperties.Compression(false)));
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
for (int i = 0; i < threadCount; i++) {
new Thread(() -> {
// triggers the buffer size insertion condition
for (int j = 0; j < writeSize; j++) {
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
}
}).start();
}
// wait for the timer to execute the task
Awaitility.await()
.pollInterval(3, TimeUnit.SECONDS)
.atMost(15, TimeUnit.SECONDS)
.untilAsserted(() ->
assertThat(postForEntityCount.get())
// minimum flushes: ensure all data is processed (threadCount * writeSize / bufferSize)
.isGreaterThanOrEqualTo(threadCount * writeSize / bufferSize));
}
@AfterEach
void stop() {
if (victoriaMetricsDataStorage != null) {
victoriaMetricsDataStorage.destroy();
}
}
// for historical data insertion (which mandates preparatory steps), instantiate a mock MetricsData object
public static CollectRep.MetricsData generateMockedMetricsData() {
CollectRep.MetricsData mockMetricsData = Mockito.mock(CollectRep.MetricsData.class);
when(mockMetricsData.getId()).thenReturn(0L);
when(mockMetricsData.getMetrics()).thenReturn("cpu");
when(mockMetricsData.getTime()).thenReturn(System.currentTimeMillis());
when(mockMetricsData.getCode()).thenReturn(CollectRep.Code.SUCCESS);
when(mockMetricsData.getApp()).thenReturn("app");
CollectRep.ValueRow mockValueRow = Mockito.mock(CollectRep.ValueRow.class);
List<String> columnValues = List.of("server-test-01", "68.7");
when(mockValueRow.getColumnsList()).thenReturn(columnValues);
when(mockValueRow.getColumns(0)).thenReturn("server-test-01");
when(mockValueRow.getColumns(1)).thenReturn("68.7");
List<CollectRep.ValueRow> mockValueRowsList = List.of(mockValueRow);
when(mockMetricsData.getValues()).thenReturn(mockValueRowsList);
CollectRep.Field instanceField = Mockito.mock(CollectRep.Field.class);
when(instanceField.getName()).thenReturn("instance");
CollectRep.Field usageField = Mockito.mock(CollectRep.Field.class);
when(usageField.getName()).thenReturn("usage");
CollectRep.Field systemField = Mockito.mock(CollectRep.Field.class);
when(systemField.getName()).thenReturn("system");
List<CollectRep.Field> mockFields = List.of(instanceField, usageField, systemField);
when(mockMetricsData.getFields()).thenReturn(mockFields);
ArrowType instanceArrowType = new ArrowType.Utf8();
FieldType instanceFieldType = new FieldType(true, instanceArrowType, null, null);
Field instanceRealField = new Field("instance", instanceFieldType, null);
ArrowCell instanceCell = Mockito.mock(ArrowCell.class);
when(instanceCell.getField()).thenReturn(instanceRealField);
when(instanceCell.getValue()).thenReturn("server-test-01");
when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true);
when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);
ArrowCell usageCell = Mockito.mock(ArrowCell.class);
when(usageCell.getField()).thenReturn(instanceRealField);
when(usageCell.getValue()).thenReturn("68.7");
when(usageCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(false);
when(usageCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_NUMBER);
List<ArrowCell> mockCells = List.of(instanceCell, usageCell);
RowWrapper mockRowWrapper = Mockito.mock(RowWrapper.class);
when(mockRowWrapper.hasNextRow()).thenReturn(true).thenReturn(false);
when(mockRowWrapper.nextRow()).thenReturn(mockRowWrapper);
when(mockRowWrapper.cellStream()).thenAnswer(invocation -> mockCells.stream());
when(mockMetricsData.readRow()).thenReturn(mockRowWrapper);
return mockMetricsData;
}
}
@@ -0,0 +1,77 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
import static org.assertj.core.api.Assertions.assertThat;
import java.util.Map;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.boot.context.properties.bind.Binder;
import org.springframework.boot.context.properties.source.MapConfigurationPropertySource;
import org.springframework.boot.test.context.SpringBootTest;
/**
* Test case for {@link VictoriaMetricsProperties}
*/
@EnableConfigurationProperties(value = VictoriaMetricsProperties.class)
@SpringBootTest(
classes = VictoriaMetricsPropertiesTest.class,
properties = {
"warehouse.store.victoria-metrics.enabled=true",
"warehouse.store.victoria-metrics.username=test_user",
"warehouse.store.victoria-metrics.insert.buffer-size=999",
"warehouse.store.victoria-metrics.insert.compression.enabled=true"
}
)
class VictoriaMetricsPropertiesTest {
@Autowired
private VictoriaMetricsProperties properties;
@Test
void propertiesShouldBeInjectedAndCorrectlyBound() {
assertThat(properties).isNotNull();
assertThat(properties.enabled()).isTrue();
// url not configured, default value injected
assertThat(properties.url()).isEqualTo("http://localhost:8428");
assertThat(properties.username()).isEqualTo("test_user");
// password not configured, null injected
assertThat(properties.password()).isNull();
assertThat(properties.insert()).isNotNull();
assertThat(properties.insert().bufferSize()).isEqualTo(999);
// flush-interval not configured, default value is 3
assertThat(properties.insert().flushInterval()).isEqualTo(3);
assertThat(properties.insert().compression()).isNotNull();
assertThat(properties.insert().compression().enabled()).isTrue();
}
@Test
void compressionShouldDefaultToDisabledWhenNotConfigured() {
var source = new MapConfigurationPropertySource(Map.of(
"warehouse.store.victoria-metrics.insert.buffer-size", "999"));
var defaultProperties = new Binder(source)
.bind("warehouse.store.victoria-metrics", VictoriaMetricsProperties.class)
.get();
assertThat(defaultProperties.insert().compression()).isNotNull();
assertThat(defaultProperties.insert().compression().enabled()).isFalse();
}
}