chore: import upstream snapshot with attribution
This commit is contained in:
+122
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse;
|
||||
|
||||
import com.google.common.util.concurrent.ThreadFactoryBuilder;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.SynchronousQueue;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.ThreadPoolExecutor;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.concurrent.ManagedExecutor;
|
||||
import org.apache.hertzbeat.common.concurrent.ManagedExecutors;
|
||||
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* warehouse worker thread pool
|
||||
*/
|
||||
@Component
|
||||
@Slf4j
|
||||
public class WarehouseWorkerPool implements DisposableBean {
|
||||
|
||||
private final ManagedExecutor workerExecutor;
|
||||
|
||||
private final ManagedExecutor longRunningExecutor;
|
||||
|
||||
public WarehouseWorkerPool() {
|
||||
this(VirtualThreadProperties.defaults());
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public WarehouseWorkerPool(VirtualThreadProperties virtualThreadProperties) {
|
||||
VirtualThreadProperties properties =
|
||||
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
this.workerExecutor = createWorkerExecutor(properties);
|
||||
this.longRunningExecutor = createLongRunningExecutor(properties, workerExecutor);
|
||||
}
|
||||
|
||||
private ManagedExecutor createWorkerExecutor(VirtualThreadProperties properties) {
|
||||
Thread.UncaughtExceptionHandler handler = (thread, throwable) -> {
|
||||
log.error("Warehouse workerExecutor has uncaughtException.");
|
||||
log.error(throwable.getMessage(), throwable);
|
||||
};
|
||||
if (properties.enabled()) {
|
||||
VirtualThreadProperties.PoolProperties poolProperties = properties.warehouse();
|
||||
return ManagedExecutors.newVirtualExecutor("warehouse-worker", "warehouse-worker-",
|
||||
poolProperties.mode(), poolProperties.maxConcurrentJobs(), handler);
|
||||
}
|
||||
return ManagedExecutors.wrap("warehouse-worker", createLegacyExecutor(handler));
|
||||
}
|
||||
|
||||
private ManagedExecutor createLongRunningExecutor(VirtualThreadProperties properties, ManagedExecutor fallback) {
|
||||
if (!properties.enabled()) {
|
||||
return fallback;
|
||||
}
|
||||
return ManagedExecutors.newPlatformExecutor("warehouse-long-running", "warehouse-long-running-",
|
||||
(thread, throwable) -> {
|
||||
log.error("Warehouse longRunningExecutor has uncaughtException.");
|
||||
log.error(throwable.getMessage(), throwable);
|
||||
});
|
||||
}
|
||||
|
||||
private ExecutorService createLegacyExecutor(Thread.UncaughtExceptionHandler handler) {
|
||||
ThreadFactory threadFactory = new ThreadFactoryBuilder()
|
||||
.setUncaughtExceptionHandler(handler)
|
||||
.setDaemon(true)
|
||||
.setNameFormat("warehouse-worker-%d")
|
||||
.build();
|
||||
return new ThreadPoolExecutor(2,
|
||||
Integer.MAX_VALUE,
|
||||
10,
|
||||
TimeUnit.SECONDS,
|
||||
new SynchronousQueue<>(),
|
||||
threadFactory,
|
||||
new ThreadPoolExecutor.AbortPolicy());
|
||||
}
|
||||
|
||||
/**
|
||||
* Run warehouse task
|
||||
* @param runnable task
|
||||
* @throws RejectedExecutionException when THREAD POOL FULL
|
||||
*/
|
||||
public void executeJob(Runnable runnable) throws RejectedExecutionException {
|
||||
workerExecutor.execute(runnable);
|
||||
}
|
||||
|
||||
/**
|
||||
* Run a long-lived warehouse consumer outside of the per-task executor.
|
||||
*
|
||||
* @param runnable consumer task
|
||||
*/
|
||||
public void executeLongRunning(Runnable runnable) {
|
||||
longRunningExecutor.execute(runnable);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
workerExecutor.close();
|
||||
if (longRunningExecutor != workerExecutor) {
|
||||
longRunningExecutor.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.config;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.springframework.boot.autoconfigure.AutoConfiguration;
|
||||
import org.springframework.context.annotation.ComponentScan;
|
||||
|
||||
/**
|
||||
* WarehouseAutoConfiguration class
|
||||
* @version 2.1
|
||||
*/
|
||||
|
||||
@AutoConfiguration
|
||||
@ComponentScan(basePackages = ConfigConstants.PkgConstant.PKG
|
||||
+ SignConstants.DOT
|
||||
+ ConfigConstants.FunctionModuleConstants.WAREHOUSE)
|
||||
public class WarehouseAutoConfiguration {
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.constants;
|
||||
|
||||
/**
|
||||
* Warehouse configuration constants.
|
||||
*/
|
||||
|
||||
public interface WarehouseConstants {
|
||||
|
||||
String STORE = "store";
|
||||
|
||||
String REAL_TIME = "real-time";
|
||||
|
||||
/**
|
||||
* History database name.
|
||||
*/
|
||||
interface HistoryName {
|
||||
String GREPTIME = "greptime";
|
||||
|
||||
String INFLUXDB = "influxdb";
|
||||
|
||||
String IOT_DB = "iot-db";
|
||||
|
||||
String JPA = "jpa";
|
||||
|
||||
String TD_ENGINE = "td-engine";
|
||||
|
||||
String VM = "victoria-metrics";
|
||||
|
||||
String VM_CLUSTER = "victoria-metrics.cluster";
|
||||
|
||||
String QUEST_DB = "questdb";
|
||||
|
||||
String DUCKDB = "duckdb";
|
||||
|
||||
String DORIS = "doris";
|
||||
}
|
||||
|
||||
/**
|
||||
* Real-time database name.
|
||||
*/
|
||||
interface RealTimeName {
|
||||
|
||||
String REDIS = "redis";
|
||||
|
||||
String MEMORY = "memory";
|
||||
}
|
||||
|
||||
String PROMQL = "promql";
|
||||
|
||||
String SQL = "sql";
|
||||
|
||||
String RANGE = "range";
|
||||
|
||||
String INSTANT = "instant";
|
||||
|
||||
String LOG_TABLE_NAME = "hertzbeat_logs";
|
||||
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.controller;
|
||||
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.common.entity.dto.Message;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
|
||||
import org.apache.hertzbeat.warehouse.service.DatasourceQueryService;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.PostMapping;
|
||||
import org.springframework.web.bind.annotation.RequestBody;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* Metrics Data Query API
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping(produces = {APPLICATION_JSON_VALUE})
|
||||
@Tag(name = "Metrics Data Query API")
|
||||
public class DataQueryController {
|
||||
|
||||
@Autowired(required = false)
|
||||
private DatasourceQueryService datasourceQueryService;
|
||||
|
||||
@PostMapping("/api/warehouse/query")
|
||||
@Operation(summary = "Warehouse Query")
|
||||
public ResponseEntity<Message<List<DatasourceQueryData>>> query(
|
||||
@Parameter(description = "Query Expr") @RequestBody List<DatasourceQuery> queries) {
|
||||
return ResponseEntity.ok(Message.success(datasourceQueryService.query(queries)));
|
||||
}
|
||||
}
|
||||
+103
@@ -0,0 +1,103 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.controller;
|
||||
|
||||
import static org.apache.hertzbeat.common.constants.CommonConstants.FAIL_CODE;
|
||||
import static org.springframework.http.MediaType.APPLICATION_JSON_VALUE;
|
||||
import io.swagger.v3.oas.annotations.Operation;
|
||||
import io.swagger.v3.oas.annotations.Parameter;
|
||||
import io.swagger.v3.oas.annotations.tags.Tag;
|
||||
import org.apache.hertzbeat.common.entity.dto.Message;
|
||||
import org.apache.hertzbeat.common.entity.dto.MetricsData;
|
||||
import org.apache.hertzbeat.common.entity.dto.MetricsHistoryData;
|
||||
import org.apache.hertzbeat.warehouse.service.MetricsDataService;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.bind.annotation.GetMapping;
|
||||
import org.springframework.web.bind.annotation.PathVariable;
|
||||
import org.springframework.web.bind.annotation.RequestMapping;
|
||||
import org.springframework.web.bind.annotation.RequestParam;
|
||||
import org.springframework.web.bind.annotation.RestController;
|
||||
|
||||
/**
|
||||
* Indicator data query interface
|
||||
*/
|
||||
@RestController
|
||||
@RequestMapping(produces = {APPLICATION_JSON_VALUE})
|
||||
@Tag(name = "Metrics Data API | The API for monitoring metric data")
|
||||
public class MetricsDataController {
|
||||
|
||||
private static final Integer METRIC_FULL_LENGTH = 3;
|
||||
|
||||
private final MetricsDataService metricsDataService;
|
||||
|
||||
public MetricsDataController(MetricsDataService metricsDataService) {
|
||||
this.metricsDataService = metricsDataService;
|
||||
}
|
||||
|
||||
@GetMapping("/api/warehouse/storage/status")
|
||||
@Operation(summary = "Query Warehouse Storage Server Status", description = "Query the availability status of the storage service under the warehouse")
|
||||
public ResponseEntity<Message<Void>> getWarehouseStorageServerStatus() {
|
||||
Boolean status = metricsDataService.getWarehouseStorageServerStatus();
|
||||
if (Boolean.TRUE.equals(status)) {
|
||||
return ResponseEntity.ok(Message.success());
|
||||
}
|
||||
|
||||
// historyDataReader does not exist or is not available
|
||||
return ResponseEntity.ok(Message.fail(FAIL_CODE, "Service not available!"));
|
||||
}
|
||||
|
||||
@GetMapping("/api/monitor/{monitorId}/metrics/{metrics}")
|
||||
@Operation(summary = "Query Real Time Metrics Data", description = "Query real-time metrics data of monitoring indicators")
|
||||
public ResponseEntity<Message<MetricsData>> getMetricsData(
|
||||
@Parameter(description = "Monitor Id", example = "343254354")
|
||||
@PathVariable Long monitorId,
|
||||
@Parameter(description = "Metrics Name", example = "cpu")
|
||||
@PathVariable String metrics) {
|
||||
MetricsData metricsData = metricsDataService.getMetricsData(monitorId, metrics);
|
||||
if (metricsData == null){
|
||||
return ResponseEntity.ok(Message.success("query metrics data is empty"));
|
||||
}
|
||||
return ResponseEntity.ok(Message.success(metricsData));
|
||||
}
|
||||
|
||||
@GetMapping("/api/monitor/{instance}/metric/{metricFull}")
|
||||
@Operation(summary = "Queries historical data for a specified metric for monitoring", description = "Queries historical data for a specified metric under monitoring")
|
||||
public ResponseEntity<Message<MetricsHistoryData>> getMetricHistoryData(
|
||||
@Parameter(description = "monitor instance", example = "127.0.0.1:8080")
|
||||
@PathVariable String instance,
|
||||
@Parameter(description = "monitor metric full path", example = "linux.cpu.usage")
|
||||
@PathVariable() String metricFull,
|
||||
@Parameter(description = "query historical time period, default 6h-6 hours: s-seconds, M-minutes, h-hours, d-days, w-weeks", example = "6h")
|
||||
@RequestParam(required = false) String history,
|
||||
@Parameter(description = "aggregate data calc. off by default; 4-hour window, query limit >1 week", example = "false")
|
||||
@RequestParam(required = false) Boolean interval
|
||||
) {
|
||||
if (!metricsDataService.getWarehouseStorageServerStatus()) {
|
||||
return ResponseEntity.ok(Message.fail(FAIL_CODE, "time series database not available"));
|
||||
}
|
||||
String[] names = metricFull.split("\\.");
|
||||
if (names.length != METRIC_FULL_LENGTH) {
|
||||
throw new IllegalArgumentException("metrics full name: " + metricFull + " is illegal.");
|
||||
}
|
||||
String app = names[0];
|
||||
String metrics = names[1];
|
||||
String metric = names[2];
|
||||
MetricsHistoryData historyData = metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, interval);
|
||||
return ResponseEntity.ok(Message.success(historyData));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,59 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.dao;
|
||||
|
||||
import org.apache.hertzbeat.common.entity.warehouse.History;
|
||||
import org.springframework.data.jpa.repository.JpaRepository;
|
||||
import org.springframework.data.jpa.repository.JpaSpecificationExecutor;
|
||||
import org.springframework.data.jpa.repository.Modifying;
|
||||
import org.springframework.data.jpa.repository.Query;
|
||||
import org.springframework.data.repository.query.Param;
|
||||
import org.springframework.transaction.annotation.Transactional;
|
||||
|
||||
/**
|
||||
* history entity dao
|
||||
*/
|
||||
public interface HistoryDao extends JpaRepository<History, Long>, JpaSpecificationExecutor<History> {
|
||||
|
||||
/**
|
||||
* delete history before expireTime
|
||||
* @param expireTime expireTime
|
||||
* @return rows deleted
|
||||
*/
|
||||
@Modifying
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
int deleteHistoriesByTimeBefore(Long expireTime);
|
||||
|
||||
/**
|
||||
* delete older history record
|
||||
* @param delNum number to be deleted
|
||||
* @return rows deleted
|
||||
*/
|
||||
@Modifying
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
@Query(value = "DELETE FROM hzb_history WHERE id IN ( SELECT t2.id from (SELECT t1.id FROM hzb_history t1 LIMIT ?1) as t2)", nativeQuery = true)
|
||||
int deleteOlderHistoriesRecord(@Param(value = "delNum") int delNum);
|
||||
|
||||
/**
|
||||
* truncateTable
|
||||
*/
|
||||
@Modifying
|
||||
@Transactional(rollbackFor = Exception.class)
|
||||
@Query(value = "TRUNCATE TABLE hzb_history", nativeQuery = true)
|
||||
void truncateTable();
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
|
||||
/**
|
||||
* query executor for victor metrics
|
||||
*/
|
||||
@Component("greptimePromqlQueryExecutor")
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class GreptimePromqlQueryExecutor extends PromqlQueryExecutor {
|
||||
|
||||
private static final String QUERY_PATH = "/v1/prometheus";
|
||||
|
||||
private static final String Datasource = "Greptime-promql";
|
||||
|
||||
private final GreptimeProperties greptimeProperties;
|
||||
|
||||
public GreptimePromqlQueryExecutor(GreptimeProperties greptimeProperties, RestTemplate restTemplate) {
|
||||
super(restTemplate, new HttpPromqlProperties(greptimeProperties.httpEndpoint() + QUERY_PATH,
|
||||
greptimeProperties.username(), greptimeProperties.password()));
|
||||
this.greptimeProperties = greptimeProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDatasource() {
|
||||
return Datasource;
|
||||
}
|
||||
|
||||
}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.constants.NetworkConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.common.util.Base64Util;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeSqlQueryContent;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* query executor for GreptimeDB SQL
|
||||
*/
|
||||
@Slf4j
|
||||
@Component("greptimeSqlQueryExecutor")
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
|
||||
public class GreptimeSqlQueryExecutor extends SqlQueryExecutor {
|
||||
|
||||
private static final String QUERY_PATH = "/v1/sql";
|
||||
private static final String DATASOURCE = "Greptime-sql";
|
||||
|
||||
private final GreptimeProperties greptimeProperties;
|
||||
|
||||
|
||||
public GreptimeSqlQueryExecutor(GreptimeProperties greptimeProperties, RestTemplate restTemplate) {
|
||||
super(restTemplate, new SqlQueryExecutor.HttpSqlProperties(greptimeProperties.httpEndpoint() + QUERY_PATH,
|
||||
greptimeProperties.username(), greptimeProperties.password()));
|
||||
this.greptimeProperties = greptimeProperties;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> execute(String queryString) {
|
||||
List<Map<String, Object>> results = new LinkedList<>();
|
||||
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_FORM_URLENCODED);
|
||||
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||
if (StringUtils.hasText(greptimeProperties.username())
|
||||
&& StringUtils.hasText(greptimeProperties.password())) {
|
||||
String authStr = greptimeProperties.username() + ":" + greptimeProperties.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
|
||||
String requestBody = "sql=" + queryString;
|
||||
HttpEntity<String> httpEntity = new HttpEntity<>(requestBody, headers);
|
||||
|
||||
String url = greptimeProperties.httpEndpoint() + QUERY_PATH;
|
||||
if (StringUtils.hasText(greptimeProperties.database())) {
|
||||
url += "?db=" + greptimeProperties.database();
|
||||
}
|
||||
|
||||
ResponseEntity<GreptimeSqlQueryContent> responseEntity;
|
||||
try {
|
||||
responseEntity = restTemplate.exchange(url,
|
||||
HttpMethod.POST, httpEntity, GreptimeSqlQueryContent.class);
|
||||
} catch (Exception e) {
|
||||
log.error("Exception occurred while querying GreptimeDB SQL: {}", e.getMessage(), e);
|
||||
throw new RuntimeException("Failed to execute GreptimeDB SQL query", e);
|
||||
}
|
||||
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
GreptimeSqlQueryContent responseBody = responseEntity.getBody();
|
||||
// GreptimeDB SQL HTTP API may not return 'code' field in successful response
|
||||
// Check if output exists and is not empty
|
||||
if (responseBody != null && responseBody.getOutput() != null && !responseBody.getOutput().isEmpty()) {
|
||||
|
||||
for (GreptimeSqlQueryContent.Output output : responseBody.getOutput()) {
|
||||
if (output.getRecords() != null && output.getRecords().getRows() != null) {
|
||||
GreptimeSqlQueryContent.Output.Records.Schema schema = output.getRecords().getSchema();
|
||||
List<List<Object>> rows = output.getRecords().getRows();
|
||||
|
||||
for (List<Object> row : rows) {
|
||||
Map<String, Object> rowMap = new HashMap<>();
|
||||
if (schema != null && schema.getColumnSchemas() != null) {
|
||||
for (int i = 0; i < Math.min(schema.getColumnSchemas().size(), row.size()); i++) {
|
||||
String columnName = schema.getColumnSchemas().get(i).getName();
|
||||
Object value = row.get(i);
|
||||
rowMap.put(columnName, value);
|
||||
}
|
||||
} else {
|
||||
for (int i = 0; i < row.size(); i++) {
|
||||
rowMap.put("col_" + i, row.get(i));
|
||||
}
|
||||
}
|
||||
results.add(rowMap);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error("query metrics data from greptime failed. {}", responseEntity);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDatasource() {
|
||||
return DATASOURCE;
|
||||
}
|
||||
}
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.hertzbeat.common.constants.NetworkConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
|
||||
import org.apache.hertzbeat.common.util.Base64Util;
|
||||
import org.apache.hertzbeat.common.util.TimePeriodUtil;
|
||||
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.INSTANT;
|
||||
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.PROMQL;
|
||||
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.RANGE;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.PromQlQueryContent;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import org.springframework.web.util.UriComponentsBuilder;
|
||||
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* abstract class for promql query executor
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class PromqlQueryExecutor implements QueryExecutor {
|
||||
|
||||
private static final String supportQueryLanguage = PROMQL;
|
||||
private static final String QUERY_RANGE_PATH = "/api/v1/query_range";
|
||||
private static final String QUERY_PATH = "/api/v1/query";
|
||||
protected static final String HTTP_QUERY_PARAM = "query";
|
||||
protected static final String HTTP_TIME_PARAM = "time";
|
||||
protected static final String HTTP_START_PARAM = "start";
|
||||
protected static final String HTTP_END_PARAM = "end";
|
||||
protected static final String HTTP_STEP_PARAM = "step";
|
||||
private static final String INNER_KEY_TIME = "__ts__";
|
||||
private static final String INNER_KEY_VALUE = "__value__";
|
||||
|
||||
private final RestTemplate restTemplate;
|
||||
|
||||
private final HttpPromqlProperties httpPromqlProperties;
|
||||
|
||||
PromqlQueryExecutor(RestTemplate restTemplate, HttpPromqlProperties httpPromqlProperties) {
|
||||
this.restTemplate = restTemplate;
|
||||
this.httpPromqlProperties = httpPromqlProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* record class for promql http connection
|
||||
*/
|
||||
protected record HttpPromqlProperties(
|
||||
String url,
|
||||
String username,
|
||||
String password
|
||||
) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> execute(String queryString) {
|
||||
List<Map<String, Object>> results = new LinkedList<>();
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||
if (StringUtils.hasText(httpPromqlProperties.username())
|
||||
&& StringUtils.hasText(httpPromqlProperties.password())) {
|
||||
String authStr = httpPromqlProperties.username() + ":" + httpPromqlProperties.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
|
||||
|
||||
UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromUriString(httpPromqlProperties.url + QUERY_PATH);
|
||||
uriComponentsBuilder.queryParam(HTTP_QUERY_PARAM, queryString);
|
||||
URI uri = uriComponentsBuilder.build().toUri();
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri,
|
||||
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
Map<String, Object> queryResult = new HashMap<>(8);
|
||||
queryResult.putAll(labels);
|
||||
if (content.getValue() != null && content.getValue().length == 2) {
|
||||
queryResult.put("__timestamp__", content.getValue()[0]);
|
||||
queryResult.put("__value__", content.getValue()[1]);
|
||||
} else if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Object> values = new LinkedList<>();
|
||||
for (Object[] valueArr : content.getValues()) {
|
||||
values.add(valueArr[1]);
|
||||
}
|
||||
queryResult.put("__value__", values);
|
||||
}
|
||||
results.add(queryResult);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error("query metrics data from greptime failed. {}", responseEntity);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(e.toString(), e);
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatasourceQueryData query(DatasourceQuery datasourceQuery) {
|
||||
DatasourceQueryData.DatasourceQueryDataBuilder queryDataBuilder = DatasourceQueryData.builder()
|
||||
.refId(datasourceQuery.getRefId()).status(200);
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||
if (StringUtils.hasText(httpPromqlProperties.username())
|
||||
&& StringUtils.hasText(httpPromqlProperties.password())) {
|
||||
String authStr = httpPromqlProperties.username() + ":" + httpPromqlProperties.password();
|
||||
String encodedAuth = new String(Base64.encodeBase64(authStr.getBytes(StandardCharsets.UTF_8)), StandardCharsets.UTF_8);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + " " + encodedAuth);
|
||||
}
|
||||
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
|
||||
URI uri;
|
||||
if (datasourceQuery.getTimeType().equals(RANGE)) {
|
||||
uri = UriComponentsBuilder.fromUriString(httpPromqlProperties.url() + QUERY_RANGE_PATH)
|
||||
.queryParam(HTTP_QUERY_PARAM, datasourceQuery.getExpr())
|
||||
.queryParam(HTTP_START_PARAM, datasourceQuery.getStart())
|
||||
.queryParam(HTTP_END_PARAM, datasourceQuery.getEnd())
|
||||
.queryParam(HTTP_STEP_PARAM, datasourceQuery.getStep())
|
||||
.build().toUri();
|
||||
} else if (datasourceQuery.getTimeType().equals(INSTANT)) {
|
||||
uri = UriComponentsBuilder.fromUriString(httpPromqlProperties.url() + QUERY_PATH)
|
||||
.queryParam(HTTP_QUERY_PARAM, datasourceQuery.getExpr())
|
||||
.build().toUri();
|
||||
} else {
|
||||
throw new IllegalArgumentException(String.format("no such time type for query id %s.", datasourceQuery.getRefId()));
|
||||
}
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity,
|
||||
PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.debug("query metrics data from promql http api success. {}", uri);
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
|
||||
List<DatasourceQueryData.SchemaData> schemaDataList = new LinkedList<>();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
DatasourceQueryData.MetricSchema.MetricSchemaBuilder schemaBuilder = DatasourceQueryData.MetricSchema
|
||||
.builder().fields(List.of(
|
||||
// todo: unit?
|
||||
DatasourceQueryData.MetricField.builder().name(INNER_KEY_TIME)
|
||||
.type("time").build(),
|
||||
DatasourceQueryData.MetricField.builder().name(INNER_KEY_VALUE)
|
||||
.type("number").build()
|
||||
)).labels(content.getMetric());
|
||||
List<Object[]> values;
|
||||
if (datasourceQuery.getTimeType().equals(RANGE)) {
|
||||
values = content.getValues();
|
||||
}
|
||||
else {
|
||||
values = List.<Object[]>of(content.getValue());
|
||||
}
|
||||
values.forEach(objects -> {
|
||||
objects[0] = TimePeriodUtil.normalizeToMilliseconds(objects[0]);
|
||||
});
|
||||
DatasourceQueryData.SchemaData.SchemaDataBuilder schemaData = DatasourceQueryData.SchemaData.builder()
|
||||
.schema(schemaBuilder.build()).data(values);
|
||||
schemaDataList.add(schemaData.build());
|
||||
}
|
||||
queryDataBuilder.frames(schemaDataList);
|
||||
}
|
||||
} else {
|
||||
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
|
||||
queryDataBuilder.msg("query metrics data from victoria-metrics failed. ");
|
||||
queryDataBuilder.status(responseEntity.getStatusCode().value());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
|
||||
queryDataBuilder.msg("query metrics data from victoria-metrics error: " + e.getMessage());
|
||||
queryDataBuilder.status(400);
|
||||
}
|
||||
return queryDataBuilder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean support(String queryLanguage) {
|
||||
return StringUtils.hasText(queryLanguage) && queryLanguage.equalsIgnoreCase(supportQueryLanguage);
|
||||
}
|
||||
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* query executor interface
|
||||
*/
|
||||
public interface QueryExecutor {
|
||||
|
||||
List<Map<String, Object>> execute(String query);
|
||||
|
||||
DatasourceQueryData query(DatasourceQuery datasourceQuery);
|
||||
|
||||
String getDatasource();
|
||||
|
||||
boolean support(String queryLanguage);
|
||||
}
|
||||
+73
@@ -0,0 +1,73 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
|
||||
|
||||
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.SQL;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
/**
|
||||
* abstract class for sql query executor
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class SqlQueryExecutor implements QueryExecutor {
|
||||
|
||||
private static final String supportQueryLanguage = SQL;
|
||||
protected final RestTemplate restTemplate;
|
||||
protected final SqlQueryExecutor.HttpSqlProperties httpSqlProperties;
|
||||
|
||||
SqlQueryExecutor(RestTemplate restTemplate, SqlQueryExecutor.HttpSqlProperties httpSqlProperties) {
|
||||
this.restTemplate = restTemplate;
|
||||
this.httpSqlProperties = httpSqlProperties;
|
||||
}
|
||||
|
||||
/**
|
||||
* record class for sql http connection
|
||||
*/
|
||||
protected record HttpSqlProperties(
|
||||
String url,
|
||||
String username,
|
||||
String password
|
||||
) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Map<String, Object>> execute(String query) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public DatasourceQueryData query(DatasourceQuery datasourceQuery) {
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean support(String queryLanguage) {
|
||||
return StringUtils.hasText(queryLanguage) && queryLanguage.equalsIgnoreCase(supportQueryLanguage);
|
||||
}
|
||||
|
||||
}
|
||||
+56
@@ -0,0 +1,56 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsClusterProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsSelectProperties;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
/**
|
||||
* query executor for victoria metrics cluster
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics.cluster", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class VictoriaMetricsClusterQueryExecutor extends PromqlQueryExecutor {
|
||||
|
||||
private static final String QUERY_PATH = "/select/%s/prometheus";
|
||||
private static final String DATASOURCE = "VictoriaMetricsCluster";
|
||||
|
||||
public VictoriaMetricsClusterQueryExecutor(VictoriaMetricsClusterProperties victoriaMetricsClusterProps,
|
||||
RestTemplate restTemplate) {
|
||||
super(restTemplate, buildHttpPromqlProperties(victoriaMetricsClusterProps));
|
||||
}
|
||||
|
||||
private static HttpPromqlProperties buildHttpPromqlProperties(VictoriaMetricsClusterProperties props) {
|
||||
VictoriaMetricsSelectProperties selectProperties = props.select();
|
||||
return new HttpPromqlProperties(
|
||||
selectProperties.url() + QUERY_PATH.formatted(props.accountID()),
|
||||
selectProperties.username(),
|
||||
selectProperties.password()
|
||||
);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDatasource() {
|
||||
return DATASOURCE;
|
||||
}
|
||||
}
|
||||
+49
@@ -0,0 +1,49 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsProperties;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
/**
|
||||
* query executor for victor metrics
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class VictoriaMetricsQueryExecutor extends PromqlQueryExecutor {
|
||||
private static final String Datasource = "VictoriaMetrics";
|
||||
|
||||
private final VictoriaMetricsProperties victoriaMetricsProp;
|
||||
|
||||
public VictoriaMetricsQueryExecutor(VictoriaMetricsProperties victoriaMetricsProp, RestTemplate restTemplate) {
|
||||
super(restTemplate, new HttpPromqlProperties(victoriaMetricsProp.url(),
|
||||
victoriaMetricsProp.username(), victoriaMetricsProp.password()));
|
||||
this.victoriaMetricsProp = victoriaMetricsProp;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String getDatasource() {
|
||||
return Datasource;
|
||||
}
|
||||
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.listener;
|
||||
|
||||
import java.util.Optional;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.springframework.boot.context.event.ApplicationReadyEvent;
|
||||
import org.springframework.context.event.EventListener;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* WareHouseApplicationReadyListener
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class WareHouseApplicationReadyListener {
|
||||
|
||||
private final Optional<AbstractHistoryDataStorage> historyDataStorage;
|
||||
|
||||
public WareHouseApplicationReadyListener(Optional<AbstractHistoryDataStorage> historyDataStorage) {
|
||||
this.historyDataStorage = historyDataStorage;
|
||||
}
|
||||
|
||||
@EventListener(classes = {ApplicationReadyEvent.class})
|
||||
public void listen() {
|
||||
if (historyDataStorage.isEmpty()) {
|
||||
log.warn("The historical data repository is not configured");
|
||||
}
|
||||
}
|
||||
}
|
||||
+35
@@ -0,0 +1,35 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.service;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
|
||||
|
||||
/**
|
||||
* metrics data query service
|
||||
*/
|
||||
public interface DatasourceQueryService {
|
||||
|
||||
/**
|
||||
* Query metrics data
|
||||
* @param queries query expr
|
||||
* @return data
|
||||
*/
|
||||
List<DatasourceQueryData> query(List<DatasourceQuery> queries);
|
||||
}
|
||||
+54
@@ -0,0 +1,54 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.service;
|
||||
|
||||
import org.apache.hertzbeat.common.entity.dto.MetricsData;
|
||||
import org.apache.hertzbeat.common.entity.dto.MetricsHistoryData;
|
||||
|
||||
/**
|
||||
* service for metrics data
|
||||
*/
|
||||
public interface MetricsDataService {
|
||||
|
||||
/**
|
||||
* warehouse storage server availability or not
|
||||
* @return true or false
|
||||
*/
|
||||
Boolean getWarehouseStorageServerStatus();
|
||||
|
||||
/**
|
||||
* Query Real Time Metrics Data
|
||||
* @param monitorId Monitor Id
|
||||
* @param metrics Metrics Name
|
||||
* @return metrics data
|
||||
*/
|
||||
MetricsData getMetricsData(Long monitorId, String metrics);
|
||||
|
||||
/**
|
||||
* Queries historical data for a specified metric for monitoring
|
||||
*
|
||||
* @param instance Instance e.g. ip:port or ip or domain
|
||||
* @param app Monitor Type
|
||||
* @param metrics Metrics Name
|
||||
* @param metric Metrics Field Name
|
||||
* @param history Query Historical Time Period
|
||||
* @param interval aggregate data calc
|
||||
* @return metrics history data
|
||||
*/
|
||||
MetricsHistoryData getMetricHistoryData(String instance, String app, String metrics, String metric, String history, Boolean interval);
|
||||
}
|
||||
+34
@@ -0,0 +1,34 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.service;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
|
||||
/**
|
||||
* service for warehouse
|
||||
*/
|
||||
public interface WarehouseService {
|
||||
|
||||
/**
|
||||
* query monitor real time metrics data by monitor id
|
||||
* @param monitorId monitor id
|
||||
* @return metrics data
|
||||
*/
|
||||
List<CollectRep.MetricsData> queryMonitorMetricsData(Long monitorId);
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.service.impl;
|
||||
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
|
||||
import org.apache.hertzbeat.warehouse.db.QueryExecutor;
|
||||
import org.apache.hertzbeat.warehouse.service.DatasourceQueryService;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
/**
|
||||
* datasource query service impl
|
||||
*/
|
||||
@Service
|
||||
public class DatasourceQueryServiceImpl implements DatasourceQueryService {
|
||||
Map<String, QueryExecutor> executorMap;
|
||||
|
||||
DatasourceQueryServiceImpl(List<QueryExecutor> executors) {
|
||||
executorMap = executors.stream().collect(Collectors.toMap(QueryExecutor::getDatasource, executor -> executor));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<DatasourceQueryData> query(List<DatasourceQuery> queries) {
|
||||
if (queries == null) {
|
||||
throw new IllegalArgumentException("No query found");
|
||||
}
|
||||
List<DatasourceQueryData> datasourceQueryDataList = new ArrayList<>();
|
||||
for (DatasourceQuery datasourceQuery : queries) {
|
||||
QueryExecutor executor = executorMap.get(datasourceQuery.getDatasource());
|
||||
if (executor == null) {
|
||||
throw new IllegalArgumentException("Unsupported datasource: " + datasourceQuery.getDatasource());
|
||||
}
|
||||
datasourceQueryDataList.add(executor.query(datasourceQuery));
|
||||
}
|
||||
|
||||
return datasourceQueryDataList;
|
||||
}
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.service.impl;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Optional;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Field;
|
||||
import org.apache.hertzbeat.common.entity.dto.MetricsData;
|
||||
import org.apache.hertzbeat.common.entity.dto.MetricsHistoryData;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.dto.ValueRow;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.support.exception.CommonException;
|
||||
import org.apache.hertzbeat.warehouse.service.MetricsDataService;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataReader;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* Metrics Data Service impl
|
||||
*/
|
||||
@Slf4j
|
||||
@Service
|
||||
public class MetricsDataServiceImpl implements MetricsDataService {
|
||||
|
||||
private final RealTimeDataReader realTimeDataReader;
|
||||
|
||||
private final Optional<HistoryDataReader> historyDataReader;
|
||||
|
||||
public MetricsDataServiceImpl(RealTimeDataReader realTimeDataReader, Optional<HistoryDataReader> historyDataReader) {
|
||||
this.realTimeDataReader = realTimeDataReader;
|
||||
this.historyDataReader = historyDataReader;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Boolean getWarehouseStorageServerStatus() {
|
||||
return historyDataReader.isPresent() && historyDataReader.get().isServerAvailable();
|
||||
}
|
||||
|
||||
@Override
|
||||
public MetricsData getMetricsData(Long monitorId, String metrics) {
|
||||
boolean available = realTimeDataReader.isServerAvailable();
|
||||
if (!available) {
|
||||
throw new CommonException("real time store not available");
|
||||
}
|
||||
CollectRep.MetricsData storageData = realTimeDataReader.getCurrentMetricsData(monitorId, metrics);
|
||||
if (storageData == null) {
|
||||
return null;
|
||||
}
|
||||
MetricsData.MetricsDataBuilder dataBuilder = MetricsData.builder();
|
||||
dataBuilder.id(storageData.getId()).app(storageData.getApp()).metrics(storageData.getMetrics())
|
||||
.time(storageData.getTime());
|
||||
dataBuilder.fields(storageData.getFields().stream()
|
||||
.map(field -> Field.builder().name(field.getName())
|
||||
.type((byte) field.getType())
|
||||
.label(field.getLabel())
|
||||
.unit(field.getUnit())
|
||||
.build())
|
||||
.toList());
|
||||
|
||||
List<ValueRow> valueRows = new ArrayList<>();
|
||||
if (storageData.rowCount() > 0) {
|
||||
RowWrapper rowWrapper = storageData.readRow();
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
|
||||
List<Value> values = new ArrayList<>();
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
String origin = cell.getValue();
|
||||
|
||||
if (CommonConstants.NULL_VALUE.equals(origin)) {
|
||||
values.add(new Value());
|
||||
} else {
|
||||
values.add(new Value(origin));
|
||||
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL)) {
|
||||
labels.put(cell.getField().getName(), origin);
|
||||
}
|
||||
}
|
||||
});
|
||||
valueRows.add(ValueRow.builder().labels(labels).values(values).build());
|
||||
}
|
||||
dataBuilder.valueRows(valueRows);
|
||||
}
|
||||
return dataBuilder.build();
|
||||
}
|
||||
|
||||
@Override
|
||||
public MetricsHistoryData getMetricHistoryData(String instance, String app, String metrics, String metric, String history, Boolean interval) {
|
||||
if (history == null) {
|
||||
history = "6h";
|
||||
}
|
||||
Map<String, List<Value>> instanceValuesMap;
|
||||
if (interval == null || !interval) {
|
||||
instanceValuesMap = historyDataReader.get().getHistoryMetricData(instance, app, metrics, metric, history);
|
||||
} else {
|
||||
instanceValuesMap = historyDataReader.get().getHistoryIntervalMetricData(instance, app, metrics, metric, history);
|
||||
}
|
||||
if (instanceValuesMap.containsKey("{}")) {
|
||||
instanceValuesMap.put("", instanceValuesMap.get("{}"));
|
||||
instanceValuesMap.remove("{}");
|
||||
}
|
||||
return MetricsHistoryData.builder()
|
||||
.instance(instance).metrics(metrics).values(instanceValuesMap)
|
||||
.field(Field.builder().name(metric).type(CommonConstants.TYPE_NUMBER).build())
|
||||
.build();
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.service.impl;
|
||||
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.service.WarehouseService;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.AbstractRealTimeDataStorage;
|
||||
import org.springframework.stereotype.Service;
|
||||
|
||||
/**
|
||||
* warehouse service impl
|
||||
*/
|
||||
@Service
|
||||
@Slf4j
|
||||
public class WarehouseServiceImpl implements WarehouseService {
|
||||
|
||||
private final AbstractRealTimeDataStorage realTimeDataStorage;
|
||||
|
||||
public WarehouseServiceImpl(AbstractRealTimeDataStorage realTimeDataStorage) {
|
||||
this.realTimeDataStorage = realTimeDataStorage;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<CollectRep.MetricsData> queryMonitorMetricsData(Long monitorId) {
|
||||
boolean available = realTimeDataStorage.isServerAvailable();
|
||||
if (!available) {
|
||||
log.error("real time store not available");
|
||||
return Collections.emptyList();
|
||||
}
|
||||
return realTimeDataStorage.getCurrentMetricsData(monitorId);
|
||||
}
|
||||
}
|
||||
+155
@@ -0,0 +1,155 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
import jakarta.persistence.EntityManager;
|
||||
import jakarta.persistence.PersistenceContext;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.entity.manager.Monitor;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.queue.CommonDataQueue;
|
||||
import org.apache.hertzbeat.common.support.exception.CommonDataQueueUnknownException;
|
||||
import org.apache.hertzbeat.common.util.BackoffUtils;
|
||||
import org.apache.hertzbeat.common.util.ExponentialBackoff;
|
||||
import org.apache.hertzbeat.plugin.PostCollectPlugin;
|
||||
import org.apache.hertzbeat.plugin.runner.PluginRunner;
|
||||
import org.apache.hertzbeat.warehouse.WarehouseWorkerPool;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataWriter;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataWriter;
|
||||
import org.springframework.jdbc.core.JdbcTemplate;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* dispatch storage metrics data
|
||||
*/
|
||||
@Slf4j
|
||||
@Component
|
||||
public class DataStorageDispatch {
|
||||
|
||||
private final CommonDataQueue commonDataQueue;
|
||||
private final WarehouseWorkerPool workerPool;
|
||||
private final JdbcTemplate jdbcTemplate;
|
||||
private final RealTimeDataWriter realTimeDataWriter;
|
||||
private final Optional<HistoryDataWriter> historyDataWriter;
|
||||
private final PluginRunner pluginRunner;
|
||||
private static final int LOG_BATCH_SIZE = 1000;
|
||||
@PersistenceContext
|
||||
private EntityManager entityManager;
|
||||
|
||||
public DataStorageDispatch(CommonDataQueue commonDataQueue,
|
||||
WarehouseWorkerPool workerPool,
|
||||
JdbcTemplate jdbcTemplate,
|
||||
Optional<HistoryDataWriter> historyDataWriter,
|
||||
RealTimeDataWriter realTimeDataWriter,
|
||||
PluginRunner pluginRunner) {
|
||||
this.commonDataQueue = commonDataQueue;
|
||||
this.workerPool = workerPool;
|
||||
this.jdbcTemplate = jdbcTemplate;
|
||||
this.realTimeDataWriter = realTimeDataWriter;
|
||||
this.historyDataWriter = historyDataWriter;
|
||||
this.pluginRunner = pluginRunner;
|
||||
startPersistentDataStorage();
|
||||
startLogDataStorage();
|
||||
}
|
||||
|
||||
protected void startPersistentDataStorage() {
|
||||
Runnable runnable = () -> {
|
||||
Thread.currentThread().setName("warehouse-persistent-data-storage");
|
||||
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
try {
|
||||
CollectRep.MetricsData metricsData = commonDataQueue.pollMetricsDataToStorage();
|
||||
if (metricsData == null) {
|
||||
continue;
|
||||
}
|
||||
backoff.reset();
|
||||
try {
|
||||
calculateMonitorStatus(metricsData);
|
||||
historyDataWriter.ifPresent(dataWriter -> dataWriter.saveData(metricsData));
|
||||
pluginRunner.pluginExecute(PostCollectPlugin.class, ((postCollectPlugin, pluginContext) -> postCollectPlugin.execute(metricsData, pluginContext)));
|
||||
} finally {
|
||||
realTimeDataWriter.saveData(metricsData);
|
||||
}
|
||||
} catch (InterruptedException interruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (CommonDataQueueUnknownException ue) {
|
||||
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
|
||||
break;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
};
|
||||
workerPool.executeLongRunning(runnable);
|
||||
}
|
||||
|
||||
protected void startLogDataStorage() {
|
||||
Runnable runnable = () -> {
|
||||
ExponentialBackoff backoff = new ExponentialBackoff(50L, 1000L);
|
||||
Thread.currentThread().setName("warehouse-log-data-storage");
|
||||
while (!Thread.currentThread().isInterrupted()) {
|
||||
try {
|
||||
List<LogEntry> logEntries = commonDataQueue.pollLogEntryToStorageBatch(LOG_BATCH_SIZE);
|
||||
if (logEntries == null || logEntries.isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
backoff.reset();
|
||||
historyDataWriter.ifPresent(dataWriter -> {
|
||||
try {
|
||||
dataWriter.saveLogDataBatch(logEntries);
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to save log entries batch: {}", e.getMessage(), e);
|
||||
}
|
||||
});
|
||||
} catch (InterruptedException interruptedException) {
|
||||
Thread.currentThread().interrupt();
|
||||
} catch (CommonDataQueueUnknownException ue) {
|
||||
if (!BackoffUtils.shouldContinueAfterBackoff(backoff)) {
|
||||
break;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error in log data storage thread: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
};
|
||||
workerPool.executeLongRunning(runnable);
|
||||
}
|
||||
|
||||
protected void calculateMonitorStatus(CollectRep.MetricsData metricsData) {
|
||||
if (metricsData.getPriority() == 0) {
|
||||
long id = metricsData.getId();
|
||||
CollectRep.Code code = metricsData.getCode();
|
||||
try {
|
||||
String sql = "UPDATE hzb_monitor SET status = ? WHERE id = ? AND status = ?";
|
||||
int status = code == CollectRep.Code.SUCCESS ? CommonConstants.MONITOR_UP_CODE : CommonConstants.MONITOR_DOWN_CODE;
|
||||
int preStatus = code == CollectRep.Code.SUCCESS ? CommonConstants.MONITOR_DOWN_CODE : CommonConstants.MONITOR_UP_CODE;
|
||||
int matchedRows = jdbcTemplate.update(sql, status, id, preStatus);
|
||||
if (matchedRows > 0) {
|
||||
entityManager.getEntityManagerFactory().getCache().evict(Monitor.class, id);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Update monitor status failed for monitor id: {}", id, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
|
||||
/**
|
||||
* data storage abstract class
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class AbstractHistoryDataStorage implements HistoryDataReader, HistoryDataWriter, DisposableBean {
|
||||
|
||||
protected boolean serverAvailable;
|
||||
|
||||
/**
|
||||
* @return data storage available
|
||||
*/
|
||||
@Override
|
||||
public boolean isServerAvailable() {
|
||||
return serverAvailable;
|
||||
}
|
||||
}
|
||||
+131
@@ -0,0 +1,131 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
|
||||
/**
|
||||
* history data reader
|
||||
*/
|
||||
public interface HistoryDataReader {
|
||||
|
||||
/**
|
||||
* @return data storage available
|
||||
*/
|
||||
boolean isServerAvailable();
|
||||
|
||||
/**
|
||||
* query history range metrics data from tsdb
|
||||
*
|
||||
* @param instance instance e.g. ip:port or ip or domain
|
||||
* @param app monitor type
|
||||
* @param metrics metrics
|
||||
* @param metric metric
|
||||
* @param history range
|
||||
* @return metrics data
|
||||
*/
|
||||
Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history);
|
||||
|
||||
/**
|
||||
* query history range interval metrics data from tsdb
|
||||
* max min mean metrics value
|
||||
*
|
||||
* @param instance instance e.g. ip:port or ip or domain
|
||||
* @param app monitor type
|
||||
* @param metrics metrics
|
||||
* @param metric metric
|
||||
* @param history history range
|
||||
* @return metrics data
|
||||
*/
|
||||
Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history);
|
||||
|
||||
/**
|
||||
* Query logs with multiple filter conditions
|
||||
* @param startTime start time in milliseconds
|
||||
* @param endTime end time in milliseconds
|
||||
* @param traceId trace ID filter
|
||||
* @param spanId span ID filter
|
||||
* @param severityNumber severity number filter
|
||||
* @param severityText severity text filter
|
||||
* @param searchContent search content in log body
|
||||
* @return filtered log entries
|
||||
*/
|
||||
default List<LogEntry> queryLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber,
|
||||
String severityText, String searchContent) {
|
||||
throw new UnsupportedOperationException("query logs by multiple conditions is not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Query logs with multiple filter conditions and pagination (Legacy)
|
||||
*/
|
||||
default List<LogEntry> queryLogsByMultipleConditionsWithPagination(Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber,
|
||||
String severityText, Integer offset, Integer limit) {
|
||||
return queryLogsByMultipleConditionsWithPagination(startTime, endTime, traceId, spanId, severityNumber, severityText, null, offset, limit);
|
||||
}
|
||||
|
||||
/**
|
||||
* Query logs with multiple filter conditions and pagination including search content
|
||||
* @param startTime start time in milliseconds
|
||||
* @param endTime end time in milliseconds
|
||||
* @param traceId trace ID filter
|
||||
* @param spanId span ID filter
|
||||
* @param severityNumber severity number filter
|
||||
* @param severityText severity text filter
|
||||
* @param searchContent search content in log body
|
||||
* @param offset pagination offset
|
||||
* @param limit pagination limit
|
||||
* @return filtered log entries with pagination
|
||||
*/
|
||||
default List<LogEntry> queryLogsByMultipleConditionsWithPagination(Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber,
|
||||
String severityText, String searchContent,
|
||||
Integer offset, Integer limit) {
|
||||
throw new UnsupportedOperationException("query logs by multiple conditions with pagination is not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Count logs with multiple filter conditions (Legacy)
|
||||
*/
|
||||
default long countLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber,
|
||||
String severityText) {
|
||||
return countLogsByMultipleConditions(startTime, endTime, traceId, spanId, severityNumber, severityText, null);
|
||||
}
|
||||
|
||||
/**
|
||||
* Count logs with multiple filter conditions including search content
|
||||
* @param startTime start time in milliseconds
|
||||
* @param endTime end time in milliseconds
|
||||
* @param traceId trace ID filter
|
||||
* @param spanId span ID filter
|
||||
* @param severityNumber severity number filter
|
||||
* @param severityText severity text filter
|
||||
* @param searchContent search content in log body
|
||||
* @return count of matching log entries
|
||||
*/
|
||||
default long countLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber,
|
||||
String severityText, String searchContent) {
|
||||
throw new UnsupportedOperationException("count logs by multiple conditions is not supported");
|
||||
}
|
||||
}
|
||||
+70
@@ -0,0 +1,70 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb;
|
||||
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* history data writer
|
||||
*/
|
||||
public interface HistoryDataWriter {
|
||||
|
||||
/**
|
||||
* @return data storage available
|
||||
*/
|
||||
boolean isServerAvailable();
|
||||
|
||||
/**
|
||||
* save metrics data
|
||||
* @param metricsData metrics data
|
||||
*/
|
||||
void saveData(CollectRep.MetricsData metricsData);
|
||||
|
||||
/**
|
||||
* default save log data
|
||||
* @param logEntry log entry
|
||||
*/
|
||||
default void saveLogData(LogEntry logEntry) {
|
||||
throw new UnsupportedOperationException("save log data is not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch delete logs by time timestamps
|
||||
* @param timeUnixNanos list of time timestamps to delete
|
||||
* @return true if deletion is successful, false otherwise
|
||||
*/
|
||||
default boolean batchDeleteLogs(List<Long> timeUnixNanos) {
|
||||
throw new UnsupportedOperationException("batch delete logs is not supported");
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch save log data
|
||||
* @param logEntries list of log entries
|
||||
*/
|
||||
default void saveLogDataBatch(List<LogEntry> logEntries) {
|
||||
if (logEntries == null || logEntries.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
for (LogEntry logEntry : logEntries) {
|
||||
saveLogData(logEntry);
|
||||
}
|
||||
}
|
||||
}
|
||||
+1244
File diff suppressed because it is too large
Load Diff
+46
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.doris;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
|
||||
/**
|
||||
* Internal class to represent a metric row for Doris storage.
|
||||
* This class is used by both DorisDataStorage and DorisStreamLoadWriter.
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class DorisMetricRow {
|
||||
public String instance;
|
||||
public String app;
|
||||
public String metrics;
|
||||
public String metric;
|
||||
public byte metricType;
|
||||
public Integer int32Value;
|
||||
public Double doubleValue;
|
||||
public String strValue;
|
||||
public Timestamp recordTime;
|
||||
public String labels;
|
||||
}
|
||||
+235
@@ -0,0 +1,235 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.doris;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* Apache Doris configuration properties
|
||||
*/
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.DORIS)
|
||||
public record DorisProperties(
|
||||
@DefaultValue("false") boolean enabled,
|
||||
@DefaultValue("jdbc:mysql://127.0.0.1:9030/hertzbeat") String url,
|
||||
String username,
|
||||
String password,
|
||||
TableConfig tableConfig,
|
||||
PoolConfig poolConfig,
|
||||
WriteConfig writeConfig) {
|
||||
/**
|
||||
* Table structure configuration
|
||||
*/
|
||||
public record TableConfig(
|
||||
// Whether to enable dynamic partitioning (default enabled)
|
||||
@DefaultValue("false") boolean enablePartition,
|
||||
// Partition time unit: DAY, HOUR, MONTH
|
||||
@DefaultValue("DAY") String partitionTimeUnit,
|
||||
// Dynamic partition retention days
|
||||
@DefaultValue("7") int partitionRetentionDays,
|
||||
// Dynamic partition creation range (future partitions to create)
|
||||
@DefaultValue("3") int partitionFutureDays,
|
||||
// Number of buckets
|
||||
@DefaultValue("8") int buckets,
|
||||
// Number of replicas (recommended 3 for production)
|
||||
@DefaultValue("1") int replicationNum,
|
||||
// Maximum length of string columns
|
||||
@DefaultValue("4096") int strColumnMaxLength) {
|
||||
public TableConfig {
|
||||
if (partitionRetentionDays <= 0) {
|
||||
partitionRetentionDays = 7;
|
||||
}
|
||||
if (partitionFutureDays <= 0) {
|
||||
partitionFutureDays = 3;
|
||||
}
|
||||
if (buckets <= 0) {
|
||||
buckets = 8;
|
||||
}
|
||||
if (replicationNum <= 0) {
|
||||
replicationNum = 1;
|
||||
}
|
||||
if (strColumnMaxLength <= 0 || strColumnMaxLength > 65533) {
|
||||
strColumnMaxLength = 4096;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Connection pool configuration (based on HikariCP)
|
||||
*/
|
||||
public record PoolConfig(
|
||||
// Minimum idle connections
|
||||
@DefaultValue("5") int minimumIdle,
|
||||
// Maximum pool size
|
||||
@DefaultValue("20") int maximumPoolSize,
|
||||
// Connection timeout in milliseconds
|
||||
@DefaultValue("30000") int connectionTimeout,
|
||||
// Maximum connection lifetime in milliseconds (0 means no limit)
|
||||
@DefaultValue("0") long maxLifetime,
|
||||
// Idle connection timeout in milliseconds (0 means never recycle)
|
||||
@DefaultValue("600000") long idleTimeout) {
|
||||
public PoolConfig {
|
||||
if (minimumIdle <= 0) {
|
||||
minimumIdle = 5;
|
||||
}
|
||||
if (maximumPoolSize <= 0) {
|
||||
maximumPoolSize = 20;
|
||||
}
|
||||
if (minimumIdle > maximumPoolSize) {
|
||||
minimumIdle = maximumPoolSize;
|
||||
}
|
||||
if (connectionTimeout <= 0) {
|
||||
connectionTimeout = 30000;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Write configuration
|
||||
*/
|
||||
public record WriteConfig(
|
||||
// Write mode: jdbc (batch insert) or stream (HTTP stream load)
|
||||
@DefaultValue("jdbc") String writeMode,
|
||||
// Batch write size (for jdbc mode)
|
||||
@DefaultValue("1000") int batchSize,
|
||||
// Batch write flush interval in seconds (for jdbc mode)
|
||||
@DefaultValue("5") int flushInterval,
|
||||
// Fallback to JDBC when stream load fails (may introduce duplicate data in ambiguous cases)
|
||||
@DefaultValue("false") boolean fallbackToJdbcOnFailure,
|
||||
// Stream load configuration (for stream mode)
|
||||
StreamLoadConfig streamLoadConfig) {
|
||||
public WriteConfig {
|
||||
String normalizedWriteMode = writeMode == null ? "" : writeMode.trim().toLowerCase();
|
||||
if (!"jdbc".equals(normalizedWriteMode) && !"stream".equals(normalizedWriteMode)) {
|
||||
writeMode = "jdbc";
|
||||
} else {
|
||||
writeMode = normalizedWriteMode;
|
||||
}
|
||||
if (batchSize <= 0) {
|
||||
batchSize = 1000;
|
||||
}
|
||||
if (flushInterval <= 0) {
|
||||
flushInterval = 5;
|
||||
}
|
||||
if (streamLoadConfig == null) {
|
||||
streamLoadConfig = StreamLoadConfig.createDefault();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stream Load configuration for HTTP-based streaming writes
|
||||
*/
|
||||
public record StreamLoadConfig(
|
||||
// Doris FE HTTP port for Stream Load API
|
||||
@DefaultValue(":8030") String httpPort,
|
||||
// Stream load timeout in seconds
|
||||
@DefaultValue("60") int timeout,
|
||||
// Max batch size in bytes for stream load
|
||||
@DefaultValue("10485760") int maxBytesPerBatch,
|
||||
// Maximum allowed filter ratio in [0,1]
|
||||
@DefaultValue("0.1") double maxFilterRatio,
|
||||
// Enable strict mode
|
||||
@DefaultValue("false") boolean strictMode,
|
||||
// Import timezone, empty means Doris default
|
||||
@DefaultValue("") String timezone,
|
||||
// Redirect policy: direct/public/private
|
||||
@DefaultValue("") String redirectPolicy,
|
||||
// Group commit mode: async_mode/sync_mode/off_mode
|
||||
@DefaultValue("") String groupCommit,
|
||||
// Send batch parallelism, 0 means Doris default
|
||||
@DefaultValue("0") int sendBatchParallelism,
|
||||
// Retry times for one label when stream load is retryable
|
||||
@DefaultValue("2") int retryTimes) {
|
||||
public StreamLoadConfig {
|
||||
if (httpPort == null || httpPort.isBlank()) {
|
||||
httpPort = ":8030";
|
||||
} else {
|
||||
httpPort = httpPort.trim();
|
||||
}
|
||||
if (timeout <= 0) {
|
||||
timeout = 60;
|
||||
}
|
||||
if (maxBytesPerBatch <= 0) {
|
||||
maxBytesPerBatch = 10485760; // 10MB
|
||||
}
|
||||
if (maxFilterRatio < 0 || maxFilterRatio > 1) {
|
||||
maxFilterRatio = 0.1;
|
||||
}
|
||||
if (sendBatchParallelism < 0) {
|
||||
sendBatchParallelism = 0;
|
||||
}
|
||||
if (retryTimes < 0) {
|
||||
retryTimes = 2;
|
||||
}
|
||||
if (!isValidRedirectPolicy(redirectPolicy)) {
|
||||
redirectPolicy = "";
|
||||
}
|
||||
if (!isValidGroupCommit(groupCommit)) {
|
||||
groupCommit = "";
|
||||
}
|
||||
timezone = timezone == null ? "" : timezone.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory method to create default StreamLoadConfig
|
||||
*/
|
||||
public static StreamLoadConfig createDefault() {
|
||||
return new StreamLoadConfig(":8030", 60, 10485760,
|
||||
0.1, false, "", "", "", 0, 2);
|
||||
}
|
||||
|
||||
private static boolean isValidRedirectPolicy(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return true;
|
||||
}
|
||||
return "direct".equalsIgnoreCase(value)
|
||||
|| "public".equalsIgnoreCase(value)
|
||||
|| "private".equalsIgnoreCase(value);
|
||||
}
|
||||
|
||||
private static boolean isValidGroupCommit(String value) {
|
||||
if (value == null || value.isBlank()) {
|
||||
return true;
|
||||
}
|
||||
return "async_mode".equalsIgnoreCase(value)
|
||||
|| "sync_mode".equalsIgnoreCase(value)
|
||||
|| "off_mode".equalsIgnoreCase(value);
|
||||
}
|
||||
}
|
||||
|
||||
// Provide default values for nested configs if null
|
||||
public DorisProperties {
|
||||
if (tableConfig == null) {
|
||||
tableConfig = new TableConfig(false, "DAY", 7, 3, 8, 1, 4096);
|
||||
}
|
||||
if (poolConfig == null) {
|
||||
poolConfig = new PoolConfig(5, 20, 30000, 0, 600000);
|
||||
}
|
||||
if (writeConfig == null) {
|
||||
writeConfig = new WriteConfig("jdbc", 1000, 5, false, StreamLoadConfig.createDefault());
|
||||
}
|
||||
}
|
||||
}
|
||||
+603
@@ -0,0 +1,603 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.doris;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
import lombok.Getter;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.codec.binary.Base64;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.http.Header;
|
||||
import org.apache.http.HttpEntityEnclosingRequest;
|
||||
import org.apache.http.HttpRequest;
|
||||
import org.apache.http.HttpResponse;
|
||||
import org.apache.http.HttpHeaders;
|
||||
import org.apache.http.protocol.HttpContext;
|
||||
import org.apache.http.ProtocolException;
|
||||
import org.apache.http.client.config.RequestConfig;
|
||||
import org.apache.http.client.methods.CloseableHttpResponse;
|
||||
import org.apache.http.client.methods.HttpPut;
|
||||
import org.apache.http.entity.StringEntity;
|
||||
import org.apache.http.impl.client.CloseableHttpClient;
|
||||
import org.apache.http.impl.client.DefaultRedirectStrategy;
|
||||
import org.apache.http.impl.client.HttpClients;
|
||||
import org.apache.http.impl.conn.PoolingHttpClientConnectionManager;
|
||||
import org.apache.http.util.EntityUtils;
|
||||
import org.apache.http.client.methods.HttpUriRequest;
|
||||
import org.apache.http.client.utils.URIBuilder;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.net.URI;
|
||||
import java.net.URISyntaxException;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneId;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.atomic.AtomicLong;
|
||||
|
||||
/**
|
||||
* Doris Stream Load writer using Apache HttpClient for high-throughput writes.
|
||||
* Based on official Apache Doris Stream Load Java example.
|
||||
* <p>
|
||||
* Reference: <a href="https://github.com/apache/doris/blob/master/samples/doris-stream-load-demo/src/main/java/DorisStreamLoad.java">...</a>
|
||||
*/
|
||||
@Slf4j
|
||||
public class DorisStreamLoadWriter {
|
||||
|
||||
private static final int CONNECT_TIMEOUT = 30000;
|
||||
private static final int SOCKET_TIMEOUT = 60000;
|
||||
private static final int CONNECTION_REQUEST_TIMEOUT = 5000;
|
||||
private static final int MAX_TOTAL_CONNECTIONS = 20;
|
||||
private static final int MAX_PER_ROUTE_CONNECTIONS = 10;
|
||||
private static final long RETRY_BACKOFF_MS = 200L;
|
||||
private static final DateTimeFormatter DORIS_DATETIME_FORMATTER =
|
||||
DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS");
|
||||
private static final String METRIC_JSON_PATHS =
|
||||
"[\"$.instance\",\"$.app\",\"$.metrics\",\"$.metric\",\"$.recordTime\",\"$.metricType\",\"$.int32Value\",\"$.doubleValue\",\"$.strValue\",\"$.labels\"]";
|
||||
private static final String METRIC_COLUMNS =
|
||||
"instance,app,metrics,metric,record_time,metric_type,int32_value,double_value,str_value,labels";
|
||||
private static final String LOG_JSON_PATHS =
|
||||
"[\"$.timeUnixNano\",\"$.observedTimeUnixNano\",\"$.eventTime\",\"$.severityNumber\",\"$.severityText\""
|
||||
+ ",\"$.body\",\"$.traceId\",\"$.spanId\",\"$.traceFlags\",\"$.attributes\",\"$.resource\""
|
||||
+ ",\"$.instrumentationScope\",\"$.droppedAttributesCount\"]";
|
||||
private static final String LOG_COLUMNS =
|
||||
"time_unix_nano,observed_time_unix_nano,event_time,severity_number,severity_text,body,trace_id,span_id,trace_flags,attributes,resource,instrumentation_scope,dropped_attributes_count";
|
||||
|
||||
public static final String STATUS_SUCCESS = "Success";
|
||||
public static final String STATUS_PUBLISH_TIMEOUT = "Publish Timeout";
|
||||
public static final String STATUS_FAIL = "Fail";
|
||||
public static final String STATUS_LABEL_ALREADY_EXISTS = "Label Already Exists";
|
||||
|
||||
private final String databaseName;
|
||||
private final String tableName;
|
||||
private final String feHost;
|
||||
private final int feHttpPort;
|
||||
private final String username;
|
||||
private final String password;
|
||||
private final DorisProperties.StreamLoadConfig config;
|
||||
|
||||
private final AtomicLong transactionId = new AtomicLong(0);
|
||||
private final CloseableHttpClient httpClient;
|
||||
|
||||
private enum LoadResult {
|
||||
SUCCESS,
|
||||
RETRYABLE_FAILURE,
|
||||
NON_RETRYABLE_FAILURE
|
||||
}
|
||||
|
||||
private static final class StreamLoadMetricRow {
|
||||
public String instance;
|
||||
public String app;
|
||||
public String metrics;
|
||||
public String metric;
|
||||
public String recordTime;
|
||||
public Byte metricType;
|
||||
public Integer int32Value;
|
||||
public Double doubleValue;
|
||||
public String strValue;
|
||||
public String labels;
|
||||
}
|
||||
|
||||
private static final class StreamLoadLogRow {
|
||||
public Long timeUnixNano;
|
||||
public Long observedTimeUnixNano;
|
||||
public String eventTime;
|
||||
public Integer severityNumber;
|
||||
public String severityText;
|
||||
public String body;
|
||||
public String traceId;
|
||||
public String spanId;
|
||||
public Integer traceFlags;
|
||||
public String attributes;
|
||||
public String resource;
|
||||
public String instrumentationScope;
|
||||
public Integer droppedAttributesCount;
|
||||
}
|
||||
|
||||
/**
|
||||
* -- GETTER --
|
||||
* Check if writer is available
|
||||
*/
|
||||
@Getter
|
||||
private volatile boolean available = true;
|
||||
|
||||
public DorisStreamLoadWriter(String databaseName, String tableName,
|
||||
String jdbcUrl, String username, String password,
|
||||
DorisProperties.StreamLoadConfig config) {
|
||||
this.databaseName = databaseName;
|
||||
this.tableName = tableName;
|
||||
this.username = username;
|
||||
this.password = password;
|
||||
this.config = config;
|
||||
|
||||
this.feHost = parseHostFromJdbcUrl(jdbcUrl);
|
||||
this.feHttpPort = parseHttpPort(config.httpPort());
|
||||
|
||||
// Create HTTP client with connection pool and redirect support
|
||||
this.httpClient = createHttpClient();
|
||||
|
||||
log.info("[Doris StreamLoad] Writer initialized for {}.{}", databaseName, tableName);
|
||||
}
|
||||
|
||||
private String parseHostFromJdbcUrl(String jdbcUrl) {
|
||||
if (jdbcUrl == null || jdbcUrl.isBlank()) {
|
||||
return "127.0.0.1";
|
||||
}
|
||||
|
||||
String hostPart = jdbcUrl;
|
||||
if (hostPart.startsWith("jdbc:mysql://")) {
|
||||
hostPart = hostPart.substring("jdbc:mysql://".length());
|
||||
}
|
||||
|
||||
int slashIndex = hostPart.indexOf('/');
|
||||
if (slashIndex > 0) {
|
||||
hostPart = hostPart.substring(0, slashIndex);
|
||||
}
|
||||
|
||||
int queryIndex = hostPart.indexOf('?');
|
||||
if (queryIndex > 0) {
|
||||
hostPart = hostPart.substring(0, queryIndex);
|
||||
}
|
||||
|
||||
// Multi-host URL: use first host
|
||||
int commaIndex = hostPart.indexOf(',');
|
||||
if (commaIndex > 0) {
|
||||
hostPart = hostPart.substring(0, commaIndex);
|
||||
}
|
||||
|
||||
int colonIndex = hostPart.lastIndexOf(':');
|
||||
if (colonIndex > 0) {
|
||||
hostPart = hostPart.substring(0, colonIndex);
|
||||
}
|
||||
|
||||
return hostPart;
|
||||
}
|
||||
|
||||
private int parseHttpPort(String portStr) {
|
||||
if (portStr == null || portStr.isBlank()) {
|
||||
return 8030;
|
||||
}
|
||||
if (portStr.startsWith(":")) {
|
||||
portStr = portStr.substring(1);
|
||||
}
|
||||
try {
|
||||
return Integer.parseInt(portStr);
|
||||
} catch (NumberFormatException e) {
|
||||
return 8030; // default
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create HTTP client with connection pool and automatic redirect handling
|
||||
* Replaces internal IPs in redirect URLs with external FE host for cloud deployments
|
||||
*/
|
||||
private CloseableHttpClient createHttpClient() {
|
||||
// Connection pool configuration
|
||||
PoolingHttpClientConnectionManager connectionManager = new PoolingHttpClientConnectionManager();
|
||||
connectionManager.setMaxTotal(MAX_TOTAL_CONNECTIONS);
|
||||
connectionManager.setDefaultMaxPerRoute(MAX_PER_ROUTE_CONNECTIONS);
|
||||
|
||||
// Request configuration
|
||||
RequestConfig requestConfig = RequestConfig.custom()
|
||||
.setConnectTimeout(CONNECT_TIMEOUT)
|
||||
.setSocketTimeout(SOCKET_TIMEOUT)
|
||||
.setConnectionRequestTimeout(CONNECTION_REQUEST_TIMEOUT)
|
||||
.build();
|
||||
|
||||
// Custom redirect strategy to handle PUT requests and replace internal IPs
|
||||
DefaultRedirectStrategy redirectStrategy = new DefaultRedirectStrategy() {
|
||||
@Override
|
||||
protected boolean isRedirectable(String method) {
|
||||
// Enable redirect for PUT method (required for Doris Stream Load)
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public HttpUriRequest getRedirect(
|
||||
HttpRequest request,
|
||||
HttpResponse response,
|
||||
HttpContext context) throws ProtocolException {
|
||||
URI uri = getLocationURI(request, response, context);
|
||||
String redirectLocation = uri.toString();
|
||||
|
||||
// Check if redirect points to localhost/internal IP
|
||||
String host = uri.getHost();
|
||||
if (host != null && ("127.0.0.1".equals(host) || "localhost".equals(host)
|
||||
|| host.startsWith("192.168.") || host.startsWith("10.")
|
||||
|| "0.0.0.0".equals(host))) {
|
||||
// Replace internal IP with external FE host, keep BE port and path
|
||||
int port = uri.getPort() > 0 ? uri.getPort() : 8040;
|
||||
String path = uri.getPath();
|
||||
String query = uri.getQuery();
|
||||
redirectLocation = "http://" + feHost + ":" + port + path
|
||||
+ (query != null ? "?" + query : "");
|
||||
log.debug("[Doris StreamLoad] Replaced internal IP {} with external {}:{}",
|
||||
host, feHost, port);
|
||||
try {
|
||||
uri = new URI(redirectLocation);
|
||||
} catch (URISyntaxException e) {
|
||||
log.warn("[Doris StreamLoad] Failed to build URI for redirect location", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove any embedded credentials like http://root:@
|
||||
if (redirectLocation.contains("@")) {
|
||||
try {
|
||||
URIBuilder uriBuilder = new URIBuilder(redirectLocation);
|
||||
uriBuilder.setUserInfo(null);
|
||||
uri = uriBuilder.build();
|
||||
} catch (URISyntaxException e) {
|
||||
log.warn("[Doris StreamLoad] Failed to remove credentials from redirect URL", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Create new redirect request
|
||||
HttpPut redirect = new HttpPut(uri);
|
||||
if (request instanceof HttpEntityEnclosingRequest sourceRequest
|
||||
&& sourceRequest.getEntity() != null) {
|
||||
redirect.setEntity(sourceRequest.getEntity());
|
||||
}
|
||||
// Copy headers from original request
|
||||
copyHeaders(request, redirect);
|
||||
|
||||
return redirect;
|
||||
}
|
||||
|
||||
private void copyHeaders(HttpRequest source, HttpUriRequest target) {
|
||||
// Keep all business headers for redirected PUT request.
|
||||
for (Header header : source.getAllHeaders()) {
|
||||
String name = header.getName();
|
||||
if (HttpHeaders.CONTENT_LENGTH.equalsIgnoreCase(name)
|
||||
|| HttpHeaders.HOST.equalsIgnoreCase(name)
|
||||
|| HttpHeaders.TRANSFER_ENCODING.equalsIgnoreCase(name)) {
|
||||
continue;
|
||||
}
|
||||
target.setHeader(name, header.getValue());
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
return HttpClients.custom()
|
||||
.setConnectionManager(connectionManager)
|
||||
.setDefaultRequestConfig(requestConfig)
|
||||
.setRedirectStrategy(redirectStrategy)
|
||||
.build();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Basic Authentication header
|
||||
*/
|
||||
private String basicAuthHeader(String username, String password) {
|
||||
String toBeEncode = username + ":" + (password != null ? password : "");
|
||||
byte[] encoded = Base64.encodeBase64(toBeEncode.getBytes(StandardCharsets.UTF_8));
|
||||
return "Basic " + new String(encoded, StandardCharsets.UTF_8);
|
||||
}
|
||||
|
||||
/**
|
||||
* Build unique label for idempotency
|
||||
*/
|
||||
private String buildLabel() {
|
||||
return "hzb_" + System.currentTimeMillis() + "_" + transactionId.incrementAndGet();
|
||||
}
|
||||
|
||||
/**
|
||||
* Build Stream Load URL
|
||||
*/
|
||||
private String buildLoadUrl() {
|
||||
return String.format("http://%s:%d/api/%s/%s/_stream_load",
|
||||
feHost, feHttpPort, databaseName, tableName);
|
||||
}
|
||||
|
||||
/**
|
||||
* Write metrics data using Stream Load API
|
||||
*
|
||||
* @param rows list of metric rows to write
|
||||
* @return true if write succeeded, false otherwise
|
||||
*/
|
||||
public boolean write(List<DorisMetricRow> rows) {
|
||||
if (!available || rows == null || rows.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
return writeMetricWithAutoSplit(new ArrayList<>(rows));
|
||||
}
|
||||
|
||||
public boolean writeLogs(List<LogEntry> logEntries) {
|
||||
if (!available || logEntries == null || logEntries.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
return writeLogWithAutoSplit(new ArrayList<>(logEntries));
|
||||
}
|
||||
|
||||
private boolean writeMetricWithAutoSplit(List<DorisMetricRow> rows) {
|
||||
List<StreamLoadMetricRow> streamLoadRows = toStreamLoadRows(rows);
|
||||
String jsonData = JsonUtil.toJson(streamLoadRows);
|
||||
if (jsonData == null) {
|
||||
log.error("[Doris StreamLoad] Failed to serialize {} rows to JSON.", rows.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
int payloadBytes = jsonData.getBytes(StandardCharsets.UTF_8).length;
|
||||
int maxBytesPerBatch = config.maxBytesPerBatch();
|
||||
if (payloadBytes > maxBytesPerBatch && rows.size() > 1) {
|
||||
int mid = rows.size() / 2;
|
||||
List<DorisMetricRow> left = new ArrayList<>(rows.subList(0, mid));
|
||||
List<DorisMetricRow> right = new ArrayList<>(rows.subList(mid, rows.size()));
|
||||
log.debug("[Doris StreamLoad] Split batch: rows={}, bytes={}, maxBytes={}",
|
||||
rows.size(), payloadBytes, maxBytesPerBatch);
|
||||
return writeMetricWithAutoSplit(left) && writeMetricWithAutoSplit(right);
|
||||
}
|
||||
|
||||
String label = buildLabel();
|
||||
return writeSingleBatch(rows.size(), jsonData, label, METRIC_JSON_PATHS, METRIC_COLUMNS);
|
||||
}
|
||||
|
||||
private boolean writeLogWithAutoSplit(List<LogEntry> logEntries) {
|
||||
List<StreamLoadLogRow> streamLoadRows = toStreamLoadLogRows(logEntries);
|
||||
String jsonData = JsonUtil.toJson(streamLoadRows);
|
||||
if (jsonData == null) {
|
||||
log.error("[Doris StreamLoad] Failed to serialize {} log entries to JSON.", logEntries.size());
|
||||
return false;
|
||||
}
|
||||
|
||||
int payloadBytes = jsonData.getBytes(StandardCharsets.UTF_8).length;
|
||||
int maxBytesPerBatch = config.maxBytesPerBatch();
|
||||
if (payloadBytes > maxBytesPerBatch && logEntries.size() > 1) {
|
||||
int mid = logEntries.size() / 2;
|
||||
List<LogEntry> left = new ArrayList<>(logEntries.subList(0, mid));
|
||||
List<LogEntry> right = new ArrayList<>(logEntries.subList(mid, logEntries.size()));
|
||||
log.debug("[Doris StreamLoad] Split log batch: rows={}, bytes={}, maxBytes={}",
|
||||
logEntries.size(), payloadBytes, maxBytesPerBatch);
|
||||
return writeLogWithAutoSplit(left) && writeLogWithAutoSplit(right);
|
||||
}
|
||||
|
||||
String label = buildLabel();
|
||||
return writeSingleBatch(logEntries.size(), jsonData, label, LOG_JSON_PATHS, LOG_COLUMNS);
|
||||
}
|
||||
|
||||
private List<StreamLoadMetricRow> toStreamLoadRows(List<DorisMetricRow> rows) {
|
||||
List<StreamLoadMetricRow> result = new ArrayList<>(rows.size());
|
||||
for (DorisMetricRow row : rows) {
|
||||
StreamLoadMetricRow item = new StreamLoadMetricRow();
|
||||
item.instance = row.instance;
|
||||
item.app = row.app;
|
||||
item.metrics = row.metrics;
|
||||
item.metric = row.metric;
|
||||
item.recordTime = formatRecordTime(row.recordTime);
|
||||
item.metricType = row.metricType;
|
||||
item.int32Value = row.int32Value;
|
||||
item.doubleValue = row.doubleValue;
|
||||
item.strValue = row.strValue;
|
||||
item.labels = row.labels;
|
||||
result.add(item);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String formatRecordTime(Timestamp timestamp) {
|
||||
if (timestamp == null) {
|
||||
return null;
|
||||
}
|
||||
return DORIS_DATETIME_FORMATTER.format(timestamp.toInstant().atZone(ZoneId.systemDefault()).toLocalDateTime());
|
||||
}
|
||||
|
||||
private List<StreamLoadLogRow> toStreamLoadLogRows(List<LogEntry> logEntries) {
|
||||
List<StreamLoadLogRow> result = new ArrayList<>(logEntries.size());
|
||||
for (LogEntry logEntry : logEntries) {
|
||||
long timeUnixNano = logEntry.getTimeUnixNano() != null
|
||||
? logEntry.getTimeUnixNano()
|
||||
: System.currentTimeMillis() * 1_000_000L;
|
||||
long observedTimeUnixNano = logEntry.getObservedTimeUnixNano() != null
|
||||
? logEntry.getObservedTimeUnixNano()
|
||||
: timeUnixNano;
|
||||
|
||||
StreamLoadLogRow row = new StreamLoadLogRow();
|
||||
row.timeUnixNano = timeUnixNano;
|
||||
row.observedTimeUnixNano = observedTimeUnixNano;
|
||||
row.eventTime = formatEpochNanos(timeUnixNano);
|
||||
row.severityNumber = logEntry.getSeverityNumber();
|
||||
row.severityText = logEntry.getSeverityText();
|
||||
row.body = JsonUtil.toJson(logEntry.getBody());
|
||||
row.traceId = logEntry.getTraceId();
|
||||
row.spanId = logEntry.getSpanId();
|
||||
row.traceFlags = logEntry.getTraceFlags();
|
||||
row.attributes = JsonUtil.toJson(logEntry.getAttributes());
|
||||
row.resource = JsonUtil.toJson(logEntry.getResource());
|
||||
row.instrumentationScope = JsonUtil.toJson(logEntry.getInstrumentationScope());
|
||||
row.droppedAttributesCount = logEntry.getDroppedAttributesCount();
|
||||
result.add(row);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
private String formatEpochNanos(long epochNanos) {
|
||||
long millis = epochNanos / 1_000_000L;
|
||||
return DORIS_DATETIME_FORMATTER.format(
|
||||
Instant.ofEpochMilli(millis).atZone(ZoneId.systemDefault()).toLocalDateTime());
|
||||
}
|
||||
|
||||
private boolean writeSingleBatch(int rowCount, String jsonData, String label,
|
||||
String jsonPaths, String columns) {
|
||||
String loadUrl = buildLoadUrl();
|
||||
int maxAttempts = Math.max(1, config.retryTimes() + 1);
|
||||
|
||||
for (int attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
LoadResult loadResult;
|
||||
String responseBody = "";
|
||||
int statusCode = -1;
|
||||
String action = String.format("label=%s attempt=%d/%d rows=%d",
|
||||
label, attempt, maxAttempts, rowCount);
|
||||
|
||||
HttpPut httpPut = new HttpPut(loadUrl);
|
||||
httpPut.setHeader(HttpHeaders.EXPECT, "100-continue");
|
||||
httpPut.setHeader(HttpHeaders.AUTHORIZATION, basicAuthHeader(username, password));
|
||||
httpPut.setHeader("label", label);
|
||||
httpPut.setHeader("format", "json");
|
||||
httpPut.setHeader("strip_outer_array", "true");
|
||||
httpPut.setHeader("timeout", String.valueOf(config.timeout()));
|
||||
httpPut.setHeader("max_filter_ratio", String.valueOf(config.maxFilterRatio()));
|
||||
httpPut.setHeader("strict_mode", String.valueOf(config.strictMode()));
|
||||
httpPut.setHeader("jsonpaths", jsonPaths);
|
||||
httpPut.setHeader("columns", columns);
|
||||
setHeaderIfHasText(httpPut, "timezone", config.timezone());
|
||||
setHeaderIfHasText(httpPut, "redirect-policy", config.redirectPolicy());
|
||||
setHeaderIfHasText(httpPut, "group_commit", config.groupCommit());
|
||||
if (config.sendBatchParallelism() > 0) {
|
||||
httpPut.setHeader("send_batch_parallelism", String.valueOf(config.sendBatchParallelism()));
|
||||
}
|
||||
httpPut.setEntity(new StringEntity(jsonData, StandardCharsets.UTF_8));
|
||||
|
||||
try (CloseableHttpResponse response = httpClient.execute(httpPut)) {
|
||||
statusCode = response.getStatusLine().getStatusCode();
|
||||
if (response.getEntity() != null) {
|
||||
responseBody = EntityUtils.toString(response.getEntity());
|
||||
}
|
||||
loadResult = handleResponse(statusCode, responseBody, rowCount);
|
||||
} catch (Exception e) {
|
||||
log.warn("[Doris StreamLoad] Request error, {}: {}", action, e.getMessage());
|
||||
loadResult = LoadResult.RETRYABLE_FAILURE;
|
||||
}
|
||||
|
||||
if (loadResult == LoadResult.SUCCESS) {
|
||||
return true;
|
||||
}
|
||||
|
||||
if (loadResult == LoadResult.NON_RETRYABLE_FAILURE || attempt >= maxAttempts) {
|
||||
log.error("[Doris StreamLoad] Batch failed, {} statusCode={} response={}",
|
||||
action, statusCode, responseBody);
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
Thread.sleep(RETRY_BACKOFF_MS * attempt);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
log.warn("[Doris StreamLoad] Retry interrupted, {}", action);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
private void setHeaderIfHasText(HttpPut httpPut, String name, String value) {
|
||||
if (value != null && !value.isBlank()) {
|
||||
httpPut.setHeader(name, value.trim());
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle Stream Load response
|
||||
*
|
||||
* Note: statusCode 200 only indicates the BE service is OK, not that stream load succeeded.
|
||||
* We must parse the response JSON to check the actual status.
|
||||
*/
|
||||
private LoadResult handleResponse(int statusCode, String body, int rowCount) {
|
||||
if (statusCode != 200) {
|
||||
log.warn("[Doris StreamLoad] HTTP request returned status {}, response: {}",
|
||||
statusCode, body);
|
||||
return statusCode >= 500 ? LoadResult.RETRYABLE_FAILURE : LoadResult.NON_RETRYABLE_FAILURE;
|
||||
}
|
||||
|
||||
JsonNode jsonNode = JsonUtil.fromJson(body);
|
||||
if (jsonNode == null || !jsonNode.has("Status")) {
|
||||
log.error("[Doris StreamLoad] Invalid response body: {}", body);
|
||||
return LoadResult.NON_RETRYABLE_FAILURE;
|
||||
}
|
||||
|
||||
String status = jsonNode.get("Status").asText("");
|
||||
|
||||
if (STATUS_SUCCESS.equalsIgnoreCase(status)) {
|
||||
log.info("[Doris StreamLoad] Successfully loaded {} rows", rowCount);
|
||||
return LoadResult.SUCCESS;
|
||||
}
|
||||
|
||||
if (STATUS_PUBLISH_TIMEOUT.equalsIgnoreCase(status)) {
|
||||
log.warn("[Doris StreamLoad] Publish Timeout for {} rows, treated as success. Response: {}",
|
||||
rowCount, body);
|
||||
return LoadResult.SUCCESS;
|
||||
}
|
||||
|
||||
if (STATUS_LABEL_ALREADY_EXISTS.equalsIgnoreCase(status)) {
|
||||
String existingStatus = jsonNode.has("ExistingJobStatus")
|
||||
? jsonNode.get("ExistingJobStatus").asText("")
|
||||
: "";
|
||||
if ("FINISHED".equalsIgnoreCase(existingStatus) || "VISIBLE".equalsIgnoreCase(existingStatus)) {
|
||||
log.info("[Doris StreamLoad] Label exists and already finished for {} rows", rowCount);
|
||||
return LoadResult.SUCCESS;
|
||||
}
|
||||
|
||||
return LoadResult.RETRYABLE_FAILURE;
|
||||
}
|
||||
|
||||
if (STATUS_FAIL.equalsIgnoreCase(status)) {
|
||||
log.error("[Doris StreamLoad] Failed to load {} rows, status={}, response={}",
|
||||
rowCount, status, body);
|
||||
return LoadResult.NON_RETRYABLE_FAILURE;
|
||||
}
|
||||
|
||||
log.warn("[Doris StreamLoad] Unexpected status={} for {} rows, response={}",
|
||||
status, rowCount, body);
|
||||
return LoadResult.RETRYABLE_FAILURE;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get current statistics
|
||||
*/
|
||||
public String getStats() {
|
||||
return String.format("transactions=%d, available=%s", transactionId.get(), available);
|
||||
}
|
||||
|
||||
/**
|
||||
* Close the HTTP client and release resources
|
||||
*/
|
||||
public void close() {
|
||||
available = false;
|
||||
try {
|
||||
if (httpClient != null) {
|
||||
httpClient.close();
|
||||
log.info("[Doris StreamLoad] HTTP client closed");
|
||||
}
|
||||
} catch (IOException e) {
|
||||
log.error("[Doris StreamLoad] Failed to close HTTP client", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
+567
@@ -0,0 +1,567 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.duckdb;
|
||||
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.common.util.TimePeriodUtil;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.sql.Connection;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.temporal.TemporalAmount;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ExecutorService;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
|
||||
/**
|
||||
* data storage by duckdb
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.duckdb", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class DuckdbDatabaseDataStorage extends AbstractHistoryDataStorage {
|
||||
|
||||
private static final String DRIVER_NAME = "org.duckdb.DuckDBDriver";
|
||||
private static final String URL_PREFIX = "jdbc:duckdb:";
|
||||
|
||||
// Ideal number of data points for charting (avoids frontend lag)
|
||||
private static final int TARGET_CHART_POINTS = 800;
|
||||
|
||||
// Regex to strictly match days format, e.g., "90d", "7D". Group 1 captures the digits.
|
||||
private static final Pattern DAY_PATTERN = Pattern.compile("^(\\d+)[dD]$");
|
||||
|
||||
private final String expireTimeStr;
|
||||
private final String dbPath;
|
||||
private final ScheduledExecutorService cleanerScheduler;
|
||||
private final ExecutorService cleanerExecutor;
|
||||
private final ScheduledDispatchTask cleanerTask;
|
||||
private HikariDataSource dataSource;
|
||||
|
||||
public DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties) {
|
||||
this(duckdbProperties, VirtualThreadProperties.defaults(), true);
|
||||
}
|
||||
|
||||
@Autowired
|
||||
public DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties,
|
||||
VirtualThreadProperties virtualThreadProperties) {
|
||||
this(duckdbProperties, virtualThreadProperties, true);
|
||||
}
|
||||
|
||||
DuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties,
|
||||
VirtualThreadProperties virtualThreadProperties,
|
||||
boolean autoStartCleaner) {
|
||||
this.dbPath = duckdbProperties.storePath();
|
||||
this.expireTimeStr = duckdbProperties.expireTime();
|
||||
this.cleanerScheduler = Executors.newSingleThreadScheduledExecutor(r -> {
|
||||
Thread thread = new Thread(r, "duckdb-cleaner");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
});
|
||||
this.cleanerExecutor = createCleanerExecutor(virtualThreadProperties);
|
||||
this.cleanerTask = new ScheduledDispatchTask(cleanerExecutor, this::runExpiredDataCleaner);
|
||||
this.serverAvailable = initDuckDb();
|
||||
if (this.serverAvailable && autoStartCleaner) {
|
||||
startExpiredDataCleaner();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean initDuckDb() {
|
||||
try {
|
||||
Class.forName(DRIVER_NAME);
|
||||
|
||||
// Initialize HikariCP
|
||||
HikariConfig config = new HikariConfig();
|
||||
config.setJdbcUrl(URL_PREFIX + dbPath);
|
||||
config.setDriverClassName(DRIVER_NAME);
|
||||
config.setPoolName("DuckDB-Pool");
|
||||
// Important: Maintain at least one connection to keep the DB file lock held
|
||||
// and avoid frequent open/close of the embedded DB file which causes lock errors.
|
||||
config.setMinimumIdle(1);
|
||||
config.setMaximumPoolSize(10);
|
||||
config.setConnectionTimeout(30000);
|
||||
config.setConnectionTestQuery("SELECT 1");
|
||||
|
||||
this.dataSource = new HikariDataSource(config);
|
||||
|
||||
try (Connection connection = this.dataSource.getConnection();
|
||||
Statement statement = connection.createStatement()) {
|
||||
|
||||
String createTableSql = """
|
||||
CREATE TABLE IF NOT EXISTS hzb_history (
|
||||
instance VARCHAR,
|
||||
app VARCHAR,
|
||||
metrics VARCHAR,
|
||||
metric VARCHAR,
|
||||
metric_type SMALLINT,
|
||||
int32_value INTEGER,
|
||||
double_value DOUBLE,
|
||||
str_value VARCHAR,
|
||||
record_time BIGINT,
|
||||
labels VARCHAR)""";
|
||||
statement.execute(createTableSql);
|
||||
// Re-add indexes for performance on queries and cleanup
|
||||
statement.execute("CREATE INDEX IF NOT EXISTS idx_hzb_history_composite ON hzb_history (instance, app, metrics, metric, record_time)");
|
||||
statement.execute("CREATE INDEX IF NOT EXISTS idx_hzb_history_record_time ON hzb_history (record_time)");
|
||||
return true;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Failed to init duckdb: {}", e.getMessage(), e);
|
||||
if (this.dataSource != null) {
|
||||
this.dataSource.close();
|
||||
}
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private void startExpiredDataCleaner() {
|
||||
// Run every 1 hour
|
||||
cleanerScheduler.scheduleAtFixedRate(this::dispatchExpiredDataCleaner, 5, 60, TimeUnit.MINUTES);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS || metricsData.getValues().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
String monitorType = metricsData.getApp();
|
||||
String metrics = metricsData.getMetrics();
|
||||
String insertSql = "INSERT INTO hzb_history VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
|
||||
|
||||
try (Connection connection = this.dataSource.getConnection();
|
||||
PreparedStatement preparedStatement = connection.prepareStatement(insertSql)) {
|
||||
|
||||
RowWrapper rowWrapper = metricsData.readRow();
|
||||
Map<String, String> labels = new HashMap<>();
|
||||
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
long time = metricsData.getTime();
|
||||
|
||||
// First pass: collect labels
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL)) {
|
||||
labels.put(cell.getField().getName(), cell.getValue());
|
||||
}
|
||||
});
|
||||
String labelsJson = JsonUtil.toJson(labels);
|
||||
|
||||
// Second pass: insert data
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
try {
|
||||
String metric = cell.getField().getName();
|
||||
String columnValue = cell.getValue();
|
||||
int fieldType = cell.getMetadataAsInteger(MetricDataConstants.TYPE);
|
||||
|
||||
preparedStatement.setString(1, metricsData.getInstance());
|
||||
preparedStatement.setString(2, monitorType);
|
||||
preparedStatement.setString(3, metrics);
|
||||
preparedStatement.setString(4, metric);
|
||||
|
||||
if (CommonConstants.NULL_VALUE.equals(columnValue)) {
|
||||
preparedStatement.setShort(5, (short) CommonConstants.TYPE_NUMBER);
|
||||
preparedStatement.setObject(6, null);
|
||||
preparedStatement.setObject(7, null);
|
||||
preparedStatement.setObject(8, null);
|
||||
} else {
|
||||
switch (fieldType) {
|
||||
case CommonConstants.TYPE_STRING -> {
|
||||
preparedStatement.setShort(5, (short) CommonConstants.TYPE_STRING);
|
||||
preparedStatement.setObject(6, null);
|
||||
preparedStatement.setObject(7, null);
|
||||
preparedStatement.setString(8, columnValue);
|
||||
}
|
||||
case CommonConstants.TYPE_TIME -> {
|
||||
preparedStatement.setShort(5, (short) CommonConstants.TYPE_TIME);
|
||||
preparedStatement.setInt(6, Integer.parseInt(columnValue));
|
||||
preparedStatement.setObject(7, null);
|
||||
preparedStatement.setObject(8, null);
|
||||
}
|
||||
default -> {
|
||||
preparedStatement.setShort(5, (short) CommonConstants.TYPE_NUMBER);
|
||||
preparedStatement.setObject(6, null);
|
||||
double v = Double.parseDouble(columnValue);
|
||||
preparedStatement.setDouble(7, v);
|
||||
preparedStatement.setObject(8, null);
|
||||
}
|
||||
}
|
||||
}
|
||||
preparedStatement.setLong(9, time);
|
||||
preparedStatement.setString(10, labelsJson);
|
||||
preparedStatement.addBatch();
|
||||
} catch (SQLException e) {
|
||||
log.error("error setting prepared statement", e);
|
||||
}
|
||||
});
|
||||
labels.clear();
|
||||
}
|
||||
preparedStatement.executeBatch();
|
||||
} catch (Exception e) {
|
||||
log.error("[duckdb] save data error: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
if (!isServerAvailable()) {
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
StringBuilder sqlBuilder = new StringBuilder("""
|
||||
SELECT record_time,
|
||||
metric_type,
|
||||
int32_value,
|
||||
double_value,
|
||||
str_value,
|
||||
labels FROM hzb_history
|
||||
WHERE instance = ?
|
||||
AND app = ?
|
||||
AND metrics = ?
|
||||
AND metric = ?
|
||||
""");
|
||||
|
||||
long timeBefore = 0;
|
||||
if (history != null) {
|
||||
try {
|
||||
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
|
||||
timeBefore = dateTime.toEpochSecond() * 1000L;
|
||||
sqlBuilder.append(" AND record_time >= ?");
|
||||
} catch (Exception e) {
|
||||
log.error("parse history time error: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
sqlBuilder.append(" ORDER BY record_time DESC LIMIT 20000"); // Add safety limit for raw data
|
||||
|
||||
try (Connection connection = this.dataSource.getConnection();
|
||||
PreparedStatement preparedStatement = connection.prepareStatement(sqlBuilder.toString())) {
|
||||
|
||||
preparedStatement.setString(1, instance);
|
||||
preparedStatement.setString(2, app);
|
||||
preparedStatement.setString(3, metrics);
|
||||
preparedStatement.setString(4, metric);
|
||||
if (timeBefore > 0) {
|
||||
preparedStatement.setLong(5, timeBefore);
|
||||
}
|
||||
|
||||
try (ResultSet resultSet = preparedStatement.executeQuery()) {
|
||||
while (resultSet.next()) {
|
||||
long time = resultSet.getLong("record_time");
|
||||
int type = resultSet.getShort("metric_type");
|
||||
String labels = resultSet.getString("labels");
|
||||
String value = formatValue(type, resultSet);
|
||||
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labels == null ? "" : labels, k -> new LinkedList<>());
|
||||
valueList.add(new Value(value, time));
|
||||
}
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
log.error("[duckdb] query data error: {}", e.getMessage(), e);
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) {
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
if (!isServerAvailable()) {
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
long timeBefore = 0;
|
||||
long endTime = System.currentTimeMillis();
|
||||
long interval = 0;
|
||||
|
||||
if (history != null) {
|
||||
try {
|
||||
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
|
||||
timeBefore = dateTime.toEpochSecond() * 1000L;
|
||||
// Calculate bucket interval based on desired target points (e.g. 800 points on chart)
|
||||
interval = (endTime - timeBefore) / TARGET_CHART_POINTS;
|
||||
// Minimum interval 1 minute
|
||||
if (interval < 60000) {
|
||||
interval = 60000;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("parse history time error: {}", e.getMessage());
|
||||
// Fallback defaults
|
||||
timeBefore = System.currentTimeMillis() - 24 * 60 * 60 * 1000L;
|
||||
interval = 60 * 1000L;
|
||||
}
|
||||
}
|
||||
|
||||
// DuckDB Aggregation Query: Group by time bucket
|
||||
// We use casting to integer division for bucketing: (time / interval) * interval
|
||||
String sql = """
|
||||
SELECT
|
||||
CAST(record_time / ? AS BIGINT) * ? AS ts_bucket,
|
||||
AVG(double_value) AS avg_val,
|
||||
MIN(double_value) AS min_val,
|
||||
MAX(double_value) AS max_val,
|
||||
FIRST(str_value) AS str_val,
|
||||
metric_type, labels
|
||||
FROM hzb_history
|
||||
WHERE instance = ? AND app = ? AND metrics = ? AND metric = ? AND record_time >= ?
|
||||
GROUP BY ts_bucket, metric_type, labels
|
||||
ORDER BY ts_bucket""";
|
||||
|
||||
try (Connection connection = this.dataSource.getConnection();
|
||||
PreparedStatement preparedStatement = connection.prepareStatement(sql)) {
|
||||
|
||||
preparedStatement.setLong(1, interval);
|
||||
preparedStatement.setLong(2, interval);
|
||||
preparedStatement.setString(3, instance);
|
||||
preparedStatement.setString(4, app);
|
||||
preparedStatement.setString(5, metrics);
|
||||
preparedStatement.setString(6, metric);
|
||||
preparedStatement.setLong(7, timeBefore);
|
||||
|
||||
try (ResultSet resultSet = preparedStatement.executeQuery()) {
|
||||
while (resultSet.next()) {
|
||||
long time = resultSet.getLong("ts_bucket");
|
||||
int type = resultSet.getShort("metric_type");
|
||||
String labels = resultSet.getString("labels");
|
||||
|
||||
Value valueObj;
|
||||
if (type == CommonConstants.TYPE_NUMBER) {
|
||||
double avg = resultSet.getDouble("avg_val");
|
||||
double min = resultSet.getDouble("min_val");
|
||||
double max = resultSet.getDouble("max_val");
|
||||
if (resultSet.wasNull()) {
|
||||
valueObj = new Value(null, time);
|
||||
} else {
|
||||
String avgStr = BigDecimal.valueOf(avg).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
String minStr = BigDecimal.valueOf(min).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
String maxStr = BigDecimal.valueOf(max).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
valueObj = new Value(avgStr, time);
|
||||
valueObj.setMean(avgStr);
|
||||
valueObj.setMin(minStr);
|
||||
valueObj.setMax(maxStr);
|
||||
}
|
||||
} else {
|
||||
// For non-numeric, we just took the FIRST value in the bucket
|
||||
String strVal = resultSet.getString("str_val");
|
||||
valueObj = new Value(strVal, time);
|
||||
}
|
||||
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labels == null ? "" : labels, k -> new LinkedList<>());
|
||||
valueList.add(valueObj);
|
||||
}
|
||||
}
|
||||
} catch (SQLException e) {
|
||||
log.error("[duckdb] query interval data error: {}", e.getMessage(), e);
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
private String formatValue(int type, ResultSet resultSet) throws SQLException {
|
||||
if (type == CommonConstants.TYPE_NUMBER) {
|
||||
double v = resultSet.getDouble("double_value");
|
||||
if (!resultSet.wasNull()) {
|
||||
return BigDecimal.valueOf(v).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
}
|
||||
} else if (type == CommonConstants.TYPE_TIME) {
|
||||
int v = resultSet.getInt("int32_value");
|
||||
if (!resultSet.wasNull()) {
|
||||
return String.valueOf(v);
|
||||
}
|
||||
} else {
|
||||
return resultSet.getString("str_value");
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
cleanerScheduler.shutdownNow();
|
||||
if (cleanerExecutor != null) {
|
||||
cleanerExecutor.shutdownNow();
|
||||
}
|
||||
if (this.dataSource != null) {
|
||||
this.dataSource.close();
|
||||
}
|
||||
}
|
||||
|
||||
void dispatchExpiredDataCleaner() {
|
||||
cleanerTask.dispatch();
|
||||
}
|
||||
|
||||
void beforeExpiredDataCleanerRun() {
|
||||
// Test hook.
|
||||
}
|
||||
|
||||
private ExecutorService createCleanerExecutor(VirtualThreadProperties virtualThreadProperties) {
|
||||
VirtualThreadProperties properties =
|
||||
virtualThreadProperties == null ? VirtualThreadProperties.defaults() : virtualThreadProperties;
|
||||
if (!properties.enabled()) {
|
||||
return null;
|
||||
}
|
||||
return Executors.newThreadPerTaskExecutor(Thread.ofVirtual()
|
||||
.name("duckdb-cleaner-vt-", 0)
|
||||
.uncaughtExceptionHandler((thread, throwable) -> {
|
||||
log.error("[duckdb] cleaner worker has uncaughtException.");
|
||||
log.error(throwable.getMessage(), throwable);
|
||||
})
|
||||
.factory());
|
||||
}
|
||||
|
||||
private void runExpiredDataCleaner() {
|
||||
log.info("[duckdb] start data cleaner and checkpoint...");
|
||||
beforeExpiredDataCleanerRun();
|
||||
long expireTime;
|
||||
try {
|
||||
// Ensure no whitespace issues
|
||||
String cleanExpireStr = expireTimeStr == null ? "" : expireTimeStr.trim();
|
||||
Matcher dayMatcher = DAY_PATTERN.matcher(cleanExpireStr);
|
||||
|
||||
if (NumberUtils.isParsable(cleanExpireStr)) {
|
||||
expireTime = NumberUtils.toLong(cleanExpireStr);
|
||||
expireTime = (ZonedDateTime.now().toEpochSecond() - expireTime) * 1000L;
|
||||
} else if (dayMatcher.matches()) {
|
||||
// Strictly matched "90d" or "90D" format
|
||||
long days = Long.parseLong(dayMatcher.group(1));
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(days));
|
||||
expireTime = dateTime.toEpochSecond() * 1000L;
|
||||
} else {
|
||||
// Fallback to existing utility for other units (h, m, s, etc.)
|
||||
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(cleanExpireStr);
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
|
||||
expireTime = dateTime.toEpochSecond() * 1000L;
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("expiredDataCleaner time error: {}. use default expire time to clean: 30d", e.getMessage());
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofDays(30));
|
||||
expireTime = dateTime.toEpochSecond() * 1000L;
|
||||
}
|
||||
|
||||
try (Connection connection = this.dataSource.getConnection()) {
|
||||
// 1. Delete expired data
|
||||
try (PreparedStatement statement = connection.prepareStatement("DELETE FROM hzb_history WHERE record_time < ?")) {
|
||||
statement.setLong(1, expireTime);
|
||||
int rows = statement.executeUpdate();
|
||||
if (rows > 0) {
|
||||
log.info("[duckdb] delete {} expired records.", rows);
|
||||
}
|
||||
}
|
||||
|
||||
// 2. Force Checkpoint to compress data and flush WAL
|
||||
// This is crucial for keeping file size small and moving data from WAL to column store
|
||||
try (Statement statement = connection.createStatement()) {
|
||||
statement.execute("CHECKPOINT");
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("[duckdb] clean expired data error: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private static final class ScheduledDispatchTask {
|
||||
|
||||
private final ExecutorService executorService;
|
||||
private final Runnable task;
|
||||
private final Object lock = new Object();
|
||||
private boolean running;
|
||||
private int pendingRuns;
|
||||
|
||||
private ScheduledDispatchTask(ExecutorService executorService, Runnable task) {
|
||||
this.executorService = executorService;
|
||||
this.task = task;
|
||||
}
|
||||
|
||||
private void dispatch() {
|
||||
if (executorService == null) {
|
||||
task.run();
|
||||
return;
|
||||
}
|
||||
synchronized (lock) {
|
||||
if (running) {
|
||||
pendingRuns++;
|
||||
return;
|
||||
}
|
||||
running = true;
|
||||
}
|
||||
submit();
|
||||
}
|
||||
|
||||
private void submit() {
|
||||
boolean submitted = false;
|
||||
try {
|
||||
executorService.execute(() -> {
|
||||
try {
|
||||
task.run();
|
||||
} finally {
|
||||
onComplete();
|
||||
}
|
||||
});
|
||||
submitted = true;
|
||||
} finally {
|
||||
if (!submitted) {
|
||||
synchronized (lock) {
|
||||
running = false;
|
||||
pendingRuns = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void onComplete() {
|
||||
boolean shouldRunAgain;
|
||||
synchronized (lock) {
|
||||
if (pendingRuns > 0) {
|
||||
pendingRuns--;
|
||||
shouldRunAgain = true;
|
||||
} else {
|
||||
running = false;
|
||||
shouldRunAgain = false;
|
||||
}
|
||||
}
|
||||
if (shouldRunAgain) {
|
||||
submit();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+40
@@ -0,0 +1,40 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.duckdb;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* Duckdb configuration information
|
||||
* @param enabled use duckdb store metrics history data
|
||||
* @param expireTime save data expire time (supports formats like '30d', '90d', or milliseconds)
|
||||
* @param storePath duckdb database file path
|
||||
*/
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.DUCKDB)
|
||||
public record DuckdbProperties(@DefaultValue("true") boolean enabled,
|
||||
@DefaultValue("90d") String expireTime,
|
||||
@DefaultValue("data/history.duckdb") String storePath) {
|
||||
}
|
||||
+864
@@ -0,0 +1,864 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
|
||||
|
||||
import io.greptime.GreptimeDB;
|
||||
import io.greptime.models.AuthInfo;
|
||||
import io.greptime.models.DataType;
|
||||
import io.greptime.models.Err;
|
||||
import io.greptime.models.Result;
|
||||
import io.greptime.models.Table;
|
||||
import io.greptime.models.TableSchema;
|
||||
import io.greptime.models.WriteOk;
|
||||
import io.greptime.options.GreptimeOptions;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.net.URI;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.time.temporal.TemporalAmount;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.function.BiConsumer;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.util.Base64Util;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.common.util.TimePeriodUtil;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.PromQlQueryContent;
|
||||
import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import org.springframework.web.util.UriComponents;
|
||||
import org.springframework.web.util.UriComponentsBuilder;
|
||||
|
||||
/**
|
||||
* GreptimeDB data storage, only supports GreptimeDB version >= v0.5
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.greptime", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class GreptimeDbDataStorage extends AbstractHistoryDataStorage {
|
||||
|
||||
private static final String BASIC = "Basic";
|
||||
private static final String QUERY_RANGE_PATH = "/v1/prometheus/api/v1/query_range";
|
||||
private static final String LABEL_KEY_NAME = "__name__";
|
||||
private static final String LABEL_KEY_FIELD = "__field__";
|
||||
private static final String LABEL_KEY_INSTANCE = "instance";
|
||||
private static final String LOG_TABLE_NAME = WarehouseConstants.LOG_TABLE_NAME;
|
||||
private static final String LABEL_KEY_START_TIME = "start";
|
||||
private static final String LABEL_KEY_END_TIME = "end";
|
||||
private static final String LABEL_KEY_TS = "ts";
|
||||
private static final int LOG_BATCH_SIZE = 500;
|
||||
|
||||
private GreptimeDB greptimeDb;
|
||||
|
||||
private final GreptimeProperties greptimeProperties;
|
||||
|
||||
private final RestTemplate restTemplate;
|
||||
|
||||
private final GreptimeSqlQueryExecutor greptimeSqlQueryExecutor;
|
||||
|
||||
public GreptimeDbDataStorage(GreptimeProperties greptimeProperties, RestTemplate restTemplate,
|
||||
GreptimeSqlQueryExecutor greptimeSqlQueryExecutor) {
|
||||
if (greptimeProperties == null) {
|
||||
log.error("init error, please config Warehouse GreptimeDB props in application.yml");
|
||||
throw new IllegalArgumentException("please config Warehouse GreptimeDB props");
|
||||
}
|
||||
this.restTemplate = restTemplate;
|
||||
this.greptimeProperties = greptimeProperties;
|
||||
this.greptimeSqlQueryExecutor = greptimeSqlQueryExecutor;
|
||||
serverAvailable = initGreptimeDbClient(greptimeProperties);
|
||||
}
|
||||
|
||||
private boolean initGreptimeDbClient(GreptimeProperties greptimeProperties) {
|
||||
String endpoints = greptimeProperties.grpcEndpoints();
|
||||
try {
|
||||
GreptimeOptions opts = GreptimeOptions.newBuilder(endpoints.split(","), greptimeProperties.database())
|
||||
.writeMaxRetries(3)
|
||||
.authInfo(new AuthInfo(greptimeProperties.username(), greptimeProperties.password()))
|
||||
.routeTableRefreshPeriodSeconds(30)
|
||||
.build();
|
||||
this.greptimeDb = GreptimeDB.create(opts);
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse greptime] Fail to start GreptimeDB client");
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
|
||||
return;
|
||||
}
|
||||
if (metricsData.getValues().isEmpty()) {
|
||||
log.info("[warehouse greptime] flush metrics data {} {}is null, ignore.", metricsData.getId(), metricsData.getMetrics());
|
||||
return;
|
||||
}
|
||||
String instance = metricsData.getInstance();
|
||||
String app = metricsData.getApp();
|
||||
String tableName = getTableName(app, metricsData.getMetrics());
|
||||
TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(tableName);
|
||||
|
||||
tableSchemaBuilder.addTag("instance", DataType.String)
|
||||
.addTimestamp("ts", DataType.TimestampMillisecond);
|
||||
List<CollectRep.Field> fields = metricsData.getFields();
|
||||
Map<String, String> customLabels = metricsData.getLabels();
|
||||
List<String> fieldNames = fields.stream().map(CollectRep.Field::getName).collect(Collectors.toList());
|
||||
fields.forEach(field -> {
|
||||
if (field.getLabel()) {
|
||||
tableSchemaBuilder.addTag(field.getName(), DataType.String);
|
||||
} else {
|
||||
if (field.getType() == CommonConstants.TYPE_NUMBER) {
|
||||
tableSchemaBuilder.addField(field.getName(), DataType.Float64);
|
||||
} else if (field.getType() == CommonConstants.TYPE_STRING) {
|
||||
tableSchemaBuilder.addField(field.getName(), DataType.String);
|
||||
}
|
||||
}
|
||||
});
|
||||
List<String> labelKeys = new LinkedList<>();
|
||||
if (!Objects.isNull(customLabels) && !customLabels.isEmpty()) {
|
||||
for (Map.Entry<String, String> label : customLabels.entrySet()) {
|
||||
String key = label.getKey();
|
||||
if (!LABEL_KEY_INSTANCE.equals(key) && !LABEL_KEY_TS.equals(key) && !fieldNames.contains(key)) {
|
||||
tableSchemaBuilder.addTag(key, DataType.String);
|
||||
labelKeys.add(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
Table table = Table.from(tableSchemaBuilder.build());
|
||||
long now = System.currentTimeMillis();
|
||||
Object[] values = new Object[2 + fields.size() + labelKeys.size()];
|
||||
values[0] = instance;
|
||||
values[1] = now;
|
||||
RowWrapper rowWrapper = metricsData.readRow();
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
|
||||
AtomicInteger index = new AtomicInteger(-1);
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
index.getAndIncrement();
|
||||
|
||||
if (CommonConstants.NULL_VALUE.equals(cell.getValue())) {
|
||||
values[2 + index.get()] = null;
|
||||
return;
|
||||
}
|
||||
|
||||
Boolean label = cell.getMetadataAsBoolean(MetricDataConstants.LABEL);
|
||||
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
|
||||
|
||||
if (label) {
|
||||
values[2 + index.get()] = cell.getValue();
|
||||
} else {
|
||||
if (type == CommonConstants.TYPE_NUMBER) {
|
||||
values[2 + index.get()] = Double.parseDouble(cell.getValue());
|
||||
} else if (type == CommonConstants.TYPE_STRING) {
|
||||
values[2 + index.get()] = cell.getValue();
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
for (int i = 0; i < labelKeys.size(); i++) {
|
||||
values[2 + fields.size() + i] = customLabels.get(labelKeys.get(i));
|
||||
}
|
||||
|
||||
table.addRow(values);
|
||||
}
|
||||
|
||||
CompletableFuture<Result<WriteOk, Err>> writeFuture = greptimeDb.write(table);
|
||||
try {
|
||||
Result<WriteOk, Err> result = writeFuture.get(10, TimeUnit.SECONDS);
|
||||
if (result.isOk()) {
|
||||
log.debug("[warehouse greptime]-Write successful");
|
||||
} else {
|
||||
log.warn("[warehouse greptime]--Write failed: {}", result.getErr());
|
||||
}
|
||||
} catch (Throwable throwable) {
|
||||
log.error("[warehouse greptime]--Error occurred: {}", throwable.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric,
|
||||
String history) {
|
||||
Map<String, Long> timeRange = getTimeRange(history);
|
||||
Long start = timeRange.get(LABEL_KEY_START_TIME);
|
||||
Long end = timeRange.get(LABEL_KEY_END_TIME);
|
||||
|
||||
String step = getTimeStep(start, end);
|
||||
|
||||
return getHistoryData(start, end, step, instance, app, metrics, metric);
|
||||
}
|
||||
|
||||
private String getTableName(String app, String metrics) {
|
||||
return app + "_" + metrics;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
|
||||
String metric, String history) {
|
||||
Map<String, Long> timeRange = getTimeRange(history);
|
||||
Long start = timeRange.get(LABEL_KEY_START_TIME);
|
||||
Long end = timeRange.get(LABEL_KEY_END_TIME);
|
||||
|
||||
String step = getTimeStep(start, end);
|
||||
|
||||
Map<String, List<Value>> instanceValuesMap = getHistoryData(start, end, step, instance, app, metrics, metric);
|
||||
|
||||
if (instanceValuesMap.isEmpty()) {
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
// Queries below this point may yield inconsistent results due to exceeding the valid data range.
|
||||
// Therefore, we restrict the valid range by obtaining the post-query timeframe.
|
||||
// Since `gretime`'s `end` excludes the specified time, we add 4 hours.
|
||||
List<Value> values = instanceValuesMap.get(instanceValuesMap.keySet().iterator().next());
|
||||
// effective time
|
||||
long effectiveStart = values.get(0).getTime() / 1000;
|
||||
long effectiveEnd = values.get(values.size() - 1).getTime() / 1000 + Duration.ofHours(4).getSeconds();
|
||||
|
||||
String name = getTableName(app, metrics);
|
||||
String timeSeriesSelector = name + "{" + LABEL_KEY_INSTANCE + "=\"" + instance + "\"";
|
||||
if (!CommonConstants.PROMETHEUS.equals(app)) {
|
||||
timeSeriesSelector = timeSeriesSelector + "," + LABEL_KEY_FIELD + "=\"" + metric + "\"";
|
||||
}
|
||||
timeSeriesSelector = timeSeriesSelector + "}";
|
||||
|
||||
try {
|
||||
// max
|
||||
String finalTimeSeriesSelector = timeSeriesSelector;
|
||||
URI uri = getUri(effectiveStart, effectiveEnd, step, uriComponents -> "max_over_time(" + finalTimeSeriesSelector + "[" + step + "])");
|
||||
requestIntervalMetricAndPutValue(uri, instanceValuesMap, Value::setMax);
|
||||
// min
|
||||
uri = getUri(effectiveStart, effectiveEnd, step, uriComponents -> "min_over_time(" + finalTimeSeriesSelector + "[" + step + "])");
|
||||
requestIntervalMetricAndPutValue(uri, instanceValuesMap, Value::setMin);
|
||||
// avg
|
||||
uri = getUri(effectiveStart, effectiveEnd, step, uriComponents -> "avg_over_time(" + finalTimeSeriesSelector + "[" + step + "])");
|
||||
requestIntervalMetricAndPutValue(uri, instanceValuesMap, Value::setMean);
|
||||
} catch (Exception e) {
|
||||
log.error("query interval metrics data from greptime error. {}", e.getMessage(), e);
|
||||
}
|
||||
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get time range
|
||||
*
|
||||
* @param history history range
|
||||
* @return time range
|
||||
*/
|
||||
private Map<String, Long> getTimeRange(String history) {
|
||||
// Build start and end times
|
||||
Instant now = Instant.now();
|
||||
long start;
|
||||
try {
|
||||
if (NumberUtils.isParsable(history)) {
|
||||
start = NumberUtils.toLong(history);
|
||||
start = (ZonedDateTime.now().toEpochSecond() - start);
|
||||
} else {
|
||||
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
|
||||
assert temporalAmount != null;
|
||||
Instant dateTime = now.minus(temporalAmount);
|
||||
start = dateTime.getEpochSecond();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("history time error: {}. use default: 6h", e.getMessage());
|
||||
start = now.minus(6, ChronoUnit.HOURS).getEpochSecond();
|
||||
}
|
||||
long end = now.getEpochSecond();
|
||||
return Map.of("start", start, "end", end);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get time step
|
||||
*
|
||||
* @param start start time
|
||||
* @param end end time
|
||||
* @return step
|
||||
*/
|
||||
private String getTimeStep(long start, long end) {
|
||||
// get step
|
||||
String step = "60s";
|
||||
if (end - start < Duration.ofDays(7).getSeconds() && end - start > Duration.ofDays(1).getSeconds()) {
|
||||
step = "1h";
|
||||
} else if (end - start >= Duration.ofDays(7).getSeconds()) {
|
||||
step = "4h";
|
||||
}
|
||||
return step;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get history metric data
|
||||
*
|
||||
* @param start start time
|
||||
* @param end end time
|
||||
* @param step step
|
||||
* @param instance monitor instance
|
||||
* @param app monitor type
|
||||
* @param metrics metrics
|
||||
* @param metric metric
|
||||
* @return history metric data
|
||||
*/
|
||||
private Map<String, List<Value>> getHistoryData(long start, long end, String step, String instance, String app, String metrics, String metric) {
|
||||
String name = getTableName(app, metrics);
|
||||
String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + name + "\""
|
||||
+ "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\"";
|
||||
if (!CommonConstants.PROMETHEUS.equals(app)) {
|
||||
timeSeriesSelector = timeSeriesSelector + "," + LABEL_KEY_FIELD + "=\"" + metric + "\"";
|
||||
}
|
||||
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
try {
|
||||
HttpEntity<Void> httpEntity = getHttpEntity();
|
||||
|
||||
String finalTimeSeriesSelector = timeSeriesSelector;
|
||||
URI uri = getUri(start, end, step, uriComponents -> {
|
||||
MultiValueMap<String, String> queryParams = uriComponents.getQueryParams();
|
||||
if (!queryParams.isEmpty()) {
|
||||
return "{" + finalTimeSeriesSelector + "}";
|
||||
}
|
||||
return null;
|
||||
});
|
||||
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = null;
|
||||
if (uri != null) {
|
||||
responseEntity = restTemplate.exchange(uri,
|
||||
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
}
|
||||
if (responseEntity != null && responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.debug("query metrics data from greptime success. {}", uri);
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
|
||||
for (Object[] valueArr : content.getValues()) {
|
||||
long timestamp = ((Double) valueArr[0]).longValue();
|
||||
String value = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
valueList.add(new Value(value, timestamp * 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error("query metrics data from greptime failed. {}", responseEntity);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("query metrics data from greptime error. {}", e.getMessage(), e);
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get HTTP instance
|
||||
*
|
||||
* @return HTTP instance
|
||||
*/
|
||||
private HttpEntity<Void> getHttpEntity() {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||
if (StringUtils.hasText(greptimeProperties.username())
|
||||
&& StringUtils.hasText(greptimeProperties.password())) {
|
||||
String authStr = greptimeProperties.username() + ":" + greptimeProperties.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, BASIC + " " + encodedAuth);
|
||||
}
|
||||
return new HttpEntity<>(headers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Request URI
|
||||
*
|
||||
* @param start start time
|
||||
* @param end end time
|
||||
* @param step interval
|
||||
* @param queryFunction request parameters
|
||||
* @return URI
|
||||
*/
|
||||
private URI getUri(long start, long end, String step, Function<UriComponents, String> queryFunction) {
|
||||
UriComponentsBuilder uriComponentsBuilder = UriComponentsBuilder.fromUriString(greptimeProperties.httpEndpoint() + QUERY_RANGE_PATH)
|
||||
.queryParam("start", start)
|
||||
.queryParam("end", end)
|
||||
.queryParam("step", step)
|
||||
.queryParam("db", greptimeProperties.database());
|
||||
UriComponents cloneUriComponents = uriComponentsBuilder.cloneBuilder().build(true);
|
||||
String queryValue = queryFunction.apply(cloneUriComponents);
|
||||
if (!StringUtils.hasText(queryValue)) {
|
||||
return null;
|
||||
}
|
||||
UriComponents uriComponents = uriComponentsBuilder
|
||||
.queryParam(
|
||||
URLEncoder.encode("query", StandardCharsets.UTF_8),
|
||||
URLEncoder.encode(queryValue, StandardCharsets.UTF_8)
|
||||
).build(true);
|
||||
return uriComponents.toUri();
|
||||
}
|
||||
|
||||
/**
|
||||
* Request greptime and assign a value
|
||||
*
|
||||
* @param uri request URI
|
||||
* @param instanceValuesMap metrics data
|
||||
* @param valueConsumer consumer used for assigning values
|
||||
*/
|
||||
private void requestIntervalMetricAndPutValue(URI uri, Map<String, List<Value>> instanceValuesMap, BiConsumer<Value, String> valueConsumer) {
|
||||
if (uri == null) {
|
||||
return;
|
||||
}
|
||||
HttpEntity<Void> httpEntity = getHttpEntity();
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri,
|
||||
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (!responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.error("query interval metrics data from greptime failed. {}", responseEntity);
|
||||
return;
|
||||
}
|
||||
log.debug("query interval metrics data from greptime success. {}", uri);
|
||||
PromQlQueryContent body = responseEntity.getBody();
|
||||
if (body == null || body.getData() == null || body.getData().getResult() == null) {
|
||||
return;
|
||||
}
|
||||
List<PromQlQueryContent.ContentData.Content> contents = body.getData().getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() == null || content.getValues().isEmpty()) {
|
||||
continue;
|
||||
}
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
|
||||
if (valueList.size() == content.getValues().size()) {
|
||||
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
|
||||
Value value = valueList.get(timestampIndex);
|
||||
Object[] valueArr = content.getValues().get(timestampIndex);
|
||||
String avgValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
valueConsumer.accept(value, avgValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (this.greptimeDb != null) {
|
||||
this.greptimeDb.shutdownGracefully();
|
||||
this.greptimeDb = null;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveLogData(LogEntry logEntry) {
|
||||
if (!isServerAvailable()) {
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
// Create table schema
|
||||
TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(LOG_TABLE_NAME);
|
||||
tableSchemaBuilder.addTimestamp("time_unix_nano", DataType.TimestampNanosecond)
|
||||
.addField("observed_time_unix_nano", DataType.TimestampNanosecond)
|
||||
.addField("severity_number", DataType.Int32)
|
||||
.addField("severity_text", DataType.String)
|
||||
.addField("body", DataType.Json)
|
||||
.addField("trace_id", DataType.String)
|
||||
.addField("span_id", DataType.String)
|
||||
.addField("trace_flags", DataType.Int32)
|
||||
.addField("attributes", DataType.Json)
|
||||
.addField("resource", DataType.Json)
|
||||
.addField("instrumentation_scope", DataType.Json)
|
||||
.addField("dropped_attributes_count", DataType.Int32);
|
||||
|
||||
Table table = Table.from(tableSchemaBuilder.build());
|
||||
|
||||
// Convert LogEntry to table row
|
||||
Object[] values = new Object[] {
|
||||
logEntry.getTimeUnixNano() != null ? logEntry.getTimeUnixNano() : System.nanoTime(),
|
||||
logEntry.getObservedTimeUnixNano() != null ? logEntry.getObservedTimeUnixNano() : System.nanoTime(),
|
||||
logEntry.getSeverityNumber(),
|
||||
logEntry.getSeverityText(),
|
||||
JsonUtil.toJson(logEntry.getBody()),
|
||||
logEntry.getTraceId(),
|
||||
logEntry.getSpanId(),
|
||||
logEntry.getTraceFlags(),
|
||||
JsonUtil.toJson(logEntry.getAttributes()),
|
||||
JsonUtil.toJson(logEntry.getResource()),
|
||||
JsonUtil.toJson(logEntry.getInstrumentationScope()),
|
||||
logEntry.getDroppedAttributesCount()
|
||||
};
|
||||
|
||||
table.addRow(values);
|
||||
|
||||
// Write to GreptimeDB
|
||||
CompletableFuture<Result<WriteOk, Err>> writeFuture = greptimeDb.write(table);
|
||||
Result<WriteOk, Err> result = writeFuture.get(10, TimeUnit.SECONDS);
|
||||
|
||||
if (result.isOk()) {
|
||||
log.debug("[warehouse greptime-log] Write successful");
|
||||
} else {
|
||||
log.warn("[warehouse greptime-log] Write failed: {}", result.getErr());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse greptime-log] Error saving log entry", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<LogEntry> queryLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber,
|
||||
String severityText, String searchContent) {
|
||||
try {
|
||||
StringBuilder sql = new StringBuilder("SELECT * FROM ").append(LOG_TABLE_NAME);
|
||||
buildWhereConditions(sql, startTime, endTime, traceId, spanId, severityNumber, severityText, searchContent);
|
||||
sql.append(" ORDER BY time_unix_nano DESC");
|
||||
|
||||
List<Map<String, Object>> rows = greptimeSqlQueryExecutor.execute(sql.toString());
|
||||
return mapRowsToLogEntries(rows);
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse greptime-log] queryLogsByMultipleConditions error: {}", e.getMessage(), e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<LogEntry> queryLogsByMultipleConditionsWithPagination(Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber,
|
||||
String severityText, String searchContent,
|
||||
Integer offset, Integer limit) {
|
||||
try {
|
||||
StringBuilder sql = new StringBuilder("SELECT * FROM ").append(LOG_TABLE_NAME);
|
||||
buildWhereConditions(sql, startTime, endTime, traceId, spanId, severityNumber, severityText, searchContent);
|
||||
sql.append(" ORDER BY time_unix_nano DESC");
|
||||
|
||||
// Add pagination
|
||||
if (limit != null && limit > 0) {
|
||||
sql.append(" LIMIT ").append(limit);
|
||||
if (offset != null && offset > 0) {
|
||||
sql.append(" OFFSET ").append(offset);
|
||||
}
|
||||
}
|
||||
|
||||
List<Map<String, Object>> rows = greptimeSqlQueryExecutor.execute(sql.toString());
|
||||
return mapRowsToLogEntries(rows);
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse greptime-log] queryLogsByMultipleConditionsWithPagination error: {}", e.getMessage(), e);
|
||||
return List.of();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long countLogsByMultipleConditions(Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber,
|
||||
String severityText, String searchContent) {
|
||||
try {
|
||||
StringBuilder sql = new StringBuilder("SELECT COUNT(*) as count FROM ").append(LOG_TABLE_NAME);
|
||||
buildWhereConditions(sql, startTime, endTime, traceId, spanId, severityNumber, severityText, searchContent);
|
||||
|
||||
List<Map<String, Object>> rows = greptimeSqlQueryExecutor.execute(sql.toString());
|
||||
if (rows != null && !rows.isEmpty()) {
|
||||
Object countObj = rows.get(0).get("count");
|
||||
if (countObj instanceof Number) {
|
||||
return ((Number) countObj).longValue();
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse greptime-log] countLogsByMultipleConditions error: {}", e.getMessage(), e);
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
|
||||
private static long msToNs(Long ms) {
|
||||
return ms * 1_000_000L;
|
||||
}
|
||||
|
||||
private static String safeString(String input) {
|
||||
if (input == null) {
|
||||
return "";
|
||||
}
|
||||
return input.replace("'", "''");
|
||||
}
|
||||
|
||||
/**
|
||||
* build WHERE conditions
|
||||
* @param sql SQL builder
|
||||
* @param startTime start time
|
||||
* @param endTime end time
|
||||
* @param traceId trace id
|
||||
* @param spanId span id
|
||||
* @param severityNumber severity number
|
||||
*/
|
||||
private void buildWhereConditions(StringBuilder sql, Long startTime, Long endTime, String traceId,
|
||||
String spanId, Integer severityNumber, String severityText, String searchContent) {
|
||||
List<String> conditions = new ArrayList<>();
|
||||
|
||||
// Time range condition
|
||||
if (startTime != null && endTime != null) {
|
||||
conditions.add("time_unix_nano >= " + msToNs(startTime) + " AND time_unix_nano <= " + msToNs(endTime));
|
||||
}
|
||||
|
||||
// TraceId condition
|
||||
if (StringUtils.hasText(traceId)) {
|
||||
conditions.add("trace_id = '" + safeString(traceId) + "'");
|
||||
}
|
||||
|
||||
// SpanId condition
|
||||
if (StringUtils.hasText(spanId)) {
|
||||
conditions.add("span_id = '" + safeString(spanId) + "'");
|
||||
}
|
||||
|
||||
// Severity condition
|
||||
if (severityNumber != null) {
|
||||
conditions.add("severity_number = " + severityNumber);
|
||||
}
|
||||
|
||||
// SeverityText condition
|
||||
if (StringUtils.hasText(severityText)) {
|
||||
conditions.add("severity_text = '" + safeString(severityText) + "'");
|
||||
}
|
||||
|
||||
// Search content condition - search in body field
|
||||
if (StringUtils.hasText(searchContent)) {
|
||||
conditions.add("body LIKE '%" + safeString(searchContent) + "%'");
|
||||
}
|
||||
|
||||
// Add WHERE clause if there are conditions
|
||||
if (!conditions.isEmpty()) {
|
||||
sql.append(" WHERE ").append(String.join(" AND ", conditions));
|
||||
}
|
||||
}
|
||||
|
||||
private List<LogEntry> mapRowsToLogEntries(List<Map<String, Object>> rows) {
|
||||
List<LogEntry> list = new LinkedList<>();
|
||||
if (rows == null || rows.isEmpty()) {
|
||||
return list;
|
||||
}
|
||||
for (Map<String, Object> row : rows) {
|
||||
try {
|
||||
LogEntry.InstrumentationScope scope = null;
|
||||
Object scopeObj = row.get("instrumentation_scope");
|
||||
if (scopeObj instanceof String scopeStr && StringUtils.hasText(scopeStr)) {
|
||||
try {
|
||||
scope = JsonUtil.fromJson(scopeStr, LogEntry.InstrumentationScope.class);
|
||||
} catch (Exception ignore) {
|
||||
scope = null;
|
||||
}
|
||||
}
|
||||
|
||||
Object bodyObj = parseJsonMaybe(row.get("body"));
|
||||
Map<String, Object> attributes = castToMap(parseJsonMaybe(row.get("attributes")));
|
||||
Map<String, Object> resource = castToMap(parseJsonMaybe(row.get("resource")));
|
||||
|
||||
LogEntry entry = LogEntry.builder()
|
||||
.timeUnixNano(castToLong(row.get("time_unix_nano")))
|
||||
.observedTimeUnixNano(castToLong(row.get("observed_time_unix_nano")))
|
||||
.severityNumber(castToInteger(row.get("severity_number")))
|
||||
.severityText(castToString(row.get("severity_text")))
|
||||
.body(bodyObj)
|
||||
.traceId(castToString(row.get("trace_id")))
|
||||
.spanId(castToString(row.get("span_id")))
|
||||
.traceFlags(castToInteger(row.get("trace_flags")))
|
||||
.attributes(attributes)
|
||||
.resource(resource)
|
||||
.instrumentationScope(scope)
|
||||
.droppedAttributesCount(castToInteger(row.get("dropped_attributes_count")))
|
||||
.build();
|
||||
list.add(entry);
|
||||
} catch (Exception e) {
|
||||
log.warn("[warehouse greptime-log] map row to LogEntry error: {}", e.getMessage());
|
||||
}
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
private static Object parseJsonMaybe(Object value) {
|
||||
if (value == null) return null;
|
||||
if (value instanceof Map) return value;
|
||||
if (value instanceof String str) {
|
||||
String s = str.trim();
|
||||
if ((s.startsWith("{") && s.endsWith("}")) || (s.startsWith("[") && s.endsWith("]"))) {
|
||||
try {
|
||||
return JsonUtil.fromJson(s, Object.class);
|
||||
} catch (Exception e) {
|
||||
return s;
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Map<String, Object> castToMap(Object obj) {
|
||||
if (obj instanceof Map) {
|
||||
return (Map<String, Object>) obj;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
private static Long castToLong(Object obj) {
|
||||
if (obj == null) return null;
|
||||
if (obj instanceof Number n) return n.longValue();
|
||||
try {
|
||||
return Long.parseLong(String.valueOf(obj));
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static Integer castToInteger(Object obj) {
|
||||
if (obj == null) return null;
|
||||
if (obj instanceof Number n) return n.intValue();
|
||||
try {
|
||||
return Integer.parseInt(String.valueOf(obj));
|
||||
} catch (Exception e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static String castToString(Object obj) {
|
||||
return obj == null ? null : String.valueOf(obj);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean batchDeleteLogs(List<Long> timeUnixNanos) {
|
||||
if (!isServerAvailable() || timeUnixNanos == null || timeUnixNanos.isEmpty()) {
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
StringBuilder sql = new StringBuilder("DELETE FROM ").append(LOG_TABLE_NAME).append(" WHERE time_unix_nano IN (");
|
||||
sql.append(timeUnixNanos.stream()
|
||||
.filter(Objects::nonNull)
|
||||
.map(String::valueOf)
|
||||
.collect(Collectors.joining(", ")));
|
||||
sql.append(")");
|
||||
|
||||
greptimeSqlQueryExecutor.execute(sql.toString());
|
||||
log.info("[warehouse greptime-log] Batch delete executed successfully for {} logs", timeUnixNanos.size());
|
||||
return true;
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse greptime-log] batchDeleteLogs error: {}", e.getMessage(), e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveLogDataBatch(List<LogEntry> logEntries) {
|
||||
if (!isServerAvailable() || logEntries == null || logEntries.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
|
||||
int total = logEntries.size();
|
||||
for (int i = 0; i < total; i += LOG_BATCH_SIZE) {
|
||||
int end = Math.min(i + LOG_BATCH_SIZE, total);
|
||||
List<LogEntry> batch = logEntries.subList(i, end);
|
||||
doSaveLogBatch(batch);
|
||||
}
|
||||
}
|
||||
|
||||
private void doSaveLogBatch(List<LogEntry> logEntries) {
|
||||
try {
|
||||
TableSchema.Builder tableSchemaBuilder = TableSchema.newBuilder(LOG_TABLE_NAME);
|
||||
tableSchemaBuilder.addTimestamp("time_unix_nano", DataType.TimestampNanosecond)
|
||||
.addField("observed_time_unix_nano", DataType.TimestampNanosecond)
|
||||
.addField("severity_number", DataType.Int32)
|
||||
.addField("severity_text", DataType.String)
|
||||
.addField("body", DataType.Json)
|
||||
.addField("trace_id", DataType.String)
|
||||
.addField("span_id", DataType.String)
|
||||
.addField("trace_flags", DataType.Int32)
|
||||
.addField("attributes", DataType.Json)
|
||||
.addField("resource", DataType.Json)
|
||||
.addField("instrumentation_scope", DataType.Json)
|
||||
.addField("dropped_attributes_count", DataType.Int32);
|
||||
|
||||
Table table = Table.from(tableSchemaBuilder.build());
|
||||
|
||||
for (LogEntry logEntry : logEntries) {
|
||||
Object[] values = new Object[] {
|
||||
logEntry.getTimeUnixNano() != null ? logEntry.getTimeUnixNano() : System.nanoTime(),
|
||||
logEntry.getObservedTimeUnixNano() != null ? logEntry.getObservedTimeUnixNano() : System.nanoTime(),
|
||||
logEntry.getSeverityNumber(),
|
||||
logEntry.getSeverityText(),
|
||||
JsonUtil.toJson(logEntry.getBody()),
|
||||
logEntry.getTraceId(),
|
||||
logEntry.getSpanId(),
|
||||
logEntry.getTraceFlags(),
|
||||
JsonUtil.toJson(logEntry.getAttributes()),
|
||||
JsonUtil.toJson(logEntry.getResource()),
|
||||
JsonUtil.toJson(logEntry.getInstrumentationScope()),
|
||||
logEntry.getDroppedAttributesCount()
|
||||
};
|
||||
table.addRow(values);
|
||||
}
|
||||
|
||||
CompletableFuture<Result<WriteOk, Err>> writeFuture = greptimeDb.write(table);
|
||||
Result<WriteOk, Err> result = writeFuture.get(10, TimeUnit.SECONDS);
|
||||
|
||||
if (result.isOk()) {
|
||||
log.debug("[warehouse greptime-log] Batch write {} logs successful", logEntries.size());
|
||||
} else {
|
||||
log.warn("[warehouse greptime-log] Batch write failed: {}", result.getErr());
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse greptime-log] Error saving log entries batch", e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* GrepTimeDB configuration information
|
||||
*/
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.GREPTIME)
|
||||
public record GreptimeProperties(@DefaultValue("false") boolean enabled,
|
||||
@DefaultValue("127.0.0.1:4001") String grpcEndpoints, @DefaultValue("http://127.0.0.1:4000") String httpEndpoint,
|
||||
@DefaultValue("public") String database, String username, String password) {
|
||||
}
|
||||
+93
@@ -0,0 +1,93 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
|
||||
|
||||
import com.fasterxml.jackson.annotation.JsonProperty;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
/**
|
||||
* GreptimeDB SQL query content
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class GreptimeSqlQueryContent {
|
||||
|
||||
private Integer code;
|
||||
|
||||
private List<Output> output;
|
||||
|
||||
@JsonProperty("execution_time_ms")
|
||||
private Long executionTimeMs;
|
||||
|
||||
/**
|
||||
* Represents the output of a GreptimeDB SQL query.
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static class Output {
|
||||
|
||||
private Records records;
|
||||
|
||||
/**
|
||||
* Represents the records returned by a GreptimeDB SQL query.
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static class Records {
|
||||
|
||||
private Schema schema;
|
||||
|
||||
private List<List<Object>> rows;
|
||||
|
||||
/**
|
||||
* Represents the schema of the records returned by a GreptimeDB SQL query.
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static class Schema {
|
||||
|
||||
@JsonProperty("column_schemas")
|
||||
private List<ColumnSchema> columnSchemas;
|
||||
|
||||
/**
|
||||
* Represents a column schema in the records returned by a GreptimeDB SQL query.
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static class ColumnSchema {
|
||||
|
||||
private String name;
|
||||
|
||||
@JsonProperty("data_type")
|
||||
private String dataType;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+364
@@ -0,0 +1,364 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.influxdb;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import javax.net.ssl.HostnameVerifier;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import javax.net.ssl.TrustManager;
|
||||
import javax.net.ssl.X509TrustManager;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.ConnectionPool;
|
||||
import okhttp3.OkHttpClient;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.constants.NetworkConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.apache.http.ssl.SSLContexts;
|
||||
import org.influxdb.InfluxDB;
|
||||
import org.influxdb.InfluxDBFactory;
|
||||
import org.influxdb.dto.BatchPoints;
|
||||
import org.influxdb.dto.Point;
|
||||
import org.influxdb.dto.Query;
|
||||
import org.influxdb.dto.QueryResult;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* HistoryInfluxdbDataStorage class
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.influxdb", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class InfluxdbDataStorage extends AbstractHistoryDataStorage {
|
||||
|
||||
private static final String DATABASE = "hertzbeat";
|
||||
|
||||
private static final String SHOW_DATABASE = "SHOW DATABASES";
|
||||
|
||||
private static final String CREATE_DATABASE = "CREATE DATABASE %s";
|
||||
|
||||
private static final String QUERY_HISTORY_SQL = "SELECT metric_labels, \"%s\" FROM \"%s\" WHERE time >= now() - %s order by time desc";
|
||||
|
||||
private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL =
|
||||
"SELECT FIRST(\"%s\"), MEAN(\"%s\"), MAX(\"%s\"), MIN(\"%s\") FROM \"%s\" WHERE metric_labels = '%s' and time >= now() - %s GROUP BY time(4h)";
|
||||
private static final String QUERY_INSTANCE_SQL = "show tag values from \"%s\" with key = \"metric_labels\"";
|
||||
|
||||
private static final String CREATE_RETENTION_POLICY = "CREATE RETENTION POLICY \"%s_retention\" ON \"%s\" DURATION %s REPLICATION %d DEFAULT";
|
||||
|
||||
private InfluxDB influxDb;
|
||||
|
||||
public InfluxdbDataStorage(InfluxdbProperties influxdbProperties) {
|
||||
this.initInfluxDb(influxdbProperties);
|
||||
}
|
||||
|
||||
public void initInfluxDb(InfluxdbProperties influxdbProperties) {
|
||||
|
||||
var client = new OkHttpClient.Builder()
|
||||
.readTimeout(NetworkConstants.HttpClientConstants.READ_TIME_OUT, TimeUnit.SECONDS)
|
||||
.writeTimeout(NetworkConstants.HttpClientConstants.WRITE_TIME_OUT, TimeUnit.SECONDS)
|
||||
.connectTimeout(NetworkConstants.HttpClientConstants.CONNECT_TIME_OUT, TimeUnit.SECONDS)
|
||||
.connectionPool(new ConnectionPool(
|
||||
NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS,
|
||||
NetworkConstants.HttpClientConstants.KEEP_ALIVE_TIMEOUT,
|
||||
TimeUnit.SECONDS)
|
||||
).sslSocketFactory(defaultSslSocketFactory(), defaultTrustManager())
|
||||
.hostnameVerifier(noopHostnameVerifier())
|
||||
.retryOnConnectionFailure(true);
|
||||
|
||||
this.influxDb = InfluxDBFactory.connect(influxdbProperties.serverUrl(), influxdbProperties.username(), influxdbProperties.password(), client);
|
||||
// Close it if your application is terminating, or you are not using it anymore.
|
||||
Runtime.getRuntime().addShutdownHook(new Thread(influxDb::close));
|
||||
|
||||
this.serverAvailable = this.createDatabase(influxdbProperties);
|
||||
}
|
||||
|
||||
private boolean createDatabase(InfluxdbProperties influxdbProperties) {
|
||||
QueryResult queryResult = this.influxDb.query(new Query(SHOW_DATABASE));
|
||||
|
||||
if (queryResult.hasError()) {
|
||||
log.error("show databases in influxdb error, msg: {}", queryResult.getError());
|
||||
return false;
|
||||
}
|
||||
|
||||
for (QueryResult.Result result : queryResult.getResults()) {
|
||||
for (QueryResult.Series series : result.getSeries()) {
|
||||
for (List<Object> values : series.getValues()) {
|
||||
if (values.contains(DATABASE)) {
|
||||
// database exists
|
||||
return true;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// create the database
|
||||
String createDatabaseSql = String.format(CREATE_DATABASE, DATABASE);
|
||||
QueryResult createDatabaseResult = this.influxDb.query(new Query(createDatabaseSql));
|
||||
if (createDatabaseResult.hasError()) {
|
||||
log.error("create database {} in influxdb error, msg: {}", DATABASE, createDatabaseResult.getError());
|
||||
return false;
|
||||
}
|
||||
// set the expiration time
|
||||
String createRetentionPolicySql = String.format(CREATE_RETENTION_POLICY, DATABASE, DATABASE,
|
||||
influxdbProperties.expireTime(), influxdbProperties.replication());
|
||||
QueryResult createRetentionPolicySqlResult = this.influxDb.query(new Query(createRetentionPolicySql));
|
||||
if (createRetentionPolicySqlResult.hasError()) {
|
||||
log.error("create retention policy in influxdb error, msg: {}", createDatabaseResult.getError());
|
||||
return false;
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
|
||||
return;
|
||||
}
|
||||
if (metricsData.getValues().isEmpty()) {
|
||||
log.info("[warehouse influxdb] flush metrics data {} is null, ignore.", metricsData.getInstance());
|
||||
return;
|
||||
}
|
||||
|
||||
String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance());
|
||||
List<Point> points = new ArrayList<>();
|
||||
|
||||
try {
|
||||
RowWrapper rowWrapper = metricsData.readRow();
|
||||
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
Point.Builder builder = Point.measurement(table);
|
||||
builder.time(metricsData.getTime(), TimeUnit.MILLISECONDS);
|
||||
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
|
||||
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
if (CommonConstants.NULL_VALUE.equals(cell.getValue())) {
|
||||
builder.addField(cell.getField().getName(), "");
|
||||
return;
|
||||
}
|
||||
|
||||
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
|
||||
if (type == CommonConstants.TYPE_NUMBER) {
|
||||
builder.addField(cell.getField().getName(), Double.parseDouble(cell.getValue()));
|
||||
} else if (type == CommonConstants.TYPE_STRING) {
|
||||
builder.addField(cell.getField().getName(), cell.getValue());
|
||||
}
|
||||
|
||||
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL)) {
|
||||
labels.put(cell.getField().getName(), cell.getValue());
|
||||
}
|
||||
});
|
||||
builder.tag("metric_labels", JsonUtil.toJson(labels));
|
||||
points.add(builder.build());
|
||||
}
|
||||
|
||||
BatchPoints.Builder builder = BatchPoints.database(DATABASE);
|
||||
builder.points(points);
|
||||
this.influxDb.write(builder.build());
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse influxdb]--Error: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
|
||||
String table = this.generateTable(app, metrics, instance);
|
||||
String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history);
|
||||
Map<String, List<Value>> instanceValueMap = new HashMap<>(8);
|
||||
try {
|
||||
QueryResult selectResult = this.influxDb.query(new Query(selectSql, DATABASE), TimeUnit.MILLISECONDS);
|
||||
for (QueryResult.Result result : selectResult.getResults()) {
|
||||
if (result.getSeries() == null) {
|
||||
continue;
|
||||
}
|
||||
for (QueryResult.Series series : result.getSeries()) {
|
||||
for (List<Object> value : series.getValues()) {
|
||||
long time = this.parseTimeToMillis(value.get(0));
|
||||
String instanceValue = value.get(1) == null ? "" : String.valueOf(value.get(1));
|
||||
String strValue = value.get(2) == null ? null : this.parseDoubleValue(value.get(2).toString());
|
||||
if (strValue == null) {
|
||||
continue;
|
||||
}
|
||||
List<Value> valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
|
||||
valueList.add(new Value(strValue, time));
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("select history metric data in influxdb error, sql:{}, msg: {}", selectSql, e.getMessage());
|
||||
}
|
||||
return instanceValueMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) {
|
||||
String table = this.generateTable(app, metrics, instance);
|
||||
Map<String, List<Value>> instanceValueMap = new HashMap<>(8);
|
||||
Set<String> instances = new HashSet<>(8);
|
||||
// query all metric_labels
|
||||
String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table);
|
||||
QueryResult instanceQueryResult = this.influxDb.query(new Query(queryInstanceSql, DATABASE), TimeUnit.MILLISECONDS);
|
||||
for (QueryResult.Result result : instanceQueryResult.getResults()) {
|
||||
if (result.getSeries() == null) {
|
||||
continue;
|
||||
}
|
||||
for (QueryResult.Series series : result.getSeries()) {
|
||||
for (List<Object> value : series.getValues()) {
|
||||
if (value != null && value.get(1) != null) {
|
||||
instances.add(value.get(1).toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
history = history.toLowerCase();
|
||||
if (instances.isEmpty()) {
|
||||
instances.add("");
|
||||
}
|
||||
for (String instanceValue : instances) {
|
||||
String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, metric, metric, metric, table, instanceValue, history);
|
||||
QueryResult selectResult = this.influxDb.query(new Query(selectSql, DATABASE), TimeUnit.MILLISECONDS);
|
||||
for (QueryResult.Result result : selectResult.getResults()) {
|
||||
if (result.getSeries() == null) {
|
||||
continue;
|
||||
}
|
||||
for (QueryResult.Series series : result.getSeries()) {
|
||||
for (List<Object> value : series.getValues()) {
|
||||
Value.ValueBuilder valueBuilder = Value.builder();
|
||||
long time = this.parseTimeToMillis(value.get(0));
|
||||
valueBuilder.time(time);
|
||||
|
||||
if (value.get(1) != null) {
|
||||
valueBuilder.origin(this.parseDoubleValue(value.get(1).toString()));
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
if (value.get(2) != null) {
|
||||
valueBuilder.mean(this.parseDoubleValue(value.get(2).toString()));
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
if (value.get(3) != null) {
|
||||
valueBuilder.max(this.parseDoubleValue(value.get(3).toString()));
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
if (value.get(4) != null) {
|
||||
valueBuilder.min(this.parseDoubleValue(value.get(4).toString()));
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
List<Value> valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
|
||||
valueList.add(valueBuilder.build());
|
||||
}
|
||||
}
|
||||
}
|
||||
List<Value> instanceValueList = instanceValueMap.get(instanceValue);
|
||||
if (instanceValueList == null || instanceValueList.isEmpty()) {
|
||||
instanceValueMap.remove(instanceValue);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("select history interval metric data in influxdb error, msg: {}", e.getMessage());
|
||||
}
|
||||
return instanceValueMap;
|
||||
}
|
||||
|
||||
private String generateTable(String app, String metrics, String instance) {
|
||||
if (instance.contains(".") || instance.contains(":") || instance.contains("[")) {
|
||||
instance = instance.replace(".", "_")
|
||||
.replace(":", "_")
|
||||
.replace("[", "_")
|
||||
.replace("]", "_");
|
||||
}
|
||||
return app + "_" + metrics + "_" + instance;
|
||||
}
|
||||
|
||||
private long parseTimeToMillis(Object time) {
|
||||
if (time == null) {
|
||||
return 0;
|
||||
}
|
||||
Double doubleTime = (Double) time;
|
||||
return doubleTime.longValue();
|
||||
}
|
||||
|
||||
private String parseDoubleValue(String value) {
|
||||
return (new BigDecimal(value)).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
}
|
||||
|
||||
private static X509TrustManager defaultTrustManager() {
|
||||
return new X509TrustManager() {
|
||||
@Override
|
||||
public X509Certificate[] getAcceptedIssuers() {
|
||||
return new X509Certificate[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkClientTrusted(X509Certificate[] certs, String authType) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkServerTrusted(X509Certificate[] certs, String authType) {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static SSLSocketFactory defaultSslSocketFactory() {
|
||||
try {
|
||||
SSLContext sslContext = SSLContexts.createDefault();
|
||||
sslContext.init(null, new TrustManager[]{
|
||||
defaultTrustManager()
|
||||
}, new SecureRandom());
|
||||
return sslContext.getSocketFactory();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private static HostnameVerifier noopHostnameVerifier() {
|
||||
return (s, sslSession) -> true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
if (this.influxDb != null) {
|
||||
this.influxDb.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.influxdb;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* Influxdb configuration information
|
||||
*/
|
||||
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.INFLUXDB)
|
||||
public record InfluxdbProperties(@DefaultValue("false") boolean enabled,
|
||||
String serverUrl,
|
||||
String username,
|
||||
String password,
|
||||
@DefaultValue("30d") String expireTime,
|
||||
@DefaultValue("1") int replication) {
|
||||
}
|
||||
+469
@@ -0,0 +1,469 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.iotdb;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.apache.iotdb.rpc.IoTDBConnectionException;
|
||||
import org.apache.iotdb.rpc.StatementExecutionException;
|
||||
import org.apache.iotdb.session.pool.SessionDataSetWrapper;
|
||||
import org.apache.iotdb.session.pool.SessionPool;
|
||||
import org.apache.iotdb.tsfile.file.metadata.enums.TSDataType;
|
||||
import org.apache.iotdb.tsfile.read.common.RowRecord;
|
||||
import org.apache.iotdb.tsfile.write.record.Tablet;
|
||||
import org.apache.iotdb.tsfile.write.schema.MeasurementSchema;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* IoTDB data storage
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.iot-db", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class IotDbDataStorage extends AbstractHistoryDataStorage {
|
||||
private static final String BACK_QUOTE = "`";
|
||||
/**
|
||||
* set ttl never expire
|
||||
*/
|
||||
private static final String NEVER_EXPIRE = "-1";
|
||||
|
||||
/**
|
||||
* storage group
|
||||
*/
|
||||
private static final String STORAGE_GROUP = "root.hertzbeat";
|
||||
|
||||
private static final String SHOW_DATABASE = "show databases %s";
|
||||
|
||||
private static final String CREATE_DATABASE = "create database %s";
|
||||
|
||||
private static final String SET_TTL = "set ttl to %s %s";
|
||||
|
||||
private static final String CANCEL_TTL = "unset ttl to %s";
|
||||
|
||||
private static final String SHOW_DEVICES = "SHOW DEVICES %s";
|
||||
|
||||
private static final String SHOW_STORAGE_GROUP = "show storage group";
|
||||
|
||||
private static final String QUERY_HISTORY_SQL =
|
||||
"SELECT %s FROM %s WHERE Time >= now() - %s order by Time desc";
|
||||
private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL =
|
||||
"SELECT FIRST_VALUE(%s), AVG(%s), MIN_VALUE(%s), MAX_VALUE(%s) FROM %s GROUP BY ([now() - %s, now()), 4h)";
|
||||
|
||||
private SessionPool sessionPool;
|
||||
|
||||
private long queryTimeoutInMs;
|
||||
|
||||
public IotDbDataStorage(IotDbProperties iotDbProperties) {
|
||||
this.serverAvailable = this.initIotDbSession(iotDbProperties);
|
||||
}
|
||||
|
||||
private boolean initIotDbSession(IotDbProperties properties) {
|
||||
SessionPool.Builder builder = new SessionPool.Builder();
|
||||
builder.host(properties.host());
|
||||
if (properties.rpcPort() != null) {
|
||||
builder.port(properties.rpcPort());
|
||||
}
|
||||
if (properties.username() != null) {
|
||||
builder.user(properties.username());
|
||||
}
|
||||
if (properties.password() != null) {
|
||||
builder.password(properties.password());
|
||||
}
|
||||
if (properties.nodeUrls() != null && !properties.nodeUrls().isEmpty()) {
|
||||
builder.nodeUrls(properties.nodeUrls());
|
||||
}
|
||||
if (properties.zoneId() != null) {
|
||||
builder.zoneId(properties.zoneId());
|
||||
}
|
||||
this.queryTimeoutInMs = properties.queryTimeoutInMs();
|
||||
this.sessionPool = builder.build();
|
||||
boolean available = checkConnection();
|
||||
if (!available) {
|
||||
log.error("IotDB session pool init error with check connection");
|
||||
return false;
|
||||
}
|
||||
available = this.createDatabase();
|
||||
if (!available) {
|
||||
log.error("IotDB session pool init error with create database");
|
||||
return false;
|
||||
}
|
||||
this.initTtl(properties.expireTime());
|
||||
log.info("IotDB session pool init success");
|
||||
return true;
|
||||
}
|
||||
|
||||
private boolean checkConnection() {
|
||||
try {
|
||||
this.sessionPool.executeNonQueryStatement(SHOW_STORAGE_GROUP);
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private boolean createDatabase() {
|
||||
SessionDataSetWrapper dataSet = null;
|
||||
try {
|
||||
// if root.hertzbeat database not exist, create database
|
||||
String showDatabaseSql = String.format(SHOW_DATABASE, STORAGE_GROUP);
|
||||
dataSet = this.sessionPool.executeQueryStatement(showDatabaseSql);
|
||||
if (!dataSet.hasNext()) {
|
||||
String createDatabaseSql = String.format(CREATE_DATABASE, STORAGE_GROUP);
|
||||
this.sessionPool.executeNonQueryStatement(createDatabaseSql);
|
||||
}
|
||||
} catch (IoTDBConnectionException | StatementExecutionException e) {
|
||||
log.error("create database error, error: {}", e.getMessage());
|
||||
return false;
|
||||
} finally {
|
||||
if (dataSet != null) {
|
||||
this.sessionPool.closeResultSet(dataSet);
|
||||
}
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
private void initTtl(String expireTime) {
|
||||
if (expireTime == null || expireTime.isEmpty()) {
|
||||
return;
|
||||
}
|
||||
try {
|
||||
if (NEVER_EXPIRE.equals(expireTime)) {
|
||||
// DELETE TTL that might already exist
|
||||
String cancelTtlSql = String.format(CANCEL_TTL, STORAGE_GROUP);
|
||||
this.sessionPool.executeNonQueryStatement(cancelTtlSql);
|
||||
} else {
|
||||
String sstTtlSql = String.format(SET_TTL, STORAGE_GROUP, expireTime);
|
||||
this.sessionPool.executeNonQueryStatement(sstTtlSql);
|
||||
}
|
||||
} catch (IoTDBConnectionException | StatementExecutionException e) {
|
||||
// Failure does not affect the primary business
|
||||
log.error("IoTDB init ttl error, expireTime: {}, error: {}", expireTime, e.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
|
||||
return;
|
||||
}
|
||||
if (metricsData.getValues().isEmpty()) {
|
||||
log.info("[warehouse iotdb] flush metrics data {} is null, ignore.", metricsData.getInstance());
|
||||
return;
|
||||
}
|
||||
List<MeasurementSchema> schemaList = new ArrayList<>();
|
||||
Map<String, Tablet> tabletMap = Maps.newHashMapWithExpectedSize(8);
|
||||
|
||||
// todo Measurement schema is a data structure that is generated on the client side, and encoding and compression have no effect
|
||||
try {
|
||||
metricsData.getFields().forEach(field -> {
|
||||
MeasurementSchema schema = new MeasurementSchema();
|
||||
schema.setMeasurementId(field.getName());
|
||||
byte type = (byte) field.getType();
|
||||
|
||||
// handle field type
|
||||
if (type == CommonConstants.TYPE_NUMBER) {
|
||||
schema.setType(TSDataType.DOUBLE);
|
||||
} else if (type == CommonConstants.TYPE_STRING) {
|
||||
schema.setType(TSDataType.TEXT);
|
||||
}
|
||||
schemaList.add(schema);
|
||||
});
|
||||
|
||||
long now = System.currentTimeMillis();
|
||||
RowWrapper rowWrapper = metricsData.readRow();
|
||||
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
|
||||
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL) && !CommonConstants.NULL_VALUE.equals(cell.getValue())) {
|
||||
labels.put(cell.getField().getName(), cell.getValue());
|
||||
}
|
||||
});
|
||||
|
||||
|
||||
String label = JsonUtil.toJson(labels);
|
||||
String deviceId = getDeviceId(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance(), label, true);
|
||||
if (tabletMap.containsKey(label)) {
|
||||
// Avoid Time repeats
|
||||
now++;
|
||||
} else {
|
||||
tabletMap.put(label, new Tablet(deviceId, schemaList));
|
||||
}
|
||||
Tablet tablet = tabletMap.get(label);
|
||||
int rowIndex = tablet.rowSize++;
|
||||
tablet.addTimestamp(rowIndex, now);
|
||||
|
||||
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
if (CommonConstants.NULL_VALUE.equals(cell.getValue())) {
|
||||
tablet.addValue(cell.getField().getName(), rowIndex, null);
|
||||
return;
|
||||
}
|
||||
|
||||
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
|
||||
if (type == CommonConstants.TYPE_NUMBER) {
|
||||
tablet.addValue(cell.getField().getName(), rowIndex, Double.parseDouble(cell.getValue()));
|
||||
} else if (type == CommonConstants.TYPE_STRING) {
|
||||
tablet.addValue(cell.getField().getName(), rowIndex, cell.getValue());
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
for (Tablet tablet : tabletMap.values()) {
|
||||
this.sessionPool.insertTablet(tablet, true);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error(e.getMessage(), e);
|
||||
} finally {
|
||||
for (Tablet tablet : tabletMap.values()) {
|
||||
tablet.reset();
|
||||
}
|
||||
tabletMap.clear();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric,
|
||||
String history) {
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
if (!isServerAvailable()) {
|
||||
log.error("""
|
||||
|
||||
\t---------------IotDb Init Failed---------------
|
||||
\t--------------Please Config IotDb--------------
|
||||
\t----------Can Not Use Metric History Now----------
|
||||
""");
|
||||
return instanceValuesMap;
|
||||
}
|
||||
String deviceId = getDeviceId(app, metrics, instance, null, true);
|
||||
String selectSql = "";
|
||||
try {
|
||||
// query all devices
|
||||
List<String> devices = queryAllDevices(deviceId);
|
||||
if (devices.isEmpty()) {
|
||||
log.warn("no iot device found for deviceId: {}", deviceId);
|
||||
return instanceValuesMap;
|
||||
}
|
||||
for (String device : devices) {
|
||||
selectSql = String.format(QUERY_HISTORY_SQL, addQuote(metric), device, history);
|
||||
String labels = extractLabelsFromDevice(device, deviceId);
|
||||
handleHistorySelect(selectSql, labels, instanceValuesMap);
|
||||
}
|
||||
} catch (StatementExecutionException | IoTDBConnectionException e) {
|
||||
log.error("select error history sql: {}", selectSql);
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
private void handleHistorySelect(String selectSql, String labels, Map<String, List<Value>> instanceValuesMap)
|
||||
throws IoTDBConnectionException, StatementExecutionException {
|
||||
SessionDataSetWrapper dataSet = null;
|
||||
try {
|
||||
dataSet = this.sessionPool.executeQueryStatement(selectSql, this.queryTimeoutInMs);
|
||||
log.debug("iot select sql: {}", selectSql);
|
||||
while (dataSet.hasNext()) {
|
||||
RowRecord rowRecord = dataSet.next();
|
||||
long timestamp = rowRecord.getTimestamp();
|
||||
double value = rowRecord.getFields().get(0).getDoubleV();
|
||||
String strValue = BigDecimal.valueOf(value).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labels, k -> new LinkedList<>());
|
||||
valueList.add(new Value(strValue, timestamp));
|
||||
}
|
||||
} finally {
|
||||
if (dataSet != null) {
|
||||
// need to close the result set! ! ! otherwise it will cause server-side heap
|
||||
this.sessionPool.closeResultSet(dataSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
|
||||
String metric, String history) {
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
if (!isServerAvailable()) {
|
||||
log.error("""
|
||||
|
||||
\t---------------IotDb Init Failed---------------
|
||||
\t--------------Please Config IotDb--------------
|
||||
\t----------Can Not Use Metric History Now----------
|
||||
""");
|
||||
return instanceValuesMap;
|
||||
}
|
||||
String deviceId = getDeviceId(app, metrics, instance, null, true);
|
||||
String selectSql;
|
||||
List<String> devices = queryAllDevices(deviceId);
|
||||
if (devices.isEmpty()) {
|
||||
log.warn("no iot device found for deviceId: {}", deviceId);
|
||||
return instanceValuesMap;
|
||||
} else {
|
||||
for (String device : devices) {
|
||||
selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL,
|
||||
addQuote(metric), addQuote(metric), addQuote(metric), addQuote(metric), device, history);
|
||||
String labels = extractLabelsFromDevice(device, deviceId);
|
||||
handleHistoryIntervalSelect(selectSql, labels, instanceValuesMap);
|
||||
}
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
private void handleHistoryIntervalSelect(String selectSql, String instance,
|
||||
Map<String, List<Value>> instanceValuesMap) {
|
||||
SessionDataSetWrapper dataSet = null;
|
||||
try {
|
||||
dataSet = this.sessionPool.executeQueryStatement(selectSql, this.queryTimeoutInMs);
|
||||
log.debug("iot select sql: {}", selectSql);
|
||||
while (dataSet.hasNext()) {
|
||||
RowRecord rowRecord = dataSet.next();
|
||||
if (rowRecord.hasNullField()) {
|
||||
continue;
|
||||
}
|
||||
long timestamp = rowRecord.getTimestamp();
|
||||
double origin = rowRecord.getFields().get(0).getDoubleV();
|
||||
String originStr = BigDecimal.valueOf(origin).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
double avg = rowRecord.getFields().get(1).getDoubleV();
|
||||
String avgStr = BigDecimal.valueOf(avg).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
double min = rowRecord.getFields().get(2).getDoubleV();
|
||||
String minStr = BigDecimal.valueOf(min).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
double max = rowRecord.getFields().get(3).getDoubleV();
|
||||
String maxStr = BigDecimal.valueOf(max).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
Value value = Value.builder()
|
||||
.origin(originStr).mean(avgStr)
|
||||
.min(minStr).max(maxStr)
|
||||
.time(timestamp)
|
||||
.build();
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(instance, k -> new LinkedList<>());
|
||||
valueList.add(value);
|
||||
}
|
||||
} catch (StatementExecutionException | IoTDBConnectionException e) {
|
||||
log.error("select error history interval sql: {}", selectSql);
|
||||
log.error(e.getMessage(), e);
|
||||
} finally {
|
||||
if (dataSet != null) {
|
||||
// need to close the result set! ! ! otherwise it will cause server-side heap
|
||||
this.sessionPool.closeResultSet(dataSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Query all devices by deviceId
|
||||
*
|
||||
* @param deviceId deviceId
|
||||
*/
|
||||
private List<String> queryAllDevices(String deviceId) {
|
||||
List<String> devices = new ArrayList<>();
|
||||
List<String> sqls = new ArrayList<>();
|
||||
sqls.add(String.format(SHOW_DEVICES, deviceId));
|
||||
sqls.add(String.format(SHOW_DEVICES, deviceId + ".*"));
|
||||
|
||||
for (String sql : sqls) {
|
||||
SessionDataSetWrapper dataSet = null;
|
||||
try {
|
||||
dataSet = this.sessionPool.executeQueryStatement(sql, this.queryTimeoutInMs);
|
||||
while (dataSet.hasNext()) {
|
||||
RowRecord rowRecord = dataSet.next();
|
||||
devices.add(rowRecord.getFields().get(0).getStringValue());
|
||||
}
|
||||
} catch (StatementExecutionException | IoTDBConnectionException e) {
|
||||
log.error("query show devices sql error. sql: {}", sql);
|
||||
log.error(e.getMessage(), e);
|
||||
} finally {
|
||||
if (dataSet != null) {
|
||||
// need to close the result set! ! ! otherwise it will cause server-side heap
|
||||
this.sessionPool.closeResultSet(dataSet);
|
||||
}
|
||||
}
|
||||
}
|
||||
return devices;
|
||||
}
|
||||
|
||||
/**
|
||||
* use ${group}.${app}.${metrics}.${monitor}.${metric_labels} to get device id if labels exist
|
||||
* otherwise use ${group}.${app}.${metrics}.${monitor}
|
||||
* Use ${group}.${app}.${metrics}.${monitor}.* to get all instance data when you tend to query
|
||||
*/
|
||||
private String getDeviceId(String app, String metrics, String instance, String labels, boolean useQuote) {
|
||||
if (instance.contains(".") || instance.contains(":") || instance.contains("[")) {
|
||||
instance = instance.replace(".", "_")
|
||||
.replace(":", "_")
|
||||
.replace("[", "_")
|
||||
.replace("]", "_");
|
||||
}
|
||||
String deviceId = STORAGE_GROUP + "."
|
||||
+ (useQuote ? addQuote(app) : app) + "."
|
||||
+ (useQuote ? addQuote(metrics) : metrics) + "."
|
||||
+ addQuote(instance);
|
||||
if (labels != null && !labels.isEmpty() && !labels.equals(CommonConstants.NULL_VALUE) && !"{}".equals(labels)) {
|
||||
deviceId += "." + addQuote(labels);
|
||||
}
|
||||
return deviceId;
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract labels from device path
|
||||
*/
|
||||
private String extractLabelsFromDevice(String device, String baseDeviceId) {
|
||||
if (device.length() > baseDeviceId.length() + 1) {
|
||||
String labelsPart = device.substring(baseDeviceId.length() + 1);
|
||||
return labelsPart.replace("`", "");
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
/**
|
||||
* add quote,prevents keyword errors during queries(eg: nodes)
|
||||
*/
|
||||
private String addQuote(String text) {
|
||||
if (text == null || text.isEmpty() || (text.startsWith(BACK_QUOTE) && text.endsWith(BACK_QUOTE))) {
|
||||
return text;
|
||||
}
|
||||
text = text.replace("*", "-");
|
||||
text = String.format("`%s`", text);
|
||||
return text;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (this.sessionPool != null) {
|
||||
this.sessionPool.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+53
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.iotdb;
|
||||
|
||||
import java.time.ZoneId;
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* IotDB configuration information
|
||||
* @param enabled Whether the iotDB data store is enabled
|
||||
* @param host iotDB host
|
||||
* @param expireTime save data expire time(ms),-1 means it never expires Data storage time (unit: ms,-1 means never expire)
|
||||
* Note: Why is String used here instead of Long? At present, the set ttl of IoTDB only supports milliseconds as a unit,
|
||||
* and other units may be added later, so the String type is used for compatibility Data storage time (unit: ms, -1 means never expires)
|
||||
* Note: Why use String instead of Long here? Currently, IoTDB's set ttl only supports milliseconds as the unit.
|
||||
* Other units may be added later. In order to be compatible with the future, the String type is used.
|
||||
*/
|
||||
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.IOT_DB)
|
||||
public record IotDbProperties(@DefaultValue("false") boolean enabled,
|
||||
@DefaultValue("127.0.0.1") String host,
|
||||
@DefaultValue("6667") Integer rpcPort,
|
||||
String username,
|
||||
String password,
|
||||
List<String> nodeUrls,
|
||||
ZoneId zoneId,
|
||||
long queryTimeoutInMs,
|
||||
String expireTime) {
|
||||
}
|
||||
+414
@@ -0,0 +1,414 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.questdb;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.net.URLEncoder;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.security.SecureRandom;
|
||||
import java.security.cert.X509Certificate;
|
||||
import java.time.Instant;
|
||||
import java.util.Base64;
|
||||
import java.util.HashMap;
|
||||
import java.util.HashSet;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Set;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import javax.net.ssl.HostnameVerifier;
|
||||
import javax.net.ssl.SSLContext;
|
||||
import javax.net.ssl.SSLSocketFactory;
|
||||
import javax.net.ssl.TrustManager;
|
||||
import javax.net.ssl.X509TrustManager;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import io.questdb.client.Sender;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import okhttp3.ConnectionPool;
|
||||
import okhttp3.OkHttpClient;
|
||||
import okhttp3.Request;
|
||||
import okhttp3.Response;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.constants.NetworkConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.apache.http.ssl.SSLContexts;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* HistoryQuestdbDataStorage class
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.questdb", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class QuestdbDataStorage extends AbstractHistoryDataStorage {
|
||||
|
||||
private static final String QUERY_HISTORY_SQL = "SELECT timestamp AS ts, metric_labels, \"%s\" AS value FROM \"%s\" WHERE timestamp >= %s ORDER BY timestamp DESC";
|
||||
|
||||
private static final String QUERY_HISTORY_SQL_WITH_INSTANCE =
|
||||
"SELECT timestamp AS ts, metric_labels, \"%s\" AS value FROM \"%s\" WHERE metric_labels = '%s' AND timestamp >= %s ORDER BY timestamp DESC";
|
||||
|
||||
private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL =
|
||||
"SELECT timestamp AS ts, first(\"%s\") AS origin, avg(\"%s\") AS mean, max(\"%s\") AS max, min(\"%s\") AS min FROM \"%s\" WHERE metric_labels = '%s' AND timestamp >= %s SAMPLE BY 4h";
|
||||
|
||||
private static final String QUERY_INSTANCE_SQL = "SELECT DISTINCT metric_labels FROM \"%s\"";
|
||||
|
||||
private Sender sender;
|
||||
|
||||
private OkHttpClient client;
|
||||
|
||||
private String queryBaseUrl;
|
||||
|
||||
private QuestdbProperties questdbProperties;
|
||||
|
||||
public QuestdbDataStorage(QuestdbProperties questdbProperties) {
|
||||
this.questdbProperties = questdbProperties;
|
||||
this.initQuestDb(questdbProperties);
|
||||
}
|
||||
|
||||
public void initQuestDb(QuestdbProperties questdbProperties) {
|
||||
String ilpAddress = questdbProperties.url(); // e.g., "localhost:9009"
|
||||
this.sender = Sender.builder(Sender.Transport.HTTP)
|
||||
.address(ilpAddress)
|
||||
.httpUsernamePassword(questdbProperties.username(), questdbProperties.password())
|
||||
.build();
|
||||
|
||||
// Parse host and set query base URL (assuming HTTP port is 9000)
|
||||
String[] parts = ilpAddress.split(":");
|
||||
String host = parts[0];
|
||||
String queryPort = "9000"; // Default QuestDB HTTP port
|
||||
this.queryBaseUrl = "http://" + host + ":" + queryPort + "/exec?query=";
|
||||
|
||||
this.client = new OkHttpClient.Builder()
|
||||
.readTimeout(NetworkConstants.HttpClientConstants.READ_TIME_OUT, TimeUnit.SECONDS)
|
||||
.writeTimeout(NetworkConstants.HttpClientConstants.WRITE_TIME_OUT, TimeUnit.SECONDS)
|
||||
.connectTimeout(NetworkConstants.HttpClientConstants.CONNECT_TIME_OUT, TimeUnit.SECONDS)
|
||||
.connectionPool(new ConnectionPool(
|
||||
NetworkConstants.HttpClientConstants.MAX_IDLE_CONNECTIONS,
|
||||
NetworkConstants.HttpClientConstants.KEEP_ALIVE_TIMEOUT,
|
||||
TimeUnit.SECONDS)
|
||||
).sslSocketFactory(defaultSslSocketFactory(), defaultTrustManager())
|
||||
.hostnameVerifier(noopHostnameVerifier())
|
||||
.retryOnConnectionFailure(true)
|
||||
.build();
|
||||
|
||||
this.serverAvailable = this.checkConnection();
|
||||
}
|
||||
|
||||
private boolean checkConnection() {
|
||||
// Test query to check if server is available
|
||||
Map<String, Object> result = executeQuery("SELECT 1");
|
||||
return result != null && result.containsKey("dataset");
|
||||
}
|
||||
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS || metricsData.getValues().isEmpty()) {
|
||||
return;
|
||||
}
|
||||
String table = this.generateTable(metricsData.getApp(), metricsData.getMetrics(), metricsData.getInstance());
|
||||
|
||||
try {
|
||||
RowWrapper rowWrapper = metricsData.readRow();
|
||||
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
// Wrap the construction of each row in a try-catch block
|
||||
// to prevent one bad row from corrupting the sender's state.
|
||||
try {
|
||||
// 1. Set the table for the new row.
|
||||
sender.table(table);
|
||||
// 2. Process and write all symbols FIRST.
|
||||
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
|
||||
rowWrapper.cellStream()
|
||||
.filter(cell -> cell.getMetadataAsBoolean(MetricDataConstants.LABEL))
|
||||
.forEach(cell -> labels.put(cell.getField().getName(), cell.getValue()));
|
||||
if (!labels.isEmpty()) {
|
||||
sender.symbol("metric_labels", JsonUtil.toJson(labels));
|
||||
} else {
|
||||
sender.symbol("metric_labels", metricsData.getApp()
|
||||
+ "_" + metricsData.getMetrics());
|
||||
}
|
||||
|
||||
// 3. Now, process and write all other columns (fields).
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
if (CommonConstants.NULL_VALUE.equals(cell.getValue())) {
|
||||
return;
|
||||
}
|
||||
String fieldName = cell.getField().getName();
|
||||
String fieldValue = cell.getValue();
|
||||
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
|
||||
|
||||
if (type == CommonConstants.TYPE_NUMBER) {
|
||||
sender.doubleColumn(fieldName, Double.parseDouble(fieldValue));
|
||||
} else if (type == CommonConstants.TYPE_STRING) {
|
||||
sender.stringColumn(fieldName, fieldValue);
|
||||
}
|
||||
});
|
||||
|
||||
// 4. Finally, set the timestamp to commit the row.
|
||||
sender.atNow();
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse questdb]--Could not process a row, cancelling it. Error: {}", e.getMessage());
|
||||
// IMPORTANT: Cancel the partially built row to reset the sender's state.
|
||||
sender.cancelRow();
|
||||
}
|
||||
}
|
||||
sender.flush();
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse questdb]--Error during batch save: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
|
||||
String table = this.generateTable(app, metrics, instance);
|
||||
String dateAdd = getDateAdd(history);
|
||||
String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, dateAdd);
|
||||
Map<String, List<Value>> instanceValueMap = new HashMap<>(8);
|
||||
try {
|
||||
Map<String, Object> selectResult = executeQuery(selectSql);
|
||||
if (selectResult == null || !selectResult.containsKey("dataset")) {
|
||||
return instanceValueMap;
|
||||
}
|
||||
List<Map<String, String>> columns = (List<Map<String, String>>) selectResult.get("columns");
|
||||
List<List<Object>> dataset = (List<List<Object>>) selectResult.get("dataset");
|
||||
|
||||
Map<String, Integer> colMap = new HashMap<>();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
colMap.put(columns.get(i).get("name"), i);
|
||||
}
|
||||
int tsIdx = colMap.get("ts");
|
||||
int metricLabelsIdx = colMap.get("metric_labels");
|
||||
int valueIdx = colMap.get("value");
|
||||
|
||||
for (List<Object> row : dataset) {
|
||||
String tsStr = (String) row.get(tsIdx);
|
||||
long time = Instant.parse(tsStr).toEpochMilli();
|
||||
String instanceValue = row.get(metricLabelsIdx) == null ? "" : (String) row.get(metricLabelsIdx);
|
||||
Object valObj = row.get(valueIdx);
|
||||
String strValue = valObj == null ? null : this.parseDoubleValue(valObj.toString());
|
||||
if (strValue == null) {
|
||||
continue;
|
||||
}
|
||||
List<Value> valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
|
||||
valueList.add(new Value(strValue, time));
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("select history metric data in questdb error, sql:{}, msg: {}", selectSql, e.getMessage());
|
||||
}
|
||||
return instanceValueMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics, String metric, String history) {
|
||||
String table = this.generateTable(app, metrics, instance);
|
||||
String dateAdd = getDateAdd(history);
|
||||
Map<String, List<Value>> instanceValueMap = new HashMap<>(8);
|
||||
Set<String> instances = new HashSet<>(8);
|
||||
// query all metric_labels
|
||||
String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table);
|
||||
Map<String, Object> instanceQueryResult = executeQuery(queryInstanceSql);
|
||||
if (instanceQueryResult != null && instanceQueryResult.containsKey("dataset")) {
|
||||
List<List<Object>> dataset = (List<List<Object>>) instanceQueryResult.get("dataset");
|
||||
for (List<Object> row : dataset) {
|
||||
if (!row.isEmpty() && row.get(0) != null) {
|
||||
instances.add(row.get(0).toString());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
if (instances.isEmpty()) {
|
||||
instances.add("");
|
||||
}
|
||||
for (String instanceValue : instances) {
|
||||
String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL, metric, metric, metric, metric, table, instanceValue.replace("'", "\\'"), dateAdd);
|
||||
Map<String, Object> selectResult = executeQuery(selectSql);
|
||||
if (selectResult == null || !selectResult.containsKey("dataset")) {
|
||||
continue;
|
||||
}
|
||||
List<Map<String, String>> columns = (List<Map<String, String>>) selectResult.get("columns");
|
||||
List<List<Object>> dataset = (List<List<Object>>) selectResult.get("dataset");
|
||||
|
||||
Map<String, Integer> colMap = new HashMap<>();
|
||||
for (int i = 0; i < columns.size(); i++) {
|
||||
colMap.put(columns.get(i).get("name"), i);
|
||||
}
|
||||
int tsIdx = colMap.get("ts");
|
||||
int originIdx = colMap.get("origin");
|
||||
int meanIdx = colMap.get("mean");
|
||||
int maxIdx = colMap.get("max");
|
||||
int minIdx = colMap.get("min");
|
||||
|
||||
for (List<Object> row : dataset) {
|
||||
String tsStr = (String) row.get(tsIdx);
|
||||
long time = Instant.parse(tsStr).toEpochMilli();
|
||||
Value.ValueBuilder valueBuilder = Value.builder();
|
||||
valueBuilder.time(time);
|
||||
|
||||
Object originObj = row.get(originIdx);
|
||||
if (originObj == null) {
|
||||
continue;
|
||||
}
|
||||
valueBuilder.origin(this.parseDoubleValue(originObj.toString()));
|
||||
|
||||
Object meanObj = row.get(meanIdx);
|
||||
if (meanObj == null) {
|
||||
continue;
|
||||
}
|
||||
valueBuilder.mean(this.parseDoubleValue(meanObj.toString()));
|
||||
|
||||
Object maxObj = row.get(maxIdx);
|
||||
if (maxObj == null) {
|
||||
continue;
|
||||
}
|
||||
valueBuilder.max(this.parseDoubleValue(maxObj.toString()));
|
||||
|
||||
Object minObj = row.get(minIdx);
|
||||
if (minObj == null) {
|
||||
continue;
|
||||
}
|
||||
valueBuilder.min(this.parseDoubleValue(minObj.toString()));
|
||||
|
||||
List<Value> valueList = instanceValueMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
|
||||
valueList.add(valueBuilder.build());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("select history interval metric data in questdb error, msg: {}", e.getMessage());
|
||||
}
|
||||
return instanceValueMap;
|
||||
}
|
||||
|
||||
private Map<String, Object> executeQuery(String sql) {
|
||||
try {
|
||||
String encodedSql = URLEncoder.encode(sql, StandardCharsets.UTF_8);
|
||||
String url = queryBaseUrl + encodedSql + "×tamptype=rfc3339";
|
||||
String authHeader = "Basic " + Base64.getEncoder().encodeToString(
|
||||
(questdbProperties.username() + ":" + questdbProperties.password())
|
||||
.getBytes(StandardCharsets.UTF_8)
|
||||
);
|
||||
Request request = new Request.Builder()
|
||||
.url(url)
|
||||
.addHeader("Authorization", authHeader)
|
||||
.get()
|
||||
.build();
|
||||
try (Response response = client.newCall(request).execute()) {
|
||||
if (!response.isSuccessful()) {
|
||||
log.error("QuestDB query failed: {} - {}", response.code(), response.message());
|
||||
return null;
|
||||
}
|
||||
String body = response.body().string();
|
||||
return JsonUtil.fromJson(body, Map.class);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("Error executing QuestDB query: {} - {}", sql, e.getMessage());
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private String getDateAdd(String history) {
|
||||
history = history.toLowerCase();
|
||||
char unitChar = history.charAt(history.length() - 1);
|
||||
int count = Integer.parseInt(history.substring(0, history.length() - 1));
|
||||
String unit;
|
||||
switch (unitChar) {
|
||||
case 'd':
|
||||
unit = "d";
|
||||
break;
|
||||
case 'h':
|
||||
unit = "h";
|
||||
break;
|
||||
case 'm': // minute
|
||||
unit = "m";
|
||||
break;
|
||||
case 's':
|
||||
unit = "s";
|
||||
break;
|
||||
default: throw new IllegalArgumentException("Invalid history unit: " + unitChar);
|
||||
}
|
||||
return String.format("dateadd('%s', %d, now())", unit, -count);
|
||||
}
|
||||
|
||||
private String generateTable(String app, String metrics, String instance) {
|
||||
if (instance.contains(".") || instance.contains(":") || instance.contains("[")) {
|
||||
instance = instance.replace(".", "_")
|
||||
.replace(":", "_")
|
||||
.replace("[", "_")
|
||||
.replace("]", "_");
|
||||
}
|
||||
return app + "_" + metrics + "_" + instance;
|
||||
}
|
||||
|
||||
private String parseDoubleValue(String value) {
|
||||
return (new BigDecimal(value)).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
}
|
||||
|
||||
private static X509TrustManager defaultTrustManager() {
|
||||
return new X509TrustManager() {
|
||||
@Override
|
||||
public X509Certificate[] getAcceptedIssuers() {
|
||||
return new X509Certificate[0];
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkClientTrusted(X509Certificate[] certs, String authType) {
|
||||
}
|
||||
|
||||
@Override
|
||||
public void checkServerTrusted(X509Certificate[] certs, String authType) {
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
private static SSLSocketFactory defaultSslSocketFactory() {
|
||||
try {
|
||||
SSLContext sslContext = SSLContexts.createDefault();
|
||||
sslContext.init(null, new TrustManager[]{
|
||||
defaultTrustManager()
|
||||
}, new SecureRandom());
|
||||
return sslContext.getSocketFactory();
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private static HostnameVerifier noopHostnameVerifier() {
|
||||
return (s, sslSession) -> true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
if (this.sender != null) {
|
||||
this.sender.close();
|
||||
}
|
||||
if (this.client != null) {
|
||||
this.client.dispatcher().executorService().shutdown();
|
||||
}
|
||||
}
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.questdb;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* QuestDB configuration information
|
||||
*/
|
||||
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.QUEST_DB)
|
||||
public record QuestdbProperties(@DefaultValue("false") boolean enabled,
|
||||
String url,
|
||||
String username,
|
||||
String password) {
|
||||
}
|
||||
+509
@@ -0,0 +1,509 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.tdengine;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import com.taosdata.jdbc.TSDBDriver;
|
||||
import com.zaxxer.hikari.HikariConfig;
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Properties;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.regex.Pattern;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.ObjectUtils;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.common.util.StrBuffer;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.jetbrains.annotations.NotNull;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* tdengine data storage
|
||||
*/
|
||||
@Primary
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.td-engine",
|
||||
name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class TdEngineDataStorage extends AbstractHistoryDataStorage {
|
||||
|
||||
private static final String CONSTANTS_URL_PREFIX = "jdbc:TAOS-RS://";
|
||||
private static final Pattern SQL_SPECIAL_STRING_PATTERN = Pattern.compile("(\\\\)|(')");
|
||||
private static final String INSTANCE_NULL = "''";
|
||||
private static final String CONSTANTS_CREATE_DATABASE = "CREATE DATABASE IF NOT EXISTS %s";
|
||||
private static final String INSERT_TABLE_DATA_SQL = "INSERT INTO `%s` USING `%s` TAGS (%s) VALUES %s";
|
||||
private static final String CREATE_SUPER_TABLE_SQL = "CREATE STABLE IF NOT EXISTS `%s` %s TAGS (instance NCHAR(128))";
|
||||
private static final String NO_SUPER_TABLE_ERROR = "Table does not exist";
|
||||
/**
|
||||
* schema version suffix - increment when TAG definition changes.
|
||||
* v1: instance BIGINT (original monitorId like 123456789)
|
||||
* v2: instance NCHAR(128) (supports string instance values like 127.0.0.1:8080)
|
||||
*/
|
||||
private static final String SUPER_TABLE_VERSION = "_v2";
|
||||
|
||||
private static final String QUERY_HISTORY_WITH_INSTANCE_SQL =
|
||||
"SELECT ts, metric_labels, `%s` FROM `%s` WHERE metric_labels = '%s' AND ts >= now - %s order by ts desc";
|
||||
private static final String QUERY_HISTORY_SQL =
|
||||
"SELECT ts, metric_labels, `%s` FROM `%s` WHERE ts >= now - %s order by ts desc";
|
||||
private static final String QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL =
|
||||
"SELECT first(ts), first(`%s`), avg(`%s`), min(`%s`), max(`%s`) FROM `%s` WHERE metric_labels = '%s' AND ts >= now - %s interval(4h)";
|
||||
private static final String QUERY_INSTANCE_SQL =
|
||||
"SELECT DISTINCT metric_labels FROM `%s` WHERE ts >= now - 1w";
|
||||
|
||||
private static final String TABLE_NOT_EXIST = "Table does not exist";
|
||||
|
||||
private static final Runnable INSTANCE_EXCEPTION_PRINT = () -> {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(
|
||||
"""
|
||||
\t---------------TdEngine Init Failed---------------
|
||||
\t--------------Please Config Tdengine--------------
|
||||
\t----------Can Not Use Metric History Now----------
|
||||
"""
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
private HikariDataSource hikariDataSource;
|
||||
private final int tableStrColumnDefineMaxLength;
|
||||
|
||||
public TdEngineDataStorage(TdEngineProperties tdEngineProperties) {
|
||||
if (tdEngineProperties == null) {
|
||||
log.error("init error, please config Warehouse TdEngine props in application.yml");
|
||||
throw new IllegalArgumentException("please config Warehouse TdEngine props");
|
||||
}
|
||||
tableStrColumnDefineMaxLength = tdEngineProperties.tableStrColumnDefineMaxLength();
|
||||
serverAvailable = initTdEngineDatasource(tdEngineProperties);
|
||||
}
|
||||
|
||||
/**
|
||||
* {@code TdEngine} init TdEngineDatabase
|
||||
*
|
||||
* @param tdEngineProperties {@link TdEngineProperties}
|
||||
*/
|
||||
private void initTdEngineDatabase(final TdEngineProperties tdEngineProperties) throws SQLException {
|
||||
final Properties parseResultProperties = com.taosdata.jdbc.utils.StringUtils.parseUrl(tdEngineProperties.url(), null);
|
||||
final String host = ObjectUtils.requireNonEmpty(parseResultProperties.getProperty(TSDBDriver.PROPERTY_KEY_HOST));
|
||||
final String port = ObjectUtils.requireNonEmpty(parseResultProperties.getProperty(TSDBDriver.PROPERTY_KEY_PORT));
|
||||
final String dbName = ObjectUtils.requireNonEmpty(parseResultProperties.getProperty(TSDBDriver.PROPERTY_KEY_DBNAME));
|
||||
|
||||
try (
|
||||
final Connection tempConnection = DriverManager.getConnection(
|
||||
CONSTANTS_URL_PREFIX + host + ":" + port,
|
||||
tdEngineProperties.username(),
|
||||
tdEngineProperties.password()
|
||||
)
|
||||
) {
|
||||
tempConnection.prepareStatement(String.format(CONSTANTS_CREATE_DATABASE, dbName))
|
||||
.execute();
|
||||
}
|
||||
}
|
||||
|
||||
private boolean initTdEngineDatasource(final TdEngineProperties tdEngineProperties) {
|
||||
try {
|
||||
initTdEngineDatabase(tdEngineProperties);
|
||||
} catch (Exception e) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
|
||||
INSTANCE_EXCEPTION_PRINT.run();
|
||||
return false;
|
||||
}
|
||||
|
||||
final HikariConfig config = new HikariConfig();
|
||||
// jdbc properties
|
||||
config.setJdbcUrl(tdEngineProperties.url());
|
||||
config.setUsername(tdEngineProperties.username());
|
||||
config.setPassword(tdEngineProperties.password());
|
||||
config.setDriverClassName(tdEngineProperties.driverClassName());
|
||||
//minimum number of idle connection
|
||||
config.setMinimumIdle(10);
|
||||
//maximum number of connection in the pool
|
||||
config.setMaximumPoolSize(10);
|
||||
//maximum wait milliseconds for get connection from pool
|
||||
config.setConnectionTimeout(30000);
|
||||
// maximum lifetime for each connection
|
||||
config.setMaxLifetime(0);
|
||||
// max idle time for recycle idle connection
|
||||
config.setIdleTimeout(0);
|
||||
//validation query
|
||||
config.setConnectionTestQuery("select server_status()");
|
||||
try {
|
||||
this.hikariDataSource = new HikariDataSource(config);
|
||||
} catch (Exception e) {
|
||||
INSTANCE_EXCEPTION_PRINT.run();
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
|
||||
return;
|
||||
}
|
||||
if (metricsData.getValues().isEmpty()) {
|
||||
|
||||
if (log.isInfoEnabled()) {
|
||||
log.info("[warehouse tdengine] flush metrics data {} is null, ignore.", metricsData.getInstance());
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
String instance = metricsData.getInstance();
|
||||
String app = metricsData.getApp();
|
||||
String metrics = metricsData.getMetrics();
|
||||
String superTable = getSuperTable(app, metrics);
|
||||
String table = getTable(app, metrics, instance);
|
||||
StringBuilder sqlBuffer = new StringBuilder();
|
||||
int i = 0;
|
||||
|
||||
try {
|
||||
RowWrapper rowWrapper = metricsData.readRow();
|
||||
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
StringBuilder sqlRowBuffer = new StringBuilder("(");
|
||||
sqlRowBuffer.append(metricsData.getTime() + i++).append(", ");
|
||||
Map<String, String> labels = Maps.newHashMapWithExpectedSize(8);
|
||||
sqlRowBuffer.append("'").append("%s").append("', ");
|
||||
|
||||
|
||||
AtomicInteger index = new AtomicInteger(-1);
|
||||
int fieldMaxSize = rowWrapper.getFieldList().size();
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
index.getAndIncrement();
|
||||
String value = cell.getValue();
|
||||
final int fieldType;
|
||||
|
||||
if ((fieldType = cell.getMetadataAsByte(MetricDataConstants.TYPE)) == CommonConstants.TYPE_NUMBER || fieldType == CommonConstants.TYPE_TIME) {
|
||||
// number data
|
||||
if (CommonConstants.NULL_VALUE.equals(value)) {
|
||||
sqlRowBuffer.append("NULL");
|
||||
} else {
|
||||
try {
|
||||
double number = Double.parseDouble(value);
|
||||
// TDengine doesn't support NaN or Infinity literals, convert to NULL
|
||||
if (Double.isNaN(number) || Double.isInfinite(number)) {
|
||||
sqlRowBuffer.append("NULL");
|
||||
} else {
|
||||
sqlRowBuffer.append(number);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
|
||||
if (log.isWarnEnabled()) {
|
||||
log.warn(e.getMessage());
|
||||
}
|
||||
|
||||
sqlRowBuffer.append("NULL");
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// string
|
||||
if (CommonConstants.NULL_VALUE.equals(value)) {
|
||||
sqlRowBuffer.append("NULL");
|
||||
} else {
|
||||
sqlRowBuffer.append("'").append(StrBuffer.escapeForFormat(formatStringValue(value))).append("'");
|
||||
}
|
||||
}
|
||||
|
||||
if (cell.getMetadataAsBoolean(MetricDataConstants.LABEL) && !CommonConstants.NULL_VALUE.equals(value)) {
|
||||
labels.put(cell.getField().getName(), formatStringValue(value));
|
||||
}
|
||||
if (index.get() != fieldMaxSize - 1) {
|
||||
sqlRowBuffer.append(", ");
|
||||
}
|
||||
});
|
||||
|
||||
sqlRowBuffer.append(")");
|
||||
sqlBuffer.append(" ").append(String.format(sqlRowBuffer.toString(), formatStringValue(JsonUtil.toJson(labels))));
|
||||
}
|
||||
|
||||
String insertDataSql = String.format(INSERT_TABLE_DATA_SQL, table, superTable, "'" + formatStringValue(instance) + "'", sqlBuffer);
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug(insertDataSql);
|
||||
}
|
||||
|
||||
Connection connection = null;
|
||||
Statement statement = null;
|
||||
try {
|
||||
connection = hikariDataSource.getConnection();
|
||||
statement = connection.createStatement();
|
||||
statement.execute(insertDataSql);
|
||||
connection.close();
|
||||
} catch (Exception e) {
|
||||
if (e.getMessage().contains(NO_SUPER_TABLE_ERROR)) {
|
||||
// stable not exists, create it
|
||||
StringBuilder fieldSqlBuilder = new StringBuilder("(");
|
||||
fieldSqlBuilder.append("ts TIMESTAMP, ");
|
||||
fieldSqlBuilder.append("metric_labels NCHAR(").append(tableStrColumnDefineMaxLength).append("), ");
|
||||
for (int index = 0; index < metricsData.getFields().size(); index++) {
|
||||
CollectRep.Field field = metricsData.getFields().get(index);
|
||||
String fieldName = field.getName();
|
||||
final int fieldType = field.getType();
|
||||
|
||||
if (fieldType == CommonConstants.TYPE_NUMBER || fieldType == CommonConstants.TYPE_TIME) {
|
||||
fieldSqlBuilder.append("`").append(fieldName).append("` ").append("DOUBLE");
|
||||
} else {
|
||||
fieldSqlBuilder.append("`").append(fieldName).append("` ").append("NCHAR(")
|
||||
.append(tableStrColumnDefineMaxLength).append(")");
|
||||
}
|
||||
if (index != metricsData.getFields().size() - 1) {
|
||||
fieldSqlBuilder.append(", ");
|
||||
}
|
||||
}
|
||||
fieldSqlBuilder.append(")");
|
||||
String createTableSql = String.format(CREATE_SUPER_TABLE_SQL, superTable, fieldSqlBuilder);
|
||||
try {
|
||||
assert statement != null;
|
||||
|
||||
if (log.isInfoEnabled()) {
|
||||
log.info("[tdengine-data]: create {} use sql: {}.", superTable, createTableSql);
|
||||
}
|
||||
|
||||
statement.execute(createTableSql);
|
||||
statement.execute(insertDataSql);
|
||||
} catch (Exception createTableException) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage(), createTableException);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
assert connection != null;
|
||||
connection.close();
|
||||
} catch (Exception e) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("[warehouse tdEngine]--Error: {}", e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static String getTable(String app, String metrics, String instance) {
|
||||
if (instance.contains(".") || instance.contains(":") || instance.contains("[")) {
|
||||
instance = instance.replace(".", "_")
|
||||
.replace(":", "_")
|
||||
.replace("[", "_")
|
||||
.replace("]", "_");
|
||||
}
|
||||
return app + "_" + metrics + "_" + instance + SUPER_TABLE_VERSION;
|
||||
}
|
||||
|
||||
@NotNull
|
||||
private static String getSuperTable(String app, String metrics) {
|
||||
return app + "_" + metrics + "_super" + SUPER_TABLE_VERSION;
|
||||
}
|
||||
|
||||
private String formatStringValue(String value) {
|
||||
String formatValue = SQL_SPECIAL_STRING_PATTERN.matcher(value).replaceAll("\\\\$0");
|
||||
// bugfix Argument list too long
|
||||
if (formatValue != null && formatValue.length() > tableStrColumnDefineMaxLength) {
|
||||
formatValue = formatValue.substring(0, tableStrColumnDefineMaxLength);
|
||||
}
|
||||
return formatValue;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (hikariDataSource != null) {
|
||||
hikariDataSource.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
|
||||
String table = getTable(app, metrics, instance);
|
||||
String selectSql = String.format(QUERY_HISTORY_SQL, metric, table, history);
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug(selectSql);
|
||||
}
|
||||
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
if (!serverAvailable) {
|
||||
|
||||
INSTANCE_EXCEPTION_PRINT.run();
|
||||
|
||||
return instanceValuesMap;
|
||||
}
|
||||
try (Connection connection = hikariDataSource.getConnection();
|
||||
Statement statement = connection.createStatement();
|
||||
ResultSet resultSet = statement.executeQuery(selectSql)) {
|
||||
while (resultSet.next()) {
|
||||
Timestamp ts = resultSet.getTimestamp(1);
|
||||
if (ts == null) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error("warehouse tdengine query result timestamp is null, ignore. {}.", selectSql);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
String instanceValue = resultSet.getString(2);
|
||||
if (instanceValue == null || StringUtils.isBlank(instanceValue)) {
|
||||
instanceValue = "";
|
||||
}
|
||||
double value = resultSet.getDouble(3);
|
||||
String strValue = new BigDecimal(value).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
|
||||
valueList.add(new Value(strValue, ts.getTime() / 100 * 100));
|
||||
}
|
||||
return instanceValuesMap;
|
||||
} catch (SQLException sqlException) {
|
||||
String msg = sqlException.getMessage();
|
||||
if (msg != null && !msg.contains(TABLE_NOT_EXIST)) {
|
||||
if (log.isWarnEnabled()) {
|
||||
log.warn(sqlException.getMessage());
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
|
||||
String metric, String history) {
|
||||
if (!serverAvailable) {
|
||||
|
||||
INSTANCE_EXCEPTION_PRINT.run();
|
||||
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
String table = getTable(app, metrics, instance);
|
||||
List<String> instances = new LinkedList<>();
|
||||
// query all metric_labels from the table
|
||||
String queryInstanceSql = String.format(QUERY_INSTANCE_SQL, table);
|
||||
Connection connection = null;
|
||||
try {
|
||||
connection = hikariDataSource.getConnection();
|
||||
Statement statement = connection.createStatement();
|
||||
ResultSet resultSet = statement.executeQuery(queryInstanceSql);
|
||||
while (resultSet.next()) {
|
||||
String instanceValue = resultSet.getString(1);
|
||||
if (instanceValue == null || StringUtils.isBlank(instanceValue)) {
|
||||
instances.add("''");
|
||||
} else {
|
||||
instances.add(instanceValue);
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
assert connection != null;
|
||||
connection.close();
|
||||
} catch (Exception e) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(instances.size());
|
||||
for (String instanceValue : instances) {
|
||||
if (INSTANCE_NULL.equals(instanceValue)) {
|
||||
instanceValue = "";
|
||||
}
|
||||
String selectSql = String.format(QUERY_HISTORY_INTERVAL_WITH_INSTANCE_SQL,
|
||||
metric, metric, metric, metric, table, instanceValue, history);
|
||||
|
||||
if (log.isDebugEnabled()) {
|
||||
log.debug(selectSql);
|
||||
}
|
||||
|
||||
List<Value> values = instanceValuesMap.computeIfAbsent(instanceValue, k -> new LinkedList<>());
|
||||
Connection conn = null;
|
||||
try {
|
||||
conn = hikariDataSource.getConnection();
|
||||
Statement statement = conn.createStatement();
|
||||
ResultSet resultSet = statement.executeQuery(selectSql);
|
||||
while (resultSet.next()) {
|
||||
Timestamp ts = resultSet.getTimestamp(1);
|
||||
double origin = resultSet.getDouble(2);
|
||||
String originStr = BigDecimal.valueOf(origin).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
double avg = resultSet.getDouble(3);
|
||||
String avgStr = BigDecimal.valueOf(avg).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
double min = resultSet.getDouble(4);
|
||||
String minStr = BigDecimal.valueOf(min).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
double max = resultSet.getDouble(5);
|
||||
String maxStr = BigDecimal.valueOf(max).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
Value value = Value.builder()
|
||||
.origin(originStr).mean(avgStr)
|
||||
.min(minStr).max(maxStr)
|
||||
.time(ts.getTime() / 100 * 100)
|
||||
.build();
|
||||
values.add(value);
|
||||
}
|
||||
resultSet.close();
|
||||
} catch (Exception e) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage(), e);
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
assert conn != null;
|
||||
conn.close();
|
||||
} catch (Exception e) {
|
||||
if (log.isErrorEnabled()) {
|
||||
log.error(e.getMessage());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
}
|
||||
+46
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.tdengine;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
*
|
||||
* @param enabled Whether the TdEngine data store is enabled
|
||||
* @param url TdEngine connect url
|
||||
* @param driverClassName tdengine driver, default restful driver
|
||||
* @param username tdengine username
|
||||
* @param password tdengine password
|
||||
* @param tableStrColumnDefineMaxLength auto create table's string column define max length : NCHAR(200)
|
||||
*/
|
||||
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.TD_ENGINE)
|
||||
public record TdEngineProperties(@DefaultValue("false") boolean enabled,
|
||||
@DefaultValue("jdbc:TAOS-RS://localhost:6041/demo") String url,
|
||||
@DefaultValue("com.taosdata.jdbc.rs.RestfulDriver") String driverClassName,
|
||||
String username, String password,
|
||||
@DefaultValue("200") int tableStrColumnDefineMaxLength) {
|
||||
}
|
||||
+100
@@ -0,0 +1,100 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
|
||||
/**
|
||||
* promql query content
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public class PromQlQueryContent {
|
||||
|
||||
private String status;
|
||||
|
||||
private ContentData data;
|
||||
|
||||
/**
|
||||
* content data
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static final class ContentData {
|
||||
|
||||
private String resultType;
|
||||
|
||||
private List<Content> result;
|
||||
|
||||
/**
|
||||
* content
|
||||
*/
|
||||
@Data
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static final class Content {
|
||||
|
||||
/**
|
||||
* metric contains metric name plus labels for a particular time series
|
||||
*/
|
||||
private Map<String, String> metric;
|
||||
|
||||
/**
|
||||
* values contains raw sample values for the given time series
|
||||
* value-timestamp
|
||||
* [1700993195,"436960986"]
|
||||
*/
|
||||
private Object[] value;
|
||||
|
||||
/**
|
||||
* values contains raw sample values for the given time series
|
||||
* value-timestamp list
|
||||
* [[1700993195,"436960986"],[1700993195,"436960986"]...]
|
||||
*/
|
||||
private List<Object[]> values;
|
||||
}
|
||||
}
|
||||
}
|
||||
+707
@@ -0,0 +1,707 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
import tools.jackson.databind.JsonNode;
|
||||
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.net.URI;
|
||||
import java.time.Duration;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.temporal.TemporalAmount;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
import java.util.stream.Stream;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.constants.NetworkConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.timer.HashedWheelTimer;
|
||||
import org.apache.hertzbeat.common.timer.Timeout;
|
||||
import org.apache.hertzbeat.common.timer.TimerTask;
|
||||
import org.apache.hertzbeat.common.util.Base64Util;
|
||||
import org.apache.hertzbeat.common.util.CommonUtil;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.common.util.TimePeriodUtil;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import org.springframework.web.util.UriComponentsBuilder;
|
||||
|
||||
import static org.apache.hertzbeat.common.constants.ConfigConstants.FunctionModuleConstants.STATUS;
|
||||
|
||||
/**
|
||||
* VictoriaMetrics data storage
|
||||
*/
|
||||
@Primary
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics.cluster", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class VictoriaMetricsClusterDataStorage extends AbstractHistoryDataStorage {
|
||||
|
||||
private static final String VM_INSERT_BASE_PATH = "/insert/%s/%s";
|
||||
private static final String VM_SELECT_BASE_PATH = "/select/%s/%s";
|
||||
private static final String IMPORT_PATH = "prometheus/api/v1/import";
|
||||
private static final String EXPORT_PATH = "prometheus/api/v1/export";
|
||||
private static final String STATUS_PATH = "prometheus/api/v1/status/tsdb";
|
||||
private static final String STATUS_SUCCESS = "success";
|
||||
private static final String QUERY_RANGE_PATH = "prometheus/api/v1/query_range";
|
||||
private static final String LABEL_KEY_NAME = "__name__";
|
||||
private static final String LABEL_KEY_JOB = "job";
|
||||
private static final String LABEL_KEY_INSTANCE = "instance";
|
||||
private static final String LABEL_KEY_MONITOR_ID = "__monitor_id__";
|
||||
private static final String SPILT = "_";
|
||||
private static final String MONITOR_METRICS_KEY = "__metrics__";
|
||||
private static final String MONITOR_METRIC_KEY = "__metric__";
|
||||
private static final long MAX_WAIT_MS = 500L;
|
||||
private static final int MAX_RETRIES = 3;
|
||||
|
||||
private final VictoriaMetricsClusterProperties vmClusterProps;
|
||||
private final VictoriaMetricsInsertProperties vmInsertProps;
|
||||
private final VictoriaMetricsSelectProperties vmSelectProps;
|
||||
private final RestTemplate restTemplate;
|
||||
private final BlockingQueue<VictoriaMetricsDataStorage.VictoriaMetricsContent> metricsBufferQueue;
|
||||
|
||||
private HashedWheelTimer metricsFlushTimer = null;
|
||||
private MetricsFlushTask metricsFlushtask = null;
|
||||
private boolean isBatchImportEnabled = false;
|
||||
|
||||
|
||||
public VictoriaMetricsClusterDataStorage(VictoriaMetricsClusterProperties vmClusterProps,
|
||||
RestTemplate restTemplate) {
|
||||
if (vmClusterProps == null) {
|
||||
log.error("init error, please config Warehouse victoriaMetrics cluster props in application.yml");
|
||||
throw new IllegalArgumentException("please config Warehouse victoriaMetrics cluster props");
|
||||
}
|
||||
this.restTemplate = restTemplate;
|
||||
this.vmClusterProps = vmClusterProps;
|
||||
this.vmInsertProps = vmClusterProps.insert();
|
||||
this.vmSelectProps = vmClusterProps.select();
|
||||
serverAvailable = checkVictoriaMetricsDatasourceAvailable();
|
||||
metricsBufferQueue = new LinkedBlockingQueue<>(vmInsertProps.bufferSize());
|
||||
isBatchImportEnabled = vmInsertProps.flushInterval() != 0 && vmInsertProps.bufferSize() != 0;
|
||||
if (isBatchImportEnabled){
|
||||
initializeFlushTimer();
|
||||
}
|
||||
}
|
||||
|
||||
private void initializeFlushTimer() {
|
||||
this.metricsFlushTimer = new HashedWheelTimer(r -> {
|
||||
Thread thread = new Thread(r, "victoria-metrics-flush-timer");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
}, 1, TimeUnit.SECONDS, 512);
|
||||
metricsFlushtask = new MetricsFlushTask();
|
||||
this.metricsFlushTimer.newTimeout(metricsFlushtask, 0, TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
private boolean checkVictoriaMetricsDatasourceAvailable() {
|
||||
// check server status
|
||||
try {
|
||||
String selectNodeStatusUrl = vmClusterProps.select().url() + VM_SELECT_BASE_PATH.formatted(vmClusterProps.accountID(), STATUS_PATH);
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
if (StringUtils.hasText(vmInsertProps.username())
|
||||
&& StringUtils.hasText(vmInsertProps.password())) {
|
||||
String authStr = vmInsertProps.username() + ":" + vmInsertProps.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
HttpEntity<Void> requestEntity = new HttpEntity<>(headers);
|
||||
ResponseEntity<String> responseEntity = restTemplate.exchange(
|
||||
selectNodeStatusUrl,
|
||||
HttpMethod.GET,
|
||||
requestEntity,
|
||||
String.class
|
||||
);
|
||||
|
||||
String result = responseEntity.getBody();
|
||||
|
||||
JsonNode jsonNode = JsonUtil.fromJson(result);
|
||||
if (jsonNode != null && STATUS_SUCCESS.equalsIgnoreCase(jsonNode.get(STATUS).asText())) {
|
||||
return true;
|
||||
}
|
||||
log.error("check victoria metrics cluster server status not success: {}.", result);
|
||||
} catch (Exception e) {
|
||||
log.error("check victoria metrics cluster server status error: {}.", e.getMessage());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
if (!isServerAvailable()) {
|
||||
serverAvailable = checkVictoriaMetricsDatasourceAvailable();
|
||||
}
|
||||
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
|
||||
return;
|
||||
}
|
||||
if (metricsData.getValues().isEmpty()) {
|
||||
log.info("[warehouse victoria-metrics] flush metrics data {} {} {} is null, ignore.",
|
||||
metricsData.getId(), metricsData.getApp(), metricsData.getMetrics());
|
||||
return;
|
||||
}
|
||||
Map<String, String> defaultLabels = Maps.newHashMapWithExpectedSize(8);
|
||||
defaultLabels.put(MONITOR_METRICS_KEY, metricsData.getMetrics());
|
||||
boolean isPrometheusAuto;
|
||||
if (metricsData.getApp().startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
|
||||
isPrometheusAuto = true;
|
||||
defaultLabels.remove(MONITOR_METRICS_KEY);
|
||||
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()
|
||||
.substring(CommonConstants.PROMETHEUS_APP_PREFIX.length()));
|
||||
} else {
|
||||
isPrometheusAuto = false;
|
||||
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp());
|
||||
}
|
||||
defaultLabels.put(LABEL_KEY_INSTANCE, metricsData.getInstance());
|
||||
|
||||
|
||||
try {
|
||||
List<CollectRep.Field> fieldList = metricsData.getFields();
|
||||
Long[] timestamp = new Long[]{metricsData.getTime()};
|
||||
Map<String, Double> fieldsValue = Maps.newHashMapWithExpectedSize(fieldList.size());
|
||||
Map<String, String> labels = Maps.newHashMapWithExpectedSize(fieldList.size());
|
||||
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> contentList = new LinkedList<>();
|
||||
|
||||
|
||||
RowWrapper rowWrapper = metricsData.readRow();
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
fieldsValue.clear();
|
||||
labels.clear();
|
||||
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
String value = cell.getValue();
|
||||
Byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
|
||||
Boolean label = cell.getMetadataAsBoolean(MetricDataConstants.LABEL);
|
||||
|
||||
if (type == CommonConstants.TYPE_NUMBER && !label) {
|
||||
// number metrics data
|
||||
if (!CommonConstants.NULL_VALUE.equals(value)) {
|
||||
fieldsValue.put(cell.getField().getName(), CommonUtil.parseStrDouble(value));
|
||||
}
|
||||
}
|
||||
// label
|
||||
if (label && !CommonConstants.NULL_VALUE.equals(value)) {
|
||||
labels.put(cell.getField().getName(), value);
|
||||
}
|
||||
|
||||
for (Map.Entry<String, Double> entry : fieldsValue.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
try {
|
||||
labels.putAll(defaultLabels);
|
||||
String labelName = isPrometheusAuto ? metricsData.getMetrics()
|
||||
: metricsData.getMetrics() + SPILT + entry.getKey();
|
||||
labels.put(LABEL_KEY_NAME, labelName);
|
||||
if (!isPrometheusAuto) {
|
||||
labels.put(MONITOR_METRIC_KEY, entry.getKey());
|
||||
}
|
||||
labels.put(LABEL_KEY_MONITOR_ID, String.valueOf(metricsData.getId()));
|
||||
// add customized labels as identifier
|
||||
var customizedLabels = metricsData.getLabels();
|
||||
if (!ObjectUtils.isEmpty(customizedLabels)) {
|
||||
labels.putAll(customizedLabels);
|
||||
}
|
||||
VictoriaMetricsDataStorage.VictoriaMetricsContent content = VictoriaMetricsDataStorage.VictoriaMetricsContent.builder()
|
||||
.metric(new HashMap<>(labels))
|
||||
.values(new Double[]{entry.getValue()})
|
||||
.timestamps(timestamp)
|
||||
.build();
|
||||
contentList.add(content);
|
||||
} catch (Exception e) {
|
||||
log.error("combine metrics data error: {}.", e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if (contentList.isEmpty()) {
|
||||
log.info("[warehouse victoria-metrics] flush metrics data {} is empty, ignore.", metricsData.getId());
|
||||
return;
|
||||
}
|
||||
if (!isBatchImportEnabled){
|
||||
doSaveData(contentList);
|
||||
return;
|
||||
}
|
||||
sendVictoriaMetrics(contentList);
|
||||
} catch (Exception e) {
|
||||
log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
|
||||
metricsFlushTimer.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric,
|
||||
String history) {
|
||||
String labelName = metrics + SPILT + metric;
|
||||
if (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
|
||||
labelName = metrics;
|
||||
}
|
||||
String timeSeriesSelector = Stream.of(
|
||||
LABEL_KEY_NAME + "=\"" + labelName + "\"",
|
||||
LABEL_KEY_INSTANCE + "=\"" + instance + "\"",
|
||||
app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX) ? null : MONITOR_METRIC_KEY + "=\"" + metric + "\""
|
||||
).filter(Objects::nonNull).collect(Collectors.joining(","));
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||
if (StringUtils.hasText(vmSelectProps.username()) && StringUtils.hasText(vmSelectProps.password())) {
|
||||
String authStr = vmSelectProps.username() + ":" + vmSelectProps.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC
|
||||
+ SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
|
||||
String exportUrl = vmClusterProps.select().url() + VM_SELECT_BASE_PATH.formatted(vmClusterProps.accountID(), EXPORT_PATH);
|
||||
URI uri = UriComponentsBuilder.fromUriString(exportUrl)
|
||||
.queryParam("match[]", "{" + timeSeriesSelector + "}")
|
||||
.queryParam("start", "now-" + history)
|
||||
.queryParam("end", "now")
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
ResponseEntity<String> responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity,
|
||||
String.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.debug("query metrics data from victoria-metrics success. {}", uri);
|
||||
if (StringUtils.hasText(responseEntity.getBody())) {
|
||||
String[] contentJsonArr = responseEntity.getBody().split("\n");
|
||||
List<VictoriaMetricsContent> contents = Arrays.stream(contentJsonArr)
|
||||
.map(item -> JsonUtil.fromJson(item, VictoriaMetricsContent.class)).toList();
|
||||
for (VictoriaMetricsContent content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && content.getTimestamps() != null) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
|
||||
k -> new LinkedList<>());
|
||||
if (content.getValues().length != content.getTimestamps().length) {
|
||||
log.error("content.getValues().length != content.getTimestamps().length");
|
||||
continue;
|
||||
}
|
||||
Double[] values = content.getValues();
|
||||
Long[] timestamps = content.getTimestamps();
|
||||
for (int index = 0; index < content.getValues().length; index++) {
|
||||
String strValue = BigDecimal.valueOf(values[index]).setScale(4, RoundingMode.HALF_UP)
|
||||
.stripTrailingZeros().toPlainString();
|
||||
// read timestamp here is ms unit
|
||||
valueList.add(new Value(strValue, timestamps[index]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
|
||||
String metric, String history) {
|
||||
if (!serverAvailable) {
|
||||
log.error("""
|
||||
|
||||
\t---------------VictoriaMetrics Init Failed---------------
|
||||
\t--------------Please Config VictoriaMetrics--------------
|
||||
\t----------Can Not Use Metric History Now----------
|
||||
""");
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
long endTime = ZonedDateTime.now().toEpochSecond();
|
||||
long startTime;
|
||||
try {
|
||||
if (NumberUtils.isParsable(history)) {
|
||||
startTime = NumberUtils.toLong(history);
|
||||
startTime = (ZonedDateTime.now().toEpochSecond() - startTime);
|
||||
} else {
|
||||
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
|
||||
startTime = dateTime.toEpochSecond();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("history time error: {}. use default: 6h", e.getMessage());
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6));
|
||||
startTime = dateTime.toEpochSecond();
|
||||
}
|
||||
String labelName = metrics + SPILT + metric;
|
||||
if (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
|
||||
labelName = metrics;
|
||||
}
|
||||
String timeSeriesSelector = Stream.of(
|
||||
LABEL_KEY_NAME + "=\"" + labelName + "\"",
|
||||
LABEL_KEY_INSTANCE + "=\"" + instance + "\"",
|
||||
app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX) ? null : MONITOR_METRIC_KEY + "=\"" + metric + "\""
|
||||
).filter(Objects::nonNull).collect(Collectors.joining(","));
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||
if (StringUtils.hasText(vmSelectProps.username()) && StringUtils.hasText(vmSelectProps.password())) {
|
||||
String authStr = vmSelectProps.username() + ":" + vmSelectProps.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC
|
||||
+ SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
|
||||
String rangeUrl = vmClusterProps.select().url() + VM_SELECT_BASE_PATH.formatted(vmClusterProps.accountID(), QUERY_RANGE_PATH);
|
||||
URI uri = UriComponentsBuilder.fromUriString(rangeUrl)
|
||||
.queryParam("query", "{" + timeSeriesSelector + "}")
|
||||
.queryParam("step", "4h")
|
||||
.queryParam("start", startTime)
|
||||
.queryParam("end", endTime)
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri, HttpMethod.GET,
|
||||
httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.debug("query metrics data from victoria-metrics success. {}", uri);
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData()
|
||||
.getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
|
||||
k -> new LinkedList<>());
|
||||
for (Object[] valueArr : content.getValues()) {
|
||||
long timestamp = Long.parseLong(String.valueOf(valueArr[0]));
|
||||
String value = new BigDecimal(String.valueOf(valueArr[1])).setScale(4,
|
||||
RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
// read timestamp here is s unit
|
||||
valueList.add(new Value(value, timestamp * 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
|
||||
}
|
||||
// max
|
||||
uri = UriComponentsBuilder.fromUriString(rangeUrl)
|
||||
.queryParam("query", "max_over_time({" + timeSeriesSelector + "})")
|
||||
.queryParam("step", "4h")
|
||||
.queryParam("start", startTime)
|
||||
.queryParam("end", endTime)
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData()
|
||||
.getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
|
||||
k -> new LinkedList<>());
|
||||
if (valueList.size() == content.getValues().size()) {
|
||||
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
|
||||
Value value = valueList.get(timestampIndex);
|
||||
Object[] valueArr = content.getValues().get(timestampIndex);
|
||||
String maxValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4,
|
||||
RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
value.setMax(maxValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// min
|
||||
uri = UriComponentsBuilder.fromUriString(rangeUrl)
|
||||
.queryParam("query", "min_over_time({" + timeSeriesSelector + "})")
|
||||
.queryParam("step", "4h")
|
||||
.queryParam("start", startTime)
|
||||
.queryParam("end", endTime)
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData()
|
||||
.getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
|
||||
k -> new LinkedList<>());
|
||||
if (valueList.size() == content.getValues().size()) {
|
||||
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
|
||||
Value value = valueList.get(timestampIndex);
|
||||
Object[] valueArr = content.getValues().get(timestampIndex);
|
||||
String minValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4,
|
||||
RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
value.setMin(minValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// avg
|
||||
uri = UriComponentsBuilder.fromUriString(rangeUrl)
|
||||
.queryParam("query", "avg_over_time({" + timeSeriesSelector + "})")
|
||||
.queryParam("step", "4h")
|
||||
.queryParam("start", startTime)
|
||||
.queryParam("end", endTime)
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
responseEntity = restTemplate.exchange(uri, HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData()
|
||||
.getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr,
|
||||
k -> new LinkedList<>());
|
||||
if (valueList.size() == content.getValues().size()) {
|
||||
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
|
||||
Value value = valueList.get(timestampIndex);
|
||||
Object[] valueArr = content.getValues().get(timestampIndex);
|
||||
String avgValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4,
|
||||
RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
value.setMean(avgValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metric data to victoria-metric via HTTP call
|
||||
*/
|
||||
public void doSaveData(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> contentList){
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
if (StringUtils.hasText(vmInsertProps.username())
|
||||
&& StringUtils.hasText(vmInsertProps.password())) {
|
||||
String authStr = vmInsertProps.username() + ":" + vmInsertProps.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
for (VictoriaMetricsDataStorage.VictoriaMetricsContent content : contentList) {
|
||||
stringBuilder.append(JsonUtil.toJson(content)).append("\n");
|
||||
}
|
||||
HttpEntity<String> httpEntity = new HttpEntity<>(stringBuilder.toString(), headers);
|
||||
String importUrl = vmClusterProps.insert().url() + VM_INSERT_BASE_PATH.formatted(vmClusterProps.accountID(), IMPORT_PATH);
|
||||
ResponseEntity<String> responseEntity = restTemplate.postForEntity(importUrl,
|
||||
httpEntity, String.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.debug("insert metrics data to victoria-metrics success.");
|
||||
} else {
|
||||
log.error("insert metrics data to victoria-metrics failed. {}", responseEntity.getBody());
|
||||
}
|
||||
} catch (Exception e){
|
||||
log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* add victoriaMetricsContent to buffer
|
||||
* @param contentList victoriaMetricsContent List
|
||||
*/
|
||||
private void sendVictoriaMetrics(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> contentList) {
|
||||
for (VictoriaMetricsDataStorage.VictoriaMetricsContent content : contentList) {
|
||||
boolean offered = false;
|
||||
int retryCount = 0;
|
||||
while (!offered && retryCount < MAX_RETRIES) {
|
||||
try {
|
||||
// Attempt to add to the queue for a limited time
|
||||
offered = metricsBufferQueue.offer(content, MAX_WAIT_MS, TimeUnit.MILLISECONDS);
|
||||
if (!offered) {
|
||||
// If the queue is still full, trigger an immediate refresh to free up space
|
||||
if (retryCount == 0) {
|
||||
log.debug("victoria metrics buffer queue is full, triggering immediate flush");
|
||||
triggerImmediateFlush();
|
||||
}
|
||||
retryCount++;
|
||||
// The short sleep allows the queue to clear out
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
Thread.sleep(100L * retryCount);
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
log.error("[Victoria Metrics] Interrupted while offering metrics to buffer queue", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// When the maximum number of retries is reached, if it still cannot be added to the queue, the data is saved directly
|
||||
if (!offered) {
|
||||
log.warn("[Victoria Metrics] Failed to add metrics to buffer after {} retries, saving directly", MAX_RETRIES);
|
||||
try {
|
||||
doSaveData(contentList);
|
||||
} catch (Exception e) {
|
||||
log.error("[Victoria Metrics] Failed to save metrics directly: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
// Refresh in advance to avoid waiting
|
||||
if (metricsBufferQueue.size() >= vmInsertProps.bufferSize() * 0.8) {
|
||||
triggerImmediateFlush();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void triggerImmediateFlush() {
|
||||
metricsFlushTimer.newTimeout(metricsFlushtask, 0, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
|
||||
/**
|
||||
* Regularly refresh the buffer queue to the vm
|
||||
*/
|
||||
private class MetricsFlushTask implements TimerTask {
|
||||
@Override
|
||||
public void run(Timeout timeout) {
|
||||
try {
|
||||
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch = new ArrayList<>(vmInsertProps.bufferSize());
|
||||
metricsBufferQueue.drainTo(batch, vmInsertProps.bufferSize());
|
||||
if (!batch.isEmpty()) {
|
||||
doSaveData(batch);
|
||||
log.debug("[Victoria Metrics] Flushed {} metrics items", batch.size());
|
||||
}
|
||||
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
|
||||
metricsFlushTimer.newTimeout(this, vmInsertProps.flushInterval(), TimeUnit.SECONDS);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("[VictoriaMetrics] flush task error: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* victoria metrics content
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static final class VictoriaMetricsContent {
|
||||
|
||||
/**
|
||||
* metric contains metric name plus labels for a particular time series
|
||||
*/
|
||||
private Map<String, String> metric;
|
||||
|
||||
/**
|
||||
* values contains raw sample values for the given time series
|
||||
*/
|
||||
private Double[] values;
|
||||
|
||||
/**
|
||||
* timestamps contains raw sample UNIX timestamps in milliseconds for the given time series
|
||||
* every timestamp is associated with the value at the corresponding position
|
||||
*/
|
||||
private Long[] timestamps;
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* Victoriametrics configuration information
|
||||
*/
|
||||
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.VM_CLUSTER)
|
||||
public record VictoriaMetricsClusterProperties(
|
||||
@DefaultValue("false") boolean enabled,
|
||||
@DefaultValue("0") String accountID,
|
||||
VictoriaMetricsInsertProperties insert,
|
||||
VictoriaMetricsSelectProperties select
|
||||
) {
|
||||
}
|
||||
+716
@@ -0,0 +1,716 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
import java.io.ByteArrayOutputStream;
|
||||
import java.math.BigDecimal;
|
||||
import java.math.RoundingMode;
|
||||
import java.net.URI;
|
||||
import java.nio.charset.StandardCharsets;
|
||||
import java.time.Duration;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.temporal.TemporalAmount;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.BlockingQueue;
|
||||
import java.util.concurrent.LinkedBlockingQueue;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.zip.GZIPOutputStream;
|
||||
|
||||
import com.google.common.collect.Maps;
|
||||
import lombok.AllArgsConstructor;
|
||||
import lombok.Builder;
|
||||
import lombok.Data;
|
||||
import lombok.NoArgsConstructor;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.constants.NetworkConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.timer.HashedWheelTimer;
|
||||
import org.apache.hertzbeat.common.timer.Timeout;
|
||||
import org.apache.hertzbeat.common.timer.TimerTask;
|
||||
import org.apache.hertzbeat.common.util.Base64Util;
|
||||
import org.apache.hertzbeat.common.util.CommonUtil;
|
||||
import org.apache.hertzbeat.common.util.JsonUtil;
|
||||
import org.apache.hertzbeat.common.util.TimePeriodUtil;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.AbstractHistoryDataStorage;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpHeaders;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.MediaType;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.stereotype.Component;
|
||||
import org.springframework.util.ObjectUtils;
|
||||
import org.springframework.util.StringUtils;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
import org.springframework.web.util.UriComponentsBuilder;
|
||||
|
||||
/**
|
||||
* VictoriaMetrics data storage
|
||||
*/
|
||||
@Primary
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.store.victoria-metrics", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class VictoriaMetricsDataStorage extends AbstractHistoryDataStorage {
|
||||
|
||||
private static final String IMPORT_PATH = "/api/v1/import";
|
||||
/**
|
||||
* <a href="https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-export-data-in-json-line-format">
|
||||
* https://docs.victoriametrics.com/Single-server-VictoriaMetrics.html#how-to-export-data-in-json-line-format
|
||||
* </a>
|
||||
*/
|
||||
private static final String EXPORT_PATH = "/api/v1/export";
|
||||
private static final String QUERY_RANGE_PATH = "/api/v1/query_range";
|
||||
private static final String STATUS_PATH = "/api/v1/status/tsdb";
|
||||
private static final String LABEL_KEY_NAME = "__name__";
|
||||
private static final String LABEL_KEY_JOB = "job";
|
||||
private static final String LABEL_KEY_INSTANCE = "instance";
|
||||
private static final String LABEL_KEY_MONITOR_ID = "__monitor_id__";
|
||||
private static final String SPILT = "_";
|
||||
private static final String MONITOR_METRICS_KEY = "__metrics__";
|
||||
private static final String MONITOR_METRIC_KEY = "__metric__";
|
||||
private static final long MAX_WAIT_MS = 500L;
|
||||
private static final int MAX_RETRIES = 3;
|
||||
|
||||
private final VictoriaMetricsProperties victoriaMetricsProp;
|
||||
private final RestTemplate restTemplate;
|
||||
private final BlockingQueue<VictoriaMetricsDataStorage.VictoriaMetricsContent> metricsBufferQueue;
|
||||
|
||||
private HashedWheelTimer metricsFlushTimer = null;
|
||||
private final VictoriaMetricsProperties.InsertConfig insertConfig;
|
||||
private final AtomicBoolean draining = new AtomicBoolean(false);
|
||||
|
||||
public VictoriaMetricsDataStorage(VictoriaMetricsProperties victoriaMetricsProperties, RestTemplate restTemplate) {
|
||||
if (victoriaMetricsProperties == null) {
|
||||
log.error("init error, please config Warehouse victoriaMetrics props in application.yml");
|
||||
throw new IllegalArgumentException("please config Warehouse victoriaMetrics props");
|
||||
}
|
||||
this.restTemplate = restTemplate;
|
||||
victoriaMetricsProp = victoriaMetricsProperties;
|
||||
serverAvailable = checkVictoriaMetricsDatasourceAvailable();
|
||||
insertConfig = victoriaMetricsProperties.insert() == null ? new VictoriaMetricsProperties.InsertConfig(100, 3,
|
||||
new VictoriaMetricsProperties.Compression(false)) : victoriaMetricsProperties.insert();
|
||||
metricsBufferQueue = new LinkedBlockingQueue<>(insertConfig.bufferSize());
|
||||
initializeFlushTimer();
|
||||
}
|
||||
|
||||
private void initializeFlushTimer() {
|
||||
this.metricsFlushTimer = new HashedWheelTimer(r -> {
|
||||
Thread thread = new Thread(r, "victoria-metrics-cluster-flush-timer");
|
||||
thread.setDaemon(true);
|
||||
return thread;
|
||||
}, 1, TimeUnit.SECONDS, 512);
|
||||
// start flush interval timer
|
||||
this.metricsFlushTimer.newTimeout(new MetricsFlushTask(null), insertConfig.flushInterval(), TimeUnit.SECONDS);
|
||||
}
|
||||
|
||||
private boolean checkVictoriaMetricsDatasourceAvailable() {
|
||||
// check server status
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
if (StringUtils.hasText(victoriaMetricsProp.username())
|
||||
&& StringUtils.hasText(victoriaMetricsProp.password())) {
|
||||
String authStr = victoriaMetricsProp.username() + SignConstants.DOUBLE_MARK + victoriaMetricsProp.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + " " + encodedAuth);
|
||||
}
|
||||
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
|
||||
ResponseEntity<String> responseEntity = restTemplate.exchange(victoriaMetricsProp.url() + STATUS_PATH,
|
||||
HttpMethod.GET, httpEntity, String.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.info("check victoria metrics server status success.");
|
||||
return true;
|
||||
}
|
||||
log.error("check victoria metrics server status not success: {}.", responseEntity.getBody());
|
||||
} catch (Exception e) {
|
||||
log.error("check victoria metrics server status error: {}.", e.getMessage());
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
if (!isServerAvailable()) {
|
||||
serverAvailable = checkVictoriaMetricsDatasourceAvailable();
|
||||
}
|
||||
if (!isServerAvailable() || metricsData.getCode() != CollectRep.Code.SUCCESS) {
|
||||
return;
|
||||
}
|
||||
if (metricsData.getValues().isEmpty()) {
|
||||
log.info("[warehouse victoria-metrics] flush metrics data {} {} {} is null, ignore.",
|
||||
metricsData.getId(), metricsData.getApp(), metricsData.getMetrics());
|
||||
return;
|
||||
}
|
||||
Map<String, String> defaultLabels = Maps.newHashMapWithExpectedSize(8);
|
||||
defaultLabels.put(MONITOR_METRICS_KEY, metricsData.getMetrics());
|
||||
boolean isPrometheusAuto = false;
|
||||
if (metricsData.getApp().startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
|
||||
isPrometheusAuto = true;
|
||||
defaultLabels.remove(MONITOR_METRICS_KEY);
|
||||
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp()
|
||||
.substring(CommonConstants.PROMETHEUS_APP_PREFIX.length()));
|
||||
} else {
|
||||
defaultLabels.put(LABEL_KEY_JOB, metricsData.getApp());
|
||||
}
|
||||
defaultLabels.put(LABEL_KEY_INSTANCE, metricsData.getInstance());
|
||||
|
||||
|
||||
List<VictoriaMetricsContent> contentList = new LinkedList<>();
|
||||
try {
|
||||
final int fieldSize = metricsData.getFields().size();
|
||||
Long[] timestamp = new Long[]{metricsData.getTime()};
|
||||
Map<String, Double> fieldsValue = Maps.newHashMapWithExpectedSize(fieldSize);
|
||||
Map<String, String> labels = Maps.newHashMapWithExpectedSize(fieldSize);
|
||||
|
||||
RowWrapper rowWrapper = metricsData.readRow();
|
||||
while (rowWrapper.hasNextRow()) {
|
||||
rowWrapper = rowWrapper.nextRow();
|
||||
fieldsValue.clear();
|
||||
labels.clear();
|
||||
|
||||
rowWrapper.cellStream().forEach(cell -> {
|
||||
String value = cell.getValue();
|
||||
boolean isLabel = cell.getMetadataAsBoolean(MetricDataConstants.LABEL);
|
||||
byte type = cell.getMetadataAsByte(MetricDataConstants.TYPE);
|
||||
|
||||
if (type == CommonConstants.TYPE_NUMBER && !isLabel) {
|
||||
// number metrics data
|
||||
if (!CommonConstants.NULL_VALUE.equals(value)) {
|
||||
fieldsValue.put(cell.getField().getName(), CommonUtil.parseStrDouble(value));
|
||||
}
|
||||
}
|
||||
// label
|
||||
if (isLabel && !CommonConstants.NULL_VALUE.equals(value)) {
|
||||
labels.put(cell.getField().getName(), value);
|
||||
}
|
||||
});
|
||||
|
||||
for (Map.Entry<String, Double> entry : fieldsValue.entrySet()) {
|
||||
if (entry.getKey() != null && entry.getValue() != null) {
|
||||
try {
|
||||
labels.putAll(defaultLabels);
|
||||
String labelName = isPrometheusAuto ? metricsData.getMetrics()
|
||||
: metricsData.getMetrics() + SPILT + entry.getKey();
|
||||
labels.put(LABEL_KEY_NAME, labelName);
|
||||
if (!isPrometheusAuto) {
|
||||
labels.put(MONITOR_METRIC_KEY, entry.getKey());
|
||||
}
|
||||
labels.put(LABEL_KEY_MONITOR_ID, String.valueOf(metricsData.getId()));
|
||||
// add customized labels as identifier
|
||||
var customizedLabels = metricsData.getLabels();
|
||||
if (!ObjectUtils.isEmpty(customizedLabels)) {
|
||||
labels.putAll(customizedLabels);
|
||||
}
|
||||
VictoriaMetricsContent content = VictoriaMetricsContent.builder()
|
||||
.metric(new HashMap<>(labels))
|
||||
.values(new Double[]{entry.getValue()})
|
||||
.timestamps(timestamp)
|
||||
.build();
|
||||
contentList.add(content);
|
||||
} catch (Exception e) {
|
||||
log.error("combine metrics data error: {}.", e.getMessage(), e);
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} catch (Exception e) {
|
||||
log.error("save metrics data to victoria error: {}.", e.getMessage(), e);
|
||||
}
|
||||
|
||||
if (contentList.isEmpty()) {
|
||||
log.info("[warehouse victoria-metrics] flush metrics data {} is empty, ignore.", metricsData.getId());
|
||||
return;
|
||||
}
|
||||
sendVictoriaMetrics(contentList);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
|
||||
metricsFlushTimer.stop();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryMetricData(String instance, String app, String metrics, String metric, String history) {
|
||||
String labelName = metrics + SPILT + metric;
|
||||
if (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
|
||||
labelName = metrics;
|
||||
}
|
||||
String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + labelName + "\""
|
||||
+ "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\""
|
||||
+ (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX) ? "" : "," + MONITOR_METRIC_KEY + "=\"" + metric + "\"");
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||
if (StringUtils.hasText(victoriaMetricsProp.username())
|
||||
&& StringUtils.hasText(victoriaMetricsProp.password())) {
|
||||
String authStr = victoriaMetricsProp.username() + ":" + victoriaMetricsProp.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
|
||||
URI uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + EXPORT_PATH)
|
||||
.queryParam("match[]", "{" + timeSeriesSelector + "}")
|
||||
.queryParam("start", "now-" + history)
|
||||
.queryParam("end", "now")
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
ResponseEntity<String> responseEntity = restTemplate.exchange(uri,
|
||||
HttpMethod.GET, httpEntity, String.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.debug("query metrics data from victoria-metrics success. {}", uri);
|
||||
if (StringUtils.hasText(responseEntity.getBody())) {
|
||||
String[] contentJsonArr = responseEntity.getBody().split("\n");
|
||||
List<VictoriaMetricsContent> contents = Arrays.stream(contentJsonArr).map(
|
||||
item -> JsonUtil.fromJson(item, VictoriaMetricsContent.class)
|
||||
).toList();
|
||||
for (VictoriaMetricsContent content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && content.getTimestamps() != null) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
|
||||
if (content.getValues().length != content.getTimestamps().length) {
|
||||
log.error("content.getValues().length != content.getTimestamps().length");
|
||||
continue;
|
||||
}
|
||||
Double[] values = content.getValues();
|
||||
Long[] timestamps = content.getTimestamps();
|
||||
for (int index = 0; index < content.getValues().length; index++) {
|
||||
String strValue = BigDecimal.valueOf(values[index]).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
// read timestamp here is ms unit
|
||||
valueList.add(new Value(strValue, timestamps[index]));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, List<Value>> getHistoryIntervalMetricData(String instance, String app, String metrics,
|
||||
String metric, String history) {
|
||||
if (!serverAvailable) {
|
||||
log.error("""
|
||||
|
||||
\t---------------VictoriaMetrics Init Failed---------------
|
||||
\t--------------Please Config VictoriaMetrics--------------
|
||||
\t----------Can Not Use Metric History Now----------
|
||||
""");
|
||||
return Collections.emptyMap();
|
||||
}
|
||||
long endTime = ZonedDateTime.now().toEpochSecond();
|
||||
long startTime;
|
||||
try {
|
||||
if (NumberUtils.isParsable(history)) {
|
||||
startTime = NumberUtils.toLong(history);
|
||||
startTime = (ZonedDateTime.now().toEpochSecond() - startTime);
|
||||
} else {
|
||||
TemporalAmount temporalAmount = TimePeriodUtil.parseTokenTime(history);
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(temporalAmount);
|
||||
startTime = dateTime.toEpochSecond();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("history time error: {}. use default: 6h", e.getMessage());
|
||||
ZonedDateTime dateTime = ZonedDateTime.now().minus(Duration.ofHours(6));
|
||||
startTime = dateTime.toEpochSecond();
|
||||
}
|
||||
String labelName = metrics + SPILT + metric;
|
||||
if (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX)) {
|
||||
labelName = metrics;
|
||||
}
|
||||
String timeSeriesSelector = LABEL_KEY_NAME + "=\"" + labelName + "\""
|
||||
+ "," + LABEL_KEY_INSTANCE + "=\"" + instance + "\""
|
||||
+ (app.startsWith(CommonConstants.PROMETHEUS_APP_PREFIX) ? "" : "," + MONITOR_METRIC_KEY + "=\"" + metric + "\"");
|
||||
Map<String, List<Value>> instanceValuesMap = new HashMap<>(8);
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
headers.setAccept(List.of(MediaType.APPLICATION_JSON));
|
||||
if (StringUtils.hasText(victoriaMetricsProp.username())
|
||||
&& StringUtils.hasText(victoriaMetricsProp.password())) {
|
||||
String authStr = victoriaMetricsProp.username() + ":" + victoriaMetricsProp.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC
|
||||
+ SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
HttpEntity<Void> httpEntity = new HttpEntity<>(headers);
|
||||
URI uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + QUERY_RANGE_PATH)
|
||||
.queryParam("query", "{" + timeSeriesSelector + "}")
|
||||
.queryParam("step", "4h")
|
||||
.queryParam("start", startTime)
|
||||
.queryParam("end", endTime)
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = restTemplate.exchange(uri,
|
||||
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.debug("query metrics data from victoria-metrics success. {}", uri);
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
|
||||
for (Object[] valueArr : content.getValues()) {
|
||||
long timestamp = Long.parseLong(String.valueOf(valueArr[0]));
|
||||
String value = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
// read timestamp here is s unit
|
||||
valueList.add(new Value(value, timestamp * 1000));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
log.error("query metrics data from victoria-metrics failed. {}", responseEntity);
|
||||
}
|
||||
// max
|
||||
uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + QUERY_RANGE_PATH)
|
||||
.queryParam("query", "max_over_time({" + timeSeriesSelector + "})")
|
||||
.queryParam("step", "4h")
|
||||
.queryParam("start", startTime)
|
||||
.queryParam("end", endTime)
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
responseEntity = restTemplate.exchange(uri,
|
||||
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
|
||||
if (valueList.size() == content.getValues().size()) {
|
||||
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
|
||||
Value value = valueList.get(timestampIndex);
|
||||
Object[] valueArr = content.getValues().get(timestampIndex);
|
||||
String maxValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
value.setMax(maxValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// min
|
||||
uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + QUERY_RANGE_PATH)
|
||||
.queryParam("query", "min_over_time({" + timeSeriesSelector + "})")
|
||||
.queryParam("step", "4h")
|
||||
.queryParam("start", startTime)
|
||||
.queryParam("end", endTime)
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
responseEntity = restTemplate.exchange(uri,
|
||||
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
|
||||
if (valueList.size() == content.getValues().size()) {
|
||||
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
|
||||
Value value = valueList.get(timestampIndex);
|
||||
Object[] valueArr = content.getValues().get(timestampIndex);
|
||||
String minValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
value.setMin(minValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
// avg
|
||||
uri = UriComponentsBuilder.fromUriString(victoriaMetricsProp.url() + QUERY_RANGE_PATH)
|
||||
.queryParam("query", "avg_over_time({" + timeSeriesSelector + "})")
|
||||
.queryParam("step", "4h")
|
||||
.queryParam("start", startTime)
|
||||
.queryParam("end", endTime)
|
||||
.build()
|
||||
.encode()
|
||||
.toUri();
|
||||
responseEntity = restTemplate.exchange(uri,
|
||||
HttpMethod.GET, httpEntity, PromQlQueryContent.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
if (responseEntity.getBody() != null && responseEntity.getBody().getData() != null
|
||||
&& responseEntity.getBody().getData().getResult() != null) {
|
||||
List<PromQlQueryContent.ContentData.Content> contents = responseEntity.getBody().getData().getResult();
|
||||
for (PromQlQueryContent.ContentData.Content content : contents) {
|
||||
Map<String, String> labels = content.getMetric();
|
||||
labels.remove(LABEL_KEY_NAME);
|
||||
labels.remove(LABEL_KEY_JOB);
|
||||
labels.remove(LABEL_KEY_INSTANCE);
|
||||
labels.remove(LABEL_KEY_MONITOR_ID);
|
||||
labels.remove(MONITOR_METRICS_KEY);
|
||||
labels.remove(MONITOR_METRIC_KEY);
|
||||
String labelStr = JsonUtil.toJson(labels);
|
||||
if (content.getValues() != null && !content.getValues().isEmpty()) {
|
||||
List<Value> valueList = instanceValuesMap.computeIfAbsent(labelStr, k -> new LinkedList<>());
|
||||
if (valueList.size() == content.getValues().size()) {
|
||||
for (int timestampIndex = 0; timestampIndex < valueList.size(); timestampIndex++) {
|
||||
Value value = valueList.get(timestampIndex);
|
||||
Object[] valueArr = content.getValues().get(timestampIndex);
|
||||
String avgValue = new BigDecimal(String.valueOf(valueArr[1])).setScale(4, RoundingMode.HALF_UP).stripTrailingZeros().toPlainString();
|
||||
value.setMean(avgValue);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("query metrics data from victoria-metrics error. {}.", e.getMessage(), e);
|
||||
}
|
||||
return instanceValuesMap;
|
||||
}
|
||||
|
||||
/**
|
||||
* victoria metrics content
|
||||
*/
|
||||
@Data
|
||||
@Builder
|
||||
@AllArgsConstructor
|
||||
@NoArgsConstructor
|
||||
public static final class VictoriaMetricsContent {
|
||||
|
||||
/**
|
||||
* metric contains metric name plus labels for a particular time series
|
||||
*/
|
||||
private Map<String, String> metric;
|
||||
|
||||
/**
|
||||
* values contains raw sample values for the given time series
|
||||
*/
|
||||
private Double[] values;
|
||||
|
||||
/**
|
||||
* timestamps contains raw sample UNIX timestamps in milliseconds for the given time series
|
||||
* every timestamp is associated with the value at the corresponding position
|
||||
*/
|
||||
private Long[] timestamps;
|
||||
}
|
||||
|
||||
/**
|
||||
* add victoriaMetricsContent to buffer
|
||||
* @param contentList victoriaMetricsContent List
|
||||
*/
|
||||
private void sendVictoriaMetrics(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> contentList) {
|
||||
for (VictoriaMetricsDataStorage.VictoriaMetricsContent content : contentList) {
|
||||
boolean offered = false;
|
||||
int retryCount = 0;
|
||||
while (!offered && retryCount < MAX_RETRIES) {
|
||||
try {
|
||||
// Attempt to add to the queue for a limited time
|
||||
offered = metricsBufferQueue.offer(content, MAX_WAIT_MS, TimeUnit.MILLISECONDS);
|
||||
if (!offered) {
|
||||
// If the queue is still full, trigger an immediate refresh to free up space
|
||||
if (retryCount == 0) {
|
||||
log.debug("victoria metrics buffer queue is full, triggering immediate flush");
|
||||
triggerImmediateFlush();
|
||||
}
|
||||
retryCount++;
|
||||
// The short sleep allows the queue to clear out
|
||||
if (retryCount < MAX_RETRIES) {
|
||||
Thread.sleep(100L * retryCount);
|
||||
}
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
log.error("[Victoria Metrics] Interrupted while offering metrics to buffer queue", e);
|
||||
break;
|
||||
}
|
||||
}
|
||||
// When the maximum number of retries is reached, if it still cannot be added to the queue, the data is saved directly
|
||||
if (!offered) {
|
||||
log.warn("[Victoria Metrics] Failed to add metrics to buffer after {} retries, saving directly", MAX_RETRIES);
|
||||
try {
|
||||
doSaveData(contentList);
|
||||
} catch (Exception e) {
|
||||
log.error("[Victoria Metrics] Failed to save metrics directly: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Refresh in advance to avoid waiting
|
||||
if (metricsBufferQueue.size() >= insertConfig.bufferSize() * 0.8
|
||||
&& draining.compareAndSet(false, true)) {
|
||||
triggerImmediateFlush();
|
||||
}
|
||||
}
|
||||
|
||||
private void triggerImmediateFlush() {
|
||||
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch = new ArrayList<>(insertConfig.bufferSize());
|
||||
metricsBufferQueue.drainTo(batch, insertConfig.bufferSize());
|
||||
draining.set(false);
|
||||
if (!batch.isEmpty()) {
|
||||
metricsFlushTimer.newTimeout(new MetricsFlushTask(batch), 0, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Regularly refresh the buffer queue to the vm
|
||||
*/
|
||||
private class MetricsFlushTask implements TimerTask {
|
||||
private final List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch;
|
||||
|
||||
public MetricsFlushTask(List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batch) {
|
||||
this.batch = batch;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void run(Timeout timeout) {
|
||||
try {
|
||||
if (batch == null) {
|
||||
// If the batch is null, it means that the timer is triggered by flush interval timer
|
||||
List<VictoriaMetricsDataStorage.VictoriaMetricsContent> batchT = new ArrayList<>(insertConfig.bufferSize());
|
||||
metricsBufferQueue.drainTo(batchT, insertConfig.bufferSize());
|
||||
triggerDoSaveData(batchT);
|
||||
// Reschedule the next flush task
|
||||
triggerIntervalFlushTimer();
|
||||
} else {
|
||||
// If the batch is not null, it means that the timer is triggered by the immediate flush
|
||||
triggerDoSaveData(batch);
|
||||
}
|
||||
} catch (Exception e) {
|
||||
log.error("[VictoriaMetrics] flush task error: {}", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
private void triggerDoSaveData(List<VictoriaMetricsContent> batch) {
|
||||
if (!batch.isEmpty()) {
|
||||
doSaveData(batch);
|
||||
log.debug("[Victoria Metrics] Flushed {} metrics items", batch.size());
|
||||
}
|
||||
}
|
||||
|
||||
private void triggerIntervalFlushTimer() {
|
||||
if (metricsFlushTimer != null && !metricsFlushTimer.isStop()) {
|
||||
metricsFlushTimer.newTimeout(new MetricsFlushTask(null), insertConfig.flushInterval(), TimeUnit.SECONDS);
|
||||
log.debug("[Victoria Metrics] Rescheduled next flush task in {} seconds.", insertConfig.flushInterval());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Save metric data to victoria-metric via HTTP call
|
||||
*/
|
||||
private void doSaveData(List<VictoriaMetricsContent> contentList) {
|
||||
try {
|
||||
HttpHeaders headers = new HttpHeaders();
|
||||
headers.setContentType(MediaType.APPLICATION_JSON);
|
||||
if (StringUtils.hasText(victoriaMetricsProp.username())
|
||||
&& StringUtils.hasText(victoriaMetricsProp.password())) {
|
||||
String authStr = victoriaMetricsProp.username() + ":" + victoriaMetricsProp.password();
|
||||
String encodedAuth = Base64Util.encode(authStr);
|
||||
headers.add(HttpHeaders.AUTHORIZATION, NetworkConstants.BASIC + SignConstants.BLANK + encodedAuth);
|
||||
}
|
||||
|
||||
StringBuilder stringBuilder = new StringBuilder();
|
||||
for (VictoriaMetricsContent content : contentList) {
|
||||
stringBuilder.append(JsonUtil.toJson(content)).append("\n");
|
||||
}
|
||||
String payload = stringBuilder.toString();
|
||||
|
||||
Object httpEntity;
|
||||
if (insertConfig.compression().enabled()) {
|
||||
// enable compression
|
||||
headers.set(HttpHeaders.CONTENT_ENCODING, "gzip");
|
||||
// compress the payload using gzip
|
||||
try (ByteArrayOutputStream bos = new ByteArrayOutputStream(payload.length());
|
||||
GZIPOutputStream gzip = new GZIPOutputStream(bos)) {
|
||||
gzip.write(payload.getBytes(StandardCharsets.UTF_8));
|
||||
// finishes writing compressed data before the gzip stream is closed.
|
||||
gzip.finish();
|
||||
httpEntity = new HttpEntity<>(bos.toByteArray(), headers);
|
||||
}
|
||||
} else {
|
||||
httpEntity = new HttpEntity<>(payload, headers);
|
||||
}
|
||||
|
||||
ResponseEntity<String> responseEntity = restTemplate.postForEntity(victoriaMetricsProp.url() + IMPORT_PATH,
|
||||
httpEntity, String.class);
|
||||
if (responseEntity.getStatusCode().is2xxSuccessful()) {
|
||||
log.debug("insert metrics data to victoria-metrics success.");
|
||||
} else {
|
||||
log.error("insert metrics data to victoria-metrics failed. {}", responseEntity.getBody());
|
||||
}
|
||||
} catch (Exception e){
|
||||
log.error("flush metrics data to victoria-metrics error: {}.", e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
}
|
||||
+32
@@ -0,0 +1,32 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* vminsert configuration information
|
||||
*/
|
||||
public record VictoriaMetricsInsertProperties(
|
||||
String url,
|
||||
String username,
|
||||
String password,
|
||||
@DefaultValue("1000") int bufferSize,
|
||||
@DefaultValue("3") int flushInterval
|
||||
) {
|
||||
}
|
||||
+47
@@ -0,0 +1,47 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* Victoria metrics configuration information.
|
||||
*/
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.STORE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.HistoryName.VM)
|
||||
public record VictoriaMetricsProperties(@DefaultValue("false") boolean enabled,
|
||||
@DefaultValue("http://localhost:8428") String url,
|
||||
String username,
|
||||
String password,
|
||||
InsertConfig insert) {
|
||||
|
||||
record InsertConfig(@DefaultValue("100") int bufferSize,
|
||||
@DefaultValue("3") int flushInterval,
|
||||
@DefaultValue Compression compression) {
|
||||
}
|
||||
|
||||
record Compression(@DefaultValue("false") boolean enabled) {
|
||||
}
|
||||
}
|
||||
+28
@@ -0,0 +1,28 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
/**
|
||||
* vmselect configuration information
|
||||
*/
|
||||
public record VictoriaMetricsSelectProperties(
|
||||
String url,
|
||||
String username,
|
||||
String password
|
||||
) {
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.springframework.beans.factory.DisposableBean;
|
||||
|
||||
/**
|
||||
* Real-time data storage abstract class
|
||||
*/
|
||||
@Slf4j
|
||||
public abstract class AbstractRealTimeDataStorage implements RealTimeDataReader, RealTimeDataWriter, DisposableBean {
|
||||
|
||||
protected volatile boolean serverAvailable;
|
||||
|
||||
/**
|
||||
* @return data Whether the storage is available
|
||||
*/
|
||||
@Override
|
||||
public boolean isServerAvailable() {
|
||||
return serverAvailable;
|
||||
}
|
||||
}
|
||||
+50
@@ -0,0 +1,50 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime;
|
||||
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.springframework.lang.NonNull;
|
||||
|
||||
|
||||
/**
|
||||
* Real-time data reading class
|
||||
*/
|
||||
public interface RealTimeDataReader {
|
||||
|
||||
/**
|
||||
* @return data storage available
|
||||
*/
|
||||
boolean isServerAvailable();
|
||||
|
||||
/**
|
||||
* query real-time last metrics data
|
||||
* @param monitorId monitorId
|
||||
* @param metric metric name
|
||||
* @return metrics data
|
||||
*/
|
||||
CollectRep.MetricsData getCurrentMetricsData(@NonNull Long monitorId, @NonNull String metric);
|
||||
|
||||
/**
|
||||
* query real-time last metrics data
|
||||
* @param monitorId monitor id
|
||||
* @return metrics data
|
||||
*/
|
||||
List<CollectRep.MetricsData> getCurrentMetricsData(@NonNull Long monitorId);
|
||||
|
||||
}
|
||||
+38
@@ -0,0 +1,38 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime;
|
||||
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
|
||||
/**
|
||||
* Real-time data writing class
|
||||
*/
|
||||
public interface RealTimeDataWriter {
|
||||
|
||||
/**
|
||||
* @return data storage available
|
||||
*/
|
||||
boolean isServerAvailable();
|
||||
|
||||
/**
|
||||
* save metrics data
|
||||
* @param metricsData metrics data
|
||||
*/
|
||||
void saveData(CollectRep.MetricsData metricsData);
|
||||
|
||||
}
|
||||
+92
@@ -0,0 +1,92 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.memory;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.ConcurrentHashMap;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.AbstractRealTimeDataStorage;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* Store and collect real-time data - memory
|
||||
*/
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.real-time.memory", name = "enabled", havingValue = "true", matchIfMissing = true)
|
||||
@Slf4j
|
||||
public class MemoryDataStorage extends AbstractRealTimeDataStorage {
|
||||
|
||||
/**
|
||||
* monitorId -> metricsName -> data
|
||||
*/
|
||||
private final Map<Long, Map<String, CollectRep.MetricsData>> monitorMetricsDataMap;
|
||||
private static final Integer DEFAULT_INIT_SIZE = 16;
|
||||
private static final Integer METRICS_SIZE = 8;
|
||||
|
||||
public MemoryDataStorage(MemoryProperties memoryProperties) {
|
||||
int initSize = DEFAULT_INIT_SIZE;
|
||||
if (memoryProperties != null
|
||||
&& memoryProperties.initSize() != null) {
|
||||
initSize = memoryProperties.initSize();
|
||||
}
|
||||
monitorMetricsDataMap = new ConcurrentHashMap<>(initSize);
|
||||
this.serverAvailable = true;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CollectRep.MetricsData getCurrentMetricsData(@NonNull Long monitorId, @NonNull String metric) {
|
||||
Map<String, CollectRep.MetricsData> metricsDataMap = monitorMetricsDataMap.computeIfAbsent(monitorId, key -> new ConcurrentHashMap<>(METRICS_SIZE));
|
||||
return metricsDataMap.get(metric);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<CollectRep.MetricsData> getCurrentMetricsData(@NonNull Long monitorId) {
|
||||
Map<String, CollectRep.MetricsData> metricsDataMap = monitorMetricsDataMap.computeIfAbsent(monitorId, key -> new ConcurrentHashMap<>(METRICS_SIZE));
|
||||
return new ArrayList<>(metricsDataMap.values());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
Long monitorId = metricsData.getId();
|
||||
String metrics = metricsData.getMetrics();
|
||||
if (metricsData.getCode() != CollectRep.Code.SUCCESS) {
|
||||
metricsData.close();
|
||||
return;
|
||||
}
|
||||
Map<String, CollectRep.MetricsData> metricsDataMap =
|
||||
monitorMetricsDataMap.computeIfAbsent(monitorId, key -> new ConcurrentHashMap<>(METRICS_SIZE));
|
||||
|
||||
CollectRep.MetricsData oldMetricsData = metricsDataMap.get(metrics);
|
||||
if (oldMetricsData != null) {
|
||||
oldMetricsData.close();
|
||||
}
|
||||
metricsDataMap.put(metrics, metricsData);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() {
|
||||
if (monitorMetricsDataMap != null) {
|
||||
monitorMetricsDataMap.clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
+39
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.memory;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* Memory storage configuration information
|
||||
* @param enabled Whether memory data storage is enabled
|
||||
* @param initSize Memory storage map initialization size
|
||||
*/
|
||||
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.REAL_TIME
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.RealTimeName.MEMORY)
|
||||
public record MemoryProperties(@DefaultValue("true") boolean enabled,
|
||||
@DefaultValue("1024") Integer initSize) {
|
||||
}
|
||||
+82
@@ -0,0 +1,82 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.redis;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.AbstractRealTimeDataStorage;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.RedisCommandDelegate;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.context.annotation.Primary;
|
||||
import org.springframework.lang.NonNull;
|
||||
import org.springframework.stereotype.Component;
|
||||
|
||||
/**
|
||||
* redis storage collects real-time data
|
||||
*/
|
||||
@Primary
|
||||
@Component
|
||||
@ConditionalOnProperty(prefix = "warehouse.real-time.redis", name = "enabled", havingValue = "true")
|
||||
@Slf4j
|
||||
public class RedisDataStorage extends AbstractRealTimeDataStorage {
|
||||
|
||||
private final RedisCommandDelegate redisCommandDelegate;
|
||||
|
||||
public RedisDataStorage(RedisProperties redisProperties) {
|
||||
final RedisCommandDelegate delegate = RedisCommandDelegate.getInstance();
|
||||
this.serverAvailable = delegate.initRedisClient(redisProperties);
|
||||
this.redisCommandDelegate = delegate;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CollectRep.MetricsData getCurrentMetricsData(@NonNull Long monitorId, @NonNull String metric) {
|
||||
return redisCommandDelegate.operate().hget(String.valueOf(monitorId), metric);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<CollectRep.MetricsData> getCurrentMetricsData(@NonNull Long monitorId) {
|
||||
Map<String, CollectRep.MetricsData> metricsDataMap = redisCommandDelegate.operate().hgetAll(String.valueOf(monitorId));
|
||||
return new ArrayList<>(metricsDataMap.values());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void saveData(CollectRep.MetricsData metricsData) {
|
||||
String key = String.valueOf(metricsData.getId());
|
||||
String hashKey = metricsData.getMetrics();
|
||||
if (metricsData.getCode() == CollectRep.Code.SUCCESS) {
|
||||
redisCommandDelegate.operate().hset(key, hashKey, metricsData, future -> future.thenAccept(response -> {
|
||||
metricsData.close();
|
||||
if (response) {
|
||||
log.debug("[warehouse] redis add new data {}:{}.", key, hashKey);
|
||||
} else {
|
||||
log.debug("[warehouse] redis replace data {}:{}.", key, hashKey);
|
||||
}
|
||||
}));
|
||||
} else {
|
||||
metricsData.close();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void destroy() throws Exception {
|
||||
redisCommandDelegate.destroy();
|
||||
}
|
||||
}
|
||||
+41
@@ -0,0 +1,41 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.redis;
|
||||
|
||||
import org.apache.hertzbeat.common.constants.ConfigConstants;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.warehouse.constants.WarehouseConstants;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.DefaultValue;
|
||||
|
||||
/**
|
||||
* Redis configuration information
|
||||
*/
|
||||
|
||||
@ConfigurationProperties(prefix = ConfigConstants.FunctionModuleConstants.WAREHOUSE
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.REAL_TIME
|
||||
+ SignConstants.DOT
|
||||
+ WarehouseConstants.RealTimeName.REDIS)
|
||||
public record RedisProperties(@DefaultValue("false") boolean enabled,
|
||||
@DefaultValue("single") String mode,
|
||||
@DefaultValue("127.0.0.1:6379") String address,
|
||||
String masterName,
|
||||
String password,
|
||||
@DefaultValue("0") Integer db) {
|
||||
}
|
||||
+37
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.redis.client;
|
||||
|
||||
import io.lettuce.core.RedisFuture;
|
||||
import io.lettuce.core.codec.RedisCodec;
|
||||
import java.util.Map;
|
||||
import java.util.function.Consumer;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
|
||||
|
||||
/**
|
||||
* Redis Client Operation
|
||||
*/
|
||||
public interface RedisClientOperation<K, V> extends AutoCloseable {
|
||||
RedisClientOperation<K, V> connect(RedisProperties redisProperties, RedisCodec<K, V> redisCodec);
|
||||
|
||||
V hget(K key, K field);
|
||||
|
||||
Map<K, V> hgetAll(K key);
|
||||
|
||||
void hset(K key, K field, V value, Consumer<RedisFuture<Boolean>> redisFutureConsumer);
|
||||
}
|
||||
+75
@@ -0,0 +1,75 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.redis.client;
|
||||
|
||||
import lombok.extern.slf4j.Slf4j;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.serialize.RedisMetricsDataCodec;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl.RedisClusterClientImpl;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl.RedisSentinelClientImpl;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl.RedisSimpleClientImpl;
|
||||
|
||||
/**
|
||||
* Redis command delegate
|
||||
*/
|
||||
@Slf4j
|
||||
public class RedisCommandDelegate {
|
||||
private static final String SINGLE_MODE = "single";
|
||||
private static final String SENTINEL_MODE = "sentinel";
|
||||
private static final String CLUSTER_MODE = "cluster";
|
||||
private static final RedisCommandDelegate INSTANCE = new RedisCommandDelegate();
|
||||
private RedisClientOperation<String, CollectRep.MetricsData> operation;
|
||||
|
||||
public static RedisCommandDelegate getInstance() {
|
||||
return INSTANCE;
|
||||
}
|
||||
|
||||
public RedisClientOperation<String, CollectRep.MetricsData> operate() {
|
||||
return operation;
|
||||
}
|
||||
|
||||
public void destroy() throws Exception {
|
||||
operation.close();
|
||||
}
|
||||
|
||||
public boolean initRedisClient(RedisProperties redisProperties) {
|
||||
if (redisProperties == null) {
|
||||
log.error("init error, please config Warehouse redis props in application.yml");
|
||||
return false;
|
||||
}
|
||||
|
||||
try {
|
||||
operation = switch (redisProperties.mode()) {
|
||||
case SINGLE_MODE -> new RedisSimpleClientImpl().connect(redisProperties, new RedisMetricsDataCodec());
|
||||
case SENTINEL_MODE -> new RedisSentinelClientImpl().connect(redisProperties, new RedisMetricsDataCodec());
|
||||
case CLUSTER_MODE -> new RedisClusterClientImpl().connect(redisProperties, new RedisMetricsDataCodec());
|
||||
default -> throw new UnsupportedOperationException("Incorrect redis mode: " + redisProperties.mode());
|
||||
};
|
||||
|
||||
return true;
|
||||
} catch (Exception e) {
|
||||
log.error("init redis error {}", e.getMessage(), e);
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private RedisCommandDelegate() {
|
||||
}
|
||||
}
|
||||
+95
@@ -0,0 +1,95 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl;
|
||||
|
||||
import io.lettuce.core.RedisFuture;
|
||||
import io.lettuce.core.RedisURI;
|
||||
import io.lettuce.core.cluster.RedisClusterClient;
|
||||
import io.lettuce.core.cluster.api.StatefulRedisClusterConnection;
|
||||
import io.lettuce.core.codec.RedisCodec;
|
||||
import java.time.Duration;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.HashSet;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.Set;
|
||||
import java.util.function.Consumer;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.RedisClientOperation;
|
||||
|
||||
/**
|
||||
* Redis Client for cluster mode
|
||||
*/
|
||||
public class RedisClusterClientImpl implements RedisClientOperation<String, CollectRep.MetricsData> {
|
||||
private RedisClusterClient redisClusterClient;
|
||||
private StatefulRedisClusterConnection<String, CollectRep.MetricsData> connection;
|
||||
|
||||
@Override
|
||||
public RedisClientOperation<String, CollectRep.MetricsData> connect(RedisProperties redisProperties,
|
||||
RedisCodec<String, CollectRep.MetricsData> redisCodec) {
|
||||
final String[] clusterAddress = redisProperties.address().split(SignConstants.COMMA);
|
||||
Set<RedisURI> clusterUri = new HashSet<>();
|
||||
for (String address : clusterAddress) {
|
||||
final String[] split = address.split(SignConstants.DOUBLE_MARK);
|
||||
RedisURI.Builder uriBuilder = RedisURI.builder()
|
||||
.withHost(split[0])
|
||||
.withPort(Integer.parseInt(split[1]))
|
||||
.withTimeout(Duration.of(10, ChronoUnit.SECONDS));
|
||||
if (StringUtils.isNotBlank(redisProperties.password())) {
|
||||
uriBuilder.withPassword(redisProperties.password().toCharArray());
|
||||
}
|
||||
|
||||
clusterUri.add(uriBuilder.build());
|
||||
}
|
||||
|
||||
redisClusterClient = RedisClusterClient.create(clusterUri);
|
||||
connection = redisClusterClient.connect(redisCodec);
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CollectRep.MetricsData hget(String key, String field) {
|
||||
return connection.sync().hget(key, field);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, CollectRep.MetricsData> hgetAll(String key) {
|
||||
return connection.sync().hgetall(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void hset(String key, String field, CollectRep.MetricsData value, Consumer<RedisFuture<Boolean>> redisFutureConsumer) {
|
||||
final RedisFuture<Boolean> redisFuture = connection.async().hset(key, field, value);
|
||||
if (Objects.nonNull(redisFutureConsumer)) {
|
||||
redisFutureConsumer.accept(redisFuture);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
if (Objects.nonNull(connection)) {
|
||||
connection.close();
|
||||
}
|
||||
if (Objects.nonNull(redisClusterClient)) {
|
||||
redisClusterClient.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+117
@@ -0,0 +1,117 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl;
|
||||
|
||||
import io.lettuce.core.RedisClient;
|
||||
import io.lettuce.core.RedisFuture;
|
||||
import io.lettuce.core.RedisURI;
|
||||
import io.lettuce.core.api.StatefulRedisConnection;
|
||||
import io.lettuce.core.codec.RedisCodec;
|
||||
import io.lettuce.core.sentinel.api.StatefulRedisSentinelConnection;
|
||||
import java.net.InetSocketAddress;
|
||||
import java.net.SocketAddress;
|
||||
import java.time.Duration;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Consumer;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.serialize.RedisMetricsDataCodec;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.RedisClientOperation;
|
||||
|
||||
/**
|
||||
* Redis Client for sentinel mode
|
||||
*/
|
||||
public class RedisSentinelClientImpl implements RedisClientOperation<String, CollectRep.MetricsData> {
|
||||
private RedisClient sentinelRedisClient;
|
||||
private StatefulRedisSentinelConnection<String, String> sentinelConnection;
|
||||
private RedisClient masterRedisClient;
|
||||
private StatefulRedisConnection<String, CollectRep.MetricsData> masterConnection;
|
||||
|
||||
@Override
|
||||
public RedisClientOperation<String, CollectRep.MetricsData> connect(RedisProperties redisProperties,
|
||||
RedisCodec<String, CollectRep.MetricsData> redisCodec) {
|
||||
final String[] address = redisProperties.address().split(SignConstants.DOUBLE_MARK);
|
||||
|
||||
RedisURI.Builder uriBuilder = RedisURI.builder()
|
||||
.withHost(address[0])
|
||||
.withPort(Integer.parseInt(address[1]))
|
||||
.withTimeout(Duration.of(10, ChronoUnit.SECONDS));
|
||||
if (StringUtils.isNotBlank(redisProperties.password())) {
|
||||
uriBuilder.withPassword(redisProperties.password().toCharArray());
|
||||
}
|
||||
|
||||
sentinelRedisClient = RedisClient.create(uriBuilder.build());
|
||||
sentinelConnection = sentinelRedisClient.connectSentinel();
|
||||
final SocketAddress masterSocketAddress = sentinelConnection.sync().getMasterAddrByName(redisProperties.masterName());
|
||||
if (masterSocketAddress instanceof InetSocketAddress masterAddress) {
|
||||
RedisURI.Builder masterUriBuilder = RedisURI.builder()
|
||||
.withHost(masterAddress.getHostName())
|
||||
.withPort(masterAddress.getPort())
|
||||
.withTimeout(Duration.of(10, ChronoUnit.SECONDS))
|
||||
.withDatabase(redisProperties.db());
|
||||
if (StringUtils.isNotBlank(redisProperties.password())) {
|
||||
masterUriBuilder.withPassword(redisProperties.password().toCharArray());
|
||||
}
|
||||
|
||||
masterRedisClient = RedisClient.create(masterUriBuilder.build());
|
||||
masterConnection = masterRedisClient.connect(new RedisMetricsDataCodec());
|
||||
} else {
|
||||
throw new UnsupportedOperationException("Incorrect type of SocketAddress, connect redis sentinel failed...");
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CollectRep.MetricsData hget(String key, String field) {
|
||||
return masterConnection.sync().hget(key, field);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, CollectRep.MetricsData> hgetAll(String key) {
|
||||
return masterConnection.sync().hgetall(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void hset(String key, String field, CollectRep.MetricsData value, Consumer<RedisFuture<Boolean>> redisFutureConsumer) {
|
||||
final RedisFuture<Boolean> redisFuture = masterConnection.async().hset(key, field, value);
|
||||
if (Objects.nonNull(redisFutureConsumer)) {
|
||||
redisFutureConsumer.accept(redisFuture);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
if (Objects.nonNull(masterConnection)) {
|
||||
masterConnection.close();
|
||||
}
|
||||
if (Objects.nonNull(masterRedisClient)) {
|
||||
masterRedisClient.close();
|
||||
}
|
||||
if (Objects.nonNull(sentinelConnection)) {
|
||||
sentinelConnection.close();
|
||||
}
|
||||
if (Objects.nonNull(sentinelRedisClient)) {
|
||||
sentinelRedisClient.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+90
@@ -0,0 +1,90 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.realtime.redis.client.impl;
|
||||
|
||||
import io.lettuce.core.RedisClient;
|
||||
import io.lettuce.core.RedisFuture;
|
||||
import io.lettuce.core.RedisURI;
|
||||
import io.lettuce.core.api.StatefulRedisConnection;
|
||||
import io.lettuce.core.codec.RedisCodec;
|
||||
import java.time.Duration;
|
||||
import java.time.temporal.ChronoUnit;
|
||||
import java.util.Map;
|
||||
import java.util.Objects;
|
||||
import java.util.function.Consumer;
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
import org.apache.hertzbeat.common.constants.SignConstants;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.serialize.RedisMetricsDataCodec;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.client.RedisClientOperation;
|
||||
|
||||
/**
|
||||
* Redis Client
|
||||
*/
|
||||
public class RedisSimpleClientImpl implements RedisClientOperation<String, CollectRep.MetricsData> {
|
||||
private RedisClient redisClient;
|
||||
private StatefulRedisConnection<String, CollectRep.MetricsData> connection;
|
||||
|
||||
@Override
|
||||
public RedisClientOperation<String, CollectRep.MetricsData> connect(RedisProperties redisProperties,
|
||||
RedisCodec<String, CollectRep.MetricsData> redisCodec) {
|
||||
final String[] address = redisProperties.address().split(SignConstants.DOUBLE_MARK);
|
||||
|
||||
RedisURI.Builder uriBuilder = RedisURI.builder()
|
||||
.withHost(address[0])
|
||||
.withPort(Integer.parseInt(address[1]))
|
||||
.withTimeout(Duration.of(10, ChronoUnit.SECONDS))
|
||||
.withDatabase(redisProperties.db());
|
||||
if (StringUtils.isNotBlank(redisProperties.password())) {
|
||||
uriBuilder.withPassword(redisProperties.password().toCharArray());
|
||||
}
|
||||
|
||||
redisClient = RedisClient.create(uriBuilder.build());
|
||||
connection = redisClient.connect(new RedisMetricsDataCodec());
|
||||
return this;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CollectRep.MetricsData hget(String key, String field) {
|
||||
return connection.sync().hget(key, field);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, CollectRep.MetricsData> hgetAll(String key) {
|
||||
return connection.sync().hgetall(key);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void hset(String key, String field, CollectRep.MetricsData value, Consumer<RedisFuture<Boolean>> redisFutureConsumer) {
|
||||
final RedisFuture<Boolean> redisFuture = connection.async().hset(key, field, value);
|
||||
if (Objects.nonNull(redisFutureConsumer)) {
|
||||
redisFutureConsumer.accept(redisFuture);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() throws Exception {
|
||||
if (Objects.nonNull(connection)) {
|
||||
connection.close();
|
||||
}
|
||||
if (Objects.nonNull(redisClient)) {
|
||||
redisClient.close();
|
||||
}
|
||||
}
|
||||
}
|
||||
+16
@@ -0,0 +1,16 @@
|
||||
# Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
# contributor license agreements. See the NOTICE file distributed with
|
||||
# this work for additional information regarding copyright ownership.
|
||||
# The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
# (the "License"); you may not use this file except in compliance with
|
||||
# the License. You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
org.apache.hertzbeat.warehouse.config.WarehouseAutoConfiguration
|
||||
+110
@@ -0,0 +1,110 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.RejectedExecutionException;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import org.apache.hertzbeat.common.concurrent.AdmissionMode;
|
||||
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
|
||||
/**
|
||||
* Test case for {@link WarehouseWorkerPool}
|
||||
*/
|
||||
class WarehouseWorkerPoolTest {
|
||||
|
||||
private static final int NUMBER_OF_THREADS = 10;
|
||||
private WarehouseWorkerPool pool;
|
||||
|
||||
@AfterEach
|
||||
void tearDown() throws Exception {
|
||||
if (pool != null) {
|
||||
pool.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeJob() throws InterruptedException {
|
||||
pool = new WarehouseWorkerPool();
|
||||
AtomicInteger counter = new AtomicInteger();
|
||||
CountDownLatch latch = new CountDownLatch(NUMBER_OF_THREADS);
|
||||
for (int i = 0; i < NUMBER_OF_THREADS; i++) {
|
||||
pool.executeJob(() -> {
|
||||
counter.incrementAndGet();
|
||||
latch.countDown();
|
||||
});
|
||||
}
|
||||
latch.await();
|
||||
|
||||
assertEquals(NUMBER_OF_THREADS, counter.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeJobRunsOnVirtualThread() throws InterruptedException {
|
||||
pool = new WarehouseWorkerPool();
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
AtomicBoolean virtualThread = new AtomicBoolean(false);
|
||||
|
||||
pool.executeJob(() -> {
|
||||
virtualThread.set(Thread.currentThread().isVirtual());
|
||||
latch.countDown();
|
||||
});
|
||||
|
||||
assertTrue(latch.await(5, TimeUnit.SECONDS));
|
||||
assertTrue(virtualThread.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void executeJobRejectsWhenConcurrencyLimitReached() throws InterruptedException {
|
||||
VirtualThreadProperties properties = new VirtualThreadProperties(
|
||||
true,
|
||||
VirtualThreadProperties.PoolProperties.collectorDefaults(),
|
||||
VirtualThreadProperties.PoolProperties.commonDefaults(),
|
||||
VirtualThreadProperties.PoolProperties.managerDefaults(),
|
||||
VirtualThreadProperties.AlerterProperties.defaults(),
|
||||
new VirtualThreadProperties.PoolProperties(AdmissionMode.LIMIT_AND_REJECT, 1),
|
||||
VirtualThreadProperties.AsyncProperties.defaults());
|
||||
pool = new WarehouseWorkerPool(properties);
|
||||
|
||||
CountDownLatch started = new CountDownLatch(1);
|
||||
CountDownLatch release = new CountDownLatch(1);
|
||||
pool.executeJob(() -> {
|
||||
started.countDown();
|
||||
try {
|
||||
release.await(5, TimeUnit.SECONDS);
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
});
|
||||
assertTrue(started.await(5, TimeUnit.SECONDS));
|
||||
|
||||
try {
|
||||
assertThrows(RejectedExecutionException.class, () -> pool.executeJob(() -> {
|
||||
}));
|
||||
} finally {
|
||||
release.countDown();
|
||||
}
|
||||
}
|
||||
}
|
||||
+171
@@ -0,0 +1,171 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.controller;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.jsonPath;
|
||||
import static org.springframework.test.web.servlet.result.MockMvcResultMatchers.status;
|
||||
import jakarta.servlet.ServletException;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.dto.Field;
|
||||
import org.apache.hertzbeat.common.entity.dto.MetricsData;
|
||||
import org.apache.hertzbeat.common.entity.dto.MetricsHistoryData;
|
||||
import org.apache.hertzbeat.warehouse.service.MetricsDataService;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.test.web.servlet.MockMvc;
|
||||
import org.springframework.test.web.servlet.request.MockMvcRequestBuilders;
|
||||
import org.springframework.test.web.servlet.setup.MockMvcBuilders;
|
||||
import org.springframework.util.LinkedMultiValueMap;
|
||||
import org.springframework.util.MultiValueMap;
|
||||
|
||||
/**
|
||||
* Test case for {@link MetricsDataController}
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class MetricsDataControllerTest {
|
||||
|
||||
private MockMvc mockMvc;
|
||||
|
||||
@InjectMocks
|
||||
MetricsDataController metricsDataController;
|
||||
|
||||
@Mock
|
||||
MetricsDataService metricsDataService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
metricsDataController = new MetricsDataController(metricsDataService);
|
||||
this.mockMvc = MockMvcBuilders.standaloneSetup(metricsDataController).build();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getWarehouseStorageServerStatus() throws Exception {
|
||||
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(true);
|
||||
this.mockMvc.perform(MockMvcRequestBuilders.get("/api/warehouse/storage/status"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
|
||||
.andExpect(jsonPath("$.data").isEmpty())
|
||||
.andExpect(jsonPath("$.msg").isEmpty())
|
||||
.andReturn();
|
||||
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(false);
|
||||
this.mockMvc.perform(MockMvcRequestBuilders.get("/api/warehouse/storage/status"))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
|
||||
.andExpect(jsonPath("$.data").isEmpty())
|
||||
.andReturn();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getMetricsData() throws Exception {
|
||||
final long monitorId = 343254354;
|
||||
final String metric = "cpu";
|
||||
final String app = "testapp";
|
||||
final long time = System.currentTimeMillis();
|
||||
final String getUrl = "/api/monitor/" + monitorId + "/metrics/" + metric;
|
||||
|
||||
when(metricsDataService.getMetricsData(eq(monitorId), eq(metric))).thenReturn(null);
|
||||
this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
|
||||
.andExpect(jsonPath("$.msg").value("query metrics data is empty"))
|
||||
.andExpect(jsonPath("$.data").isEmpty())
|
||||
.andReturn();
|
||||
|
||||
MetricsData metricsData = MetricsData.builder()
|
||||
.id(monitorId)
|
||||
.app(app)
|
||||
.metrics(metric)
|
||||
.time(time)
|
||||
.build();
|
||||
|
||||
when(metricsDataService.getMetricsData(eq(monitorId), eq(metric))).thenReturn(metricsData);
|
||||
this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
|
||||
.andExpect(jsonPath("$.msg").isEmpty())
|
||||
.andExpect(jsonPath("$.data.id").value(monitorId))
|
||||
.andExpect(jsonPath("$.data.metrics").value(metric))
|
||||
.andExpect(jsonPath("$.data.app").value(app))
|
||||
.andExpect(jsonPath("$.data.time").value(time))
|
||||
.andReturn();
|
||||
}
|
||||
|
||||
@Test
|
||||
void getMetricHistoryData() throws Exception {
|
||||
final String instance = "127.0.0.1:8081";
|
||||
final String app = "linux";
|
||||
final String metrics = "cpu";
|
||||
final String metric = "usage";
|
||||
final String metricFull = "linux.cpu.usage";
|
||||
final String metricFullFail = "linux.usage";
|
||||
final String label = "disk2";
|
||||
final String history = "6h";
|
||||
final Boolean interval = false;
|
||||
final String getUrl = "/api/monitor/" + instance + "/metric/" + metricFull;
|
||||
final String getUrlFail = "/api/monitor/" + instance + "/metric/" + metricFullFail;
|
||||
|
||||
MultiValueMap<String, String> params = new LinkedMultiValueMap<>();
|
||||
params.add("instance", instance);
|
||||
params.add("label", label);
|
||||
params.add("history", history);
|
||||
params.add("interval", String.valueOf(interval));
|
||||
|
||||
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(false);
|
||||
this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl).params(params))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
|
||||
.andExpect(jsonPath("$.msg").value("time series database not available"))
|
||||
.andExpect(jsonPath("$.data").isEmpty())
|
||||
.andReturn();
|
||||
|
||||
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(true);
|
||||
ServletException exception = assertThrows(ServletException.class, () -> this.mockMvc.perform(MockMvcRequestBuilders.get(getUrlFail).params(params))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value((int) CommonConstants.FAIL_CODE))
|
||||
.andExpect(jsonPath("$.data").isEmpty())
|
||||
.andReturn());
|
||||
assertTrue(exception.getMessage().contains("IllegalArgumentException"));
|
||||
|
||||
MetricsHistoryData metricsHistoryData = MetricsHistoryData.builder()
|
||||
.instance(instance)
|
||||
.metrics(metrics)
|
||||
.field(Field.builder().name(metric).type(CommonConstants.TYPE_NUMBER).build())
|
||||
.build();
|
||||
when(metricsDataService.getWarehouseStorageServerStatus()).thenReturn(true);
|
||||
lenient().when(metricsDataService.getMetricHistoryData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history), eq(interval)))
|
||||
.thenReturn(metricsHistoryData);
|
||||
this.mockMvc.perform(MockMvcRequestBuilders.get(getUrl).params(params))
|
||||
.andExpect(status().isOk())
|
||||
.andExpect(jsonPath("$.code").value((int) CommonConstants.SUCCESS_CODE))
|
||||
.andExpect(jsonPath("$.data.instance").value(instance))
|
||||
.andExpect(jsonPath("$.data.metrics").value(metrics))
|
||||
.andExpect(jsonPath("$.data.field.name").value(metric))
|
||||
.andExpect(jsonPath("$.data.field.type").value(String.valueOf(CommonConstants.TYPE_NUMBER)))
|
||||
.andExpect(jsonPath("$.msg").isEmpty())
|
||||
.andReturn();
|
||||
}
|
||||
}
|
||||
+136
@@ -0,0 +1,136 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.greptime.GreptimeSqlQueryContent;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
/**
|
||||
* Test case for {@link GreptimeSqlQueryExecutor}
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class GreptimeSqlQueryExecutorTest {
|
||||
|
||||
@Mock
|
||||
private GreptimeProperties greptimeProperties;
|
||||
|
||||
@Mock
|
||||
private RestTemplate restTemplate;
|
||||
|
||||
private GreptimeSqlQueryExecutor greptimeSqlQueryExecutor;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
when(greptimeProperties.httpEndpoint()).thenReturn("http://127.0.0.1:4000");
|
||||
when(greptimeProperties.database()).thenReturn("hertzbeat");
|
||||
when(greptimeProperties.username()).thenReturn("username");
|
||||
when(greptimeProperties.password()).thenReturn("password");
|
||||
|
||||
greptimeSqlQueryExecutor = new GreptimeSqlQueryExecutor(greptimeProperties, restTemplate);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExecuteSuccess() {
|
||||
// Mock successful response
|
||||
GreptimeSqlQueryContent mockResponse = createMockResponse();
|
||||
ResponseEntity<GreptimeSqlQueryContent> responseEntity =
|
||||
new ResponseEntity<>(mockResponse, HttpStatus.OK);
|
||||
|
||||
when(restTemplate.exchange(
|
||||
any(String.class),
|
||||
eq(HttpMethod.POST),
|
||||
any(HttpEntity.class),
|
||||
eq(GreptimeSqlQueryContent.class)
|
||||
)).thenReturn(responseEntity);
|
||||
|
||||
// Execute
|
||||
List<Map<String, Object>> result = greptimeSqlQueryExecutor.execute("SELECT * FROM metrics");
|
||||
|
||||
// Verify
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("cpu", result.get(0).get("metric_name"));
|
||||
assertEquals(85.5, result.get(0).get("value"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExecuteError() {
|
||||
// Mock error response
|
||||
when(restTemplate.exchange(
|
||||
any(String.class),
|
||||
eq(HttpMethod.POST),
|
||||
any(HttpEntity.class),
|
||||
eq(GreptimeSqlQueryContent.class)
|
||||
)).thenThrow(new RuntimeException("Connection error"));
|
||||
|
||||
// Execute
|
||||
assertThrows(RuntimeException.class, () -> greptimeSqlQueryExecutor.execute("SELECT * FROM metrics"));
|
||||
}
|
||||
|
||||
private GreptimeSqlQueryContent createMockResponse() {
|
||||
GreptimeSqlQueryContent response = new GreptimeSqlQueryContent();
|
||||
response.setCode(0);
|
||||
|
||||
// Create simple schema
|
||||
List<GreptimeSqlQueryContent.Output.Records.Schema.ColumnSchema> columnSchemas = new ArrayList<>();
|
||||
columnSchemas.add(new GreptimeSqlQueryContent.Output.Records.Schema.ColumnSchema("metric_name", "String"));
|
||||
columnSchemas.add(new GreptimeSqlQueryContent.Output.Records.Schema.ColumnSchema("value", "Float64"));
|
||||
|
||||
GreptimeSqlQueryContent.Output.Records.Schema schema =
|
||||
new GreptimeSqlQueryContent.Output.Records.Schema();
|
||||
schema.setColumnSchemas(columnSchemas);
|
||||
|
||||
// Create simple row
|
||||
List<List<Object>> rows = new ArrayList<>();
|
||||
rows.add(List.of("cpu", 85.5));
|
||||
|
||||
// Build response structure
|
||||
GreptimeSqlQueryContent.Output.Records records =
|
||||
new GreptimeSqlQueryContent.Output.Records();
|
||||
records.setSchema(schema);
|
||||
records.setRows(rows);
|
||||
|
||||
GreptimeSqlQueryContent.Output output = new GreptimeSqlQueryContent.Output();
|
||||
output.setRecords(records);
|
||||
|
||||
response.setOutput(List.of(output));
|
||||
return response;
|
||||
}
|
||||
}
|
||||
+207
@@ -0,0 +1,207 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one
|
||||
* or more contributor license agreements. See the NOTICE file
|
||||
* distributed with this work for additional information
|
||||
* regarding copyright ownership. The ASF licenses this file
|
||||
* to you under the Apache License, Version 2.0 (the
|
||||
* "License"); you may not use this file except in compliance
|
||||
* with the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing,
|
||||
* software distributed under the License is distributed on an
|
||||
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
||||
* KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations
|
||||
* under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.db;
|
||||
|
||||
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.PROMQL;
|
||||
import static org.apache.hertzbeat.warehouse.constants.WarehouseConstants.RANGE;
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import java.net.URI;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQuery;
|
||||
import org.apache.hertzbeat.common.entity.dto.query.DatasourceQueryData;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.PromQlQueryContent;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsClusterProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsInsertProperties;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.VictoriaMetricsSelectProperties;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
/**
|
||||
* Test case for {@link VictoriaMetricsClusterQueryExecutor}
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class VictoriaMetricsClusterQueryExecutorTest {
|
||||
|
||||
@Mock
|
||||
private RestTemplate restTemplate;
|
||||
|
||||
private VictoriaMetricsClusterQueryExecutor queryExecutor;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
VictoriaMetricsClusterProperties properties = new VictoriaMetricsClusterProperties(
|
||||
true,
|
||||
"42",
|
||||
new VictoriaMetricsInsertProperties("http://vm-insert:8480", "insert-user", "insert-pass", 1000, 3),
|
||||
new VictoriaMetricsSelectProperties("http://vm-select:8481", "select-user", "select-pass")
|
||||
);
|
||||
queryExecutor = new VictoriaMetricsClusterQueryExecutor(properties, restTemplate);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetDatasource() {
|
||||
assertEquals("VictoriaMetricsCluster", queryExecutor.getDatasource());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExecuteSuccess() {
|
||||
PromQlQueryContent mockResponse = createInstantResponse();
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = new ResponseEntity<>(mockResponse, HttpStatus.OK);
|
||||
when(restTemplate.exchange(
|
||||
any(URI.class),
|
||||
eq(HttpMethod.GET),
|
||||
any(HttpEntity.class),
|
||||
eq(PromQlQueryContent.class)
|
||||
)).thenReturn(responseEntity);
|
||||
|
||||
List<Map<String, Object>> result = queryExecutor.execute("up");
|
||||
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("cpu_usage", result.getFirst().get("__name__"));
|
||||
assertEquals("server1", result.getFirst().get("instance"));
|
||||
assertEquals(1712300000.0d, result.getFirst().get("__timestamp__"));
|
||||
assertEquals("85.5", result.getFirst().get("__value__"));
|
||||
|
||||
ArgumentCaptor<URI> uriCaptor = ArgumentCaptor.forClass(URI.class);
|
||||
verify(restTemplate).exchange(
|
||||
uriCaptor.capture(),
|
||||
eq(HttpMethod.GET),
|
||||
any(HttpEntity.class),
|
||||
eq(PromQlQueryContent.class)
|
||||
);
|
||||
assertEquals("http", uriCaptor.getValue().getScheme());
|
||||
assertEquals("vm-select", uriCaptor.getValue().getHost());
|
||||
assertEquals(8481, uriCaptor.getValue().getPort());
|
||||
assertEquals("/select/42/prometheus/api/v1/query", uriCaptor.getValue().getPath());
|
||||
assertTrue(uriCaptor.getValue().getQuery().contains("query=up"));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testExecuteErrorReturnsEmptyResult() {
|
||||
when(restTemplate.exchange(
|
||||
any(URI.class),
|
||||
eq(HttpMethod.GET),
|
||||
any(HttpEntity.class),
|
||||
eq(PromQlQueryContent.class)
|
||||
)).thenThrow(new RuntimeException("Connection error"));
|
||||
|
||||
List<Map<String, Object>> result = queryExecutor.execute("up");
|
||||
|
||||
assertNotNull(result);
|
||||
assertEquals(0, result.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQueryRangeSuccess() {
|
||||
PromQlQueryContent mockResponse = createRangeResponse();
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = new ResponseEntity<>(mockResponse, HttpStatus.OK);
|
||||
when(restTemplate.exchange(
|
||||
any(URI.class),
|
||||
eq(HttpMethod.GET),
|
||||
any(HttpEntity.class),
|
||||
eq(PromQlQueryContent.class)
|
||||
)).thenReturn(responseEntity);
|
||||
|
||||
DatasourceQuery datasourceQuery = DatasourceQuery.builder()
|
||||
.refId("A")
|
||||
.datasource(queryExecutor.getDatasource())
|
||||
.expr("avg_over_time(up[5m])")
|
||||
.exprType(PROMQL)
|
||||
.timeType(RANGE)
|
||||
.start(1712300000L)
|
||||
.end(1712300300L)
|
||||
.step("60s")
|
||||
.build();
|
||||
|
||||
DatasourceQueryData result = queryExecutor.query(datasourceQuery);
|
||||
|
||||
assertNotNull(result);
|
||||
assertEquals("A", result.getRefId());
|
||||
assertEquals(200, result.getStatus());
|
||||
assertNotNull(result.getFrames());
|
||||
assertEquals(1, result.getFrames().size());
|
||||
assertEquals("cpu_usage", result.getFrames().getFirst().getSchema().getLabels().get("__name__"));
|
||||
assertEquals("server1", result.getFrames().getFirst().getSchema().getLabels().get("instance"));
|
||||
assertEquals(1712300000000L, result.getFrames().getFirst().getData().get(0)[0]);
|
||||
assertEquals("85.5", result.getFrames().getFirst().getData().get(0)[1]);
|
||||
assertEquals(1712300060000L, result.getFrames().getFirst().getData().get(1)[0]);
|
||||
assertEquals("86.1", result.getFrames().getFirst().getData().get(1)[1]);
|
||||
|
||||
ArgumentCaptor<URI> uriCaptor = ArgumentCaptor.forClass(URI.class);
|
||||
verify(restTemplate).exchange(
|
||||
uriCaptor.capture(),
|
||||
eq(HttpMethod.GET),
|
||||
any(HttpEntity.class),
|
||||
eq(PromQlQueryContent.class)
|
||||
);
|
||||
assertEquals("/select/42/prometheus/api/v1/query_range", uriCaptor.getValue().getPath());
|
||||
assertTrue(uriCaptor.getValue().getQuery().contains("query=avg_over_time"));
|
||||
assertTrue(uriCaptor.getValue().getQuery().contains("start=1712300000"));
|
||||
assertTrue(uriCaptor.getValue().getQuery().contains("end=1712300300"));
|
||||
assertTrue(uriCaptor.getValue().getQuery().contains("step=60s"));
|
||||
}
|
||||
|
||||
private PromQlQueryContent createInstantResponse() {
|
||||
PromQlQueryContent response = new PromQlQueryContent();
|
||||
PromQlQueryContent.ContentData data = new PromQlQueryContent.ContentData();
|
||||
PromQlQueryContent.ContentData.Content content = new PromQlQueryContent.ContentData.Content();
|
||||
|
||||
content.setMetric(Map.of("__name__", "cpu_usage", "instance", "server1"));
|
||||
content.setValue(new Object[] {1712300000.0d, "85.5"});
|
||||
|
||||
data.setResult(List.of(content));
|
||||
response.setData(data);
|
||||
return response;
|
||||
}
|
||||
|
||||
private PromQlQueryContent createRangeResponse() {
|
||||
PromQlQueryContent response = new PromQlQueryContent();
|
||||
PromQlQueryContent.ContentData data = new PromQlQueryContent.ContentData();
|
||||
PromQlQueryContent.ContentData.Content content = new PromQlQueryContent.ContentData.Content();
|
||||
|
||||
content.setMetric(Map.of("__name__", "cpu_usage", "instance", "server1"));
|
||||
content.setValues(List.of(
|
||||
new Object[] {1712300000L, "85.5"},
|
||||
new Object[] {1712300060L, "86.1"}
|
||||
));
|
||||
|
||||
data.setResult(List.of(content));
|
||||
response.setData(data);
|
||||
return response;
|
||||
}
|
||||
}
|
||||
+115
@@ -0,0 +1,115 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.eq;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import java.util.HashMap;
|
||||
import java.util.Optional;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.common.support.exception.CommonException;
|
||||
import org.apache.hertzbeat.warehouse.service.impl.MetricsDataServiceImpl;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.HistoryDataReader;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.RealTimeDataReader;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
|
||||
/**
|
||||
* Test case for {@link MetricsDataService}
|
||||
*/
|
||||
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
public class MetricsDataServiceTest {
|
||||
|
||||
@InjectMocks
|
||||
private MetricsDataServiceImpl metricsDataService;
|
||||
|
||||
@Mock
|
||||
private RealTimeDataReader realTimeDataReader;
|
||||
|
||||
@Mock
|
||||
private HistoryDataReader historyDataReader;
|
||||
|
||||
@BeforeEach
|
||||
public void setUp() {
|
||||
metricsDataService = new MetricsDataServiceImpl(realTimeDataReader, Optional.of(historyDataReader));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetWarehouseStorageServerStatus() {
|
||||
when(historyDataReader.isServerAvailable()).thenReturn(false);
|
||||
assertFalse(metricsDataService.getWarehouseStorageServerStatus());
|
||||
|
||||
when(historyDataReader.isServerAvailable()).thenReturn(true);
|
||||
assertTrue(metricsDataService.getWarehouseStorageServerStatus());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetMetricsData() {
|
||||
Long monitorId = 1L;
|
||||
String metrics = "disk";
|
||||
|
||||
when(realTimeDataReader.isServerAvailable()).thenReturn(false);
|
||||
assertThrows(CommonException.class, () -> metricsDataService.getMetricsData(monitorId, metrics), "real time store not available");
|
||||
|
||||
|
||||
when(realTimeDataReader.isServerAvailable()).thenReturn(true);
|
||||
when(realTimeDataReader.getCurrentMetricsData(eq(monitorId), eq(metrics))).thenReturn(null);
|
||||
assertNull(metricsDataService.getMetricsData(monitorId, metrics));
|
||||
|
||||
CollectRep.MetricsData storageData = CollectRep.MetricsData.newBuilder()
|
||||
.setId(monitorId)
|
||||
.setMetrics(metrics)
|
||||
.build();
|
||||
when(realTimeDataReader.isServerAvailable()).thenReturn(true);
|
||||
when(realTimeDataReader.getCurrentMetricsData(eq(monitorId), eq(metrics))).thenReturn(storageData);
|
||||
assertNotNull(metricsDataService.getMetricsData(monitorId, metrics));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetMetricHistoryData() {
|
||||
String instance = "127.0.0.1:8080";
|
||||
String app = "linux";
|
||||
String metrics = "disk";
|
||||
String metric = "used";
|
||||
String label = "label";
|
||||
String history = "6h";
|
||||
Boolean intervalFalse = false;
|
||||
Boolean intervalTrue = true;
|
||||
|
||||
when(historyDataReader.getHistoryMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history))).thenReturn(new HashMap<>());
|
||||
assertNotNull(metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, intervalFalse));
|
||||
verify(historyDataReader, times(1)).getHistoryMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history));
|
||||
|
||||
when(historyDataReader.getHistoryIntervalMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history))).thenReturn(new HashMap<>());
|
||||
assertNotNull(metricsDataService.getMetricHistoryData(instance, app, metrics, metric, history, intervalTrue));
|
||||
verify(historyDataReader, times(1)).getHistoryIntervalMetricData(eq(instance), eq(app), eq(metrics), eq(metric), eq(history));
|
||||
}
|
||||
}
|
||||
+88
@@ -0,0 +1,88 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.service;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.anyLong;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.service.impl.WarehouseServiceImpl;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.AbstractRealTimeDataStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
import org.springframework.test.context.junit.jupiter.SpringExtension;
|
||||
|
||||
/**
|
||||
* test case for {@link WarehouseServiceImpl}
|
||||
*/
|
||||
|
||||
@ExtendWith(SpringExtension.class)
|
||||
class WarehouseServiceTest {
|
||||
|
||||
@Mock
|
||||
private AbstractRealTimeDataStorage realTimeDataStorage;
|
||||
|
||||
@InjectMocks
|
||||
private WarehouseServiceImpl warehouseService;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
|
||||
MockitoAnnotations.openMocks(this);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQueryMonitorMetricsData() {
|
||||
|
||||
Long monitorId = 1L;
|
||||
List<CollectRep.MetricsData> expectedData = Collections.emptyList();
|
||||
|
||||
when(realTimeDataStorage.isServerAvailable()).thenReturn(true);
|
||||
when(realTimeDataStorage.getCurrentMetricsData(monitorId)).thenReturn(expectedData);
|
||||
|
||||
List<CollectRep.MetricsData> result = warehouseService.queryMonitorMetricsData(monitorId);
|
||||
|
||||
assertEquals(expectedData, result);
|
||||
verify(realTimeDataStorage, times(1)).isServerAvailable();
|
||||
verify(realTimeDataStorage, times(1)).getCurrentMetricsData(monitorId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQueryMonitorMetricsDataNotAvailable() {
|
||||
|
||||
Long monitorId = 1L;
|
||||
|
||||
when(realTimeDataStorage.isServerAvailable()).thenReturn(false);
|
||||
|
||||
List<CollectRep.MetricsData> result = warehouseService.queryMonitorMetricsData(monitorId);
|
||||
|
||||
assertTrue(result.isEmpty());
|
||||
verify(realTimeDataStorage, times(1)).isServerAvailable();
|
||||
verify(realTimeDataStorage, never()).getCurrentMetricsData(anyLong());
|
||||
}
|
||||
}
|
||||
+122
@@ -0,0 +1,122 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.when;
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.memory.MemoryDataStorage;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.memory.MemoryProperties;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.InjectMocks;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
|
||||
/**
|
||||
* Test case for {@link MemoryDataStorage}
|
||||
*/
|
||||
|
||||
class MemoryDataStorageTest {
|
||||
|
||||
@Mock
|
||||
private MemoryProperties memoryProperties;
|
||||
|
||||
@InjectMocks
|
||||
private MemoryDataStorage memoryDataStorage;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
|
||||
MockitoAnnotations.openMocks(this);
|
||||
|
||||
when(memoryProperties.initSize()).thenReturn(null);
|
||||
memoryDataStorage = new MemoryDataStorage(memoryProperties);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetCurrentMetricsDataByMetric() {
|
||||
|
||||
Long monitorId = 1L;
|
||||
String metric = "cpuUsage";
|
||||
CollectRep.MetricsData metricsData = mock(CollectRep.MetricsData.class);
|
||||
|
||||
memoryDataStorage.saveData(metricsData);
|
||||
|
||||
CollectRep.MetricsData result = memoryDataStorage.getCurrentMetricsData(monitorId, metric);
|
||||
|
||||
assertNull(result);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetCurrentMetricsData() {
|
||||
|
||||
Long monitorId = 1L;
|
||||
CollectRep.MetricsData metricsData1 = mock(CollectRep.MetricsData.class);
|
||||
CollectRep.MetricsData metricsData2 = mock(CollectRep.MetricsData.class);
|
||||
|
||||
when(metricsData1.getId()).thenReturn(monitorId);
|
||||
when(metricsData1.getMetrics()).thenReturn("cpuUsage");
|
||||
when(metricsData1.getCode()).thenReturn(CollectRep.Code.SUCCESS);
|
||||
|
||||
when(metricsData2.getId()).thenReturn(monitorId);
|
||||
when(metricsData2.getMetrics()).thenReturn("memoryUsage");
|
||||
when(metricsData2.getCode()).thenReturn(CollectRep.Code.SUCCESS);
|
||||
|
||||
memoryDataStorage.saveData(metricsData1);
|
||||
memoryDataStorage.saveData(metricsData2);
|
||||
|
||||
List<CollectRep.MetricsData> result = memoryDataStorage.getCurrentMetricsData(monitorId);
|
||||
|
||||
assertEquals(2, result.size());
|
||||
assertTrue(result.contains(metricsData1));
|
||||
assertTrue(result.contains(metricsData2));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveDataFailure() {
|
||||
|
||||
CollectRep.MetricsData metricsData = mock(CollectRep.MetricsData.class);
|
||||
when(metricsData.getCode()).thenReturn(CollectRep.Code.FAIL);
|
||||
|
||||
memoryDataStorage.saveData(metricsData);
|
||||
|
||||
List<CollectRep.MetricsData> result = memoryDataStorage.getCurrentMetricsData(metricsData.getId());
|
||||
assertTrue(result.isEmpty());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDestroy() {
|
||||
|
||||
CollectRep.MetricsData metricsData = mock(CollectRep.MetricsData.class);
|
||||
when(metricsData.getId()).thenReturn(1L);
|
||||
when(metricsData.getMetrics()).thenReturn("cpuUsage");
|
||||
when(metricsData.getCode()).thenReturn(CollectRep.Code.SUCCESS);
|
||||
|
||||
memoryDataStorage.saveData(metricsData);
|
||||
memoryDataStorage.destroy();
|
||||
|
||||
List<CollectRep.MetricsData> result = memoryDataStorage.getCurrentMetricsData(1L);
|
||||
assertTrue(result.isEmpty());
|
||||
}
|
||||
|
||||
}
|
||||
+85
@@ -0,0 +1,85 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import com.google.common.collect.Lists;
|
||||
import java.util.List;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.store.realtime.redis.RedisDataStorage;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.MockitoAnnotations;
|
||||
|
||||
/**
|
||||
* Test case for {@link RedisDataStorage}
|
||||
*/
|
||||
class RedisDataStorageTest {
|
||||
@Mock
|
||||
private RedisDataStorage redisDataStorage;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
MockitoAnnotations.openMocks(this);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetCurrentMetricsData() {
|
||||
Long monitorId = 1L;
|
||||
String metric = "testMetric";
|
||||
CollectRep.MetricsData expectedMetricsData = CollectRep.MetricsData.newBuilder().setMetrics(metric).setId(monitorId).build();
|
||||
Mockito.doReturn(expectedMetricsData).when(redisDataStorage).getCurrentMetricsData(monitorId, metric);
|
||||
|
||||
CollectRep.MetricsData actualMetricsData = redisDataStorage.getCurrentMetricsData(monitorId, metric);
|
||||
|
||||
assertNotNull(actualMetricsData);
|
||||
assertEquals(expectedMetricsData, actualMetricsData);
|
||||
|
||||
verify(redisDataStorage, times(1)).getCurrentMetricsData(monitorId, metric);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetCurrentMetricsDataByMonitorId() {
|
||||
Long monitorId = 1L;
|
||||
String metric = "testMetric";
|
||||
CollectRep.MetricsData expectedMetricsData = CollectRep.MetricsData.newBuilder().setMetrics(metric).setId(monitorId).build();
|
||||
Mockito.doReturn(Lists.newArrayList(expectedMetricsData)).when(redisDataStorage).getCurrentMetricsData(monitorId);
|
||||
|
||||
List<CollectRep.MetricsData> actualMetricsData = redisDataStorage.getCurrentMetricsData(monitorId);
|
||||
|
||||
assertNotNull(actualMetricsData);
|
||||
assertEquals(expectedMetricsData, actualMetricsData.get(0));
|
||||
|
||||
verify(redisDataStorage, times(1)).getCurrentMetricsData(monitorId);
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveData() {
|
||||
long monitorId = 1L;
|
||||
String metric = "testMetric";
|
||||
CollectRep.MetricsData expectedMetricsData = CollectRep.MetricsData.newBuilder().setMetrics(metric).setId(monitorId).build();
|
||||
redisDataStorage.saveData(expectedMetricsData);
|
||||
|
||||
verify(redisDataStorage, times(1)).saveData(expectedMetricsData);
|
||||
}
|
||||
}
|
||||
+206
@@ -0,0 +1,206 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import org.apache.arrow.vector.types.pojo.ArrowType;
|
||||
import org.apache.arrow.vector.types.pojo.Field;
|
||||
import org.apache.arrow.vector.types.pojo.FieldType;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.ArrowCell;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.tdengine.TdEngineDataStorage;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.tdengine.TdEngineProperties;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
import org.mockito.quality.Strictness;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.Statement;
|
||||
import java.util.List;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.Mockito.atLeastOnce;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
/**
|
||||
* Test case for {@link TdEngineDataStorage}
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@MockitoSettings(strictness = Strictness.LENIENT)
|
||||
class TdEngineDataStorageTest {
|
||||
|
||||
@Mock
|
||||
private TdEngineProperties tdEngineProperties;
|
||||
|
||||
@Mock
|
||||
private HikariDataSource mockHikariDataSource;
|
||||
|
||||
@Mock
|
||||
private Connection mockConnection;
|
||||
|
||||
@Mock
|
||||
private Statement mockStatement;
|
||||
|
||||
private TdEngineDataStorage tdEngineDataStorage;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() throws Exception {
|
||||
when(tdEngineProperties.enabled()).thenReturn(true);
|
||||
when(tdEngineProperties.url()).thenReturn("jdbc:TAOS-RS://localhost:6041/demo");
|
||||
when(tdEngineProperties.username()).thenReturn("root");
|
||||
when(tdEngineProperties.password()).thenReturn("root");
|
||||
when(tdEngineProperties.tableStrColumnDefineMaxLength()).thenReturn(200);
|
||||
when(mockHikariDataSource.getConnection()).thenReturn(mockConnection);
|
||||
when(mockConnection.createStatement()).thenReturn(mockStatement);
|
||||
}
|
||||
|
||||
@Test
|
||||
void isServerAvailable() {
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveData() throws Exception {
|
||||
tdEngineDataStorage = new TdEngineDataStorage(tdEngineProperties);
|
||||
setPrivateField(tdEngineDataStorage, "hikariDataSource", mockHikariDataSource);
|
||||
setParentPrivateField(tdEngineDataStorage, "serverAvailable", true);
|
||||
|
||||
CollectRep.MetricsData metricsData = generateMockedMetricsData();
|
||||
tdEngineDataStorage.saveData(metricsData);
|
||||
|
||||
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
|
||||
verify(mockStatement, atLeastOnce()).execute(sqlCaptor.capture());
|
||||
|
||||
String executedSql = sqlCaptor.getValue();
|
||||
|
||||
// Verify SQL structure
|
||||
assertNotNull(executedSql);
|
||||
assertTrue(executedSql.startsWith("INSERT INTO"), "SQL should start with INSERT INTO");
|
||||
assertTrue(executedSql.contains("USING"), "SQL should contain USING clause");
|
||||
assertTrue(executedSql.contains("TAGS"), "SQL should contain TAGS clause");
|
||||
assertTrue(executedSql.contains("VALUES"), "SQL should contain VALUES clause");
|
||||
|
||||
// Verify table name format: app_metrics_instance_v2
|
||||
assertTrue(executedSql.contains("app_cpu_test-%server-01_v2"), "Should contain correct table name");
|
||||
// Verify super table name format: app_metrics_super_v2
|
||||
assertTrue(executedSql.contains("app_cpu_super_v2"), "Should contain correct super table name");
|
||||
// Verify tags format: test-%server-01
|
||||
assertTrue(executedSql.contains("TAGS ('test-%server-01')"), "Should contain correct super table name");
|
||||
// Verify VALUES clause structure (timestamp + data values)
|
||||
assertTrue(executedSql.matches(".*VALUES\\s+\\(\\d+.*68\\.7\\)"), "Should contain timestamp and value 68.7");
|
||||
}
|
||||
|
||||
@Test
|
||||
void destroy() {
|
||||
}
|
||||
|
||||
@Test
|
||||
void getHistoryMetricData() {
|
||||
}
|
||||
|
||||
@Test
|
||||
void getHistoryIntervalMetricData() {
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to set private field using reflection
|
||||
*/
|
||||
private void setPrivateField(Object target, String fieldName, Object value) throws Exception {
|
||||
java.lang.reflect.Field field = target.getClass().getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
|
||||
/**
|
||||
* Helper method to set private field from parent class using reflection
|
||||
*/
|
||||
private void setParentPrivateField(Object target, String fieldName, Object value) throws Exception {
|
||||
java.lang.reflect.Field field = target.getClass().getSuperclass().getDeclaredField(fieldName);
|
||||
field.setAccessible(true);
|
||||
field.set(target, value);
|
||||
}
|
||||
|
||||
public static CollectRep.MetricsData generateMockedMetricsData() {
|
||||
CollectRep.MetricsData mockMetricsData = Mockito.mock(CollectRep.MetricsData.class);
|
||||
|
||||
when(mockMetricsData.getId()).thenReturn(0L);
|
||||
when(mockMetricsData.getMetrics()).thenReturn("cpu");
|
||||
when(mockMetricsData.getTime()).thenReturn(System.currentTimeMillis());
|
||||
when(mockMetricsData.getCode()).thenReturn(CollectRep.Code.SUCCESS);
|
||||
when(mockMetricsData.getApp()).thenReturn("app");
|
||||
when(mockMetricsData.getInstance()).thenReturn("test-%server-01");
|
||||
|
||||
CollectRep.ValueRow mockValueRow = Mockito.mock(CollectRep.ValueRow.class);
|
||||
List<String> columnValues = List.of("test-%server-01", "68.7");
|
||||
when(mockValueRow.getColumnsList()).thenReturn(columnValues);
|
||||
when(mockValueRow.getColumns(0)).thenReturn("test-%server-01");
|
||||
when(mockValueRow.getColumns(1)).thenReturn("68.7");
|
||||
List<CollectRep.ValueRow> mockValueRowsList = List.of(mockValueRow);
|
||||
when(mockMetricsData.getValues()).thenReturn(mockValueRowsList);
|
||||
|
||||
CollectRep.Field instanceField = Mockito.mock(CollectRep.Field.class);
|
||||
when(instanceField.getName()).thenReturn("instance");
|
||||
CollectRep.Field usageField = Mockito.mock(CollectRep.Field.class);
|
||||
when(usageField.getName()).thenReturn("usage");
|
||||
CollectRep.Field systemField = Mockito.mock(CollectRep.Field.class);
|
||||
when(systemField.getName()).thenReturn("system");
|
||||
List<CollectRep.Field> mockFields = List.of(instanceField, usageField, systemField);
|
||||
when(mockMetricsData.getFields()).thenReturn(mockFields);
|
||||
|
||||
ArrowType instanceArrowType = new ArrowType.Utf8();
|
||||
FieldType instanceFieldType = new FieldType(true, instanceArrowType, null, null);
|
||||
Field instanceArrowField = new Field("instance", instanceFieldType, null);
|
||||
ArrowCell instanceCell = Mockito.mock(ArrowCell.class);
|
||||
when(instanceCell.getField()).thenReturn(instanceArrowField);
|
||||
when(instanceCell.getValue()).thenReturn("test-%server-01");
|
||||
when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true);
|
||||
when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);
|
||||
|
||||
ArrowType usageArrowType = new ArrowType.Utf8();
|
||||
FieldType usageFieldType = new FieldType(true, usageArrowType, null, null);
|
||||
Field usageArrowField = new Field("usage", usageFieldType, null);
|
||||
ArrowCell usageCell = Mockito.mock(ArrowCell.class);
|
||||
when(usageCell.getField()).thenReturn(usageArrowField);
|
||||
when(usageCell.getValue()).thenReturn("68.7");
|
||||
when(usageCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(false);
|
||||
when(usageCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_NUMBER);
|
||||
List<ArrowCell> mockCells = List.of(instanceCell, usageCell);
|
||||
|
||||
// Create Arrow Field list for RowWrapper
|
||||
List<org.apache.arrow.vector.types.pojo.Field> arrowFields = List.of(instanceArrowField, usageArrowField);
|
||||
|
||||
RowWrapper mockRowWrapper = Mockito.mock(RowWrapper.class);
|
||||
when(mockRowWrapper.hasNextRow()).thenReturn(true).thenReturn(false);
|
||||
when(mockRowWrapper.nextRow()).thenReturn(mockRowWrapper);
|
||||
when(mockRowWrapper.cellStream()).thenAnswer(invocation -> mockCells.stream());
|
||||
when(mockRowWrapper.getFieldList()).thenReturn(arrowFields);
|
||||
when(mockMetricsData.readRow()).thenReturn(mockRowWrapper);
|
||||
return mockMetricsData;
|
||||
}
|
||||
|
||||
}
|
||||
+314
@@ -0,0 +1,314 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.doris;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockConstruction;
|
||||
import static org.mockito.Mockito.mockStatic;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import com.zaxxer.hikari.HikariDataSource;
|
||||
import java.sql.Connection;
|
||||
import java.sql.DriverManager;
|
||||
import java.sql.PreparedStatement;
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.Statement;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.warehouse.WarehouseWorkerPool;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockedConstruction;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
|
||||
/**
|
||||
* Tests for {@link DorisDataStorage}.
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class DorisDataStorageTest {
|
||||
|
||||
private static final long NANOS_PER_MILLISECOND = 1_000_000L;
|
||||
|
||||
@Mock
|
||||
private WarehouseWorkerPool workerPool;
|
||||
|
||||
@Test
|
||||
void queryLogsWithPaginationShouldContainLimitAndOffsetClauses() throws Exception {
|
||||
QueryStorageContext context = createQueryStorageContext(createProperties(false));
|
||||
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
|
||||
when(context.resultSet().next()).thenReturn(false);
|
||||
|
||||
context.storage().queryLogsByMultipleConditionsWithPagination(
|
||||
1000L, 2000L, null, null, null, null, null, 5, 20
|
||||
);
|
||||
|
||||
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
|
||||
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
|
||||
String sql = sqlCaptor.getValue();
|
||||
|
||||
assertThat(sql).contains("ORDER BY time_unix_nano DESC");
|
||||
assertThat(sql).contains("LIMIT ?");
|
||||
assertThat(sql).contains("OFFSET ?");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryLogsShouldBindAllConditionsWithConvertedTimeInOrder() throws Exception {
|
||||
QueryStorageContext context = createQueryStorageContext(createProperties(false));
|
||||
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
|
||||
when(context.resultSet().next()).thenReturn(false);
|
||||
|
||||
context.storage().queryLogsByMultipleConditions(
|
||||
1000L, 2000L, "trace-1", "span-1", 9, "INFO", "error"
|
||||
);
|
||||
|
||||
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
|
||||
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
|
||||
String sql = sqlCaptor.getValue();
|
||||
|
||||
assertThat(sql).contains("WHERE");
|
||||
assertThat(sql).contains("time_unix_nano >= ?");
|
||||
assertThat(sql).contains("time_unix_nano <= ?");
|
||||
assertThat(sql).contains("trace_id = ?");
|
||||
assertThat(sql).contains("span_id = ?");
|
||||
assertThat(sql).contains("severity_number = ?");
|
||||
assertThat(sql).contains("severity_text = ?");
|
||||
assertThat(sql).contains("body LIKE ?");
|
||||
assertThat(sql).contains("ORDER BY time_unix_nano DESC");
|
||||
|
||||
verify(context.queryPreparedStatement()).setObject(1, 1000L * NANOS_PER_MILLISECOND);
|
||||
verify(context.queryPreparedStatement()).setObject(2, 2000L * NANOS_PER_MILLISECOND);
|
||||
verify(context.queryPreparedStatement()).setObject(3, "trace-1");
|
||||
verify(context.queryPreparedStatement()).setObject(4, "span-1");
|
||||
verify(context.queryPreparedStatement()).setObject(5, 9);
|
||||
verify(context.queryPreparedStatement()).setObject(6, "INFO");
|
||||
verify(context.queryPreparedStatement()).setObject(7, "%error%");
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryLogsWithPaginationShouldNotAppendOffsetWhenOffsetIsZero() throws Exception {
|
||||
QueryStorageContext context = createQueryStorageContext(createProperties(false));
|
||||
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
|
||||
when(context.resultSet().next()).thenReturn(false);
|
||||
|
||||
context.storage().queryLogsByMultipleConditionsWithPagination(
|
||||
null, null, null, null, null, null, null, 0, 20
|
||||
);
|
||||
|
||||
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
|
||||
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
|
||||
String sql = sqlCaptor.getValue();
|
||||
|
||||
assertThat(sql).contains("LIMIT ?");
|
||||
assertThat(sql).doesNotContain("OFFSET ?");
|
||||
verify(context.queryPreparedStatement()).setObject(1, 20);
|
||||
}
|
||||
|
||||
@Test
|
||||
void countLogsShouldReturnCountAndBindConditions() throws Exception {
|
||||
QueryStorageContext context = createQueryStorageContext(createProperties(false));
|
||||
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
|
||||
when(context.resultSet().next()).thenReturn(true);
|
||||
when(context.resultSet().getLong("count")).thenReturn(5L);
|
||||
|
||||
long count = context.storage().countLogsByMultipleConditions(
|
||||
1000L, null, "trace-1", null, null, null, null
|
||||
);
|
||||
|
||||
assertThat(count).isEqualTo(5L);
|
||||
|
||||
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
|
||||
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
|
||||
String sql = sqlCaptor.getValue();
|
||||
|
||||
assertThat(sql).contains("SELECT COUNT(*) AS count");
|
||||
assertThat(sql).contains("time_unix_nano >= ?");
|
||||
assertThat(sql).contains("trace_id = ?");
|
||||
verify(context.queryPreparedStatement()).setObject(1, 1000L * NANOS_PER_MILLISECOND);
|
||||
verify(context.queryPreparedStatement()).setObject(2, "trace-1");
|
||||
}
|
||||
|
||||
@Test
|
||||
void batchDeleteLogsShouldFilterNullValuesAndExecuteDelete() throws Exception {
|
||||
QueryStorageContext context = createQueryStorageContext(createProperties(false));
|
||||
when(context.queryPreparedStatement().executeUpdate()).thenReturn(2);
|
||||
|
||||
List<Long> timeUnixNanos = new ArrayList<>();
|
||||
timeUnixNanos.add(111L);
|
||||
timeUnixNanos.add(null);
|
||||
timeUnixNanos.add(333L);
|
||||
|
||||
boolean deleted = context.storage().batchDeleteLogs(timeUnixNanos);
|
||||
|
||||
assertThat(deleted).isTrue();
|
||||
|
||||
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
|
||||
verify(context.queryConnection()).prepareStatement(sqlCaptor.capture());
|
||||
assertThat(sqlCaptor.getValue()).contains("time_unix_nano IN (?,?)");
|
||||
verify(context.queryPreparedStatement()).setLong(1, 111L);
|
||||
verify(context.queryPreparedStatement()).setLong(2, 333L);
|
||||
}
|
||||
|
||||
@Test
|
||||
void batchDeleteLogsShouldReturnFalseWhenAllValuesAreNull() throws Exception {
|
||||
DorisDataStorage storage = createStorageContext(createProperties(false));
|
||||
|
||||
List<Long> timeUnixNanos = new ArrayList<>();
|
||||
timeUnixNanos.add(null);
|
||||
timeUnixNanos.add(null);
|
||||
|
||||
boolean deleted = storage.batchDeleteLogs(timeUnixNanos);
|
||||
|
||||
assertThat(deleted).isFalse();
|
||||
}
|
||||
|
||||
@Test
|
||||
void queryLogsShouldMapJsonColumnsToLogEntryFields() throws Exception {
|
||||
QueryStorageContext context = createQueryStorageContext(createProperties(false));
|
||||
when(context.queryPreparedStatement().executeQuery()).thenReturn(context.resultSet());
|
||||
ResultSet resultSet = context.resultSet();
|
||||
|
||||
when(resultSet.next()).thenReturn(true, false);
|
||||
when(resultSet.getLong("time_unix_nano")).thenReturn(111L);
|
||||
when(resultSet.getLong("observed_time_unix_nano")).thenReturn(222L);
|
||||
when(resultSet.getInt("severity_number")).thenReturn(9);
|
||||
when(resultSet.getString("severity_text")).thenReturn("INFO");
|
||||
when(resultSet.getString("body")).thenReturn("{\"message\":\"ok\"}");
|
||||
when(resultSet.getString("trace_id")).thenReturn("trace-1");
|
||||
when(resultSet.getString("span_id")).thenReturn("span-1");
|
||||
when(resultSet.getInt("trace_flags")).thenReturn(1);
|
||||
when(resultSet.getString("attributes")).thenReturn("{\"k\":\"v\"}");
|
||||
when(resultSet.getString("resource")).thenReturn("{\"service\":\"warehouse\"}");
|
||||
when(resultSet.getString("instrumentation_scope")).thenReturn("{\"name\":\"scope\",\"version\":\"1.0.0\"}");
|
||||
when(resultSet.getInt("dropped_attributes_count")).thenReturn(3);
|
||||
when(resultSet.wasNull()).thenReturn(false, false, false);
|
||||
|
||||
List<LogEntry> entries = context.storage().queryLogsByMultipleConditions(
|
||||
null, null, null, null, null, null, null
|
||||
);
|
||||
|
||||
assertThat(entries).hasSize(1);
|
||||
LogEntry entry = entries.get(0);
|
||||
assertThat(entry.getTimeUnixNano()).isEqualTo(111L);
|
||||
assertThat(entry.getObservedTimeUnixNano()).isEqualTo(222L);
|
||||
assertThat(entry.getSeverityNumber()).isEqualTo(9);
|
||||
assertThat(entry.getSeverityText()).isEqualTo("INFO");
|
||||
assertThat(entry.getTraceId()).isEqualTo("trace-1");
|
||||
assertThat(entry.getSpanId()).isEqualTo("span-1");
|
||||
assertThat(entry.getBody()).isInstanceOf(Map.class);
|
||||
@SuppressWarnings("unchecked")
|
||||
Map<String, Object> body = (Map<String, Object>) entry.getBody();
|
||||
assertThat(body).containsEntry("message", "ok");
|
||||
assertThat(entry.getAttributes()).containsEntry("k", "v");
|
||||
assertThat(entry.getResource()).containsEntry("service", "warehouse");
|
||||
assertThat(entry.getInstrumentationScope()).isNotNull();
|
||||
assertThat(entry.getInstrumentationScope().getName()).isEqualTo("scope");
|
||||
assertThat(entry.getInstrumentationScope().getVersion()).isEqualTo("1.0.0");
|
||||
}
|
||||
|
||||
private QueryStorageContext createQueryStorageContext(DorisProperties properties) {
|
||||
Connection initConnection = mock(Connection.class);
|
||||
Statement initStatement = mock(Statement.class);
|
||||
Connection tableConnection = mock(Connection.class);
|
||||
Statement tableStatement = mock(Statement.class);
|
||||
Connection queryConnection = mock(Connection.class);
|
||||
PreparedStatement queryPreparedStatement = mock(PreparedStatement.class);
|
||||
ResultSet resultSet = mock(ResultSet.class);
|
||||
AtomicInteger dataSourceConnectionCalls = new AtomicInteger(0);
|
||||
String baseUrl = properties.url();
|
||||
String username = properties.username();
|
||||
String password = properties.password();
|
||||
try {
|
||||
when(initConnection.createStatement()).thenReturn(initStatement);
|
||||
when(tableConnection.createStatement()).thenReturn(tableStatement);
|
||||
when(queryConnection.prepareStatement(anyString())).thenReturn(queryPreparedStatement);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
try (MockedStatic<DriverManager> driverManagerMock = mockStatic(DriverManager.class);
|
||||
MockedConstruction<HikariDataSource> hikariDataSourceMockedConstruction = mockConstruction(
|
||||
HikariDataSource.class,
|
||||
(mock, context) -> when(mock.getConnection()).thenAnswer(invocation -> {
|
||||
int call = dataSourceConnectionCalls.getAndIncrement();
|
||||
return call == 0 ? tableConnection : queryConnection;
|
||||
}))) {
|
||||
driverManagerMock.when(() -> DriverManager.getConnection(baseUrl, username, password))
|
||||
.thenReturn(initConnection);
|
||||
DorisDataStorage storage = new DorisDataStorage(properties, workerPool);
|
||||
return new QueryStorageContext(storage, queryConnection, queryPreparedStatement, resultSet);
|
||||
}
|
||||
}
|
||||
|
||||
private DorisDataStorage createStorageContext(DorisProperties properties) {
|
||||
Connection initConnection = mock(Connection.class);
|
||||
Statement initStatement = mock(Statement.class);
|
||||
Connection tableConnection = mock(Connection.class);
|
||||
Statement tableStatement = mock(Statement.class);
|
||||
String baseUrl = properties.url();
|
||||
String username = properties.username();
|
||||
String password = properties.password();
|
||||
try {
|
||||
when(initConnection.createStatement()).thenReturn(initStatement);
|
||||
when(tableConnection.createStatement()).thenReturn(tableStatement);
|
||||
} catch (Exception e) {
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
|
||||
try (MockedStatic<DriverManager> driverManagerMock = mockStatic(DriverManager.class);
|
||||
MockedConstruction<HikariDataSource> hikariDataSourceMockedConstruction = mockConstruction(
|
||||
HikariDataSource.class, (mock, context) -> when(mock.getConnection()).thenReturn(tableConnection))) {
|
||||
driverManagerMock.when(() -> DriverManager.getConnection(baseUrl, username, password))
|
||||
.thenReturn(initConnection);
|
||||
return new DorisDataStorage(properties, workerPool);
|
||||
}
|
||||
}
|
||||
|
||||
private DorisProperties createProperties(boolean enablePartition) {
|
||||
DorisProperties.TableConfig tableConfig = new DorisProperties.TableConfig(
|
||||
enablePartition, "HOUR", 2, 1, 12, 1, 4096
|
||||
);
|
||||
DorisProperties.PoolConfig poolConfig = new DorisProperties.PoolConfig(
|
||||
1, 2, 500, 0, 60_000
|
||||
);
|
||||
DorisProperties.WriteConfig writeConfig = new DorisProperties.WriteConfig(
|
||||
"jdbc", 1000, 5, false, DorisProperties.StreamLoadConfig.createDefault()
|
||||
);
|
||||
return new DorisProperties(
|
||||
true,
|
||||
"jdbc:mysql://127.0.0.1:9030/hertzbeat",
|
||||
"root",
|
||||
"123456",
|
||||
tableConfig,
|
||||
poolConfig,
|
||||
writeConfig
|
||||
);
|
||||
}
|
||||
|
||||
private record QueryStorageContext(DorisDataStorage storage, Connection queryConnection,
|
||||
PreparedStatement queryPreparedStatement, ResultSet resultSet) {
|
||||
}
|
||||
}
|
||||
+148
@@ -0,0 +1,148 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.duckdb;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
|
||||
import java.nio.file.Path;
|
||||
import java.util.concurrent.CountDownLatch;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicBoolean;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
import org.apache.hertzbeat.common.config.VirtualThreadProperties;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.io.TempDir;
|
||||
|
||||
/**
|
||||
* Test case for {@link DuckdbDatabaseDataStorage}.
|
||||
*/
|
||||
class DuckdbDatabaseDataStorageTest {
|
||||
|
||||
private TestDuckdbDatabaseDataStorage dataStorage;
|
||||
|
||||
@TempDir
|
||||
Path tempDir;
|
||||
|
||||
@AfterEach
|
||||
void tearDown() throws Exception {
|
||||
if (dataStorage != null) {
|
||||
dataStorage.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void dispatchExpiredDataCleanerRunsOnVirtualThread() throws Exception {
|
||||
CountDownLatch latch = new CountDownLatch(1);
|
||||
AtomicBoolean virtualThread = new AtomicBoolean(false);
|
||||
dataStorage = new TestDuckdbDatabaseDataStorage(properties(), latch, virtualThread,
|
||||
null, null, null, null, null, null);
|
||||
|
||||
dataStorage.dispatchExpiredDataCleaner();
|
||||
|
||||
assertTrue(latch.await(5, TimeUnit.SECONDS));
|
||||
assertTrue(virtualThread.get());
|
||||
}
|
||||
|
||||
@Test
|
||||
void dispatchExpiredDataCleanerDoesNotRunConcurrently() throws Exception {
|
||||
CountDownLatch firstStarted = new CountDownLatch(1);
|
||||
CountDownLatch releaseFirst = new CountDownLatch(1);
|
||||
CountDownLatch secondStarted = new CountDownLatch(1);
|
||||
AtomicInteger concurrent = new AtomicInteger();
|
||||
AtomicInteger maxConcurrent = new AtomicInteger();
|
||||
AtomicInteger invocations = new AtomicInteger();
|
||||
dataStorage = new TestDuckdbDatabaseDataStorage(properties(), null, null,
|
||||
firstStarted, releaseFirst, secondStarted, concurrent, maxConcurrent, invocations);
|
||||
|
||||
dataStorage.dispatchExpiredDataCleaner();
|
||||
assertTrue(firstStarted.await(5, TimeUnit.SECONDS));
|
||||
|
||||
dataStorage.dispatchExpiredDataCleaner();
|
||||
assertFalse(secondStarted.await(200, TimeUnit.MILLISECONDS));
|
||||
|
||||
releaseFirst.countDown();
|
||||
assertTrue(secondStarted.await(5, TimeUnit.SECONDS));
|
||||
assertEquals(1, maxConcurrent.get());
|
||||
}
|
||||
|
||||
private DuckdbProperties properties() {
|
||||
return new DuckdbProperties(true, "1d", tempDir.resolve("history.duckdb").toString());
|
||||
}
|
||||
|
||||
private static final class TestDuckdbDatabaseDataStorage extends DuckdbDatabaseDataStorage {
|
||||
|
||||
private final CountDownLatch virtualThreadLatch;
|
||||
private final AtomicBoolean virtualThread;
|
||||
private final CountDownLatch firstStarted;
|
||||
private final CountDownLatch releaseFirst;
|
||||
private final CountDownLatch secondStarted;
|
||||
private final AtomicInteger concurrent;
|
||||
private final AtomicInteger maxConcurrent;
|
||||
private final AtomicInteger invocations;
|
||||
|
||||
private TestDuckdbDatabaseDataStorage(DuckdbProperties duckdbProperties,
|
||||
CountDownLatch virtualThreadLatch,
|
||||
AtomicBoolean virtualThread,
|
||||
CountDownLatch firstStarted,
|
||||
CountDownLatch releaseFirst,
|
||||
CountDownLatch secondStarted,
|
||||
AtomicInteger concurrent,
|
||||
AtomicInteger maxConcurrent,
|
||||
AtomicInteger invocations) {
|
||||
super(duckdbProperties, new VirtualThreadProperties(), false);
|
||||
this.virtualThreadLatch = virtualThreadLatch;
|
||||
this.virtualThread = virtualThread;
|
||||
this.firstStarted = firstStarted;
|
||||
this.releaseFirst = releaseFirst;
|
||||
this.secondStarted = secondStarted;
|
||||
this.concurrent = concurrent;
|
||||
this.maxConcurrent = maxConcurrent;
|
||||
this.invocations = invocations;
|
||||
}
|
||||
|
||||
@Override
|
||||
void beforeExpiredDataCleanerRun() {
|
||||
if (virtualThreadLatch != null && virtualThread != null) {
|
||||
virtualThread.set(Thread.currentThread().isVirtual());
|
||||
virtualThreadLatch.countDown();
|
||||
}
|
||||
if (firstStarted == null || releaseFirst == null || secondStarted == null
|
||||
|| concurrent == null || maxConcurrent == null || invocations == null) {
|
||||
return;
|
||||
}
|
||||
int running = concurrent.incrementAndGet();
|
||||
maxConcurrent.accumulateAndGet(running, Math::max);
|
||||
int currentInvocation = invocations.incrementAndGet();
|
||||
try {
|
||||
if (currentInvocation == 1) {
|
||||
firstStarted.countDown();
|
||||
releaseFirst.await(5, TimeUnit.SECONDS);
|
||||
} else if (currentInvocation == 2) {
|
||||
secondStarted.countDown();
|
||||
}
|
||||
} catch (InterruptedException e) {
|
||||
Thread.currentThread().interrupt();
|
||||
} finally {
|
||||
concurrent.decrementAndGet();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
+460
@@ -0,0 +1,460 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.greptime;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.junit.jupiter.api.Assertions.assertFalse;
|
||||
import static org.junit.jupiter.api.Assertions.assertNotNull;
|
||||
import static org.junit.jupiter.api.Assertions.assertThrows;
|
||||
import static org.junit.jupiter.api.Assertions.assertTrue;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.Mockito.lenient;
|
||||
import static org.mockito.Mockito.mock;
|
||||
import static org.mockito.Mockito.mockStatic;
|
||||
import static org.mockito.Mockito.never;
|
||||
import static org.mockito.Mockito.times;
|
||||
import static org.mockito.Mockito.verify;
|
||||
import static org.mockito.Mockito.when;
|
||||
|
||||
import io.greptime.GreptimeDB;
|
||||
import io.greptime.models.Err;
|
||||
import io.greptime.models.Result;
|
||||
import io.greptime.models.Table;
|
||||
import io.greptime.models.WriteOk;
|
||||
import io.greptime.v1.Common;
|
||||
import io.greptime.v1.RowData;
|
||||
import java.lang.reflect.Field;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Collections;
|
||||
import java.util.HashMap;
|
||||
import java.util.LinkedHashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.CompletableFuture;
|
||||
import java.util.stream.Collectors;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.ArrowCell;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.dto.Value;
|
||||
import org.apache.hertzbeat.common.entity.log.LogEntry;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.apache.hertzbeat.warehouse.db.GreptimeSqlQueryExecutor;
|
||||
import org.apache.hertzbeat.warehouse.store.history.tsdb.vm.PromQlQueryContent;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
import org.mockito.ArgumentCaptor;
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.MockedStatic;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
/**
|
||||
* Test case for {@link GreptimeDbDataStorage}
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
class GreptimeDbDataStorageTest {
|
||||
|
||||
@Mock
|
||||
private GreptimeProperties greptimeProperties;
|
||||
|
||||
@Mock
|
||||
private RestTemplate restTemplate;
|
||||
|
||||
@Mock
|
||||
private GreptimeSqlQueryExecutor greptimeSqlQueryExecutor;
|
||||
|
||||
@Mock
|
||||
private GreptimeDB greptimeDb;
|
||||
|
||||
private GreptimeDbDataStorage greptimeDbDataStorage;
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
lenient().when(greptimeProperties.grpcEndpoints()).thenReturn("127.0.0.1:4001");
|
||||
lenient().when(greptimeProperties.database()).thenReturn("hertzbeat");
|
||||
lenient().when(greptimeProperties.username()).thenReturn("username");
|
||||
lenient().when(greptimeProperties.password()).thenReturn("password");
|
||||
lenient().when(greptimeProperties.httpEndpoint()).thenReturn("http://127.0.0.1:4000");
|
||||
}
|
||||
|
||||
@Test
|
||||
void testConstructor() {
|
||||
// Test constructor with null properties
|
||||
assertThrows(IllegalArgumentException.class, () -> new GreptimeDbDataStorage(null, restTemplate, greptimeSqlQueryExecutor));
|
||||
|
||||
// Test successful constructor initialization
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
GreptimeDbDataStorage storage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
assertNotNull(storage);
|
||||
assertTrue(storage.isServerAvailable());
|
||||
}
|
||||
|
||||
// Test constructor when GreptimeDB.create throws an exception
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenThrow(new RuntimeException("Connection failed"));
|
||||
GreptimeDbDataStorage storage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
assertNotNull(storage);
|
||||
assertFalse(storage.isServerAvailable());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveData() {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
|
||||
// Mock the write result
|
||||
@SuppressWarnings("unchecked")
|
||||
Result<WriteOk, Err> mockResult = mock(Result.class);
|
||||
when(mockResult.isOk()).thenReturn(true);
|
||||
CompletableFuture<Result<WriteOk, Err>> mockFuture = CompletableFuture.completedFuture(mockResult);
|
||||
when(greptimeDb.write(any(Table.class))).thenReturn(mockFuture);
|
||||
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
|
||||
// Test with valid metrics data
|
||||
CollectRep.MetricsData metricsData = createMockMetricsData(true);
|
||||
greptimeDbDataStorage.saveData(metricsData);
|
||||
verify(greptimeDb, times(1)).write(any(Table.class));
|
||||
|
||||
// Test with failure code
|
||||
CollectRep.MetricsData failMetricsData = mock(CollectRep.MetricsData.class);
|
||||
when(failMetricsData.getCode()).thenReturn(CollectRep.Code.FAIL);
|
||||
greptimeDbDataStorage.saveData(failMetricsData);
|
||||
// Verify write was not called again
|
||||
verify(greptimeDb, times(1)).write(any(Table.class));
|
||||
|
||||
// Test with empty values
|
||||
CollectRep.MetricsData emptyMetricsData = createMockMetricsData(false);
|
||||
greptimeDbDataStorage.saveData(emptyMetricsData);
|
||||
// Verify write was not called again
|
||||
verify(greptimeDb, times(1)).write(any(Table.class));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveDataWithCustomLabels() throws Exception {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
|
||||
Result<WriteOk, Err> mockResult = mock(Result.class);
|
||||
when(mockResult.isOk()).thenReturn(true);
|
||||
CompletableFuture<Result<WriteOk, Err>> mockFuture = CompletableFuture.completedFuture(mockResult);
|
||||
when(greptimeDb.write(any(Table.class))).thenReturn(mockFuture);
|
||||
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
|
||||
CollectRep.MetricsData metricsData = createMockMetricsData(true);
|
||||
Map<String, String> customLabels = new LinkedHashMap<>();
|
||||
customLabels.put("instance", "bad_instance");
|
||||
customLabels.put("ts", "bad_ts");
|
||||
customLabels.put("usage", "bad_usage");
|
||||
customLabels.put("env", "prod");
|
||||
when(metricsData.getLabels()).thenReturn(customLabels);
|
||||
when(metricsData.getInstance()).thenReturn("server1");
|
||||
|
||||
ArgumentCaptor<Table> tableCaptor = ArgumentCaptor.forClass(Table.class);
|
||||
greptimeDbDataStorage.saveData(metricsData);
|
||||
|
||||
verify(greptimeDb).write(tableCaptor.capture());
|
||||
Table capturedTable = tableCaptor.getValue();
|
||||
|
||||
List<RowData.ColumnSchema> columnSchemas = getColumnSchemas(capturedTable);
|
||||
List<String> columnNames = columnSchemas.stream()
|
||||
.map(RowData.ColumnSchema::getColumnName)
|
||||
.collect(Collectors.toList());
|
||||
assertEquals(5, columnNames.size());
|
||||
assertEquals(2, Collections.frequency(columnNames, "instance"));
|
||||
assertEquals(1, Collections.frequency(columnNames, "ts"));
|
||||
assertEquals(1, Collections.frequency(columnNames, "usage"));
|
||||
|
||||
List<RowData.ColumnSchema> envColumnSchemas = columnSchemas.stream()
|
||||
.filter(columnSchema -> "env".equals(columnSchema.getColumnName()))
|
||||
.collect(Collectors.toList());
|
||||
assertEquals(1, envColumnSchemas.size());
|
||||
assertEquals(Common.SemanticType.TAG, envColumnSchemas.get(0).getSemanticType());
|
||||
|
||||
List<RowData.Row> rows = getRows(capturedTable);
|
||||
assertEquals(1, rows.size());
|
||||
RowData.Row row = rows.get(0);
|
||||
assertEquals("prod", row.getValuesList().get(4).getStringValue());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testGetHistoryMetricData() {
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
|
||||
PromQlQueryContent content = createMockPromQlQueryContent();
|
||||
ResponseEntity<PromQlQueryContent> responseEntity = new ResponseEntity<>(content, HttpStatus.OK);
|
||||
|
||||
when(restTemplate.exchange(any(), eq(HttpMethod.GET), any(HttpEntity.class), eq(PromQlQueryContent.class)))
|
||||
.thenReturn(responseEntity);
|
||||
|
||||
Map<String, List<Value>> result = greptimeDbDataStorage.getHistoryMetricData("127.0.0.1:8080", "test_app", "test_metrics", "test_metric", "6h");
|
||||
|
||||
assertNotNull(result);
|
||||
assertFalse(result.isEmpty());
|
||||
// Verify that the mapping logic correctly extracted the value
|
||||
assertEquals("85.5", result.values().iterator().next().get(0).getOrigin());
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveLogData() {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
|
||||
// Mock the write result
|
||||
@SuppressWarnings("unchecked")
|
||||
Result<WriteOk, Err> mockResult = mock(Result.class);
|
||||
when(mockResult.isOk()).thenReturn(true);
|
||||
CompletableFuture<Result<WriteOk, Err>> mockFuture = CompletableFuture.completedFuture(mockResult);
|
||||
when(greptimeDb.write(any(Table.class))).thenReturn(mockFuture);
|
||||
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
|
||||
LogEntry logEntry = createMockLogEntry();
|
||||
greptimeDbDataStorage.saveLogData(logEntry);
|
||||
|
||||
verify(greptimeDb, times(1)).write(any(Table.class));
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQueryAndCountLogs() {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
List<Map<String, Object>> mockLogRows = createMockLogRows();
|
||||
when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(mockLogRows);
|
||||
|
||||
// Test basic query
|
||||
List<LogEntry> result = greptimeDbDataStorage.queryLogsByMultipleConditions(
|
||||
System.currentTimeMillis() - 3600000, System.currentTimeMillis(), "trace123", "span456", 1, "INFO", "content"
|
||||
);
|
||||
assertNotNull(result);
|
||||
assertEquals(1, result.size());
|
||||
assertEquals("trace123", result.get(0).getTraceId());
|
||||
|
||||
// Test count query
|
||||
List<Map<String, Object>> mockCountResult = List.of(Map.of("count", 5L));
|
||||
when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(mockCountResult);
|
||||
long count = greptimeDbDataStorage.countLogsByMultipleConditions(
|
||||
System.currentTimeMillis() - 3600000, System.currentTimeMillis(), null, null, null, null, null
|
||||
);
|
||||
assertEquals(5L, count);
|
||||
|
||||
// Test count query with executor error
|
||||
when(greptimeSqlQueryExecutor.execute(anyString())).thenThrow(new RuntimeException("Database error"));
|
||||
long errorCount = greptimeDbDataStorage.countLogsByMultipleConditions(System.currentTimeMillis() - 3600000, System.currentTimeMillis(), null, null, null, null, null);
|
||||
assertEquals(0L, errorCount);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testQueryLogsWithPagination() {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
when(greptimeSqlQueryExecutor.execute(anyString())).thenReturn(createMockLogRows());
|
||||
|
||||
ArgumentCaptor<String> sqlCaptor = ArgumentCaptor.forClass(String.class);
|
||||
|
||||
greptimeDbDataStorage.queryLogsByMultipleConditionsWithPagination(
|
||||
System.currentTimeMillis() - 3600000, System.currentTimeMillis(),
|
||||
null, null, null, null, null, 1, 10
|
||||
);
|
||||
|
||||
verify(greptimeSqlQueryExecutor).execute(sqlCaptor.capture());
|
||||
String capturedSql = sqlCaptor.getValue();
|
||||
|
||||
assertTrue(capturedSql.toLowerCase().contains("limit 10"));
|
||||
assertTrue(capturedSql.toLowerCase().contains("offset 1"));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@Test
|
||||
void testBatchDeleteLogsWithValidList() {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
|
||||
// Test with valid list
|
||||
boolean result = greptimeDbDataStorage.batchDeleteLogs(List.of(1L, 2L));
|
||||
assertTrue(result);
|
||||
verify(greptimeSqlQueryExecutor, times(1)).execute(anyString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBatchDeleteLogsWithEmptyList() {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
|
||||
// Test with empty list
|
||||
boolean emptyResult = greptimeDbDataStorage.batchDeleteLogs(Collections.emptyList());
|
||||
assertFalse(emptyResult);
|
||||
verify(greptimeSqlQueryExecutor, never()).execute(anyString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testBatchDeleteLogsWithNullList() {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
|
||||
// Test with null list
|
||||
boolean nullResult = greptimeDbDataStorage.batchDeleteLogs(null);
|
||||
assertFalse(nullResult);
|
||||
verify(greptimeSqlQueryExecutor, never()).execute(anyString());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
void testDestroy() {
|
||||
try (MockedStatic<GreptimeDB> mockedStatic = mockStatic(GreptimeDB.class)) {
|
||||
mockedStatic.when(() -> GreptimeDB.create(any())).thenReturn(greptimeDb);
|
||||
greptimeDbDataStorage = new GreptimeDbDataStorage(greptimeProperties, restTemplate, greptimeSqlQueryExecutor);
|
||||
|
||||
greptimeDbDataStorage.destroy();
|
||||
|
||||
verify(greptimeDb, times(1)).shutdownGracefully();
|
||||
}
|
||||
}
|
||||
|
||||
private CollectRep.MetricsData createMockMetricsData(boolean hasValues) {
|
||||
CollectRep.MetricsData mockMetricsData = mock(CollectRep.MetricsData.class);
|
||||
lenient().when(mockMetricsData.getCode()).thenReturn(CollectRep.Code.SUCCESS);
|
||||
lenient().when(mockMetricsData.getMetrics()).thenReturn("cpu");
|
||||
lenient().when(mockMetricsData.getId()).thenReturn(1L);
|
||||
|
||||
if (!hasValues) {
|
||||
lenient().when(mockMetricsData.getValues()).thenReturn(Collections.emptyList());
|
||||
return mockMetricsData;
|
||||
}
|
||||
|
||||
// Only create detailed mocks when hasValues = true
|
||||
// Mock fields
|
||||
CollectRep.Field mockField1 = mock(CollectRep.Field.class);
|
||||
lenient().when(mockField1.getName()).thenReturn("usage");
|
||||
lenient().when(mockField1.getLabel()).thenReturn(false);
|
||||
lenient().when(mockField1.getType()).thenReturn((int) CommonConstants.TYPE_NUMBER);
|
||||
|
||||
CollectRep.Field mockField2 = mock(CollectRep.Field.class);
|
||||
lenient().when(mockField2.getName()).thenReturn("instance");
|
||||
lenient().when(mockField2.getLabel()).thenReturn(true);
|
||||
lenient().when(mockField2.getType()).thenReturn((int) CommonConstants.TYPE_STRING);
|
||||
|
||||
lenient().when(mockMetricsData.getFields()).thenReturn(List.of(mockField1, mockField2));
|
||||
|
||||
// Create ValueRow mock
|
||||
CollectRep.ValueRow mockValueRow = mock(CollectRep.ValueRow.class);
|
||||
lenient().when(mockValueRow.getColumnsList()).thenReturn(List.of("server1", "85.5"));
|
||||
|
||||
lenient().when(mockMetricsData.getValues()).thenReturn(List.of(mockValueRow));
|
||||
|
||||
// Mock RowWrapper for readRow()
|
||||
RowWrapper mockRowWrapper = mock(RowWrapper.class);
|
||||
lenient().when(mockRowWrapper.hasNextRow()).thenReturn(true, false);
|
||||
lenient().when(mockRowWrapper.nextRow()).thenReturn(mockRowWrapper);
|
||||
|
||||
// Mock cell stream
|
||||
ArrowCell mockCell1 = mock(ArrowCell.class);
|
||||
lenient().when(mockCell1.getValue()).thenReturn("85.5");
|
||||
lenient().when(mockCell1.getMetadataAsBoolean(any())).thenReturn(false);
|
||||
lenient().when(mockCell1.getMetadataAsByte(any())).thenReturn(CommonConstants.TYPE_NUMBER);
|
||||
|
||||
ArrowCell mockCell2 = mock(ArrowCell.class);
|
||||
lenient().when(mockCell2.getValue()).thenReturn("server1");
|
||||
lenient().when(mockCell2.getMetadataAsBoolean(any())).thenReturn(true);
|
||||
lenient().when(mockCell2.getMetadataAsByte(any())).thenReturn(CommonConstants.TYPE_STRING);
|
||||
|
||||
lenient().when(mockRowWrapper.cellStream()).thenReturn(java.util.stream.Stream.of(mockCell1, mockCell2));
|
||||
lenient().when(mockMetricsData.readRow()).thenReturn(mockRowWrapper);
|
||||
|
||||
return mockMetricsData;
|
||||
}
|
||||
|
||||
private PromQlQueryContent createMockPromQlQueryContent() {
|
||||
PromQlQueryContent content = new PromQlQueryContent();
|
||||
PromQlQueryContent.ContentData data = new PromQlQueryContent.ContentData();
|
||||
PromQlQueryContent.ContentData.Content result = new PromQlQueryContent.ContentData.Content();
|
||||
|
||||
Map<String, String> metric = new HashMap<>();
|
||||
metric.put("__name__", "cpu");
|
||||
metric.put("instance", "1");
|
||||
result.setMetric(metric);
|
||||
|
||||
List<Object[]> values = new ArrayList<>();
|
||||
values.add(new Object[]{System.currentTimeMillis() / 1000.0, "85.5"});
|
||||
result.setValues(values);
|
||||
|
||||
data.setResult(List.of(result));
|
||||
content.setData(data);
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
private LogEntry createMockLogEntry() {
|
||||
return LogEntry.builder()
|
||||
.timeUnixNano(System.nanoTime())
|
||||
.severityText("INFO")
|
||||
.body("Test log message")
|
||||
.traceId("trace123")
|
||||
.spanId("span456")
|
||||
.build();
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> createMockLogRows() {
|
||||
Map<String, Object> row = new HashMap<>();
|
||||
row.put("time_unix_nano", System.nanoTime());
|
||||
row.put("severity_text", "INFO");
|
||||
row.put("body", "\"Test log message\"");
|
||||
row.put("trace_id", "trace123");
|
||||
row.put("span_id", "span456");
|
||||
|
||||
return List.of(row);
|
||||
}
|
||||
|
||||
private List<RowData.ColumnSchema> getColumnSchemas(Table table) throws Exception {
|
||||
Field columnSchemasField = table.getClass().getDeclaredField("columnSchemas");
|
||||
columnSchemasField.setAccessible(true);
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RowData.ColumnSchema> columnSchemas = (List<RowData.ColumnSchema>) columnSchemasField.get(table);
|
||||
return columnSchemas;
|
||||
}
|
||||
|
||||
private List<RowData.Row> getRows(Table table) throws Exception {
|
||||
Field rowsField = table.getClass().getDeclaredField("rows");
|
||||
rowsField.setAccessible(true);
|
||||
@SuppressWarnings("unchecked")
|
||||
List<RowData.Row> rows = (List<RowData.Row>) rowsField.get(table);
|
||||
return rows;
|
||||
}
|
||||
}
|
||||
+245
@@ -0,0 +1,245 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
import static org.mockito.ArgumentMatchers.anyString;
|
||||
import static org.mockito.ArgumentMatchers.eq;
|
||||
import static org.mockito.ArgumentMatchers.any;
|
||||
import static org.mockito.ArgumentMatchers.startsWith;
|
||||
import static org.mockito.Mockito.when;
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import org.apache.arrow.vector.types.pojo.ArrowType;
|
||||
import org.apache.arrow.vector.types.pojo.Field;
|
||||
import org.apache.arrow.vector.types.pojo.FieldType;
|
||||
import org.apache.hertzbeat.common.constants.CommonConstants;
|
||||
import org.apache.hertzbeat.common.constants.MetricDataConstants;
|
||||
import org.apache.hertzbeat.common.entity.arrow.ArrowCell;
|
||||
import org.apache.hertzbeat.common.entity.arrow.RowWrapper;
|
||||
import org.apache.hertzbeat.common.entity.message.CollectRep;
|
||||
import org.awaitility.Awaitility;
|
||||
import org.junit.jupiter.api.AfterEach;
|
||||
import org.junit.jupiter.api.BeforeEach;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.junit.jupiter.api.extension.ExtendWith;
|
||||
|
||||
import org.mockito.Mock;
|
||||
import org.mockito.Mockito;
|
||||
import org.mockito.junit.jupiter.MockitoExtension;
|
||||
import org.mockito.junit.jupiter.MockitoSettings;
|
||||
import org.mockito.quality.Strictness;
|
||||
|
||||
import org.springframework.http.HttpEntity;
|
||||
import org.springframework.http.HttpMethod;
|
||||
import org.springframework.http.HttpStatus;
|
||||
import org.springframework.http.ResponseEntity;
|
||||
import org.springframework.web.client.RestTemplate;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
/**
|
||||
* Test case for {@link VictoriaMetricsDataStorage}
|
||||
*/
|
||||
@ExtendWith(MockitoExtension.class)
|
||||
@MockitoSettings(strictness = Strictness.LENIENT)
|
||||
class VictoriaMetricsDataStorageTest {
|
||||
|
||||
@Mock
|
||||
private VictoriaMetricsProperties victoriaMetricsProperties;
|
||||
|
||||
@Mock
|
||||
private RestTemplate restTemplate;
|
||||
|
||||
@Mock
|
||||
private ResponseEntity<String> responseEntity;
|
||||
|
||||
private VictoriaMetricsDataStorage victoriaMetricsDataStorage;
|
||||
|
||||
private final AtomicInteger postForEntityCount = new AtomicInteger(0);
|
||||
|
||||
@BeforeEach
|
||||
void setUp() {
|
||||
when(victoriaMetricsProperties.enabled()).thenReturn(true);
|
||||
when(victoriaMetricsProperties.url()).thenReturn("http://localhost:8428");
|
||||
when(victoriaMetricsProperties.username()).thenReturn("root");
|
||||
when(victoriaMetricsProperties.password()).thenReturn("root");
|
||||
|
||||
// on successful write, VictoriaMetrics returns HTTP 204 (No Content)
|
||||
when(responseEntity.getStatusCode()).thenReturn(HttpStatus.NO_CONTENT);
|
||||
|
||||
when(restTemplate.exchange(
|
||||
anyString(),
|
||||
eq(HttpMethod.GET),
|
||||
any(HttpEntity.class),
|
||||
eq(String.class)
|
||||
)).thenReturn(responseEntity);
|
||||
|
||||
when(restTemplate.postForEntity(
|
||||
startsWith(victoriaMetricsProperties.url()),
|
||||
any(HttpEntity.class),
|
||||
eq(String.class)
|
||||
)).thenAnswer(invocation -> {
|
||||
postForEntityCount.incrementAndGet();
|
||||
return responseEntity;
|
||||
});
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveDataInitialization() {
|
||||
// using the default insertion policy, schedule a task once upon object instantiation (with second-level time granularity)
|
||||
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
|
||||
// execute one-time data insertion
|
||||
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
|
||||
// wait for the timer's first insertion task execution and verify if it was called once (default 3 seconds)
|
||||
Awaitility.await()
|
||||
.pollInterval(2, TimeUnit.SECONDS)
|
||||
.atMost(7, TimeUnit.SECONDS)
|
||||
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveDataBySize() {
|
||||
// verify insert process for buffer size, with the flush interval defined as an unreachable state
|
||||
when(victoriaMetricsProperties.insert()).thenReturn(new VictoriaMetricsProperties.InsertConfig(
|
||||
10, Integer.MAX_VALUE, new VictoriaMetricsProperties.Compression(false)));
|
||||
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
|
||||
|
||||
// triggers the buffer size insertion condition
|
||||
for (int i = 0; i < 10 * 0.8; i++) {
|
||||
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
|
||||
}
|
||||
// wait for the timer to execute the task
|
||||
Awaitility.await()
|
||||
.pollInterval(1, TimeUnit.SECONDS)
|
||||
.atMost(5, TimeUnit.SECONDS)
|
||||
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testSaveDataByTime() {
|
||||
// verify insert process for flush interval and set the buffer size to an unreachable state
|
||||
when(victoriaMetricsProperties.insert()).thenReturn(new VictoriaMetricsProperties.InsertConfig(
|
||||
10000, 2, new VictoriaMetricsProperties.Compression(false)));
|
||||
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
|
||||
|
||||
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
|
||||
// wait for the timer to execute its first insertion task
|
||||
Awaitility.await()
|
||||
.pollInterval(500, TimeUnit.MILLISECONDS)
|
||||
.atMost(5, TimeUnit.SECONDS)
|
||||
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(1));
|
||||
|
||||
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
|
||||
// wait for the flush interval to be triggered again
|
||||
Awaitility.await()
|
||||
.pollInterval(500, TimeUnit.MILLISECONDS)
|
||||
.atMost(5, TimeUnit.SECONDS)
|
||||
.untilAsserted(() -> assertThat(postForEntityCount.get()).isEqualTo(2));
|
||||
}
|
||||
|
||||
@Test
|
||||
void testMultiThreadSaveDataBySize() {
|
||||
int threadCount = 100;
|
||||
int bufferSize = 10;
|
||||
int writeSize = (int) (bufferSize * 0.8);
|
||||
|
||||
// verify insert process for buffer size, with the flush interval defined as an unreachable state
|
||||
when(victoriaMetricsProperties.insert()).thenReturn(new VictoriaMetricsProperties.InsertConfig(
|
||||
bufferSize, Integer.MAX_VALUE, new VictoriaMetricsProperties.Compression(false)));
|
||||
victoriaMetricsDataStorage = new VictoriaMetricsDataStorage(victoriaMetricsProperties, restTemplate);
|
||||
|
||||
for (int i = 0; i < threadCount; i++) {
|
||||
new Thread(() -> {
|
||||
// triggers the buffer size insertion condition
|
||||
for (int j = 0; j < writeSize; j++) {
|
||||
victoriaMetricsDataStorage.saveData(generateMockedMetricsData());
|
||||
}
|
||||
}).start();
|
||||
}
|
||||
|
||||
// wait for the timer to execute the task
|
||||
Awaitility.await()
|
||||
.pollInterval(3, TimeUnit.SECONDS)
|
||||
.atMost(15, TimeUnit.SECONDS)
|
||||
.untilAsserted(() ->
|
||||
assertThat(postForEntityCount.get())
|
||||
// minimum flushes: ensure all data is processed (threadCount * writeSize / bufferSize)
|
||||
.isGreaterThanOrEqualTo(threadCount * writeSize / bufferSize));
|
||||
}
|
||||
|
||||
@AfterEach
|
||||
void stop() {
|
||||
if (victoriaMetricsDataStorage != null) {
|
||||
victoriaMetricsDataStorage.destroy();
|
||||
}
|
||||
}
|
||||
|
||||
// for historical data insertion (which mandates preparatory steps), instantiate a mock MetricsData object
|
||||
public static CollectRep.MetricsData generateMockedMetricsData() {
|
||||
CollectRep.MetricsData mockMetricsData = Mockito.mock(CollectRep.MetricsData.class);
|
||||
|
||||
when(mockMetricsData.getId()).thenReturn(0L);
|
||||
when(mockMetricsData.getMetrics()).thenReturn("cpu");
|
||||
when(mockMetricsData.getTime()).thenReturn(System.currentTimeMillis());
|
||||
when(mockMetricsData.getCode()).thenReturn(CollectRep.Code.SUCCESS);
|
||||
when(mockMetricsData.getApp()).thenReturn("app");
|
||||
|
||||
CollectRep.ValueRow mockValueRow = Mockito.mock(CollectRep.ValueRow.class);
|
||||
List<String> columnValues = List.of("server-test-01", "68.7");
|
||||
when(mockValueRow.getColumnsList()).thenReturn(columnValues);
|
||||
when(mockValueRow.getColumns(0)).thenReturn("server-test-01");
|
||||
when(mockValueRow.getColumns(1)).thenReturn("68.7");
|
||||
List<CollectRep.ValueRow> mockValueRowsList = List.of(mockValueRow);
|
||||
when(mockMetricsData.getValues()).thenReturn(mockValueRowsList);
|
||||
|
||||
CollectRep.Field instanceField = Mockito.mock(CollectRep.Field.class);
|
||||
when(instanceField.getName()).thenReturn("instance");
|
||||
CollectRep.Field usageField = Mockito.mock(CollectRep.Field.class);
|
||||
when(usageField.getName()).thenReturn("usage");
|
||||
CollectRep.Field systemField = Mockito.mock(CollectRep.Field.class);
|
||||
when(systemField.getName()).thenReturn("system");
|
||||
List<CollectRep.Field> mockFields = List.of(instanceField, usageField, systemField);
|
||||
when(mockMetricsData.getFields()).thenReturn(mockFields);
|
||||
|
||||
ArrowType instanceArrowType = new ArrowType.Utf8();
|
||||
FieldType instanceFieldType = new FieldType(true, instanceArrowType, null, null);
|
||||
Field instanceRealField = new Field("instance", instanceFieldType, null);
|
||||
ArrowCell instanceCell = Mockito.mock(ArrowCell.class);
|
||||
when(instanceCell.getField()).thenReturn(instanceRealField);
|
||||
when(instanceCell.getValue()).thenReturn("server-test-01");
|
||||
when(instanceCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(true);
|
||||
when(instanceCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_STRING);
|
||||
|
||||
ArrowCell usageCell = Mockito.mock(ArrowCell.class);
|
||||
when(usageCell.getField()).thenReturn(instanceRealField);
|
||||
when(usageCell.getValue()).thenReturn("68.7");
|
||||
when(usageCell.getMetadataAsBoolean(MetricDataConstants.LABEL)).thenReturn(false);
|
||||
when(usageCell.getMetadataAsByte(MetricDataConstants.TYPE)).thenReturn(CommonConstants.TYPE_NUMBER);
|
||||
List<ArrowCell> mockCells = List.of(instanceCell, usageCell);
|
||||
|
||||
RowWrapper mockRowWrapper = Mockito.mock(RowWrapper.class);
|
||||
when(mockRowWrapper.hasNextRow()).thenReturn(true).thenReturn(false);
|
||||
when(mockRowWrapper.nextRow()).thenReturn(mockRowWrapper);
|
||||
when(mockRowWrapper.cellStream()).thenAnswer(invocation -> mockCells.stream());
|
||||
when(mockMetricsData.readRow()).thenReturn(mockRowWrapper);
|
||||
return mockMetricsData;
|
||||
}
|
||||
|
||||
}
|
||||
+77
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* Licensed to the Apache Software Foundation (ASF) under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The ASF licenses this file to You under the Apache License, Version 2.0
|
||||
* (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*/
|
||||
|
||||
package org.apache.hertzbeat.warehouse.store.history.tsdb.vm;
|
||||
|
||||
import static org.assertj.core.api.Assertions.assertThat;
|
||||
|
||||
import java.util.Map;
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.bind.Binder;
|
||||
import org.springframework.boot.context.properties.source.MapConfigurationPropertySource;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
|
||||
/**
|
||||
* Test case for {@link VictoriaMetricsProperties}
|
||||
*/
|
||||
@EnableConfigurationProperties(value = VictoriaMetricsProperties.class)
|
||||
@SpringBootTest(
|
||||
classes = VictoriaMetricsPropertiesTest.class,
|
||||
properties = {
|
||||
"warehouse.store.victoria-metrics.enabled=true",
|
||||
"warehouse.store.victoria-metrics.username=test_user",
|
||||
"warehouse.store.victoria-metrics.insert.buffer-size=999",
|
||||
"warehouse.store.victoria-metrics.insert.compression.enabled=true"
|
||||
}
|
||||
)
|
||||
class VictoriaMetricsPropertiesTest {
|
||||
|
||||
@Autowired
|
||||
private VictoriaMetricsProperties properties;
|
||||
|
||||
@Test
|
||||
void propertiesShouldBeInjectedAndCorrectlyBound() {
|
||||
assertThat(properties).isNotNull();
|
||||
assertThat(properties.enabled()).isTrue();
|
||||
// url not configured, default value injected
|
||||
assertThat(properties.url()).isEqualTo("http://localhost:8428");
|
||||
assertThat(properties.username()).isEqualTo("test_user");
|
||||
// password not configured, null injected
|
||||
assertThat(properties.password()).isNull();
|
||||
assertThat(properties.insert()).isNotNull();
|
||||
assertThat(properties.insert().bufferSize()).isEqualTo(999);
|
||||
// flush-interval not configured, default value is 3
|
||||
assertThat(properties.insert().flushInterval()).isEqualTo(3);
|
||||
assertThat(properties.insert().compression()).isNotNull();
|
||||
assertThat(properties.insert().compression().enabled()).isTrue();
|
||||
}
|
||||
|
||||
@Test
|
||||
void compressionShouldDefaultToDisabledWhenNotConfigured() {
|
||||
var source = new MapConfigurationPropertySource(Map.of(
|
||||
"warehouse.store.victoria-metrics.insert.buffer-size", "999"));
|
||||
var defaultProperties = new Binder(source)
|
||||
.bind("warehouse.store.victoria-metrics", VictoriaMetricsProperties.class)
|
||||
.get();
|
||||
|
||||
assertThat(defaultProperties.insert().compression()).isNotNull();
|
||||
assertThat(defaultProperties.insert().compression().enabled()).isFalse();
|
||||
}
|
||||
|
||||
}
|
||||
Reference in New Issue
Block a user