chore: import upstream snapshot with attribution
CI / unit-test (push) Has been cancelled
CI / detect-changes (push) Has been cancelled
CI / build (push) Has been cancelled
Publish docs via GitHub Pages / Deploy docs (push) Has been cancelled
CI / test-harness (push) Has been cancelled
CI / generate-e2e-matrix (push) Has been cancelled
CI / e2e (push) Has been cancelled
CI / build-ui (push) Has been cancelled
Release Drafter / update_release_draft (push) Has been cancelled
UI v2 Integration CI / E2E (Integration) (push) Has been cancelled
UI v2 CI / Lint, Format & Test (push) Has been cancelled
UI v2 CI / E2E (Mocked) (push) Has been cancelled
CI / unit-test (push) Has been cancelled
CI / detect-changes (push) Has been cancelled
CI / build (push) Has been cancelled
Publish docs via GitHub Pages / Deploy docs (push) Has been cancelled
CI / test-harness (push) Has been cancelled
CI / generate-e2e-matrix (push) Has been cancelled
CI / e2e (push) Has been cancelled
CI / build-ui (push) Has been cancelled
Release Drafter / update_release_draft (push) Has been cancelled
UI v2 Integration CI / E2E (Integration) (push) Has been cancelled
UI v2 CI / Lint, Format & Test (push) Has been cancelled
UI v2 CI / E2E (Mocked) (push) Has been cancelled
This commit is contained in:
+294
@@ -0,0 +1,294 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.config;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.util.Optional;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.conductoross.conductor.sqlite.dao.SqliteFileMetadataDAO;
|
||||
import org.conductoross.conductor.sqlite.dao.SqliteSkillMetadataDAO;
|
||||
import org.conductoross.conductor.sqlite.dao.SqliteSkillPackageDAO;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.flywaydb.core.api.configuration.FluentConfiguration;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
|
||||
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
import org.springframework.boot.context.properties.EnableConfigurationProperties;
|
||||
import org.springframework.context.annotation.Bean;
|
||||
import org.springframework.context.annotation.Configuration;
|
||||
import org.springframework.context.annotation.DependsOn;
|
||||
import org.springframework.context.annotation.Import;
|
||||
import org.springframework.retry.RetryContext;
|
||||
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
|
||||
import org.springframework.retry.policy.SimpleRetryPolicy;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.core.sync.Lock;
|
||||
import com.netflix.conductor.core.sync.local.LocalOnlyLock;
|
||||
import com.netflix.conductor.sqlite.dao.*;
|
||||
import com.netflix.conductor.sqlite.dao.metadata.SqliteEventHandlerMetadataDAO;
|
||||
import com.netflix.conductor.sqlite.dao.metadata.SqliteMetadataDAO;
|
||||
import com.netflix.conductor.sqlite.dao.metadata.SqliteTaskMetadataDAO;
|
||||
import com.netflix.conductor.sqlite.dao.metadata.SqliteWorkflowMetadataDAO;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import jakarta.annotation.PostConstruct;
|
||||
|
||||
@Configuration(proxyBeanMethods = false)
|
||||
@EnableConfigurationProperties(SqliteProperties.class)
|
||||
@ConditionalOnProperty(name = "conductor.db.type", havingValue = "sqlite")
|
||||
@Import(DataSourceAutoConfiguration.class)
|
||||
@ConfigurationProperties(prefix = "conductor.sqlite")
|
||||
public class SqliteConfiguration {
|
||||
|
||||
DataSource dataSource;
|
||||
|
||||
private final SqliteProperties properties;
|
||||
|
||||
public SqliteConfiguration(DataSource dataSource, SqliteProperties properties) {
|
||||
this.dataSource = dataSource;
|
||||
this.properties = properties;
|
||||
}
|
||||
|
||||
@PostConstruct
|
||||
public void initializeSqlite() {
|
||||
try (Connection conn = dataSource.getConnection();
|
||||
Statement stmt = conn.createStatement()) {
|
||||
|
||||
// Enable WAL mode for better concurrent access
|
||||
stmt.execute("PRAGMA journal_mode=WAL");
|
||||
|
||||
// Set busy timeout to 30 seconds (30000 ms)
|
||||
// This makes SQLite wait up to 30s when encountering locks
|
||||
stmt.execute("PRAGMA busy_timeout=30000");
|
||||
|
||||
// Enable foreign keys
|
||||
stmt.execute("PRAGMA foreign_keys=ON");
|
||||
|
||||
// Optimize for concurrency
|
||||
stmt.execute("PRAGMA synchronous=NORMAL");
|
||||
|
||||
// Use memory for temporary storage
|
||||
stmt.execute("PRAGMA temp_store=MEMORY");
|
||||
|
||||
} catch (SQLException e) {
|
||||
throw new RuntimeException("Failed to initialize SQLite database", e);
|
||||
}
|
||||
}
|
||||
|
||||
@Bean(initMethod = "migrate")
|
||||
public Flyway flywayForPrimaryDb() {
|
||||
FluentConfiguration config =
|
||||
Flyway.configure()
|
||||
.dataSource(dataSource) // SQLite doesn't need username/password
|
||||
.locations("classpath:db/migration_sqlite") // Location of migration files
|
||||
.sqlMigrationPrefix("V") // V1, V2, etc.
|
||||
.sqlMigrationSeparator("__") // V1__description
|
||||
.mixed(true) // Allow mixed migrations (both versioned and repeatable)
|
||||
.validateOnMigrate(true)
|
||||
.cleanDisabled(false);
|
||||
|
||||
return new Flyway(config);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
public SqliteMetadataDAO sqliteMetadataDAO(
|
||||
SqliteTaskMetadataDAO taskMetadataDAO,
|
||||
SqliteWorkflowMetadataDAO workflowMetadataDAO,
|
||||
SqliteEventHandlerMetadataDAO eventHandlerMetadataDAO) {
|
||||
return new SqliteMetadataDAO(taskMetadataDAO, workflowMetadataDAO, eventHandlerMetadataDAO);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
public SqliteEventHandlerMetadataDAO sqliteEventHandlerMetadataDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper) {
|
||||
return new SqliteEventHandlerMetadataDAO(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
public SqliteWorkflowMetadataDAO sqliteWorkflowMetadataDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper) {
|
||||
return new SqliteWorkflowMetadataDAO(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
public SqliteTaskMetadataDAO sqliteTaskMetadataDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper) {
|
||||
return new SqliteTaskMetadataDAO(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
public SqliteExecutionDAO sqliteExecutionDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
SqliteQueueDAO queueDAO) {
|
||||
return new SqliteExecutionDAO(retryTemplate, objectMapper, dataSource, queueDAO);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
public SqlitePollDataDAO sqlitePollDataDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper) {
|
||||
return new SqlitePollDataDAO(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
public SqliteQueueDAO sqliteQueueDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
SqliteProperties properties) {
|
||||
return new SqliteQueueDAO(retryTemplate, objectMapper, dataSource, properties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
@ConditionalOnProperty(name = "conductor.indexing.type", havingValue = "sqlite")
|
||||
public SqliteIndexDAO sqliteIndexDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
SqliteProperties properties) {
|
||||
return new SqliteIndexDAO(retryTemplate, objectMapper, dataSource, properties);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
@ConditionalOnProperty(name = "conductor.workflow-execution-lock.type", havingValue = "sqlite")
|
||||
public Lock sqliteLockDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper) {
|
||||
return new LocalOnlyLock();
|
||||
}
|
||||
|
||||
@Bean
|
||||
public RetryTemplate sqliteRetryTemplate(SqliteProperties properties) {
|
||||
CustomRetryPolicy retryPolicy = new CustomRetryPolicy();
|
||||
retryPolicy.setMaxAttempts(10); // Increased for SQLite locking scenarios
|
||||
|
||||
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
|
||||
backOffPolicy.setInitialInterval(50L); // Start with 50ms
|
||||
backOffPolicy.setMultiplier(2.0); // Double each time
|
||||
backOffPolicy.setMaxInterval(5000L); // Max 5 seconds
|
||||
|
||||
RetryTemplate retryTemplate = new RetryTemplate();
|
||||
retryTemplate.setRetryPolicy(retryPolicy);
|
||||
retryTemplate.setBackOffPolicy(backOffPolicy);
|
||||
return retryTemplate;
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
@ConditionalOnProperty(name = "conductor.file-storage.enabled", havingValue = "true")
|
||||
public SqliteFileMetadataDAO sqliteFileMetadataDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper) {
|
||||
return new SqliteFileMetadataDAO(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
@ConditionalOnProperty(name = "conductor.integrations.ai.enabled", havingValue = "true")
|
||||
public SqliteSkillMetadataDAO sqliteSkillMetadataDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper) {
|
||||
return new SqliteSkillMetadataDAO(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Bean
|
||||
@DependsOn({"flywayForPrimaryDb"})
|
||||
@ConditionalOnProperty(name = "conductor.integrations.ai.enabled", havingValue = "true")
|
||||
public SqliteSkillPackageDAO sqliteSkillPackageDAO(
|
||||
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper) {
|
||||
return new SqliteSkillPackageDAO(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
public static class CustomRetryPolicy extends SimpleRetryPolicy {
|
||||
|
||||
// PostgreSQL error codes
|
||||
private static final String ER_LOCK_DEADLOCK = "40P01";
|
||||
private static final String ER_SERIALIZATION_FAILURE = "40001";
|
||||
|
||||
// SQLite error codes
|
||||
private static final int SQLITE_BUSY = 5;
|
||||
private static final int SQLITE_LOCKED = 6;
|
||||
|
||||
@Override
|
||||
public boolean canRetry(final RetryContext context) {
|
||||
final Optional<Throwable> lastThrowable =
|
||||
Optional.ofNullable(context.getLastThrowable());
|
||||
return lastThrowable
|
||||
.map(
|
||||
throwable ->
|
||||
super.canRetry(context)
|
||||
&& (isDeadLockError(throwable)
|
||||
|| isSqliteBusyError(throwable)))
|
||||
.orElseGet(() -> super.canRetry(context));
|
||||
}
|
||||
|
||||
private boolean isDeadLockError(Throwable throwable) {
|
||||
SQLException sqlException = findCauseSQLException(throwable);
|
||||
if (sqlException == null) {
|
||||
return false;
|
||||
}
|
||||
return ER_LOCK_DEADLOCK.equals(sqlException.getSQLState())
|
||||
|| ER_SERIALIZATION_FAILURE.equals(sqlException.getSQLState());
|
||||
}
|
||||
|
||||
private boolean isSqliteBusyError(Throwable throwable) {
|
||||
SQLException sqlException = findCauseSQLException(throwable);
|
||||
if (sqlException == null) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Check for SQLite BUSY (5) or LOCKED (6) error codes
|
||||
int errorCode = sqlException.getErrorCode();
|
||||
if (errorCode == SQLITE_BUSY || errorCode == SQLITE_LOCKED) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Also check message for SQLite busy/locked indicators
|
||||
String message = sqlException.getMessage();
|
||||
if (message != null) {
|
||||
return message.contains("SQLITE_BUSY")
|
||||
|| message.contains("database is locked")
|
||||
|| message.contains("SQLITE_LOCKED")
|
||||
|| message.contains("table is locked");
|
||||
}
|
||||
|
||||
return false;
|
||||
}
|
||||
|
||||
private SQLException findCauseSQLException(Throwable throwable) {
|
||||
Throwable causeException = throwable;
|
||||
while (null != causeException && !(causeException instanceof SQLException)) {
|
||||
causeException = causeException.getCause();
|
||||
}
|
||||
return (SQLException) causeException;
|
||||
}
|
||||
}
|
||||
}
|
||||
+60
@@ -0,0 +1,60 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.config;
|
||||
|
||||
import java.time.Duration;
|
||||
|
||||
import org.springframework.boot.context.properties.ConfigurationProperties;
|
||||
|
||||
@ConfigurationProperties("conductor.sqlite")
|
||||
public class SqliteProperties {
|
||||
|
||||
/** The time (in seconds) after which the in-memory task definitions cache will be refreshed */
|
||||
private Duration taskDefCacheRefreshInterval = Duration.ofSeconds(60);
|
||||
|
||||
private Integer deadlockRetryMax = 3;
|
||||
|
||||
private boolean onlyIndexOnStatusChange = false;
|
||||
|
||||
private Integer asyncMaxPoolSize = 10;
|
||||
|
||||
private Integer asyncWorkerQueueSize = 10;
|
||||
|
||||
public Duration getTaskDefCacheRefreshInterval() {
|
||||
return taskDefCacheRefreshInterval;
|
||||
}
|
||||
|
||||
public void setTaskDefCacheRefreshInterval(Duration taskDefCacheRefreshInterval) {
|
||||
this.taskDefCacheRefreshInterval = taskDefCacheRefreshInterval;
|
||||
}
|
||||
|
||||
public Integer getDeadlockRetryMax() {
|
||||
return deadlockRetryMax;
|
||||
}
|
||||
|
||||
public void setDeadlockRetryMax(Integer deadlockRetryMax) {
|
||||
this.deadlockRetryMax = deadlockRetryMax;
|
||||
}
|
||||
|
||||
public int getAsyncMaxPoolSize() {
|
||||
return asyncMaxPoolSize;
|
||||
}
|
||||
|
||||
public int getAsyncWorkerQueueSize() {
|
||||
return asyncWorkerQueueSize;
|
||||
}
|
||||
|
||||
public boolean getOnlyIndexOnStatusChange() {
|
||||
return onlyIndexOnStatusChange;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,206 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Statement;
|
||||
import java.time.Duration;
|
||||
import java.time.Instant;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.function.Consumer;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.core.exception.NonTransientException;
|
||||
import com.netflix.conductor.sqlite.util.*;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.core.type.TypeReference;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
public abstract class SqliteBaseDAO {
|
||||
|
||||
private static final List<String> EXCLUDED_STACKTRACE_CLASS =
|
||||
List.of(SqliteBaseDAO.class.getName(), Thread.class.getName());
|
||||
|
||||
protected final Logger logger = LoggerFactory.getLogger(getClass());
|
||||
protected final ObjectMapper objectMapper;
|
||||
protected final DataSource dataSource;
|
||||
|
||||
private final RetryTemplate retryTemplate;
|
||||
|
||||
protected SqliteBaseDAO(
|
||||
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
|
||||
this.retryTemplate = retryTemplate;
|
||||
this.objectMapper = objectMapper;
|
||||
this.dataSource = dataSource;
|
||||
}
|
||||
|
||||
protected String toJson(Object value) {
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (JsonProcessingException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
protected <T> T readValue(String json, Class<T> tClass) {
|
||||
try {
|
||||
return objectMapper.readValue(json, tClass);
|
||||
} catch (IOException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
protected <T> T readValue(String json, TypeReference<T> typeReference) {
|
||||
try {
|
||||
return objectMapper.readValue(json, typeReference);
|
||||
} catch (IOException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private <R> R getWithTransaction(final TransactionalFunction<R> function) {
|
||||
final Instant start = Instant.now();
|
||||
LazyToString callingMethod = getCallingMethod();
|
||||
logger.trace("{} : starting transaction", callingMethod);
|
||||
|
||||
try (Connection tx = dataSource.getConnection()) {
|
||||
boolean previousAutoCommitMode = tx.getAutoCommit();
|
||||
tx.setAutoCommit(false);
|
||||
tx.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
|
||||
|
||||
try {
|
||||
// Enable foreign keys for SQLite
|
||||
try (Statement stmt = tx.createStatement()) {
|
||||
stmt.execute("PRAGMA foreign_keys = ON");
|
||||
}
|
||||
|
||||
R result = function.apply(tx);
|
||||
tx.commit();
|
||||
return result;
|
||||
} catch (Throwable th) {
|
||||
tx.rollback();
|
||||
if (th instanceof NonTransientException) {
|
||||
throw th;
|
||||
}
|
||||
throw new NonTransientException(th.getMessage(), th);
|
||||
} finally {
|
||||
tx.setAutoCommit(previousAutoCommitMode);
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
} finally {
|
||||
logger.trace(
|
||||
"{} : took {}ms",
|
||||
callingMethod,
|
||||
Duration.between(start, Instant.now()).toMillis());
|
||||
}
|
||||
}
|
||||
|
||||
protected <R> R getWithRetriedTransactions(final TransactionalFunction<R> function) {
|
||||
try {
|
||||
return retryTemplate.execute(context -> getWithTransaction(function));
|
||||
} catch (Exception e) {
|
||||
throw new NonTransientException(e.getMessage(), e);
|
||||
}
|
||||
}
|
||||
|
||||
protected <R> R getWithTransactionWithOutErrorPropagation(TransactionalFunction<R> function) {
|
||||
Instant start = Instant.now();
|
||||
LazyToString callingMethod = getCallingMethod();
|
||||
logger.trace("{} : starting transaction", callingMethod);
|
||||
|
||||
try (Connection tx = dataSource.getConnection()) {
|
||||
boolean previousAutoCommitMode = tx.getAutoCommit();
|
||||
tx.setAutoCommit(false);
|
||||
tx.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
|
||||
|
||||
try {
|
||||
// Enable foreign keys for SQLite
|
||||
try (Statement stmt = tx.createStatement()) {
|
||||
stmt.execute("PRAGMA foreign_keys = ON");
|
||||
}
|
||||
|
||||
R result = function.apply(tx);
|
||||
tx.commit();
|
||||
return result;
|
||||
} catch (Throwable th) {
|
||||
tx.rollback();
|
||||
logger.info(th.getMessage());
|
||||
return null;
|
||||
} finally {
|
||||
tx.setAutoCommit(previousAutoCommitMode);
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
} finally {
|
||||
logger.trace(
|
||||
"{} : took {}ms",
|
||||
callingMethod,
|
||||
Duration.between(start, Instant.now()).toMillis());
|
||||
}
|
||||
}
|
||||
|
||||
protected void withTransaction(Consumer<Connection> consumer) {
|
||||
getWithRetriedTransactions(
|
||||
connection -> {
|
||||
consumer.accept(connection);
|
||||
return null;
|
||||
});
|
||||
}
|
||||
|
||||
protected <R> R queryWithTransaction(String query, QueryFunction<R> function) {
|
||||
return getWithRetriedTransactions(tx -> query(tx, query, function));
|
||||
}
|
||||
|
||||
protected <R> R query(Connection tx, String query, QueryFunction<R> function) {
|
||||
try (Query q = new Query(objectMapper, tx, query)) {
|
||||
return function.apply(q);
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
protected void execute(Connection tx, String query, ExecuteFunction function) {
|
||||
try (Query q = new Query(objectMapper, tx, query)) {
|
||||
function.apply(q);
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
protected void executeWithTransaction(String query, ExecuteFunction function) {
|
||||
withTransaction(tx -> execute(tx, query, function));
|
||||
}
|
||||
|
||||
protected final LazyToString getCallingMethod() {
|
||||
return new LazyToString(
|
||||
() ->
|
||||
Arrays.stream(Thread.currentThread().getStackTrace())
|
||||
.filter(
|
||||
ste ->
|
||||
!EXCLUDED_STACKTRACE_CLASS.contains(
|
||||
ste.getClassName()))
|
||||
.findFirst()
|
||||
.map(StackTraceElement::getMethodName)
|
||||
.orElseThrow(() -> new NullPointerException("Cannot find Caller")));
|
||||
}
|
||||
}
|
||||
+1032
File diff suppressed because it is too large
Load Diff
+428
@@ -0,0 +1,428 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.TemporalAccessor;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.concurrent.*;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.common.metadata.events.EventExecution;
|
||||
import com.netflix.conductor.common.metadata.tasks.TaskExecLog;
|
||||
import com.netflix.conductor.common.run.SearchResult;
|
||||
import com.netflix.conductor.common.run.TaskSummary;
|
||||
import com.netflix.conductor.common.run.WorkflowSummary;
|
||||
import com.netflix.conductor.core.events.queue.Message;
|
||||
import com.netflix.conductor.dao.IndexDAO;
|
||||
import com.netflix.conductor.metrics.Monitors;
|
||||
import com.netflix.conductor.sqlite.config.SqliteProperties;
|
||||
import com.netflix.conductor.sqlite.util.SqliteIndexQueryBuilder;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
public class SqliteIndexDAO extends SqliteBaseDAO implements IndexDAO {
|
||||
|
||||
private final SqliteProperties properties;
|
||||
private final ExecutorService executorService;
|
||||
|
||||
private static final int CORE_POOL_SIZE = 6;
|
||||
private static final long KEEP_ALIVE_TIME = 1L;
|
||||
|
||||
private boolean onlyIndexOnStatusChange;
|
||||
|
||||
public SqliteIndexDAO(
|
||||
RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
DataSource dataSource,
|
||||
SqliteProperties properties) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
this.properties = properties;
|
||||
this.onlyIndexOnStatusChange = properties.getOnlyIndexOnStatusChange();
|
||||
|
||||
int maximumPoolSize = properties.getAsyncMaxPoolSize();
|
||||
int workerQueueSize = properties.getAsyncWorkerQueueSize();
|
||||
|
||||
// Set up a workerpool for performing async operations.
|
||||
this.executorService =
|
||||
new ThreadPoolExecutor(
|
||||
CORE_POOL_SIZE,
|
||||
maximumPoolSize,
|
||||
KEEP_ALIVE_TIME,
|
||||
TimeUnit.MINUTES,
|
||||
new LinkedBlockingQueue<>(workerQueueSize),
|
||||
(runnable, executor) -> {
|
||||
logger.warn(
|
||||
"Request {} to async dao discarded in executor {}",
|
||||
runnable,
|
||||
executor);
|
||||
Monitors.recordDiscardedIndexingCount("indexQueue");
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void indexWorkflow(WorkflowSummary workflow) {
|
||||
String INSERT_WORKFLOW_INDEX_SQL =
|
||||
"INSERT INTO workflow_index (workflow_id, correlation_id, workflow_type, start_time, update_time, status, parent_workflow_id, classifier, json_data) "
|
||||
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (workflow_id) "
|
||||
+ " DO UPDATE SET correlation_id = excluded.correlation_id, workflow_type = excluded.workflow_type, "
|
||||
+ " start_time = excluded.start_time, status = excluded.status, json_data = excluded.json_data, "
|
||||
+ " update_time = excluded.update_time, parent_workflow_id = excluded.parent_workflow_id, "
|
||||
+ " classifier = excluded.classifier "
|
||||
+ " WHERE excluded.update_time >= workflow_index.update_time";
|
||||
|
||||
if (onlyIndexOnStatusChange) {
|
||||
INSERT_WORKFLOW_INDEX_SQL += " AND workflow_index.status != excluded.status";
|
||||
}
|
||||
|
||||
TemporalAccessor updateTa = DateTimeFormatter.ISO_INSTANT.parse(workflow.getUpdateTime());
|
||||
Timestamp updateTime = Timestamp.from(Instant.from(updateTa));
|
||||
|
||||
TemporalAccessor ta = DateTimeFormatter.ISO_INSTANT.parse(workflow.getStartTime());
|
||||
Timestamp startTime = Timestamp.from(Instant.from(ta));
|
||||
|
||||
int rowsUpdated =
|
||||
queryWithTransaction(
|
||||
INSERT_WORKFLOW_INDEX_SQL,
|
||||
q ->
|
||||
q.addParameter(workflow.getWorkflowId())
|
||||
.addParameter(workflow.getCorrelationId())
|
||||
.addParameter(workflow.getWorkflowType())
|
||||
.addParameter(startTime.toString())
|
||||
.addParameter(updateTime.toString())
|
||||
.addParameter(workflow.getStatus().toString())
|
||||
.addParameter(
|
||||
workflow.getParentWorkflowId() != null
|
||||
? workflow.getParentWorkflowId()
|
||||
: "")
|
||||
.addParameter(workflow.getClassifier())
|
||||
.addJsonParameter(workflow)
|
||||
.executeUpdate());
|
||||
logger.debug("Sqlite index workflow rows updated: {}", rowsUpdated);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SearchResult<WorkflowSummary> searchWorkflowSummary(
|
||||
String query, String freeText, int start, int count, List<String> sort) {
|
||||
SqliteIndexQueryBuilder queryBuilder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"workflow_index", query, freeText, start, count, sort, properties);
|
||||
|
||||
List<WorkflowSummary> results =
|
||||
queryWithTransaction(
|
||||
queryBuilder.getQuery(),
|
||||
q -> {
|
||||
queryBuilder.addParameters(q);
|
||||
queryBuilder.addPagingParameters(q);
|
||||
return q.executeAndFetch(WorkflowSummary.class);
|
||||
});
|
||||
|
||||
List<String> totalHitResults =
|
||||
queryWithTransaction(
|
||||
queryBuilder.getCountQuery(),
|
||||
q -> {
|
||||
queryBuilder.addParameters(q);
|
||||
return q.executeAndFetch(String.class);
|
||||
});
|
||||
|
||||
int totalHits = Integer.valueOf(totalHitResults.get(0));
|
||||
return new SearchResult<>(totalHits, results);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void indexTask(TaskSummary task) {
|
||||
String INSERT_TASK_INDEX_SQL =
|
||||
"INSERT INTO task_index (task_id, task_type, task_def_name, status, start_time, update_time, workflow_type, json_data)"
|
||||
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (task_id) "
|
||||
+ "DO UPDATE SET task_type = excluded.task_type, task_def_name = excluded.task_def_name, "
|
||||
+ "status = excluded.status, update_time = excluded.update_time, json_data = excluded.json_data "
|
||||
+ "WHERE excluded.update_time >= task_index.update_time";
|
||||
|
||||
if (onlyIndexOnStatusChange) {
|
||||
INSERT_TASK_INDEX_SQL += " AND task_index.status != excluded.status";
|
||||
}
|
||||
|
||||
TemporalAccessor updateTa = DateTimeFormatter.ISO_INSTANT.parse(task.getUpdateTime());
|
||||
Timestamp updateTime = Timestamp.from(Instant.from(updateTa));
|
||||
|
||||
TemporalAccessor startTa = DateTimeFormatter.ISO_INSTANT.parse(task.getStartTime());
|
||||
Timestamp startTime = Timestamp.from(Instant.from(startTa));
|
||||
|
||||
int rowsUpdated =
|
||||
queryWithTransaction(
|
||||
INSERT_TASK_INDEX_SQL,
|
||||
q ->
|
||||
q.addParameter(task.getTaskId())
|
||||
.addParameter(task.getTaskType())
|
||||
.addParameter(task.getTaskDefName())
|
||||
.addParameter(task.getStatus().toString())
|
||||
.addParameter(startTime.toString())
|
||||
.addParameter(updateTime.toString())
|
||||
.addParameter(task.getWorkflowType())
|
||||
.addJsonParameter(task)
|
||||
.executeUpdate());
|
||||
logger.debug("Sqlite index task rows updated: {}", rowsUpdated);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SearchResult<TaskSummary> searchTaskSummary(
|
||||
String query, String freeText, int start, int count, List<String> sort) {
|
||||
SqliteIndexQueryBuilder queryBuilder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"task_index", query, freeText, start, count, sort, properties);
|
||||
|
||||
List<TaskSummary> results =
|
||||
queryWithTransaction(
|
||||
queryBuilder.getQuery(),
|
||||
q -> {
|
||||
queryBuilder.addParameters(q);
|
||||
queryBuilder.addPagingParameters(q);
|
||||
return q.executeAndFetch(TaskSummary.class);
|
||||
});
|
||||
|
||||
List<String> totalHitResults =
|
||||
queryWithTransaction(
|
||||
queryBuilder.getCountQuery(),
|
||||
q -> {
|
||||
queryBuilder.addParameters(q);
|
||||
return q.executeAndFetch(String.class);
|
||||
});
|
||||
|
||||
int totalHits = Integer.valueOf(totalHitResults.get(0));
|
||||
return new SearchResult<>(totalHits, results);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addTaskExecutionLogs(List<TaskExecLog> logs) {
|
||||
String INSERT_LOG =
|
||||
"INSERT INTO task_execution_logs (task_id, created_time, log) VALUES (?, ?, ?)";
|
||||
for (TaskExecLog log : logs) {
|
||||
queryWithTransaction(
|
||||
INSERT_LOG,
|
||||
q ->
|
||||
q.addParameter(log.getTaskId())
|
||||
.addParameter(new Timestamp(log.getCreatedTime()))
|
||||
.addParameter(log.getLog())
|
||||
.executeUpdate());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<TaskExecLog> getTaskExecutionLogs(String taskId) {
|
||||
return queryWithTransaction(
|
||||
"SELECT log, task_id, created_time FROM task_execution_logs WHERE task_id = ? ORDER BY created_time ASC",
|
||||
q ->
|
||||
q.addParameter(taskId)
|
||||
.executeAndFetch(
|
||||
rs -> {
|
||||
List<TaskExecLog> result = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
TaskExecLog log = new TaskExecLog();
|
||||
log.setLog(rs.getString("log"));
|
||||
log.setTaskId(rs.getString("task_id"));
|
||||
log.setCreatedTime(
|
||||
rs.getTimestamp("created_time").getTime());
|
||||
result.add(log);
|
||||
}
|
||||
return result;
|
||||
}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void setup() {}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncIndexWorkflow(WorkflowSummary workflow) {
|
||||
logger.info("asyncIndexWorkflow is not supported for Sqlite indexing");
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncIndexTask(TaskSummary task) {
|
||||
logger.info("asyncIndexTask is not supported for Sqlite indexing");
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SearchResult<String> searchWorkflows(
|
||||
String query, String freeText, int start, int count, List<String> sort) {
|
||||
SqliteIndexQueryBuilder queryBuilder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"workflow_index", query, freeText, start, count, sort, properties);
|
||||
|
||||
List<String> results =
|
||||
queryWithTransaction(
|
||||
queryBuilder.getQuery("workflow_id"),
|
||||
q -> {
|
||||
queryBuilder.addParameters(q);
|
||||
queryBuilder.addPagingParameters(q);
|
||||
return q.executeScalarList(String.class);
|
||||
});
|
||||
|
||||
List<String> totalHitResults =
|
||||
queryWithTransaction(
|
||||
queryBuilder.getCountQuery(),
|
||||
q -> {
|
||||
queryBuilder.addParameters(q);
|
||||
return q.executeAndFetch(String.class);
|
||||
});
|
||||
|
||||
int totalHits = Integer.valueOf(totalHitResults.get(0));
|
||||
return new SearchResult<>(totalHits, results);
|
||||
}
|
||||
|
||||
@Override
|
||||
public SearchResult<String> searchTasks(
|
||||
String query, String freeText, int start, int count, List<String> sort) {
|
||||
SqliteIndexQueryBuilder queryBuilder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"task_index", query, freeText, start, count, sort, properties);
|
||||
|
||||
List<String> results =
|
||||
queryWithTransaction(
|
||||
queryBuilder.getQuery("task_id"),
|
||||
q -> {
|
||||
queryBuilder.addParameters(q);
|
||||
queryBuilder.addPagingParameters(q);
|
||||
return q.executeScalarList(String.class);
|
||||
});
|
||||
|
||||
List<String> totalHitResults =
|
||||
queryWithTransaction(
|
||||
queryBuilder.getCountQuery(),
|
||||
q -> {
|
||||
queryBuilder.addParameters(q);
|
||||
return q.executeAndFetch(String.class);
|
||||
});
|
||||
|
||||
int totalHits = Integer.valueOf(totalHitResults.get(0));
|
||||
return new SearchResult<>(totalHits, results);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeWorkflow(String workflowId) {
|
||||
String REMOVE_WORKFLOW_SQL = "DELETE FROM workflow_index WHERE workflow_id = ?";
|
||||
|
||||
queryWithTransaction(REMOVE_WORKFLOW_SQL, q -> q.addParameter(workflowId).executeUpdate());
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncRemoveWorkflow(String workflowId) {
|
||||
return CompletableFuture.runAsync(() -> removeWorkflow(workflowId), executorService);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) {
|
||||
logger.info("updateWorkflow is not supported for Sqlite indexing");
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncUpdateWorkflow(
|
||||
String workflowInstanceId, String[] keys, Object[] values) {
|
||||
logger.info("asyncUpdateWorkflow is not supported for Sqlite indexing");
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeTask(String workflowId, String taskId) {
|
||||
String REMOVE_TASK_SQL = "DELETE FROM task_index WHERE task_id = ?";
|
||||
String REMOVE_TASK_EXECUTION_SQL = "DELETE FROM task_execution_logs WHERE task_id =?";
|
||||
withTransaction(
|
||||
connection -> {
|
||||
queryWithTransaction(
|
||||
REMOVE_TASK_SQL, q -> q.addParameter(taskId).executeUpdate());
|
||||
queryWithTransaction(
|
||||
REMOVE_TASK_EXECUTION_SQL, q -> q.addParameter(taskId).executeUpdate());
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncRemoveTask(String workflowId, String taskId) {
|
||||
return CompletableFuture.runAsync(() -> removeTask(workflowId, taskId), executorService);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateTask(String workflowId, String taskId, String[] keys, Object[] values) {
|
||||
logger.info("updateTask is not supported for Sqlite indexing");
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncUpdateTask(
|
||||
String workflowId, String taskId, String[] keys, Object[] values) {
|
||||
logger.info("asyncUpdateTask is not supported for Sqlite indexing");
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public String get(String workflowInstanceId, String key) {
|
||||
logger.info("get is not supported for Sqlite indexing");
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncAddTaskExecutionLogs(List<TaskExecLog> logs) {
|
||||
logger.info("asyncAddTaskExecutionLogs is not supported for Sqlite indexing");
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addEventExecution(EventExecution eventExecution) {
|
||||
logger.info("addEventExecution is not supported for Sqlite indexing");
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EventExecution> getEventExecutions(String event) {
|
||||
logger.info("getEventExecutions is not supported for Sqlite indexing");
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncAddEventExecution(EventExecution eventExecution) {
|
||||
logger.info("asyncAddEventExecution is not supported for Sqlite indexing");
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addMessage(String queue, Message msg) {
|
||||
logger.info("addMessage is not supported for Sqlite indexing");
|
||||
}
|
||||
|
||||
@Override
|
||||
public CompletableFuture<Void> asyncAddMessage(String queue, Message message) {
|
||||
logger.info("asyncAddMessage is not supported for Sqlite indexing");
|
||||
return CompletableFuture.completedFuture(null);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Message> getMessages(String queue) {
|
||||
logger.info("getMessages is not supported for Sqlite indexing");
|
||||
return null;
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> searchArchivableWorkflows(String indexName, long archiveTtlDays) {
|
||||
logger.info("searchArchivableWorkflows is not supported for Sqlite indexing");
|
||||
return null;
|
||||
}
|
||||
|
||||
public long getWorkflowCount(String query, String freeText) {
|
||||
logger.info("getWorkflowCount is not supported for Sqlite indexing");
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
+133
@@ -0,0 +1,133 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.List;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.common.metadata.tasks.PollData;
|
||||
import com.netflix.conductor.core.exception.NonTransientException;
|
||||
import com.netflix.conductor.dao.PollDataDAO;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
public class SqlitePollDataDAO extends SqliteBaseDAO implements PollDataDAO {
|
||||
|
||||
public SqlitePollDataDAO(
|
||||
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateLastPollData(String taskDefName, String domain, String workerId) {
|
||||
Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
|
||||
|
||||
String effectiveDomain = domain == null ? "DEFAULT" : domain;
|
||||
PollData pollData = new PollData(taskDefName, domain, workerId, System.currentTimeMillis());
|
||||
|
||||
withTransaction(tx -> insertOrUpdatePollData(tx, pollData, effectiveDomain));
|
||||
}
|
||||
|
||||
@Override
|
||||
public PollData getPollData(String taskDefName, String domain) {
|
||||
Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
|
||||
String effectiveDomain = (domain == null) ? "DEFAULT" : domain;
|
||||
return getWithRetriedTransactions(tx -> readPollData(tx, taskDefName, effectiveDomain));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<PollData> getPollData(String taskDefName) {
|
||||
Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
|
||||
return readAllPollData(taskDefName);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<PollData> getAllPollData() {
|
||||
try (Connection tx = dataSource.getConnection()) {
|
||||
boolean previousAutoCommitMode = tx.getAutoCommit();
|
||||
tx.setAutoCommit(true);
|
||||
try {
|
||||
String GET_ALL_POLL_DATA = "SELECT json_data FROM poll_data ORDER BY queue_name";
|
||||
return query(tx, GET_ALL_POLL_DATA, q -> q.executeAndFetch(PollData.class));
|
||||
} catch (Throwable th) {
|
||||
throw new NonTransientException(th.getMessage(), th);
|
||||
} finally {
|
||||
tx.setAutoCommit(previousAutoCommitMode);
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
private void insertOrUpdatePollData(Connection connection, PollData pollData, String domain) {
|
||||
try {
|
||||
/*
|
||||
* Most times the row will be updated so let's try the update first. This used to be an 'INSERT/ON CONFLICT do update' sql statement. The problem with that
|
||||
* is that if we try the INSERT first, the sequence will be increased even if the ON CONFLICT happens. Since polling happens *a lot*, the sequence can increase
|
||||
* dramatically even though it won't be used.
|
||||
*/
|
||||
String UPDATE_POLL_DATA =
|
||||
"UPDATE poll_data SET json_data=?, modified_on=CURRENT_TIMESTAMP WHERE queue_name=? AND domain=?";
|
||||
int rowsUpdated =
|
||||
query(
|
||||
connection,
|
||||
UPDATE_POLL_DATA,
|
||||
q ->
|
||||
q.addJsonParameter(pollData)
|
||||
.addParameter(pollData.getQueueName())
|
||||
.addParameter(domain)
|
||||
.executeUpdate());
|
||||
|
||||
if (rowsUpdated == 0) {
|
||||
String INSERT_POLL_DATA =
|
||||
"INSERT INTO poll_data (queue_name, domain, json_data, modified_on) VALUES (?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT (queue_name,domain) DO UPDATE SET json_data=excluded.json_data, modified_on=excluded.modified_on";
|
||||
execute(
|
||||
connection,
|
||||
INSERT_POLL_DATA,
|
||||
q ->
|
||||
q.addParameter(pollData.getQueueName())
|
||||
.addParameter(domain)
|
||||
.addJsonParameter(pollData)
|
||||
.executeUpdate());
|
||||
}
|
||||
} catch (NonTransientException e) {
|
||||
if (!e.getMessage().startsWith("ERROR: lastPollTime cannot be set to a lower value")) {
|
||||
throw e;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private PollData readPollData(Connection connection, String queueName, String domain) {
|
||||
String GET_POLL_DATA =
|
||||
"SELECT json_data FROM poll_data WHERE queue_name = ? AND domain = ?";
|
||||
return query(
|
||||
connection,
|
||||
GET_POLL_DATA,
|
||||
q ->
|
||||
q.addParameter(queueName)
|
||||
.addParameter(domain)
|
||||
.executeAndFetchFirst(PollData.class));
|
||||
}
|
||||
|
||||
private List<PollData> readAllPollData(String queueName) {
|
||||
String GET_ALL_POLL_DATA = "SELECT json_data FROM poll_data WHERE queue_name = ?";
|
||||
return queryWithTransaction(
|
||||
GET_ALL_POLL_DATA, q -> q.addParameter(queueName).executeAndFetch(PollData.class));
|
||||
}
|
||||
}
|
||||
+508
@@ -0,0 +1,508 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.HashMap;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ScheduledExecutorService;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.core.events.queue.Message;
|
||||
import com.netflix.conductor.dao.QueueDAO;
|
||||
import com.netflix.conductor.sqlite.config.SqliteProperties;
|
||||
import com.netflix.conductor.sqlite.util.ExecutorsUtil;
|
||||
import com.netflix.conductor.sqlite.util.Query;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.ImmutableMap;
|
||||
import com.google.common.collect.Maps;
|
||||
import com.google.common.util.concurrent.Uninterruptibles;
|
||||
import jakarta.annotation.PreDestroy;
|
||||
|
||||
public class SqliteQueueDAO extends SqliteBaseDAO implements QueueDAO {
|
||||
|
||||
private static final Long UNACK_SCHEDULE_MS = 60_000L;
|
||||
|
||||
private final ScheduledExecutorService scheduledExecutorService;
|
||||
|
||||
public SqliteQueueDAO(
|
||||
RetryTemplate retryTemplate,
|
||||
ObjectMapper objectMapper,
|
||||
DataSource dataSource,
|
||||
SqliteProperties properties) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
|
||||
this.scheduledExecutorService =
|
||||
Executors.newSingleThreadScheduledExecutor(
|
||||
ExecutorsUtil.newNamedThreadFactory("sqlite-queue-"));
|
||||
this.scheduledExecutorService.scheduleAtFixedRate(
|
||||
this::processAllUnacks,
|
||||
UNACK_SCHEDULE_MS,
|
||||
UNACK_SCHEDULE_MS,
|
||||
TimeUnit.MILLISECONDS);
|
||||
logger.debug("{} is ready to serve", SqliteQueueDAO.class.getName());
|
||||
}
|
||||
|
||||
@PreDestroy
|
||||
public void destroy() {
|
||||
try {
|
||||
this.scheduledExecutorService.shutdown();
|
||||
if (scheduledExecutorService.awaitTermination(30, TimeUnit.SECONDS)) {
|
||||
logger.debug("tasks completed, shutting down");
|
||||
} else {
|
||||
logger.warn("Forcing shutdown after waiting for 30 seconds");
|
||||
scheduledExecutorService.shutdownNow();
|
||||
}
|
||||
} catch (InterruptedException ie) {
|
||||
logger.warn(
|
||||
"Shutdown interrupted, invoking shutdownNow on scheduledExecutorService for processAllUnacks",
|
||||
ie);
|
||||
scheduledExecutorService.shutdownNow();
|
||||
Thread.currentThread().interrupt();
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void push(String queueName, String id, long offsetTimeInSecond) {
|
||||
push(queueName, id, 0, offsetTimeInSecond);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void push(String queueName, String id, int priority, long offsetTimeInSecond) {
|
||||
withTransaction(tx -> pushMessage(tx, queueName, id, null, priority, offsetTimeInSecond));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void push(String queueName, List<Message> messages) {
|
||||
withTransaction(
|
||||
tx ->
|
||||
messages.forEach(
|
||||
message ->
|
||||
pushMessage(
|
||||
tx,
|
||||
queueName,
|
||||
message.getId(),
|
||||
message.getPayload(),
|
||||
message.getPriority(),
|
||||
0)));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean pushIfNotExists(String queueName, String id, long offsetTimeInSecond) {
|
||||
return pushIfNotExists(queueName, id, 0, offsetTimeInSecond);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean pushIfNotExists(
|
||||
String queueName, String id, int priority, long offsetTimeInSecond) {
|
||||
return getWithRetriedTransactions(
|
||||
tx -> {
|
||||
if (!existsMessage(tx, queueName, id)) {
|
||||
pushMessage(tx, queueName, id, null, priority, offsetTimeInSecond);
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> pop(String queueName, int count, int timeout) {
|
||||
return pollMessages(queueName, count, timeout).stream()
|
||||
.map(Message::getId)
|
||||
.collect(Collectors.toList());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> peekFirstIds(String queueName, int count) {
|
||||
final String SQL =
|
||||
"SELECT message_id FROM queue_message "
|
||||
+ "WHERE queue_name = ? AND popped = false "
|
||||
+ "ORDER BY deliver_on, priority DESC, created_on LIMIT ?";
|
||||
return queryWithTransaction(
|
||||
SQL,
|
||||
q -> q.addParameter(queueName).addParameter(count).executeScalarList(String.class));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<Message> pollMessages(String queueName, int count, int timeout) {
|
||||
if (timeout < 1) {
|
||||
List<Message> messages =
|
||||
getWithTransactionWithOutErrorPropagation(
|
||||
tx -> popMessages(tx, queueName, count, timeout));
|
||||
if (messages == null) {
|
||||
return new ArrayList<>();
|
||||
}
|
||||
return messages;
|
||||
}
|
||||
|
||||
long start = System.currentTimeMillis();
|
||||
final List<Message> messages = new ArrayList<>();
|
||||
|
||||
while (true) {
|
||||
List<Message> messagesSlice =
|
||||
getWithTransactionWithOutErrorPropagation(
|
||||
tx -> popMessages(tx, queueName, count - messages.size(), timeout));
|
||||
if (messagesSlice == null) {
|
||||
logger.warn(
|
||||
"Unable to poll {} messages from {} due to tx conflict, only {} popped",
|
||||
count,
|
||||
queueName,
|
||||
messages.size());
|
||||
// conflict could have happened, returned messages popped so far
|
||||
return messages;
|
||||
}
|
||||
|
||||
messages.addAll(messagesSlice);
|
||||
if (messages.size() >= count || ((System.currentTimeMillis() - start) > timeout)) {
|
||||
return messages;
|
||||
}
|
||||
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void remove(String queueName, String messageId) {
|
||||
withTransaction(tx -> removeMessage(tx, queueName, messageId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public int getSize(String queueName) {
|
||||
final String GET_QUEUE_SIZE = "SELECT COUNT(*) FROM queue_message WHERE queue_name = ?";
|
||||
return queryWithTransaction(
|
||||
GET_QUEUE_SIZE, q -> ((Long) q.addParameter(queueName).executeCount()).intValue());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean ack(String queueName, String messageId) {
|
||||
return getWithRetriedTransactions(tx -> removeMessage(tx, queueName, messageId));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean setUnackTimeout(String queueName, String messageId, long unackTimeout) {
|
||||
long updatedOffsetTimeInSecond = unackTimeout / 1000;
|
||||
|
||||
final String UPDATE_UNACK_TIMEOUT =
|
||||
"UPDATE queue_message SET offset_time_seconds = ?, deliver_on = strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds') WHERE queue_name = ? AND message_id = ?";
|
||||
|
||||
return queryWithTransaction(
|
||||
UPDATE_UNACK_TIMEOUT,
|
||||
q ->
|
||||
q.addParameter(updatedOffsetTimeInSecond)
|
||||
.addParameter(updatedOffsetTimeInSecond)
|
||||
.addParameter(queueName)
|
||||
.addParameter(messageId)
|
||||
.executeUpdate())
|
||||
== 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean setUnackTimeoutIfShorter(String queueName, String messageId, long unackTimeout) {
|
||||
long updatedOffsetTimeInSecond = unackTimeout / 1000;
|
||||
|
||||
// Only update when the proposed deliver_on is earlier than what is already set,
|
||||
// mirroring the ZADD LT semantics used by the Redis implementation.
|
||||
final String UPDATE_UNACK_TIMEOUT_IF_SHORTER =
|
||||
"UPDATE queue_message SET offset_time_seconds = ?, deliver_on = strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds')"
|
||||
+ " WHERE queue_name = ? AND message_id = ? AND deliver_on > strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds')";
|
||||
|
||||
return queryWithTransaction(
|
||||
UPDATE_UNACK_TIMEOUT_IF_SHORTER,
|
||||
q ->
|
||||
q.addParameter(updatedOffsetTimeInSecond)
|
||||
.addParameter(updatedOffsetTimeInSecond)
|
||||
.addParameter(queueName)
|
||||
.addParameter(messageId)
|
||||
.addParameter(updatedOffsetTimeInSecond)
|
||||
.executeUpdate())
|
||||
== 1;
|
||||
}
|
||||
|
||||
@Override
|
||||
public void flush(String queueName) {
|
||||
final String FLUSH_QUEUE = "DELETE FROM queue_message WHERE queue_name = ?";
|
||||
executeWithTransaction(FLUSH_QUEUE, q -> q.addParameter(queueName).executeDelete());
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Long> queuesDetail() {
|
||||
final String GET_QUEUES_DETAIL =
|
||||
"SELECT queue_name, (SELECT count(*) FROM queue_message WHERE popped = false AND queue_name = q.queue_name) AS size FROM queue q";
|
||||
return queryWithTransaction(
|
||||
GET_QUEUES_DETAIL,
|
||||
q ->
|
||||
q.executeAndFetch(
|
||||
rs -> {
|
||||
Map<String, Long> detail = Maps.newHashMap();
|
||||
while (rs.next()) {
|
||||
String queueName = rs.getString("queue_name");
|
||||
Long size = rs.getLong("size");
|
||||
detail.put(queueName, size);
|
||||
}
|
||||
return detail;
|
||||
}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public Map<String, Map<String, Map<String, Long>>> queuesDetailVerbose() {
|
||||
// @formatter:off
|
||||
final String GET_QUEUES_DETAIL_VERBOSE =
|
||||
"SELECT queue_name, \n"
|
||||
+ " (SELECT count(*) FROM queue_message WHERE popped = false AND queue_name = q.queue_name) AS size,\n"
|
||||
+ " (SELECT count(*) FROM queue_message WHERE popped = true AND queue_name = q.queue_name) AS uacked \n"
|
||||
+ "FROM queue q";
|
||||
// @formatter:on
|
||||
|
||||
return queryWithTransaction(
|
||||
GET_QUEUES_DETAIL_VERBOSE,
|
||||
q ->
|
||||
q.executeAndFetch(
|
||||
rs -> {
|
||||
Map<String, Map<String, Map<String, Long>>> result =
|
||||
Maps.newHashMap();
|
||||
while (rs.next()) {
|
||||
String queueName = rs.getString("queue_name");
|
||||
Long size = rs.getLong("size");
|
||||
Long queueUnacked = rs.getLong("uacked");
|
||||
result.put(
|
||||
queueName,
|
||||
ImmutableMap.of(
|
||||
"a",
|
||||
ImmutableMap
|
||||
.of( // sharding not implemented,
|
||||
// returning only
|
||||
// one shard with all the
|
||||
// info
|
||||
"size",
|
||||
size,
|
||||
"uacked",
|
||||
queueUnacked)));
|
||||
}
|
||||
return result;
|
||||
}));
|
||||
}
|
||||
|
||||
public void processAllUnacks() {
|
||||
logger.trace("processAllUnacks started");
|
||||
|
||||
getWithRetriedTransactions(
|
||||
tx -> {
|
||||
String LOCK_TASKS =
|
||||
"SELECT queue_name, message_id FROM queue_message WHERE popped = true AND strftime('%Y-%m-%d %H:%M:%f', deliver_on, '+60 seconds') < strftime('%Y-%m-%d %H:%M:%f', 'now') limit 1000";
|
||||
|
||||
List<QueueMessage> messages =
|
||||
query(
|
||||
tx,
|
||||
LOCK_TASKS,
|
||||
p ->
|
||||
p.executeAndFetch(
|
||||
rs -> {
|
||||
List<QueueMessage> results =
|
||||
new ArrayList<QueueMessage>();
|
||||
while (rs.next()) {
|
||||
QueueMessage qm = new QueueMessage();
|
||||
qm.queueName =
|
||||
rs.getString("queue_name");
|
||||
qm.messageId =
|
||||
rs.getString("message_id");
|
||||
results.add(qm);
|
||||
}
|
||||
return results;
|
||||
}));
|
||||
|
||||
if (messages.size() == 0) {
|
||||
return 0;
|
||||
}
|
||||
|
||||
Map<String, List<String>> queueMessageMap = new HashMap<String, List<String>>();
|
||||
for (QueueMessage qm : messages) {
|
||||
if (!queueMessageMap.containsKey(qm.queueName)) {
|
||||
queueMessageMap.put(qm.queueName, new ArrayList<String>());
|
||||
}
|
||||
queueMessageMap.get(qm.queueName).add(qm.messageId);
|
||||
}
|
||||
|
||||
int totalUnacked = 0;
|
||||
for (String queueName : queueMessageMap.keySet()) {
|
||||
Integer unacked = 0;
|
||||
try {
|
||||
final List<String> msgIds = queueMessageMap.get(queueName);
|
||||
final String UPDATE_POPPED =
|
||||
String.format(
|
||||
"UPDATE queue_message SET popped = false WHERE queue_name = ? and message_id IN (%s)",
|
||||
Query.generateInBindings(msgIds.size()));
|
||||
|
||||
unacked =
|
||||
query(
|
||||
tx,
|
||||
UPDATE_POPPED,
|
||||
q ->
|
||||
q.addParameter(queueName)
|
||||
.addParameters(msgIds)
|
||||
.executeUpdate());
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
}
|
||||
totalUnacked += unacked;
|
||||
logger.debug("Unacked {} messages from all queues", unacked);
|
||||
}
|
||||
|
||||
if (totalUnacked > 0) {
|
||||
logger.debug("Unacked {} messages from all queues", totalUnacked);
|
||||
}
|
||||
return totalUnacked;
|
||||
});
|
||||
}
|
||||
|
||||
@Override
|
||||
public void processUnacks(String queueName) {
|
||||
final String PROCESS_UNACKS =
|
||||
"UPDATE queue_message SET popped = false WHERE queue_name = ? AND popped = true AND strftime('%Y-%m-%d %H:%M:%f', 'now', '-60 seconds') > deliver_on";
|
||||
executeWithTransaction(PROCESS_UNACKS, q -> q.addParameter(queueName).executeUpdate());
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean resetOffsetTime(String queueName, String id) {
|
||||
long offsetTimeInSecond = 0; // Reset to 0
|
||||
final String SET_OFFSET_TIME =
|
||||
"UPDATE queue_message SET offset_time_seconds = ?, deliver_on = strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds') "
|
||||
+ "WHERE queue_name = ? AND message_id = ?";
|
||||
|
||||
return queryWithTransaction(
|
||||
SET_OFFSET_TIME,
|
||||
q ->
|
||||
q.addParameter(offsetTimeInSecond)
|
||||
.addParameter(offsetTimeInSecond)
|
||||
.addParameter(queueName)
|
||||
.addParameter(id)
|
||||
.executeUpdate()
|
||||
== 1);
|
||||
}
|
||||
|
||||
private boolean existsMessage(Connection connection, String queueName, String messageId) {
|
||||
final String EXISTS_MESSAGE =
|
||||
"SELECT EXISTS(SELECT 1 FROM queue_message WHERE queue_name = ? AND message_id = ?)";
|
||||
return query(
|
||||
connection,
|
||||
EXISTS_MESSAGE,
|
||||
q -> q.addParameter(queueName).addParameter(messageId).exists());
|
||||
}
|
||||
|
||||
private void pushMessage(
|
||||
Connection connection,
|
||||
String queueName,
|
||||
String messageId,
|
||||
String payload,
|
||||
Integer priority,
|
||||
long offsetTimeInSecond) {
|
||||
|
||||
createQueueIfNotExists(connection, queueName);
|
||||
|
||||
String UPDATE_MESSAGE =
|
||||
"UPDATE queue_message SET payload=?, deliver_on=strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds'), popped=false WHERE queue_name = ? AND message_id = ?";
|
||||
int rowsUpdated =
|
||||
query(
|
||||
connection,
|
||||
UPDATE_MESSAGE,
|
||||
q ->
|
||||
q.addParameter(payload)
|
||||
.addParameter(offsetTimeInSecond)
|
||||
.addParameter(queueName)
|
||||
.addParameter(messageId)
|
||||
.executeUpdate());
|
||||
|
||||
if (rowsUpdated == 0) {
|
||||
String PUSH_MESSAGE =
|
||||
"INSERT INTO queue_message (deliver_on, queue_name, message_id, priority, offset_time_seconds, payload) VALUES (strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds'), ?,?,?,?,?) ON CONFLICT (queue_name,message_id) DO UPDATE SET payload=excluded.payload, deliver_on=excluded.deliver_on, popped=false";
|
||||
execute(
|
||||
connection,
|
||||
PUSH_MESSAGE,
|
||||
q ->
|
||||
q.addParameter(offsetTimeInSecond)
|
||||
.addParameter(queueName)
|
||||
.addParameter(messageId)
|
||||
.addParameter(priority)
|
||||
.addParameter(offsetTimeInSecond)
|
||||
.addParameter(payload)
|
||||
.executeUpdate());
|
||||
}
|
||||
}
|
||||
|
||||
private boolean removeMessage(Connection connection, String queueName, String messageId) {
|
||||
final String REMOVE_MESSAGE =
|
||||
"DELETE FROM queue_message WHERE queue_name = ? AND message_id = ?";
|
||||
return query(
|
||||
connection,
|
||||
REMOVE_MESSAGE,
|
||||
q -> q.addParameter(queueName).addParameter(messageId).executeDelete());
|
||||
}
|
||||
|
||||
private List<Message> popMessages(
|
||||
Connection connection, String queueName, int count, int timeout) {
|
||||
|
||||
String POP_QUERY =
|
||||
"UPDATE queue_message SET popped = true WHERE message_id IN ("
|
||||
+ "SELECT message_id FROM queue_message WHERE queue_name = ? AND popped = false AND "
|
||||
+ "deliver_on <= strftime('%Y-%m-%d %H:%M:%f', 'now') "
|
||||
+ "ORDER BY priority DESC, deliver_on, created_on LIMIT ?"
|
||||
+ ") RETURNING message_id, priority, payload";
|
||||
|
||||
return query(
|
||||
connection,
|
||||
POP_QUERY,
|
||||
p ->
|
||||
p.addParameter(queueName)
|
||||
.addParameter(count)
|
||||
.executeAndFetch(
|
||||
rs -> {
|
||||
List<Message> results = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
Message m = new Message();
|
||||
m.setId(rs.getString("message_id"));
|
||||
m.setPriority(rs.getInt("priority"));
|
||||
m.setPayload(rs.getString("payload"));
|
||||
results.add(m);
|
||||
}
|
||||
return results;
|
||||
}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean containsMessage(String queueName, String messageId) {
|
||||
return getWithRetriedTransactions(tx -> existsMessage(tx, queueName, messageId));
|
||||
}
|
||||
|
||||
private void createQueueIfNotExists(Connection connection, String queueName) {
|
||||
logger.trace("Creating new queue '{}'", queueName);
|
||||
final String EXISTS_QUEUE = "SELECT EXISTS(SELECT 1 FROM queue WHERE queue_name = ?)";
|
||||
boolean exists = query(connection, EXISTS_QUEUE, q -> q.addParameter(queueName).exists());
|
||||
if (!exists) {
|
||||
final String CREATE_QUEUE =
|
||||
"INSERT INTO queue (queue_name) VALUES (?) ON CONFLICT (queue_name) DO NOTHING";
|
||||
execute(connection, CREATE_QUEUE, q -> q.addParameter(queueName).executeUpdate());
|
||||
}
|
||||
}
|
||||
|
||||
private class QueueMessage {
|
||||
public String queueName;
|
||||
public String messageId;
|
||||
}
|
||||
}
|
||||
+151
@@ -0,0 +1,151 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao.metadata;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.common.metadata.events.EventHandler;
|
||||
import com.netflix.conductor.core.exception.ConflictException;
|
||||
import com.netflix.conductor.core.exception.NotFoundException;
|
||||
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
public class SqliteEventHandlerMetadataDAO extends SqliteBaseDAO {
|
||||
|
||||
public SqliteEventHandlerMetadataDAO(
|
||||
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
public void addEventHandler(EventHandler eventHandler) {
|
||||
Preconditions.checkNotNull(eventHandler.getName(), "EventHandler name cannot be null");
|
||||
|
||||
final String INSERT_EVENT_HANDLER_QUERY =
|
||||
"INSERT INTO meta_event_handler (name, event, active, json_data) "
|
||||
+ "VALUES (?, ?, ?, ?)";
|
||||
|
||||
withTransaction(
|
||||
tx -> {
|
||||
if (getEventHandler(tx, eventHandler.getName()) != null) {
|
||||
throw new ConflictException(
|
||||
"EventHandler with name "
|
||||
+ eventHandler.getName()
|
||||
+ " already exists!");
|
||||
}
|
||||
|
||||
execute(
|
||||
tx,
|
||||
INSERT_EVENT_HANDLER_QUERY,
|
||||
q ->
|
||||
q.addParameter(eventHandler.getName())
|
||||
.addParameter(eventHandler.getEvent())
|
||||
.addParameter(eventHandler.isActive())
|
||||
.addJsonParameter(eventHandler)
|
||||
.executeUpdate());
|
||||
});
|
||||
}
|
||||
|
||||
public void updateEventHandler(EventHandler eventHandler) {
|
||||
Preconditions.checkNotNull(eventHandler.getName(), "EventHandler name cannot be null");
|
||||
|
||||
// @formatter:off
|
||||
final String UPDATE_EVENT_HANDLER_QUERY =
|
||||
"UPDATE meta_event_handler SET "
|
||||
+ "event = ?, active = ?, json_data = ?, "
|
||||
+ "modified_on = CURRENT_TIMESTAMP WHERE name = ?";
|
||||
// @formatter:on
|
||||
|
||||
withTransaction(
|
||||
tx -> {
|
||||
EventHandler existing = getEventHandler(tx, eventHandler.getName());
|
||||
if (existing == null) {
|
||||
throw new NotFoundException(
|
||||
"EventHandler with name " + eventHandler.getName() + " not found!");
|
||||
}
|
||||
|
||||
execute(
|
||||
tx,
|
||||
UPDATE_EVENT_HANDLER_QUERY,
|
||||
q ->
|
||||
q.addParameter(eventHandler.getEvent())
|
||||
.addParameter(eventHandler.isActive())
|
||||
.addJsonParameter(eventHandler)
|
||||
.addParameter(eventHandler.getName())
|
||||
.executeUpdate());
|
||||
});
|
||||
}
|
||||
|
||||
public void removeEventHandler(String name) {
|
||||
final String DELETE_EVENT_HANDLER_QUERY = "DELETE FROM meta_event_handler WHERE name = ?";
|
||||
|
||||
withTransaction(
|
||||
tx -> {
|
||||
EventHandler existing = getEventHandler(tx, name);
|
||||
if (existing == null) {
|
||||
throw new NotFoundException(
|
||||
"EventHandler with name " + name + " not found!");
|
||||
}
|
||||
|
||||
execute(
|
||||
tx,
|
||||
DELETE_EVENT_HANDLER_QUERY,
|
||||
q -> q.addParameter(name).executeDelete());
|
||||
});
|
||||
}
|
||||
|
||||
public List<EventHandler> getEventHandlersForEvent(String event, boolean activeOnly) {
|
||||
final String READ_ALL_EVENT_HANDLER_BY_EVENT_QUERY =
|
||||
"SELECT json_data FROM meta_event_handler WHERE event = ?";
|
||||
return queryWithTransaction(
|
||||
READ_ALL_EVENT_HANDLER_BY_EVENT_QUERY,
|
||||
q -> {
|
||||
q.addParameter(event);
|
||||
return q.executeAndFetch(
|
||||
rs -> {
|
||||
List<EventHandler> handlers = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
EventHandler h = readValue(rs.getString(1), EventHandler.class);
|
||||
if (!activeOnly || h.isActive()) {
|
||||
handlers.add(h);
|
||||
}
|
||||
}
|
||||
|
||||
return handlers;
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
public List<EventHandler> getAllEventHandlers() {
|
||||
final String READ_ALL_EVENT_HANDLER_QUERY = "SELECT json_data FROM meta_event_handler";
|
||||
return queryWithTransaction(
|
||||
READ_ALL_EVENT_HANDLER_QUERY, q -> q.executeAndFetch(EventHandler.class));
|
||||
}
|
||||
|
||||
private EventHandler getEventHandler(Connection connection, String name) {
|
||||
final String READ_ONE_EVENT_HANDLER_QUERY =
|
||||
"SELECT json_data FROM meta_event_handler WHERE name = ?";
|
||||
|
||||
return query(
|
||||
connection,
|
||||
READ_ONE_EVENT_HANDLER_QUERY,
|
||||
q -> q.addParameter(name).executeAndFetchFirst(EventHandler.class));
|
||||
}
|
||||
}
|
||||
+129
@@ -0,0 +1,129 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao.metadata;
|
||||
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import com.netflix.conductor.common.metadata.events.EventHandler;
|
||||
import com.netflix.conductor.common.metadata.tasks.TaskDef;
|
||||
import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
|
||||
import com.netflix.conductor.dao.EventHandlerDAO;
|
||||
import com.netflix.conductor.dao.MetadataDAO;
|
||||
|
||||
import lombok.RequiredArgsConstructor;
|
||||
|
||||
@RequiredArgsConstructor
|
||||
public class SqliteMetadataDAO implements MetadataDAO, EventHandlerDAO {
|
||||
|
||||
private final SqliteTaskMetadataDAO taskMetadataDAO;
|
||||
private final SqliteWorkflowMetadataDAO workflowMetadataDAO;
|
||||
private final SqliteEventHandlerMetadataDAO eventHandlerMetadataDAO;
|
||||
|
||||
@Override
|
||||
public TaskDef createTaskDef(TaskDef taskDef) {
|
||||
return taskMetadataDAO.createTaskDef(taskDef);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TaskDef updateTaskDef(TaskDef taskDef) {
|
||||
return taskMetadataDAO.updateTaskDef(taskDef);
|
||||
}
|
||||
|
||||
@Override
|
||||
public TaskDef getTaskDef(String name) {
|
||||
return taskMetadataDAO.getTaskDef(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<TaskDef> getAllTaskDefs() {
|
||||
return taskMetadataDAO.getAllTaskDefs();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeTaskDef(String name) {
|
||||
taskMetadataDAO.removeTaskDef(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createWorkflowDef(WorkflowDef def) {
|
||||
workflowMetadataDAO.createWorkflowDef(def);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateWorkflowDef(WorkflowDef def) {
|
||||
workflowMetadataDAO.updateWorkflowDef(def);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<WorkflowDef> getLatestWorkflowDef(String name) {
|
||||
return workflowMetadataDAO.getLatestWorkflowDef(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<WorkflowDef> getWorkflowDef(String name, int version) {
|
||||
return workflowMetadataDAO.getWorkflowDef(name, version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeWorkflowDef(String name, Integer version) {
|
||||
workflowMetadataDAO.removeWorkflowDef(name, version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<WorkflowDef> getAllWorkflowDefs() {
|
||||
return workflowMetadataDAO.getAllWorkflowDefs();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<WorkflowDef> getAllWorkflowDefsLatestVersions() {
|
||||
return workflowMetadataDAO.getAllWorkflowDefsLatestVersions();
|
||||
}
|
||||
|
||||
@Override
|
||||
public void addEventHandler(EventHandler eventHandler) {
|
||||
eventHandlerMetadataDAO.addEventHandler(eventHandler);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateEventHandler(EventHandler eventHandler) {
|
||||
eventHandlerMetadataDAO.updateEventHandler(eventHandler);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void removeEventHandler(String name) {
|
||||
eventHandlerMetadataDAO.removeEventHandler(name);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EventHandler> getAllEventHandlers() {
|
||||
return eventHandlerMetadataDAO.getAllEventHandlers();
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<EventHandler> getEventHandlersForEvent(String event, boolean activeOnly) {
|
||||
return eventHandlerMetadataDAO.getEventHandlersForEvent(event, activeOnly);
|
||||
}
|
||||
|
||||
public List<String> findAll() {
|
||||
return workflowMetadataDAO.findAll();
|
||||
}
|
||||
|
||||
public List<WorkflowDef> getAllLatest() {
|
||||
return workflowMetadataDAO.getAllLatest();
|
||||
}
|
||||
|
||||
public List<WorkflowDef> getAllVersions(String name) {
|
||||
return workflowMetadataDAO.getAllVersions(name);
|
||||
}
|
||||
}
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao.metadata;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.List;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.common.metadata.tasks.TaskDef;
|
||||
import com.netflix.conductor.core.exception.NotFoundException;
|
||||
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
public class SqliteTaskMetadataDAO extends SqliteBaseDAO {
|
||||
|
||||
public SqliteTaskMetadataDAO(
|
||||
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
public TaskDef createTaskDef(TaskDef taskDef) {
|
||||
validate(taskDef);
|
||||
insertOrUpdateTaskDef(taskDef);
|
||||
return taskDef;
|
||||
}
|
||||
|
||||
public TaskDef updateTaskDef(TaskDef taskDef) {
|
||||
validate(taskDef);
|
||||
insertOrUpdateTaskDef(taskDef);
|
||||
return taskDef;
|
||||
}
|
||||
|
||||
public TaskDef getTaskDef(String name) {
|
||||
Preconditions.checkNotNull(name, "TaskDef name cannot be null");
|
||||
return getTaskDefFromDB(name);
|
||||
}
|
||||
|
||||
public List<TaskDef> getAllTaskDefs() {
|
||||
return getWithRetriedTransactions(this::findAllTaskDefs);
|
||||
}
|
||||
|
||||
public void removeTaskDef(String name) {
|
||||
final String DELETE_TASKDEF_QUERY = "DELETE FROM meta_task_def WHERE name = ?";
|
||||
|
||||
executeWithTransaction(
|
||||
DELETE_TASKDEF_QUERY,
|
||||
q -> {
|
||||
if (!q.addParameter(name).executeDelete()) {
|
||||
throw new NotFoundException("No such task definition");
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
private void validate(TaskDef taskDef) {
|
||||
Preconditions.checkNotNull(taskDef, "TaskDef object cannot be null");
|
||||
Preconditions.checkNotNull(taskDef.getName(), "TaskDef name cannot be null");
|
||||
}
|
||||
|
||||
private TaskDef getTaskDefFromDB(String name) {
|
||||
final String READ_ONE_TASKDEF_QUERY = "SELECT json_data FROM meta_task_def WHERE name = ?";
|
||||
|
||||
return queryWithTransaction(
|
||||
READ_ONE_TASKDEF_QUERY,
|
||||
q -> q.addParameter(name).executeAndFetchFirst(TaskDef.class));
|
||||
}
|
||||
|
||||
private String insertOrUpdateTaskDef(TaskDef taskDef) {
|
||||
final String UPDATE_TASKDEF_QUERY =
|
||||
"UPDATE meta_task_def SET json_data = ?, modified_on = CURRENT_TIMESTAMP WHERE name = ?";
|
||||
|
||||
final String INSERT_TASKDEF_QUERY =
|
||||
"INSERT INTO meta_task_def (name, json_data) VALUES (?, ?)";
|
||||
|
||||
return getWithRetriedTransactions(
|
||||
tx -> {
|
||||
execute(
|
||||
tx,
|
||||
UPDATE_TASKDEF_QUERY,
|
||||
update -> {
|
||||
int result =
|
||||
update.addJsonParameter(taskDef)
|
||||
.addParameter(taskDef.getName())
|
||||
.executeUpdate();
|
||||
if (result == 0) {
|
||||
execute(
|
||||
tx,
|
||||
INSERT_TASKDEF_QUERY,
|
||||
insert ->
|
||||
insert.addParameter(taskDef.getName())
|
||||
.addJsonParameter(taskDef)
|
||||
.executeUpdate());
|
||||
}
|
||||
});
|
||||
return taskDef.getName();
|
||||
});
|
||||
}
|
||||
|
||||
private List<TaskDef> findAllTaskDefs(Connection tx) {
|
||||
final String READ_ALL_TASKDEF_QUERY = "SELECT json_data FROM meta_task_def";
|
||||
|
||||
return query(tx, READ_ALL_TASKDEF_QUERY, q -> q.executeAndFetch(TaskDef.class));
|
||||
}
|
||||
}
|
||||
+229
@@ -0,0 +1,229 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao.metadata;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
|
||||
import com.netflix.conductor.core.exception.ConflictException;
|
||||
import com.netflix.conductor.core.exception.NotFoundException;
|
||||
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.base.Preconditions;
|
||||
|
||||
public class SqliteWorkflowMetadataDAO extends SqliteBaseDAO {
|
||||
|
||||
public SqliteWorkflowMetadataDAO(
|
||||
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
public void createWorkflowDef(WorkflowDef def) {
|
||||
validate(def);
|
||||
|
||||
withTransaction(
|
||||
tx -> {
|
||||
if (workflowExists(tx, def)) {
|
||||
throw new ConflictException(
|
||||
"Workflow with " + def.key() + " already exists!");
|
||||
}
|
||||
|
||||
insertOrUpdateWorkflowDef(tx, def);
|
||||
});
|
||||
}
|
||||
|
||||
public void updateWorkflowDef(WorkflowDef def) {
|
||||
validate(def);
|
||||
withTransaction(tx -> insertOrUpdateWorkflowDef(tx, def));
|
||||
}
|
||||
|
||||
public Optional<WorkflowDef> getLatestWorkflowDef(String name) {
|
||||
final String GET_LATEST_WORKFLOW_DEF_QUERY =
|
||||
"SELECT json_data FROM meta_workflow_def WHERE NAME = ? AND "
|
||||
+ "version = latest_version";
|
||||
|
||||
return Optional.ofNullable(
|
||||
queryWithTransaction(
|
||||
GET_LATEST_WORKFLOW_DEF_QUERY,
|
||||
q -> q.addParameter(name).executeAndFetchFirst(WorkflowDef.class)));
|
||||
}
|
||||
|
||||
public Optional<WorkflowDef> getWorkflowDef(String name, int version) {
|
||||
final String GET_WORKFLOW_DEF_QUERY =
|
||||
"SELECT json_data FROM meta_workflow_def WHERE NAME = ? AND version = ?";
|
||||
return Optional.ofNullable(
|
||||
queryWithTransaction(
|
||||
GET_WORKFLOW_DEF_QUERY,
|
||||
q ->
|
||||
q.addParameter(name)
|
||||
.addParameter(version)
|
||||
.executeAndFetchFirst(WorkflowDef.class)));
|
||||
}
|
||||
|
||||
public void removeWorkflowDef(String name, Integer version) {
|
||||
final String DELETE_WORKFLOW_QUERY =
|
||||
"DELETE from meta_workflow_def WHERE name = ? AND version = ?";
|
||||
|
||||
withTransaction(
|
||||
tx -> {
|
||||
// remove specified workflow
|
||||
execute(
|
||||
tx,
|
||||
DELETE_WORKFLOW_QUERY,
|
||||
q -> {
|
||||
if (!q.addParameter(name).addParameter(version).executeDelete()) {
|
||||
throw new NotFoundException(
|
||||
String.format(
|
||||
"No such workflow definition: %s version: %d",
|
||||
name, version));
|
||||
}
|
||||
});
|
||||
// reset latest version based on remaining rows for this workflow
|
||||
Optional<Integer> maxVersion = getLatestVersion(tx, name);
|
||||
maxVersion.ifPresent(newVersion -> updateLatestVersion(tx, name, newVersion));
|
||||
});
|
||||
}
|
||||
|
||||
public List<WorkflowDef> getAllWorkflowDefs() {
|
||||
final String GET_ALL_WORKFLOW_DEF_QUERY =
|
||||
"SELECT json_data FROM meta_workflow_def ORDER BY name, version";
|
||||
|
||||
return queryWithTransaction(
|
||||
GET_ALL_WORKFLOW_DEF_QUERY, q -> q.executeAndFetch(WorkflowDef.class));
|
||||
}
|
||||
|
||||
public List<WorkflowDef> getAllWorkflowDefsLatestVersions() {
|
||||
final String GET_ALL_WORKFLOW_DEF_LATEST_VERSIONS_QUERY =
|
||||
"SELECT json_data FROM meta_workflow_def wd WHERE wd.version = (SELECT MAX(version) FROM meta_workflow_def wd2 WHERE wd2.name = wd.name)";
|
||||
return queryWithTransaction(
|
||||
GET_ALL_WORKFLOW_DEF_LATEST_VERSIONS_QUERY,
|
||||
q -> q.executeAndFetch(WorkflowDef.class));
|
||||
}
|
||||
|
||||
public List<String> findAll() {
|
||||
final String FIND_ALL_WORKFLOW_DEF_QUERY = "SELECT DISTINCT name FROM meta_workflow_def";
|
||||
return queryWithTransaction(
|
||||
FIND_ALL_WORKFLOW_DEF_QUERY, q -> q.executeAndFetch(String.class));
|
||||
}
|
||||
|
||||
public List<WorkflowDef> getAllLatest() {
|
||||
final String GET_ALL_LATEST_WORKFLOW_DEF_QUERY =
|
||||
"SELECT json_data FROM meta_workflow_def WHERE version = " + "latest_version";
|
||||
|
||||
return queryWithTransaction(
|
||||
GET_ALL_LATEST_WORKFLOW_DEF_QUERY, q -> q.executeAndFetch(WorkflowDef.class));
|
||||
}
|
||||
|
||||
public List<WorkflowDef> getAllVersions(String name) {
|
||||
final String GET_ALL_VERSIONS_WORKFLOW_DEF_QUERY =
|
||||
"SELECT json_data FROM meta_workflow_def WHERE name = ? " + "ORDER BY version";
|
||||
|
||||
return queryWithTransaction(
|
||||
GET_ALL_VERSIONS_WORKFLOW_DEF_QUERY,
|
||||
q -> q.addParameter(name).executeAndFetch(WorkflowDef.class));
|
||||
}
|
||||
|
||||
private Boolean workflowExists(Connection connection, WorkflowDef def) {
|
||||
final String CHECK_WORKFLOW_DEF_EXISTS_QUERY =
|
||||
"SELECT COUNT(*) FROM meta_workflow_def WHERE name = ? AND " + "version = ?";
|
||||
|
||||
return query(
|
||||
connection,
|
||||
CHECK_WORKFLOW_DEF_EXISTS_QUERY,
|
||||
q -> q.addParameter(def.getName()).addParameter(def.getVersion()).exists());
|
||||
}
|
||||
|
||||
private void insertOrUpdateWorkflowDef(Connection tx, WorkflowDef def) {
|
||||
final String INSERT_WORKFLOW_DEF_QUERY =
|
||||
"INSERT INTO meta_workflow_def (name, version, json_data) VALUES (?," + " ?, ?)";
|
||||
|
||||
Optional<Integer> version = getLatestVersion(tx, def.getName());
|
||||
if (!workflowExists(tx, def)) {
|
||||
execute(
|
||||
tx,
|
||||
INSERT_WORKFLOW_DEF_QUERY,
|
||||
q ->
|
||||
q.addParameter(def.getName())
|
||||
.addParameter(def.getVersion())
|
||||
.addJsonParameter(def)
|
||||
.executeUpdate());
|
||||
} else {
|
||||
// @formatter:off
|
||||
final String UPDATE_WORKFLOW_DEF_QUERY =
|
||||
"UPDATE meta_workflow_def "
|
||||
+ "SET json_data = ?, modified_on = CURRENT_TIMESTAMP "
|
||||
+ "WHERE name = ? AND version = ?";
|
||||
// @formatter:on
|
||||
|
||||
execute(
|
||||
tx,
|
||||
UPDATE_WORKFLOW_DEF_QUERY,
|
||||
q ->
|
||||
q.addJsonParameter(def)
|
||||
.addParameter(def.getName())
|
||||
.addParameter(def.getVersion())
|
||||
.executeUpdate());
|
||||
}
|
||||
int maxVersion = def.getVersion();
|
||||
if (version.isPresent() && version.get() > def.getVersion()) {
|
||||
maxVersion = version.get();
|
||||
}
|
||||
|
||||
updateLatestVersion(tx, def.getName(), maxVersion);
|
||||
}
|
||||
|
||||
private Optional<Integer> getLatestVersion(Connection tx, String name) {
|
||||
final String GET_LATEST_WORKFLOW_DEF_VERSION =
|
||||
"SELECT max(version) AS version FROM meta_workflow_def WHERE " + "name = ?";
|
||||
|
||||
Integer val =
|
||||
query(
|
||||
tx,
|
||||
GET_LATEST_WORKFLOW_DEF_VERSION,
|
||||
q -> {
|
||||
q.addParameter(name);
|
||||
return q.executeAndFetch(
|
||||
rs -> {
|
||||
if (!rs.next()) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return rs.getInt(1);
|
||||
});
|
||||
});
|
||||
|
||||
return Optional.ofNullable(val);
|
||||
}
|
||||
|
||||
private void updateLatestVersion(Connection tx, String name, int version) {
|
||||
final String UPDATE_WORKFLOW_DEF_LATEST_VERSION_QUERY =
|
||||
"UPDATE meta_workflow_def SET latest_version = ? " + "WHERE name = ?";
|
||||
|
||||
execute(
|
||||
tx,
|
||||
UPDATE_WORKFLOW_DEF_LATEST_VERSION_QUERY,
|
||||
q -> q.addParameter(version).addParameter(name).executeUpdate());
|
||||
}
|
||||
|
||||
private void validate(WorkflowDef def) {
|
||||
Preconditions.checkNotNull(def, "WorkflowDef object cannot be null");
|
||||
Preconditions.checkNotNull(def.getName(), "WorkflowDef name cannot be null");
|
||||
}
|
||||
}
|
||||
+26
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.sql.SQLException;
|
||||
|
||||
/**
|
||||
* Functional interface for {@link Query} executions with no expected result.
|
||||
*
|
||||
* @author mustafa
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface ExecuteFunction {
|
||||
|
||||
void apply(Query query) throws SQLException;
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.util.concurrent.Executors;
|
||||
import java.util.concurrent.ThreadFactory;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
public class ExecutorsUtil {
|
||||
|
||||
private ExecutorsUtil() {}
|
||||
|
||||
public static ThreadFactory newNamedThreadFactory(final String threadNamePrefix) {
|
||||
return new ThreadFactory() {
|
||||
|
||||
private final AtomicInteger counter = new AtomicInteger();
|
||||
|
||||
@SuppressWarnings("NullableProblems")
|
||||
@Override
|
||||
public Thread newThread(Runnable r) {
|
||||
Thread thread = Executors.defaultThreadFactory().newThread(r);
|
||||
thread.setName(threadNamePrefix + counter.getAndIncrement());
|
||||
return thread;
|
||||
}
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,33 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.util.function.Supplier;
|
||||
|
||||
/** Functional class to support the lazy execution of a String result. */
|
||||
public class LazyToString {
|
||||
|
||||
private final Supplier<String> supplier;
|
||||
|
||||
/**
|
||||
* @param supplier Supplier to execute when {@link #toString()} is called.
|
||||
*/
|
||||
public LazyToString(Supplier<String> supplier) {
|
||||
this.supplier = supplier;
|
||||
}
|
||||
|
||||
@Override
|
||||
public String toString() {
|
||||
return supplier.get();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,648 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.io.IOException;
|
||||
import java.sql.*;
|
||||
import java.sql.Date;
|
||||
import java.util.*;
|
||||
import java.util.concurrent.atomic.AtomicInteger;
|
||||
|
||||
import org.apache.commons.lang3.math.NumberUtils;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
|
||||
import com.netflix.conductor.core.exception.NonTransientException;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
/**
|
||||
* Represents a {@link PreparedStatement} that is wrapped with convenience methods and utilities.
|
||||
*
|
||||
* <p>This class simulates a parameter building pattern and all {@literal addParameter(*)} methods
|
||||
* must be called in the proper order of their expected binding sequence.
|
||||
*
|
||||
* @author mustafa
|
||||
*/
|
||||
public class Query implements AutoCloseable {
|
||||
|
||||
private final Logger logger = LoggerFactory.getLogger(getClass());
|
||||
|
||||
/** The {@link ObjectMapper} instance to use for serializing/deserializing JSON. */
|
||||
protected final ObjectMapper objectMapper;
|
||||
|
||||
/** The initial supplied query String that was used to prepare {@link #statement}. */
|
||||
private final String rawQuery;
|
||||
|
||||
/**
|
||||
* Parameter index for the {@code ResultSet#set*(*)} methods, gets incremented every time a
|
||||
* parameter is added to the {@code PreparedStatement} {@link #statement}.
|
||||
*/
|
||||
private final AtomicInteger index = new AtomicInteger(1);
|
||||
|
||||
/** The {@link PreparedStatement} that will be managed and executed by this class. */
|
||||
private final PreparedStatement statement;
|
||||
|
||||
private final Connection connection;
|
||||
|
||||
public Query(ObjectMapper objectMapper, Connection connection, String query) {
|
||||
this.rawQuery = query;
|
||||
this.objectMapper = objectMapper;
|
||||
this.connection = connection;
|
||||
|
||||
try {
|
||||
this.statement = connection.prepareStatement(query);
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(
|
||||
"Cannot prepare statement for query: " + ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a String with {@literal count} number of '?' placeholders for {@link
|
||||
* PreparedStatement} queries.
|
||||
*
|
||||
* @param count The number of '?' chars to generate.
|
||||
* @return a comma delimited string of {@literal count} '?' binding placeholders.
|
||||
*/
|
||||
public static String generateInBindings(int count) {
|
||||
String[] questions = new String[count];
|
||||
for (int i = 0; i < count; i++) {
|
||||
questions[i] = "?";
|
||||
}
|
||||
|
||||
return String.join(", ", questions);
|
||||
}
|
||||
|
||||
public Query addParameter(final String value) {
|
||||
return addParameterInternal((ps, idx) -> ps.setString(idx, value));
|
||||
}
|
||||
|
||||
public Query addParameter(final List<String> value) throws SQLException {
|
||||
String[] valueStringArray = value.toArray(new String[0]);
|
||||
Array valueArray = this.connection.createArrayOf("VARCHAR", valueStringArray);
|
||||
return addParameterInternal((ps, idx) -> ps.setArray(idx, valueArray));
|
||||
}
|
||||
|
||||
public Query addParameter(final int value) {
|
||||
return addParameterInternal((ps, idx) -> ps.setInt(idx, value));
|
||||
}
|
||||
|
||||
public Query addParameter(final boolean value) {
|
||||
return addParameterInternal(((ps, idx) -> ps.setBoolean(idx, value)));
|
||||
}
|
||||
|
||||
public Query addParameter(final long value) {
|
||||
return addParameterInternal((ps, idx) -> ps.setLong(idx, value));
|
||||
}
|
||||
|
||||
public Query addParameter(final double value) {
|
||||
return addParameterInternal((ps, idx) -> ps.setDouble(idx, value));
|
||||
}
|
||||
|
||||
public Query addParameter(Date date) {
|
||||
return addParameterInternal((ps, idx) -> ps.setDate(idx, date));
|
||||
}
|
||||
|
||||
public Query addParameter(Timestamp timestamp) {
|
||||
return addParameterInternal((ps, idx) -> ps.setTimestamp(idx, timestamp));
|
||||
}
|
||||
|
||||
/**
|
||||
* Serializes {@literal value} to a JSON string for persistence.
|
||||
*
|
||||
* @param value The value to serialize.
|
||||
* @return {@literal this}
|
||||
*/
|
||||
public Query addJsonParameter(Object value) {
|
||||
return addParameter(toJson(value));
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind the given {@link java.util.Date} to the PreparedStatement as a {@link Date}.
|
||||
*
|
||||
* @param date The {@literal java.util.Date} to bind.
|
||||
* @return {@literal this}
|
||||
*/
|
||||
public Query addDateParameter(java.util.Date date) {
|
||||
return addParameter(new Date(date.getTime()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind the given {@link java.util.Date} to the PreparedStatement as a {@link Timestamp}.
|
||||
*
|
||||
* @param date The {@literal java.util.Date} to bind.
|
||||
* @return {@literal this}
|
||||
*/
|
||||
public Query addTimestampParameter(java.util.Date date) {
|
||||
return addParameter(new Timestamp(date.getTime()));
|
||||
}
|
||||
|
||||
/**
|
||||
* Bind the given epoch millis to the PreparedStatement as a {@link Timestamp}.
|
||||
*
|
||||
* @param epochMillis The epoch ms to create a new {@literal Timestamp} from.
|
||||
* @return {@literal this}
|
||||
*/
|
||||
public Query addTimestampParameter(long epochMillis) {
|
||||
return addParameter(new Timestamp(epochMillis));
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a collection of primitive values at once, in the order of the collection.
|
||||
*
|
||||
* @param values The values to bind to the prepared statement.
|
||||
* @return {@literal this}
|
||||
* @throws IllegalArgumentException If a non-primitive/unsupported type is encountered in the
|
||||
* collection.
|
||||
* @see #addParameters(Object...)
|
||||
*/
|
||||
public Query addParameters(Collection values) {
|
||||
return addParameters(values.toArray());
|
||||
}
|
||||
|
||||
/**
|
||||
* Add many primitive values at once.
|
||||
*
|
||||
* @param values The values to bind to the prepared statement.
|
||||
* @return {@literal this}
|
||||
* @throws IllegalArgumentException If a non-primitive/unsupported type is encountered.
|
||||
*/
|
||||
public Query addParameters(Object... values) {
|
||||
for (Object v : values) {
|
||||
if (v instanceof String) {
|
||||
addParameter((String) v);
|
||||
} else if (v instanceof Integer) {
|
||||
addParameter((Integer) v);
|
||||
} else if (v instanceof Long) {
|
||||
addParameter((Long) v);
|
||||
} else if (v instanceof Double) {
|
||||
addParameter((Double) v);
|
||||
} else if (v instanceof Boolean) {
|
||||
addParameter((Boolean) v);
|
||||
} else if (v instanceof Date) {
|
||||
addParameter((Date) v);
|
||||
} else if (v instanceof Timestamp) {
|
||||
addParameter((Timestamp) v);
|
||||
} else {
|
||||
throw new IllegalArgumentException(
|
||||
"Type "
|
||||
+ v.getClass().getName()
|
||||
+ " is not supported by automatic property assignment");
|
||||
}
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Utility method for evaluating the prepared statement as a query to check the existence of a
|
||||
* record using a numeric count or boolean return value.
|
||||
*
|
||||
* <p>The {@link #rawQuery} provided must result in a {@link Number} or {@link Boolean} result.
|
||||
*
|
||||
* @return {@literal true} If a count query returned more than 0 or an exists query returns
|
||||
* {@literal true}.
|
||||
* @throws NonTransientException If an unexpected return type cannot be evaluated to a {@code
|
||||
* Boolean} result.
|
||||
*/
|
||||
public boolean exists() {
|
||||
Object val = executeScalar();
|
||||
if (null == val) {
|
||||
return false;
|
||||
}
|
||||
|
||||
if (val instanceof Number) {
|
||||
return convertLong(val) > 0;
|
||||
}
|
||||
|
||||
if (val instanceof Boolean) {
|
||||
return (Boolean) val;
|
||||
}
|
||||
|
||||
if (val instanceof String) {
|
||||
return convertBoolean(val);
|
||||
}
|
||||
|
||||
throw new NonTransientException(
|
||||
"Expected a Numeric or Boolean scalar return value from the query, received "
|
||||
+ val.getClass().getName());
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience method for executing delete statements.
|
||||
*
|
||||
* @return {@literal true} if the statement affected 1 or more rows.
|
||||
* @see #executeUpdate()
|
||||
*/
|
||||
public boolean executeDelete() {
|
||||
int count = executeUpdate();
|
||||
if (count > 1) {
|
||||
logger.trace("Removed {} row(s) for query {}", count, rawQuery);
|
||||
}
|
||||
|
||||
return count > 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* Convenience method for executing statements that return a single numeric value, typically
|
||||
* {@literal SELECT COUNT...} style queries.
|
||||
*
|
||||
* @return The result of the query as a {@literal long}.
|
||||
*/
|
||||
public long executeCount() {
|
||||
return executeScalar(Long.class);
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The result of {@link PreparedStatement#executeUpdate()}
|
||||
*/
|
||||
public int executeUpdate() {
|
||||
try {
|
||||
|
||||
Long start = null;
|
||||
if (logger.isTraceEnabled()) {
|
||||
start = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
final int val = this.statement.executeUpdate();
|
||||
|
||||
if (null != start && logger.isTraceEnabled()) {
|
||||
long end = System.currentTimeMillis();
|
||||
logger.trace("[{}ms] {}: {}", (end - start), val, rawQuery);
|
||||
}
|
||||
|
||||
return val;
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a query from the PreparedStatement and return the ResultSet.
|
||||
*
|
||||
* <p><em>NOTE:</em> The returned ResultSet must be closed/managed by the calling methods.
|
||||
*
|
||||
* @return {@link PreparedStatement#executeQuery()}
|
||||
* @throws NonTransientException If any SQL errors occur.
|
||||
*/
|
||||
public ResultSet executeQuery() {
|
||||
Long start = null;
|
||||
if (logger.isTraceEnabled()) {
|
||||
start = System.currentTimeMillis();
|
||||
}
|
||||
|
||||
try {
|
||||
return this.statement.executeQuery();
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
} finally {
|
||||
if (null != start && logger.isTraceEnabled()) {
|
||||
long end = System.currentTimeMillis();
|
||||
logger.trace("[{}ms] {}", (end - start), rawQuery);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* @return The single result of the query as an Object.
|
||||
*/
|
||||
public Object executeScalar() {
|
||||
try (ResultSet rs = executeQuery()) {
|
||||
if (!rs.next()) {
|
||||
return null;
|
||||
}
|
||||
return rs.getObject(1);
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the PreparedStatement and return a single 'primitive' value from the ResultSet.
|
||||
*
|
||||
* @param returnType The type to return.
|
||||
* @param <V> The type parameter to return a List of.
|
||||
* @return A single result from the execution of the statement, as a type of {@literal
|
||||
* returnType}.
|
||||
* @throws NonTransientException {@literal returnType} is unsupported, cannot be cast to from
|
||||
* the result, or any SQL errors occur.
|
||||
*/
|
||||
public <V> V executeScalar(Class<V> returnType) {
|
||||
try (ResultSet rs = executeQuery()) {
|
||||
if (!rs.next()) {
|
||||
Object value = null;
|
||||
if (Integer.class == returnType) {
|
||||
value = 0;
|
||||
} else if (Long.class == returnType) {
|
||||
value = 0L;
|
||||
} else if (Boolean.class == returnType) {
|
||||
value = false;
|
||||
}
|
||||
return returnType.cast(value);
|
||||
} else {
|
||||
return getScalarFromResultSet(rs, returnType);
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the PreparedStatement and return a List of 'primitive' values from the ResultSet.
|
||||
*
|
||||
* @param returnType The type Class return a List of.
|
||||
* @param <V> The type parameter to return a List of.
|
||||
* @return A {@code List<returnType>}.
|
||||
* @throws NonTransientException {@literal returnType} is unsupported, cannot be cast to from
|
||||
* the result, or any SQL errors occur.
|
||||
*/
|
||||
public <V> List<V> executeScalarList(Class<V> returnType) {
|
||||
try (ResultSet rs = executeQuery()) {
|
||||
List<V> values = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
values.add(getScalarFromResultSet(rs, returnType));
|
||||
}
|
||||
return values;
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the statement and return only the first record from the result set.
|
||||
*
|
||||
* @param returnType The Class to return.
|
||||
* @param <V> The type parameter.
|
||||
* @return An instance of {@literal <V>} from the result set.
|
||||
*/
|
||||
public <V> V executeAndFetchFirst(Class<V> returnType) {
|
||||
Object o = executeScalar();
|
||||
if (null == o) {
|
||||
return null;
|
||||
}
|
||||
return convert(o, returnType);
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the PreparedStatement and return a List of {@literal returnType} values from the
|
||||
* ResultSet.
|
||||
*
|
||||
* @param returnType The type Class return a List of.
|
||||
* @param <V> The type parameter to return a List of.
|
||||
* @return A {@code List<returnType>}.
|
||||
* @throws NonTransientException {@literal returnType} is unsupported, cannot be cast to from
|
||||
* the result, or any SQL errors occur.
|
||||
*/
|
||||
public <V> List<V> executeAndFetch(Class<V> returnType) {
|
||||
try (ResultSet rs = executeQuery()) {
|
||||
List<V> list = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
list.add(convert(rs.getObject(1), returnType));
|
||||
}
|
||||
return list;
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the PreparedStatement and return a List of {@literal Map} values from the ResultSet.
|
||||
*
|
||||
* @return A {@code List<Map>}.
|
||||
* @throws SQLException if any SQL errors occur.
|
||||
* @throws NonTransientException if any SQL errors occur.
|
||||
*/
|
||||
public List<Map<String, Object>> executeAndFetchMap() {
|
||||
try (ResultSet rs = executeQuery()) {
|
||||
List<Map<String, Object>> result = new ArrayList<>();
|
||||
ResultSetMetaData metadata = rs.getMetaData();
|
||||
int columnCount = metadata.getColumnCount();
|
||||
while (rs.next()) {
|
||||
HashMap<String, Object> row = new HashMap<>();
|
||||
for (int i = 1; i <= columnCount; i++) {
|
||||
row.put(metadata.getColumnLabel(i), rs.getObject(i));
|
||||
}
|
||||
result.add(row);
|
||||
}
|
||||
return result;
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute the query and pass the {@link ResultSet} to the given handler.
|
||||
*
|
||||
* @param handler The {@link ResultSetHandler} to execute.
|
||||
* @param <V> The return type of this method.
|
||||
* @return The results of {@link ResultSetHandler#apply(ResultSet)}.
|
||||
*/
|
||||
public <V> V executeAndFetch(ResultSetHandler<V> handler) {
|
||||
try (ResultSet rs = executeQuery()) {
|
||||
return handler.apply(rs);
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public void close() {
|
||||
try {
|
||||
if (null != statement && !statement.isClosed()) {
|
||||
statement.close();
|
||||
}
|
||||
} catch (SQLException ex) {
|
||||
logger.warn("Error closing prepared statement: {}", ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
protected final Query addParameterInternal(InternalParameterSetter setter) {
|
||||
int index = getAndIncrementIndex();
|
||||
try {
|
||||
setter.apply(this.statement, index);
|
||||
return this;
|
||||
} catch (SQLException ex) {
|
||||
throw new NonTransientException("Could not apply bind parameter at index " + index, ex);
|
||||
}
|
||||
}
|
||||
|
||||
protected <V> V getScalarFromResultSet(ResultSet rs, Class<V> returnType) throws SQLException {
|
||||
Object value = null;
|
||||
|
||||
if (Integer.class == returnType) {
|
||||
value = rs.getInt(1);
|
||||
} else if (Long.class == returnType) {
|
||||
value = rs.getLong(1);
|
||||
} else if (String.class == returnType) {
|
||||
value = rs.getString(1);
|
||||
} else if (Boolean.class == returnType) {
|
||||
value = rs.getBoolean(1);
|
||||
} else if (Double.class == returnType) {
|
||||
value = rs.getDouble(1);
|
||||
} else if (Date.class == returnType) {
|
||||
value = rs.getDate(1);
|
||||
} else if (Timestamp.class == returnType) {
|
||||
value = rs.getTimestamp(1);
|
||||
} else {
|
||||
value = rs.getObject(1);
|
||||
}
|
||||
|
||||
if (null == value) {
|
||||
throw new NullPointerException(
|
||||
"Cannot get value from ResultSet of type " + returnType.getName());
|
||||
}
|
||||
|
||||
return returnType.cast(value);
|
||||
}
|
||||
|
||||
protected <V> V convert(Object value, Class<V> returnType) {
|
||||
if (Boolean.class == returnType) {
|
||||
return returnType.cast(convertBoolean(value));
|
||||
} else if (Integer.class == returnType) {
|
||||
return returnType.cast(convertInt(value));
|
||||
} else if (Long.class == returnType) {
|
||||
return returnType.cast(convertLong(value));
|
||||
} else if (Double.class == returnType) {
|
||||
return returnType.cast(convertDouble(value));
|
||||
} else if (String.class == returnType) {
|
||||
return returnType.cast(convertString(value));
|
||||
} else if (value instanceof String) {
|
||||
return fromJson((String) value, returnType);
|
||||
}
|
||||
|
||||
final String vName = value.getClass().getName();
|
||||
final String rName = returnType.getName();
|
||||
throw new NonTransientException("Cannot convert type " + vName + " to " + rName);
|
||||
}
|
||||
|
||||
protected Integer convertInt(Object value) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof Integer) {
|
||||
return (Integer) value;
|
||||
}
|
||||
|
||||
if (value instanceof Number) {
|
||||
return ((Number) value).intValue();
|
||||
}
|
||||
|
||||
return NumberUtils.toInt(value.toString());
|
||||
}
|
||||
|
||||
protected Double convertDouble(Object value) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof Double) {
|
||||
return (Double) value;
|
||||
}
|
||||
|
||||
if (value instanceof Number) {
|
||||
return ((Number) value).doubleValue();
|
||||
}
|
||||
|
||||
return NumberUtils.toDouble(value.toString());
|
||||
}
|
||||
|
||||
protected Long convertLong(Object value) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof Long) {
|
||||
return (Long) value;
|
||||
}
|
||||
|
||||
if (value instanceof Number) {
|
||||
return ((Number) value).longValue();
|
||||
}
|
||||
return NumberUtils.toLong(value.toString());
|
||||
}
|
||||
|
||||
protected String convertString(Object value) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof String) {
|
||||
return (String) value;
|
||||
}
|
||||
|
||||
return value.toString().trim();
|
||||
}
|
||||
|
||||
protected Boolean convertBoolean(Object value) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
if (value instanceof Boolean) {
|
||||
return (Boolean) value;
|
||||
}
|
||||
|
||||
if (value instanceof Number) {
|
||||
return ((Number) value).intValue() != 0;
|
||||
}
|
||||
|
||||
String text = value.toString().trim();
|
||||
return "Y".equalsIgnoreCase(text)
|
||||
|| "YES".equalsIgnoreCase(text)
|
||||
|| "TRUE".equalsIgnoreCase(text)
|
||||
|| "T".equalsIgnoreCase(text)
|
||||
|| "1".equalsIgnoreCase(text);
|
||||
}
|
||||
|
||||
protected String toJson(Object value) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return objectMapper.writeValueAsString(value);
|
||||
} catch (JsonProcessingException ex) {
|
||||
throw new NonTransientException(ex.getMessage(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
protected <V> V fromJson(String value, Class<V> returnType) {
|
||||
if (null == value) {
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
return objectMapper.readValue(value, returnType);
|
||||
} catch (IOException ex) {
|
||||
throw new NonTransientException(
|
||||
"Could not convert JSON '" + value + "' to " + returnType.getName(), ex);
|
||||
}
|
||||
}
|
||||
|
||||
protected final int getIndex() {
|
||||
return index.get();
|
||||
}
|
||||
|
||||
protected final int getAndIncrementIndex() {
|
||||
return index.getAndIncrement();
|
||||
}
|
||||
|
||||
@FunctionalInterface
|
||||
private interface InternalParameterSetter {
|
||||
|
||||
void apply(PreparedStatement ps, int idx) throws SQLException;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,26 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.sql.SQLException;
|
||||
|
||||
/**
|
||||
* Functional interface for {@link Query} executions that return results.
|
||||
*
|
||||
* @author mustafa
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface QueryFunction<R> {
|
||||
|
||||
R apply(Query query) throws SQLException;
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
/*
|
||||
* Copyright 2024 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
public class QueueStats {
|
||||
private Integer depth;
|
||||
|
||||
private long nextDelivery;
|
||||
|
||||
public void setDepth(Integer depth) {
|
||||
this.depth = depth;
|
||||
}
|
||||
|
||||
public Integer getDepth() {
|
||||
return depth;
|
||||
}
|
||||
|
||||
public void setNextDelivery(long nextDelivery) {
|
||||
this.nextDelivery = nextDelivery;
|
||||
}
|
||||
|
||||
public long getNextDelivery() {
|
||||
return nextDelivery;
|
||||
}
|
||||
|
||||
public String toString() {
|
||||
return "{nextDelivery: " + nextDelivery + " depth: " + depth + "}";
|
||||
}
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
|
||||
/**
|
||||
* Functional interface for {@link Query#executeAndFetch(ResultSetHandler)}.
|
||||
*
|
||||
* @author mustafa
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface ResultSetHandler<R> {
|
||||
|
||||
R apply(ResultSet resultSet) throws SQLException;
|
||||
}
|
||||
+297
@@ -0,0 +1,297 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.time.Instant;
|
||||
import java.time.ZoneOffset;
|
||||
import java.time.ZonedDateTime;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.util.*;
|
||||
import java.util.regex.Matcher;
|
||||
import java.util.regex.Pattern;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.StringUtils;
|
||||
|
||||
import com.netflix.conductor.common.metadata.workflow.WorkflowClassifier;
|
||||
import com.netflix.conductor.sqlite.config.SqliteProperties;
|
||||
|
||||
public class SqliteIndexQueryBuilder {
|
||||
|
||||
private final String table;
|
||||
private final String freeText;
|
||||
private final int start;
|
||||
private final int count;
|
||||
private final List<String> sort;
|
||||
private final List<Condition> conditions = new ArrayList<>();
|
||||
|
||||
private boolean allowJsonQueries;
|
||||
private boolean allowFullTextQueries;
|
||||
|
||||
private static final String[] VALID_FIELDS = {
|
||||
"workflow_id",
|
||||
"correlation_id",
|
||||
"workflow_type",
|
||||
"start_time",
|
||||
"status",
|
||||
"task_id",
|
||||
"task_type",
|
||||
"task_def_name",
|
||||
"update_time",
|
||||
"json_data",
|
||||
"parent_workflow_id",
|
||||
"classifier"
|
||||
};
|
||||
|
||||
private static final String[] VALID_SORT_ORDER = {"ASC", "DESC"};
|
||||
|
||||
private static class Condition {
|
||||
private String attribute;
|
||||
private String operator;
|
||||
private List<String> values;
|
||||
private final String CONDITION_REGEX = "([a-zA-Z]+)\\s?(=|>|<|IN)\\s?(.*)";
|
||||
|
||||
public Condition() {}
|
||||
|
||||
public Condition(String query) {
|
||||
Pattern conditionRegex = Pattern.compile(CONDITION_REGEX);
|
||||
Matcher conditionMatcher = conditionRegex.matcher(query);
|
||||
if (conditionMatcher.find()) {
|
||||
String[] valueArr = conditionMatcher.group(3).replaceAll("[\"'()]", "").split(",");
|
||||
ArrayList<String> values = new ArrayList<>(Arrays.asList(valueArr));
|
||||
this.attribute = camelToSnake(conditionMatcher.group(1));
|
||||
this.values = values;
|
||||
this.operator = getOperator(conditionMatcher.group(2));
|
||||
if (this.attribute.endsWith("_time")) {
|
||||
values.set(0, millisToUtc(values.get(0)));
|
||||
}
|
||||
} else {
|
||||
throw new IllegalArgumentException("Incorrectly formatted query string: " + query);
|
||||
}
|
||||
}
|
||||
|
||||
public String getQueryFragment() {
|
||||
if (operator.equals("IN")) {
|
||||
// Create proper IN clause for SQLite
|
||||
String inClause =
|
||||
attribute
|
||||
+ " IN ("
|
||||
+ String.join(",", Collections.nCopies(values.size(), "?"))
|
||||
+ ")";
|
||||
if (classifierMatchesUntagged()) {
|
||||
return "(" + inClause + " OR " + attribute + " IS NULL)";
|
||||
}
|
||||
return inClause;
|
||||
} else if (operator.equals("MATCH")) {
|
||||
// SQLite FTS5 full-text search
|
||||
return "json_data MATCH ?";
|
||||
} else if (operator.equals("JSON_CONTAINS")) {
|
||||
// SQLite JSON1 extension query
|
||||
return "json_extract(json_data, ?) IS NOT NULL";
|
||||
} else if (operator.equals("LIKE")) {
|
||||
return "lower(" + attribute + ") LIKE ?";
|
||||
} else {
|
||||
if (attribute.endsWith("_time")) {
|
||||
return attribute + " " + operator + " datetime(?)";
|
||||
} else if (operator.equals("=")
|
||||
&& values.size() == 1
|
||||
&& values.get(0).contains("*")) {
|
||||
return "lower(" + attribute + ") LIKE lower(?)";
|
||||
} else if (operator.equals("=") && classifierMatchesUntagged()) {
|
||||
return "(" + attribute + " = ? OR " + attribute + " IS NULL)";
|
||||
} else {
|
||||
return attribute + " " + operator + " ?";
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Rows indexed before the classifier column existed have a NULL classifier but are
|
||||
* semantically untagged, i.e. plain workflows. When a filter asks for the untagged token
|
||||
* ({@link WorkflowClassifier#WORKFLOW}), widen the predicate to also match those legacy
|
||||
* NULL rows.
|
||||
*/
|
||||
private boolean classifierMatchesUntagged() {
|
||||
return "classifier".equals(attribute)
|
||||
&& values != null
|
||||
&& values.stream().anyMatch(WorkflowClassifier.WORKFLOW::equalsIgnoreCase);
|
||||
}
|
||||
|
||||
private String getOperator(String op) {
|
||||
if (op.equals("IN") && values.size() == 1) {
|
||||
return "=";
|
||||
}
|
||||
return op;
|
||||
}
|
||||
|
||||
public void addParameter(Query q) throws SQLException {
|
||||
if (values.size() > 1) {
|
||||
// For IN clause, add each value separately
|
||||
for (String value : values) {
|
||||
q.addParameter(value);
|
||||
}
|
||||
} else {
|
||||
String val = values.get(0);
|
||||
if (val.contains("*")) {
|
||||
val = val.replace("*", "%");
|
||||
}
|
||||
q.addParameter(val);
|
||||
}
|
||||
}
|
||||
|
||||
private String millisToUtc(String millis) {
|
||||
Long startTimeMilli = Long.parseLong(millis);
|
||||
ZonedDateTime startDate =
|
||||
ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTimeMilli), ZoneOffset.UTC);
|
||||
return DateTimeFormatter.ISO_DATE_TIME.format(startDate);
|
||||
}
|
||||
|
||||
private boolean isValid() {
|
||||
return Arrays.asList(VALID_FIELDS).contains(attribute);
|
||||
}
|
||||
|
||||
public void setAttribute(String attribute) {
|
||||
this.attribute = attribute;
|
||||
}
|
||||
|
||||
public void setOperator(String operator) {
|
||||
this.operator = operator;
|
||||
}
|
||||
|
||||
public void setValues(List<String> values) {
|
||||
this.values = values;
|
||||
}
|
||||
}
|
||||
|
||||
public SqliteIndexQueryBuilder(
|
||||
String table,
|
||||
String query,
|
||||
String freeText,
|
||||
int start,
|
||||
int count,
|
||||
List<String> sort,
|
||||
SqliteProperties properties) {
|
||||
this.table = table;
|
||||
this.freeText = freeText;
|
||||
this.start = start;
|
||||
this.count = count;
|
||||
this.sort = sort != null ? sort : Collections.emptyList();
|
||||
this.allowFullTextQueries = true;
|
||||
this.allowJsonQueries = true;
|
||||
this.parseQuery(query);
|
||||
this.parseFreeText(freeText);
|
||||
}
|
||||
|
||||
public String getQuery() {
|
||||
return getQuery("json_data");
|
||||
}
|
||||
|
||||
public String getQuery(String selectColumn) {
|
||||
String queryString = "";
|
||||
List<Condition> validConditions =
|
||||
conditions.stream().filter(c -> c.isValid()).collect(Collectors.toList());
|
||||
if (validConditions.size() > 0) {
|
||||
queryString =
|
||||
" WHERE "
|
||||
+ String.join(
|
||||
" AND ",
|
||||
validConditions.stream()
|
||||
.map(c -> c.getQueryFragment())
|
||||
.collect(Collectors.toList()));
|
||||
}
|
||||
return "SELECT "
|
||||
+ selectColumn
|
||||
+ " FROM "
|
||||
+ table
|
||||
+ queryString
|
||||
+ getSort()
|
||||
+ " LIMIT ? OFFSET ?";
|
||||
}
|
||||
|
||||
public String getCountQuery() {
|
||||
String queryString = "";
|
||||
List<Condition> validConditions =
|
||||
conditions.stream().filter(c -> c.isValid()).collect(Collectors.toList());
|
||||
if (validConditions.size() > 0) {
|
||||
queryString =
|
||||
" WHERE "
|
||||
+ String.join(
|
||||
" AND ",
|
||||
validConditions.stream()
|
||||
.map(c -> c.getQueryFragment())
|
||||
.collect(Collectors.toList()));
|
||||
}
|
||||
return "SELECT COUNT(*) FROM " + table + queryString;
|
||||
}
|
||||
|
||||
public void addParameters(Query q) throws SQLException {
|
||||
for (Condition condition : conditions) {
|
||||
if (condition.isValid()) {
|
||||
condition.addParameter(q);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void addPagingParameters(Query q) throws SQLException {
|
||||
q.addParameter(count);
|
||||
q.addParameter(start);
|
||||
}
|
||||
|
||||
private void parseQuery(String query) {
|
||||
if (!StringUtils.isEmpty(query)) {
|
||||
for (String s : query.split(" AND ")) {
|
||||
conditions.add(new Condition(s));
|
||||
}
|
||||
Collections.sort(conditions, Comparator.comparing(Condition::getQueryFragment));
|
||||
}
|
||||
}
|
||||
|
||||
private void parseFreeText(String freeText) {
|
||||
if (!StringUtils.isEmpty(freeText) && !freeText.equals("*")) {
|
||||
Condition cond = new Condition();
|
||||
cond.setAttribute("json_data");
|
||||
cond.setOperator("LIKE");
|
||||
String[] values = {freeText};
|
||||
cond.setValues(
|
||||
Arrays.stream(values)
|
||||
.map(v -> "%" + v.toLowerCase() + "%")
|
||||
.collect(Collectors.toList()));
|
||||
conditions.add(cond);
|
||||
}
|
||||
}
|
||||
|
||||
private String getSort() {
|
||||
ArrayList<String> sortConds = new ArrayList<>();
|
||||
for (String s : sort) {
|
||||
String[] splitCond = s.split(":");
|
||||
if (splitCond.length == 2) {
|
||||
String attribute = camelToSnake(splitCond[0]);
|
||||
String order = splitCond[1].toUpperCase();
|
||||
if (Arrays.asList(VALID_FIELDS).contains(attribute)
|
||||
&& Arrays.asList(VALID_SORT_ORDER).contains(order)) {
|
||||
sortConds.add(attribute + " " + order);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (sortConds.size() > 0) {
|
||||
return " ORDER BY " + String.join(", ", sortConds);
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
private static String camelToSnake(String camel) {
|
||||
return camel.replaceAll("\\B([A-Z])", "_$1").toLowerCase();
|
||||
}
|
||||
}
|
||||
+27
@@ -0,0 +1,27 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
|
||||
/**
|
||||
* Functional interface for operations within a transactional context.
|
||||
*
|
||||
* @author mustafa
|
||||
*/
|
||||
@FunctionalInterface
|
||||
public interface TransactionalFunction<R> {
|
||||
|
||||
R apply(Connection tx) throws SQLException;
|
||||
}
|
||||
+152
@@ -0,0 +1,152 @@
|
||||
/*
|
||||
* Copyright 2026 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package org.conductoross.conductor.sqlite.dao;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.conductoross.conductor.dao.FileMetadataDAO;
|
||||
import org.conductoross.conductor.model.FileModel;
|
||||
import org.conductoross.conductor.model.file.FileUploadStatus;
|
||||
import org.conductoross.conductor.model.file.StorageType;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
public class SqliteFileMetadataDAO extends SqliteBaseDAO implements FileMetadataDAO {
|
||||
|
||||
private static final String INSERT_FILE =
|
||||
"INSERT INTO file_metadata (file_id, file_name, content_type, "
|
||||
+ "storage_type, storage_path, upload_status, workflow_id, task_id, "
|
||||
+ "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
|
||||
|
||||
private static final String SELECT_BY_ID = "SELECT * FROM file_metadata WHERE file_id = ?";
|
||||
|
||||
private static final String UPDATE_STATUS =
|
||||
"UPDATE file_metadata SET upload_status = ?, updated_at = CURRENT_TIMESTAMP "
|
||||
+ "WHERE file_id = ?";
|
||||
|
||||
private static final String UPDATE_UPLOAD_COMPLETE =
|
||||
"UPDATE file_metadata SET upload_status = ?, storage_content_hash = ?, "
|
||||
+ "storage_content_size = ?, updated_at = CURRENT_TIMESTAMP "
|
||||
+ "WHERE file_id = ?";
|
||||
|
||||
private static final String SELECT_BY_WORKFLOW =
|
||||
"SELECT * FROM file_metadata WHERE workflow_id = ?";
|
||||
|
||||
private static final String SELECT_BY_TASK = "SELECT * FROM file_metadata WHERE task_id = ?";
|
||||
|
||||
public SqliteFileMetadataDAO(
|
||||
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void createFileMetadata(FileModel fileModel) {
|
||||
executeWithTransaction(
|
||||
INSERT_FILE,
|
||||
q ->
|
||||
q.addParameter(fileModel.getFileId())
|
||||
.addParameter(fileModel.getFileName())
|
||||
.addParameter(fileModel.getContentType())
|
||||
.addParameter(fileModel.getStorageType().name())
|
||||
.addParameter(fileModel.getStoragePath())
|
||||
.addParameter(fileModel.getUploadStatus().name())
|
||||
.addParameter(fileModel.getWorkflowId())
|
||||
.addParameter(fileModel.getTaskId())
|
||||
.addParameter(Timestamp.from(fileModel.getCreatedAt()))
|
||||
.addParameter(Timestamp.from(fileModel.getUpdatedAt()))
|
||||
.executeUpdate());
|
||||
}
|
||||
|
||||
@Override
|
||||
public FileModel getFileMetadata(String fileId) {
|
||||
return queryWithTransaction(
|
||||
SELECT_BY_ID,
|
||||
q ->
|
||||
q.addParameter(fileId)
|
||||
.executeAndFetch(
|
||||
rs -> {
|
||||
if (!rs.next()) return null;
|
||||
return toFileModel(rs);
|
||||
}));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateUploadStatus(String fileId, FileUploadStatus status) {
|
||||
executeWithTransaction(
|
||||
UPDATE_STATUS,
|
||||
q -> q.addParameter(status.name()).addParameter(fileId).executeUpdate());
|
||||
}
|
||||
|
||||
@Override
|
||||
public void updateUploadComplete(
|
||||
String fileId, FileUploadStatus status, String contentHash, long contentSize) {
|
||||
executeWithTransaction(
|
||||
UPDATE_UPLOAD_COMPLETE,
|
||||
q ->
|
||||
q.addParameter(status.name())
|
||||
.addParameter(contentHash)
|
||||
.addParameter(contentSize)
|
||||
.addParameter(fileId)
|
||||
.executeUpdate());
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<FileModel> getFilesByWorkflowId(String workflowId) {
|
||||
return queryWithTransaction(
|
||||
SELECT_BY_WORKFLOW,
|
||||
q -> q.addParameter(workflowId).executeAndFetch(this::toFileModelList));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<FileModel> getFilesByTaskId(String taskId) {
|
||||
return queryWithTransaction(
|
||||
SELECT_BY_TASK, q -> q.addParameter(taskId).executeAndFetch(this::toFileModelList));
|
||||
}
|
||||
|
||||
private List<FileModel> toFileModelList(ResultSet rs) throws SQLException {
|
||||
List<FileModel> list = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
list.add(toFileModel(rs));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
|
||||
private FileModel toFileModel(ResultSet rs) throws SQLException {
|
||||
FileModel model = new FileModel();
|
||||
model.setFileId(rs.getString("file_id"));
|
||||
model.setFileName(rs.getString("file_name"));
|
||||
model.setContentType(rs.getString("content_type"));
|
||||
model.setStorageContentHash(rs.getString("storage_content_hash"));
|
||||
long scs = rs.getLong("storage_content_size");
|
||||
model.setStorageContentSize(rs.wasNull() ? 0 : scs);
|
||||
model.setStorageType(StorageType.valueOf(rs.getString("storage_type")));
|
||||
model.setStoragePath(rs.getString("storage_path"));
|
||||
model.setUploadStatus(FileUploadStatus.valueOf(rs.getString("upload_status")));
|
||||
model.setWorkflowId(rs.getString("workflow_id"));
|
||||
model.setTaskId(rs.getString("task_id"));
|
||||
Timestamp createdAt = rs.getTimestamp("created_at");
|
||||
if (createdAt != null) model.setCreatedAt(createdAt.toInstant());
|
||||
Timestamp updatedAt = rs.getTimestamp("updated_at");
|
||||
if (updatedAt != null) model.setUpdatedAt(updatedAt.toInstant());
|
||||
return model;
|
||||
}
|
||||
}
|
||||
+205
@@ -0,0 +1,205 @@
|
||||
/*
|
||||
* Copyright 2026 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package org.conductoross.conductor.sqlite.dao;
|
||||
|
||||
import java.sql.ResultSet;
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
import java.util.List;
|
||||
import java.util.Optional;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.conductoross.conductor.dao.SkillMetadataDAO;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
/** SQLite {@link SkillMetadataDAO} — table {@code skill_metadata}. */
|
||||
public class SqliteSkillMetadataDAO extends SqliteBaseDAO implements SkillMetadataDAO {
|
||||
|
||||
private static final String CLEAR_LATEST =
|
||||
"UPDATE skill_metadata SET is_latest = ? WHERE owner_id = ? AND name = ?";
|
||||
|
||||
private static final String UPSERT_LATEST =
|
||||
"INSERT INTO skill_metadata (owner_id, name, version, is_latest, detail_json, created_at, updated_at) "
|
||||
+ "VALUES (?, ?, ?, ?, ?, ?, ?) "
|
||||
+ "ON CONFLICT (owner_id, name, version) DO UPDATE SET "
|
||||
+ "is_latest = excluded.is_latest, detail_json = excluded.detail_json, "
|
||||
+ "updated_at = excluded.updated_at";
|
||||
|
||||
private static final String UPSERT_NO_LATEST =
|
||||
"INSERT INTO skill_metadata (owner_id, name, version, is_latest, detail_json, created_at, updated_at) "
|
||||
+ "VALUES (?, ?, ?, ?, ?, ?, ?) "
|
||||
+ "ON CONFLICT (owner_id, name, version) DO UPDATE SET "
|
||||
+ "detail_json = excluded.detail_json, updated_at = excluded.updated_at";
|
||||
|
||||
private static final String SELECT_DETAIL =
|
||||
"SELECT detail_json FROM skill_metadata WHERE owner_id = ? AND name = ? AND version = ?";
|
||||
|
||||
private static final String SELECT_LATEST_VERSION =
|
||||
"SELECT version FROM skill_metadata WHERE owner_id = ? AND name = ? AND is_latest = ?";
|
||||
|
||||
private static final String SELECT_VERSIONS =
|
||||
"SELECT detail_json FROM skill_metadata WHERE owner_id = ? AND name = ?";
|
||||
|
||||
private static final String SELECT_ALL =
|
||||
"SELECT detail_json FROM skill_metadata WHERE owner_id = ?";
|
||||
|
||||
private static final String SELECT_LATEST_ALL =
|
||||
"SELECT detail_json FROM skill_metadata WHERE owner_id = ? AND is_latest = ?";
|
||||
|
||||
private static final String DELETE_ONE =
|
||||
"DELETE FROM skill_metadata WHERE owner_id = ? AND name = ? AND version = ?";
|
||||
|
||||
// In SQLite, NULL sorts lowest, so DESC places non-null updated_at first (NULLs last).
|
||||
private static final String SELECT_NEWEST_REMAINING =
|
||||
"SELECT version FROM skill_metadata WHERE owner_id = ? AND name = ? ORDER BY updated_at DESC LIMIT 1";
|
||||
|
||||
private static final String SET_LATEST =
|
||||
"UPDATE skill_metadata SET is_latest = ? WHERE owner_id = ? AND name = ? AND version = ?";
|
||||
|
||||
public SqliteSkillMetadataDAO(
|
||||
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void save(
|
||||
String ownerId,
|
||||
String name,
|
||||
String version,
|
||||
boolean makeLatest,
|
||||
String detailJson,
|
||||
Long createdAt,
|
||||
Long updatedAt) {
|
||||
if (makeLatest) {
|
||||
executeWithTransaction(
|
||||
CLEAR_LATEST,
|
||||
q ->
|
||||
q.addParameter(false)
|
||||
.addParameter(ownerId)
|
||||
.addParameter(name)
|
||||
.executeUpdate());
|
||||
executeWithTransaction(
|
||||
UPSERT_LATEST,
|
||||
q ->
|
||||
q.addParameter(ownerId)
|
||||
.addParameter(name)
|
||||
.addParameter(version)
|
||||
.addParameter(true)
|
||||
.addParameter(detailJson)
|
||||
.addParameter(createdAt)
|
||||
.addParameter(updatedAt)
|
||||
.executeUpdate());
|
||||
} else {
|
||||
executeWithTransaction(
|
||||
UPSERT_NO_LATEST,
|
||||
q ->
|
||||
q.addParameter(ownerId)
|
||||
.addParameter(name)
|
||||
.addParameter(version)
|
||||
.addParameter(false)
|
||||
.addParameter(detailJson)
|
||||
.addParameter(createdAt)
|
||||
.addParameter(updatedAt)
|
||||
.executeUpdate());
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> find(String ownerId, String name, String version) {
|
||||
String json =
|
||||
queryWithTransaction(
|
||||
SELECT_DETAIL,
|
||||
q ->
|
||||
q.addParameter(ownerId)
|
||||
.addParameter(name)
|
||||
.addParameter(version)
|
||||
.executeAndFetch(rs -> rs.next() ? rs.getString(1) : null));
|
||||
return Optional.ofNullable(json);
|
||||
}
|
||||
|
||||
@Override
|
||||
public Optional<String> latestVersion(String ownerId, String name) {
|
||||
String version =
|
||||
queryWithTransaction(
|
||||
SELECT_LATEST_VERSION,
|
||||
q ->
|
||||
q.addParameter(ownerId)
|
||||
.addParameter(name)
|
||||
.addParameter(true)
|
||||
.executeAndFetch(rs -> rs.next() ? rs.getString(1) : null));
|
||||
return Optional.ofNullable(version);
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> listVersions(String ownerId, String name) {
|
||||
return queryWithTransaction(
|
||||
SELECT_VERSIONS,
|
||||
q -> q.addParameter(ownerId).addParameter(name).executeAndFetch(this::toJsonList));
|
||||
}
|
||||
|
||||
@Override
|
||||
public List<String> list(String ownerId, boolean allVersions) {
|
||||
if (allVersions) {
|
||||
return queryWithTransaction(
|
||||
SELECT_ALL, q -> q.addParameter(ownerId).executeAndFetch(this::toJsonList));
|
||||
}
|
||||
return queryWithTransaction(
|
||||
SELECT_LATEST_ALL,
|
||||
q -> q.addParameter(ownerId).addParameter(true).executeAndFetch(this::toJsonList));
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(String ownerId, String name, String version) {
|
||||
Optional<String> latest = latestVersion(ownerId, name);
|
||||
executeWithTransaction(
|
||||
DELETE_ONE,
|
||||
q ->
|
||||
q.addParameter(ownerId)
|
||||
.addParameter(name)
|
||||
.addParameter(version)
|
||||
.executeUpdate());
|
||||
if (latest.isPresent() && latest.get().equals(version)) {
|
||||
String newest =
|
||||
queryWithTransaction(
|
||||
SELECT_NEWEST_REMAINING,
|
||||
q ->
|
||||
q.addParameter(ownerId)
|
||||
.addParameter(name)
|
||||
.executeAndFetch(
|
||||
rs -> rs.next() ? rs.getString(1) : null));
|
||||
if (newest != null) {
|
||||
executeWithTransaction(
|
||||
SET_LATEST,
|
||||
q ->
|
||||
q.addParameter(true)
|
||||
.addParameter(ownerId)
|
||||
.addParameter(name)
|
||||
.addParameter(newest)
|
||||
.executeUpdate());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private List<String> toJsonList(ResultSet rs) throws SQLException {
|
||||
List<String> list = new ArrayList<>();
|
||||
while (rs.next()) {
|
||||
list.add(rs.getString(1));
|
||||
}
|
||||
return list;
|
||||
}
|
||||
}
|
||||
+81
@@ -0,0 +1,81 @@
|
||||
/*
|
||||
* Copyright 2026 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package org.conductoross.conductor.sqlite.dao;
|
||||
|
||||
import java.util.Base64;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.conductoross.conductor.dao.SkillPackageDAO;
|
||||
import org.springframework.retry.support.RetryTemplate;
|
||||
|
||||
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
/** SQLite {@link SkillPackageDAO} — table {@code skill_package} (Base64-encoded bytes). */
|
||||
public class SqliteSkillPackageDAO extends SqliteBaseDAO implements SkillPackageDAO {
|
||||
|
||||
private static final String UPSERT =
|
||||
"INSERT INTO skill_package (handle, data, size_bytes, created_at) VALUES (?, ?, ?, ?) "
|
||||
+ "ON CONFLICT (handle) DO UPDATE SET data = excluded.data, size_bytes = excluded.size_bytes";
|
||||
|
||||
private static final String SELECT = "SELECT data FROM skill_package WHERE handle = ?";
|
||||
|
||||
private static final String EXISTS = "SELECT 1 FROM skill_package WHERE handle = ?";
|
||||
|
||||
private static final String DELETE = "DELETE FROM skill_package WHERE handle = ?";
|
||||
|
||||
public SqliteSkillPackageDAO(
|
||||
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
|
||||
super(retryTemplate, objectMapper, dataSource);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void put(String handle, byte[] data) {
|
||||
String encoded = Base64.getEncoder().encodeToString(data);
|
||||
executeWithTransaction(
|
||||
UPSERT,
|
||||
q ->
|
||||
q.addParameter(handle)
|
||||
.addParameter(encoded)
|
||||
.addParameter((long) data.length)
|
||||
.addParameter(System.currentTimeMillis())
|
||||
.executeUpdate());
|
||||
}
|
||||
|
||||
@Override
|
||||
public byte[] get(String handle) {
|
||||
String encoded =
|
||||
queryWithTransaction(
|
||||
SELECT,
|
||||
q ->
|
||||
q.addParameter(handle)
|
||||
.executeAndFetch(rs -> rs.next() ? rs.getString(1) : null));
|
||||
return encoded == null ? null : Base64.getDecoder().decode(encoded);
|
||||
}
|
||||
|
||||
@Override
|
||||
public boolean exists(String handle) {
|
||||
Boolean present =
|
||||
queryWithTransaction(
|
||||
EXISTS,
|
||||
q -> q.addParameter(handle).executeAndFetch(java.sql.ResultSet::next));
|
||||
return Boolean.TRUE.equals(present);
|
||||
}
|
||||
|
||||
@Override
|
||||
public void delete(String handle) {
|
||||
executeWithTransaction(DELETE, q -> q.addParameter(handle).executeUpdate());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,187 @@
|
||||
CREATE TABLE meta_event_handler (
|
||||
id integer PRIMARY KEY AUTOINCREMENT,
|
||||
name TEXT NOT NULL,
|
||||
event TEXT NOT NULL,
|
||||
active INTEGER NOT NULL,
|
||||
json_data TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE locks (
|
||||
lock_id TEXT NOT NULL,
|
||||
lease_expiration DATETIME NOT NULL,
|
||||
PRIMARY KEY (lock_id)
|
||||
);
|
||||
|
||||
CREATE TABLE meta_workflow_def (
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
name TEXT NOT NULL,
|
||||
version INTEGER NOT NULL,
|
||||
latest_version INTEGER DEFAULT 0 NOT NULL,
|
||||
json_data TEXT NOT NULL,
|
||||
PRIMARY KEY (name, version)
|
||||
);
|
||||
CREATE INDEX workflow_def_name_index ON meta_workflow_def(name);
|
||||
|
||||
CREATE TABLE meta_task_def (
|
||||
name TEXT NOT NULL PRIMARY KEY,
|
||||
json_data TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
-- Execution Tables
|
||||
CREATE TABLE event_execution (
|
||||
event_handler_name TEXT NOT NULL,
|
||||
event_name TEXT NOT NULL,
|
||||
execution_id TEXT NOT NULL,
|
||||
message_id TEXT NOT NULL,
|
||||
json_data TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (event_handler_name, event_name, execution_id)
|
||||
);
|
||||
|
||||
CREATE TABLE poll_data (
|
||||
queue_name TEXT NOT NULL,
|
||||
domain TEXT NOT NULL,
|
||||
json_data TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (queue_name, domain)
|
||||
);
|
||||
CREATE INDEX poll_data_queue_name_idx ON poll_data(queue_name);
|
||||
|
||||
CREATE TABLE task_scheduled (
|
||||
workflow_id TEXT NOT NULL,
|
||||
task_key TEXT NOT NULL,
|
||||
task_id TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (workflow_id, task_key)
|
||||
);
|
||||
|
||||
CREATE TABLE task_in_progress (
|
||||
task_def_name TEXT NOT NULL,
|
||||
task_id TEXT NOT NULL,
|
||||
workflow_id TEXT NOT NULL,
|
||||
in_progress_status INTEGER NOT NULL DEFAULT 0,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (task_def_name, task_id)
|
||||
);
|
||||
|
||||
CREATE TABLE task (
|
||||
task_id TEXT NOT NULL PRIMARY KEY,
|
||||
json_data TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE workflow (
|
||||
workflow_id TEXT NOT NULL PRIMARY KEY,
|
||||
correlation_id TEXT,
|
||||
json_data TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE workflow_def_to_workflow (
|
||||
workflow_def TEXT NOT NULL,
|
||||
date_str TEXT,
|
||||
workflow_id TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (workflow_def, date_str, workflow_id)
|
||||
);
|
||||
|
||||
CREATE TABLE workflow_pending (
|
||||
workflow_type TEXT NOT NULL,
|
||||
workflow_id TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (workflow_type, workflow_id)
|
||||
);
|
||||
CREATE INDEX workflow_type_index ON workflow_pending (workflow_type);
|
||||
|
||||
CREATE TABLE workflow_to_task (
|
||||
workflow_id TEXT NOT NULL,
|
||||
task_id TEXT NOT NULL,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (workflow_id, task_id)
|
||||
);
|
||||
CREATE INDEX workflow_id_index ON workflow_to_task(workflow_id);
|
||||
|
||||
-- Queue Tables
|
||||
CREATE TABLE queue (
|
||||
queue_name TEXT NOT NULL PRIMARY KEY,
|
||||
created_on DATETIME DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE queue_message (
|
||||
queue_name TEXT NOT NULL,
|
||||
message_id TEXT NOT NULL,
|
||||
deliver_on DATETIME DEFAULT CURRENT_TIMESTAMP,
|
||||
priority INTEGER DEFAULT 0,
|
||||
popped INTEGER DEFAULT 0,
|
||||
offset_time_seconds INTEGER,
|
||||
payload TEXT,
|
||||
created_on DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
PRIMARY KEY (queue_name, message_id)
|
||||
);
|
||||
|
||||
CREATE TABLE task_execution_logs (
|
||||
log_id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
task_id TEXT NOT NULL,
|
||||
log TEXT NOT NULL,
|
||||
created_time DATETIME NOT NULL
|
||||
);
|
||||
|
||||
CREATE INDEX task_execution_logs_task_id_idx ON task_execution_logs(task_id);
|
||||
|
||||
CREATE TABLE task_index (
|
||||
task_id TEXT NOT NULL,
|
||||
task_type TEXT NOT NULL,
|
||||
task_def_name TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
start_time DATETIME NOT NULL,
|
||||
update_time DATETIME NOT NULL,
|
||||
workflow_type TEXT NOT NULL,
|
||||
json_data TEXT NOT NULL,
|
||||
PRIMARY KEY (task_id)
|
||||
);
|
||||
|
||||
CREATE TABLE workflow_index (
|
||||
workflow_id TEXT NOT NULL,
|
||||
correlation_id TEXT,
|
||||
workflow_type TEXT NOT NULL,
|
||||
start_time DATETIME NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
json_data TEXT NOT NULL, -- SQLite doesn't have JSONB, storing as TEXT
|
||||
update_time DATETIME NOT NULL DEFAULT '1970-01-01 00:00:00',
|
||||
PRIMARY KEY (workflow_id)
|
||||
);
|
||||
|
||||
-- Regular indexes
|
||||
CREATE INDEX workflow_index_correlation_id_idx ON workflow_index(correlation_id);
|
||||
CREATE INDEX workflow_index_start_time_idx ON workflow_index(start_time);
|
||||
CREATE INDEX workflow_index_status_idx ON workflow_index(status);
|
||||
CREATE INDEX workflow_index_workflow_type_idx ON workflow_index(workflow_type);
|
||||
|
||||
-- Regular indexes for columns
|
||||
CREATE INDEX task_index_status_idx ON task_index(status);
|
||||
CREATE INDEX task_index_task_def_name_idx ON task_index(task_def_name);
|
||||
CREATE INDEX task_index_task_id_idx ON task_index(task_id);
|
||||
CREATE INDEX task_index_task_type_idx ON task_index(task_type);
|
||||
CREATE INDEX task_index_update_time_idx ON task_index(update_time);
|
||||
CREATE INDEX task_index_workflow_type_idx ON task_index(workflow_type);
|
||||
|
||||
-- Indexes
|
||||
CREATE INDEX IF NOT EXISTS idx_event_handler_name ON meta_event_handler (name);
|
||||
CREATE INDEX IF NOT EXISTS idx_event_handler_event ON meta_event_handler (event);
|
||||
CREATE INDEX IF NOT EXISTS idx_workflow_def_name ON meta_workflow_def (name);
|
||||
CREATE INDEX IF NOT EXISTS idx_workflow_correlation ON workflow (correlation_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_queue_message_combo ON queue_message (queue_name, priority DESC, popped, deliver_on, created_on);
|
||||
@@ -0,0 +1,2 @@
|
||||
ALTER TABLE workflow_index ADD COLUMN parent_workflow_id TEXT NOT NULL DEFAULT '';
|
||||
CREATE INDEX workflow_index_parent_workflow_id_idx ON workflow_index (parent_workflow_id);
|
||||
@@ -0,0 +1,18 @@
|
||||
CREATE TABLE IF NOT EXISTS file_metadata (
|
||||
file_id VARCHAR(255) NOT NULL PRIMARY KEY,
|
||||
file_name VARCHAR(1024) NOT NULL,
|
||||
content_type VARCHAR(255) NOT NULL,
|
||||
storage_content_hash VARCHAR(255),
|
||||
storage_content_size BIGINT,
|
||||
storage_type VARCHAR(50) NOT NULL,
|
||||
storage_path VARCHAR(2048) NOT NULL,
|
||||
upload_status VARCHAR(50) NOT NULL DEFAULT 'UPLOADING',
|
||||
workflow_id VARCHAR(255) NOT NULL,
|
||||
task_id VARCHAR(255),
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_file_metadata_workflow_id ON file_metadata (workflow_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_file_metadata_task_id ON file_metadata (task_id);
|
||||
CREATE INDEX IF NOT EXISTS idx_file_metadata_upload_status ON file_metadata (upload_status);
|
||||
@@ -0,0 +1,21 @@
|
||||
-- AgentSpan skill storage (conductor.integrations.ai.enabled). Metadata + package bytes.
|
||||
CREATE TABLE IF NOT EXISTS skill_metadata (
|
||||
owner_id VARCHAR(255) NOT NULL,
|
||||
name VARCHAR(255) NOT NULL,
|
||||
version VARCHAR(255) NOT NULL,
|
||||
is_latest BOOLEAN NOT NULL DEFAULT 0,
|
||||
detail_json TEXT NOT NULL,
|
||||
created_at BIGINT,
|
||||
updated_at BIGINT,
|
||||
PRIMARY KEY (owner_id, name, version)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_skill_metadata_owner_name ON skill_metadata (owner_id, name);
|
||||
|
||||
-- Package bytes are stored Base64-encoded so the value binds uniformly across backends.
|
||||
CREATE TABLE IF NOT EXISTS skill_package (
|
||||
handle VARCHAR(255) NOT NULL PRIMARY KEY,
|
||||
data TEXT NOT NULL,
|
||||
size_bytes BIGINT,
|
||||
created_at BIGINT
|
||||
);
|
||||
+6
@@ -0,0 +1,6 @@
|
||||
-- Classifier of the workflow definition an execution was started from (e.g. 'workflow' for a
|
||||
-- plain workflow, 'agent' for AgentSpan agents). Derived from WorkflowDef.metadata at index time.
|
||||
-- Rows indexed before this migration keep a NULL classifier and are treated as untagged
|
||||
-- ('workflow') by the search query builder.
|
||||
ALTER TABLE workflow_index ADD COLUMN classifier TEXT;
|
||||
CREATE INDEX workflow_index_classifier_idx ON workflow_index (classifier, start_time);
|
||||
+114
@@ -0,0 +1,114 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.util.List;
|
||||
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.mockito.Mockito;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
|
||||
import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
|
||||
import com.netflix.conductor.dao.ExecutionDAO;
|
||||
import com.netflix.conductor.dao.ExecutionDAOTest;
|
||||
import com.netflix.conductor.model.WorkflowModel;
|
||||
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
|
||||
|
||||
import com.google.common.collect.Iterables;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
|
||||
@ContextConfiguration(
|
||||
classes = {
|
||||
TestObjectMapperConfiguration.class,
|
||||
SqliteConfiguration.class,
|
||||
FlywayAutoConfiguration.class
|
||||
})
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(properties = "spring.flyway.clean-disabled=false")
|
||||
public class SqliteExecutionDAOTest extends ExecutionDAOTest {
|
||||
|
||||
@Autowired private SqliteExecutionDAO executionDAO;
|
||||
|
||||
@Autowired Flyway flyway;
|
||||
|
||||
// clean the database between tests.
|
||||
@Before
|
||||
public void before() {
|
||||
flyway.migrate();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testPendingByCorrelationId() {
|
||||
|
||||
WorkflowDef def = new WorkflowDef();
|
||||
def.setName("pending_count_correlation_jtest");
|
||||
|
||||
WorkflowModel workflow = createTestWorkflow();
|
||||
workflow.setWorkflowDefinition(def);
|
||||
|
||||
generateWorkflows(workflow, 10);
|
||||
|
||||
List<WorkflowModel> bycorrelationId =
|
||||
getExecutionDAO()
|
||||
.getWorkflowsByCorrelationId(
|
||||
"pending_count_correlation_jtest", "corr001", true);
|
||||
assertNotNull(bycorrelationId);
|
||||
assertEquals(10, bycorrelationId.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoveWorkflow() {
|
||||
WorkflowDef def = new WorkflowDef();
|
||||
def.setName("workflow");
|
||||
|
||||
WorkflowModel workflow = createTestWorkflow();
|
||||
workflow.setWorkflowDefinition(def);
|
||||
|
||||
List<String> ids = generateWorkflows(workflow, 1);
|
||||
|
||||
assertEquals(1, getExecutionDAO().getPendingWorkflowCount("workflow"));
|
||||
ids.forEach(wfId -> getExecutionDAO().removeWorkflow(wfId));
|
||||
assertEquals(0, getExecutionDAO().getPendingWorkflowCount("workflow"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoveWorkflowWithExpiry() {
|
||||
WorkflowDef def = new WorkflowDef();
|
||||
def.setName("workflow");
|
||||
|
||||
WorkflowModel workflow = createTestWorkflow();
|
||||
workflow.setWorkflowDefinition(def);
|
||||
|
||||
List<String> ids = generateWorkflows(workflow, 1);
|
||||
|
||||
final ExecutionDAO execDao = Mockito.spy(getExecutionDAO());
|
||||
assertEquals(1, execDao.getPendingWorkflowCount("workflow"));
|
||||
ids.forEach(wfId -> execDao.removeWorkflowWithExpiry(wfId, 1));
|
||||
Mockito.verify(execDao, Mockito.timeout(10 * 1000)).removeWorkflow(Iterables.getLast(ids));
|
||||
}
|
||||
|
||||
@Override
|
||||
public ExecutionDAO getExecutionDAO() {
|
||||
return executionDAO;
|
||||
}
|
||||
}
|
||||
+863
@@ -0,0 +1,863 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.io.File;
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.sql.Timestamp;
|
||||
import java.time.Instant;
|
||||
import java.time.format.DateTimeFormatter;
|
||||
import java.time.temporal.TemporalAccessor;
|
||||
import java.util.*;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.Before;
|
||||
import org.junit.Ignore;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.annotation.DirtiesContext;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
|
||||
import com.netflix.conductor.common.metadata.tasks.Task;
|
||||
import com.netflix.conductor.common.metadata.tasks.TaskExecLog;
|
||||
import com.netflix.conductor.common.run.SearchResult;
|
||||
import com.netflix.conductor.common.run.TaskSummary;
|
||||
import com.netflix.conductor.common.run.Workflow;
|
||||
import com.netflix.conductor.common.run.WorkflowSummary;
|
||||
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
|
||||
import com.netflix.conductor.sqlite.util.Query;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
@ContextConfiguration(
|
||||
classes = {
|
||||
TestObjectMapperConfiguration.class,
|
||||
SqliteConfiguration.class,
|
||||
FlywayAutoConfiguration.class
|
||||
})
|
||||
@RunWith(SpringRunner.class)
|
||||
@TestPropertySource(
|
||||
properties = {
|
||||
"conductor.app.asyncIndexingEnabled=false",
|
||||
"conductor.elasticsearch.version=0",
|
||||
"conductor.indexing.type=sqlite",
|
||||
"conductor.db.type=sqlite",
|
||||
"spring.flyway.clean-disabled=false"
|
||||
})
|
||||
@SpringBootTest
|
||||
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
|
||||
public class SqliteIndexDAOTest {
|
||||
|
||||
@Autowired private SqliteIndexDAO indexDAO;
|
||||
|
||||
@Autowired private ObjectMapper objectMapper;
|
||||
|
||||
@Qualifier("dataSource")
|
||||
@Autowired
|
||||
private DataSource dataSource;
|
||||
|
||||
@Autowired Flyway flyway;
|
||||
|
||||
// clean the database between tests.
|
||||
@Before
|
||||
public void before() {
|
||||
// Delete the database file if it exists
|
||||
File dbFile = new File("conductorosstest.db");
|
||||
if (dbFile.exists()) {
|
||||
dbFile.delete();
|
||||
}
|
||||
|
||||
// Also delete SQLite journal files if they exist
|
||||
File dbJournal = new File("conductorosstest.db-journal");
|
||||
if (dbJournal.exists()) {
|
||||
dbJournal.delete();
|
||||
}
|
||||
File dbShm = new File("conductorosstest.db-shm");
|
||||
if (dbShm.exists()) {
|
||||
dbShm.delete();
|
||||
}
|
||||
File dbWal = new File("conductorosstest.db-wal");
|
||||
if (dbWal.exists()) {
|
||||
dbWal.delete();
|
||||
}
|
||||
|
||||
flyway.clean();
|
||||
flyway.migrate();
|
||||
}
|
||||
|
||||
private WorkflowSummary getMockWorkflowSummary(String id) {
|
||||
WorkflowSummary wfs = new WorkflowSummary();
|
||||
wfs.setWorkflowId(id);
|
||||
wfs.setCorrelationId("correlation-id");
|
||||
wfs.setWorkflowType("workflow-type");
|
||||
wfs.setStartTime("2023-02-07T08:42:45Z");
|
||||
wfs.setUpdateTime("2023-02-07T08:43:45Z");
|
||||
wfs.setStatus(Workflow.WorkflowStatus.COMPLETED);
|
||||
return wfs;
|
||||
}
|
||||
|
||||
private TaskSummary getMockTaskSummary(String taskId) {
|
||||
TaskSummary ts = new TaskSummary();
|
||||
ts.setTaskId(taskId);
|
||||
ts.setTaskType("task-type1");
|
||||
ts.setTaskDefName("task-def-name1");
|
||||
ts.setStatus(Task.Status.COMPLETED);
|
||||
ts.setStartTime("2023-02-07T09:41:45Z");
|
||||
ts.setUpdateTime("2023-02-07T09:42:45Z");
|
||||
ts.setWorkflowType("workflow-type");
|
||||
return ts;
|
||||
}
|
||||
|
||||
private TaskExecLog getMockTaskExecutionLog(String taskId, long createdTime, String log) {
|
||||
TaskExecLog tse = new TaskExecLog();
|
||||
tse.setTaskId(taskId);
|
||||
tse.setLog(log);
|
||||
tse.setCreatedTime(createdTime);
|
||||
return tse;
|
||||
}
|
||||
|
||||
private void compareWorkflowSummary(WorkflowSummary wfs) throws SQLException {
|
||||
List<Map<String, Object>> result =
|
||||
queryDb(
|
||||
String.format(
|
||||
"SELECT * FROM workflow_index WHERE workflow_id = '%s'",
|
||||
wfs.getWorkflowId()));
|
||||
assertEquals("Wrong number of rows returned", 1, result.size());
|
||||
assertEquals(
|
||||
"Workflow id does not match",
|
||||
wfs.getWorkflowId(),
|
||||
result.get(0).get("workflow_id"));
|
||||
assertEquals(
|
||||
"Correlation id does not match",
|
||||
wfs.getCorrelationId(),
|
||||
result.get(0).get("correlation_id"));
|
||||
assertEquals(
|
||||
"Workflow type does not match",
|
||||
wfs.getWorkflowType(),
|
||||
result.get(0).get("workflow_type"));
|
||||
TemporalAccessor ta = DateTimeFormatter.ISO_INSTANT.parse(wfs.getStartTime());
|
||||
Timestamp startTime = Timestamp.from(Instant.from(ta));
|
||||
assertEquals(
|
||||
"Start time does not match", startTime.toString(), result.get(0).get("start_time"));
|
||||
assertEquals(
|
||||
"Status does not match", wfs.getStatus().toString(), result.get(0).get("status"));
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> queryDb(String query) throws SQLException {
|
||||
try (Connection c = dataSource.getConnection()) {
|
||||
try (Query q = new Query(objectMapper, c, query)) {
|
||||
return q.executeAndFetchMap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void compareTaskSummary(TaskSummary ts) throws SQLException {
|
||||
List<Map<String, Object>> result =
|
||||
queryDb(
|
||||
String.format(
|
||||
"SELECT * FROM task_index WHERE task_id = '%s'", ts.getTaskId()));
|
||||
assertEquals("Wrong number of rows returned", 1, result.size());
|
||||
assertEquals("Task id does not match", ts.getTaskId(), result.get(0).get("task_id"));
|
||||
assertEquals("Task type does not match", ts.getTaskType(), result.get(0).get("task_type"));
|
||||
assertEquals(
|
||||
"Task def name does not match",
|
||||
ts.getTaskDefName(),
|
||||
result.get(0).get("task_def_name"));
|
||||
TemporalAccessor startTa = DateTimeFormatter.ISO_INSTANT.parse(ts.getStartTime());
|
||||
Timestamp startTime = Timestamp.from(Instant.from(startTa));
|
||||
assertEquals(
|
||||
"Start time does not match", startTime.toString(), result.get(0).get("start_time"));
|
||||
TemporalAccessor updateTa = DateTimeFormatter.ISO_INSTANT.parse(ts.getUpdateTime());
|
||||
Timestamp updateTime = Timestamp.from(Instant.from(updateTa));
|
||||
assertEquals(
|
||||
"Update time does not match",
|
||||
updateTime.toString(),
|
||||
result.get(0).get("update_time"));
|
||||
assertEquals(
|
||||
"Status does not match", ts.getStatus().toString(), result.get(0).get("status"));
|
||||
assertEquals(
|
||||
"Workflow type does not match",
|
||||
ts.getWorkflowType().toString(),
|
||||
result.get(0).get("workflow_type"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIndexNewWorkflow() throws SQLException {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-new");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
compareWorkflowSummary(wfs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIndexExistingWorkflow() throws SQLException {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-existing");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
compareWorkflowSummary(wfs);
|
||||
|
||||
wfs.setStatus(Workflow.WorkflowStatus.FAILED);
|
||||
wfs.setUpdateTime("2023-02-07T08:44:45Z");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
compareWorkflowSummary(wfs);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWhenWorkflowIsIndexedOutOfOrderOnlyLatestIsIndexed() throws SQLException {
|
||||
WorkflowSummary firstWorkflowUpdate =
|
||||
getMockWorkflowSummary("workflow-id-existing-no-index");
|
||||
firstWorkflowUpdate.setUpdateTime("2023-02-07T08:42:45Z");
|
||||
|
||||
WorkflowSummary secondWorkflowUpdateSummary =
|
||||
getMockWorkflowSummary("workflow-id-existing-no-index");
|
||||
secondWorkflowUpdateSummary.setUpdateTime("2023-02-07T08:43:45Z");
|
||||
secondWorkflowUpdateSummary.setStatus(Workflow.WorkflowStatus.FAILED);
|
||||
|
||||
indexDAO.indexWorkflow(secondWorkflowUpdateSummary);
|
||||
|
||||
compareWorkflowSummary(secondWorkflowUpdateSummary);
|
||||
|
||||
indexDAO.indexWorkflow(firstWorkflowUpdate);
|
||||
|
||||
compareWorkflowSummary(secondWorkflowUpdateSummary);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWhenWorkflowUpdatesHaveTheSameUpdateTimeTheLastIsIndexed() throws SQLException {
|
||||
WorkflowSummary firstWorkflowUpdate =
|
||||
getMockWorkflowSummary("workflow-id-existing-same-time-index");
|
||||
firstWorkflowUpdate.setUpdateTime("2023-02-07T08:42:45Z");
|
||||
|
||||
WorkflowSummary secondWorkflowUpdateSummary =
|
||||
getMockWorkflowSummary("workflow-id-existing-same-time-index");
|
||||
secondWorkflowUpdateSummary.setUpdateTime("2023-02-07T08:42:45Z");
|
||||
secondWorkflowUpdateSummary.setStatus(Workflow.WorkflowStatus.FAILED);
|
||||
|
||||
indexDAO.indexWorkflow(firstWorkflowUpdate);
|
||||
|
||||
compareWorkflowSummary(firstWorkflowUpdate);
|
||||
|
||||
indexDAO.indexWorkflow(secondWorkflowUpdateSummary);
|
||||
|
||||
compareWorkflowSummary(secondWorkflowUpdateSummary);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIndexNewTask() throws SQLException {
|
||||
TaskSummary ts = getMockTaskSummary("task-id-new");
|
||||
|
||||
indexDAO.indexTask(ts);
|
||||
|
||||
compareTaskSummary(ts);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testIndexExistingTask() throws SQLException {
|
||||
TaskSummary ts = getMockTaskSummary("task-id-existing");
|
||||
|
||||
indexDAO.indexTask(ts);
|
||||
|
||||
compareTaskSummary(ts);
|
||||
|
||||
ts.setUpdateTime("2023-02-07T09:43:45Z");
|
||||
ts.setStatus(Task.Status.FAILED);
|
||||
|
||||
indexDAO.indexTask(ts);
|
||||
|
||||
compareTaskSummary(ts);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWhenTaskIsIndexedOutOfOrderOnlyLatestIsIndexed() throws SQLException {
|
||||
TaskSummary firstTaskState = getMockTaskSummary("task-id-exiting-no-update");
|
||||
firstTaskState.setUpdateTime("2023-02-07T09:41:45Z");
|
||||
firstTaskState.setStatus(Task.Status.FAILED);
|
||||
|
||||
TaskSummary secondTaskState = getMockTaskSummary("task-id-exiting-no-update");
|
||||
secondTaskState.setUpdateTime("2023-02-07T09:42:45Z");
|
||||
|
||||
indexDAO.indexTask(secondTaskState);
|
||||
|
||||
compareTaskSummary(secondTaskState);
|
||||
|
||||
indexDAO.indexTask(firstTaskState);
|
||||
|
||||
compareTaskSummary(secondTaskState);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWhenTaskUpdatesHaveTheSameUpdateTimeTheLastIsIndexed() throws SQLException {
|
||||
TaskSummary firstTaskState = getMockTaskSummary("task-id-exiting-same-time-update");
|
||||
firstTaskState.setUpdateTime("2023-02-07T09:42:45Z");
|
||||
firstTaskState.setStatus(Task.Status.FAILED);
|
||||
|
||||
TaskSummary secondTaskState = getMockTaskSummary("task-id-exiting-same-time-update");
|
||||
secondTaskState.setUpdateTime("2023-02-07T09:42:45Z");
|
||||
|
||||
indexDAO.indexTask(firstTaskState);
|
||||
|
||||
compareTaskSummary(firstTaskState);
|
||||
|
||||
indexDAO.indexTask(secondTaskState);
|
||||
|
||||
compareTaskSummary(secondTaskState);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testAddTaskExecutionLogs() throws SQLException {
|
||||
List<TaskExecLog> logs = new ArrayList<>();
|
||||
String taskId = UUID.randomUUID().toString();
|
||||
logs.add(getMockTaskExecutionLog(taskId, 1675845986000L, "Log 1"));
|
||||
logs.add(getMockTaskExecutionLog(taskId, 1675845987000L, "Log 2"));
|
||||
|
||||
indexDAO.addTaskExecutionLogs(logs);
|
||||
|
||||
List<Map<String, Object>> records =
|
||||
queryDb(
|
||||
"SELECT * FROM task_execution_logs where task_id = '"
|
||||
+ taskId
|
||||
+ "' ORDER BY created_time ASC");
|
||||
assertEquals("Wrong number of logs returned", 2, records.size());
|
||||
assertEquals(logs.get(0).getLog(), records.get(0).get("log"));
|
||||
assertEquals(1675845986000L, records.get(0).get("created_time"));
|
||||
assertEquals(logs.get(1).getLog(), records.get(1).get("log"));
|
||||
assertEquals(1675845987000L, records.get(1).get("created_time"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflowSummary() {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
String query = String.format("workflowId=\"%s\"", wfs.getWorkflowId());
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList());
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong workflow returned",
|
||||
wfs.getWorkflowId(),
|
||||
results.getResults().get(0).getWorkflowId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflowSummaryByClassifier() {
|
||||
String correlationId = "classifier-search-correlation-id";
|
||||
|
||||
WorkflowSummary agentWfs = getMockWorkflowSummary("workflow-id-classifier-agent");
|
||||
agentWfs.setCorrelationId(correlationId);
|
||||
agentWfs.setClassifier("agent");
|
||||
indexDAO.indexWorkflow(agentWfs);
|
||||
|
||||
WorkflowSummary plainWfs = getMockWorkflowSummary("workflow-id-classifier-plain");
|
||||
plainWfs.setCorrelationId(correlationId);
|
||||
plainWfs.setClassifier("workflow");
|
||||
indexDAO.indexWorkflow(plainWfs);
|
||||
|
||||
// Simulates a row indexed before the classifier column existed (NULL classifier).
|
||||
WorkflowSummary legacyWfs = getMockWorkflowSummary("workflow-id-classifier-legacy");
|
||||
legacyWfs.setCorrelationId(correlationId);
|
||||
indexDAO.indexWorkflow(legacyWfs);
|
||||
|
||||
String agentQuery =
|
||||
String.format("correlationId='%s' AND classifier='agent'", correlationId);
|
||||
SearchResult<WorkflowSummary> agentResults =
|
||||
indexDAO.searchWorkflowSummary(agentQuery, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Wrong number of agent results", 1, agentResults.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong workflow returned",
|
||||
agentWfs.getWorkflowId(),
|
||||
agentResults.getResults().get(0).getWorkflowId());
|
||||
|
||||
// The untagged token must match both explicitly tagged plain workflows and legacy
|
||||
// NULL rows.
|
||||
String workflowQuery =
|
||||
String.format("correlationId='%s' AND classifier='workflow'", correlationId);
|
||||
SearchResult<WorkflowSummary> workflowResults =
|
||||
indexDAO.searchWorkflowSummary(workflowQuery, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Wrong number of untagged results", 2, workflowResults.getResults().size());
|
||||
|
||||
String inQuery =
|
||||
String.format("correlationId='%s' AND classifier IN (agent,other)", correlationId);
|
||||
SearchResult<WorkflowSummary> inResults =
|
||||
indexDAO.searchWorkflowSummary(inQuery, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Wrong number of IN clause results", 1, inResults.getResults().size());
|
||||
|
||||
indexDAO.removeWorkflow(agentWfs.getWorkflowId());
|
||||
indexDAO.removeWorkflow(plainWfs.getWorkflowId());
|
||||
indexDAO.removeWorkflow(legacyWfs.getWorkflowId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testFullTextSearchWorkflowSummary() {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
String freeText = "notworkflow-id";
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary("", freeText, 0, 15, new ArrayList<>());
|
||||
assertEquals("Wrong number of results returned", 0, results.getResults().size());
|
||||
|
||||
freeText = "workflow-id";
|
||||
results = indexDAO.searchWorkflowSummary("", freeText, 0, 15, new ArrayList<>());
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong workflow returned",
|
||||
wfs.getWorkflowId(),
|
||||
results.getResults().getFirst().getWorkflowId());
|
||||
}
|
||||
|
||||
// json working not working
|
||||
// @Test
|
||||
// public void testJsonSearchWorkflowSummary() {
|
||||
// WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-summary");
|
||||
// wfs.setVersion(3);
|
||||
//
|
||||
// indexDAO.indexWorkflow(wfs);
|
||||
//
|
||||
// String freeText = "{\"correlationId\":\"not-the-id\"}";
|
||||
// SearchResult<WorkflowSummary> results =
|
||||
// indexDAO.searchWorkflowSummary("", freeText, 0, 15, new ArrayList());
|
||||
// assertEquals("Wrong number of results returned", 0, results.getResults().size());
|
||||
//
|
||||
// freeText = "{\"correlationId\":\"correlation-id\", \"version\":3}";
|
||||
// results = indexDAO.searchWorkflowSummary("", freeText, 0, 15, new ArrayList());
|
||||
// assertEquals("No results returned", 1, results.getResults().size());
|
||||
// assertEquals(
|
||||
// "Wrong workflow returned",
|
||||
// wfs.getWorkflowId(),
|
||||
// results.getResults().get(0).getWorkflowId());
|
||||
// }
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflowSummaryPagination() {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-pagination-" + i);
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
}
|
||||
|
||||
List<String> orderBy = Arrays.asList(new String[] {"workflowId:DESC"});
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary("", "workflow-id-pagination", 0, 2, orderBy);
|
||||
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
|
||||
assertEquals("Wrong number of results returned", 2, results.getResults().size());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"workflow-id-pagination-4",
|
||||
results.getResults().get(0).getWorkflowId());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"workflow-id-pagination-3",
|
||||
results.getResults().get(1).getWorkflowId());
|
||||
results = indexDAO.searchWorkflowSummary("", "*", 2, 2, orderBy);
|
||||
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
|
||||
assertEquals("Wrong number of results returned", 2, results.getResults().size());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"workflow-id-pagination-2",
|
||||
results.getResults().get(0).getWorkflowId());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"workflow-id-pagination-1",
|
||||
results.getResults().get(1).getWorkflowId());
|
||||
results = indexDAO.searchWorkflowSummary("", "*", 4, 2, orderBy);
|
||||
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
|
||||
assertEquals("Wrong number of results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"workflow-id-pagination-0",
|
||||
results.getResults().get(0).getWorkflowId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflows() {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-v2");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
String query = String.format("workflowId=\"%s\"", wfs.getWorkflowId());
|
||||
SearchResult<String> results =
|
||||
indexDAO.searchWorkflows(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong workflow id returned", wfs.getWorkflowId(), results.getResults().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflowsPagination() {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("wf-v2-pagination-" + i);
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
}
|
||||
|
||||
List<String> orderBy = Arrays.asList(new String[] {"workflowId:DESC"});
|
||||
SearchResult<String> results = indexDAO.searchWorkflows("", "*", 0, 2, orderBy);
|
||||
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
|
||||
assertEquals("Wrong number of results returned", 2, results.getResults().size());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"wf-v2-pagination-4",
|
||||
results.getResults().get(0));
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"wf-v2-pagination-3",
|
||||
results.getResults().get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflowsWithNullSort() {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-null-sort");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
String query = String.format("workflowId=\"%s\"", wfs.getWorkflowId());
|
||||
SearchResult<String> results = indexDAO.searchWorkflows(query, "*", 0, 15, null);
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong workflow id returned", wfs.getWorkflowId(), results.getResults().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflowSummaryWithSingleQuotes() {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-single-quote");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
String query = String.format("workflowId='%s'", wfs.getWorkflowId());
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong workflow returned",
|
||||
wfs.getWorkflowId(),
|
||||
results.getResults().get(0).getWorkflowId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflowsWithSingleQuotes() {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-v2-single-quote");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
String query = String.format("workflowId='%s'", wfs.getWorkflowId());
|
||||
SearchResult<String> results =
|
||||
indexDAO.searchWorkflows(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong workflow id returned", wfs.getWorkflowId(), results.getResults().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchWorkflowsWithSingleQuotedMultiCondition() {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-multi-cond");
|
||||
wfs.setCorrelationId("test-correlation-id");
|
||||
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
String query =
|
||||
String.format(
|
||||
"correlationId='%s' AND workflowType='%s'",
|
||||
wfs.getCorrelationId(), wfs.getWorkflowType());
|
||||
SearchResult<String> results =
|
||||
indexDAO.searchWorkflows(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong workflow id returned", wfs.getWorkflowId(), results.getResults().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchTasks() {
|
||||
TaskSummary ts = getMockTaskSummary("task-id-v2");
|
||||
|
||||
indexDAO.indexTask(ts);
|
||||
|
||||
String query = String.format("taskId=\"%s\"", ts.getTaskId());
|
||||
SearchResult<String> results = indexDAO.searchTasks(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals("Wrong task id returned", ts.getTaskId(), results.getResults().get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchTasksPagination() {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
TaskSummary ts = getMockTaskSummary("task-v2-pagination-" + i);
|
||||
indexDAO.indexTask(ts);
|
||||
}
|
||||
|
||||
List<String> orderBy = Arrays.asList(new String[] {"taskId:DESC"});
|
||||
SearchResult<String> results = indexDAO.searchTasks("", "*", 0, 2, orderBy);
|
||||
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
|
||||
assertEquals("Wrong number of results returned", 2, results.getResults().size());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"task-v2-pagination-4",
|
||||
results.getResults().get(0));
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"task-v2-pagination-3",
|
||||
results.getResults().get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchTaskSummary() {
|
||||
TaskSummary ts = getMockTaskSummary("task-id");
|
||||
|
||||
indexDAO.indexTask(ts);
|
||||
|
||||
String query = String.format("taskId=\"%s\"", ts.getTaskId());
|
||||
SearchResult<TaskSummary> results =
|
||||
indexDAO.searchTaskSummary(query, "*", 0, 15, new ArrayList());
|
||||
assertEquals("No results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Wrong task returned", ts.getTaskId(), results.getResults().get(0).getTaskId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchTaskSummaryPagination() {
|
||||
for (int i = 0; i < 5; i++) {
|
||||
TaskSummary ts = getMockTaskSummary("task-id-pagination-" + i);
|
||||
indexDAO.indexTask(ts);
|
||||
}
|
||||
|
||||
List<String> orderBy = Arrays.asList(new String[] {"taskId:DESC"});
|
||||
SearchResult<TaskSummary> results = indexDAO.searchTaskSummary("", "*", 0, 2, orderBy);
|
||||
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
|
||||
assertEquals("Wrong number of results returned", 2, results.getResults().size());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"task-id-pagination-4",
|
||||
results.getResults().get(0).getTaskId());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"task-id-pagination-3",
|
||||
results.getResults().get(1).getTaskId());
|
||||
results = indexDAO.searchTaskSummary("", "*", 2, 2, orderBy);
|
||||
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
|
||||
assertEquals("Wrong number of results returned", 2, results.getResults().size());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"task-id-pagination-2",
|
||||
results.getResults().get(0).getTaskId());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"task-id-pagination-1",
|
||||
results.getResults().get(1).getTaskId());
|
||||
results = indexDAO.searchTaskSummary("", "*", 4, 2, orderBy);
|
||||
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
|
||||
assertEquals("Wrong number of results returned", 1, results.getResults().size());
|
||||
assertEquals(
|
||||
"Results returned in wrong order",
|
||||
"task-id-pagination-0",
|
||||
results.getResults().get(0).getTaskId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetTaskExecutionLogs() throws SQLException {
|
||||
List<TaskExecLog> logs = new ArrayList<>();
|
||||
String taskId = UUID.randomUUID().toString();
|
||||
logs.add(getMockTaskExecutionLog(taskId, new Date(1675845986000L).getTime(), "Log 1"));
|
||||
logs.add(getMockTaskExecutionLog(taskId, new Date(1675845987000L).getTime(), "Log 2"));
|
||||
|
||||
indexDAO.addTaskExecutionLogs(logs);
|
||||
|
||||
List<TaskExecLog> records = indexDAO.getTaskExecutionLogs(logs.get(0).getTaskId());
|
||||
assertEquals("Wrong number of logs returned", 2, records.size());
|
||||
assertEquals(logs.get(0).getLog(), records.get(0).getLog());
|
||||
assertEquals(logs.get(0).getCreatedTime(), 1675845986000L);
|
||||
assertEquals(logs.get(1).getLog(), records.get(1).getLog());
|
||||
assertEquals(logs.get(1).getCreatedTime(), 1675845987000L);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoveWorkflow() throws SQLException {
|
||||
String workflowId = UUID.randomUUID().toString();
|
||||
WorkflowSummary wfs = getMockWorkflowSummary(workflowId);
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
List<Map<String, Object>> workflow_records =
|
||||
queryDb("SELECT * FROM workflow_index WHERE workflow_id = '" + workflowId + "'");
|
||||
assertEquals("Workflow index record was not created", 1, workflow_records.size());
|
||||
|
||||
indexDAO.removeWorkflow(workflowId);
|
||||
|
||||
workflow_records =
|
||||
queryDb("SELECT * FROM workflow_index WHERE workflow_id = '" + workflowId + "'");
|
||||
assertEquals("Workflow index record was not deleted", 0, workflow_records.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
@Ignore("Skipping due to SQLite database connection issues in test environment")
|
||||
public void testRemoveTask() throws SQLException {
|
||||
// Ensure database is properly initialized
|
||||
flyway.clean();
|
||||
flyway.migrate();
|
||||
|
||||
String workflowId = UUID.randomUUID().toString();
|
||||
|
||||
String taskId = UUID.randomUUID().toString();
|
||||
TaskSummary ts = getMockTaskSummary(taskId);
|
||||
indexDAO.indexTask(ts);
|
||||
|
||||
List<TaskExecLog> logs = new ArrayList<>();
|
||||
logs.add(getMockTaskExecutionLog(taskId, new Date(1675845986000L).getTime(), "Log 1"));
|
||||
logs.add(getMockTaskExecutionLog(taskId, new Date(1675845987000L).getTime(), "Log 2"));
|
||||
indexDAO.addTaskExecutionLogs(logs);
|
||||
|
||||
List<Map<String, Object>> task_records =
|
||||
queryDb("SELECT * FROM task_index WHERE task_id = '" + taskId + "'");
|
||||
assertEquals("Task index record was not created", 1, task_records.size());
|
||||
|
||||
List<Map<String, Object>> log_records =
|
||||
queryDb("SELECT * FROM task_execution_logs WHERE task_id = '" + taskId + "'");
|
||||
assertEquals("Task execution logs were not created", 2, log_records.size());
|
||||
|
||||
indexDAO.removeTask(workflowId, taskId);
|
||||
|
||||
task_records = queryDb("SELECT * FROM task_index WHERE task_id = '" + taskId + "'");
|
||||
assertEquals("Task index record was not deleted", 0, task_records.size());
|
||||
|
||||
log_records = queryDb("SELECT * FROM task_execution_logs WHERE task_id = '" + taskId + "'");
|
||||
assertEquals("Task execution logs were not deleted", 0, log_records.size());
|
||||
}
|
||||
|
||||
private WorkflowSummary getMockWorkflowSummary(String id, String parentWorkflowId) {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary(id);
|
||||
wfs.setParentWorkflowId(parentWorkflowId);
|
||||
return wfs;
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWildcardSearchWorkflowSummaryByType() {
|
||||
WorkflowSummary wfs1 = getMockWorkflowSummary("wf-wildcard-1");
|
||||
wfs1.setWorkflowType("order_processing_v1");
|
||||
indexDAO.indexWorkflow(wfs1);
|
||||
|
||||
WorkflowSummary wfs2 = getMockWorkflowSummary("wf-wildcard-2");
|
||||
wfs2.setWorkflowType("order_processing_v2");
|
||||
indexDAO.indexWorkflow(wfs2);
|
||||
|
||||
WorkflowSummary wfs3 = getMockWorkflowSummary("wf-wildcard-3");
|
||||
wfs3.setWorkflowType("payment_processing_v1");
|
||||
indexDAO.indexWorkflow(wfs3);
|
||||
|
||||
String query = "workflowType=order_processing*";
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Should find 2 order_processing workflows", 2, results.getResults().size());
|
||||
|
||||
query = "workflowType=*processing*";
|
||||
results = indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Should find 3 processing workflows", 3, results.getResults().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWildcardSearchNoMatches() {
|
||||
WorkflowSummary wfs = getMockWorkflowSummary("wf-wildcard-nomatch");
|
||||
wfs.setWorkflowType("order_processing_v1");
|
||||
indexDAO.indexWorkflow(wfs);
|
||||
|
||||
String query = "workflowType=payment*";
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Should find 0 workflows", 0, results.getResults().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchExcludeSubWorkflows() {
|
||||
WorkflowSummary topLevel1 = getMockWorkflowSummary("wf-top-1", "");
|
||||
indexDAO.indexWorkflow(topLevel1);
|
||||
|
||||
WorkflowSummary topLevel2 = getMockWorkflowSummary("wf-top-2", "");
|
||||
indexDAO.indexWorkflow(topLevel2);
|
||||
|
||||
WorkflowSummary subWf1 = getMockWorkflowSummary("wf-sub-1", "wf-top-1");
|
||||
indexDAO.indexWorkflow(subWf1);
|
||||
|
||||
WorkflowSummary subWf2 = getMockWorkflowSummary("wf-sub-2", "wf-top-1");
|
||||
indexDAO.indexWorkflow(subWf2);
|
||||
|
||||
String query = "parentWorkflowId=\"\"";
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Should find only 2 top-level workflows", 2, results.getResults().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchSubWorkflowsOfParent() {
|
||||
WorkflowSummary topLevel = getMockWorkflowSummary("wf-parent-1", "");
|
||||
indexDAO.indexWorkflow(topLevel);
|
||||
|
||||
WorkflowSummary subWf1 = getMockWorkflowSummary("wf-child-1", "wf-parent-1");
|
||||
indexDAO.indexWorkflow(subWf1);
|
||||
|
||||
WorkflowSummary subWf2 = getMockWorkflowSummary("wf-child-2", "wf-parent-1");
|
||||
indexDAO.indexWorkflow(subWf2);
|
||||
|
||||
WorkflowSummary subWf3 = getMockWorkflowSummary("wf-child-3", "wf-parent-other");
|
||||
indexDAO.indexWorkflow(subWf3);
|
||||
|
||||
String query = "parentWorkflowId=\"wf-parent-1\"";
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Should find 2 child workflows", 2, results.getResults().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testSearchSubWorkflowsWrongParentReturnsEmpty() {
|
||||
WorkflowSummary subWf1 = getMockWorkflowSummary("wf-orphan-1", "wf-parent-1");
|
||||
indexDAO.indexWorkflow(subWf1);
|
||||
|
||||
String query = "parentWorkflowId=\"wf-nonexistent-parent\"";
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals(
|
||||
"Should find 0 workflows for nonexistent parent", 0, results.getResults().size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWildcardWithParentWorkflowIdFilter() {
|
||||
WorkflowSummary topOrder = getMockWorkflowSummary("wf-combined-1", "");
|
||||
topOrder.setWorkflowType("order_processing_v1");
|
||||
indexDAO.indexWorkflow(topOrder);
|
||||
|
||||
WorkflowSummary topPayment = getMockWorkflowSummary("wf-combined-2", "");
|
||||
topPayment.setWorkflowType("payment_processing_v1");
|
||||
indexDAO.indexWorkflow(topPayment);
|
||||
|
||||
WorkflowSummary subOrder = getMockWorkflowSummary("wf-combined-3", "wf-combined-1");
|
||||
subOrder.setWorkflowType("order_processing_v1");
|
||||
indexDAO.indexWorkflow(subOrder);
|
||||
|
||||
String query = "parentWorkflowId=\"\" AND workflowType=order*";
|
||||
SearchResult<WorkflowSummary> results =
|
||||
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
|
||||
assertEquals("Should find 1 top-level order workflow", 1, results.getResults().size());
|
||||
assertEquals("wf-combined-1", results.getResults().get(0).getWorkflowId());
|
||||
}
|
||||
}
|
||||
+318
@@ -0,0 +1,318 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.util.*;
|
||||
import java.util.function.Function;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import org.apache.commons.lang3.builder.EqualsBuilder;
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestName;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
|
||||
import com.netflix.conductor.common.metadata.events.EventHandler;
|
||||
import com.netflix.conductor.common.metadata.tasks.TaskDef;
|
||||
import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
|
||||
import com.netflix.conductor.core.exception.NonTransientException;
|
||||
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
|
||||
import com.netflix.conductor.sqlite.dao.metadata.SqliteMetadataDAO;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.junit.Assert.assertEquals;
|
||||
|
||||
@ContextConfiguration(
|
||||
classes = {
|
||||
TestObjectMapperConfiguration.class,
|
||||
SqliteConfiguration.class,
|
||||
FlywayAutoConfiguration.class
|
||||
})
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(properties = "spring.flyway.clean-disabled=false")
|
||||
public class SqliteMetadataDAOTest {
|
||||
|
||||
@Autowired private SqliteMetadataDAO metadataDAO;
|
||||
|
||||
@Rule public TestName name = new TestName();
|
||||
|
||||
@Autowired private Flyway flyway;
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
flyway.migrate();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testDuplicateWorkflowDef() {
|
||||
WorkflowDef def = new WorkflowDef();
|
||||
def.setName("testDuplicate");
|
||||
def.setVersion(1);
|
||||
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
|
||||
NonTransientException applicationException =
|
||||
assertThrows(NonTransientException.class, () -> metadataDAO.createWorkflowDef(def));
|
||||
assertEquals(
|
||||
"Workflow with testDuplicate.1 already exists!", applicationException.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoveNotExistingWorkflowDef() {
|
||||
NonTransientException applicationException =
|
||||
assertThrows(
|
||||
NonTransientException.class,
|
||||
() -> metadataDAO.removeWorkflowDef("test", 1));
|
||||
assertEquals(
|
||||
"No such workflow definition: test version: 1", applicationException.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testWorkflowDefOperations() {
|
||||
WorkflowDef def = new WorkflowDef();
|
||||
def.setName("test");
|
||||
def.setVersion(1);
|
||||
def.setDescription("description");
|
||||
def.setCreatedBy("unit_test");
|
||||
def.setCreateTime(1L);
|
||||
def.setOwnerApp("ownerApp");
|
||||
def.setUpdatedBy("unit_test2");
|
||||
def.setUpdateTime(2L);
|
||||
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
|
||||
List<WorkflowDef> all = metadataDAO.getAllWorkflowDefs();
|
||||
assertNotNull(all);
|
||||
assertEquals(1, all.size());
|
||||
assertEquals("test", all.get(0).getName());
|
||||
assertEquals(1, all.get(0).getVersion());
|
||||
|
||||
WorkflowDef found = metadataDAO.getWorkflowDef("test", 1).get();
|
||||
assertTrue(EqualsBuilder.reflectionEquals(def, found));
|
||||
|
||||
def.setVersion(3);
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
|
||||
all = metadataDAO.getAllWorkflowDefs();
|
||||
assertNotNull(all);
|
||||
assertEquals(2, all.size());
|
||||
assertEquals("test", all.get(0).getName());
|
||||
assertEquals(1, all.get(0).getVersion());
|
||||
|
||||
found = metadataDAO.getLatestWorkflowDef(def.getName()).get();
|
||||
assertEquals(def.getName(), found.getName());
|
||||
assertEquals(def.getVersion(), found.getVersion());
|
||||
assertEquals(3, found.getVersion());
|
||||
|
||||
all = metadataDAO.getAllLatest();
|
||||
assertNotNull(all);
|
||||
assertEquals(1, all.size());
|
||||
assertEquals("test", all.get(0).getName());
|
||||
assertEquals(3, all.get(0).getVersion());
|
||||
|
||||
all = metadataDAO.getAllVersions(def.getName());
|
||||
assertNotNull(all);
|
||||
assertEquals(2, all.size());
|
||||
assertEquals("test", all.get(0).getName());
|
||||
assertEquals("test", all.get(1).getName());
|
||||
assertEquals(1, all.get(0).getVersion());
|
||||
assertEquals(3, all.get(1).getVersion());
|
||||
|
||||
def.setDescription("updated");
|
||||
metadataDAO.updateWorkflowDef(def);
|
||||
found = metadataDAO.getWorkflowDef(def.getName(), def.getVersion()).get();
|
||||
assertEquals(def.getDescription(), found.getDescription());
|
||||
|
||||
List<String> allnames = metadataDAO.findAll();
|
||||
assertNotNull(allnames);
|
||||
assertEquals(1, allnames.size());
|
||||
assertEquals(def.getName(), allnames.get(0));
|
||||
|
||||
def.setVersion(2);
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
|
||||
found = metadataDAO.getLatestWorkflowDef(def.getName()).get();
|
||||
assertEquals(def.getName(), found.getName());
|
||||
assertEquals(3, found.getVersion());
|
||||
|
||||
metadataDAO.removeWorkflowDef("test", 3);
|
||||
Optional<WorkflowDef> deleted = metadataDAO.getWorkflowDef("test", 3);
|
||||
assertFalse(deleted.isPresent());
|
||||
|
||||
found = metadataDAO.getLatestWorkflowDef(def.getName()).get();
|
||||
assertEquals(def.getName(), found.getName());
|
||||
assertEquals(2, found.getVersion());
|
||||
|
||||
metadataDAO.removeWorkflowDef("test", 1);
|
||||
deleted = metadataDAO.getWorkflowDef("test", 1);
|
||||
assertFalse(deleted.isPresent());
|
||||
|
||||
found = metadataDAO.getLatestWorkflowDef(def.getName()).get();
|
||||
assertEquals(def.getName(), found.getName());
|
||||
assertEquals(2, found.getVersion());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testTaskDefOperations() {
|
||||
TaskDef def = new TaskDef("taskA");
|
||||
def.setDescription("description");
|
||||
def.setCreatedBy("unit_test");
|
||||
def.setCreateTime(1L);
|
||||
def.setInputKeys(Arrays.asList("a", "b", "c"));
|
||||
def.setOutputKeys(Arrays.asList("01", "o2"));
|
||||
def.setOwnerApp("ownerApp");
|
||||
def.setRetryCount(3);
|
||||
def.setRetryDelaySeconds(100);
|
||||
def.setRetryLogic(TaskDef.RetryLogic.FIXED);
|
||||
def.setTimeoutPolicy(TaskDef.TimeoutPolicy.ALERT_ONLY);
|
||||
def.setUpdatedBy("unit_test2");
|
||||
def.setUpdateTime(2L);
|
||||
def.setRateLimitFrequencyInSeconds(1);
|
||||
def.setRateLimitPerFrequency(1);
|
||||
|
||||
metadataDAO.createTaskDef(def);
|
||||
|
||||
TaskDef found = metadataDAO.getTaskDef(def.getName());
|
||||
assertTrue(EqualsBuilder.reflectionEquals(def, found));
|
||||
|
||||
def.setDescription("updated description");
|
||||
metadataDAO.updateTaskDef(def);
|
||||
found = metadataDAO.getTaskDef(def.getName());
|
||||
assertTrue(EqualsBuilder.reflectionEquals(def, found));
|
||||
assertEquals("updated description", found.getDescription());
|
||||
|
||||
for (int i = 0; i < 9; i++) {
|
||||
TaskDef tdf = new TaskDef("taskA" + i);
|
||||
metadataDAO.createTaskDef(tdf);
|
||||
}
|
||||
|
||||
List<TaskDef> all = metadataDAO.getAllTaskDefs();
|
||||
assertNotNull(all);
|
||||
assertEquals(10, all.size());
|
||||
Set<String> allnames = all.stream().map(TaskDef::getName).collect(Collectors.toSet());
|
||||
assertEquals(10, allnames.size());
|
||||
List<String> sorted = allnames.stream().sorted().collect(Collectors.toList());
|
||||
assertEquals(def.getName(), sorted.get(0));
|
||||
|
||||
for (int i = 0; i < 9; i++) {
|
||||
assertEquals(def.getName() + i, sorted.get(i + 1));
|
||||
}
|
||||
|
||||
for (int i = 0; i < 9; i++) {
|
||||
metadataDAO.removeTaskDef(def.getName() + i);
|
||||
}
|
||||
all = metadataDAO.getAllTaskDefs();
|
||||
assertNotNull(all);
|
||||
assertEquals(1, all.size());
|
||||
assertEquals(def.getName(), all.get(0).getName());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testRemoveNotExistingTaskDef() {
|
||||
NonTransientException applicationException =
|
||||
assertThrows(
|
||||
NonTransientException.class,
|
||||
() -> metadataDAO.removeTaskDef("test" + UUID.randomUUID().toString()));
|
||||
assertEquals("No such task definition", applicationException.getMessage());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testEventHandlers() {
|
||||
String event1 = "SQS::arn:account090:sqstest1";
|
||||
String event2 = "SQS::arn:account090:sqstest2";
|
||||
|
||||
EventHandler eventHandler = new EventHandler();
|
||||
eventHandler.setName(UUID.randomUUID().toString());
|
||||
eventHandler.setActive(false);
|
||||
EventHandler.Action action = new EventHandler.Action();
|
||||
action.setAction(EventHandler.Action.Type.start_workflow);
|
||||
action.setStart_workflow(new EventHandler.StartWorkflow());
|
||||
action.getStart_workflow().setName("workflow_x");
|
||||
eventHandler.getActions().add(action);
|
||||
eventHandler.setEvent(event1);
|
||||
|
||||
metadataDAO.addEventHandler(eventHandler);
|
||||
List<EventHandler> all = metadataDAO.getAllEventHandlers();
|
||||
assertNotNull(all);
|
||||
assertEquals(1, all.size());
|
||||
assertEquals(eventHandler.getName(), all.get(0).getName());
|
||||
assertEquals(eventHandler.getEvent(), all.get(0).getEvent());
|
||||
|
||||
List<EventHandler> byEvents = metadataDAO.getEventHandlersForEvent(event1, true);
|
||||
assertNotNull(byEvents);
|
||||
assertEquals(0, byEvents.size()); // event is marked as in-active
|
||||
|
||||
eventHandler.setActive(true);
|
||||
eventHandler.setEvent(event2);
|
||||
metadataDAO.updateEventHandler(eventHandler);
|
||||
|
||||
all = metadataDAO.getAllEventHandlers();
|
||||
assertNotNull(all);
|
||||
assertEquals(1, all.size());
|
||||
|
||||
byEvents = metadataDAO.getEventHandlersForEvent(event1, true);
|
||||
assertNotNull(byEvents);
|
||||
assertEquals(0, byEvents.size());
|
||||
|
||||
byEvents = metadataDAO.getEventHandlersForEvent(event2, true);
|
||||
assertNotNull(byEvents);
|
||||
assertEquals(1, byEvents.size());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void testGetAllWorkflowDefsLatestVersions() {
|
||||
WorkflowDef def = new WorkflowDef();
|
||||
def.setName("test1");
|
||||
def.setVersion(1);
|
||||
def.setDescription("description");
|
||||
def.setCreatedBy("unit_test");
|
||||
def.setCreateTime(1L);
|
||||
def.setOwnerApp("ownerApp");
|
||||
def.setUpdatedBy("unit_test2");
|
||||
def.setUpdateTime(2L);
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
|
||||
def.setName("test2");
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
def.setVersion(2);
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
|
||||
def.setName("test3");
|
||||
def.setVersion(1);
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
def.setVersion(2);
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
def.setVersion(3);
|
||||
metadataDAO.createWorkflowDef(def);
|
||||
|
||||
// Placed the values in a map because they might not be stored in order of defName.
|
||||
// To test, needed to confirm that the versions are correct for the definitions.
|
||||
Map<String, WorkflowDef> allMap =
|
||||
metadataDAO.getAllWorkflowDefsLatestVersions().stream()
|
||||
.collect(Collectors.toMap(WorkflowDef::getName, Function.identity()));
|
||||
|
||||
assertNotNull(allMap);
|
||||
assertEquals(4, allMap.size());
|
||||
assertEquals(1, allMap.get("test1").getVersion());
|
||||
assertEquals(2, allMap.get("test2").getVersion());
|
||||
assertEquals(3, allMap.get("test3").getVersion());
|
||||
}
|
||||
}
|
||||
+201
@@ -0,0 +1,201 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.sql.SQLException;
|
||||
import java.util.Collections;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.Before;
|
||||
import org.junit.Test;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.TestPropertySource;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
|
||||
import com.netflix.conductor.common.metadata.tasks.PollData;
|
||||
import com.netflix.conductor.dao.PollDataDAO;
|
||||
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
|
||||
import com.netflix.conductor.sqlite.util.Query;
|
||||
|
||||
import com.fasterxml.jackson.core.JsonProcessingException;
|
||||
import com.fasterxml.jackson.databind.JsonNode;
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
|
||||
import static org.junit.Assert.*;
|
||||
import static org.junit.Assert.assertTrue;
|
||||
|
||||
@ContextConfiguration(
|
||||
classes = {
|
||||
TestObjectMapperConfiguration.class,
|
||||
SqliteConfiguration.class,
|
||||
FlywayAutoConfiguration.class
|
||||
})
|
||||
@RunWith(SpringRunner.class)
|
||||
@TestPropertySource(
|
||||
properties = {
|
||||
"conductor.app.asyncIndexingEnabled=false",
|
||||
"conductor.elasticsearch.version=0",
|
||||
"conductor.indexing.type=sqlite",
|
||||
"spring.flyway.clean-disabled=false"
|
||||
})
|
||||
@SpringBootTest
|
||||
public class SqlitePollDataTest {
|
||||
|
||||
@Autowired private PollDataDAO pollDataDAO;
|
||||
|
||||
@Autowired private ObjectMapper objectMapper;
|
||||
|
||||
@Qualifier("dataSource")
|
||||
@Autowired
|
||||
private DataSource dataSource;
|
||||
|
||||
@Autowired Flyway flyway;
|
||||
|
||||
// clean the database between tests.
|
||||
@Before
|
||||
public void before() {
|
||||
try (Connection conn = dataSource.getConnection()) {
|
||||
conn.setAutoCommit(true);
|
||||
conn.prepareStatement("delete from poll_data").executeUpdate();
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
private List<Map<String, Object>> queryDb(String query) throws SQLException {
|
||||
try (Connection c = dataSource.getConnection()) {
|
||||
try (Query q = new Query(objectMapper, c, query)) {
|
||||
return q.executeAndFetchMap();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void updateLastPollDataTest() throws SQLException, JsonProcessingException {
|
||||
pollDataDAO.updateLastPollData("dummy-task", "dummy-domain", "dummy-worker-id");
|
||||
|
||||
List<Map<String, Object>> records =
|
||||
queryDb("SELECT * FROM poll_data WHERE queue_name = 'dummy-task'");
|
||||
|
||||
assertEquals("More than one poll data records returned", 1, records.size());
|
||||
assertEquals("Wrong domain set", "dummy-domain", records.get(0).get("domain"));
|
||||
|
||||
JsonNode jsonData = objectMapper.readTree(records.get(0).get("json_data").toString());
|
||||
assertEquals(
|
||||
"Poll data is incorrect", "dummy-worker-id", jsonData.get("workerId").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void updateLastPollDataNullDomainTest() throws SQLException, JsonProcessingException {
|
||||
pollDataDAO.updateLastPollData("dummy-task", null, "dummy-worker-id");
|
||||
|
||||
List<Map<String, Object>> records =
|
||||
queryDb("SELECT * FROM poll_data WHERE queue_name = 'dummy-task'");
|
||||
|
||||
assertEquals("More than one poll data records returned", 1, records.size());
|
||||
assertEquals("Wrong domain set", "DEFAULT", records.get(0).get("domain"));
|
||||
|
||||
JsonNode jsonData = objectMapper.readTree(records.get(0).get("json_data").toString());
|
||||
assertEquals(
|
||||
"Poll data is incorrect", "dummy-worker-id", jsonData.get("workerId").asText());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getPollDataByDomainTest() {
|
||||
pollDataDAO.updateLastPollData("dummy-task", "dummy-domain", "dummy-worker-id");
|
||||
|
||||
PollData pollData = pollDataDAO.getPollData("dummy-task", "dummy-domain");
|
||||
assertEquals("dummy-task", pollData.getQueueName());
|
||||
assertEquals("dummy-domain", pollData.getDomain());
|
||||
assertEquals("dummy-worker-id", pollData.getWorkerId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getPollDataByNullDomainTest() {
|
||||
pollDataDAO.updateLastPollData("dummy-task", null, "dummy-worker-id");
|
||||
|
||||
PollData pollData = pollDataDAO.getPollData("dummy-task", null);
|
||||
assertEquals("dummy-task", pollData.getQueueName());
|
||||
assertNull(pollData.getDomain());
|
||||
assertEquals("dummy-worker-id", pollData.getWorkerId());
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getPollDataByTaskTest() {
|
||||
pollDataDAO.updateLastPollData("dummy-task1", "domain1", "dummy-worker-id1");
|
||||
pollDataDAO.updateLastPollData("dummy-task1", "domain2", "dummy-worker-id2");
|
||||
pollDataDAO.updateLastPollData("dummy-task1", null, "dummy-worker-id3");
|
||||
pollDataDAO.updateLastPollData("dummy-task2", "domain2", "dummy-worker-id4");
|
||||
|
||||
List<PollData> pollData = pollDataDAO.getPollData("dummy-task1");
|
||||
assertEquals("Wrong number of records returned", 3, pollData.size());
|
||||
|
||||
List<String> queueNames =
|
||||
pollData.stream().map(x -> x.getQueueName()).collect(Collectors.toList());
|
||||
assertEquals(3, Collections.frequency(queueNames, "dummy-task1"));
|
||||
|
||||
List<String> domains =
|
||||
pollData.stream().map(x -> x.getDomain()).collect(Collectors.toList());
|
||||
assertTrue(domains.contains("domain1"));
|
||||
assertTrue(domains.contains("domain2"));
|
||||
assertTrue(domains.contains(null));
|
||||
|
||||
List<String> workerIds =
|
||||
pollData.stream().map(x -> x.getWorkerId()).collect(Collectors.toList());
|
||||
assertTrue(workerIds.contains("dummy-worker-id1"));
|
||||
assertTrue(workerIds.contains("dummy-worker-id2"));
|
||||
assertTrue(workerIds.contains("dummy-worker-id3"));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void getAllPollDataTest() {
|
||||
pollDataDAO.updateLastPollData("dummy-task1", "domain1", "dummy-worker-id1");
|
||||
pollDataDAO.updateLastPollData("dummy-task1", "domain2", "dummy-worker-id2");
|
||||
pollDataDAO.updateLastPollData("dummy-task1", null, "dummy-worker-id3");
|
||||
pollDataDAO.updateLastPollData("dummy-task2", "domain2", "dummy-worker-id4");
|
||||
|
||||
List<PollData> pollData = pollDataDAO.getAllPollData();
|
||||
assertEquals("Wrong number of records returned", 4, pollData.size());
|
||||
|
||||
List<String> queueNames =
|
||||
pollData.stream().map(x -> x.getQueueName()).collect(Collectors.toList());
|
||||
assertEquals(3, Collections.frequency(queueNames, "dummy-task1"));
|
||||
assertEquals(1, Collections.frequency(queueNames, "dummy-task2"));
|
||||
|
||||
List<String> domains =
|
||||
pollData.stream().map(x -> x.getDomain()).collect(Collectors.toList());
|
||||
assertEquals(1, Collections.frequency(domains, "domain1"));
|
||||
assertEquals(2, Collections.frequency(domains, "domain2"));
|
||||
assertEquals(1, Collections.frequency(domains, null));
|
||||
|
||||
List<String> workerIds =
|
||||
pollData.stream().map(x -> x.getWorkerId()).collect(Collectors.toList());
|
||||
assertTrue(workerIds.contains("dummy-worker-id1"));
|
||||
assertTrue(workerIds.contains("dummy-worker-id2"));
|
||||
assertTrue(workerIds.contains("dummy-worker-id3"));
|
||||
assertTrue(workerIds.contains("dummy-worker-id4"));
|
||||
}
|
||||
}
|
||||
+505
@@ -0,0 +1,505 @@
|
||||
/*
|
||||
* Copyright 2025 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.dao;
|
||||
|
||||
import java.sql.Connection;
|
||||
import java.util.ArrayList;
|
||||
import java.util.Arrays;
|
||||
import java.util.List;
|
||||
import java.util.Map;
|
||||
import java.util.concurrent.TimeUnit;
|
||||
import java.util.stream.Collectors;
|
||||
|
||||
import javax.sql.DataSource;
|
||||
|
||||
import org.flywaydb.core.Flyway;
|
||||
import org.junit.Before;
|
||||
import org.junit.Rule;
|
||||
import org.junit.Test;
|
||||
import org.junit.rules.TestName;
|
||||
import org.junit.runner.RunWith;
|
||||
import org.slf4j.Logger;
|
||||
import org.slf4j.LoggerFactory;
|
||||
import org.springframework.beans.factory.annotation.Autowired;
|
||||
import org.springframework.beans.factory.annotation.Qualifier;
|
||||
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
|
||||
import org.springframework.boot.test.context.SpringBootTest;
|
||||
import org.springframework.test.context.ContextConfiguration;
|
||||
import org.springframework.test.context.junit4.SpringRunner;
|
||||
|
||||
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
|
||||
import com.netflix.conductor.core.events.queue.Message;
|
||||
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
|
||||
import com.netflix.conductor.sqlite.util.Query;
|
||||
|
||||
import com.fasterxml.jackson.databind.ObjectMapper;
|
||||
import com.google.common.collect.ImmutableList;
|
||||
|
||||
import static org.awaitility.Awaitility.await;
|
||||
import static org.junit.Assert.*;
|
||||
import static org.junit.Assert.assertNotNull;
|
||||
|
||||
@ContextConfiguration(
|
||||
classes = {
|
||||
TestObjectMapperConfiguration.class,
|
||||
SqliteConfiguration.class,
|
||||
FlywayAutoConfiguration.class
|
||||
})
|
||||
@RunWith(SpringRunner.class)
|
||||
@SpringBootTest(properties = "spring.flyway.clean-disabled=false")
|
||||
public class SqliteQueueDAOTest {
|
||||
|
||||
private static final Logger LOGGER = LoggerFactory.getLogger(SqliteQueueDAOTest.class);
|
||||
|
||||
@Autowired private SqliteQueueDAO queueDAO;
|
||||
|
||||
@Qualifier("dataSource")
|
||||
@Autowired
|
||||
private DataSource dataSource;
|
||||
|
||||
@Autowired private ObjectMapper objectMapper;
|
||||
|
||||
@Rule public TestName name = new TestName();
|
||||
|
||||
@Autowired Flyway flyway;
|
||||
|
||||
@Before
|
||||
public void before() {
|
||||
try (Connection conn = dataSource.getConnection()) {
|
||||
conn.setAutoCommit(true);
|
||||
String[] stmts = new String[] {"delete from queue;", "delete from queue_message;"};
|
||||
for (String stmt : stmts) {
|
||||
conn.prepareStatement(stmt).executeUpdate();
|
||||
}
|
||||
} catch (Exception e) {
|
||||
e.printStackTrace();
|
||||
throw new RuntimeException(e);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void complexQueueTest() {
|
||||
String queueName = "TestQueue";
|
||||
long offsetTimeInSecond = 0;
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String messageId = "msg" + i;
|
||||
queueDAO.push(queueName, messageId, offsetTimeInSecond);
|
||||
}
|
||||
int size = queueDAO.getSize(queueName);
|
||||
assertEquals(10, size);
|
||||
Map<String, Long> details = queueDAO.queuesDetail();
|
||||
assertEquals(1, details.size());
|
||||
assertEquals(10L, details.get(queueName).longValue());
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String messageId = "msg" + i;
|
||||
queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond);
|
||||
}
|
||||
|
||||
List<String> popped = queueDAO.pop(queueName, 10, 100);
|
||||
assertNotNull(popped);
|
||||
assertEquals(10, popped.size());
|
||||
|
||||
Map<String, Map<String, Map<String, Long>>> verbose = queueDAO.queuesDetailVerbose();
|
||||
assertEquals(1, verbose.size());
|
||||
long shardSize = verbose.get(queueName).get("a").get("size");
|
||||
long unackedSize = verbose.get(queueName).get("a").get("uacked");
|
||||
assertEquals(0, shardSize);
|
||||
assertEquals(10, unackedSize);
|
||||
|
||||
popped.forEach(messageId -> queueDAO.ack(queueName, messageId));
|
||||
|
||||
verbose = queueDAO.queuesDetailVerbose();
|
||||
assertEquals(1, verbose.size());
|
||||
shardSize = verbose.get(queueName).get("a").get("size");
|
||||
unackedSize = verbose.get(queueName).get("a").get("uacked");
|
||||
assertEquals(0, shardSize);
|
||||
assertEquals(0, unackedSize);
|
||||
|
||||
popped = queueDAO.pop(queueName, 10, 100);
|
||||
assertNotNull(popped);
|
||||
assertEquals(0, popped.size());
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String messageId = "msg" + i;
|
||||
queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond);
|
||||
}
|
||||
size = queueDAO.getSize(queueName);
|
||||
assertEquals(10, size);
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String messageId = "msg" + i;
|
||||
assertTrue(queueDAO.containsMessage(queueName, messageId));
|
||||
queueDAO.remove(queueName, messageId);
|
||||
}
|
||||
|
||||
size = queueDAO.getSize(queueName);
|
||||
assertEquals(0, size);
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String messageId = "msg" + i;
|
||||
queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond);
|
||||
}
|
||||
queueDAO.flush(queueName);
|
||||
size = queueDAO.getSize(queueName);
|
||||
assertEquals(0, size);
|
||||
}
|
||||
|
||||
/**
|
||||
* Test fix for https://github.com/Netflix/conductor/issues/399
|
||||
*
|
||||
* @since 1.8.2-rc5
|
||||
*/
|
||||
@Test
|
||||
public void pollMessagesTest() {
|
||||
final List<Message> messages = new ArrayList<>();
|
||||
final String queueName = "issue399_testQueue";
|
||||
final int totalSize = 10;
|
||||
|
||||
for (int i = 0; i < totalSize; i++) {
|
||||
String payload = "{\"id\": " + i + ", \"msg\":\"test " + i + "\"}";
|
||||
Message m = new Message("testmsg-" + i, payload, "");
|
||||
if (i % 2 == 0) {
|
||||
// Set priority on message with pair id
|
||||
m.setPriority(99 - i);
|
||||
}
|
||||
messages.add(m);
|
||||
}
|
||||
|
||||
// Populate the queue with our test message batch
|
||||
queueDAO.push(queueName, ImmutableList.copyOf(messages));
|
||||
|
||||
// Assert that all messages were persisted and no extras are in there
|
||||
assertEquals("Queue size mismatch", totalSize, queueDAO.getSize(queueName));
|
||||
|
||||
List<Message> zeroPoll = queueDAO.pollMessages(queueName, 0, 10_000);
|
||||
assertTrue("Zero poll should be empty", zeroPoll.isEmpty());
|
||||
|
||||
final int firstPollSize = 3;
|
||||
List<Message> firstPoll = queueDAO.pollMessages(queueName, firstPollSize, 10_000);
|
||||
assertNotNull("First poll was null", firstPoll);
|
||||
assertFalse("First poll was empty", firstPoll.isEmpty());
|
||||
assertEquals("First poll size mismatch", firstPollSize, firstPoll.size());
|
||||
|
||||
final int secondPollSize = 4;
|
||||
List<Message> secondPoll = queueDAO.pollMessages(queueName, secondPollSize, 10_000);
|
||||
assertNotNull("Second poll was null", secondPoll);
|
||||
assertFalse("Second poll was empty", secondPoll.isEmpty());
|
||||
assertEquals("Second poll size mismatch", secondPollSize, secondPoll.size());
|
||||
|
||||
// Assert that the total queue size hasn't changed
|
||||
assertEquals(
|
||||
"Total queue size should have remained the same",
|
||||
totalSize,
|
||||
queueDAO.getSize(queueName));
|
||||
|
||||
// Assert that our un-popped messages match our expected size
|
||||
final long expectedSize = totalSize - firstPollSize - secondPollSize;
|
||||
try (Connection c = dataSource.getConnection()) {
|
||||
String UNPOPPED =
|
||||
"SELECT COUNT(*) FROM queue_message WHERE queue_name = ? AND popped = false";
|
||||
try (Query q = new Query(objectMapper, c, UNPOPPED)) {
|
||||
long count = q.addParameter(queueName).executeCount();
|
||||
assertEquals("Remaining queue size mismatch", expectedSize, count);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
fail(ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
/** Test fix for https://github.com/Netflix/conductor/issues/1892 */
|
||||
@Test
|
||||
public void containsMessageTest() {
|
||||
String queueName = "TestQueue";
|
||||
long offsetTimeInSecond = 0;
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String messageId = "msg" + i;
|
||||
queueDAO.push(queueName, messageId, offsetTimeInSecond);
|
||||
}
|
||||
int size = queueDAO.getSize(queueName);
|
||||
assertEquals(10, size);
|
||||
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String messageId = "msg" + i;
|
||||
assertTrue(queueDAO.containsMessage(queueName, messageId));
|
||||
queueDAO.remove(queueName, messageId);
|
||||
}
|
||||
for (int i = 0; i < 10; i++) {
|
||||
String messageId = "msg" + i;
|
||||
assertFalse(queueDAO.containsMessage(queueName, messageId));
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Test fix for https://github.com/Netflix/conductor/issues/448
|
||||
*
|
||||
* @since 1.8.2-rc5
|
||||
*/
|
||||
@Test
|
||||
public void pollDeferredMessagesTest() {
|
||||
final List<Message> messages = new ArrayList<>();
|
||||
final String queueName = "issue448_testQueue";
|
||||
final int totalSize = 10;
|
||||
|
||||
for (int i = 0; i < totalSize; i++) {
|
||||
int offset = 0;
|
||||
if (i < 5) {
|
||||
offset = 0;
|
||||
} else if (i == 6 || i == 7) {
|
||||
// Purposefully skipping id:5 to test out of order deliveries
|
||||
// Set id:6 and id:7 for a 2s delay to be picked up in the second polling batch
|
||||
offset = 5;
|
||||
} else {
|
||||
// Set all other queue messages to have enough of a delay that they won't
|
||||
// accidentally
|
||||
// be picked up.
|
||||
offset = 10_000 + i;
|
||||
}
|
||||
|
||||
String payload = "{\"id\": " + i + ",\"offset_time_seconds\":" + offset + "}";
|
||||
Message m = new Message("testmsg-" + i, payload, "");
|
||||
messages.add(m);
|
||||
queueDAO.push(queueName, "testmsg-" + i, offset);
|
||||
}
|
||||
|
||||
// Assert that all messages were persisted and no extras are in there
|
||||
assertEquals("Queue size mismatch", totalSize, queueDAO.getSize(queueName));
|
||||
|
||||
final int firstPollSize = 4;
|
||||
List<Message> firstPoll = queueDAO.pollMessages(queueName, firstPollSize, 100);
|
||||
assertNotNull("First poll was null", firstPoll);
|
||||
assertFalse("First poll was empty", firstPoll.isEmpty());
|
||||
assertEquals("First poll size mismatch", firstPollSize, firstPoll.size());
|
||||
|
||||
List<String> firstPollMessageIds =
|
||||
messages.stream()
|
||||
.map(Message::getId)
|
||||
.collect(Collectors.toList())
|
||||
.subList(0, firstPollSize + 1);
|
||||
|
||||
for (int i = 0; i < firstPollSize; i++) {
|
||||
String actual = firstPoll.get(i).getId();
|
||||
assertTrue("Unexpected Id: " + actual, firstPollMessageIds.contains(actual));
|
||||
}
|
||||
|
||||
final int secondPollSize = 3;
|
||||
|
||||
// Wait for delayed messages to become available for polling
|
||||
final String COUNT_READY =
|
||||
"SELECT COUNT(*) FROM queue_message WHERE queue_name = ? AND popped = false AND deliver_on <= strftime('%Y-%m-%d %H:%M:%f', 'now')";
|
||||
await().atMost(10, TimeUnit.SECONDS)
|
||||
.pollInterval(500, TimeUnit.MILLISECONDS)
|
||||
.until(
|
||||
() -> {
|
||||
try (Connection c = dataSource.getConnection()) {
|
||||
try (Query q = new Query(objectMapper, c, COUNT_READY)) {
|
||||
return q.addParameter(queueName).executeCount()
|
||||
>= secondPollSize;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
// Poll for many more messages than expected
|
||||
List<Message> secondPoll = queueDAO.pollMessages(queueName, secondPollSize + 10, 100);
|
||||
assertNotNull("Second poll was null", secondPoll);
|
||||
assertFalse("Second poll was empty", secondPoll.isEmpty());
|
||||
assertEquals("Second poll size mismatch", secondPollSize, secondPoll.size());
|
||||
|
||||
List<String> expectedIds = Arrays.asList("testmsg-4", "testmsg-6", "testmsg-7");
|
||||
for (int i = 0; i < secondPollSize; i++) {
|
||||
String actual = secondPoll.get(i).getId();
|
||||
assertTrue("Unexpected Id: " + actual, expectedIds.contains(actual));
|
||||
}
|
||||
|
||||
// Assert that the total queue size hasn't changed
|
||||
assertEquals(
|
||||
"Total queue size should have remained the same",
|
||||
totalSize,
|
||||
queueDAO.getSize(queueName));
|
||||
|
||||
// Assert that our un-popped messages match our expected size
|
||||
final long expectedSize = totalSize - firstPollSize - secondPollSize;
|
||||
try (Connection c = dataSource.getConnection()) {
|
||||
String UNPOPPED =
|
||||
"SELECT COUNT(*) FROM queue_message WHERE queue_name = ? AND popped = false";
|
||||
try (Query q = new Query(objectMapper, c, UNPOPPED)) {
|
||||
long count = q.addParameter(queueName).executeCount();
|
||||
assertEquals("Remaining queue size mismatch", expectedSize, count);
|
||||
}
|
||||
} catch (Exception ex) {
|
||||
fail(ex.getMessage());
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setUnackTimeoutIfShorterShortensDueTime() {
|
||||
String queueName = "setUnackIfShorter_shorten";
|
||||
String messageId = "msg-shorten";
|
||||
|
||||
// Push with a 1-hour delay so it is not immediately poppable
|
||||
queueDAO.push(queueName, messageId, 3600L);
|
||||
assertEquals(0, queueDAO.pop(queueName, 1, 100).size());
|
||||
|
||||
// Shorten delivery to now (0 s) — should succeed and return true
|
||||
boolean shortened = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 0L);
|
||||
assertTrue(
|
||||
"setUnackTimeoutIfShorter should return true when it actually shortens", shortened);
|
||||
|
||||
// Message must now be immediately poppable
|
||||
List<String> popped = queueDAO.pop(queueName, 1, 100);
|
||||
assertEquals(1, popped.size());
|
||||
assertEquals(messageId, popped.get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setUnackTimeoutIfShorterDoesNotExtend() {
|
||||
String queueName = "setUnackIfShorter_noextend";
|
||||
String messageId = "msg-noextend";
|
||||
|
||||
// Push with offset 0 so it is immediately available
|
||||
queueDAO.push(queueName, messageId, 0L);
|
||||
|
||||
// Try to push deliver_on far into the future — must be rejected
|
||||
boolean extended = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 3_600_000L);
|
||||
assertFalse(
|
||||
"setUnackTimeoutIfShorter must not extend an already-closer delivery time",
|
||||
extended);
|
||||
|
||||
// Message must still be immediately poppable
|
||||
List<String> popped = queueDAO.pop(queueName, 1, 100);
|
||||
assertEquals(1, popped.size());
|
||||
assertEquals(messageId, popped.get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setUnackTimeoutIfShorterReturnsFalseForNonExistent() {
|
||||
String queueName = "setUnackIfShorter_nonexistent";
|
||||
boolean updated = queueDAO.setUnackTimeoutIfShorter(queueName, "no-such-message", 0L);
|
||||
assertFalse(
|
||||
"setUnackTimeoutIfShorter must return false for a non-existent message", updated);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setUnackTimeoutIfShorterReturnsFalseWhenEqualTimeout() {
|
||||
String queueName = "setUnackIfShorter_equal";
|
||||
String messageId = "msg-equal";
|
||||
queueDAO.push(queueName, messageId, 0L);
|
||||
|
||||
boolean result = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 0L);
|
||||
assertFalse("Equal timeout must not update deliver_on — returns false", result);
|
||||
|
||||
List<String> popped = queueDAO.pop(queueName, 1, 100);
|
||||
assertEquals(1, popped.size());
|
||||
assertEquals(messageId, popped.get(0));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void setUnackTimeoutIfShorterRejectsExtensionThenAcceptsShortening() {
|
||||
String queueName = "setUnackIfShorter_reject_then_accept";
|
||||
String messageId = "msg-reject-accept";
|
||||
queueDAO.push(queueName, messageId, 3600L);
|
||||
|
||||
boolean extended = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 7_200_000L);
|
||||
assertFalse("Extending must return false", extended);
|
||||
|
||||
boolean shortened = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 0L);
|
||||
assertTrue("Shortening to now must return true", shortened);
|
||||
|
||||
List<String> popped = queueDAO.pop(queueName, 1, 100);
|
||||
assertEquals(1, popped.size());
|
||||
assertEquals(messageId, popped.get(0));
|
||||
}
|
||||
|
||||
// @Test
|
||||
public void processUnacksTest() {
|
||||
processUnacks(
|
||||
() -> {
|
||||
// Process unacks
|
||||
queueDAO.processUnacks("process_unacks_test");
|
||||
},
|
||||
"process_unacks_test");
|
||||
}
|
||||
|
||||
// @Test
|
||||
public void processAllUnacksTest() {
|
||||
processUnacks(
|
||||
() -> {
|
||||
// Process all unacks
|
||||
queueDAO.processAllUnacks();
|
||||
},
|
||||
"process_unacks_test");
|
||||
}
|
||||
|
||||
private void processUnacks(Runnable unack, String queueName) {
|
||||
// Count of messages in the queue(s)
|
||||
final int count = 10;
|
||||
// Number of messages to process acks for
|
||||
final int unackedCount = 4;
|
||||
// A secondary queue to make sure we don't accidentally process other queues
|
||||
final String otherQueueName = "process_unacks_test_other_queue";
|
||||
|
||||
// Create testing queue with some messages (but not all) that will be popped/acked.
|
||||
for (int i = 0; i < count; i++) {
|
||||
int offset = 0;
|
||||
if (i >= unackedCount) {
|
||||
offset = 1_000_000;
|
||||
}
|
||||
|
||||
queueDAO.push(queueName, "unack-" + i, offset);
|
||||
}
|
||||
|
||||
// Create a second queue to make sure that unacks don't occur for it
|
||||
for (int i = 0; i < count; i++) {
|
||||
queueDAO.push(otherQueueName, "other-" + i, 0);
|
||||
}
|
||||
|
||||
// Poll for first batch of messages (should be equal to unackedCount)
|
||||
List<Message> polled = queueDAO.pollMessages(queueName, 100, 10_000);
|
||||
assertNotNull(polled);
|
||||
assertFalse(polled.isEmpty());
|
||||
assertEquals(unackedCount, polled.size());
|
||||
|
||||
// Poll messages from the other queue so we know they don't get unacked later
|
||||
queueDAO.pollMessages(otherQueueName, 100, 10_000);
|
||||
|
||||
// Ack one of the polled messages
|
||||
assertTrue(queueDAO.ack(queueName, "unack-1"));
|
||||
|
||||
// Should have one less un-acked popped message in the queue
|
||||
Long uacked = queueDAO.queuesDetailVerbose().get(queueName).get("a").get("uacked");
|
||||
assertNotNull(uacked);
|
||||
assertEquals(uacked.longValue(), unackedCount - 1);
|
||||
|
||||
unack.run();
|
||||
|
||||
// Check uacks for both queues after processing
|
||||
Map<String, Map<String, Map<String, Long>>> details = queueDAO.queuesDetailVerbose();
|
||||
uacked = details.get(queueName).get("a").get("uacked");
|
||||
assertNotNull(uacked);
|
||||
assertEquals(
|
||||
"The messages that were polled should be unacked still",
|
||||
uacked.longValue(),
|
||||
unackedCount - 1);
|
||||
|
||||
Long otherUacked = details.get(otherQueueName).get("a").get("uacked");
|
||||
assertNotNull(otherUacked);
|
||||
assertEquals(
|
||||
"Other queue should have all unacked messages", otherUacked.longValue(), count);
|
||||
|
||||
Long size = queueDAO.queuesDetail().get(queueName);
|
||||
assertNotNull(size);
|
||||
assertEquals(size.longValue(), count - unackedCount);
|
||||
}
|
||||
}
|
||||
+214
@@ -0,0 +1,214 @@
|
||||
/*
|
||||
* Copyright 2023 Conductor Authors.
|
||||
* <p>
|
||||
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
|
||||
* the License. You may obtain a copy of the License at
|
||||
* <p>
|
||||
* http://www.apache.org/licenses/LICENSE-2.0
|
||||
* <p>
|
||||
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
|
||||
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
|
||||
* specific language governing permissions and limitations under the License.
|
||||
*/
|
||||
package com.netflix.conductor.sqlite.util;
|
||||
|
||||
import java.sql.SQLException;
|
||||
import java.util.ArrayList;
|
||||
|
||||
import org.junit.jupiter.api.Test;
|
||||
import org.mockito.InOrder;
|
||||
import org.mockito.Mockito;
|
||||
|
||||
import com.netflix.conductor.sqlite.config.SqliteProperties;
|
||||
|
||||
import static org.junit.jupiter.api.Assertions.assertEquals;
|
||||
import static org.mockito.Mockito.*;
|
||||
|
||||
public class SqliteIndexQueryBuilderTest {
|
||||
|
||||
private SqliteProperties properties = new SqliteProperties();
|
||||
|
||||
@Test
|
||||
void shouldGenerateQueryForEmptyString() throws SQLException {
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", "", "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals("SELECT json_data FROM table_name LIMIT ? OFFSET ?", generatedQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGenerateQueryForExactMatch() throws SQLException {
|
||||
String inputQuery = "workflowId=\"abc123\"";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE workflow_id = ? LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
Query mockQuery = mock(Query.class);
|
||||
builder.addParameters(mockQuery);
|
||||
builder.addPagingParameters(mockQuery);
|
||||
InOrder inOrder = Mockito.inOrder(mockQuery);
|
||||
inOrder.verify(mockQuery).addParameter("abc123");
|
||||
inOrder.verify(mockQuery).addParameter(15);
|
||||
inOrder.verify(mockQuery).addParameter(0);
|
||||
verifyNoMoreInteractions(mockQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGenerateQueryWithWildcardPrefix() throws SQLException {
|
||||
String inputQuery = "workflowType=abc*";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE lower(workflow_type) LIKE lower(?) LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
Query mockQuery = mock(Query.class);
|
||||
builder.addParameters(mockQuery);
|
||||
builder.addPagingParameters(mockQuery);
|
||||
InOrder inOrder = Mockito.inOrder(mockQuery);
|
||||
inOrder.verify(mockQuery).addParameter("abc%");
|
||||
inOrder.verify(mockQuery).addParameter(15);
|
||||
inOrder.verify(mockQuery).addParameter(0);
|
||||
verifyNoMoreInteractions(mockQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGenerateQueryWithWildcardContains() throws SQLException {
|
||||
String inputQuery = "correlationId=\"*order*\"";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE lower(correlation_id) LIKE lower(?) LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
Query mockQuery = mock(Query.class);
|
||||
builder.addParameters(mockQuery);
|
||||
builder.addPagingParameters(mockQuery);
|
||||
InOrder inOrder = Mockito.inOrder(mockQuery);
|
||||
inOrder.verify(mockQuery).addParameter("%order%");
|
||||
inOrder.verify(mockQuery).addParameter(15);
|
||||
inOrder.verify(mockQuery).addParameter(0);
|
||||
verifyNoMoreInteractions(mockQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGenerateExactMatchQueryWhenNoWildcard() throws SQLException {
|
||||
String inputQuery = "workflowType=abc";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE workflow_type = ? LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
Query mockQuery = mock(Query.class);
|
||||
builder.addParameters(mockQuery);
|
||||
builder.addPagingParameters(mockQuery);
|
||||
InOrder inOrder = Mockito.inOrder(mockQuery);
|
||||
inOrder.verify(mockQuery).addParameter("abc");
|
||||
inOrder.verify(mockQuery).addParameter(15);
|
||||
inOrder.verify(mockQuery).addParameter(0);
|
||||
verifyNoMoreInteractions(mockQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldNotExpandWildcardInINClause() throws SQLException {
|
||||
String inputQuery = "status IN (COMP*,RUNNING)";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE status IN (?,?) LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGenerateQueryForClassifier() throws SQLException {
|
||||
String inputQuery = "classifier=\"agent\"";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE classifier = ? LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
Query mockQuery = mock(Query.class);
|
||||
builder.addParameters(mockQuery);
|
||||
builder.addPagingParameters(mockQuery);
|
||||
InOrder inOrder = Mockito.inOrder(mockQuery);
|
||||
inOrder.verify(mockQuery).addParameter("agent");
|
||||
inOrder.verify(mockQuery).addParameter(15);
|
||||
inOrder.verify(mockQuery).addParameter(0);
|
||||
verifyNoMoreInteractions(mockQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldMatchLegacyNullRowsForUntaggedClassifier() throws SQLException {
|
||||
// Rows indexed before the classifier column existed are untagged plain workflows;
|
||||
// filtering for the "workflow" token must also match those NULL rows.
|
||||
String inputQuery = "classifier=\"workflow\"";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE (classifier = ? OR classifier IS NULL) LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
Query mockQuery = mock(Query.class);
|
||||
builder.addParameters(mockQuery);
|
||||
builder.addPagingParameters(mockQuery);
|
||||
InOrder inOrder = Mockito.inOrder(mockQuery);
|
||||
inOrder.verify(mockQuery).addParameter("workflow");
|
||||
inOrder.verify(mockQuery).addParameter(15);
|
||||
inOrder.verify(mockQuery).addParameter(0);
|
||||
verifyNoMoreInteractions(mockQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldMatchLegacyNullRowsForUntaggedClassifierInClause() throws SQLException {
|
||||
String inputQuery = "classifier IN (agent,workflow)";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE (classifier IN (?,?) OR classifier IS NULL) LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
Query mockQuery = mock(Query.class);
|
||||
builder.addParameters(mockQuery);
|
||||
builder.addPagingParameters(mockQuery);
|
||||
InOrder inOrder = Mockito.inOrder(mockQuery);
|
||||
inOrder.verify(mockQuery).addParameter("agent");
|
||||
inOrder.verify(mockQuery).addParameter("workflow");
|
||||
inOrder.verify(mockQuery).addParameter(15);
|
||||
inOrder.verify(mockQuery).addParameter(0);
|
||||
verifyNoMoreInteractions(mockQuery);
|
||||
}
|
||||
|
||||
@Test
|
||||
void shouldGenerateQueryForParentWorkflowId() throws SQLException {
|
||||
String inputQuery = "parentWorkflowId=\"\"";
|
||||
SqliteIndexQueryBuilder builder =
|
||||
new SqliteIndexQueryBuilder(
|
||||
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
|
||||
String generatedQuery = builder.getQuery();
|
||||
assertEquals(
|
||||
"SELECT json_data FROM table_name WHERE parent_workflow_id = ? LIMIT ? OFFSET ?",
|
||||
generatedQuery);
|
||||
Query mockQuery = mock(Query.class);
|
||||
builder.addParameters(mockQuery);
|
||||
builder.addPagingParameters(mockQuery);
|
||||
InOrder inOrder = Mockito.inOrder(mockQuery);
|
||||
inOrder.verify(mockQuery).addParameter("");
|
||||
inOrder.verify(mockQuery).addParameter(15);
|
||||
inOrder.verify(mockQuery).addParameter(0);
|
||||
verifyNoMoreInteractions(mockQuery);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
spring.datasource.driver-class-name=org.sqlite.JDBC
|
||||
spring.datasource.url=jdbc:sqlite:conductorosstest.db
|
||||
spring.datasource.username=
|
||||
spring.datasource.password=
|
||||
# Hibernate SQLite Dialect
|
||||
spring.jpa.database-platform=org.hibernate.community.dialect.SQLiteDialect
|
||||
|
||||
# db type
|
||||
conductor.db.type=sqlite
|
||||
Reference in New Issue
Block a user