chore: import upstream snapshot with attribution
CI / unit-test (push) Has been cancelled
CI / detect-changes (push) Has been cancelled
CI / build (push) Has been cancelled
Publish docs via GitHub Pages / Deploy docs (push) Has been cancelled
CI / test-harness (push) Has been cancelled
CI / generate-e2e-matrix (push) Has been cancelled
CI / e2e (push) Has been cancelled
CI / build-ui (push) Has been cancelled
Release Drafter / update_release_draft (push) Has been cancelled
UI v2 Integration CI / E2E (Integration) (push) Has been cancelled
UI v2 CI / Lint, Format & Test (push) Has been cancelled
UI v2 CI / E2E (Mocked) (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:37:56 +08:00
commit a9cd7750f4
3727 changed files with 627706 additions and 0 deletions
@@ -0,0 +1,294 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.config;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.util.Optional;
import javax.sql.DataSource;
import org.conductoross.conductor.sqlite.dao.SqliteFileMetadataDAO;
import org.conductoross.conductor.sqlite.dao.SqliteSkillMetadataDAO;
import org.conductoross.conductor.sqlite.dao.SqliteSkillPackageDAO;
import org.flywaydb.core.Flyway;
import org.flywaydb.core.api.configuration.FluentConfiguration;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
import org.springframework.boot.autoconfigure.jdbc.DataSourceAutoConfiguration;
import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.context.annotation.DependsOn;
import org.springframework.context.annotation.Import;
import org.springframework.retry.RetryContext;
import org.springframework.retry.backoff.ExponentialBackOffPolicy;
import org.springframework.retry.policy.SimpleRetryPolicy;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.core.sync.Lock;
import com.netflix.conductor.core.sync.local.LocalOnlyLock;
import com.netflix.conductor.sqlite.dao.*;
import com.netflix.conductor.sqlite.dao.metadata.SqliteEventHandlerMetadataDAO;
import com.netflix.conductor.sqlite.dao.metadata.SqliteMetadataDAO;
import com.netflix.conductor.sqlite.dao.metadata.SqliteTaskMetadataDAO;
import com.netflix.conductor.sqlite.dao.metadata.SqliteWorkflowMetadataDAO;
import com.fasterxml.jackson.databind.ObjectMapper;
import jakarta.annotation.PostConstruct;
@Configuration(proxyBeanMethods = false)
@EnableConfigurationProperties(SqliteProperties.class)
@ConditionalOnProperty(name = "conductor.db.type", havingValue = "sqlite")
@Import(DataSourceAutoConfiguration.class)
@ConfigurationProperties(prefix = "conductor.sqlite")
public class SqliteConfiguration {
DataSource dataSource;
private final SqliteProperties properties;
public SqliteConfiguration(DataSource dataSource, SqliteProperties properties) {
this.dataSource = dataSource;
this.properties = properties;
}
@PostConstruct
public void initializeSqlite() {
try (Connection conn = dataSource.getConnection();
Statement stmt = conn.createStatement()) {
// Enable WAL mode for better concurrent access
stmt.execute("PRAGMA journal_mode=WAL");
// Set busy timeout to 30 seconds (30000 ms)
// This makes SQLite wait up to 30s when encountering locks
stmt.execute("PRAGMA busy_timeout=30000");
// Enable foreign keys
stmt.execute("PRAGMA foreign_keys=ON");
// Optimize for concurrency
stmt.execute("PRAGMA synchronous=NORMAL");
// Use memory for temporary storage
stmt.execute("PRAGMA temp_store=MEMORY");
} catch (SQLException e) {
throw new RuntimeException("Failed to initialize SQLite database", e);
}
}
@Bean(initMethod = "migrate")
public Flyway flywayForPrimaryDb() {
FluentConfiguration config =
Flyway.configure()
.dataSource(dataSource) // SQLite doesn't need username/password
.locations("classpath:db/migration_sqlite") // Location of migration files
.sqlMigrationPrefix("V") // V1, V2, etc.
.sqlMigrationSeparator("__") // V1__description
.mixed(true) // Allow mixed migrations (both versioned and repeatable)
.validateOnMigrate(true)
.cleanDisabled(false);
return new Flyway(config);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
public SqliteMetadataDAO sqliteMetadataDAO(
SqliteTaskMetadataDAO taskMetadataDAO,
SqliteWorkflowMetadataDAO workflowMetadataDAO,
SqliteEventHandlerMetadataDAO eventHandlerMetadataDAO) {
return new SqliteMetadataDAO(taskMetadataDAO, workflowMetadataDAO, eventHandlerMetadataDAO);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
public SqliteEventHandlerMetadataDAO sqliteEventHandlerMetadataDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper) {
return new SqliteEventHandlerMetadataDAO(retryTemplate, objectMapper, dataSource);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
public SqliteWorkflowMetadataDAO sqliteWorkflowMetadataDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper) {
return new SqliteWorkflowMetadataDAO(retryTemplate, objectMapper, dataSource);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
public SqliteTaskMetadataDAO sqliteTaskMetadataDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper) {
return new SqliteTaskMetadataDAO(retryTemplate, objectMapper, dataSource);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
public SqliteExecutionDAO sqliteExecutionDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper,
SqliteQueueDAO queueDAO) {
return new SqliteExecutionDAO(retryTemplate, objectMapper, dataSource, queueDAO);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
public SqlitePollDataDAO sqlitePollDataDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper) {
return new SqlitePollDataDAO(retryTemplate, objectMapper, dataSource);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
public SqliteQueueDAO sqliteQueueDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper,
SqliteProperties properties) {
return new SqliteQueueDAO(retryTemplate, objectMapper, dataSource, properties);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
@ConditionalOnProperty(name = "conductor.indexing.type", havingValue = "sqlite")
public SqliteIndexDAO sqliteIndexDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper,
SqliteProperties properties) {
return new SqliteIndexDAO(retryTemplate, objectMapper, dataSource, properties);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
@ConditionalOnProperty(name = "conductor.workflow-execution-lock.type", havingValue = "sqlite")
public Lock sqliteLockDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper) {
return new LocalOnlyLock();
}
@Bean
public RetryTemplate sqliteRetryTemplate(SqliteProperties properties) {
CustomRetryPolicy retryPolicy = new CustomRetryPolicy();
retryPolicy.setMaxAttempts(10); // Increased for SQLite locking scenarios
ExponentialBackOffPolicy backOffPolicy = new ExponentialBackOffPolicy();
backOffPolicy.setInitialInterval(50L); // Start with 50ms
backOffPolicy.setMultiplier(2.0); // Double each time
backOffPolicy.setMaxInterval(5000L); // Max 5 seconds
RetryTemplate retryTemplate = new RetryTemplate();
retryTemplate.setRetryPolicy(retryPolicy);
retryTemplate.setBackOffPolicy(backOffPolicy);
return retryTemplate;
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
@ConditionalOnProperty(name = "conductor.file-storage.enabled", havingValue = "true")
public SqliteFileMetadataDAO sqliteFileMetadataDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper) {
return new SqliteFileMetadataDAO(retryTemplate, objectMapper, dataSource);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
@ConditionalOnProperty(name = "conductor.integrations.ai.enabled", havingValue = "true")
public SqliteSkillMetadataDAO sqliteSkillMetadataDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper) {
return new SqliteSkillMetadataDAO(retryTemplate, objectMapper, dataSource);
}
@Bean
@DependsOn({"flywayForPrimaryDb"})
@ConditionalOnProperty(name = "conductor.integrations.ai.enabled", havingValue = "true")
public SqliteSkillPackageDAO sqliteSkillPackageDAO(
@Qualifier("sqliteRetryTemplate") RetryTemplate retryTemplate,
ObjectMapper objectMapper) {
return new SqliteSkillPackageDAO(retryTemplate, objectMapper, dataSource);
}
public static class CustomRetryPolicy extends SimpleRetryPolicy {
// PostgreSQL error codes
private static final String ER_LOCK_DEADLOCK = "40P01";
private static final String ER_SERIALIZATION_FAILURE = "40001";
// SQLite error codes
private static final int SQLITE_BUSY = 5;
private static final int SQLITE_LOCKED = 6;
@Override
public boolean canRetry(final RetryContext context) {
final Optional<Throwable> lastThrowable =
Optional.ofNullable(context.getLastThrowable());
return lastThrowable
.map(
throwable ->
super.canRetry(context)
&& (isDeadLockError(throwable)
|| isSqliteBusyError(throwable)))
.orElseGet(() -> super.canRetry(context));
}
private boolean isDeadLockError(Throwable throwable) {
SQLException sqlException = findCauseSQLException(throwable);
if (sqlException == null) {
return false;
}
return ER_LOCK_DEADLOCK.equals(sqlException.getSQLState())
|| ER_SERIALIZATION_FAILURE.equals(sqlException.getSQLState());
}
private boolean isSqliteBusyError(Throwable throwable) {
SQLException sqlException = findCauseSQLException(throwable);
if (sqlException == null) {
return false;
}
// Check for SQLite BUSY (5) or LOCKED (6) error codes
int errorCode = sqlException.getErrorCode();
if (errorCode == SQLITE_BUSY || errorCode == SQLITE_LOCKED) {
return true;
}
// Also check message for SQLite busy/locked indicators
String message = sqlException.getMessage();
if (message != null) {
return message.contains("SQLITE_BUSY")
|| message.contains("database is locked")
|| message.contains("SQLITE_LOCKED")
|| message.contains("table is locked");
}
return false;
}
private SQLException findCauseSQLException(Throwable throwable) {
Throwable causeException = throwable;
while (null != causeException && !(causeException instanceof SQLException)) {
causeException = causeException.getCause();
}
return (SQLException) causeException;
}
}
}
@@ -0,0 +1,60 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.config;
import java.time.Duration;
import org.springframework.boot.context.properties.ConfigurationProperties;
@ConfigurationProperties("conductor.sqlite")
public class SqliteProperties {
/** The time (in seconds) after which the in-memory task definitions cache will be refreshed */
private Duration taskDefCacheRefreshInterval = Duration.ofSeconds(60);
private Integer deadlockRetryMax = 3;
private boolean onlyIndexOnStatusChange = false;
private Integer asyncMaxPoolSize = 10;
private Integer asyncWorkerQueueSize = 10;
public Duration getTaskDefCacheRefreshInterval() {
return taskDefCacheRefreshInterval;
}
public void setTaskDefCacheRefreshInterval(Duration taskDefCacheRefreshInterval) {
this.taskDefCacheRefreshInterval = taskDefCacheRefreshInterval;
}
public Integer getDeadlockRetryMax() {
return deadlockRetryMax;
}
public void setDeadlockRetryMax(Integer deadlockRetryMax) {
this.deadlockRetryMax = deadlockRetryMax;
}
public int getAsyncMaxPoolSize() {
return asyncMaxPoolSize;
}
public int getAsyncWorkerQueueSize() {
return asyncWorkerQueueSize;
}
public boolean getOnlyIndexOnStatusChange() {
return onlyIndexOnStatusChange;
}
}
@@ -0,0 +1,206 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.io.IOException;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Statement;
import java.time.Duration;
import java.time.Instant;
import java.util.Arrays;
import java.util.List;
import java.util.function.Consumer;
import javax.sql.DataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.core.exception.NonTransientException;
import com.netflix.conductor.sqlite.util.*;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.core.type.TypeReference;
import com.fasterxml.jackson.databind.ObjectMapper;
public abstract class SqliteBaseDAO {
private static final List<String> EXCLUDED_STACKTRACE_CLASS =
List.of(SqliteBaseDAO.class.getName(), Thread.class.getName());
protected final Logger logger = LoggerFactory.getLogger(getClass());
protected final ObjectMapper objectMapper;
protected final DataSource dataSource;
private final RetryTemplate retryTemplate;
protected SqliteBaseDAO(
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
this.retryTemplate = retryTemplate;
this.objectMapper = objectMapper;
this.dataSource = dataSource;
}
protected String toJson(Object value) {
try {
return objectMapper.writeValueAsString(value);
} catch (JsonProcessingException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
protected <T> T readValue(String json, Class<T> tClass) {
try {
return objectMapper.readValue(json, tClass);
} catch (IOException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
protected <T> T readValue(String json, TypeReference<T> typeReference) {
try {
return objectMapper.readValue(json, typeReference);
} catch (IOException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
private <R> R getWithTransaction(final TransactionalFunction<R> function) {
final Instant start = Instant.now();
LazyToString callingMethod = getCallingMethod();
logger.trace("{} : starting transaction", callingMethod);
try (Connection tx = dataSource.getConnection()) {
boolean previousAutoCommitMode = tx.getAutoCommit();
tx.setAutoCommit(false);
tx.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
try {
// Enable foreign keys for SQLite
try (Statement stmt = tx.createStatement()) {
stmt.execute("PRAGMA foreign_keys = ON");
}
R result = function.apply(tx);
tx.commit();
return result;
} catch (Throwable th) {
tx.rollback();
if (th instanceof NonTransientException) {
throw th;
}
throw new NonTransientException(th.getMessage(), th);
} finally {
tx.setAutoCommit(previousAutoCommitMode);
}
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
} finally {
logger.trace(
"{} : took {}ms",
callingMethod,
Duration.between(start, Instant.now()).toMillis());
}
}
protected <R> R getWithRetriedTransactions(final TransactionalFunction<R> function) {
try {
return retryTemplate.execute(context -> getWithTransaction(function));
} catch (Exception e) {
throw new NonTransientException(e.getMessage(), e);
}
}
protected <R> R getWithTransactionWithOutErrorPropagation(TransactionalFunction<R> function) {
Instant start = Instant.now();
LazyToString callingMethod = getCallingMethod();
logger.trace("{} : starting transaction", callingMethod);
try (Connection tx = dataSource.getConnection()) {
boolean previousAutoCommitMode = tx.getAutoCommit();
tx.setAutoCommit(false);
tx.setTransactionIsolation(Connection.TRANSACTION_SERIALIZABLE);
try {
// Enable foreign keys for SQLite
try (Statement stmt = tx.createStatement()) {
stmt.execute("PRAGMA foreign_keys = ON");
}
R result = function.apply(tx);
tx.commit();
return result;
} catch (Throwable th) {
tx.rollback();
logger.info(th.getMessage());
return null;
} finally {
tx.setAutoCommit(previousAutoCommitMode);
}
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
} finally {
logger.trace(
"{} : took {}ms",
callingMethod,
Duration.between(start, Instant.now()).toMillis());
}
}
protected void withTransaction(Consumer<Connection> consumer) {
getWithRetriedTransactions(
connection -> {
consumer.accept(connection);
return null;
});
}
protected <R> R queryWithTransaction(String query, QueryFunction<R> function) {
return getWithRetriedTransactions(tx -> query(tx, query, function));
}
protected <R> R query(Connection tx, String query, QueryFunction<R> function) {
try (Query q = new Query(objectMapper, tx, query)) {
return function.apply(q);
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
protected void execute(Connection tx, String query, ExecuteFunction function) {
try (Query q = new Query(objectMapper, tx, query)) {
function.apply(q);
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
protected void executeWithTransaction(String query, ExecuteFunction function) {
withTransaction(tx -> execute(tx, query, function));
}
protected final LazyToString getCallingMethod() {
return new LazyToString(
() ->
Arrays.stream(Thread.currentThread().getStackTrace())
.filter(
ste ->
!EXCLUDED_STACKTRACE_CLASS.contains(
ste.getClassName()))
.findFirst()
.map(StackTraceElement::getMethodName)
.orElseThrow(() -> new NullPointerException("Cannot find Caller")));
}
}
@@ -0,0 +1,428 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.time.temporal.TemporalAccessor;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.*;
import javax.sql.DataSource;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.common.metadata.events.EventExecution;
import com.netflix.conductor.common.metadata.tasks.TaskExecLog;
import com.netflix.conductor.common.run.SearchResult;
import com.netflix.conductor.common.run.TaskSummary;
import com.netflix.conductor.common.run.WorkflowSummary;
import com.netflix.conductor.core.events.queue.Message;
import com.netflix.conductor.dao.IndexDAO;
import com.netflix.conductor.metrics.Monitors;
import com.netflix.conductor.sqlite.config.SqliteProperties;
import com.netflix.conductor.sqlite.util.SqliteIndexQueryBuilder;
import com.fasterxml.jackson.databind.ObjectMapper;
public class SqliteIndexDAO extends SqliteBaseDAO implements IndexDAO {
private final SqliteProperties properties;
private final ExecutorService executorService;
private static final int CORE_POOL_SIZE = 6;
private static final long KEEP_ALIVE_TIME = 1L;
private boolean onlyIndexOnStatusChange;
public SqliteIndexDAO(
RetryTemplate retryTemplate,
ObjectMapper objectMapper,
DataSource dataSource,
SqliteProperties properties) {
super(retryTemplate, objectMapper, dataSource);
this.properties = properties;
this.onlyIndexOnStatusChange = properties.getOnlyIndexOnStatusChange();
int maximumPoolSize = properties.getAsyncMaxPoolSize();
int workerQueueSize = properties.getAsyncWorkerQueueSize();
// Set up a workerpool for performing async operations.
this.executorService =
new ThreadPoolExecutor(
CORE_POOL_SIZE,
maximumPoolSize,
KEEP_ALIVE_TIME,
TimeUnit.MINUTES,
new LinkedBlockingQueue<>(workerQueueSize),
(runnable, executor) -> {
logger.warn(
"Request {} to async dao discarded in executor {}",
runnable,
executor);
Monitors.recordDiscardedIndexingCount("indexQueue");
});
}
@Override
public void indexWorkflow(WorkflowSummary workflow) {
String INSERT_WORKFLOW_INDEX_SQL =
"INSERT INTO workflow_index (workflow_id, correlation_id, workflow_type, start_time, update_time, status, parent_workflow_id, classifier, json_data) "
+ " VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (workflow_id) "
+ " DO UPDATE SET correlation_id = excluded.correlation_id, workflow_type = excluded.workflow_type, "
+ " start_time = excluded.start_time, status = excluded.status, json_data = excluded.json_data, "
+ " update_time = excluded.update_time, parent_workflow_id = excluded.parent_workflow_id, "
+ " classifier = excluded.classifier "
+ " WHERE excluded.update_time >= workflow_index.update_time";
if (onlyIndexOnStatusChange) {
INSERT_WORKFLOW_INDEX_SQL += " AND workflow_index.status != excluded.status";
}
TemporalAccessor updateTa = DateTimeFormatter.ISO_INSTANT.parse(workflow.getUpdateTime());
Timestamp updateTime = Timestamp.from(Instant.from(updateTa));
TemporalAccessor ta = DateTimeFormatter.ISO_INSTANT.parse(workflow.getStartTime());
Timestamp startTime = Timestamp.from(Instant.from(ta));
int rowsUpdated =
queryWithTransaction(
INSERT_WORKFLOW_INDEX_SQL,
q ->
q.addParameter(workflow.getWorkflowId())
.addParameter(workflow.getCorrelationId())
.addParameter(workflow.getWorkflowType())
.addParameter(startTime.toString())
.addParameter(updateTime.toString())
.addParameter(workflow.getStatus().toString())
.addParameter(
workflow.getParentWorkflowId() != null
? workflow.getParentWorkflowId()
: "")
.addParameter(workflow.getClassifier())
.addJsonParameter(workflow)
.executeUpdate());
logger.debug("Sqlite index workflow rows updated: {}", rowsUpdated);
}
@Override
public SearchResult<WorkflowSummary> searchWorkflowSummary(
String query, String freeText, int start, int count, List<String> sort) {
SqliteIndexQueryBuilder queryBuilder =
new SqliteIndexQueryBuilder(
"workflow_index", query, freeText, start, count, sort, properties);
List<WorkflowSummary> results =
queryWithTransaction(
queryBuilder.getQuery(),
q -> {
queryBuilder.addParameters(q);
queryBuilder.addPagingParameters(q);
return q.executeAndFetch(WorkflowSummary.class);
});
List<String> totalHitResults =
queryWithTransaction(
queryBuilder.getCountQuery(),
q -> {
queryBuilder.addParameters(q);
return q.executeAndFetch(String.class);
});
int totalHits = Integer.valueOf(totalHitResults.get(0));
return new SearchResult<>(totalHits, results);
}
@Override
public void indexTask(TaskSummary task) {
String INSERT_TASK_INDEX_SQL =
"INSERT INTO task_index (task_id, task_type, task_def_name, status, start_time, update_time, workflow_type, json_data)"
+ "VALUES (?, ?, ?, ?, ?, ?, ?, ?) ON CONFLICT (task_id) "
+ "DO UPDATE SET task_type = excluded.task_type, task_def_name = excluded.task_def_name, "
+ "status = excluded.status, update_time = excluded.update_time, json_data = excluded.json_data "
+ "WHERE excluded.update_time >= task_index.update_time";
if (onlyIndexOnStatusChange) {
INSERT_TASK_INDEX_SQL += " AND task_index.status != excluded.status";
}
TemporalAccessor updateTa = DateTimeFormatter.ISO_INSTANT.parse(task.getUpdateTime());
Timestamp updateTime = Timestamp.from(Instant.from(updateTa));
TemporalAccessor startTa = DateTimeFormatter.ISO_INSTANT.parse(task.getStartTime());
Timestamp startTime = Timestamp.from(Instant.from(startTa));
int rowsUpdated =
queryWithTransaction(
INSERT_TASK_INDEX_SQL,
q ->
q.addParameter(task.getTaskId())
.addParameter(task.getTaskType())
.addParameter(task.getTaskDefName())
.addParameter(task.getStatus().toString())
.addParameter(startTime.toString())
.addParameter(updateTime.toString())
.addParameter(task.getWorkflowType())
.addJsonParameter(task)
.executeUpdate());
logger.debug("Sqlite index task rows updated: {}", rowsUpdated);
}
@Override
public SearchResult<TaskSummary> searchTaskSummary(
String query, String freeText, int start, int count, List<String> sort) {
SqliteIndexQueryBuilder queryBuilder =
new SqliteIndexQueryBuilder(
"task_index", query, freeText, start, count, sort, properties);
List<TaskSummary> results =
queryWithTransaction(
queryBuilder.getQuery(),
q -> {
queryBuilder.addParameters(q);
queryBuilder.addPagingParameters(q);
return q.executeAndFetch(TaskSummary.class);
});
List<String> totalHitResults =
queryWithTransaction(
queryBuilder.getCountQuery(),
q -> {
queryBuilder.addParameters(q);
return q.executeAndFetch(String.class);
});
int totalHits = Integer.valueOf(totalHitResults.get(0));
return new SearchResult<>(totalHits, results);
}
@Override
public void addTaskExecutionLogs(List<TaskExecLog> logs) {
String INSERT_LOG =
"INSERT INTO task_execution_logs (task_id, created_time, log) VALUES (?, ?, ?)";
for (TaskExecLog log : logs) {
queryWithTransaction(
INSERT_LOG,
q ->
q.addParameter(log.getTaskId())
.addParameter(new Timestamp(log.getCreatedTime()))
.addParameter(log.getLog())
.executeUpdate());
}
}
@Override
public List<TaskExecLog> getTaskExecutionLogs(String taskId) {
return queryWithTransaction(
"SELECT log, task_id, created_time FROM task_execution_logs WHERE task_id = ? ORDER BY created_time ASC",
q ->
q.addParameter(taskId)
.executeAndFetch(
rs -> {
List<TaskExecLog> result = new ArrayList<>();
while (rs.next()) {
TaskExecLog log = new TaskExecLog();
log.setLog(rs.getString("log"));
log.setTaskId(rs.getString("task_id"));
log.setCreatedTime(
rs.getTimestamp("created_time").getTime());
result.add(log);
}
return result;
}));
}
@Override
public void setup() {}
@Override
public CompletableFuture<Void> asyncIndexWorkflow(WorkflowSummary workflow) {
logger.info("asyncIndexWorkflow is not supported for Sqlite indexing");
return CompletableFuture.completedFuture(null);
}
@Override
public CompletableFuture<Void> asyncIndexTask(TaskSummary task) {
logger.info("asyncIndexTask is not supported for Sqlite indexing");
return CompletableFuture.completedFuture(null);
}
@Override
public SearchResult<String> searchWorkflows(
String query, String freeText, int start, int count, List<String> sort) {
SqliteIndexQueryBuilder queryBuilder =
new SqliteIndexQueryBuilder(
"workflow_index", query, freeText, start, count, sort, properties);
List<String> results =
queryWithTransaction(
queryBuilder.getQuery("workflow_id"),
q -> {
queryBuilder.addParameters(q);
queryBuilder.addPagingParameters(q);
return q.executeScalarList(String.class);
});
List<String> totalHitResults =
queryWithTransaction(
queryBuilder.getCountQuery(),
q -> {
queryBuilder.addParameters(q);
return q.executeAndFetch(String.class);
});
int totalHits = Integer.valueOf(totalHitResults.get(0));
return new SearchResult<>(totalHits, results);
}
@Override
public SearchResult<String> searchTasks(
String query, String freeText, int start, int count, List<String> sort) {
SqliteIndexQueryBuilder queryBuilder =
new SqliteIndexQueryBuilder(
"task_index", query, freeText, start, count, sort, properties);
List<String> results =
queryWithTransaction(
queryBuilder.getQuery("task_id"),
q -> {
queryBuilder.addParameters(q);
queryBuilder.addPagingParameters(q);
return q.executeScalarList(String.class);
});
List<String> totalHitResults =
queryWithTransaction(
queryBuilder.getCountQuery(),
q -> {
queryBuilder.addParameters(q);
return q.executeAndFetch(String.class);
});
int totalHits = Integer.valueOf(totalHitResults.get(0));
return new SearchResult<>(totalHits, results);
}
@Override
public void removeWorkflow(String workflowId) {
String REMOVE_WORKFLOW_SQL = "DELETE FROM workflow_index WHERE workflow_id = ?";
queryWithTransaction(REMOVE_WORKFLOW_SQL, q -> q.addParameter(workflowId).executeUpdate());
}
@Override
public CompletableFuture<Void> asyncRemoveWorkflow(String workflowId) {
return CompletableFuture.runAsync(() -> removeWorkflow(workflowId), executorService);
}
@Override
public void updateWorkflow(String workflowInstanceId, String[] keys, Object[] values) {
logger.info("updateWorkflow is not supported for Sqlite indexing");
}
@Override
public CompletableFuture<Void> asyncUpdateWorkflow(
String workflowInstanceId, String[] keys, Object[] values) {
logger.info("asyncUpdateWorkflow is not supported for Sqlite indexing");
return CompletableFuture.completedFuture(null);
}
@Override
public void removeTask(String workflowId, String taskId) {
String REMOVE_TASK_SQL = "DELETE FROM task_index WHERE task_id = ?";
String REMOVE_TASK_EXECUTION_SQL = "DELETE FROM task_execution_logs WHERE task_id =?";
withTransaction(
connection -> {
queryWithTransaction(
REMOVE_TASK_SQL, q -> q.addParameter(taskId).executeUpdate());
queryWithTransaction(
REMOVE_TASK_EXECUTION_SQL, q -> q.addParameter(taskId).executeUpdate());
});
}
@Override
public CompletableFuture<Void> asyncRemoveTask(String workflowId, String taskId) {
return CompletableFuture.runAsync(() -> removeTask(workflowId, taskId), executorService);
}
@Override
public void updateTask(String workflowId, String taskId, String[] keys, Object[] values) {
logger.info("updateTask is not supported for Sqlite indexing");
}
@Override
public CompletableFuture<Void> asyncUpdateTask(
String workflowId, String taskId, String[] keys, Object[] values) {
logger.info("asyncUpdateTask is not supported for Sqlite indexing");
return CompletableFuture.completedFuture(null);
}
@Override
public String get(String workflowInstanceId, String key) {
logger.info("get is not supported for Sqlite indexing");
return null;
}
@Override
public CompletableFuture<Void> asyncAddTaskExecutionLogs(List<TaskExecLog> logs) {
logger.info("asyncAddTaskExecutionLogs is not supported for Sqlite indexing");
return CompletableFuture.completedFuture(null);
}
@Override
public void addEventExecution(EventExecution eventExecution) {
logger.info("addEventExecution is not supported for Sqlite indexing");
}
@Override
public List<EventExecution> getEventExecutions(String event) {
logger.info("getEventExecutions is not supported for Sqlite indexing");
return null;
}
@Override
public CompletableFuture<Void> asyncAddEventExecution(EventExecution eventExecution) {
logger.info("asyncAddEventExecution is not supported for Sqlite indexing");
return CompletableFuture.completedFuture(null);
}
@Override
public void addMessage(String queue, Message msg) {
logger.info("addMessage is not supported for Sqlite indexing");
}
@Override
public CompletableFuture<Void> asyncAddMessage(String queue, Message message) {
logger.info("asyncAddMessage is not supported for Sqlite indexing");
return CompletableFuture.completedFuture(null);
}
@Override
public List<Message> getMessages(String queue) {
logger.info("getMessages is not supported for Sqlite indexing");
return null;
}
@Override
public List<String> searchArchivableWorkflows(String indexName, long archiveTtlDays) {
logger.info("searchArchivableWorkflows is not supported for Sqlite indexing");
return null;
}
public long getWorkflowCount(String query, String freeText) {
logger.info("getWorkflowCount is not supported for Sqlite indexing");
return 0;
}
}
@@ -0,0 +1,133 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
import javax.sql.DataSource;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.common.metadata.tasks.PollData;
import com.netflix.conductor.core.exception.NonTransientException;
import com.netflix.conductor.dao.PollDataDAO;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
public class SqlitePollDataDAO extends SqliteBaseDAO implements PollDataDAO {
public SqlitePollDataDAO(
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
super(retryTemplate, objectMapper, dataSource);
}
@Override
public void updateLastPollData(String taskDefName, String domain, String workerId) {
Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
String effectiveDomain = domain == null ? "DEFAULT" : domain;
PollData pollData = new PollData(taskDefName, domain, workerId, System.currentTimeMillis());
withTransaction(tx -> insertOrUpdatePollData(tx, pollData, effectiveDomain));
}
@Override
public PollData getPollData(String taskDefName, String domain) {
Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
String effectiveDomain = (domain == null) ? "DEFAULT" : domain;
return getWithRetriedTransactions(tx -> readPollData(tx, taskDefName, effectiveDomain));
}
@Override
public List<PollData> getPollData(String taskDefName) {
Preconditions.checkNotNull(taskDefName, "taskDefName name cannot be null");
return readAllPollData(taskDefName);
}
@Override
public List<PollData> getAllPollData() {
try (Connection tx = dataSource.getConnection()) {
boolean previousAutoCommitMode = tx.getAutoCommit();
tx.setAutoCommit(true);
try {
String GET_ALL_POLL_DATA = "SELECT json_data FROM poll_data ORDER BY queue_name";
return query(tx, GET_ALL_POLL_DATA, q -> q.executeAndFetch(PollData.class));
} catch (Throwable th) {
throw new NonTransientException(th.getMessage(), th);
} finally {
tx.setAutoCommit(previousAutoCommitMode);
}
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
private void insertOrUpdatePollData(Connection connection, PollData pollData, String domain) {
try {
/*
* Most times the row will be updated so let's try the update first. This used to be an 'INSERT/ON CONFLICT do update' sql statement. The problem with that
* is that if we try the INSERT first, the sequence will be increased even if the ON CONFLICT happens. Since polling happens *a lot*, the sequence can increase
* dramatically even though it won't be used.
*/
String UPDATE_POLL_DATA =
"UPDATE poll_data SET json_data=?, modified_on=CURRENT_TIMESTAMP WHERE queue_name=? AND domain=?";
int rowsUpdated =
query(
connection,
UPDATE_POLL_DATA,
q ->
q.addJsonParameter(pollData)
.addParameter(pollData.getQueueName())
.addParameter(domain)
.executeUpdate());
if (rowsUpdated == 0) {
String INSERT_POLL_DATA =
"INSERT INTO poll_data (queue_name, domain, json_data, modified_on) VALUES (?, ?, ?, CURRENT_TIMESTAMP) ON CONFLICT (queue_name,domain) DO UPDATE SET json_data=excluded.json_data, modified_on=excluded.modified_on";
execute(
connection,
INSERT_POLL_DATA,
q ->
q.addParameter(pollData.getQueueName())
.addParameter(domain)
.addJsonParameter(pollData)
.executeUpdate());
}
} catch (NonTransientException e) {
if (!e.getMessage().startsWith("ERROR: lastPollTime cannot be set to a lower value")) {
throw e;
}
}
}
private PollData readPollData(Connection connection, String queueName, String domain) {
String GET_POLL_DATA =
"SELECT json_data FROM poll_data WHERE queue_name = ? AND domain = ?";
return query(
connection,
GET_POLL_DATA,
q ->
q.addParameter(queueName)
.addParameter(domain)
.executeAndFetchFirst(PollData.class));
}
private List<PollData> readAllPollData(String queueName) {
String GET_ALL_POLL_DATA = "SELECT json_data FROM poll_data WHERE queue_name = ?";
return queryWithTransaction(
GET_ALL_POLL_DATA, q -> q.addParameter(queueName).executeAndFetch(PollData.class));
}
}
@@ -0,0 +1,508 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.core.events.queue.Message;
import com.netflix.conductor.dao.QueueDAO;
import com.netflix.conductor.sqlite.config.SqliteProperties;
import com.netflix.conductor.sqlite.util.ExecutorsUtil;
import com.netflix.conductor.sqlite.util.Query;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableMap;
import com.google.common.collect.Maps;
import com.google.common.util.concurrent.Uninterruptibles;
import jakarta.annotation.PreDestroy;
public class SqliteQueueDAO extends SqliteBaseDAO implements QueueDAO {
private static final Long UNACK_SCHEDULE_MS = 60_000L;
private final ScheduledExecutorService scheduledExecutorService;
public SqliteQueueDAO(
RetryTemplate retryTemplate,
ObjectMapper objectMapper,
DataSource dataSource,
SqliteProperties properties) {
super(retryTemplate, objectMapper, dataSource);
this.scheduledExecutorService =
Executors.newSingleThreadScheduledExecutor(
ExecutorsUtil.newNamedThreadFactory("sqlite-queue-"));
this.scheduledExecutorService.scheduleAtFixedRate(
this::processAllUnacks,
UNACK_SCHEDULE_MS,
UNACK_SCHEDULE_MS,
TimeUnit.MILLISECONDS);
logger.debug("{} is ready to serve", SqliteQueueDAO.class.getName());
}
@PreDestroy
public void destroy() {
try {
this.scheduledExecutorService.shutdown();
if (scheduledExecutorService.awaitTermination(30, TimeUnit.SECONDS)) {
logger.debug("tasks completed, shutting down");
} else {
logger.warn("Forcing shutdown after waiting for 30 seconds");
scheduledExecutorService.shutdownNow();
}
} catch (InterruptedException ie) {
logger.warn(
"Shutdown interrupted, invoking shutdownNow on scheduledExecutorService for processAllUnacks",
ie);
scheduledExecutorService.shutdownNow();
Thread.currentThread().interrupt();
}
}
@Override
public void push(String queueName, String id, long offsetTimeInSecond) {
push(queueName, id, 0, offsetTimeInSecond);
}
@Override
public void push(String queueName, String id, int priority, long offsetTimeInSecond) {
withTransaction(tx -> pushMessage(tx, queueName, id, null, priority, offsetTimeInSecond));
}
@Override
public void push(String queueName, List<Message> messages) {
withTransaction(
tx ->
messages.forEach(
message ->
pushMessage(
tx,
queueName,
message.getId(),
message.getPayload(),
message.getPriority(),
0)));
}
@Override
public boolean pushIfNotExists(String queueName, String id, long offsetTimeInSecond) {
return pushIfNotExists(queueName, id, 0, offsetTimeInSecond);
}
@Override
public boolean pushIfNotExists(
String queueName, String id, int priority, long offsetTimeInSecond) {
return getWithRetriedTransactions(
tx -> {
if (!existsMessage(tx, queueName, id)) {
pushMessage(tx, queueName, id, null, priority, offsetTimeInSecond);
return true;
}
return false;
});
}
@Override
public List<String> pop(String queueName, int count, int timeout) {
return pollMessages(queueName, count, timeout).stream()
.map(Message::getId)
.collect(Collectors.toList());
}
@Override
public List<String> peekFirstIds(String queueName, int count) {
final String SQL =
"SELECT message_id FROM queue_message "
+ "WHERE queue_name = ? AND popped = false "
+ "ORDER BY deliver_on, priority DESC, created_on LIMIT ?";
return queryWithTransaction(
SQL,
q -> q.addParameter(queueName).addParameter(count).executeScalarList(String.class));
}
@Override
public List<Message> pollMessages(String queueName, int count, int timeout) {
if (timeout < 1) {
List<Message> messages =
getWithTransactionWithOutErrorPropagation(
tx -> popMessages(tx, queueName, count, timeout));
if (messages == null) {
return new ArrayList<>();
}
return messages;
}
long start = System.currentTimeMillis();
final List<Message> messages = new ArrayList<>();
while (true) {
List<Message> messagesSlice =
getWithTransactionWithOutErrorPropagation(
tx -> popMessages(tx, queueName, count - messages.size(), timeout));
if (messagesSlice == null) {
logger.warn(
"Unable to poll {} messages from {} due to tx conflict, only {} popped",
count,
queueName,
messages.size());
// conflict could have happened, returned messages popped so far
return messages;
}
messages.addAll(messagesSlice);
if (messages.size() >= count || ((System.currentTimeMillis() - start) > timeout)) {
return messages;
}
Uninterruptibles.sleepUninterruptibly(100, TimeUnit.MILLISECONDS);
}
}
@Override
public void remove(String queueName, String messageId) {
withTransaction(tx -> removeMessage(tx, queueName, messageId));
}
@Override
public int getSize(String queueName) {
final String GET_QUEUE_SIZE = "SELECT COUNT(*) FROM queue_message WHERE queue_name = ?";
return queryWithTransaction(
GET_QUEUE_SIZE, q -> ((Long) q.addParameter(queueName).executeCount()).intValue());
}
@Override
public boolean ack(String queueName, String messageId) {
return getWithRetriedTransactions(tx -> removeMessage(tx, queueName, messageId));
}
@Override
public boolean setUnackTimeout(String queueName, String messageId, long unackTimeout) {
long updatedOffsetTimeInSecond = unackTimeout / 1000;
final String UPDATE_UNACK_TIMEOUT =
"UPDATE queue_message SET offset_time_seconds = ?, deliver_on = strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds') WHERE queue_name = ? AND message_id = ?";
return queryWithTransaction(
UPDATE_UNACK_TIMEOUT,
q ->
q.addParameter(updatedOffsetTimeInSecond)
.addParameter(updatedOffsetTimeInSecond)
.addParameter(queueName)
.addParameter(messageId)
.executeUpdate())
== 1;
}
@Override
public boolean setUnackTimeoutIfShorter(String queueName, String messageId, long unackTimeout) {
long updatedOffsetTimeInSecond = unackTimeout / 1000;
// Only update when the proposed deliver_on is earlier than what is already set,
// mirroring the ZADD LT semantics used by the Redis implementation.
final String UPDATE_UNACK_TIMEOUT_IF_SHORTER =
"UPDATE queue_message SET offset_time_seconds = ?, deliver_on = strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds')"
+ " WHERE queue_name = ? AND message_id = ? AND deliver_on > strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds')";
return queryWithTransaction(
UPDATE_UNACK_TIMEOUT_IF_SHORTER,
q ->
q.addParameter(updatedOffsetTimeInSecond)
.addParameter(updatedOffsetTimeInSecond)
.addParameter(queueName)
.addParameter(messageId)
.addParameter(updatedOffsetTimeInSecond)
.executeUpdate())
== 1;
}
@Override
public void flush(String queueName) {
final String FLUSH_QUEUE = "DELETE FROM queue_message WHERE queue_name = ?";
executeWithTransaction(FLUSH_QUEUE, q -> q.addParameter(queueName).executeDelete());
}
@Override
public Map<String, Long> queuesDetail() {
final String GET_QUEUES_DETAIL =
"SELECT queue_name, (SELECT count(*) FROM queue_message WHERE popped = false AND queue_name = q.queue_name) AS size FROM queue q";
return queryWithTransaction(
GET_QUEUES_DETAIL,
q ->
q.executeAndFetch(
rs -> {
Map<String, Long> detail = Maps.newHashMap();
while (rs.next()) {
String queueName = rs.getString("queue_name");
Long size = rs.getLong("size");
detail.put(queueName, size);
}
return detail;
}));
}
@Override
public Map<String, Map<String, Map<String, Long>>> queuesDetailVerbose() {
// @formatter:off
final String GET_QUEUES_DETAIL_VERBOSE =
"SELECT queue_name, \n"
+ " (SELECT count(*) FROM queue_message WHERE popped = false AND queue_name = q.queue_name) AS size,\n"
+ " (SELECT count(*) FROM queue_message WHERE popped = true AND queue_name = q.queue_name) AS uacked \n"
+ "FROM queue q";
// @formatter:on
return queryWithTransaction(
GET_QUEUES_DETAIL_VERBOSE,
q ->
q.executeAndFetch(
rs -> {
Map<String, Map<String, Map<String, Long>>> result =
Maps.newHashMap();
while (rs.next()) {
String queueName = rs.getString("queue_name");
Long size = rs.getLong("size");
Long queueUnacked = rs.getLong("uacked");
result.put(
queueName,
ImmutableMap.of(
"a",
ImmutableMap
.of( // sharding not implemented,
// returning only
// one shard with all the
// info
"size",
size,
"uacked",
queueUnacked)));
}
return result;
}));
}
public void processAllUnacks() {
logger.trace("processAllUnacks started");
getWithRetriedTransactions(
tx -> {
String LOCK_TASKS =
"SELECT queue_name, message_id FROM queue_message WHERE popped = true AND strftime('%Y-%m-%d %H:%M:%f', deliver_on, '+60 seconds') < strftime('%Y-%m-%d %H:%M:%f', 'now') limit 1000";
List<QueueMessage> messages =
query(
tx,
LOCK_TASKS,
p ->
p.executeAndFetch(
rs -> {
List<QueueMessage> results =
new ArrayList<QueueMessage>();
while (rs.next()) {
QueueMessage qm = new QueueMessage();
qm.queueName =
rs.getString("queue_name");
qm.messageId =
rs.getString("message_id");
results.add(qm);
}
return results;
}));
if (messages.size() == 0) {
return 0;
}
Map<String, List<String>> queueMessageMap = new HashMap<String, List<String>>();
for (QueueMessage qm : messages) {
if (!queueMessageMap.containsKey(qm.queueName)) {
queueMessageMap.put(qm.queueName, new ArrayList<String>());
}
queueMessageMap.get(qm.queueName).add(qm.messageId);
}
int totalUnacked = 0;
for (String queueName : queueMessageMap.keySet()) {
Integer unacked = 0;
try {
final List<String> msgIds = queueMessageMap.get(queueName);
final String UPDATE_POPPED =
String.format(
"UPDATE queue_message SET popped = false WHERE queue_name = ? and message_id IN (%s)",
Query.generateInBindings(msgIds.size()));
unacked =
query(
tx,
UPDATE_POPPED,
q ->
q.addParameter(queueName)
.addParameters(msgIds)
.executeUpdate());
} catch (Exception e) {
e.printStackTrace();
}
totalUnacked += unacked;
logger.debug("Unacked {} messages from all queues", unacked);
}
if (totalUnacked > 0) {
logger.debug("Unacked {} messages from all queues", totalUnacked);
}
return totalUnacked;
});
}
@Override
public void processUnacks(String queueName) {
final String PROCESS_UNACKS =
"UPDATE queue_message SET popped = false WHERE queue_name = ? AND popped = true AND strftime('%Y-%m-%d %H:%M:%f', 'now', '-60 seconds') > deliver_on";
executeWithTransaction(PROCESS_UNACKS, q -> q.addParameter(queueName).executeUpdate());
}
@Override
public boolean resetOffsetTime(String queueName, String id) {
long offsetTimeInSecond = 0; // Reset to 0
final String SET_OFFSET_TIME =
"UPDATE queue_message SET offset_time_seconds = ?, deliver_on = strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds') "
+ "WHERE queue_name = ? AND message_id = ?";
return queryWithTransaction(
SET_OFFSET_TIME,
q ->
q.addParameter(offsetTimeInSecond)
.addParameter(offsetTimeInSecond)
.addParameter(queueName)
.addParameter(id)
.executeUpdate()
== 1);
}
private boolean existsMessage(Connection connection, String queueName, String messageId) {
final String EXISTS_MESSAGE =
"SELECT EXISTS(SELECT 1 FROM queue_message WHERE queue_name = ? AND message_id = ?)";
return query(
connection,
EXISTS_MESSAGE,
q -> q.addParameter(queueName).addParameter(messageId).exists());
}
private void pushMessage(
Connection connection,
String queueName,
String messageId,
String payload,
Integer priority,
long offsetTimeInSecond) {
createQueueIfNotExists(connection, queueName);
String UPDATE_MESSAGE =
"UPDATE queue_message SET payload=?, deliver_on=strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds'), popped=false WHERE queue_name = ? AND message_id = ?";
int rowsUpdated =
query(
connection,
UPDATE_MESSAGE,
q ->
q.addParameter(payload)
.addParameter(offsetTimeInSecond)
.addParameter(queueName)
.addParameter(messageId)
.executeUpdate());
if (rowsUpdated == 0) {
String PUSH_MESSAGE =
"INSERT INTO queue_message (deliver_on, queue_name, message_id, priority, offset_time_seconds, payload) VALUES (strftime('%Y-%m-%d %H:%M:%f', 'now', '+' || ? || ' seconds'), ?,?,?,?,?) ON CONFLICT (queue_name,message_id) DO UPDATE SET payload=excluded.payload, deliver_on=excluded.deliver_on, popped=false";
execute(
connection,
PUSH_MESSAGE,
q ->
q.addParameter(offsetTimeInSecond)
.addParameter(queueName)
.addParameter(messageId)
.addParameter(priority)
.addParameter(offsetTimeInSecond)
.addParameter(payload)
.executeUpdate());
}
}
private boolean removeMessage(Connection connection, String queueName, String messageId) {
final String REMOVE_MESSAGE =
"DELETE FROM queue_message WHERE queue_name = ? AND message_id = ?";
return query(
connection,
REMOVE_MESSAGE,
q -> q.addParameter(queueName).addParameter(messageId).executeDelete());
}
private List<Message> popMessages(
Connection connection, String queueName, int count, int timeout) {
String POP_QUERY =
"UPDATE queue_message SET popped = true WHERE message_id IN ("
+ "SELECT message_id FROM queue_message WHERE queue_name = ? AND popped = false AND "
+ "deliver_on <= strftime('%Y-%m-%d %H:%M:%f', 'now') "
+ "ORDER BY priority DESC, deliver_on, created_on LIMIT ?"
+ ") RETURNING message_id, priority, payload";
return query(
connection,
POP_QUERY,
p ->
p.addParameter(queueName)
.addParameter(count)
.executeAndFetch(
rs -> {
List<Message> results = new ArrayList<>();
while (rs.next()) {
Message m = new Message();
m.setId(rs.getString("message_id"));
m.setPriority(rs.getInt("priority"));
m.setPayload(rs.getString("payload"));
results.add(m);
}
return results;
}));
}
@Override
public boolean containsMessage(String queueName, String messageId) {
return getWithRetriedTransactions(tx -> existsMessage(tx, queueName, messageId));
}
private void createQueueIfNotExists(Connection connection, String queueName) {
logger.trace("Creating new queue '{}'", queueName);
final String EXISTS_QUEUE = "SELECT EXISTS(SELECT 1 FROM queue WHERE queue_name = ?)";
boolean exists = query(connection, EXISTS_QUEUE, q -> q.addParameter(queueName).exists());
if (!exists) {
final String CREATE_QUEUE =
"INSERT INTO queue (queue_name) VALUES (?) ON CONFLICT (queue_name) DO NOTHING";
execute(connection, CREATE_QUEUE, q -> q.addParameter(queueName).executeUpdate());
}
}
private class QueueMessage {
public String queueName;
public String messageId;
}
}
@@ -0,0 +1,151 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao.metadata;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.List;
import javax.sql.DataSource;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.common.metadata.events.EventHandler;
import com.netflix.conductor.core.exception.ConflictException;
import com.netflix.conductor.core.exception.NotFoundException;
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
public class SqliteEventHandlerMetadataDAO extends SqliteBaseDAO {
public SqliteEventHandlerMetadataDAO(
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
super(retryTemplate, objectMapper, dataSource);
}
public void addEventHandler(EventHandler eventHandler) {
Preconditions.checkNotNull(eventHandler.getName(), "EventHandler name cannot be null");
final String INSERT_EVENT_HANDLER_QUERY =
"INSERT INTO meta_event_handler (name, event, active, json_data) "
+ "VALUES (?, ?, ?, ?)";
withTransaction(
tx -> {
if (getEventHandler(tx, eventHandler.getName()) != null) {
throw new ConflictException(
"EventHandler with name "
+ eventHandler.getName()
+ " already exists!");
}
execute(
tx,
INSERT_EVENT_HANDLER_QUERY,
q ->
q.addParameter(eventHandler.getName())
.addParameter(eventHandler.getEvent())
.addParameter(eventHandler.isActive())
.addJsonParameter(eventHandler)
.executeUpdate());
});
}
public void updateEventHandler(EventHandler eventHandler) {
Preconditions.checkNotNull(eventHandler.getName(), "EventHandler name cannot be null");
// @formatter:off
final String UPDATE_EVENT_HANDLER_QUERY =
"UPDATE meta_event_handler SET "
+ "event = ?, active = ?, json_data = ?, "
+ "modified_on = CURRENT_TIMESTAMP WHERE name = ?";
// @formatter:on
withTransaction(
tx -> {
EventHandler existing = getEventHandler(tx, eventHandler.getName());
if (existing == null) {
throw new NotFoundException(
"EventHandler with name " + eventHandler.getName() + " not found!");
}
execute(
tx,
UPDATE_EVENT_HANDLER_QUERY,
q ->
q.addParameter(eventHandler.getEvent())
.addParameter(eventHandler.isActive())
.addJsonParameter(eventHandler)
.addParameter(eventHandler.getName())
.executeUpdate());
});
}
public void removeEventHandler(String name) {
final String DELETE_EVENT_HANDLER_QUERY = "DELETE FROM meta_event_handler WHERE name = ?";
withTransaction(
tx -> {
EventHandler existing = getEventHandler(tx, name);
if (existing == null) {
throw new NotFoundException(
"EventHandler with name " + name + " not found!");
}
execute(
tx,
DELETE_EVENT_HANDLER_QUERY,
q -> q.addParameter(name).executeDelete());
});
}
public List<EventHandler> getEventHandlersForEvent(String event, boolean activeOnly) {
final String READ_ALL_EVENT_HANDLER_BY_EVENT_QUERY =
"SELECT json_data FROM meta_event_handler WHERE event = ?";
return queryWithTransaction(
READ_ALL_EVENT_HANDLER_BY_EVENT_QUERY,
q -> {
q.addParameter(event);
return q.executeAndFetch(
rs -> {
List<EventHandler> handlers = new ArrayList<>();
while (rs.next()) {
EventHandler h = readValue(rs.getString(1), EventHandler.class);
if (!activeOnly || h.isActive()) {
handlers.add(h);
}
}
return handlers;
});
});
}
public List<EventHandler> getAllEventHandlers() {
final String READ_ALL_EVENT_HANDLER_QUERY = "SELECT json_data FROM meta_event_handler";
return queryWithTransaction(
READ_ALL_EVENT_HANDLER_QUERY, q -> q.executeAndFetch(EventHandler.class));
}
private EventHandler getEventHandler(Connection connection, String name) {
final String READ_ONE_EVENT_HANDLER_QUERY =
"SELECT json_data FROM meta_event_handler WHERE name = ?";
return query(
connection,
READ_ONE_EVENT_HANDLER_QUERY,
q -> q.addParameter(name).executeAndFetchFirst(EventHandler.class));
}
}
@@ -0,0 +1,129 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao.metadata;
import java.util.List;
import java.util.Optional;
import com.netflix.conductor.common.metadata.events.EventHandler;
import com.netflix.conductor.common.metadata.tasks.TaskDef;
import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
import com.netflix.conductor.dao.EventHandlerDAO;
import com.netflix.conductor.dao.MetadataDAO;
import lombok.RequiredArgsConstructor;
@RequiredArgsConstructor
public class SqliteMetadataDAO implements MetadataDAO, EventHandlerDAO {
private final SqliteTaskMetadataDAO taskMetadataDAO;
private final SqliteWorkflowMetadataDAO workflowMetadataDAO;
private final SqliteEventHandlerMetadataDAO eventHandlerMetadataDAO;
@Override
public TaskDef createTaskDef(TaskDef taskDef) {
return taskMetadataDAO.createTaskDef(taskDef);
}
@Override
public TaskDef updateTaskDef(TaskDef taskDef) {
return taskMetadataDAO.updateTaskDef(taskDef);
}
@Override
public TaskDef getTaskDef(String name) {
return taskMetadataDAO.getTaskDef(name);
}
@Override
public List<TaskDef> getAllTaskDefs() {
return taskMetadataDAO.getAllTaskDefs();
}
@Override
public void removeTaskDef(String name) {
taskMetadataDAO.removeTaskDef(name);
}
@Override
public void createWorkflowDef(WorkflowDef def) {
workflowMetadataDAO.createWorkflowDef(def);
}
@Override
public void updateWorkflowDef(WorkflowDef def) {
workflowMetadataDAO.updateWorkflowDef(def);
}
@Override
public Optional<WorkflowDef> getLatestWorkflowDef(String name) {
return workflowMetadataDAO.getLatestWorkflowDef(name);
}
@Override
public Optional<WorkflowDef> getWorkflowDef(String name, int version) {
return workflowMetadataDAO.getWorkflowDef(name, version);
}
@Override
public void removeWorkflowDef(String name, Integer version) {
workflowMetadataDAO.removeWorkflowDef(name, version);
}
@Override
public List<WorkflowDef> getAllWorkflowDefs() {
return workflowMetadataDAO.getAllWorkflowDefs();
}
@Override
public List<WorkflowDef> getAllWorkflowDefsLatestVersions() {
return workflowMetadataDAO.getAllWorkflowDefsLatestVersions();
}
@Override
public void addEventHandler(EventHandler eventHandler) {
eventHandlerMetadataDAO.addEventHandler(eventHandler);
}
@Override
public void updateEventHandler(EventHandler eventHandler) {
eventHandlerMetadataDAO.updateEventHandler(eventHandler);
}
@Override
public void removeEventHandler(String name) {
eventHandlerMetadataDAO.removeEventHandler(name);
}
@Override
public List<EventHandler> getAllEventHandlers() {
return eventHandlerMetadataDAO.getAllEventHandlers();
}
@Override
public List<EventHandler> getEventHandlersForEvent(String event, boolean activeOnly) {
return eventHandlerMetadataDAO.getEventHandlersForEvent(event, activeOnly);
}
public List<String> findAll() {
return workflowMetadataDAO.findAll();
}
public List<WorkflowDef> getAllLatest() {
return workflowMetadataDAO.getAllLatest();
}
public List<WorkflowDef> getAllVersions(String name) {
return workflowMetadataDAO.getAllVersions(name);
}
}
@@ -0,0 +1,118 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao.metadata;
import java.sql.Connection;
import java.util.List;
import javax.sql.DataSource;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.common.metadata.tasks.TaskDef;
import com.netflix.conductor.core.exception.NotFoundException;
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
public class SqliteTaskMetadataDAO extends SqliteBaseDAO {
public SqliteTaskMetadataDAO(
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
super(retryTemplate, objectMapper, dataSource);
}
public TaskDef createTaskDef(TaskDef taskDef) {
validate(taskDef);
insertOrUpdateTaskDef(taskDef);
return taskDef;
}
public TaskDef updateTaskDef(TaskDef taskDef) {
validate(taskDef);
insertOrUpdateTaskDef(taskDef);
return taskDef;
}
public TaskDef getTaskDef(String name) {
Preconditions.checkNotNull(name, "TaskDef name cannot be null");
return getTaskDefFromDB(name);
}
public List<TaskDef> getAllTaskDefs() {
return getWithRetriedTransactions(this::findAllTaskDefs);
}
public void removeTaskDef(String name) {
final String DELETE_TASKDEF_QUERY = "DELETE FROM meta_task_def WHERE name = ?";
executeWithTransaction(
DELETE_TASKDEF_QUERY,
q -> {
if (!q.addParameter(name).executeDelete()) {
throw new NotFoundException("No such task definition");
}
});
}
private void validate(TaskDef taskDef) {
Preconditions.checkNotNull(taskDef, "TaskDef object cannot be null");
Preconditions.checkNotNull(taskDef.getName(), "TaskDef name cannot be null");
}
private TaskDef getTaskDefFromDB(String name) {
final String READ_ONE_TASKDEF_QUERY = "SELECT json_data FROM meta_task_def WHERE name = ?";
return queryWithTransaction(
READ_ONE_TASKDEF_QUERY,
q -> q.addParameter(name).executeAndFetchFirst(TaskDef.class));
}
private String insertOrUpdateTaskDef(TaskDef taskDef) {
final String UPDATE_TASKDEF_QUERY =
"UPDATE meta_task_def SET json_data = ?, modified_on = CURRENT_TIMESTAMP WHERE name = ?";
final String INSERT_TASKDEF_QUERY =
"INSERT INTO meta_task_def (name, json_data) VALUES (?, ?)";
return getWithRetriedTransactions(
tx -> {
execute(
tx,
UPDATE_TASKDEF_QUERY,
update -> {
int result =
update.addJsonParameter(taskDef)
.addParameter(taskDef.getName())
.executeUpdate();
if (result == 0) {
execute(
tx,
INSERT_TASKDEF_QUERY,
insert ->
insert.addParameter(taskDef.getName())
.addJsonParameter(taskDef)
.executeUpdate());
}
});
return taskDef.getName();
});
}
private List<TaskDef> findAllTaskDefs(Connection tx) {
final String READ_ALL_TASKDEF_QUERY = "SELECT json_data FROM meta_task_def";
return query(tx, READ_ALL_TASKDEF_QUERY, q -> q.executeAndFetch(TaskDef.class));
}
}
@@ -0,0 +1,229 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao.metadata;
import java.sql.Connection;
import java.util.List;
import java.util.Optional;
import javax.sql.DataSource;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
import com.netflix.conductor.core.exception.ConflictException;
import com.netflix.conductor.core.exception.NotFoundException;
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
public class SqliteWorkflowMetadataDAO extends SqliteBaseDAO {
public SqliteWorkflowMetadataDAO(
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
super(retryTemplate, objectMapper, dataSource);
}
public void createWorkflowDef(WorkflowDef def) {
validate(def);
withTransaction(
tx -> {
if (workflowExists(tx, def)) {
throw new ConflictException(
"Workflow with " + def.key() + " already exists!");
}
insertOrUpdateWorkflowDef(tx, def);
});
}
public void updateWorkflowDef(WorkflowDef def) {
validate(def);
withTransaction(tx -> insertOrUpdateWorkflowDef(tx, def));
}
public Optional<WorkflowDef> getLatestWorkflowDef(String name) {
final String GET_LATEST_WORKFLOW_DEF_QUERY =
"SELECT json_data FROM meta_workflow_def WHERE NAME = ? AND "
+ "version = latest_version";
return Optional.ofNullable(
queryWithTransaction(
GET_LATEST_WORKFLOW_DEF_QUERY,
q -> q.addParameter(name).executeAndFetchFirst(WorkflowDef.class)));
}
public Optional<WorkflowDef> getWorkflowDef(String name, int version) {
final String GET_WORKFLOW_DEF_QUERY =
"SELECT json_data FROM meta_workflow_def WHERE NAME = ? AND version = ?";
return Optional.ofNullable(
queryWithTransaction(
GET_WORKFLOW_DEF_QUERY,
q ->
q.addParameter(name)
.addParameter(version)
.executeAndFetchFirst(WorkflowDef.class)));
}
public void removeWorkflowDef(String name, Integer version) {
final String DELETE_WORKFLOW_QUERY =
"DELETE from meta_workflow_def WHERE name = ? AND version = ?";
withTransaction(
tx -> {
// remove specified workflow
execute(
tx,
DELETE_WORKFLOW_QUERY,
q -> {
if (!q.addParameter(name).addParameter(version).executeDelete()) {
throw new NotFoundException(
String.format(
"No such workflow definition: %s version: %d",
name, version));
}
});
// reset latest version based on remaining rows for this workflow
Optional<Integer> maxVersion = getLatestVersion(tx, name);
maxVersion.ifPresent(newVersion -> updateLatestVersion(tx, name, newVersion));
});
}
public List<WorkflowDef> getAllWorkflowDefs() {
final String GET_ALL_WORKFLOW_DEF_QUERY =
"SELECT json_data FROM meta_workflow_def ORDER BY name, version";
return queryWithTransaction(
GET_ALL_WORKFLOW_DEF_QUERY, q -> q.executeAndFetch(WorkflowDef.class));
}
public List<WorkflowDef> getAllWorkflowDefsLatestVersions() {
final String GET_ALL_WORKFLOW_DEF_LATEST_VERSIONS_QUERY =
"SELECT json_data FROM meta_workflow_def wd WHERE wd.version = (SELECT MAX(version) FROM meta_workflow_def wd2 WHERE wd2.name = wd.name)";
return queryWithTransaction(
GET_ALL_WORKFLOW_DEF_LATEST_VERSIONS_QUERY,
q -> q.executeAndFetch(WorkflowDef.class));
}
public List<String> findAll() {
final String FIND_ALL_WORKFLOW_DEF_QUERY = "SELECT DISTINCT name FROM meta_workflow_def";
return queryWithTransaction(
FIND_ALL_WORKFLOW_DEF_QUERY, q -> q.executeAndFetch(String.class));
}
public List<WorkflowDef> getAllLatest() {
final String GET_ALL_LATEST_WORKFLOW_DEF_QUERY =
"SELECT json_data FROM meta_workflow_def WHERE version = " + "latest_version";
return queryWithTransaction(
GET_ALL_LATEST_WORKFLOW_DEF_QUERY, q -> q.executeAndFetch(WorkflowDef.class));
}
public List<WorkflowDef> getAllVersions(String name) {
final String GET_ALL_VERSIONS_WORKFLOW_DEF_QUERY =
"SELECT json_data FROM meta_workflow_def WHERE name = ? " + "ORDER BY version";
return queryWithTransaction(
GET_ALL_VERSIONS_WORKFLOW_DEF_QUERY,
q -> q.addParameter(name).executeAndFetch(WorkflowDef.class));
}
private Boolean workflowExists(Connection connection, WorkflowDef def) {
final String CHECK_WORKFLOW_DEF_EXISTS_QUERY =
"SELECT COUNT(*) FROM meta_workflow_def WHERE name = ? AND " + "version = ?";
return query(
connection,
CHECK_WORKFLOW_DEF_EXISTS_QUERY,
q -> q.addParameter(def.getName()).addParameter(def.getVersion()).exists());
}
private void insertOrUpdateWorkflowDef(Connection tx, WorkflowDef def) {
final String INSERT_WORKFLOW_DEF_QUERY =
"INSERT INTO meta_workflow_def (name, version, json_data) VALUES (?," + " ?, ?)";
Optional<Integer> version = getLatestVersion(tx, def.getName());
if (!workflowExists(tx, def)) {
execute(
tx,
INSERT_WORKFLOW_DEF_QUERY,
q ->
q.addParameter(def.getName())
.addParameter(def.getVersion())
.addJsonParameter(def)
.executeUpdate());
} else {
// @formatter:off
final String UPDATE_WORKFLOW_DEF_QUERY =
"UPDATE meta_workflow_def "
+ "SET json_data = ?, modified_on = CURRENT_TIMESTAMP "
+ "WHERE name = ? AND version = ?";
// @formatter:on
execute(
tx,
UPDATE_WORKFLOW_DEF_QUERY,
q ->
q.addJsonParameter(def)
.addParameter(def.getName())
.addParameter(def.getVersion())
.executeUpdate());
}
int maxVersion = def.getVersion();
if (version.isPresent() && version.get() > def.getVersion()) {
maxVersion = version.get();
}
updateLatestVersion(tx, def.getName(), maxVersion);
}
private Optional<Integer> getLatestVersion(Connection tx, String name) {
final String GET_LATEST_WORKFLOW_DEF_VERSION =
"SELECT max(version) AS version FROM meta_workflow_def WHERE " + "name = ?";
Integer val =
query(
tx,
GET_LATEST_WORKFLOW_DEF_VERSION,
q -> {
q.addParameter(name);
return q.executeAndFetch(
rs -> {
if (!rs.next()) {
return null;
}
return rs.getInt(1);
});
});
return Optional.ofNullable(val);
}
private void updateLatestVersion(Connection tx, String name, int version) {
final String UPDATE_WORKFLOW_DEF_LATEST_VERSION_QUERY =
"UPDATE meta_workflow_def SET latest_version = ? " + "WHERE name = ?";
execute(
tx,
UPDATE_WORKFLOW_DEF_LATEST_VERSION_QUERY,
q -> q.addParameter(version).addParameter(name).executeUpdate());
}
private void validate(WorkflowDef def) {
Preconditions.checkNotNull(def, "WorkflowDef object cannot be null");
Preconditions.checkNotNull(def.getName(), "WorkflowDef name cannot be null");
}
}
@@ -0,0 +1,26 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.sql.SQLException;
/**
* Functional interface for {@link Query} executions with no expected result.
*
* @author mustafa
*/
@FunctionalInterface
public interface ExecuteFunction {
void apply(Query query) throws SQLException;
}
@@ -0,0 +1,37 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.atomic.AtomicInteger;
public class ExecutorsUtil {
private ExecutorsUtil() {}
public static ThreadFactory newNamedThreadFactory(final String threadNamePrefix) {
return new ThreadFactory() {
private final AtomicInteger counter = new AtomicInteger();
@SuppressWarnings("NullableProblems")
@Override
public Thread newThread(Runnable r) {
Thread thread = Executors.defaultThreadFactory().newThread(r);
thread.setName(threadNamePrefix + counter.getAndIncrement());
return thread;
}
};
}
}
@@ -0,0 +1,33 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.util.function.Supplier;
/** Functional class to support the lazy execution of a String result. */
public class LazyToString {
private final Supplier<String> supplier;
/**
* @param supplier Supplier to execute when {@link #toString()} is called.
*/
public LazyToString(Supplier<String> supplier) {
this.supplier = supplier;
}
@Override
public String toString() {
return supplier.get();
}
}
@@ -0,0 +1,648 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.io.IOException;
import java.sql.*;
import java.sql.Date;
import java.util.*;
import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.lang3.math.NumberUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import com.netflix.conductor.core.exception.NonTransientException;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.ObjectMapper;
/**
* Represents a {@link PreparedStatement} that is wrapped with convenience methods and utilities.
*
* <p>This class simulates a parameter building pattern and all {@literal addParameter(*)} methods
* must be called in the proper order of their expected binding sequence.
*
* @author mustafa
*/
public class Query implements AutoCloseable {
private final Logger logger = LoggerFactory.getLogger(getClass());
/** The {@link ObjectMapper} instance to use for serializing/deserializing JSON. */
protected final ObjectMapper objectMapper;
/** The initial supplied query String that was used to prepare {@link #statement}. */
private final String rawQuery;
/**
* Parameter index for the {@code ResultSet#set*(*)} methods, gets incremented every time a
* parameter is added to the {@code PreparedStatement} {@link #statement}.
*/
private final AtomicInteger index = new AtomicInteger(1);
/** The {@link PreparedStatement} that will be managed and executed by this class. */
private final PreparedStatement statement;
private final Connection connection;
public Query(ObjectMapper objectMapper, Connection connection, String query) {
this.rawQuery = query;
this.objectMapper = objectMapper;
this.connection = connection;
try {
this.statement = connection.prepareStatement(query);
} catch (SQLException ex) {
throw new NonTransientException(
"Cannot prepare statement for query: " + ex.getMessage(), ex);
}
}
/**
* Generate a String with {@literal count} number of '?' placeholders for {@link
* PreparedStatement} queries.
*
* @param count The number of '?' chars to generate.
* @return a comma delimited string of {@literal count} '?' binding placeholders.
*/
public static String generateInBindings(int count) {
String[] questions = new String[count];
for (int i = 0; i < count; i++) {
questions[i] = "?";
}
return String.join(", ", questions);
}
public Query addParameter(final String value) {
return addParameterInternal((ps, idx) -> ps.setString(idx, value));
}
public Query addParameter(final List<String> value) throws SQLException {
String[] valueStringArray = value.toArray(new String[0]);
Array valueArray = this.connection.createArrayOf("VARCHAR", valueStringArray);
return addParameterInternal((ps, idx) -> ps.setArray(idx, valueArray));
}
public Query addParameter(final int value) {
return addParameterInternal((ps, idx) -> ps.setInt(idx, value));
}
public Query addParameter(final boolean value) {
return addParameterInternal(((ps, idx) -> ps.setBoolean(idx, value)));
}
public Query addParameter(final long value) {
return addParameterInternal((ps, idx) -> ps.setLong(idx, value));
}
public Query addParameter(final double value) {
return addParameterInternal((ps, idx) -> ps.setDouble(idx, value));
}
public Query addParameter(Date date) {
return addParameterInternal((ps, idx) -> ps.setDate(idx, date));
}
public Query addParameter(Timestamp timestamp) {
return addParameterInternal((ps, idx) -> ps.setTimestamp(idx, timestamp));
}
/**
* Serializes {@literal value} to a JSON string for persistence.
*
* @param value The value to serialize.
* @return {@literal this}
*/
public Query addJsonParameter(Object value) {
return addParameter(toJson(value));
}
/**
* Bind the given {@link java.util.Date} to the PreparedStatement as a {@link Date}.
*
* @param date The {@literal java.util.Date} to bind.
* @return {@literal this}
*/
public Query addDateParameter(java.util.Date date) {
return addParameter(new Date(date.getTime()));
}
/**
* Bind the given {@link java.util.Date} to the PreparedStatement as a {@link Timestamp}.
*
* @param date The {@literal java.util.Date} to bind.
* @return {@literal this}
*/
public Query addTimestampParameter(java.util.Date date) {
return addParameter(new Timestamp(date.getTime()));
}
/**
* Bind the given epoch millis to the PreparedStatement as a {@link Timestamp}.
*
* @param epochMillis The epoch ms to create a new {@literal Timestamp} from.
* @return {@literal this}
*/
public Query addTimestampParameter(long epochMillis) {
return addParameter(new Timestamp(epochMillis));
}
/**
* Add a collection of primitive values at once, in the order of the collection.
*
* @param values The values to bind to the prepared statement.
* @return {@literal this}
* @throws IllegalArgumentException If a non-primitive/unsupported type is encountered in the
* collection.
* @see #addParameters(Object...)
*/
public Query addParameters(Collection values) {
return addParameters(values.toArray());
}
/**
* Add many primitive values at once.
*
* @param values The values to bind to the prepared statement.
* @return {@literal this}
* @throws IllegalArgumentException If a non-primitive/unsupported type is encountered.
*/
public Query addParameters(Object... values) {
for (Object v : values) {
if (v instanceof String) {
addParameter((String) v);
} else if (v instanceof Integer) {
addParameter((Integer) v);
} else if (v instanceof Long) {
addParameter((Long) v);
} else if (v instanceof Double) {
addParameter((Double) v);
} else if (v instanceof Boolean) {
addParameter((Boolean) v);
} else if (v instanceof Date) {
addParameter((Date) v);
} else if (v instanceof Timestamp) {
addParameter((Timestamp) v);
} else {
throw new IllegalArgumentException(
"Type "
+ v.getClass().getName()
+ " is not supported by automatic property assignment");
}
}
return this;
}
/**
* Utility method for evaluating the prepared statement as a query to check the existence of a
* record using a numeric count or boolean return value.
*
* <p>The {@link #rawQuery} provided must result in a {@link Number} or {@link Boolean} result.
*
* @return {@literal true} If a count query returned more than 0 or an exists query returns
* {@literal true}.
* @throws NonTransientException If an unexpected return type cannot be evaluated to a {@code
* Boolean} result.
*/
public boolean exists() {
Object val = executeScalar();
if (null == val) {
return false;
}
if (val instanceof Number) {
return convertLong(val) > 0;
}
if (val instanceof Boolean) {
return (Boolean) val;
}
if (val instanceof String) {
return convertBoolean(val);
}
throw new NonTransientException(
"Expected a Numeric or Boolean scalar return value from the query, received "
+ val.getClass().getName());
}
/**
* Convenience method for executing delete statements.
*
* @return {@literal true} if the statement affected 1 or more rows.
* @see #executeUpdate()
*/
public boolean executeDelete() {
int count = executeUpdate();
if (count > 1) {
logger.trace("Removed {} row(s) for query {}", count, rawQuery);
}
return count > 0;
}
/**
* Convenience method for executing statements that return a single numeric value, typically
* {@literal SELECT COUNT...} style queries.
*
* @return The result of the query as a {@literal long}.
*/
public long executeCount() {
return executeScalar(Long.class);
}
/**
* @return The result of {@link PreparedStatement#executeUpdate()}
*/
public int executeUpdate() {
try {
Long start = null;
if (logger.isTraceEnabled()) {
start = System.currentTimeMillis();
}
final int val = this.statement.executeUpdate();
if (null != start && logger.isTraceEnabled()) {
long end = System.currentTimeMillis();
logger.trace("[{}ms] {}: {}", (end - start), val, rawQuery);
}
return val;
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
/**
* Execute a query from the PreparedStatement and return the ResultSet.
*
* <p><em>NOTE:</em> The returned ResultSet must be closed/managed by the calling methods.
*
* @return {@link PreparedStatement#executeQuery()}
* @throws NonTransientException If any SQL errors occur.
*/
public ResultSet executeQuery() {
Long start = null;
if (logger.isTraceEnabled()) {
start = System.currentTimeMillis();
}
try {
return this.statement.executeQuery();
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
} finally {
if (null != start && logger.isTraceEnabled()) {
long end = System.currentTimeMillis();
logger.trace("[{}ms] {}", (end - start), rawQuery);
}
}
}
/**
* @return The single result of the query as an Object.
*/
public Object executeScalar() {
try (ResultSet rs = executeQuery()) {
if (!rs.next()) {
return null;
}
return rs.getObject(1);
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
/**
* Execute the PreparedStatement and return a single 'primitive' value from the ResultSet.
*
* @param returnType The type to return.
* @param <V> The type parameter to return a List of.
* @return A single result from the execution of the statement, as a type of {@literal
* returnType}.
* @throws NonTransientException {@literal returnType} is unsupported, cannot be cast to from
* the result, or any SQL errors occur.
*/
public <V> V executeScalar(Class<V> returnType) {
try (ResultSet rs = executeQuery()) {
if (!rs.next()) {
Object value = null;
if (Integer.class == returnType) {
value = 0;
} else if (Long.class == returnType) {
value = 0L;
} else if (Boolean.class == returnType) {
value = false;
}
return returnType.cast(value);
} else {
return getScalarFromResultSet(rs, returnType);
}
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
/**
* Execute the PreparedStatement and return a List of 'primitive' values from the ResultSet.
*
* @param returnType The type Class return a List of.
* @param <V> The type parameter to return a List of.
* @return A {@code List<returnType>}.
* @throws NonTransientException {@literal returnType} is unsupported, cannot be cast to from
* the result, or any SQL errors occur.
*/
public <V> List<V> executeScalarList(Class<V> returnType) {
try (ResultSet rs = executeQuery()) {
List<V> values = new ArrayList<>();
while (rs.next()) {
values.add(getScalarFromResultSet(rs, returnType));
}
return values;
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
/**
* Execute the statement and return only the first record from the result set.
*
* @param returnType The Class to return.
* @param <V> The type parameter.
* @return An instance of {@literal <V>} from the result set.
*/
public <V> V executeAndFetchFirst(Class<V> returnType) {
Object o = executeScalar();
if (null == o) {
return null;
}
return convert(o, returnType);
}
/**
* Execute the PreparedStatement and return a List of {@literal returnType} values from the
* ResultSet.
*
* @param returnType The type Class return a List of.
* @param <V> The type parameter to return a List of.
* @return A {@code List<returnType>}.
* @throws NonTransientException {@literal returnType} is unsupported, cannot be cast to from
* the result, or any SQL errors occur.
*/
public <V> List<V> executeAndFetch(Class<V> returnType) {
try (ResultSet rs = executeQuery()) {
List<V> list = new ArrayList<>();
while (rs.next()) {
list.add(convert(rs.getObject(1), returnType));
}
return list;
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
/**
* Execute the PreparedStatement and return a List of {@literal Map} values from the ResultSet.
*
* @return A {@code List<Map>}.
* @throws SQLException if any SQL errors occur.
* @throws NonTransientException if any SQL errors occur.
*/
public List<Map<String, Object>> executeAndFetchMap() {
try (ResultSet rs = executeQuery()) {
List<Map<String, Object>> result = new ArrayList<>();
ResultSetMetaData metadata = rs.getMetaData();
int columnCount = metadata.getColumnCount();
while (rs.next()) {
HashMap<String, Object> row = new HashMap<>();
for (int i = 1; i <= columnCount; i++) {
row.put(metadata.getColumnLabel(i), rs.getObject(i));
}
result.add(row);
}
return result;
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
/**
* Execute the query and pass the {@link ResultSet} to the given handler.
*
* @param handler The {@link ResultSetHandler} to execute.
* @param <V> The return type of this method.
* @return The results of {@link ResultSetHandler#apply(ResultSet)}.
*/
public <V> V executeAndFetch(ResultSetHandler<V> handler) {
try (ResultSet rs = executeQuery()) {
return handler.apply(rs);
} catch (SQLException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
@Override
public void close() {
try {
if (null != statement && !statement.isClosed()) {
statement.close();
}
} catch (SQLException ex) {
logger.warn("Error closing prepared statement: {}", ex.getMessage());
}
}
protected final Query addParameterInternal(InternalParameterSetter setter) {
int index = getAndIncrementIndex();
try {
setter.apply(this.statement, index);
return this;
} catch (SQLException ex) {
throw new NonTransientException("Could not apply bind parameter at index " + index, ex);
}
}
protected <V> V getScalarFromResultSet(ResultSet rs, Class<V> returnType) throws SQLException {
Object value = null;
if (Integer.class == returnType) {
value = rs.getInt(1);
} else if (Long.class == returnType) {
value = rs.getLong(1);
} else if (String.class == returnType) {
value = rs.getString(1);
} else if (Boolean.class == returnType) {
value = rs.getBoolean(1);
} else if (Double.class == returnType) {
value = rs.getDouble(1);
} else if (Date.class == returnType) {
value = rs.getDate(1);
} else if (Timestamp.class == returnType) {
value = rs.getTimestamp(1);
} else {
value = rs.getObject(1);
}
if (null == value) {
throw new NullPointerException(
"Cannot get value from ResultSet of type " + returnType.getName());
}
return returnType.cast(value);
}
protected <V> V convert(Object value, Class<V> returnType) {
if (Boolean.class == returnType) {
return returnType.cast(convertBoolean(value));
} else if (Integer.class == returnType) {
return returnType.cast(convertInt(value));
} else if (Long.class == returnType) {
return returnType.cast(convertLong(value));
} else if (Double.class == returnType) {
return returnType.cast(convertDouble(value));
} else if (String.class == returnType) {
return returnType.cast(convertString(value));
} else if (value instanceof String) {
return fromJson((String) value, returnType);
}
final String vName = value.getClass().getName();
final String rName = returnType.getName();
throw new NonTransientException("Cannot convert type " + vName + " to " + rName);
}
protected Integer convertInt(Object value) {
if (null == value) {
return null;
}
if (value instanceof Integer) {
return (Integer) value;
}
if (value instanceof Number) {
return ((Number) value).intValue();
}
return NumberUtils.toInt(value.toString());
}
protected Double convertDouble(Object value) {
if (null == value) {
return null;
}
if (value instanceof Double) {
return (Double) value;
}
if (value instanceof Number) {
return ((Number) value).doubleValue();
}
return NumberUtils.toDouble(value.toString());
}
protected Long convertLong(Object value) {
if (null == value) {
return null;
}
if (value instanceof Long) {
return (Long) value;
}
if (value instanceof Number) {
return ((Number) value).longValue();
}
return NumberUtils.toLong(value.toString());
}
protected String convertString(Object value) {
if (null == value) {
return null;
}
if (value instanceof String) {
return (String) value;
}
return value.toString().trim();
}
protected Boolean convertBoolean(Object value) {
if (null == value) {
return null;
}
if (value instanceof Boolean) {
return (Boolean) value;
}
if (value instanceof Number) {
return ((Number) value).intValue() != 0;
}
String text = value.toString().trim();
return "Y".equalsIgnoreCase(text)
|| "YES".equalsIgnoreCase(text)
|| "TRUE".equalsIgnoreCase(text)
|| "T".equalsIgnoreCase(text)
|| "1".equalsIgnoreCase(text);
}
protected String toJson(Object value) {
if (null == value) {
return null;
}
try {
return objectMapper.writeValueAsString(value);
} catch (JsonProcessingException ex) {
throw new NonTransientException(ex.getMessage(), ex);
}
}
protected <V> V fromJson(String value, Class<V> returnType) {
if (null == value) {
return null;
}
try {
return objectMapper.readValue(value, returnType);
} catch (IOException ex) {
throw new NonTransientException(
"Could not convert JSON '" + value + "' to " + returnType.getName(), ex);
}
}
protected final int getIndex() {
return index.get();
}
protected final int getAndIncrementIndex() {
return index.getAndIncrement();
}
@FunctionalInterface
private interface InternalParameterSetter {
void apply(PreparedStatement ps, int idx) throws SQLException;
}
}
@@ -0,0 +1,26 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.sql.SQLException;
/**
* Functional interface for {@link Query} executions that return results.
*
* @author mustafa
*/
@FunctionalInterface
public interface QueryFunction<R> {
R apply(Query query) throws SQLException;
}
@@ -0,0 +1,39 @@
/*
* Copyright 2024 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
public class QueueStats {
private Integer depth;
private long nextDelivery;
public void setDepth(Integer depth) {
this.depth = depth;
}
public Integer getDepth() {
return depth;
}
public void setNextDelivery(long nextDelivery) {
this.nextDelivery = nextDelivery;
}
public long getNextDelivery() {
return nextDelivery;
}
public String toString() {
return "{nextDelivery: " + nextDelivery + " depth: " + depth + "}";
}
}
@@ -0,0 +1,27 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.sql.ResultSet;
import java.sql.SQLException;
/**
* Functional interface for {@link Query#executeAndFetch(ResultSetHandler)}.
*
* @author mustafa
*/
@FunctionalInterface
public interface ResultSetHandler<R> {
R apply(ResultSet resultSet) throws SQLException;
}
@@ -0,0 +1,297 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.sql.SQLException;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.ZonedDateTime;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.commons.lang3.StringUtils;
import com.netflix.conductor.common.metadata.workflow.WorkflowClassifier;
import com.netflix.conductor.sqlite.config.SqliteProperties;
public class SqliteIndexQueryBuilder {
private final String table;
private final String freeText;
private final int start;
private final int count;
private final List<String> sort;
private final List<Condition> conditions = new ArrayList<>();
private boolean allowJsonQueries;
private boolean allowFullTextQueries;
private static final String[] VALID_FIELDS = {
"workflow_id",
"correlation_id",
"workflow_type",
"start_time",
"status",
"task_id",
"task_type",
"task_def_name",
"update_time",
"json_data",
"parent_workflow_id",
"classifier"
};
private static final String[] VALID_SORT_ORDER = {"ASC", "DESC"};
private static class Condition {
private String attribute;
private String operator;
private List<String> values;
private final String CONDITION_REGEX = "([a-zA-Z]+)\\s?(=|>|<|IN)\\s?(.*)";
public Condition() {}
public Condition(String query) {
Pattern conditionRegex = Pattern.compile(CONDITION_REGEX);
Matcher conditionMatcher = conditionRegex.matcher(query);
if (conditionMatcher.find()) {
String[] valueArr = conditionMatcher.group(3).replaceAll("[\"'()]", "").split(",");
ArrayList<String> values = new ArrayList<>(Arrays.asList(valueArr));
this.attribute = camelToSnake(conditionMatcher.group(1));
this.values = values;
this.operator = getOperator(conditionMatcher.group(2));
if (this.attribute.endsWith("_time")) {
values.set(0, millisToUtc(values.get(0)));
}
} else {
throw new IllegalArgumentException("Incorrectly formatted query string: " + query);
}
}
public String getQueryFragment() {
if (operator.equals("IN")) {
// Create proper IN clause for SQLite
String inClause =
attribute
+ " IN ("
+ String.join(",", Collections.nCopies(values.size(), "?"))
+ ")";
if (classifierMatchesUntagged()) {
return "(" + inClause + " OR " + attribute + " IS NULL)";
}
return inClause;
} else if (operator.equals("MATCH")) {
// SQLite FTS5 full-text search
return "json_data MATCH ?";
} else if (operator.equals("JSON_CONTAINS")) {
// SQLite JSON1 extension query
return "json_extract(json_data, ?) IS NOT NULL";
} else if (operator.equals("LIKE")) {
return "lower(" + attribute + ") LIKE ?";
} else {
if (attribute.endsWith("_time")) {
return attribute + " " + operator + " datetime(?)";
} else if (operator.equals("=")
&& values.size() == 1
&& values.get(0).contains("*")) {
return "lower(" + attribute + ") LIKE lower(?)";
} else if (operator.equals("=") && classifierMatchesUntagged()) {
return "(" + attribute + " = ? OR " + attribute + " IS NULL)";
} else {
return attribute + " " + operator + " ?";
}
}
}
/**
* Rows indexed before the classifier column existed have a NULL classifier but are
* semantically untagged, i.e. plain workflows. When a filter asks for the untagged token
* ({@link WorkflowClassifier#WORKFLOW}), widen the predicate to also match those legacy
* NULL rows.
*/
private boolean classifierMatchesUntagged() {
return "classifier".equals(attribute)
&& values != null
&& values.stream().anyMatch(WorkflowClassifier.WORKFLOW::equalsIgnoreCase);
}
private String getOperator(String op) {
if (op.equals("IN") && values.size() == 1) {
return "=";
}
return op;
}
public void addParameter(Query q) throws SQLException {
if (values.size() > 1) {
// For IN clause, add each value separately
for (String value : values) {
q.addParameter(value);
}
} else {
String val = values.get(0);
if (val.contains("*")) {
val = val.replace("*", "%");
}
q.addParameter(val);
}
}
private String millisToUtc(String millis) {
Long startTimeMilli = Long.parseLong(millis);
ZonedDateTime startDate =
ZonedDateTime.ofInstant(Instant.ofEpochMilli(startTimeMilli), ZoneOffset.UTC);
return DateTimeFormatter.ISO_DATE_TIME.format(startDate);
}
private boolean isValid() {
return Arrays.asList(VALID_FIELDS).contains(attribute);
}
public void setAttribute(String attribute) {
this.attribute = attribute;
}
public void setOperator(String operator) {
this.operator = operator;
}
public void setValues(List<String> values) {
this.values = values;
}
}
public SqliteIndexQueryBuilder(
String table,
String query,
String freeText,
int start,
int count,
List<String> sort,
SqliteProperties properties) {
this.table = table;
this.freeText = freeText;
this.start = start;
this.count = count;
this.sort = sort != null ? sort : Collections.emptyList();
this.allowFullTextQueries = true;
this.allowJsonQueries = true;
this.parseQuery(query);
this.parseFreeText(freeText);
}
public String getQuery() {
return getQuery("json_data");
}
public String getQuery(String selectColumn) {
String queryString = "";
List<Condition> validConditions =
conditions.stream().filter(c -> c.isValid()).collect(Collectors.toList());
if (validConditions.size() > 0) {
queryString =
" WHERE "
+ String.join(
" AND ",
validConditions.stream()
.map(c -> c.getQueryFragment())
.collect(Collectors.toList()));
}
return "SELECT "
+ selectColumn
+ " FROM "
+ table
+ queryString
+ getSort()
+ " LIMIT ? OFFSET ?";
}
public String getCountQuery() {
String queryString = "";
List<Condition> validConditions =
conditions.stream().filter(c -> c.isValid()).collect(Collectors.toList());
if (validConditions.size() > 0) {
queryString =
" WHERE "
+ String.join(
" AND ",
validConditions.stream()
.map(c -> c.getQueryFragment())
.collect(Collectors.toList()));
}
return "SELECT COUNT(*) FROM " + table + queryString;
}
public void addParameters(Query q) throws SQLException {
for (Condition condition : conditions) {
if (condition.isValid()) {
condition.addParameter(q);
}
}
}
public void addPagingParameters(Query q) throws SQLException {
q.addParameter(count);
q.addParameter(start);
}
private void parseQuery(String query) {
if (!StringUtils.isEmpty(query)) {
for (String s : query.split(" AND ")) {
conditions.add(new Condition(s));
}
Collections.sort(conditions, Comparator.comparing(Condition::getQueryFragment));
}
}
private void parseFreeText(String freeText) {
if (!StringUtils.isEmpty(freeText) && !freeText.equals("*")) {
Condition cond = new Condition();
cond.setAttribute("json_data");
cond.setOperator("LIKE");
String[] values = {freeText};
cond.setValues(
Arrays.stream(values)
.map(v -> "%" + v.toLowerCase() + "%")
.collect(Collectors.toList()));
conditions.add(cond);
}
}
private String getSort() {
ArrayList<String> sortConds = new ArrayList<>();
for (String s : sort) {
String[] splitCond = s.split(":");
if (splitCond.length == 2) {
String attribute = camelToSnake(splitCond[0]);
String order = splitCond[1].toUpperCase();
if (Arrays.asList(VALID_FIELDS).contains(attribute)
&& Arrays.asList(VALID_SORT_ORDER).contains(order)) {
sortConds.add(attribute + " " + order);
}
}
}
if (sortConds.size() > 0) {
return " ORDER BY " + String.join(", ", sortConds);
}
return "";
}
private static String camelToSnake(String camel) {
return camel.replaceAll("\\B([A-Z])", "_$1").toLowerCase();
}
}
@@ -0,0 +1,27 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.sql.Connection;
import java.sql.SQLException;
/**
* Functional interface for operations within a transactional context.
*
* @author mustafa
*/
@FunctionalInterface
public interface TransactionalFunction<R> {
R apply(Connection tx) throws SQLException;
}
@@ -0,0 +1,152 @@
/*
* Copyright 2026 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package org.conductoross.conductor.sqlite.dao;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.List;
import javax.sql.DataSource;
import org.conductoross.conductor.dao.FileMetadataDAO;
import org.conductoross.conductor.model.FileModel;
import org.conductoross.conductor.model.file.FileUploadStatus;
import org.conductoross.conductor.model.file.StorageType;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
import com.fasterxml.jackson.databind.ObjectMapper;
public class SqliteFileMetadataDAO extends SqliteBaseDAO implements FileMetadataDAO {
private static final String INSERT_FILE =
"INSERT INTO file_metadata (file_id, file_name, content_type, "
+ "storage_type, storage_path, upload_status, workflow_id, task_id, "
+ "created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)";
private static final String SELECT_BY_ID = "SELECT * FROM file_metadata WHERE file_id = ?";
private static final String UPDATE_STATUS =
"UPDATE file_metadata SET upload_status = ?, updated_at = CURRENT_TIMESTAMP "
+ "WHERE file_id = ?";
private static final String UPDATE_UPLOAD_COMPLETE =
"UPDATE file_metadata SET upload_status = ?, storage_content_hash = ?, "
+ "storage_content_size = ?, updated_at = CURRENT_TIMESTAMP "
+ "WHERE file_id = ?";
private static final String SELECT_BY_WORKFLOW =
"SELECT * FROM file_metadata WHERE workflow_id = ?";
private static final String SELECT_BY_TASK = "SELECT * FROM file_metadata WHERE task_id = ?";
public SqliteFileMetadataDAO(
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
super(retryTemplate, objectMapper, dataSource);
}
@Override
public void createFileMetadata(FileModel fileModel) {
executeWithTransaction(
INSERT_FILE,
q ->
q.addParameter(fileModel.getFileId())
.addParameter(fileModel.getFileName())
.addParameter(fileModel.getContentType())
.addParameter(fileModel.getStorageType().name())
.addParameter(fileModel.getStoragePath())
.addParameter(fileModel.getUploadStatus().name())
.addParameter(fileModel.getWorkflowId())
.addParameter(fileModel.getTaskId())
.addParameter(Timestamp.from(fileModel.getCreatedAt()))
.addParameter(Timestamp.from(fileModel.getUpdatedAt()))
.executeUpdate());
}
@Override
public FileModel getFileMetadata(String fileId) {
return queryWithTransaction(
SELECT_BY_ID,
q ->
q.addParameter(fileId)
.executeAndFetch(
rs -> {
if (!rs.next()) return null;
return toFileModel(rs);
}));
}
@Override
public void updateUploadStatus(String fileId, FileUploadStatus status) {
executeWithTransaction(
UPDATE_STATUS,
q -> q.addParameter(status.name()).addParameter(fileId).executeUpdate());
}
@Override
public void updateUploadComplete(
String fileId, FileUploadStatus status, String contentHash, long contentSize) {
executeWithTransaction(
UPDATE_UPLOAD_COMPLETE,
q ->
q.addParameter(status.name())
.addParameter(contentHash)
.addParameter(contentSize)
.addParameter(fileId)
.executeUpdate());
}
@Override
public List<FileModel> getFilesByWorkflowId(String workflowId) {
return queryWithTransaction(
SELECT_BY_WORKFLOW,
q -> q.addParameter(workflowId).executeAndFetch(this::toFileModelList));
}
@Override
public List<FileModel> getFilesByTaskId(String taskId) {
return queryWithTransaction(
SELECT_BY_TASK, q -> q.addParameter(taskId).executeAndFetch(this::toFileModelList));
}
private List<FileModel> toFileModelList(ResultSet rs) throws SQLException {
List<FileModel> list = new ArrayList<>();
while (rs.next()) {
list.add(toFileModel(rs));
}
return list;
}
private FileModel toFileModel(ResultSet rs) throws SQLException {
FileModel model = new FileModel();
model.setFileId(rs.getString("file_id"));
model.setFileName(rs.getString("file_name"));
model.setContentType(rs.getString("content_type"));
model.setStorageContentHash(rs.getString("storage_content_hash"));
long scs = rs.getLong("storage_content_size");
model.setStorageContentSize(rs.wasNull() ? 0 : scs);
model.setStorageType(StorageType.valueOf(rs.getString("storage_type")));
model.setStoragePath(rs.getString("storage_path"));
model.setUploadStatus(FileUploadStatus.valueOf(rs.getString("upload_status")));
model.setWorkflowId(rs.getString("workflow_id"));
model.setTaskId(rs.getString("task_id"));
Timestamp createdAt = rs.getTimestamp("created_at");
if (createdAt != null) model.setCreatedAt(createdAt.toInstant());
Timestamp updatedAt = rs.getTimestamp("updated_at");
if (updatedAt != null) model.setUpdatedAt(updatedAt.toInstant());
return model;
}
}
@@ -0,0 +1,205 @@
/*
* Copyright 2026 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package org.conductoross.conductor.sqlite.dao;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import java.util.Optional;
import javax.sql.DataSource;
import org.conductoross.conductor.dao.SkillMetadataDAO;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
import com.fasterxml.jackson.databind.ObjectMapper;
/** SQLite {@link SkillMetadataDAO} — table {@code skill_metadata}. */
public class SqliteSkillMetadataDAO extends SqliteBaseDAO implements SkillMetadataDAO {
private static final String CLEAR_LATEST =
"UPDATE skill_metadata SET is_latest = ? WHERE owner_id = ? AND name = ?";
private static final String UPSERT_LATEST =
"INSERT INTO skill_metadata (owner_id, name, version, is_latest, detail_json, created_at, updated_at) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?) "
+ "ON CONFLICT (owner_id, name, version) DO UPDATE SET "
+ "is_latest = excluded.is_latest, detail_json = excluded.detail_json, "
+ "updated_at = excluded.updated_at";
private static final String UPSERT_NO_LATEST =
"INSERT INTO skill_metadata (owner_id, name, version, is_latest, detail_json, created_at, updated_at) "
+ "VALUES (?, ?, ?, ?, ?, ?, ?) "
+ "ON CONFLICT (owner_id, name, version) DO UPDATE SET "
+ "detail_json = excluded.detail_json, updated_at = excluded.updated_at";
private static final String SELECT_DETAIL =
"SELECT detail_json FROM skill_metadata WHERE owner_id = ? AND name = ? AND version = ?";
private static final String SELECT_LATEST_VERSION =
"SELECT version FROM skill_metadata WHERE owner_id = ? AND name = ? AND is_latest = ?";
private static final String SELECT_VERSIONS =
"SELECT detail_json FROM skill_metadata WHERE owner_id = ? AND name = ?";
private static final String SELECT_ALL =
"SELECT detail_json FROM skill_metadata WHERE owner_id = ?";
private static final String SELECT_LATEST_ALL =
"SELECT detail_json FROM skill_metadata WHERE owner_id = ? AND is_latest = ?";
private static final String DELETE_ONE =
"DELETE FROM skill_metadata WHERE owner_id = ? AND name = ? AND version = ?";
// In SQLite, NULL sorts lowest, so DESC places non-null updated_at first (NULLs last).
private static final String SELECT_NEWEST_REMAINING =
"SELECT version FROM skill_metadata WHERE owner_id = ? AND name = ? ORDER BY updated_at DESC LIMIT 1";
private static final String SET_LATEST =
"UPDATE skill_metadata SET is_latest = ? WHERE owner_id = ? AND name = ? AND version = ?";
public SqliteSkillMetadataDAO(
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
super(retryTemplate, objectMapper, dataSource);
}
@Override
public void save(
String ownerId,
String name,
String version,
boolean makeLatest,
String detailJson,
Long createdAt,
Long updatedAt) {
if (makeLatest) {
executeWithTransaction(
CLEAR_LATEST,
q ->
q.addParameter(false)
.addParameter(ownerId)
.addParameter(name)
.executeUpdate());
executeWithTransaction(
UPSERT_LATEST,
q ->
q.addParameter(ownerId)
.addParameter(name)
.addParameter(version)
.addParameter(true)
.addParameter(detailJson)
.addParameter(createdAt)
.addParameter(updatedAt)
.executeUpdate());
} else {
executeWithTransaction(
UPSERT_NO_LATEST,
q ->
q.addParameter(ownerId)
.addParameter(name)
.addParameter(version)
.addParameter(false)
.addParameter(detailJson)
.addParameter(createdAt)
.addParameter(updatedAt)
.executeUpdate());
}
}
@Override
public Optional<String> find(String ownerId, String name, String version) {
String json =
queryWithTransaction(
SELECT_DETAIL,
q ->
q.addParameter(ownerId)
.addParameter(name)
.addParameter(version)
.executeAndFetch(rs -> rs.next() ? rs.getString(1) : null));
return Optional.ofNullable(json);
}
@Override
public Optional<String> latestVersion(String ownerId, String name) {
String version =
queryWithTransaction(
SELECT_LATEST_VERSION,
q ->
q.addParameter(ownerId)
.addParameter(name)
.addParameter(true)
.executeAndFetch(rs -> rs.next() ? rs.getString(1) : null));
return Optional.ofNullable(version);
}
@Override
public List<String> listVersions(String ownerId, String name) {
return queryWithTransaction(
SELECT_VERSIONS,
q -> q.addParameter(ownerId).addParameter(name).executeAndFetch(this::toJsonList));
}
@Override
public List<String> list(String ownerId, boolean allVersions) {
if (allVersions) {
return queryWithTransaction(
SELECT_ALL, q -> q.addParameter(ownerId).executeAndFetch(this::toJsonList));
}
return queryWithTransaction(
SELECT_LATEST_ALL,
q -> q.addParameter(ownerId).addParameter(true).executeAndFetch(this::toJsonList));
}
@Override
public void delete(String ownerId, String name, String version) {
Optional<String> latest = latestVersion(ownerId, name);
executeWithTransaction(
DELETE_ONE,
q ->
q.addParameter(ownerId)
.addParameter(name)
.addParameter(version)
.executeUpdate());
if (latest.isPresent() && latest.get().equals(version)) {
String newest =
queryWithTransaction(
SELECT_NEWEST_REMAINING,
q ->
q.addParameter(ownerId)
.addParameter(name)
.executeAndFetch(
rs -> rs.next() ? rs.getString(1) : null));
if (newest != null) {
executeWithTransaction(
SET_LATEST,
q ->
q.addParameter(true)
.addParameter(ownerId)
.addParameter(name)
.addParameter(newest)
.executeUpdate());
}
}
}
private List<String> toJsonList(ResultSet rs) throws SQLException {
List<String> list = new ArrayList<>();
while (rs.next()) {
list.add(rs.getString(1));
}
return list;
}
}
@@ -0,0 +1,81 @@
/*
* Copyright 2026 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package org.conductoross.conductor.sqlite.dao;
import java.util.Base64;
import javax.sql.DataSource;
import org.conductoross.conductor.dao.SkillPackageDAO;
import org.springframework.retry.support.RetryTemplate;
import com.netflix.conductor.sqlite.dao.SqliteBaseDAO;
import com.fasterxml.jackson.databind.ObjectMapper;
/** SQLite {@link SkillPackageDAO} — table {@code skill_package} (Base64-encoded bytes). */
public class SqliteSkillPackageDAO extends SqliteBaseDAO implements SkillPackageDAO {
private static final String UPSERT =
"INSERT INTO skill_package (handle, data, size_bytes, created_at) VALUES (?, ?, ?, ?) "
+ "ON CONFLICT (handle) DO UPDATE SET data = excluded.data, size_bytes = excluded.size_bytes";
private static final String SELECT = "SELECT data FROM skill_package WHERE handle = ?";
private static final String EXISTS = "SELECT 1 FROM skill_package WHERE handle = ?";
private static final String DELETE = "DELETE FROM skill_package WHERE handle = ?";
public SqliteSkillPackageDAO(
RetryTemplate retryTemplate, ObjectMapper objectMapper, DataSource dataSource) {
super(retryTemplate, objectMapper, dataSource);
}
@Override
public void put(String handle, byte[] data) {
String encoded = Base64.getEncoder().encodeToString(data);
executeWithTransaction(
UPSERT,
q ->
q.addParameter(handle)
.addParameter(encoded)
.addParameter((long) data.length)
.addParameter(System.currentTimeMillis())
.executeUpdate());
}
@Override
public byte[] get(String handle) {
String encoded =
queryWithTransaction(
SELECT,
q ->
q.addParameter(handle)
.executeAndFetch(rs -> rs.next() ? rs.getString(1) : null));
return encoded == null ? null : Base64.getDecoder().decode(encoded);
}
@Override
public boolean exists(String handle) {
Boolean present =
queryWithTransaction(
EXISTS,
q -> q.addParameter(handle).executeAndFetch(java.sql.ResultSet::next));
return Boolean.TRUE.equals(present);
}
@Override
public void delete(String handle) {
executeWithTransaction(DELETE, q -> q.addParameter(handle).executeUpdate());
}
}
@@ -0,0 +1,187 @@
CREATE TABLE meta_event_handler (
id integer PRIMARY KEY AUTOINCREMENT,
name TEXT NOT NULL,
event TEXT NOT NULL,
active INTEGER NOT NULL,
json_data TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE locks (
lock_id TEXT NOT NULL,
lease_expiration DATETIME NOT NULL,
PRIMARY KEY (lock_id)
);
CREATE TABLE meta_workflow_def (
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
name TEXT NOT NULL,
version INTEGER NOT NULL,
latest_version INTEGER DEFAULT 0 NOT NULL,
json_data TEXT NOT NULL,
PRIMARY KEY (name, version)
);
CREATE INDEX workflow_def_name_index ON meta_workflow_def(name);
CREATE TABLE meta_task_def (
name TEXT NOT NULL PRIMARY KEY,
json_data TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP
);
-- Execution Tables
CREATE TABLE event_execution (
event_handler_name TEXT NOT NULL,
event_name TEXT NOT NULL,
execution_id TEXT NOT NULL,
message_id TEXT NOT NULL,
json_data TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (event_handler_name, event_name, execution_id)
);
CREATE TABLE poll_data (
queue_name TEXT NOT NULL,
domain TEXT NOT NULL,
json_data TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (queue_name, domain)
);
CREATE INDEX poll_data_queue_name_idx ON poll_data(queue_name);
CREATE TABLE task_scheduled (
workflow_id TEXT NOT NULL,
task_key TEXT NOT NULL,
task_id TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (workflow_id, task_key)
);
CREATE TABLE task_in_progress (
task_def_name TEXT NOT NULL,
task_id TEXT NOT NULL,
workflow_id TEXT NOT NULL,
in_progress_status INTEGER NOT NULL DEFAULT 0,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (task_def_name, task_id)
);
CREATE TABLE task (
task_id TEXT NOT NULL PRIMARY KEY,
json_data TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE workflow (
workflow_id TEXT NOT NULL PRIMARY KEY,
correlation_id TEXT,
json_data TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE workflow_def_to_workflow (
workflow_def TEXT NOT NULL,
date_str TEXT,
workflow_id TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (workflow_def, date_str, workflow_id)
);
CREATE TABLE workflow_pending (
workflow_type TEXT NOT NULL,
workflow_id TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (workflow_type, workflow_id)
);
CREATE INDEX workflow_type_index ON workflow_pending (workflow_type);
CREATE TABLE workflow_to_task (
workflow_id TEXT NOT NULL,
task_id TEXT NOT NULL,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP,
modified_on DATETIME DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (workflow_id, task_id)
);
CREATE INDEX workflow_id_index ON workflow_to_task(workflow_id);
-- Queue Tables
CREATE TABLE queue (
queue_name TEXT NOT NULL PRIMARY KEY,
created_on DATETIME DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE queue_message (
queue_name TEXT NOT NULL,
message_id TEXT NOT NULL,
deliver_on DATETIME DEFAULT CURRENT_TIMESTAMP,
priority INTEGER DEFAULT 0,
popped INTEGER DEFAULT 0,
offset_time_seconds INTEGER,
payload TEXT,
created_on DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP,
PRIMARY KEY (queue_name, message_id)
);
CREATE TABLE task_execution_logs (
log_id INTEGER PRIMARY KEY AUTOINCREMENT,
task_id TEXT NOT NULL,
log TEXT NOT NULL,
created_time DATETIME NOT NULL
);
CREATE INDEX task_execution_logs_task_id_idx ON task_execution_logs(task_id);
CREATE TABLE task_index (
task_id TEXT NOT NULL,
task_type TEXT NOT NULL,
task_def_name TEXT NOT NULL,
status TEXT NOT NULL,
start_time DATETIME NOT NULL,
update_time DATETIME NOT NULL,
workflow_type TEXT NOT NULL,
json_data TEXT NOT NULL,
PRIMARY KEY (task_id)
);
CREATE TABLE workflow_index (
workflow_id TEXT NOT NULL,
correlation_id TEXT,
workflow_type TEXT NOT NULL,
start_time DATETIME NOT NULL,
status TEXT NOT NULL,
json_data TEXT NOT NULL, -- SQLite doesn't have JSONB, storing as TEXT
update_time DATETIME NOT NULL DEFAULT '1970-01-01 00:00:00',
PRIMARY KEY (workflow_id)
);
-- Regular indexes
CREATE INDEX workflow_index_correlation_id_idx ON workflow_index(correlation_id);
CREATE INDEX workflow_index_start_time_idx ON workflow_index(start_time);
CREATE INDEX workflow_index_status_idx ON workflow_index(status);
CREATE INDEX workflow_index_workflow_type_idx ON workflow_index(workflow_type);
-- Regular indexes for columns
CREATE INDEX task_index_status_idx ON task_index(status);
CREATE INDEX task_index_task_def_name_idx ON task_index(task_def_name);
CREATE INDEX task_index_task_id_idx ON task_index(task_id);
CREATE INDEX task_index_task_type_idx ON task_index(task_type);
CREATE INDEX task_index_update_time_idx ON task_index(update_time);
CREATE INDEX task_index_workflow_type_idx ON task_index(workflow_type);
-- Indexes
CREATE INDEX IF NOT EXISTS idx_event_handler_name ON meta_event_handler (name);
CREATE INDEX IF NOT EXISTS idx_event_handler_event ON meta_event_handler (event);
CREATE INDEX IF NOT EXISTS idx_workflow_def_name ON meta_workflow_def (name);
CREATE INDEX IF NOT EXISTS idx_workflow_correlation ON workflow (correlation_id);
CREATE INDEX IF NOT EXISTS idx_queue_message_combo ON queue_message (queue_name, priority DESC, popped, deliver_on, created_on);
@@ -0,0 +1,2 @@
ALTER TABLE workflow_index ADD COLUMN parent_workflow_id TEXT NOT NULL DEFAULT '';
CREATE INDEX workflow_index_parent_workflow_id_idx ON workflow_index (parent_workflow_id);
@@ -0,0 +1,18 @@
CREATE TABLE IF NOT EXISTS file_metadata (
file_id VARCHAR(255) NOT NULL PRIMARY KEY,
file_name VARCHAR(1024) NOT NULL,
content_type VARCHAR(255) NOT NULL,
storage_content_hash VARCHAR(255),
storage_content_size BIGINT,
storage_type VARCHAR(50) NOT NULL,
storage_path VARCHAR(2048) NOT NULL,
upload_status VARCHAR(50) NOT NULL DEFAULT 'UPLOADING',
workflow_id VARCHAR(255) NOT NULL,
task_id VARCHAR(255),
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_file_metadata_workflow_id ON file_metadata (workflow_id);
CREATE INDEX IF NOT EXISTS idx_file_metadata_task_id ON file_metadata (task_id);
CREATE INDEX IF NOT EXISTS idx_file_metadata_upload_status ON file_metadata (upload_status);
@@ -0,0 +1,21 @@
-- AgentSpan skill storage (conductor.integrations.ai.enabled). Metadata + package bytes.
CREATE TABLE IF NOT EXISTS skill_metadata (
owner_id VARCHAR(255) NOT NULL,
name VARCHAR(255) NOT NULL,
version VARCHAR(255) NOT NULL,
is_latest BOOLEAN NOT NULL DEFAULT 0,
detail_json TEXT NOT NULL,
created_at BIGINT,
updated_at BIGINT,
PRIMARY KEY (owner_id, name, version)
);
CREATE INDEX IF NOT EXISTS idx_skill_metadata_owner_name ON skill_metadata (owner_id, name);
-- Package bytes are stored Base64-encoded so the value binds uniformly across backends.
CREATE TABLE IF NOT EXISTS skill_package (
handle VARCHAR(255) NOT NULL PRIMARY KEY,
data TEXT NOT NULL,
size_bytes BIGINT,
created_at BIGINT
);
@@ -0,0 +1,6 @@
-- Classifier of the workflow definition an execution was started from (e.g. 'workflow' for a
-- plain workflow, 'agent' for AgentSpan agents). Derived from WorkflowDef.metadata at index time.
-- Rows indexed before this migration keep a NULL classifier and are treated as untagged
-- ('workflow') by the search query builder.
ALTER TABLE workflow_index ADD COLUMN classifier TEXT;
CREATE INDEX workflow_index_classifier_idx ON workflow_index (classifier, start_time);
@@ -0,0 +1,114 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.util.List;
import org.flywaydb.core.Flyway;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
import com.netflix.conductor.dao.ExecutionDAO;
import com.netflix.conductor.dao.ExecutionDAOTest;
import com.netflix.conductor.model.WorkflowModel;
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
import com.google.common.collect.Iterables;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@ContextConfiguration(
classes = {
TestObjectMapperConfiguration.class,
SqliteConfiguration.class,
FlywayAutoConfiguration.class
})
@RunWith(SpringRunner.class)
@SpringBootTest(properties = "spring.flyway.clean-disabled=false")
public class SqliteExecutionDAOTest extends ExecutionDAOTest {
@Autowired private SqliteExecutionDAO executionDAO;
@Autowired Flyway flyway;
// clean the database between tests.
@Before
public void before() {
flyway.migrate();
}
@Test
public void testPendingByCorrelationId() {
WorkflowDef def = new WorkflowDef();
def.setName("pending_count_correlation_jtest");
WorkflowModel workflow = createTestWorkflow();
workflow.setWorkflowDefinition(def);
generateWorkflows(workflow, 10);
List<WorkflowModel> bycorrelationId =
getExecutionDAO()
.getWorkflowsByCorrelationId(
"pending_count_correlation_jtest", "corr001", true);
assertNotNull(bycorrelationId);
assertEquals(10, bycorrelationId.size());
}
@Test
public void testRemoveWorkflow() {
WorkflowDef def = new WorkflowDef();
def.setName("workflow");
WorkflowModel workflow = createTestWorkflow();
workflow.setWorkflowDefinition(def);
List<String> ids = generateWorkflows(workflow, 1);
assertEquals(1, getExecutionDAO().getPendingWorkflowCount("workflow"));
ids.forEach(wfId -> getExecutionDAO().removeWorkflow(wfId));
assertEquals(0, getExecutionDAO().getPendingWorkflowCount("workflow"));
}
@Test
public void testRemoveWorkflowWithExpiry() {
WorkflowDef def = new WorkflowDef();
def.setName("workflow");
WorkflowModel workflow = createTestWorkflow();
workflow.setWorkflowDefinition(def);
List<String> ids = generateWorkflows(workflow, 1);
final ExecutionDAO execDao = Mockito.spy(getExecutionDAO());
assertEquals(1, execDao.getPendingWorkflowCount("workflow"));
ids.forEach(wfId -> execDao.removeWorkflowWithExpiry(wfId, 1));
Mockito.verify(execDao, Mockito.timeout(10 * 1000)).removeWorkflow(Iterables.getLast(ids));
}
@Override
public ExecutionDAO getExecutionDAO() {
return executionDAO;
}
}
@@ -0,0 +1,863 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.io.File;
import java.sql.Connection;
import java.sql.SQLException;
import java.sql.Timestamp;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.time.temporal.TemporalAccessor;
import java.util.*;
import javax.sql.DataSource;
import org.flywaydb.core.Flyway;
import org.junit.Before;
import org.junit.Ignore;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
import com.netflix.conductor.common.metadata.tasks.Task;
import com.netflix.conductor.common.metadata.tasks.TaskExecLog;
import com.netflix.conductor.common.run.SearchResult;
import com.netflix.conductor.common.run.TaskSummary;
import com.netflix.conductor.common.run.Workflow;
import com.netflix.conductor.common.run.WorkflowSummary;
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
import com.netflix.conductor.sqlite.util.Query;
import com.fasterxml.jackson.databind.ObjectMapper;
import static org.junit.Assert.assertEquals;
@ContextConfiguration(
classes = {
TestObjectMapperConfiguration.class,
SqliteConfiguration.class,
FlywayAutoConfiguration.class
})
@RunWith(SpringRunner.class)
@TestPropertySource(
properties = {
"conductor.app.asyncIndexingEnabled=false",
"conductor.elasticsearch.version=0",
"conductor.indexing.type=sqlite",
"conductor.db.type=sqlite",
"spring.flyway.clean-disabled=false"
})
@SpringBootTest
@DirtiesContext(classMode = DirtiesContext.ClassMode.AFTER_EACH_TEST_METHOD)
public class SqliteIndexDAOTest {
@Autowired private SqliteIndexDAO indexDAO;
@Autowired private ObjectMapper objectMapper;
@Qualifier("dataSource")
@Autowired
private DataSource dataSource;
@Autowired Flyway flyway;
// clean the database between tests.
@Before
public void before() {
// Delete the database file if it exists
File dbFile = new File("conductorosstest.db");
if (dbFile.exists()) {
dbFile.delete();
}
// Also delete SQLite journal files if they exist
File dbJournal = new File("conductorosstest.db-journal");
if (dbJournal.exists()) {
dbJournal.delete();
}
File dbShm = new File("conductorosstest.db-shm");
if (dbShm.exists()) {
dbShm.delete();
}
File dbWal = new File("conductorosstest.db-wal");
if (dbWal.exists()) {
dbWal.delete();
}
flyway.clean();
flyway.migrate();
}
private WorkflowSummary getMockWorkflowSummary(String id) {
WorkflowSummary wfs = new WorkflowSummary();
wfs.setWorkflowId(id);
wfs.setCorrelationId("correlation-id");
wfs.setWorkflowType("workflow-type");
wfs.setStartTime("2023-02-07T08:42:45Z");
wfs.setUpdateTime("2023-02-07T08:43:45Z");
wfs.setStatus(Workflow.WorkflowStatus.COMPLETED);
return wfs;
}
private TaskSummary getMockTaskSummary(String taskId) {
TaskSummary ts = new TaskSummary();
ts.setTaskId(taskId);
ts.setTaskType("task-type1");
ts.setTaskDefName("task-def-name1");
ts.setStatus(Task.Status.COMPLETED);
ts.setStartTime("2023-02-07T09:41:45Z");
ts.setUpdateTime("2023-02-07T09:42:45Z");
ts.setWorkflowType("workflow-type");
return ts;
}
private TaskExecLog getMockTaskExecutionLog(String taskId, long createdTime, String log) {
TaskExecLog tse = new TaskExecLog();
tse.setTaskId(taskId);
tse.setLog(log);
tse.setCreatedTime(createdTime);
return tse;
}
private void compareWorkflowSummary(WorkflowSummary wfs) throws SQLException {
List<Map<String, Object>> result =
queryDb(
String.format(
"SELECT * FROM workflow_index WHERE workflow_id = '%s'",
wfs.getWorkflowId()));
assertEquals("Wrong number of rows returned", 1, result.size());
assertEquals(
"Workflow id does not match",
wfs.getWorkflowId(),
result.get(0).get("workflow_id"));
assertEquals(
"Correlation id does not match",
wfs.getCorrelationId(),
result.get(0).get("correlation_id"));
assertEquals(
"Workflow type does not match",
wfs.getWorkflowType(),
result.get(0).get("workflow_type"));
TemporalAccessor ta = DateTimeFormatter.ISO_INSTANT.parse(wfs.getStartTime());
Timestamp startTime = Timestamp.from(Instant.from(ta));
assertEquals(
"Start time does not match", startTime.toString(), result.get(0).get("start_time"));
assertEquals(
"Status does not match", wfs.getStatus().toString(), result.get(0).get("status"));
}
private List<Map<String, Object>> queryDb(String query) throws SQLException {
try (Connection c = dataSource.getConnection()) {
try (Query q = new Query(objectMapper, c, query)) {
return q.executeAndFetchMap();
}
}
}
private void compareTaskSummary(TaskSummary ts) throws SQLException {
List<Map<String, Object>> result =
queryDb(
String.format(
"SELECT * FROM task_index WHERE task_id = '%s'", ts.getTaskId()));
assertEquals("Wrong number of rows returned", 1, result.size());
assertEquals("Task id does not match", ts.getTaskId(), result.get(0).get("task_id"));
assertEquals("Task type does not match", ts.getTaskType(), result.get(0).get("task_type"));
assertEquals(
"Task def name does not match",
ts.getTaskDefName(),
result.get(0).get("task_def_name"));
TemporalAccessor startTa = DateTimeFormatter.ISO_INSTANT.parse(ts.getStartTime());
Timestamp startTime = Timestamp.from(Instant.from(startTa));
assertEquals(
"Start time does not match", startTime.toString(), result.get(0).get("start_time"));
TemporalAccessor updateTa = DateTimeFormatter.ISO_INSTANT.parse(ts.getUpdateTime());
Timestamp updateTime = Timestamp.from(Instant.from(updateTa));
assertEquals(
"Update time does not match",
updateTime.toString(),
result.get(0).get("update_time"));
assertEquals(
"Status does not match", ts.getStatus().toString(), result.get(0).get("status"));
assertEquals(
"Workflow type does not match",
ts.getWorkflowType().toString(),
result.get(0).get("workflow_type"));
}
@Test
public void testIndexNewWorkflow() throws SQLException {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-new");
indexDAO.indexWorkflow(wfs);
compareWorkflowSummary(wfs);
}
@Test
public void testIndexExistingWorkflow() throws SQLException {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-existing");
indexDAO.indexWorkflow(wfs);
compareWorkflowSummary(wfs);
wfs.setStatus(Workflow.WorkflowStatus.FAILED);
wfs.setUpdateTime("2023-02-07T08:44:45Z");
indexDAO.indexWorkflow(wfs);
compareWorkflowSummary(wfs);
}
@Test
public void testWhenWorkflowIsIndexedOutOfOrderOnlyLatestIsIndexed() throws SQLException {
WorkflowSummary firstWorkflowUpdate =
getMockWorkflowSummary("workflow-id-existing-no-index");
firstWorkflowUpdate.setUpdateTime("2023-02-07T08:42:45Z");
WorkflowSummary secondWorkflowUpdateSummary =
getMockWorkflowSummary("workflow-id-existing-no-index");
secondWorkflowUpdateSummary.setUpdateTime("2023-02-07T08:43:45Z");
secondWorkflowUpdateSummary.setStatus(Workflow.WorkflowStatus.FAILED);
indexDAO.indexWorkflow(secondWorkflowUpdateSummary);
compareWorkflowSummary(secondWorkflowUpdateSummary);
indexDAO.indexWorkflow(firstWorkflowUpdate);
compareWorkflowSummary(secondWorkflowUpdateSummary);
}
@Test
public void testWhenWorkflowUpdatesHaveTheSameUpdateTimeTheLastIsIndexed() throws SQLException {
WorkflowSummary firstWorkflowUpdate =
getMockWorkflowSummary("workflow-id-existing-same-time-index");
firstWorkflowUpdate.setUpdateTime("2023-02-07T08:42:45Z");
WorkflowSummary secondWorkflowUpdateSummary =
getMockWorkflowSummary("workflow-id-existing-same-time-index");
secondWorkflowUpdateSummary.setUpdateTime("2023-02-07T08:42:45Z");
secondWorkflowUpdateSummary.setStatus(Workflow.WorkflowStatus.FAILED);
indexDAO.indexWorkflow(firstWorkflowUpdate);
compareWorkflowSummary(firstWorkflowUpdate);
indexDAO.indexWorkflow(secondWorkflowUpdateSummary);
compareWorkflowSummary(secondWorkflowUpdateSummary);
}
@Test
public void testIndexNewTask() throws SQLException {
TaskSummary ts = getMockTaskSummary("task-id-new");
indexDAO.indexTask(ts);
compareTaskSummary(ts);
}
@Test
public void testIndexExistingTask() throws SQLException {
TaskSummary ts = getMockTaskSummary("task-id-existing");
indexDAO.indexTask(ts);
compareTaskSummary(ts);
ts.setUpdateTime("2023-02-07T09:43:45Z");
ts.setStatus(Task.Status.FAILED);
indexDAO.indexTask(ts);
compareTaskSummary(ts);
}
@Test
public void testWhenTaskIsIndexedOutOfOrderOnlyLatestIsIndexed() throws SQLException {
TaskSummary firstTaskState = getMockTaskSummary("task-id-exiting-no-update");
firstTaskState.setUpdateTime("2023-02-07T09:41:45Z");
firstTaskState.setStatus(Task.Status.FAILED);
TaskSummary secondTaskState = getMockTaskSummary("task-id-exiting-no-update");
secondTaskState.setUpdateTime("2023-02-07T09:42:45Z");
indexDAO.indexTask(secondTaskState);
compareTaskSummary(secondTaskState);
indexDAO.indexTask(firstTaskState);
compareTaskSummary(secondTaskState);
}
@Test
public void testWhenTaskUpdatesHaveTheSameUpdateTimeTheLastIsIndexed() throws SQLException {
TaskSummary firstTaskState = getMockTaskSummary("task-id-exiting-same-time-update");
firstTaskState.setUpdateTime("2023-02-07T09:42:45Z");
firstTaskState.setStatus(Task.Status.FAILED);
TaskSummary secondTaskState = getMockTaskSummary("task-id-exiting-same-time-update");
secondTaskState.setUpdateTime("2023-02-07T09:42:45Z");
indexDAO.indexTask(firstTaskState);
compareTaskSummary(firstTaskState);
indexDAO.indexTask(secondTaskState);
compareTaskSummary(secondTaskState);
}
@Test
public void testAddTaskExecutionLogs() throws SQLException {
List<TaskExecLog> logs = new ArrayList<>();
String taskId = UUID.randomUUID().toString();
logs.add(getMockTaskExecutionLog(taskId, 1675845986000L, "Log 1"));
logs.add(getMockTaskExecutionLog(taskId, 1675845987000L, "Log 2"));
indexDAO.addTaskExecutionLogs(logs);
List<Map<String, Object>> records =
queryDb(
"SELECT * FROM task_execution_logs where task_id = '"
+ taskId
+ "' ORDER BY created_time ASC");
assertEquals("Wrong number of logs returned", 2, records.size());
assertEquals(logs.get(0).getLog(), records.get(0).get("log"));
assertEquals(1675845986000L, records.get(0).get("created_time"));
assertEquals(logs.get(1).getLog(), records.get(1).get("log"));
assertEquals(1675845987000L, records.get(1).get("created_time"));
}
@Test
public void testSearchWorkflowSummary() {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id");
indexDAO.indexWorkflow(wfs);
String query = String.format("workflowId=\"%s\"", wfs.getWorkflowId());
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList());
assertEquals("No results returned", 1, results.getResults().size());
assertEquals(
"Wrong workflow returned",
wfs.getWorkflowId(),
results.getResults().get(0).getWorkflowId());
}
@Test
public void testSearchWorkflowSummaryByClassifier() {
String correlationId = "classifier-search-correlation-id";
WorkflowSummary agentWfs = getMockWorkflowSummary("workflow-id-classifier-agent");
agentWfs.setCorrelationId(correlationId);
agentWfs.setClassifier("agent");
indexDAO.indexWorkflow(agentWfs);
WorkflowSummary plainWfs = getMockWorkflowSummary("workflow-id-classifier-plain");
plainWfs.setCorrelationId(correlationId);
plainWfs.setClassifier("workflow");
indexDAO.indexWorkflow(plainWfs);
// Simulates a row indexed before the classifier column existed (NULL classifier).
WorkflowSummary legacyWfs = getMockWorkflowSummary("workflow-id-classifier-legacy");
legacyWfs.setCorrelationId(correlationId);
indexDAO.indexWorkflow(legacyWfs);
String agentQuery =
String.format("correlationId='%s' AND classifier='agent'", correlationId);
SearchResult<WorkflowSummary> agentResults =
indexDAO.searchWorkflowSummary(agentQuery, "*", 0, 15, new ArrayList<>());
assertEquals("Wrong number of agent results", 1, agentResults.getResults().size());
assertEquals(
"Wrong workflow returned",
agentWfs.getWorkflowId(),
agentResults.getResults().get(0).getWorkflowId());
// The untagged token must match both explicitly tagged plain workflows and legacy
// NULL rows.
String workflowQuery =
String.format("correlationId='%s' AND classifier='workflow'", correlationId);
SearchResult<WorkflowSummary> workflowResults =
indexDAO.searchWorkflowSummary(workflowQuery, "*", 0, 15, new ArrayList<>());
assertEquals("Wrong number of untagged results", 2, workflowResults.getResults().size());
String inQuery =
String.format("correlationId='%s' AND classifier IN (agent,other)", correlationId);
SearchResult<WorkflowSummary> inResults =
indexDAO.searchWorkflowSummary(inQuery, "*", 0, 15, new ArrayList<>());
assertEquals("Wrong number of IN clause results", 1, inResults.getResults().size());
indexDAO.removeWorkflow(agentWfs.getWorkflowId());
indexDAO.removeWorkflow(plainWfs.getWorkflowId());
indexDAO.removeWorkflow(legacyWfs.getWorkflowId());
}
@Test
public void testFullTextSearchWorkflowSummary() {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id");
indexDAO.indexWorkflow(wfs);
String freeText = "notworkflow-id";
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary("", freeText, 0, 15, new ArrayList<>());
assertEquals("Wrong number of results returned", 0, results.getResults().size());
freeText = "workflow-id";
results = indexDAO.searchWorkflowSummary("", freeText, 0, 15, new ArrayList<>());
assertEquals("No results returned", 1, results.getResults().size());
assertEquals(
"Wrong workflow returned",
wfs.getWorkflowId(),
results.getResults().getFirst().getWorkflowId());
}
// json working not working
// @Test
// public void testJsonSearchWorkflowSummary() {
// WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-summary");
// wfs.setVersion(3);
//
// indexDAO.indexWorkflow(wfs);
//
// String freeText = "{\"correlationId\":\"not-the-id\"}";
// SearchResult<WorkflowSummary> results =
// indexDAO.searchWorkflowSummary("", freeText, 0, 15, new ArrayList());
// assertEquals("Wrong number of results returned", 0, results.getResults().size());
//
// freeText = "{\"correlationId\":\"correlation-id\", \"version\":3}";
// results = indexDAO.searchWorkflowSummary("", freeText, 0, 15, new ArrayList());
// assertEquals("No results returned", 1, results.getResults().size());
// assertEquals(
// "Wrong workflow returned",
// wfs.getWorkflowId(),
// results.getResults().get(0).getWorkflowId());
// }
@Test
public void testSearchWorkflowSummaryPagination() {
for (int i = 0; i < 5; i++) {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-pagination-" + i);
indexDAO.indexWorkflow(wfs);
}
List<String> orderBy = Arrays.asList(new String[] {"workflowId:DESC"});
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary("", "workflow-id-pagination", 0, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
"workflow-id-pagination-4",
results.getResults().get(0).getWorkflowId());
assertEquals(
"Results returned in wrong order",
"workflow-id-pagination-3",
results.getResults().get(1).getWorkflowId());
results = indexDAO.searchWorkflowSummary("", "*", 2, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
"workflow-id-pagination-2",
results.getResults().get(0).getWorkflowId());
assertEquals(
"Results returned in wrong order",
"workflow-id-pagination-1",
results.getResults().get(1).getWorkflowId());
results = indexDAO.searchWorkflowSummary("", "*", 4, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 1, results.getResults().size());
assertEquals(
"Results returned in wrong order",
"workflow-id-pagination-0",
results.getResults().get(0).getWorkflowId());
}
@Test
public void testSearchWorkflows() {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-v2");
indexDAO.indexWorkflow(wfs);
String query = String.format("workflowId=\"%s\"", wfs.getWorkflowId());
SearchResult<String> results =
indexDAO.searchWorkflows(query, "*", 0, 15, new ArrayList<>());
assertEquals("No results returned", 1, results.getResults().size());
assertEquals(
"Wrong workflow id returned", wfs.getWorkflowId(), results.getResults().get(0));
}
@Test
public void testSearchWorkflowsPagination() {
for (int i = 0; i < 5; i++) {
WorkflowSummary wfs = getMockWorkflowSummary("wf-v2-pagination-" + i);
indexDAO.indexWorkflow(wfs);
}
List<String> orderBy = Arrays.asList(new String[] {"workflowId:DESC"});
SearchResult<String> results = indexDAO.searchWorkflows("", "*", 0, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
"wf-v2-pagination-4",
results.getResults().get(0));
assertEquals(
"Results returned in wrong order",
"wf-v2-pagination-3",
results.getResults().get(1));
}
@Test
public void testSearchWorkflowsWithNullSort() {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-null-sort");
indexDAO.indexWorkflow(wfs);
String query = String.format("workflowId=\"%s\"", wfs.getWorkflowId());
SearchResult<String> results = indexDAO.searchWorkflows(query, "*", 0, 15, null);
assertEquals("No results returned", 1, results.getResults().size());
assertEquals(
"Wrong workflow id returned", wfs.getWorkflowId(), results.getResults().get(0));
}
@Test
public void testSearchWorkflowSummaryWithSingleQuotes() {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-single-quote");
indexDAO.indexWorkflow(wfs);
String query = String.format("workflowId='%s'", wfs.getWorkflowId());
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
assertEquals("No results returned", 1, results.getResults().size());
assertEquals(
"Wrong workflow returned",
wfs.getWorkflowId(),
results.getResults().get(0).getWorkflowId());
}
@Test
public void testSearchWorkflowsWithSingleQuotes() {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-v2-single-quote");
indexDAO.indexWorkflow(wfs);
String query = String.format("workflowId='%s'", wfs.getWorkflowId());
SearchResult<String> results =
indexDAO.searchWorkflows(query, "*", 0, 15, new ArrayList<>());
assertEquals("No results returned", 1, results.getResults().size());
assertEquals(
"Wrong workflow id returned", wfs.getWorkflowId(), results.getResults().get(0));
}
@Test
public void testSearchWorkflowsWithSingleQuotedMultiCondition() {
WorkflowSummary wfs = getMockWorkflowSummary("workflow-id-multi-cond");
wfs.setCorrelationId("test-correlation-id");
indexDAO.indexWorkflow(wfs);
String query =
String.format(
"correlationId='%s' AND workflowType='%s'",
wfs.getCorrelationId(), wfs.getWorkflowType());
SearchResult<String> results =
indexDAO.searchWorkflows(query, "*", 0, 15, new ArrayList<>());
assertEquals("No results returned", 1, results.getResults().size());
assertEquals(
"Wrong workflow id returned", wfs.getWorkflowId(), results.getResults().get(0));
}
@Test
public void testSearchTasks() {
TaskSummary ts = getMockTaskSummary("task-id-v2");
indexDAO.indexTask(ts);
String query = String.format("taskId=\"%s\"", ts.getTaskId());
SearchResult<String> results = indexDAO.searchTasks(query, "*", 0, 15, new ArrayList<>());
assertEquals("No results returned", 1, results.getResults().size());
assertEquals("Wrong task id returned", ts.getTaskId(), results.getResults().get(0));
}
@Test
public void testSearchTasksPagination() {
for (int i = 0; i < 5; i++) {
TaskSummary ts = getMockTaskSummary("task-v2-pagination-" + i);
indexDAO.indexTask(ts);
}
List<String> orderBy = Arrays.asList(new String[] {"taskId:DESC"});
SearchResult<String> results = indexDAO.searchTasks("", "*", 0, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
"task-v2-pagination-4",
results.getResults().get(0));
assertEquals(
"Results returned in wrong order",
"task-v2-pagination-3",
results.getResults().get(1));
}
@Test
public void testSearchTaskSummary() {
TaskSummary ts = getMockTaskSummary("task-id");
indexDAO.indexTask(ts);
String query = String.format("taskId=\"%s\"", ts.getTaskId());
SearchResult<TaskSummary> results =
indexDAO.searchTaskSummary(query, "*", 0, 15, new ArrayList());
assertEquals("No results returned", 1, results.getResults().size());
assertEquals(
"Wrong task returned", ts.getTaskId(), results.getResults().get(0).getTaskId());
}
@Test
public void testSearchTaskSummaryPagination() {
for (int i = 0; i < 5; i++) {
TaskSummary ts = getMockTaskSummary("task-id-pagination-" + i);
indexDAO.indexTask(ts);
}
List<String> orderBy = Arrays.asList(new String[] {"taskId:DESC"});
SearchResult<TaskSummary> results = indexDAO.searchTaskSummary("", "*", 0, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
"task-id-pagination-4",
results.getResults().get(0).getTaskId());
assertEquals(
"Results returned in wrong order",
"task-id-pagination-3",
results.getResults().get(1).getTaskId());
results = indexDAO.searchTaskSummary("", "*", 2, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 2, results.getResults().size());
assertEquals(
"Results returned in wrong order",
"task-id-pagination-2",
results.getResults().get(0).getTaskId());
assertEquals(
"Results returned in wrong order",
"task-id-pagination-1",
results.getResults().get(1).getTaskId());
results = indexDAO.searchTaskSummary("", "*", 4, 2, orderBy);
assertEquals("Wrong totalHits returned", 5, results.getTotalHits());
assertEquals("Wrong number of results returned", 1, results.getResults().size());
assertEquals(
"Results returned in wrong order",
"task-id-pagination-0",
results.getResults().get(0).getTaskId());
}
@Test
public void testGetTaskExecutionLogs() throws SQLException {
List<TaskExecLog> logs = new ArrayList<>();
String taskId = UUID.randomUUID().toString();
logs.add(getMockTaskExecutionLog(taskId, new Date(1675845986000L).getTime(), "Log 1"));
logs.add(getMockTaskExecutionLog(taskId, new Date(1675845987000L).getTime(), "Log 2"));
indexDAO.addTaskExecutionLogs(logs);
List<TaskExecLog> records = indexDAO.getTaskExecutionLogs(logs.get(0).getTaskId());
assertEquals("Wrong number of logs returned", 2, records.size());
assertEquals(logs.get(0).getLog(), records.get(0).getLog());
assertEquals(logs.get(0).getCreatedTime(), 1675845986000L);
assertEquals(logs.get(1).getLog(), records.get(1).getLog());
assertEquals(logs.get(1).getCreatedTime(), 1675845987000L);
}
@Test
public void testRemoveWorkflow() throws SQLException {
String workflowId = UUID.randomUUID().toString();
WorkflowSummary wfs = getMockWorkflowSummary(workflowId);
indexDAO.indexWorkflow(wfs);
List<Map<String, Object>> workflow_records =
queryDb("SELECT * FROM workflow_index WHERE workflow_id = '" + workflowId + "'");
assertEquals("Workflow index record was not created", 1, workflow_records.size());
indexDAO.removeWorkflow(workflowId);
workflow_records =
queryDb("SELECT * FROM workflow_index WHERE workflow_id = '" + workflowId + "'");
assertEquals("Workflow index record was not deleted", 0, workflow_records.size());
}
@Test
@Ignore("Skipping due to SQLite database connection issues in test environment")
public void testRemoveTask() throws SQLException {
// Ensure database is properly initialized
flyway.clean();
flyway.migrate();
String workflowId = UUID.randomUUID().toString();
String taskId = UUID.randomUUID().toString();
TaskSummary ts = getMockTaskSummary(taskId);
indexDAO.indexTask(ts);
List<TaskExecLog> logs = new ArrayList<>();
logs.add(getMockTaskExecutionLog(taskId, new Date(1675845986000L).getTime(), "Log 1"));
logs.add(getMockTaskExecutionLog(taskId, new Date(1675845987000L).getTime(), "Log 2"));
indexDAO.addTaskExecutionLogs(logs);
List<Map<String, Object>> task_records =
queryDb("SELECT * FROM task_index WHERE task_id = '" + taskId + "'");
assertEquals("Task index record was not created", 1, task_records.size());
List<Map<String, Object>> log_records =
queryDb("SELECT * FROM task_execution_logs WHERE task_id = '" + taskId + "'");
assertEquals("Task execution logs were not created", 2, log_records.size());
indexDAO.removeTask(workflowId, taskId);
task_records = queryDb("SELECT * FROM task_index WHERE task_id = '" + taskId + "'");
assertEquals("Task index record was not deleted", 0, task_records.size());
log_records = queryDb("SELECT * FROM task_execution_logs WHERE task_id = '" + taskId + "'");
assertEquals("Task execution logs were not deleted", 0, log_records.size());
}
private WorkflowSummary getMockWorkflowSummary(String id, String parentWorkflowId) {
WorkflowSummary wfs = getMockWorkflowSummary(id);
wfs.setParentWorkflowId(parentWorkflowId);
return wfs;
}
@Test
public void testWildcardSearchWorkflowSummaryByType() {
WorkflowSummary wfs1 = getMockWorkflowSummary("wf-wildcard-1");
wfs1.setWorkflowType("order_processing_v1");
indexDAO.indexWorkflow(wfs1);
WorkflowSummary wfs2 = getMockWorkflowSummary("wf-wildcard-2");
wfs2.setWorkflowType("order_processing_v2");
indexDAO.indexWorkflow(wfs2);
WorkflowSummary wfs3 = getMockWorkflowSummary("wf-wildcard-3");
wfs3.setWorkflowType("payment_processing_v1");
indexDAO.indexWorkflow(wfs3);
String query = "workflowType=order_processing*";
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
assertEquals("Should find 2 order_processing workflows", 2, results.getResults().size());
query = "workflowType=*processing*";
results = indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
assertEquals("Should find 3 processing workflows", 3, results.getResults().size());
}
@Test
public void testWildcardSearchNoMatches() {
WorkflowSummary wfs = getMockWorkflowSummary("wf-wildcard-nomatch");
wfs.setWorkflowType("order_processing_v1");
indexDAO.indexWorkflow(wfs);
String query = "workflowType=payment*";
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
assertEquals("Should find 0 workflows", 0, results.getResults().size());
}
@Test
public void testSearchExcludeSubWorkflows() {
WorkflowSummary topLevel1 = getMockWorkflowSummary("wf-top-1", "");
indexDAO.indexWorkflow(topLevel1);
WorkflowSummary topLevel2 = getMockWorkflowSummary("wf-top-2", "");
indexDAO.indexWorkflow(topLevel2);
WorkflowSummary subWf1 = getMockWorkflowSummary("wf-sub-1", "wf-top-1");
indexDAO.indexWorkflow(subWf1);
WorkflowSummary subWf2 = getMockWorkflowSummary("wf-sub-2", "wf-top-1");
indexDAO.indexWorkflow(subWf2);
String query = "parentWorkflowId=\"\"";
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
assertEquals("Should find only 2 top-level workflows", 2, results.getResults().size());
}
@Test
public void testSearchSubWorkflowsOfParent() {
WorkflowSummary topLevel = getMockWorkflowSummary("wf-parent-1", "");
indexDAO.indexWorkflow(topLevel);
WorkflowSummary subWf1 = getMockWorkflowSummary("wf-child-1", "wf-parent-1");
indexDAO.indexWorkflow(subWf1);
WorkflowSummary subWf2 = getMockWorkflowSummary("wf-child-2", "wf-parent-1");
indexDAO.indexWorkflow(subWf2);
WorkflowSummary subWf3 = getMockWorkflowSummary("wf-child-3", "wf-parent-other");
indexDAO.indexWorkflow(subWf3);
String query = "parentWorkflowId=\"wf-parent-1\"";
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
assertEquals("Should find 2 child workflows", 2, results.getResults().size());
}
@Test
public void testSearchSubWorkflowsWrongParentReturnsEmpty() {
WorkflowSummary subWf1 = getMockWorkflowSummary("wf-orphan-1", "wf-parent-1");
indexDAO.indexWorkflow(subWf1);
String query = "parentWorkflowId=\"wf-nonexistent-parent\"";
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
assertEquals(
"Should find 0 workflows for nonexistent parent", 0, results.getResults().size());
}
@Test
public void testWildcardWithParentWorkflowIdFilter() {
WorkflowSummary topOrder = getMockWorkflowSummary("wf-combined-1", "");
topOrder.setWorkflowType("order_processing_v1");
indexDAO.indexWorkflow(topOrder);
WorkflowSummary topPayment = getMockWorkflowSummary("wf-combined-2", "");
topPayment.setWorkflowType("payment_processing_v1");
indexDAO.indexWorkflow(topPayment);
WorkflowSummary subOrder = getMockWorkflowSummary("wf-combined-3", "wf-combined-1");
subOrder.setWorkflowType("order_processing_v1");
indexDAO.indexWorkflow(subOrder);
String query = "parentWorkflowId=\"\" AND workflowType=order*";
SearchResult<WorkflowSummary> results =
indexDAO.searchWorkflowSummary(query, "*", 0, 15, new ArrayList<>());
assertEquals("Should find 1 top-level order workflow", 1, results.getResults().size());
assertEquals("wf-combined-1", results.getResults().get(0).getWorkflowId());
}
}
@@ -0,0 +1,318 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.util.*;
import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.commons.lang3.builder.EqualsBuilder;
import org.flywaydb.core.Flyway;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
import com.netflix.conductor.common.metadata.events.EventHandler;
import com.netflix.conductor.common.metadata.tasks.TaskDef;
import com.netflix.conductor.common.metadata.workflow.WorkflowDef;
import com.netflix.conductor.core.exception.NonTransientException;
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
import com.netflix.conductor.sqlite.dao.metadata.SqliteMetadataDAO;
import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
@ContextConfiguration(
classes = {
TestObjectMapperConfiguration.class,
SqliteConfiguration.class,
FlywayAutoConfiguration.class
})
@RunWith(SpringRunner.class)
@SpringBootTest(properties = "spring.flyway.clean-disabled=false")
public class SqliteMetadataDAOTest {
@Autowired private SqliteMetadataDAO metadataDAO;
@Rule public TestName name = new TestName();
@Autowired private Flyway flyway;
@Before
public void before() {
flyway.migrate();
}
@Test
public void testDuplicateWorkflowDef() {
WorkflowDef def = new WorkflowDef();
def.setName("testDuplicate");
def.setVersion(1);
metadataDAO.createWorkflowDef(def);
NonTransientException applicationException =
assertThrows(NonTransientException.class, () -> metadataDAO.createWorkflowDef(def));
assertEquals(
"Workflow with testDuplicate.1 already exists!", applicationException.getMessage());
}
@Test
public void testRemoveNotExistingWorkflowDef() {
NonTransientException applicationException =
assertThrows(
NonTransientException.class,
() -> metadataDAO.removeWorkflowDef("test", 1));
assertEquals(
"No such workflow definition: test version: 1", applicationException.getMessage());
}
@Test
public void testWorkflowDefOperations() {
WorkflowDef def = new WorkflowDef();
def.setName("test");
def.setVersion(1);
def.setDescription("description");
def.setCreatedBy("unit_test");
def.setCreateTime(1L);
def.setOwnerApp("ownerApp");
def.setUpdatedBy("unit_test2");
def.setUpdateTime(2L);
metadataDAO.createWorkflowDef(def);
List<WorkflowDef> all = metadataDAO.getAllWorkflowDefs();
assertNotNull(all);
assertEquals(1, all.size());
assertEquals("test", all.get(0).getName());
assertEquals(1, all.get(0).getVersion());
WorkflowDef found = metadataDAO.getWorkflowDef("test", 1).get();
assertTrue(EqualsBuilder.reflectionEquals(def, found));
def.setVersion(3);
metadataDAO.createWorkflowDef(def);
all = metadataDAO.getAllWorkflowDefs();
assertNotNull(all);
assertEquals(2, all.size());
assertEquals("test", all.get(0).getName());
assertEquals(1, all.get(0).getVersion());
found = metadataDAO.getLatestWorkflowDef(def.getName()).get();
assertEquals(def.getName(), found.getName());
assertEquals(def.getVersion(), found.getVersion());
assertEquals(3, found.getVersion());
all = metadataDAO.getAllLatest();
assertNotNull(all);
assertEquals(1, all.size());
assertEquals("test", all.get(0).getName());
assertEquals(3, all.get(0).getVersion());
all = metadataDAO.getAllVersions(def.getName());
assertNotNull(all);
assertEquals(2, all.size());
assertEquals("test", all.get(0).getName());
assertEquals("test", all.get(1).getName());
assertEquals(1, all.get(0).getVersion());
assertEquals(3, all.get(1).getVersion());
def.setDescription("updated");
metadataDAO.updateWorkflowDef(def);
found = metadataDAO.getWorkflowDef(def.getName(), def.getVersion()).get();
assertEquals(def.getDescription(), found.getDescription());
List<String> allnames = metadataDAO.findAll();
assertNotNull(allnames);
assertEquals(1, allnames.size());
assertEquals(def.getName(), allnames.get(0));
def.setVersion(2);
metadataDAO.createWorkflowDef(def);
found = metadataDAO.getLatestWorkflowDef(def.getName()).get();
assertEquals(def.getName(), found.getName());
assertEquals(3, found.getVersion());
metadataDAO.removeWorkflowDef("test", 3);
Optional<WorkflowDef> deleted = metadataDAO.getWorkflowDef("test", 3);
assertFalse(deleted.isPresent());
found = metadataDAO.getLatestWorkflowDef(def.getName()).get();
assertEquals(def.getName(), found.getName());
assertEquals(2, found.getVersion());
metadataDAO.removeWorkflowDef("test", 1);
deleted = metadataDAO.getWorkflowDef("test", 1);
assertFalse(deleted.isPresent());
found = metadataDAO.getLatestWorkflowDef(def.getName()).get();
assertEquals(def.getName(), found.getName());
assertEquals(2, found.getVersion());
}
@Test
public void testTaskDefOperations() {
TaskDef def = new TaskDef("taskA");
def.setDescription("description");
def.setCreatedBy("unit_test");
def.setCreateTime(1L);
def.setInputKeys(Arrays.asList("a", "b", "c"));
def.setOutputKeys(Arrays.asList("01", "o2"));
def.setOwnerApp("ownerApp");
def.setRetryCount(3);
def.setRetryDelaySeconds(100);
def.setRetryLogic(TaskDef.RetryLogic.FIXED);
def.setTimeoutPolicy(TaskDef.TimeoutPolicy.ALERT_ONLY);
def.setUpdatedBy("unit_test2");
def.setUpdateTime(2L);
def.setRateLimitFrequencyInSeconds(1);
def.setRateLimitPerFrequency(1);
metadataDAO.createTaskDef(def);
TaskDef found = metadataDAO.getTaskDef(def.getName());
assertTrue(EqualsBuilder.reflectionEquals(def, found));
def.setDescription("updated description");
metadataDAO.updateTaskDef(def);
found = metadataDAO.getTaskDef(def.getName());
assertTrue(EqualsBuilder.reflectionEquals(def, found));
assertEquals("updated description", found.getDescription());
for (int i = 0; i < 9; i++) {
TaskDef tdf = new TaskDef("taskA" + i);
metadataDAO.createTaskDef(tdf);
}
List<TaskDef> all = metadataDAO.getAllTaskDefs();
assertNotNull(all);
assertEquals(10, all.size());
Set<String> allnames = all.stream().map(TaskDef::getName).collect(Collectors.toSet());
assertEquals(10, allnames.size());
List<String> sorted = allnames.stream().sorted().collect(Collectors.toList());
assertEquals(def.getName(), sorted.get(0));
for (int i = 0; i < 9; i++) {
assertEquals(def.getName() + i, sorted.get(i + 1));
}
for (int i = 0; i < 9; i++) {
metadataDAO.removeTaskDef(def.getName() + i);
}
all = metadataDAO.getAllTaskDefs();
assertNotNull(all);
assertEquals(1, all.size());
assertEquals(def.getName(), all.get(0).getName());
}
@Test
public void testRemoveNotExistingTaskDef() {
NonTransientException applicationException =
assertThrows(
NonTransientException.class,
() -> metadataDAO.removeTaskDef("test" + UUID.randomUUID().toString()));
assertEquals("No such task definition", applicationException.getMessage());
}
@Test
public void testEventHandlers() {
String event1 = "SQS::arn:account090:sqstest1";
String event2 = "SQS::arn:account090:sqstest2";
EventHandler eventHandler = new EventHandler();
eventHandler.setName(UUID.randomUUID().toString());
eventHandler.setActive(false);
EventHandler.Action action = new EventHandler.Action();
action.setAction(EventHandler.Action.Type.start_workflow);
action.setStart_workflow(new EventHandler.StartWorkflow());
action.getStart_workflow().setName("workflow_x");
eventHandler.getActions().add(action);
eventHandler.setEvent(event1);
metadataDAO.addEventHandler(eventHandler);
List<EventHandler> all = metadataDAO.getAllEventHandlers();
assertNotNull(all);
assertEquals(1, all.size());
assertEquals(eventHandler.getName(), all.get(0).getName());
assertEquals(eventHandler.getEvent(), all.get(0).getEvent());
List<EventHandler> byEvents = metadataDAO.getEventHandlersForEvent(event1, true);
assertNotNull(byEvents);
assertEquals(0, byEvents.size()); // event is marked as in-active
eventHandler.setActive(true);
eventHandler.setEvent(event2);
metadataDAO.updateEventHandler(eventHandler);
all = metadataDAO.getAllEventHandlers();
assertNotNull(all);
assertEquals(1, all.size());
byEvents = metadataDAO.getEventHandlersForEvent(event1, true);
assertNotNull(byEvents);
assertEquals(0, byEvents.size());
byEvents = metadataDAO.getEventHandlersForEvent(event2, true);
assertNotNull(byEvents);
assertEquals(1, byEvents.size());
}
@Test
public void testGetAllWorkflowDefsLatestVersions() {
WorkflowDef def = new WorkflowDef();
def.setName("test1");
def.setVersion(1);
def.setDescription("description");
def.setCreatedBy("unit_test");
def.setCreateTime(1L);
def.setOwnerApp("ownerApp");
def.setUpdatedBy("unit_test2");
def.setUpdateTime(2L);
metadataDAO.createWorkflowDef(def);
def.setName("test2");
metadataDAO.createWorkflowDef(def);
def.setVersion(2);
metadataDAO.createWorkflowDef(def);
def.setName("test3");
def.setVersion(1);
metadataDAO.createWorkflowDef(def);
def.setVersion(2);
metadataDAO.createWorkflowDef(def);
def.setVersion(3);
metadataDAO.createWorkflowDef(def);
// Placed the values in a map because they might not be stored in order of defName.
// To test, needed to confirm that the versions are correct for the definitions.
Map<String, WorkflowDef> allMap =
metadataDAO.getAllWorkflowDefsLatestVersions().stream()
.collect(Collectors.toMap(WorkflowDef::getName, Function.identity()));
assertNotNull(allMap);
assertEquals(4, allMap.size());
assertEquals(1, allMap.get("test1").getVersion());
assertEquals(2, allMap.get("test2").getVersion());
assertEquals(3, allMap.get("test3").getVersion());
}
}
@@ -0,0 +1,201 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import org.flywaydb.core.Flyway;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
import com.netflix.conductor.common.metadata.tasks.PollData;
import com.netflix.conductor.dao.PollDataDAO;
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
import com.netflix.conductor.sqlite.util.Query;
import com.fasterxml.jackson.core.JsonProcessingException;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import static org.junit.Assert.*;
import static org.junit.Assert.assertTrue;
@ContextConfiguration(
classes = {
TestObjectMapperConfiguration.class,
SqliteConfiguration.class,
FlywayAutoConfiguration.class
})
@RunWith(SpringRunner.class)
@TestPropertySource(
properties = {
"conductor.app.asyncIndexingEnabled=false",
"conductor.elasticsearch.version=0",
"conductor.indexing.type=sqlite",
"spring.flyway.clean-disabled=false"
})
@SpringBootTest
public class SqlitePollDataTest {
@Autowired private PollDataDAO pollDataDAO;
@Autowired private ObjectMapper objectMapper;
@Qualifier("dataSource")
@Autowired
private DataSource dataSource;
@Autowired Flyway flyway;
// clean the database between tests.
@Before
public void before() {
try (Connection conn = dataSource.getConnection()) {
conn.setAutoCommit(true);
conn.prepareStatement("delete from poll_data").executeUpdate();
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
private List<Map<String, Object>> queryDb(String query) throws SQLException {
try (Connection c = dataSource.getConnection()) {
try (Query q = new Query(objectMapper, c, query)) {
return q.executeAndFetchMap();
}
}
}
@Test
public void updateLastPollDataTest() throws SQLException, JsonProcessingException {
pollDataDAO.updateLastPollData("dummy-task", "dummy-domain", "dummy-worker-id");
List<Map<String, Object>> records =
queryDb("SELECT * FROM poll_data WHERE queue_name = 'dummy-task'");
assertEquals("More than one poll data records returned", 1, records.size());
assertEquals("Wrong domain set", "dummy-domain", records.get(0).get("domain"));
JsonNode jsonData = objectMapper.readTree(records.get(0).get("json_data").toString());
assertEquals(
"Poll data is incorrect", "dummy-worker-id", jsonData.get("workerId").asText());
}
@Test
public void updateLastPollDataNullDomainTest() throws SQLException, JsonProcessingException {
pollDataDAO.updateLastPollData("dummy-task", null, "dummy-worker-id");
List<Map<String, Object>> records =
queryDb("SELECT * FROM poll_data WHERE queue_name = 'dummy-task'");
assertEquals("More than one poll data records returned", 1, records.size());
assertEquals("Wrong domain set", "DEFAULT", records.get(0).get("domain"));
JsonNode jsonData = objectMapper.readTree(records.get(0).get("json_data").toString());
assertEquals(
"Poll data is incorrect", "dummy-worker-id", jsonData.get("workerId").asText());
}
@Test
public void getPollDataByDomainTest() {
pollDataDAO.updateLastPollData("dummy-task", "dummy-domain", "dummy-worker-id");
PollData pollData = pollDataDAO.getPollData("dummy-task", "dummy-domain");
assertEquals("dummy-task", pollData.getQueueName());
assertEquals("dummy-domain", pollData.getDomain());
assertEquals("dummy-worker-id", pollData.getWorkerId());
}
@Test
public void getPollDataByNullDomainTest() {
pollDataDAO.updateLastPollData("dummy-task", null, "dummy-worker-id");
PollData pollData = pollDataDAO.getPollData("dummy-task", null);
assertEquals("dummy-task", pollData.getQueueName());
assertNull(pollData.getDomain());
assertEquals("dummy-worker-id", pollData.getWorkerId());
}
@Test
public void getPollDataByTaskTest() {
pollDataDAO.updateLastPollData("dummy-task1", "domain1", "dummy-worker-id1");
pollDataDAO.updateLastPollData("dummy-task1", "domain2", "dummy-worker-id2");
pollDataDAO.updateLastPollData("dummy-task1", null, "dummy-worker-id3");
pollDataDAO.updateLastPollData("dummy-task2", "domain2", "dummy-worker-id4");
List<PollData> pollData = pollDataDAO.getPollData("dummy-task1");
assertEquals("Wrong number of records returned", 3, pollData.size());
List<String> queueNames =
pollData.stream().map(x -> x.getQueueName()).collect(Collectors.toList());
assertEquals(3, Collections.frequency(queueNames, "dummy-task1"));
List<String> domains =
pollData.stream().map(x -> x.getDomain()).collect(Collectors.toList());
assertTrue(domains.contains("domain1"));
assertTrue(domains.contains("domain2"));
assertTrue(domains.contains(null));
List<String> workerIds =
pollData.stream().map(x -> x.getWorkerId()).collect(Collectors.toList());
assertTrue(workerIds.contains("dummy-worker-id1"));
assertTrue(workerIds.contains("dummy-worker-id2"));
assertTrue(workerIds.contains("dummy-worker-id3"));
}
@Test
public void getAllPollDataTest() {
pollDataDAO.updateLastPollData("dummy-task1", "domain1", "dummy-worker-id1");
pollDataDAO.updateLastPollData("dummy-task1", "domain2", "dummy-worker-id2");
pollDataDAO.updateLastPollData("dummy-task1", null, "dummy-worker-id3");
pollDataDAO.updateLastPollData("dummy-task2", "domain2", "dummy-worker-id4");
List<PollData> pollData = pollDataDAO.getAllPollData();
assertEquals("Wrong number of records returned", 4, pollData.size());
List<String> queueNames =
pollData.stream().map(x -> x.getQueueName()).collect(Collectors.toList());
assertEquals(3, Collections.frequency(queueNames, "dummy-task1"));
assertEquals(1, Collections.frequency(queueNames, "dummy-task2"));
List<String> domains =
pollData.stream().map(x -> x.getDomain()).collect(Collectors.toList());
assertEquals(1, Collections.frequency(domains, "domain1"));
assertEquals(2, Collections.frequency(domains, "domain2"));
assertEquals(1, Collections.frequency(domains, null));
List<String> workerIds =
pollData.stream().map(x -> x.getWorkerId()).collect(Collectors.toList());
assertTrue(workerIds.contains("dummy-worker-id1"));
assertTrue(workerIds.contains("dummy-worker-id2"));
assertTrue(workerIds.contains("dummy-worker-id3"));
assertTrue(workerIds.contains("dummy-worker-id4"));
}
}
@@ -0,0 +1,505 @@
/*
* Copyright 2025 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.dao;
import java.sql.Connection;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import org.flywaydb.core.Flyway;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
import org.junit.runner.RunWith;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.boot.autoconfigure.flyway.FlywayAutoConfiguration;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ContextConfiguration;
import org.springframework.test.context.junit4.SpringRunner;
import com.netflix.conductor.common.config.TestObjectMapperConfiguration;
import com.netflix.conductor.core.events.queue.Message;
import com.netflix.conductor.sqlite.config.SqliteConfiguration;
import com.netflix.conductor.sqlite.util.Query;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.collect.ImmutableList;
import static org.awaitility.Awaitility.await;
import static org.junit.Assert.*;
import static org.junit.Assert.assertNotNull;
@ContextConfiguration(
classes = {
TestObjectMapperConfiguration.class,
SqliteConfiguration.class,
FlywayAutoConfiguration.class
})
@RunWith(SpringRunner.class)
@SpringBootTest(properties = "spring.flyway.clean-disabled=false")
public class SqliteQueueDAOTest {
private static final Logger LOGGER = LoggerFactory.getLogger(SqliteQueueDAOTest.class);
@Autowired private SqliteQueueDAO queueDAO;
@Qualifier("dataSource")
@Autowired
private DataSource dataSource;
@Autowired private ObjectMapper objectMapper;
@Rule public TestName name = new TestName();
@Autowired Flyway flyway;
@Before
public void before() {
try (Connection conn = dataSource.getConnection()) {
conn.setAutoCommit(true);
String[] stmts = new String[] {"delete from queue;", "delete from queue_message;"};
for (String stmt : stmts) {
conn.prepareStatement(stmt).executeUpdate();
}
} catch (Exception e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
@Test
public void complexQueueTest() {
String queueName = "TestQueue";
long offsetTimeInSecond = 0;
for (int i = 0; i < 10; i++) {
String messageId = "msg" + i;
queueDAO.push(queueName, messageId, offsetTimeInSecond);
}
int size = queueDAO.getSize(queueName);
assertEquals(10, size);
Map<String, Long> details = queueDAO.queuesDetail();
assertEquals(1, details.size());
assertEquals(10L, details.get(queueName).longValue());
for (int i = 0; i < 10; i++) {
String messageId = "msg" + i;
queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond);
}
List<String> popped = queueDAO.pop(queueName, 10, 100);
assertNotNull(popped);
assertEquals(10, popped.size());
Map<String, Map<String, Map<String, Long>>> verbose = queueDAO.queuesDetailVerbose();
assertEquals(1, verbose.size());
long shardSize = verbose.get(queueName).get("a").get("size");
long unackedSize = verbose.get(queueName).get("a").get("uacked");
assertEquals(0, shardSize);
assertEquals(10, unackedSize);
popped.forEach(messageId -> queueDAO.ack(queueName, messageId));
verbose = queueDAO.queuesDetailVerbose();
assertEquals(1, verbose.size());
shardSize = verbose.get(queueName).get("a").get("size");
unackedSize = verbose.get(queueName).get("a").get("uacked");
assertEquals(0, shardSize);
assertEquals(0, unackedSize);
popped = queueDAO.pop(queueName, 10, 100);
assertNotNull(popped);
assertEquals(0, popped.size());
for (int i = 0; i < 10; i++) {
String messageId = "msg" + i;
queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond);
}
size = queueDAO.getSize(queueName);
assertEquals(10, size);
for (int i = 0; i < 10; i++) {
String messageId = "msg" + i;
assertTrue(queueDAO.containsMessage(queueName, messageId));
queueDAO.remove(queueName, messageId);
}
size = queueDAO.getSize(queueName);
assertEquals(0, size);
for (int i = 0; i < 10; i++) {
String messageId = "msg" + i;
queueDAO.pushIfNotExists(queueName, messageId, offsetTimeInSecond);
}
queueDAO.flush(queueName);
size = queueDAO.getSize(queueName);
assertEquals(0, size);
}
/**
* Test fix for https://github.com/Netflix/conductor/issues/399
*
* @since 1.8.2-rc5
*/
@Test
public void pollMessagesTest() {
final List<Message> messages = new ArrayList<>();
final String queueName = "issue399_testQueue";
final int totalSize = 10;
for (int i = 0; i < totalSize; i++) {
String payload = "{\"id\": " + i + ", \"msg\":\"test " + i + "\"}";
Message m = new Message("testmsg-" + i, payload, "");
if (i % 2 == 0) {
// Set priority on message with pair id
m.setPriority(99 - i);
}
messages.add(m);
}
// Populate the queue with our test message batch
queueDAO.push(queueName, ImmutableList.copyOf(messages));
// Assert that all messages were persisted and no extras are in there
assertEquals("Queue size mismatch", totalSize, queueDAO.getSize(queueName));
List<Message> zeroPoll = queueDAO.pollMessages(queueName, 0, 10_000);
assertTrue("Zero poll should be empty", zeroPoll.isEmpty());
final int firstPollSize = 3;
List<Message> firstPoll = queueDAO.pollMessages(queueName, firstPollSize, 10_000);
assertNotNull("First poll was null", firstPoll);
assertFalse("First poll was empty", firstPoll.isEmpty());
assertEquals("First poll size mismatch", firstPollSize, firstPoll.size());
final int secondPollSize = 4;
List<Message> secondPoll = queueDAO.pollMessages(queueName, secondPollSize, 10_000);
assertNotNull("Second poll was null", secondPoll);
assertFalse("Second poll was empty", secondPoll.isEmpty());
assertEquals("Second poll size mismatch", secondPollSize, secondPoll.size());
// Assert that the total queue size hasn't changed
assertEquals(
"Total queue size should have remained the same",
totalSize,
queueDAO.getSize(queueName));
// Assert that our un-popped messages match our expected size
final long expectedSize = totalSize - firstPollSize - secondPollSize;
try (Connection c = dataSource.getConnection()) {
String UNPOPPED =
"SELECT COUNT(*) FROM queue_message WHERE queue_name = ? AND popped = false";
try (Query q = new Query(objectMapper, c, UNPOPPED)) {
long count = q.addParameter(queueName).executeCount();
assertEquals("Remaining queue size mismatch", expectedSize, count);
}
} catch (Exception ex) {
fail(ex.getMessage());
}
}
/** Test fix for https://github.com/Netflix/conductor/issues/1892 */
@Test
public void containsMessageTest() {
String queueName = "TestQueue";
long offsetTimeInSecond = 0;
for (int i = 0; i < 10; i++) {
String messageId = "msg" + i;
queueDAO.push(queueName, messageId, offsetTimeInSecond);
}
int size = queueDAO.getSize(queueName);
assertEquals(10, size);
for (int i = 0; i < 10; i++) {
String messageId = "msg" + i;
assertTrue(queueDAO.containsMessage(queueName, messageId));
queueDAO.remove(queueName, messageId);
}
for (int i = 0; i < 10; i++) {
String messageId = "msg" + i;
assertFalse(queueDAO.containsMessage(queueName, messageId));
}
}
/**
* Test fix for https://github.com/Netflix/conductor/issues/448
*
* @since 1.8.2-rc5
*/
@Test
public void pollDeferredMessagesTest() {
final List<Message> messages = new ArrayList<>();
final String queueName = "issue448_testQueue";
final int totalSize = 10;
for (int i = 0; i < totalSize; i++) {
int offset = 0;
if (i < 5) {
offset = 0;
} else if (i == 6 || i == 7) {
// Purposefully skipping id:5 to test out of order deliveries
// Set id:6 and id:7 for a 2s delay to be picked up in the second polling batch
offset = 5;
} else {
// Set all other queue messages to have enough of a delay that they won't
// accidentally
// be picked up.
offset = 10_000 + i;
}
String payload = "{\"id\": " + i + ",\"offset_time_seconds\":" + offset + "}";
Message m = new Message("testmsg-" + i, payload, "");
messages.add(m);
queueDAO.push(queueName, "testmsg-" + i, offset);
}
// Assert that all messages were persisted and no extras are in there
assertEquals("Queue size mismatch", totalSize, queueDAO.getSize(queueName));
final int firstPollSize = 4;
List<Message> firstPoll = queueDAO.pollMessages(queueName, firstPollSize, 100);
assertNotNull("First poll was null", firstPoll);
assertFalse("First poll was empty", firstPoll.isEmpty());
assertEquals("First poll size mismatch", firstPollSize, firstPoll.size());
List<String> firstPollMessageIds =
messages.stream()
.map(Message::getId)
.collect(Collectors.toList())
.subList(0, firstPollSize + 1);
for (int i = 0; i < firstPollSize; i++) {
String actual = firstPoll.get(i).getId();
assertTrue("Unexpected Id: " + actual, firstPollMessageIds.contains(actual));
}
final int secondPollSize = 3;
// Wait for delayed messages to become available for polling
final String COUNT_READY =
"SELECT COUNT(*) FROM queue_message WHERE queue_name = ? AND popped = false AND deliver_on <= strftime('%Y-%m-%d %H:%M:%f', 'now')";
await().atMost(10, TimeUnit.SECONDS)
.pollInterval(500, TimeUnit.MILLISECONDS)
.until(
() -> {
try (Connection c = dataSource.getConnection()) {
try (Query q = new Query(objectMapper, c, COUNT_READY)) {
return q.addParameter(queueName).executeCount()
>= secondPollSize;
}
}
});
// Poll for many more messages than expected
List<Message> secondPoll = queueDAO.pollMessages(queueName, secondPollSize + 10, 100);
assertNotNull("Second poll was null", secondPoll);
assertFalse("Second poll was empty", secondPoll.isEmpty());
assertEquals("Second poll size mismatch", secondPollSize, secondPoll.size());
List<String> expectedIds = Arrays.asList("testmsg-4", "testmsg-6", "testmsg-7");
for (int i = 0; i < secondPollSize; i++) {
String actual = secondPoll.get(i).getId();
assertTrue("Unexpected Id: " + actual, expectedIds.contains(actual));
}
// Assert that the total queue size hasn't changed
assertEquals(
"Total queue size should have remained the same",
totalSize,
queueDAO.getSize(queueName));
// Assert that our un-popped messages match our expected size
final long expectedSize = totalSize - firstPollSize - secondPollSize;
try (Connection c = dataSource.getConnection()) {
String UNPOPPED =
"SELECT COUNT(*) FROM queue_message WHERE queue_name = ? AND popped = false";
try (Query q = new Query(objectMapper, c, UNPOPPED)) {
long count = q.addParameter(queueName).executeCount();
assertEquals("Remaining queue size mismatch", expectedSize, count);
}
} catch (Exception ex) {
fail(ex.getMessage());
}
}
@Test
public void setUnackTimeoutIfShorterShortensDueTime() {
String queueName = "setUnackIfShorter_shorten";
String messageId = "msg-shorten";
// Push with a 1-hour delay so it is not immediately poppable
queueDAO.push(queueName, messageId, 3600L);
assertEquals(0, queueDAO.pop(queueName, 1, 100).size());
// Shorten delivery to now (0 s) — should succeed and return true
boolean shortened = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 0L);
assertTrue(
"setUnackTimeoutIfShorter should return true when it actually shortens", shortened);
// Message must now be immediately poppable
List<String> popped = queueDAO.pop(queueName, 1, 100);
assertEquals(1, popped.size());
assertEquals(messageId, popped.get(0));
}
@Test
public void setUnackTimeoutIfShorterDoesNotExtend() {
String queueName = "setUnackIfShorter_noextend";
String messageId = "msg-noextend";
// Push with offset 0 so it is immediately available
queueDAO.push(queueName, messageId, 0L);
// Try to push deliver_on far into the future — must be rejected
boolean extended = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 3_600_000L);
assertFalse(
"setUnackTimeoutIfShorter must not extend an already-closer delivery time",
extended);
// Message must still be immediately poppable
List<String> popped = queueDAO.pop(queueName, 1, 100);
assertEquals(1, popped.size());
assertEquals(messageId, popped.get(0));
}
@Test
public void setUnackTimeoutIfShorterReturnsFalseForNonExistent() {
String queueName = "setUnackIfShorter_nonexistent";
boolean updated = queueDAO.setUnackTimeoutIfShorter(queueName, "no-such-message", 0L);
assertFalse(
"setUnackTimeoutIfShorter must return false for a non-existent message", updated);
}
@Test
public void setUnackTimeoutIfShorterReturnsFalseWhenEqualTimeout() {
String queueName = "setUnackIfShorter_equal";
String messageId = "msg-equal";
queueDAO.push(queueName, messageId, 0L);
boolean result = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 0L);
assertFalse("Equal timeout must not update deliver_on — returns false", result);
List<String> popped = queueDAO.pop(queueName, 1, 100);
assertEquals(1, popped.size());
assertEquals(messageId, popped.get(0));
}
@Test
public void setUnackTimeoutIfShorterRejectsExtensionThenAcceptsShortening() {
String queueName = "setUnackIfShorter_reject_then_accept";
String messageId = "msg-reject-accept";
queueDAO.push(queueName, messageId, 3600L);
boolean extended = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 7_200_000L);
assertFalse("Extending must return false", extended);
boolean shortened = queueDAO.setUnackTimeoutIfShorter(queueName, messageId, 0L);
assertTrue("Shortening to now must return true", shortened);
List<String> popped = queueDAO.pop(queueName, 1, 100);
assertEquals(1, popped.size());
assertEquals(messageId, popped.get(0));
}
// @Test
public void processUnacksTest() {
processUnacks(
() -> {
// Process unacks
queueDAO.processUnacks("process_unacks_test");
},
"process_unacks_test");
}
// @Test
public void processAllUnacksTest() {
processUnacks(
() -> {
// Process all unacks
queueDAO.processAllUnacks();
},
"process_unacks_test");
}
private void processUnacks(Runnable unack, String queueName) {
// Count of messages in the queue(s)
final int count = 10;
// Number of messages to process acks for
final int unackedCount = 4;
// A secondary queue to make sure we don't accidentally process other queues
final String otherQueueName = "process_unacks_test_other_queue";
// Create testing queue with some messages (but not all) that will be popped/acked.
for (int i = 0; i < count; i++) {
int offset = 0;
if (i >= unackedCount) {
offset = 1_000_000;
}
queueDAO.push(queueName, "unack-" + i, offset);
}
// Create a second queue to make sure that unacks don't occur for it
for (int i = 0; i < count; i++) {
queueDAO.push(otherQueueName, "other-" + i, 0);
}
// Poll for first batch of messages (should be equal to unackedCount)
List<Message> polled = queueDAO.pollMessages(queueName, 100, 10_000);
assertNotNull(polled);
assertFalse(polled.isEmpty());
assertEquals(unackedCount, polled.size());
// Poll messages from the other queue so we know they don't get unacked later
queueDAO.pollMessages(otherQueueName, 100, 10_000);
// Ack one of the polled messages
assertTrue(queueDAO.ack(queueName, "unack-1"));
// Should have one less un-acked popped message in the queue
Long uacked = queueDAO.queuesDetailVerbose().get(queueName).get("a").get("uacked");
assertNotNull(uacked);
assertEquals(uacked.longValue(), unackedCount - 1);
unack.run();
// Check uacks for both queues after processing
Map<String, Map<String, Map<String, Long>>> details = queueDAO.queuesDetailVerbose();
uacked = details.get(queueName).get("a").get("uacked");
assertNotNull(uacked);
assertEquals(
"The messages that were polled should be unacked still",
uacked.longValue(),
unackedCount - 1);
Long otherUacked = details.get(otherQueueName).get("a").get("uacked");
assertNotNull(otherUacked);
assertEquals(
"Other queue should have all unacked messages", otherUacked.longValue(), count);
Long size = queueDAO.queuesDetail().get(queueName);
assertNotNull(size);
assertEquals(size.longValue(), count - unackedCount);
}
}
@@ -0,0 +1,214 @@
/*
* Copyright 2023 Conductor Authors.
* <p>
* Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* <p>
* http://www.apache.org/licenses/LICENSE-2.0
* <p>
* Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
* an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
* specific language governing permissions and limitations under the License.
*/
package com.netflix.conductor.sqlite.util;
import java.sql.SQLException;
import java.util.ArrayList;
import org.junit.jupiter.api.Test;
import org.mockito.InOrder;
import org.mockito.Mockito;
import com.netflix.conductor.sqlite.config.SqliteProperties;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.Mockito.*;
public class SqliteIndexQueryBuilderTest {
private SqliteProperties properties = new SqliteProperties();
@Test
void shouldGenerateQueryForEmptyString() throws SQLException {
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", "", "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals("SELECT json_data FROM table_name LIMIT ? OFFSET ?", generatedQuery);
}
@Test
void shouldGenerateQueryForExactMatch() throws SQLException {
String inputQuery = "workflowId=\"abc123\"";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE workflow_id = ? LIMIT ? OFFSET ?",
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("abc123");
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
verifyNoMoreInteractions(mockQuery);
}
@Test
void shouldGenerateQueryWithWildcardPrefix() throws SQLException {
String inputQuery = "workflowType=abc*";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE lower(workflow_type) LIKE lower(?) LIMIT ? OFFSET ?",
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("abc%");
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
verifyNoMoreInteractions(mockQuery);
}
@Test
void shouldGenerateQueryWithWildcardContains() throws SQLException {
String inputQuery = "correlationId=\"*order*\"";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE lower(correlation_id) LIKE lower(?) LIMIT ? OFFSET ?",
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("%order%");
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
verifyNoMoreInteractions(mockQuery);
}
@Test
void shouldGenerateExactMatchQueryWhenNoWildcard() throws SQLException {
String inputQuery = "workflowType=abc";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE workflow_type = ? LIMIT ? OFFSET ?",
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("abc");
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
verifyNoMoreInteractions(mockQuery);
}
@Test
void shouldNotExpandWildcardInINClause() throws SQLException {
String inputQuery = "status IN (COMP*,RUNNING)";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE status IN (?,?) LIMIT ? OFFSET ?",
generatedQuery);
}
@Test
void shouldGenerateQueryForClassifier() throws SQLException {
String inputQuery = "classifier=\"agent\"";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE classifier = ? LIMIT ? OFFSET ?",
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("agent");
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
verifyNoMoreInteractions(mockQuery);
}
@Test
void shouldMatchLegacyNullRowsForUntaggedClassifier() throws SQLException {
// Rows indexed before the classifier column existed are untagged plain workflows;
// filtering for the "workflow" token must also match those NULL rows.
String inputQuery = "classifier=\"workflow\"";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE (classifier = ? OR classifier IS NULL) LIMIT ? OFFSET ?",
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("workflow");
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
verifyNoMoreInteractions(mockQuery);
}
@Test
void shouldMatchLegacyNullRowsForUntaggedClassifierInClause() throws SQLException {
String inputQuery = "classifier IN (agent,workflow)";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE (classifier IN (?,?) OR classifier IS NULL) LIMIT ? OFFSET ?",
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("agent");
inOrder.verify(mockQuery).addParameter("workflow");
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
verifyNoMoreInteractions(mockQuery);
}
@Test
void shouldGenerateQueryForParentWorkflowId() throws SQLException {
String inputQuery = "parentWorkflowId=\"\"";
SqliteIndexQueryBuilder builder =
new SqliteIndexQueryBuilder(
"table_name", inputQuery, "", 0, 15, new ArrayList<>(), properties);
String generatedQuery = builder.getQuery();
assertEquals(
"SELECT json_data FROM table_name WHERE parent_workflow_id = ? LIMIT ? OFFSET ?",
generatedQuery);
Query mockQuery = mock(Query.class);
builder.addParameters(mockQuery);
builder.addPagingParameters(mockQuery);
InOrder inOrder = Mockito.inOrder(mockQuery);
inOrder.verify(mockQuery).addParameter("");
inOrder.verify(mockQuery).addParameter(15);
inOrder.verify(mockQuery).addParameter(0);
verifyNoMoreInteractions(mockQuery);
}
}
@@ -0,0 +1,9 @@
spring.datasource.driver-class-name=org.sqlite.JDBC
spring.datasource.url=jdbc:sqlite:conductorosstest.db
spring.datasource.username=
spring.datasource.password=
# Hibernate SQLite Dialect
spring.jpa.database-platform=org.hibernate.community.dialect.SQLiteDialect
# db type
conductor.db.type=sqlite