wehub-resource-sync
120 lines
3.4 KiB
TypeScript
120 lines
3.4 KiB
TypeScript
import { IntervalService } from "@trigger.dev/core/v3";
|
|
import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger";
|
|
import type { Registry } from "prom-client";
|
|
import { Counter, Gauge } from "prom-client";
|
|
import type { K8sApi } from "../clients/kubernetes.js";
|
|
import { createK8sApi } from "../clients/kubernetes.js";
|
|
import { register } from "../metrics.js";
|
|
|
|
export type PodCleanerOptions = {
|
|
namespace: string;
|
|
k8s?: K8sApi;
|
|
register?: Registry;
|
|
batchSize?: number;
|
|
intervalMs?: number;
|
|
};
|
|
|
|
export class PodCleaner {
|
|
private readonly logger = new SimpleStructuredLogger("pod-cleaner");
|
|
private readonly k8s: K8sApi;
|
|
private readonly namespace: string;
|
|
|
|
private readonly batchSize: number;
|
|
private readonly deletionInterval: IntervalService;
|
|
|
|
// Metrics
|
|
private readonly register: Registry;
|
|
private readonly deletionCyclesTotal: Counter;
|
|
private readonly lastDeletionTimestamp: Gauge;
|
|
|
|
constructor(opts: PodCleanerOptions) {
|
|
this.k8s = opts.k8s ?? createK8sApi();
|
|
|
|
this.namespace = opts.namespace;
|
|
this.batchSize = opts.batchSize ?? 500;
|
|
|
|
this.deletionInterval = new IntervalService({
|
|
intervalMs: opts.intervalMs ?? 10000,
|
|
leadingEdge: true,
|
|
onInterval: this.deleteCompletedPods.bind(this),
|
|
});
|
|
|
|
// Initialize metrics
|
|
this.register = opts.register ?? register;
|
|
|
|
this.deletionCyclesTotal = new Counter({
|
|
name: "pod_cleaner_deletion_cycles_total",
|
|
help: "Total number of pod deletion cycles run",
|
|
labelNames: ["namespace", "status", "batch_size"],
|
|
registers: [this.register],
|
|
});
|
|
|
|
this.lastDeletionTimestamp = new Gauge({
|
|
name: "pod_cleaner_last_deletion_timestamp",
|
|
help: "Timestamp of the last deletion cycle",
|
|
labelNames: ["namespace"],
|
|
registers: [this.register],
|
|
});
|
|
}
|
|
|
|
async start() {
|
|
this.deletionInterval.start();
|
|
}
|
|
|
|
async stop() {
|
|
this.deletionInterval.stop();
|
|
}
|
|
|
|
private async deleteCompletedPods() {
|
|
let continuationToken: string | undefined;
|
|
|
|
do {
|
|
try {
|
|
const result = await this.k8s.core.deleteCollectionNamespacedPod({
|
|
namespace: this.namespace,
|
|
labelSelector: "app=task-run",
|
|
fieldSelector: "status.phase=Succeeded",
|
|
limit: this.batchSize,
|
|
_continue: continuationToken,
|
|
gracePeriodSeconds: 0,
|
|
propagationPolicy: "Background",
|
|
timeoutSeconds: 30,
|
|
});
|
|
|
|
// Update continuation token for next batch
|
|
continuationToken = result.metadata?._continue;
|
|
|
|
// Increment the deletion cycles counter
|
|
this.deletionCyclesTotal.inc({
|
|
namespace: this.namespace,
|
|
batch_size: this.batchSize,
|
|
status: "succeeded",
|
|
});
|
|
|
|
this.logger.debug("Deleted batch of pods", { continuationToken });
|
|
} catch (err) {
|
|
this.logger.error("Failed to delete batch of pods", {
|
|
err: err instanceof Error ? err.message : String(err),
|
|
});
|
|
|
|
this.deletionCyclesTotal.inc({
|
|
namespace: this.namespace,
|
|
batch_size: this.batchSize,
|
|
status: "failed",
|
|
});
|
|
break;
|
|
}
|
|
} while (continuationToken);
|
|
|
|
this.lastDeletionTimestamp.set({ namespace: this.namespace }, Date.now());
|
|
}
|
|
|
|
// Method to expose metrics for testing
|
|
public getMetrics() {
|
|
return {
|
|
deletionCyclesTotal: this.deletionCyclesTotal,
|
|
lastDeletionTimestamp: this.lastDeletionTimestamp,
|
|
};
|
|
}
|
|
}
|