wehub-resource-sync
423 lines
21 KiB
Ruby
423 lines
21 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require_relative "spec_helper"
|
|
|
|
# Covers nine CR fixes against PromoteCommand:
|
|
# FIX-1: @promote_refs is RESET per check_p1_ghcr_digests run (not memoized
|
|
# across A→B preflight invocations), and execute_promotion HARD-GUARDS
|
|
# against a nil @promote_refs.
|
|
# FIX-2: P2 in-flight race compares deployed_digest vs the digest portion of
|
|
# @promote_refs[name] (not the snapshot's tag-form nil `digest`).
|
|
# Also: JSON-string `meta` is parsed (not WARN-skipped); and
|
|
# fetch_latest_staging_deployments sorts createdAt-desc so `.first`
|
|
# is genuinely latest.
|
|
# FIX-3: execute_promotion pre-validates ALL prod-matched services have a
|
|
# digest-shaped @promote_refs entry BEFORE pinning anything.
|
|
# FIX-4: execute_promotion rescue broadens to MutationError + GraphQL::Error
|
|
# + StandardError, retains PARTIAL-PROMOTION report, and drops the
|
|
# duplicate `warn e.message`.
|
|
# FIX-5: check_p1_ghcr_digests emits REFUSE: P1 ... "no image" when staging
|
|
# service has nil/empty image (instead of silent skip).
|
|
# FIX-6: P1 per-service rescue broadens to StandardError so non-GHCR errors
|
|
# don't bypass the loop.
|
|
# FIX-7: pin_and_verify raises ArgumentError immediately when called with a
|
|
# tag-form (non-digest) image.
|
|
# FIX-8: pin_and_verify timestamp gate is non-vacuous: a nil observed
|
|
# updatedAt does NOT declare success, even when pre_update_ts is nil.
|
|
# FIX-9: run_staging_probe rescues launch failures (Errno::ENOENT, etc.)
|
|
# and returns ok:false with a descriptive summary.
|
|
class PromoteCRFixesTest < Minitest::Test
|
|
# ----- shared fakes -----
|
|
|
|
class NullGQL
|
|
def query(*); {}; end
|
|
end
|
|
|
|
# GQL that records calls and returns canned responses for P2/preflight only
|
|
# (no mutations).
|
|
class P2GQL
|
|
attr_reader :calls
|
|
def initialize(deployments_by_svc)
|
|
@deployments_by_svc = deployments_by_svc
|
|
@calls = []
|
|
end
|
|
def query(q, vars = {})
|
|
@calls << [q, vars]
|
|
if q.include?("query Deployments")
|
|
edges = (@deployments_by_svc[vars[:serviceId]] || []).map { |n| { "node" => n } }
|
|
return { "deployments" => { "edges" => edges } }
|
|
end
|
|
{}
|
|
end
|
|
end
|
|
|
|
# GHCR fake that always reports :exists and resolves any :latest -> a digest.
|
|
class PassGHCR
|
|
def initialize(resolve_map: {}); @resolve_map = resolve_map; end
|
|
def resolve_digest(ref)
|
|
return ref.split("@", 2).last if ref.include?("@sha256:")
|
|
@resolve_map[ref] || "sha256:resolved_for_#{ref}"
|
|
end
|
|
def manifest_exists(_); :exists; end
|
|
def parse_image_ref(ref); Railway::GHCR.allocate.parse_image_ref(ref); end
|
|
end
|
|
|
|
def make_svc(name, image:)
|
|
{
|
|
"name" => name, "service_id" => "svc-stg-#{name}",
|
|
"image" => image,
|
|
"env_keys" => [],
|
|
"start_command" => "node server.js", "healthcheck_path" => "/health",
|
|
"region" => "us-west", "replicas" => 1, "restart_policy" => "ON_FAILURE",
|
|
}
|
|
end
|
|
|
|
# =================== FIX-1: @promote_refs is reset, not memoized ===================
|
|
|
|
def test_fix1_promote_refs_resets_between_p1_runs
|
|
# Reuse a single PromoteCommand instance across two distinct snapshots
|
|
# (snapshot A and snapshot B). After the second P1 run, @promote_refs
|
|
# must reflect ONLY snapshot B's services — no stale A entries.
|
|
cmd = Railway::PromoteCommand.new([])
|
|
cmd.instance_variable_set(:@ghcr, PassGHCR.new)
|
|
# resolved_prod_image now pins staging's RUNNING digest (meta.imageDigest
|
|
# from the latest SUCCESS deployment), not resolve_digest(:latest). Stub
|
|
# the deployment lookup so a digest is resolvable for each tag-form svc.
|
|
cmd.define_singleton_method(:fetch_latest_staging_deployments) do |svc_id|
|
|
name = svc_id.sub("svc-stg-", "")
|
|
[{ "id" => "d", "status" => "SUCCESS",
|
|
"meta" => { "image" => "ghcr.io/copilotkit/#{name}:latest",
|
|
"imageDigest" => "sha256:running_#{name}" } }]
|
|
end
|
|
|
|
snapshot_a = { "services" => [make_svc("alpha", image: "ghcr.io/copilotkit/alpha:latest")] }
|
|
snapshot_b = { "services" => [make_svc("beta", image: "ghcr.io/copilotkit/beta:latest")] }
|
|
|
|
cmd.send(:check_p1_ghcr_digests, snapshot_a)
|
|
refs_after_a = cmd.instance_variable_get(:@promote_refs).keys.sort
|
|
assert_equal ["alpha"], refs_after_a
|
|
|
|
cmd.send(:check_p1_ghcr_digests, snapshot_b)
|
|
refs_after_b = cmd.instance_variable_get(:@promote_refs).keys.sort
|
|
assert_equal ["beta"], refs_after_b,
|
|
"stale entries from snapshot A must be cleared; got #{refs_after_b.inspect}"
|
|
end
|
|
|
|
def test_fix1_execute_promotion_raises_when_promote_refs_nil
|
|
# execute_promotion called WITHOUT a prior preflight must raise an
|
|
# internal-error exception (not silently treat refs as empty).
|
|
cmd = Railway::PromoteCommand.new([])
|
|
cmd.instance_variable_set(:@ghcr, PassGHCR.new)
|
|
# Deliberately do NOT call check_p1_ghcr_digests; @promote_refs stays nil.
|
|
staging = { "services" => [make_svc("x", image: "ghcr.io/copilotkit/x:latest")] }
|
|
prod = { "services" => [{ "name" => "x", "service_id" => "svc-prod-x",
|
|
"image" => "ghcr.io/copilotkit/x@sha256:OLD" }] }
|
|
err = assert_raises(RuntimeError) do
|
|
cmd.send(:execute_promotion, staging, prod)
|
|
end
|
|
assert_match(/internal error.*execute_promotion.*preflight/i, err.message)
|
|
end
|
|
|
|
# =================== FIX-2: P2 race-check is alive ===================
|
|
|
|
def test_fix2_p2_race_check_uses_promote_refs_not_snapshot_digest
|
|
# Staging service is tag-form (so svc["digest"] is nil — pre-fix the
|
|
# race-check was DEAD CODE). After fix: P2 compares against the digest
|
|
# captured in @promote_refs[name].
|
|
cmd = Railway::PromoteCommand.new([])
|
|
cmd.instance_variable_set(:@promote_refs, {
|
|
"x" => "ghcr.io/copilotkit/x@sha256:YYY",
|
|
})
|
|
cmd.define_singleton_method(:fetch_latest_staging_deployments) do |_svc_id|
|
|
[{ "id" => "d1", "status" => "SUCCESS",
|
|
"meta" => { "image" => "ghcr.io/copilotkit/x@sha256:XXX" },
|
|
"createdAt" => "2026-05-28T01:00:00Z" }]
|
|
end
|
|
staging = { "services" => [{
|
|
"name" => "x", "service_id" => "svc-1",
|
|
"image" => "ghcr.io/copilotkit/x:latest", # tag-form; no "digest"
|
|
"env_keys" => [],
|
|
}] }
|
|
findings = cmd.send(:check_p2_staging_deployments, staging)
|
|
assert(findings.any? { |f| f =~ /REFUSE: P2 \(x\).*in-flight.*sha256:XXX.*sha256:YYY/ },
|
|
"expected REFUSE: P2 comparing deployed XXX vs P1-resolved YYY; got: #{findings.inspect}")
|
|
end
|
|
|
|
def test_fix2_p2_parses_meta_when_it_is_a_json_string
|
|
# Some Railway responses deserialize Deployment.meta as a JSON String
|
|
# (not a Hash). P2 must parse it before falling back to the WARN branch.
|
|
cmd = Railway::PromoteCommand.new([])
|
|
cmd.instance_variable_set(:@promote_refs, {
|
|
"x" => "ghcr.io/copilotkit/x@sha256:abc",
|
|
})
|
|
cmd.define_singleton_method(:fetch_latest_staging_deployments) do |_svc_id|
|
|
[{ "id" => "d1", "status" => "SUCCESS",
|
|
"meta" => '{"image":"ghcr.io/copilotkit/x@sha256:abc"}',
|
|
"createdAt" => "2026-05-28T01:00:00Z" }]
|
|
end
|
|
staging = { "services" => [{
|
|
"name" => "x", "service_id" => "svc-1",
|
|
"image" => "ghcr.io/copilotkit/x:latest",
|
|
"env_keys" => [],
|
|
}] }
|
|
findings = cmd.send(:check_p2_staging_deployments, staging)
|
|
refute(findings.any? { |f| f =~ /WARN: P2 \(x\)/ },
|
|
"JSON-string meta must be parsed (not WARN-skipped); got: #{findings.inspect}")
|
|
refute(findings.any? { |f| f =~ /REFUSE: P2/ },
|
|
"matching digest in parsed meta must not REFUSE; got: #{findings.inspect}")
|
|
end
|
|
|
|
def test_fix2_fetch_latest_staging_deployments_sorts_newest_first
|
|
# Stub gql.query to return deployments in OLDEST-first order — the
|
|
# helper must sort by createdAt DESC so `.first` is the newest.
|
|
cmd = Railway::PromoteCommand.new([])
|
|
nodes = [
|
|
{ "id" => "d-old", "status" => "SUCCESS", "createdAt" => "2026-05-01T00:00:00Z" },
|
|
{ "id" => "d-new", "status" => "SUCCESS", "createdAt" => "2026-05-28T00:00:00Z" },
|
|
{ "id" => "d-mid", "status" => "FAILED", "createdAt" => "2026-05-15T00:00:00Z" },
|
|
]
|
|
edges = nodes.map { |n| { "node" => n } }
|
|
fake_gql = Object.new
|
|
fake_gql.define_singleton_method(:query) do |_q, _vars = {}|
|
|
{ "deployments" => { "edges" => edges } }
|
|
end
|
|
cmd.instance_variable_set(:@gql, fake_gql)
|
|
deployments = cmd.send(:fetch_latest_staging_deployments, "svc-1")
|
|
assert_equal "d-new", deployments.first["id"],
|
|
"fetch_latest_staging_deployments must sort newest-first; got: #{deployments.map { |d| d['id'] }.inspect}"
|
|
end
|
|
|
|
# =================== FIX-3: execute_promotion pre-validation ===================
|
|
|
|
def test_fix3_execute_promotion_pre_validates_all_refs_before_any_pin
|
|
# Two prod-matched services; ONE missing from @promote_refs. The
|
|
# pre-validation must REFUSE+return 1 BEFORE any serviceInstanceUpdate
|
|
# mutation is issued.
|
|
cmd = Railway::PromoteCommand.new([])
|
|
recorded = []
|
|
fake_gql = Object.new
|
|
fake_gql.define_singleton_method(:query) do |q, vars = {}|
|
|
recorded << [q, vars]
|
|
{ "serviceInstanceUpdate" => true, "serviceInstanceDeployV2" => "dep-new" }
|
|
end
|
|
cmd.instance_variable_set(:@gql, fake_gql)
|
|
cmd.instance_variable_set(:@promote_refs, {
|
|
"a" => "ghcr.io/copilotkit/a@sha256:aaa",
|
|
# "b" is MISSING
|
|
})
|
|
staging = { "services" => [make_svc("a", image: "ghcr.io/copilotkit/a:latest"),
|
|
make_svc("b", image: "ghcr.io/copilotkit/b:latest")] }
|
|
prod = { "services" => [
|
|
{ "name" => "a", "service_id" => "svc-prod-a", "image" => "ghcr.io/copilotkit/a@sha256:OLD" },
|
|
{ "name" => "b", "service_id" => "svc-prod-b", "image" => "ghcr.io/copilotkit/b@sha256:OLD" },
|
|
] }
|
|
_out, _err = capture_io { @rc = cmd.send(:execute_promotion, staging, prod) }
|
|
assert_equal 1, @rc, "execute_promotion must return 1 when a ref is missing"
|
|
assert(recorded.none? { |q, _| q.include?("serviceInstanceUpdate") },
|
|
"no serviceInstanceUpdate mutations should be issued; got: #{recorded.map { |q, _| q[0, 30] }.inspect}")
|
|
end
|
|
|
|
# =================== FIX-4: broadened rescue + partial-promotion report ===================
|
|
|
|
def test_fix4_execute_promotion_rescues_graphql_error_with_partial_report
|
|
# First service pins successfully; second raises Railway::GraphQL::Error
|
|
# on its serviceInstanceUpdate mutation. The broadened rescue must
|
|
# catch it and still emit the PARTIAL-PROMOTION report.
|
|
cmd = Railway::PromoteCommand.new([])
|
|
call_count = 0
|
|
# FakeGQL that records calls and raises GraphQL::Error on the SECOND
|
|
# serviceInstanceUpdate mutation.
|
|
fake_gql = Object.new
|
|
@pinned = nil
|
|
@pre_ts = "2026-05-28T00:00:00Z"
|
|
pinned_ref = nil
|
|
fake_gql.define_singleton_method(:query) do |q, vars = {}|
|
|
if q.include?("serviceInstanceUpdate")
|
|
call_count += 1
|
|
raise Railway::GraphQL::Error, "boom on update #2" if call_count == 2
|
|
pinned_ref = vars.dig(:input, :source, :image)
|
|
{ "serviceInstanceUpdate" => true }
|
|
elsif q.include?("serviceInstanceDeployV2")
|
|
{ "serviceInstanceDeployV2" => "dep-new" }
|
|
elsif q.include?("ServiceInstanceRecheck")
|
|
if pinned_ref
|
|
{ "serviceInstance" => { "id" => "i",
|
|
"source" => { "image" => pinned_ref },
|
|
"updatedAt" => "2026-05-29T00:00:01Z",
|
|
"latestDeployment" => {
|
|
"id" => "dep-new", "status" => "SUCCESS",
|
|
"meta" => { "imageDigest" => (pinned_ref.include?("@") ? pinned_ref.split("@", 2).last : nil) },
|
|
} } }
|
|
else
|
|
{ "serviceInstance" => { "id" => "i",
|
|
"source" => { "image" => "ghcr.io/copilotkit/x@sha256:OLD" },
|
|
"updatedAt" => "2026-05-28T00:00:00Z" } }
|
|
end
|
|
else
|
|
{}
|
|
end
|
|
end
|
|
cmd.instance_variable_set(:@gql, fake_gql)
|
|
cmd.instance_variable_set(:@promote_refs, {
|
|
"a" => "ghcr.io/copilotkit/a@sha256:aaa",
|
|
"b" => "ghcr.io/copilotkit/b@sha256:bbb",
|
|
})
|
|
# Silence pin_and_verify retries.
|
|
original = Railway::PromoteCommand.const_get(:RETRY_DELAY_SEC)
|
|
Railway::PromoteCommand.send(:remove_const, :RETRY_DELAY_SEC)
|
|
Railway::PromoteCommand.const_set(:RETRY_DELAY_SEC, 0)
|
|
begin
|
|
staging = { "services" => [make_svc("a", image: "ghcr.io/copilotkit/a:latest"),
|
|
make_svc("b", image: "ghcr.io/copilotkit/b:latest")] }
|
|
prod = { "services" => [
|
|
{ "name" => "a", "service_id" => "svc-prod-a", "image" => "ghcr.io/copilotkit/a@sha256:OLD" },
|
|
{ "name" => "b", "service_id" => "svc-prod-b", "image" => "ghcr.io/copilotkit/b@sha256:OLD" },
|
|
] }
|
|
out, err = capture_io { @rc = cmd.send(:execute_promotion, staging, prod) }
|
|
combined = out + err
|
|
assert_equal 1, @rc
|
|
assert_match(/PARTIAL PROMOTION/i, combined,
|
|
"must emit partial-promotion report on GraphQL::Error; combined=#{combined}")
|
|
assert_match(/already pinned.*\ba\b/m, combined,
|
|
"report must name 'a' as already-pinned; combined=#{combined}")
|
|
assert_match(/FAILED on b/, combined,
|
|
"report must name 'b' as failed; combined=#{combined}")
|
|
# Dedup check: the inner e.message should appear ONLY inside the
|
|
# composed PARTIAL-PROMOTION line, not on its own line as well.
|
|
assert_equal 1, combined.scan(/boom on update #2/).size,
|
|
"duplicate `warn e.message` line must be removed; combined=#{combined}"
|
|
ensure
|
|
Railway::PromoteCommand.send(:remove_const, :RETRY_DELAY_SEC)
|
|
Railway::PromoteCommand.const_set(:RETRY_DELAY_SEC, original)
|
|
end
|
|
end
|
|
|
|
# =================== FIX-5: P1 REFUSE on imageless service ===================
|
|
|
|
def test_fix5_p1_refuses_when_staging_service_has_no_image
|
|
cmd = Railway::PromoteCommand.new([])
|
|
cmd.instance_variable_set(:@ghcr, PassGHCR.new)
|
|
staging = { "services" => [
|
|
{ "name" => "x", "service_id" => "svc-1", "image" => nil,
|
|
"env_keys" => [] },
|
|
] }
|
|
findings = cmd.send(:check_p1_ghcr_digests, staging)
|
|
assert(findings.any? { |f| f =~ /REFUSE: P1 \(x\).*no image/i },
|
|
"expected REFUSE: P1 (x) about missing image; got: #{findings.inspect}")
|
|
end
|
|
|
|
# =================== FIX-6: P1 per-service rescue broadens to StandardError ===================
|
|
|
|
class ArgumentErrorGHCR
|
|
def resolve_digest(_); "sha256:fake"; end
|
|
def manifest_exists(_); raise ArgumentError, "non-ghcr error"; end
|
|
def parse_image_ref(ref); Railway::GHCR.allocate.parse_image_ref(ref); end
|
|
end
|
|
|
|
def test_fix6_p1_rescue_catches_non_ghcr_errors
|
|
# ArgumentError raised inside manifest_exists must be caught by the
|
|
# per-service rescue (broadened to StandardError) — the loop must
|
|
# continue and the service must get a per-service REFUSE.
|
|
cmd = Railway::PromoteCommand.new([])
|
|
cmd.instance_variable_set(:@ghcr, ArgumentErrorGHCR.new)
|
|
# Stub the running-digest lookup so resolved_prod_image succeeds and the
|
|
# flow reaches manifest_exists (which raises the ArgumentError under test).
|
|
cmd.define_singleton_method(:fetch_latest_staging_deployments) do |svc_id|
|
|
name = svc_id.sub("svc-stg-", "")
|
|
[{ "id" => "d", "status" => "SUCCESS",
|
|
"meta" => { "image" => "ghcr.io/copilotkit/#{name}:latest",
|
|
"imageDigest" => "sha256:running_#{name}" } }]
|
|
end
|
|
staging = { "services" => [
|
|
make_svc("a", image: "ghcr.io/copilotkit/a:latest"),
|
|
make_svc("b", image: "ghcr.io/copilotkit/b:latest"),
|
|
] }
|
|
findings = cmd.send(:check_p1_ghcr_digests, staging)
|
|
# Two services, each one should fail with ArgumentError-bearing REFUSE.
|
|
assert(findings.any? { |f| f =~ /REFUSE: P1 \(a\).*ArgumentError.*non-ghcr error/ },
|
|
"service 'a' must record a per-service REFUSE for ArgumentError; got: #{findings.inspect}")
|
|
assert(findings.any? { |f| f =~ /REFUSE: P1 \(b\).*ArgumentError.*non-ghcr error/ },
|
|
"service 'b' must record a per-service REFUSE for ArgumentError; got: #{findings.inspect}")
|
|
end
|
|
|
|
# =================== FIX-7: pin_and_verify upfront digest guard ===================
|
|
|
|
def test_fix7_pin_and_verify_raises_arg_error_on_tag_form_image
|
|
# Pre-fix: pin_and_verify would dutifully attempt N retries before
|
|
# raising a misleading MutationError. After fix: ArgumentError fires
|
|
# immediately, no retries.
|
|
gql = Object.new
|
|
gql.define_singleton_method(:query) { |*| raise "no query should be issued" }
|
|
err = assert_raises(ArgumentError) do
|
|
Railway::PromoteCommand.pin_and_verify(gql,
|
|
service_id: "svc-x", env_id: "env-prod",
|
|
image: "ghcr.io/copilotkit/x:latest",
|
|
sleeper: ->(_) {})
|
|
end
|
|
assert_match(/pin_and_verify.*@sha256.*pinned/i, err.message,
|
|
"ArgumentError message must explain the digest requirement; got: #{err.message}")
|
|
end
|
|
|
|
# =================== FIX-8: pin_and_verify ts gate is non-vacuous ===================
|
|
|
|
def test_fix8_pin_and_verify_requires_non_nil_updated_at_even_when_pre_ts_nil
|
|
# pre_update_ts is nil (new prod instance). Recheck returns matching
|
|
# digest but a nil updatedAt. Pre-fix: ts_ok was vacuously true → success.
|
|
# After fix: ts_ok requires actual_ts to be non-nil → kept retrying →
|
|
# MutationError after RETRY_COUNT attempts.
|
|
gql = Object.new
|
|
pinned = nil
|
|
gql.define_singleton_method(:query) do |q, vars = {}|
|
|
if q.include?("serviceInstanceUpdate")
|
|
pinned = vars.dig(:input, :source, :image)
|
|
{ "serviceInstanceUpdate" => true }
|
|
elsif q.include?("serviceInstanceDeployV2")
|
|
{ "serviceInstanceDeployV2" => "dep-new" }
|
|
elsif q.include?("ServiceInstanceRecheck")
|
|
if pinned.nil?
|
|
# Pre-update: brand-new instance — both fields are nil.
|
|
{ "serviceInstance" => nil }
|
|
else
|
|
{ "serviceInstance" => { "id" => "i",
|
|
"source" => { "image" => pinned },
|
|
"updatedAt" => nil } }
|
|
end
|
|
else
|
|
{}
|
|
end
|
|
end
|
|
err = assert_raises(Railway::PromoteCommand::MutationError) do
|
|
Railway::PromoteCommand.pin_and_verify(gql,
|
|
service_id: "svc-x", env_id: "env-prod",
|
|
image: "ghcr.io/copilotkit/x@sha256:abc",
|
|
sleeper: ->(_) {})
|
|
end
|
|
assert_match(/did not observe image advance/i, err.message,
|
|
"expected timeout-style MutationError; got: #{err.message}")
|
|
end
|
|
|
|
# =================== FIX-9: run_staging_probe rescue on launch failure ===================
|
|
|
|
def test_fix9_run_staging_probe_returns_clean_failure_on_io_popen_error
|
|
cmd = Railway::PromoteCommand.new([])
|
|
# The probe binary must APPEAR present so we reach the IO.popen call.
|
|
# (Skip the File.exist? early-return.)
|
|
original_exist = File.method(:exist?)
|
|
File.define_singleton_method(:exist?) { |_path| true }
|
|
# Stub IO.popen to raise Errno::ENOENT (npx missing on PATH).
|
|
original_popen = IO.method(:popen)
|
|
IO.define_singleton_method(:popen) do |*_args, **_kw, &_blk|
|
|
raise Errno::ENOENT, "npx"
|
|
end
|
|
begin
|
|
result = cmd.send(:run_staging_probe, services: ["x"])
|
|
assert_equal false, result[:ok], "must return ok:false on launch failure"
|
|
assert_match(/staging probe failed to launch.*ENOENT/i, result[:summary],
|
|
"summary must describe the launch failure; got: #{result[:summary].inspect}")
|
|
ensure
|
|
IO.define_singleton_method(:popen, &original_popen)
|
|
File.define_singleton_method(:exist?, &original_exist)
|
|
end
|
|
end
|
|
end
|