wehub-resource-sync
318 lines
14 KiB
Ruby
318 lines
14 KiB
Ruby
# frozen_string_literal: true
|
|
|
|
require_relative "spec_helper"
|
|
|
|
# Covers five CR fixes against PromoteCommand:
|
|
# FIX-1: digest is resolved EXACTLY ONCE per staging service across the
|
|
# whole preflight+execute, and the SAME ref P1 verified is what
|
|
# gets pinned by execute_promotion (no TOCTOU).
|
|
# FIX-2: a Railway::GHCR::Error raised mid-loop in P1 produces a per-
|
|
# service REFUSE finding and the loop continues for remaining
|
|
# services — earlier findings are NOT discarded.
|
|
# FIX-3: pin_and_verify asserts the serviceInstanceDeployV2 result is a
|
|
# non-empty deployment id (symmetric with the serviceInstanceUpdate
|
|
# result check).
|
|
# FIX-4: P2 emits a WARN finding when deployment meta is not a Hash, so
|
|
# the silent skip of the in-flight race-check is visible.
|
|
# FIX-5: when pin_and_verify raises mid-loop in execute_promotion, a
|
|
# loud PARTIAL-PROMOTION report names the already-pinned services.
|
|
class PromoteResolveOnceTest < Minitest::Test
|
|
# FakeGHCR that counts resolve_digest calls per ref.
|
|
class CountingGHCR
|
|
attr_reader :resolve_calls
|
|
|
|
def initialize(resolve_map:, exists_set: nil)
|
|
@resolve_map = resolve_map
|
|
@exists_set = exists_set
|
|
@resolve_calls = Hash.new(0)
|
|
end
|
|
|
|
def resolve_digest(ref)
|
|
@resolve_calls[ref] += 1
|
|
return ref.split("@", 2).last if ref.include?("@sha256:")
|
|
@resolve_map[ref]
|
|
end
|
|
|
|
def manifest_exists(ref)
|
|
return :missing if @exists_set && !@exists_set.include?(ref)
|
|
:exists
|
|
end
|
|
|
|
def parse_image_ref(ref)
|
|
Railway::GHCR.allocate.parse_image_ref(ref)
|
|
end
|
|
end
|
|
|
|
# GQL fake reusing the pattern from PromoteExecuteTest::RecordingGQL,
|
|
# but with optional knobs for FIX-3 (redeploy returns false) and FIX-5
|
|
# (per-service update failure).
|
|
class RecordingGQL
|
|
def initialize(redeploy_result: true, update_fail_for: nil)
|
|
@calls = []
|
|
# Per-service post-update state so multi-service test cases don't
|
|
# leak each other's pinned image into pre-update rechecks. Keyed by
|
|
# serviceId. Value is { image:, post_ts: }. Pre-update returns the
|
|
# "OLD" snapshot (with an early ts) for any service not yet pinned.
|
|
@post = {}
|
|
@ts_counter = 0
|
|
@redeploy_result = redeploy_result
|
|
@update_fail_for = update_fail_for # service_id whose update returns false
|
|
end
|
|
attr_reader :calls
|
|
|
|
def query(q, vars = {})
|
|
@calls << [q, vars]
|
|
sid = vars[:serviceId]
|
|
if q.include?("serviceInstanceUpdate")
|
|
if @update_fail_for && sid == @update_fail_for
|
|
return { "serviceInstanceUpdate" => false }
|
|
end
|
|
@ts_counter += 1
|
|
@post[sid] = { image: vars.dig(:input, :source, :image), ts: "2026-05-29T00:00:%02dZ" % @ts_counter }
|
|
{ "serviceInstanceUpdate" => true }
|
|
elsif q.include?("serviceInstanceDeployV2")
|
|
# @redeploy_result false → return nil id so the DeployV2 guard
|
|
# fails loud (mirrors the prior redeploy-false fail path).
|
|
{ "serviceInstanceDeployV2" => (@redeploy_result ? "dep-#{sid}" : nil) }
|
|
elsif q.include?("ServiceInstanceRecheck")
|
|
if (entry = @post[sid])
|
|
{
|
|
"serviceInstance" => {
|
|
"id" => "i",
|
|
"source" => { "image" => entry[:image] },
|
|
"updatedAt" => entry[:ts],
|
|
"latestDeployment" => {
|
|
"id" => "dep-#{sid}", "status" => "SUCCESS",
|
|
"meta" => {
|
|
"imageDigest" => (entry[:image].include?("@") ? entry[:image].split("@", 2).last : nil),
|
|
},
|
|
},
|
|
},
|
|
}
|
|
else
|
|
{
|
|
"serviceInstance" => {
|
|
"id" => "i",
|
|
"source" => { "image" => "ghcr.io/copilotkit/x@sha256:OLD" },
|
|
"updatedAt" => "2026-05-28T00:00:00Z",
|
|
},
|
|
}
|
|
end
|
|
else
|
|
{}
|
|
end
|
|
end
|
|
|
|
def pinned_images
|
|
@calls.select { |q, _| q.include?("serviceInstanceUpdate") }.map { |_, v| v.dig(:input, :source, :image) }
|
|
end
|
|
end
|
|
|
|
# Silence pin_and_verify's 10s-per-retry waits.
|
|
def with_fast_sleeper
|
|
original = Railway::PromoteCommand.const_get(:RETRY_DELAY_SEC)
|
|
Railway::PromoteCommand.send(:remove_const, :RETRY_DELAY_SEC)
|
|
Railway::PromoteCommand.const_set(:RETRY_DELAY_SEC, 0)
|
|
yield
|
|
ensure
|
|
Railway::PromoteCommand.send(:remove_const, :RETRY_DELAY_SEC)
|
|
Railway::PromoteCommand.const_set(:RETRY_DELAY_SEC, original)
|
|
end
|
|
|
|
def make_svc(name, image:)
|
|
{
|
|
"name" => name, "service_id" => "svc-stg-#{name}",
|
|
"image" => image,
|
|
# All CRITICAL_ENV_KEYS present so the (now unconditional) critical
|
|
# env-key presence assertion does not fire — these tests isolate
|
|
# the resolve-once / pin behavior, not env-key parity.
|
|
"env_keys" => Railway::CRITICAL_ENV_KEYS.dup,
|
|
"start_command" => "node server.js", "healthcheck_path" => "/health",
|
|
"region" => "us-west", "replicas" => 1, "restart_policy" => "ON_FAILURE",
|
|
}
|
|
end
|
|
|
|
def make_prod_svc(name)
|
|
{
|
|
"name" => name, "service_id" => "svc-prod-#{name}",
|
|
"image" => "ghcr.io/copilotkit/#{name}@sha256:OLD",
|
|
"env_keys" => Railway::CRITICAL_ENV_KEYS.dup,
|
|
"start_command" => "node server.js", "healthcheck_path" => "/health",
|
|
"region" => "us-west", "replicas" => 1, "restart_policy" => "ON_FAILURE",
|
|
}
|
|
end
|
|
|
|
# =================== FIX-1 ===================
|
|
|
|
def test_fix1_resolve_digest_called_exactly_once_per_service
|
|
# Single :latest staging service. After full preflight + execute, the
|
|
# ghcr.resolve_digest call counter MUST be exactly 1 for that ref —
|
|
# not 2 (which was the pre-fix TOCTOU duplicate-resolve behavior).
|
|
ghcr = CountingGHCR.new(resolve_map: { "ghcr.io/copilotkit/x:latest" => "sha256:abc123" })
|
|
gql = RecordingGQL.new
|
|
|
|
cmd = Railway::PromoteCommand.new(["--non-interactive", "--yes", "--confirm-divergence"])
|
|
cmd.parser.parse!(cmd.argv)
|
|
cmd.instance_variable_set(:@staging_snapshot, {
|
|
"services" => [make_svc("x", image: "ghcr.io/copilotkit/x:latest")],
|
|
})
|
|
cmd.instance_variable_set(:@prod_snapshot, { "services" => [make_prod_svc("x")] })
|
|
cmd.instance_variable_set(:@gql, gql)
|
|
cmd.instance_variable_set(:@ghcr, ghcr)
|
|
cmd.define_singleton_method(:fetch_latest_staging_deployments) do |_svc_id|
|
|
[{ "id" => "d", "status" => "SUCCESS",
|
|
"meta" => { "image" => "ghcr.io/copilotkit/x@sha256:abc123" } }]
|
|
end
|
|
cmd.define_singleton_method(:run_staging_probe) { |services:| { ok: true, summary: "" } }
|
|
|
|
out, _ = with_fast_sleeper { capture_io { @rc = cmd.run_with_preflight_only } }
|
|
assert_equal 0, @rc, "promote should succeed; got out=#{out}"
|
|
|
|
# The exact ref pinned MUST be what P1 verified (resolve-once).
|
|
assert_equal ["ghcr.io/copilotkit/x@sha256:abc123"], gql.pinned_images,
|
|
"must pin the SAME digest-form ref P1 verified"
|
|
|
|
# The :latest tag must have been resolved EXACTLY ONCE — not twice.
|
|
assert_equal 1, ghcr.resolve_calls["ghcr.io/copilotkit/x:latest"],
|
|
"resolve_digest must be called exactly once for the staging tag; calls=#{ghcr.resolve_calls.inspect}"
|
|
end
|
|
|
|
# =================== FIX-2 ===================
|
|
|
|
# GHCR fake whose manifest_exists raises Railway::GHCR::Error on the
|
|
# SECOND service. The first service's findings must survive.
|
|
class RaisingOnSecondGHCR
|
|
def initialize
|
|
@manifest_calls = 0
|
|
end
|
|
|
|
def resolve_digest(ref)
|
|
return ref.split("@", 2).last if ref.include?("@sha256:")
|
|
"sha256:resolved_for_#{ref}"
|
|
end
|
|
|
|
def manifest_exists(_ref)
|
|
@manifest_calls += 1
|
|
raise Railway::GHCR::Error, "boom on call ##{@manifest_calls}" if @manifest_calls == 2
|
|
:exists
|
|
end
|
|
|
|
def parse_image_ref(ref); Railway::GHCR.allocate.parse_image_ref(ref); end
|
|
end
|
|
|
|
def test_fix2_p1_rescue_is_per_service_not_method_level
|
|
# Two services. The SECOND triggers a GHCR::Error in manifest_exists.
|
|
# Before the fix: method-level rescue replaced ALL findings with a
|
|
# single "REFUSE: P1: GHCR check raised ..." entry, dropping the
|
|
# first service's clean record AND scoping the error to "P1" with no
|
|
# service name. After the fix: the first service has no finding
|
|
# (it passed), the second service has a per-service REFUSE finding
|
|
# that names the service.
|
|
cmd = Railway::PromoteCommand.new([])
|
|
cmd.instance_variable_set(:@ghcr, RaisingOnSecondGHCR.new)
|
|
# resolved_prod_image now pins staging's RUNNING digest (meta.imageDigest);
|
|
# stub the deployment lookup so the flow reaches manifest_exists (which
|
|
# raises on the 2nd service under test).
|
|
cmd.define_singleton_method(:fetch_latest_staging_deployments) do |svc_id|
|
|
name = svc_id.sub("svc-stg-", "")
|
|
[{ "id" => "d", "status" => "SUCCESS",
|
|
"meta" => { "image" => "ghcr.io/copilotkit/#{name}:latest",
|
|
"imageDigest" => "sha256:running_#{name}" } }]
|
|
end
|
|
staging = {
|
|
"services" => [
|
|
make_svc("a", image: "ghcr.io/copilotkit/a:latest"),
|
|
make_svc("b", image: "ghcr.io/copilotkit/b:latest"),
|
|
],
|
|
}
|
|
findings = cmd.send(:check_p1_ghcr_digests, staging)
|
|
|
|
refute(findings.any? { |f| f =~ /REFUSE: P1 \(a\)/ },
|
|
"service 'a' passed P1 and must have no REFUSE; findings=#{findings.inspect}")
|
|
assert(findings.any? { |f| f =~ /REFUSE: P1 \(b\).*boom on call #2/ },
|
|
"service 'b' must have a per-service P1 REFUSE naming it; findings=#{findings.inspect}")
|
|
end
|
|
|
|
# =================== FIX-3 ===================
|
|
|
|
def test_fix3_pin_and_verify_raises_when_deploy_returns_no_id
|
|
gql = RecordingGQL.new(redeploy_result: false)
|
|
err = assert_raises(Railway::PromoteCommand::MutationError) do
|
|
Railway::PromoteCommand.pin_and_verify(gql,
|
|
service_id: "svc-x", env_id: "env-prod",
|
|
image: "ghcr.io/copilotkit/x@sha256:abc",
|
|
sleeper: ->(_) {})
|
|
end
|
|
assert_match(/serviceInstanceDeployV2/i, err.message,
|
|
"MutationError must reference the deploy mutation; got: #{err.message}")
|
|
end
|
|
|
|
# =================== FIX-4 ===================
|
|
|
|
def test_fix4_p2_warns_when_meta_is_a_string
|
|
# P2 already guards meta.is_a?(Hash) (no crash), but the silent skip
|
|
# of the race-check should produce a WARN finding so it's visible.
|
|
cmd = Railway::PromoteCommand.new([])
|
|
cmd.define_singleton_method(:fetch_latest_staging_deployments) do |_svc_id|
|
|
[{ "id" => "d", "status" => "SUCCESS",
|
|
"meta" => "raw-string-not-a-hash" }]
|
|
end
|
|
staging = {
|
|
"services" => [{
|
|
"name" => "x", "service_id" => "svc-1",
|
|
"image" => "ghcr.io/copilotkit/x@sha256:abc", "digest" => "sha256:abc",
|
|
"env_keys" => [],
|
|
}],
|
|
}
|
|
findings = cmd.send(:check_p2_staging_deployments, staging)
|
|
assert(findings.any? { |f| f =~ /WARN: P2 \(x\).*meta.*String.*Hash/ },
|
|
"expected WARN finding for non-Hash meta; got: #{findings.inspect}")
|
|
end
|
|
|
|
# =================== FIX-5 ===================
|
|
|
|
def test_fix5_partial_promotion_report_on_midloop_failure
|
|
# Three staging services [a, b, c]; pin succeeds for a and b, fails
|
|
# for c (serviceInstanceUpdate returns false → MutationError).
|
|
# The output must name a and b as already-pinned and c as failed.
|
|
ghcr = CountingGHCR.new(resolve_map: {
|
|
"ghcr.io/copilotkit/a:latest" => "sha256:aaa",
|
|
"ghcr.io/copilotkit/b:latest" => "sha256:bbb",
|
|
"ghcr.io/copilotkit/c:latest" => "sha256:ccc",
|
|
})
|
|
gql = RecordingGQL.new(update_fail_for: "svc-prod-c")
|
|
|
|
cmd = Railway::PromoteCommand.new(["--non-interactive", "--yes", "--confirm-divergence"])
|
|
cmd.parser.parse!(cmd.argv)
|
|
cmd.instance_variable_set(:@staging_snapshot, {
|
|
"services" => [
|
|
make_svc("a", image: "ghcr.io/copilotkit/a:latest"),
|
|
make_svc("b", image: "ghcr.io/copilotkit/b:latest"),
|
|
make_svc("c", image: "ghcr.io/copilotkit/c:latest"),
|
|
],
|
|
})
|
|
cmd.instance_variable_set(:@prod_snapshot, {
|
|
"services" => [make_prod_svc("a"), make_prod_svc("b"), make_prod_svc("c")],
|
|
})
|
|
cmd.instance_variable_set(:@gql, gql)
|
|
cmd.instance_variable_set(:@ghcr, ghcr)
|
|
cmd.define_singleton_method(:fetch_latest_staging_deployments) do |svc_id|
|
|
svc = svc_id.sub("svc-stg-", "")
|
|
digest = { "a" => "sha256:aaa", "b" => "sha256:bbb", "c" => "sha256:ccc" }.fetch(svc)
|
|
[{ "id" => "d", "status" => "SUCCESS",
|
|
"meta" => { "image" => "ghcr.io/copilotkit/#{svc}@#{digest}" } }]
|
|
end
|
|
cmd.define_singleton_method(:run_staging_probe) { |services:| { ok: true, summary: "" } }
|
|
|
|
out, err = with_fast_sleeper { capture_io { @rc = cmd.run_with_preflight_only } }
|
|
assert_equal 1, @rc, "execute_promotion must return 1 when a mid-loop pin fails"
|
|
combined = out + err
|
|
assert_match(/PARTIAL PROMOTION/i, combined,
|
|
"must emit a loud PARTIAL PROMOTION report; output=#{combined}")
|
|
assert_match(/already pinned.*\ba\b.*\bb\b/m, combined,
|
|
"report must name the already-pinned services a and b; output=#{combined}")
|
|
assert_match(/FAILED on c/, combined,
|
|
"report must name the failing service c; output=#{combined}")
|
|
end
|
|
end
|