chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,179 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
//
|
||||
|
||||
import ContainerizationError
|
||||
import ContainerizationExtras
|
||||
import ContainerizationIO
|
||||
import ContainerizationOCI
|
||||
import Crypto
|
||||
import Foundation
|
||||
|
||||
extension ImageStore {
|
||||
public struct ExportOperation: Sendable {
|
||||
let name: String
|
||||
let tag: String
|
||||
let contentStore: ContentStore
|
||||
let client: ContentClient
|
||||
let progress: ProgressHandler?
|
||||
|
||||
public init(name: String, tag: String, contentStore: ContentStore, client: ContentClient, progress: ProgressHandler? = nil) {
|
||||
self.contentStore = contentStore
|
||||
self.client = client
|
||||
self.progress = progress
|
||||
self.name = name
|
||||
self.tag = tag
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
public func export(index: Descriptor, platforms: (Platform) -> Bool, filter: (Descriptor) -> Bool = { _ in true }) async throws -> Descriptor {
|
||||
var pushQueue: [[Descriptor]] = []
|
||||
var current: [Descriptor] = [index]
|
||||
while !current.isEmpty {
|
||||
let children = try await self.getChildren(descs: current)
|
||||
let matches = try filterPlatforms(matcher: platforms, children).uniqued { $0.digest }
|
||||
pushQueue.append(matches)
|
||||
current = matches
|
||||
}
|
||||
let localIndexData = try await self.createIndex(from: index, matching: platforms)
|
||||
|
||||
await updatePushProgress(pushQueue: pushQueue, localIndexData: localIndexData)
|
||||
|
||||
// We need to work bottom up when pushing an image.
|
||||
// First, the tar blobs / config layers, then, the manifests and so on...
|
||||
// When processing a given "level", the requests maybe made in parallel.
|
||||
// We need to ensure that the child level has been uploaded fully
|
||||
// before uploading the parent level.
|
||||
try await withThrowingTaskGroup(of: Void.self) { group in
|
||||
for layerGroup in pushQueue.reversed() {
|
||||
for chunk in layerGroup.chunks(ofCount: 8) {
|
||||
for desc in chunk.filter(filter) {
|
||||
guard let content = try await self.contentStore.get(digest: desc.digest) else {
|
||||
throw ContainerizationError(.notFound, message: "content with digest \(desc.digest)")
|
||||
}
|
||||
group.addTask {
|
||||
let readStream = try ReadStream(url: content.path)
|
||||
try await self.pushContent(descriptor: desc, stream: readStream)
|
||||
}
|
||||
}
|
||||
try await group.waitForAll()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Lastly, we need to construct and push a new index, since we may
|
||||
// have pushed content only for specific platforms.
|
||||
let digest = SHA256.hash(data: localIndexData)
|
||||
// The descriptor's mediaType becomes the HTTP Content-Type in
|
||||
// RegistryClient.push and must match the mediaType field inside
|
||||
// localIndexData. Registries reject mismatches with MANIFEST_INVALID.
|
||||
let descriptor = Descriptor(
|
||||
mediaType: index.mediaType,
|
||||
digest: digest.digestString,
|
||||
size: Int64(localIndexData.count))
|
||||
let stream = ReadStream(data: localIndexData)
|
||||
try await self.pushContent(descriptor: descriptor, stream: stream)
|
||||
return descriptor
|
||||
}
|
||||
|
||||
private func updatePushProgress(pushQueue: [[Descriptor]], localIndexData: Data) async {
|
||||
for layerGroup in pushQueue {
|
||||
for desc in layerGroup {
|
||||
await progress?([
|
||||
.addTotalSize(desc.size),
|
||||
.addTotalItems(1),
|
||||
])
|
||||
}
|
||||
}
|
||||
await progress?([
|
||||
.addTotalSize(Int64(localIndexData.count)),
|
||||
.addTotalItems(1),
|
||||
])
|
||||
}
|
||||
|
||||
private func createIndex(from index: Descriptor, matching: (Platform) -> Bool) async throws -> Data {
|
||||
guard let content = try await self.contentStore.get(digest: index.digest) else {
|
||||
throw ContainerizationError(.notFound, message: "content with digest \(index.digest)")
|
||||
}
|
||||
var idx: Index = try content.decode()
|
||||
let manifests = idx.manifests
|
||||
var matchedManifests: [Descriptor] = []
|
||||
var skippedPlatforms = false
|
||||
for manifest in manifests {
|
||||
guard let p = manifest.platform else {
|
||||
continue
|
||||
}
|
||||
if matching(p) {
|
||||
matchedManifests.append(manifest)
|
||||
} else {
|
||||
skippedPlatforms = true
|
||||
}
|
||||
}
|
||||
if !skippedPlatforms {
|
||||
return try content.data()
|
||||
}
|
||||
idx.manifests = matchedManifests
|
||||
return try JSONEncoder().encode(idx)
|
||||
}
|
||||
|
||||
private func pushContent(descriptor: Descriptor, stream: ReadStream) async throws {
|
||||
do {
|
||||
let generator = {
|
||||
try stream.reset()
|
||||
return stream.stream
|
||||
}
|
||||
try await client.push(name: name, ref: tag, descriptor: descriptor, streamGenerator: generator, progress: progress)
|
||||
await progress?([
|
||||
.addSize(descriptor.size),
|
||||
.addItems(1),
|
||||
])
|
||||
} catch let err as ContainerizationError {
|
||||
guard err.code != .exists else {
|
||||
// We reported the total items and size and have to account for them in existing content.
|
||||
await progress?([
|
||||
.addSize(descriptor.size),
|
||||
.addItems(1),
|
||||
])
|
||||
return
|
||||
}
|
||||
throw err
|
||||
}
|
||||
}
|
||||
|
||||
private func getChildren(descs: [Descriptor]) async throws -> [Descriptor] {
|
||||
var out: [Descriptor] = []
|
||||
for desc in descs {
|
||||
let mediaType = desc.mediaType
|
||||
guard let content = try await self.contentStore.get(digest: desc.digest) else {
|
||||
throw ContainerizationError(.notFound, message: "content with digest \(desc.digest)")
|
||||
}
|
||||
switch mediaType {
|
||||
case MediaTypes.index, MediaTypes.dockerManifestList:
|
||||
let index: Index = try content.decode()
|
||||
out.append(contentsOf: index.manifests)
|
||||
case MediaTypes.imageManifest, MediaTypes.dockerManifest:
|
||||
let manifest: Manifest = try content.decode()
|
||||
out.append(manifest.config)
|
||||
out.append(contentsOf: manifest.layers)
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,257 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
//
|
||||
|
||||
import ContainerizationError
|
||||
import ContainerizationExtras
|
||||
import ContainerizationOCI
|
||||
import Foundation
|
||||
|
||||
extension ImageStore {
|
||||
public struct ImportOperation: Sendable {
|
||||
static let decoder = JSONDecoder()
|
||||
|
||||
let client: ContentClient
|
||||
let ingestDir: URL
|
||||
let contentStore: ContentStore
|
||||
let progress: ProgressHandler?
|
||||
let name: String
|
||||
let maxConcurrentDownloads: Int
|
||||
|
||||
public init(name: String, contentStore: ContentStore, client: ContentClient, ingestDir: URL, progress: ProgressHandler? = nil, maxConcurrentDownloads: Int = 3) {
|
||||
self.client = client
|
||||
self.ingestDir = ingestDir
|
||||
self.contentStore = contentStore
|
||||
self.progress = progress
|
||||
self.name = name
|
||||
self.maxConcurrentDownloads = maxConcurrentDownloads
|
||||
}
|
||||
|
||||
/// Pull the required image layers for the provided descriptor and platform(s) into the given directory using the provided client. Returns a descriptor to the Index manifest.
|
||||
public func `import`(root: Descriptor, matcher: (ContainerizationOCI.Platform) -> Bool) async throws -> Descriptor {
|
||||
var toProcess = [root]
|
||||
while !toProcess.isEmpty {
|
||||
// Count the total number of blobs and their size
|
||||
if let progress {
|
||||
var size: Int64 = 0
|
||||
for desc in toProcess {
|
||||
size += desc.size
|
||||
}
|
||||
await progress([
|
||||
.addTotalSize(size),
|
||||
.addTotalItems(toProcess.count),
|
||||
])
|
||||
}
|
||||
|
||||
try await self.fetchAll(toProcess)
|
||||
let children = try await self.walk(toProcess)
|
||||
let filtered = try filterPlatforms(matcher: matcher, children)
|
||||
toProcess = filtered.uniqued { $0.digest }
|
||||
}
|
||||
|
||||
guard root.mediaType != MediaTypes.dockerManifestList && root.mediaType != MediaTypes.index else {
|
||||
return root
|
||||
}
|
||||
|
||||
// Create an index for the root descriptor and write it to the content store
|
||||
let index = try await self.createIndex(for: root)
|
||||
// In cases where the root descriptor pointed to `MediaTypes.imageManifest`
|
||||
// Or `MediaTypes.dockerManifest`, it is required that we check the supported platform
|
||||
// matches the platforms we were asked to pull. This can be done only after we created
|
||||
// the Index.
|
||||
let supportedPlatforms = index.manifests.compactMap { $0.platform }
|
||||
guard supportedPlatforms.allSatisfy(matcher) else {
|
||||
throw ContainerizationError(.unsupported, message: "image \(root.digest) does not support required platforms")
|
||||
}
|
||||
let writer = try ContentWriter(for: self.ingestDir)
|
||||
let result = try writer.create(from: index)
|
||||
return Descriptor(
|
||||
mediaType: MediaTypes.index,
|
||||
digest: result.digest.digestString,
|
||||
size: Int64(result.size))
|
||||
}
|
||||
|
||||
private func getManifestContent<T: Sendable & Codable>(descriptor: Descriptor) async throws -> T {
|
||||
do {
|
||||
if let content = try await self.contentStore.get(digest: descriptor.digest.trimmingDigestPrefix) {
|
||||
return try content.decode()
|
||||
}
|
||||
if let content = try? LocalContent(path: ingestDir.appending(path: descriptor.digest.trimmingDigestPrefix)) {
|
||||
return try content.decode()
|
||||
}
|
||||
return try await self.client.fetch(name: name, descriptor: descriptor)
|
||||
} catch {
|
||||
throw ContainerizationError(.internalError, message: "cannot fetch content with digest \(descriptor.digest)", cause: error)
|
||||
}
|
||||
}
|
||||
|
||||
private func walk(_ descriptors: [Descriptor]) async throws -> [Descriptor] {
|
||||
var out: [Descriptor] = []
|
||||
for desc in descriptors {
|
||||
let mediaType = desc.mediaType
|
||||
switch mediaType {
|
||||
case MediaTypes.index, MediaTypes.dockerManifestList:
|
||||
let index: Index = try await self.getManifestContent(descriptor: desc)
|
||||
out.append(contentsOf: index.manifests)
|
||||
case MediaTypes.imageManifest, MediaTypes.dockerManifest:
|
||||
let manifest: Manifest = try await self.getManifestContent(descriptor: desc)
|
||||
out.append(manifest.config)
|
||||
out.append(contentsOf: manifest.layers)
|
||||
default:
|
||||
// TODO: Explicitly handle other content types
|
||||
continue
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
private func fetchAll(_ descriptors: [Descriptor]) async throws {
|
||||
try await withThrowingTaskGroup(of: Void.self) { group in
|
||||
var iterator = descriptors.makeIterator()
|
||||
// Start initial batch of concurrent downloads based on maxConcurrentDownloads
|
||||
for _ in 0..<self.maxConcurrentDownloads {
|
||||
if let desc = iterator.next() {
|
||||
group.addTask {
|
||||
try await self.fetch(desc)
|
||||
}
|
||||
}
|
||||
}
|
||||
// As tasks complete, add new ones to maintain concurrency
|
||||
for try await _ in group {
|
||||
if let desc = iterator.next() {
|
||||
group.addTask {
|
||||
try await self.fetch(desc)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private func fetch(_ descriptor: Descriptor) async throws {
|
||||
if let found = try await self.contentStore.get(digest: descriptor.digest) {
|
||||
try FileManager.default.copyItem(at: found.path, to: ingestDir.appendingPathComponent(descriptor.digest.trimmingDigestPrefix))
|
||||
await progress?([
|
||||
// Count the size of the blob
|
||||
.addSize(descriptor.size),
|
||||
// Count the number of blobs
|
||||
.addItems(1),
|
||||
])
|
||||
return
|
||||
}
|
||||
|
||||
if descriptor.size > 1.mib() {
|
||||
try await self.fetchBlob(descriptor)
|
||||
} else {
|
||||
try await self.fetchData(descriptor)
|
||||
}
|
||||
// Count the number of blobs
|
||||
await progress?([
|
||||
.addItems(1)
|
||||
])
|
||||
}
|
||||
|
||||
private func fetchBlob(_ descriptor: Descriptor) async throws {
|
||||
let id = UUID().uuidString
|
||||
let fm = FileManager.default
|
||||
let tempFile = ingestDir.appendingPathComponent(id)
|
||||
let (_, digest) = try await client.fetchBlob(name: name, descriptor: descriptor, into: tempFile, progress: progress)
|
||||
guard digest.digestString == descriptor.digest else {
|
||||
throw ContainerizationError(.internalError, message: "digest mismatch expected \(descriptor.digest), got \(digest.digestString)")
|
||||
}
|
||||
do {
|
||||
try fm.moveItem(at: tempFile, to: ingestDir.appendingPathComponent(digest.encoded))
|
||||
} catch let err as NSError {
|
||||
guard err.code == NSFileWriteFileExistsError else {
|
||||
throw err
|
||||
}
|
||||
try fm.removeItem(at: tempFile)
|
||||
}
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
private func fetchData(_ descriptor: Descriptor) async throws -> Data {
|
||||
let data = try await client.fetchData(name: name, descriptor: descriptor)
|
||||
let writer = try ContentWriter(for: ingestDir)
|
||||
let result = try writer.write(data)
|
||||
if let progress {
|
||||
let size = Int64(result.size)
|
||||
await progress([
|
||||
.addSize(size)
|
||||
])
|
||||
}
|
||||
guard result.digest.digestString == descriptor.digest else {
|
||||
throw ContainerizationError(.internalError, message: "digest mismatch expected \(descriptor.digest), got \(result.digest.digestString)")
|
||||
}
|
||||
return data
|
||||
}
|
||||
|
||||
private func createIndex(for root: Descriptor) async throws -> Index {
|
||||
switch root.mediaType {
|
||||
case MediaTypes.index, MediaTypes.dockerManifestList:
|
||||
return try await self.getManifestContent(descriptor: root)
|
||||
case MediaTypes.imageManifest, MediaTypes.dockerManifest:
|
||||
let supportedPlatforms = try await getSupportedPlatforms(for: root)
|
||||
guard supportedPlatforms.count == 1 else {
|
||||
throw ContainerizationError(
|
||||
.internalError,
|
||||
message:
|
||||
"descriptor \(root.mediaType) with digest \(root.digest) does not list any supported platform or supports more than one platform, supported platforms: \(supportedPlatforms)"
|
||||
)
|
||||
}
|
||||
let platform = supportedPlatforms.first!
|
||||
var root = root
|
||||
root.platform = platform
|
||||
let index = ContainerizationOCI.Index(
|
||||
schemaVersion: 2, manifests: [root],
|
||||
annotations: [
|
||||
// indicate that this is a synthesized index which is not directly user facing
|
||||
AnnotationKeys.containerizationIndexIndirect: "true"
|
||||
])
|
||||
return index
|
||||
default:
|
||||
throw ContainerizationError(.internalError, message: "failed to create index for descriptor \(root.digest), media type \(root.mediaType)")
|
||||
}
|
||||
}
|
||||
|
||||
private func getSupportedPlatforms(for root: Descriptor) async throws -> [ContainerizationOCI.Platform] {
|
||||
var supportedPlatforms: [ContainerizationOCI.Platform] = []
|
||||
var toProcess = [root]
|
||||
while !toProcess.isEmpty {
|
||||
let children = try await self.walk(toProcess)
|
||||
for child in children {
|
||||
if let p = child.platform {
|
||||
supportedPlatforms.append(p)
|
||||
continue
|
||||
}
|
||||
switch child.mediaType {
|
||||
case MediaTypes.imageConfig, MediaTypes.dockerImageConfig:
|
||||
let config: ContainerizationOCI.Image = try await self.getManifestContent(descriptor: child)
|
||||
let p = ContainerizationOCI.Platform(
|
||||
arch: config.architecture, os: config.os, osFeatures: config.osFeatures, variant: config.variant
|
||||
)
|
||||
supportedPlatforms.append(p)
|
||||
default:
|
||||
continue
|
||||
}
|
||||
}
|
||||
toProcess = children
|
||||
}
|
||||
return supportedPlatforms
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import ContainerizationError
|
||||
import ContainerizationExtras
|
||||
import ContainerizationOCI
|
||||
import Foundation
|
||||
|
||||
extension ImageStore {
|
||||
/// Exports the specified images and their associated layers to an OCI Image Layout directory.
|
||||
/// This function saves the images identified by the `references` array, including their
|
||||
/// manifests and layer blobs, into a directory structure compliant with the OCI Image Layout specification at the given `out` URL.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - references: A list image references that exists in the `ImageStore` that are to be saved in the OCI Image Layout format.
|
||||
/// - out: A URL to a directory on disk at which the OCI Image Layout structure will be created.
|
||||
/// - platform: An optional parameter to indicate the platform to be saved for the images.
|
||||
/// Defaults to `nil` signifying that layers for all supported platforms by the images will be saved.
|
||||
///
|
||||
public func save(references: [String], out: URL, platform: Platform? = nil) async throws {
|
||||
let matcher = createPlatformMatcher(for: platform)
|
||||
let fileManager = FileManager.default
|
||||
let tempDir = fileManager.uniqueTemporaryDirectory()
|
||||
defer {
|
||||
try? fileManager.removeItem(at: tempDir)
|
||||
}
|
||||
|
||||
var toSave: [Image] = []
|
||||
for reference in references {
|
||||
let image = try await self.get(reference: reference)
|
||||
let allowedMediaTypes = [MediaTypes.dockerManifestList, MediaTypes.index]
|
||||
guard allowedMediaTypes.contains(image.mediaType) else {
|
||||
throw ContainerizationError(.internalError, message: "cannot save image \(image.reference) with Index media type \(image.mediaType)")
|
||||
}
|
||||
toSave.append(image)
|
||||
}
|
||||
let client = try LocalOCILayoutClient(root: out)
|
||||
var saved: [Descriptor] = []
|
||||
|
||||
for image in toSave {
|
||||
let ref = try Reference.parse(image.reference)
|
||||
let name = ref.path
|
||||
guard let tag = ref.tag ?? ref.digest else {
|
||||
throw ContainerizationError(.invalidArgument, message: "invalid tag/digest for image reference \(image.reference)")
|
||||
}
|
||||
let operation = ExportOperation(name: name, tag: tag, contentStore: self.contentStore, client: client, progress: nil)
|
||||
var descriptor = try await operation.export(index: image.descriptor, platforms: matcher)
|
||||
client.setImageReferenceAnnotation(descriptor: &descriptor, reference: image.reference)
|
||||
saved.append(descriptor)
|
||||
}
|
||||
try client.createOCILayoutStructure(directory: out, manifests: saved)
|
||||
}
|
||||
|
||||
/// Imports one or more images and their associated layers from an OCI Image Layout directory.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - directory: A URL to a directory on disk at that follows the OCI Image Layout structure.
|
||||
/// - progress: An optional handler over which progress update events about the load operation can be received.
|
||||
/// - Returns: The list of images that were loaded into the `ImageStore`.
|
||||
///
|
||||
public func load(from directory: URL, progress: ProgressHandler? = nil) async throws -> [Image] {
|
||||
let client = try LocalOCILayoutClient(root: directory)
|
||||
let index = try client.loadIndexFromOCILayout(directory: directory)
|
||||
let matcher = createPlatformMatcher(for: nil)
|
||||
|
||||
var loaded: [Image.Description] = []
|
||||
let (id, tempDir) = try await self.contentStore.newIngestSession()
|
||||
do {
|
||||
for descriptor in index.manifests {
|
||||
let reference = client.getImageReferencefromDescriptor(descriptor: descriptor)
|
||||
let ref = try Reference.parse(reference)
|
||||
let name = ref.path
|
||||
let operation = ImportOperation(name: name, contentStore: self.contentStore, client: client, ingestDir: tempDir, progress: progress)
|
||||
let indexDesc = try await operation.import(root: descriptor, matcher: matcher)
|
||||
loaded.append(Image.Description(reference: reference, descriptor: indexDesc))
|
||||
}
|
||||
|
||||
let loadedImages = loaded
|
||||
let importedImages = try await self.lock.withLock { lock in
|
||||
var images: [Image] = []
|
||||
try await self.contentStore.completeIngestSession(id)
|
||||
for description in loadedImages {
|
||||
let img = try await self._create(description: description, lock: lock)
|
||||
images.append(img)
|
||||
}
|
||||
return images
|
||||
}
|
||||
guard importedImages.count > 0 else {
|
||||
throw ContainerizationError(.internalError, message: "failed to import image")
|
||||
}
|
||||
return importedImages
|
||||
} catch {
|
||||
try? await self.contentStore.cancelIngestSession(id)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,90 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import ContainerizationError
|
||||
import ContainerizationOCI
|
||||
import Foundation
|
||||
|
||||
extension ImageStore {
|
||||
/// A ReferenceManager handles the mappings between an image's
|
||||
/// reference and the underlying descriptor inside of a content store.
|
||||
internal actor ReferenceManager {
|
||||
private let path: URL
|
||||
|
||||
private typealias State = [String: Descriptor]
|
||||
private var images: State
|
||||
|
||||
public init(path: URL) throws {
|
||||
try FileManager.default.createDirectory(at: path, withIntermediateDirectories: true)
|
||||
|
||||
self.path = path
|
||||
self.images = [:]
|
||||
}
|
||||
|
||||
private func load() throws -> State {
|
||||
let statePath = self.path.appendingPathComponent("state.json")
|
||||
guard FileManager.default.fileExists(atPath: statePath.absolutePath()) else {
|
||||
return [:]
|
||||
}
|
||||
do {
|
||||
let data = try Data(contentsOf: statePath)
|
||||
return try JSONDecoder().decode(State.self, from: data)
|
||||
} catch {
|
||||
throw ContainerizationError(.internalError, message: "failed to load image state \(error.localizedDescription)")
|
||||
}
|
||||
}
|
||||
|
||||
private func save(_ state: State) throws {
|
||||
let statePath = self.path.appendingPathComponent("state.json")
|
||||
try JSONEncoder().encode(state).write(to: statePath)
|
||||
}
|
||||
|
||||
public func delete(reference: String) throws {
|
||||
var state = try self.load()
|
||||
state.removeValue(forKey: reference)
|
||||
try self.save(state)
|
||||
}
|
||||
|
||||
public func delete(image: Image.Description) throws {
|
||||
try self.delete(reference: image.reference)
|
||||
}
|
||||
|
||||
public func create(description: Image.Description) throws {
|
||||
var state = try self.load()
|
||||
state[description.reference] = description.descriptor
|
||||
try self.save(state)
|
||||
}
|
||||
|
||||
public func list() throws -> [Image.Description] {
|
||||
let state = try self.load()
|
||||
return state.map { key, val in
|
||||
let description = Image.Description(reference: key, descriptor: val)
|
||||
return description
|
||||
}
|
||||
}
|
||||
|
||||
public func get(reference: String) throws -> Image.Description {
|
||||
let images = try self.list()
|
||||
let hit = images.first(where: { image in
|
||||
image.reference == reference
|
||||
})
|
||||
guard let hit else {
|
||||
throw ContainerizationError(.notFound, message: "image \(reference) not found")
|
||||
}
|
||||
return hit
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,395 @@
|
||||
//===----------------------------------------------------------------------===//
|
||||
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
|
||||
//
|
||||
// Licensed under the Apache License, Version 2.0 (the "License");
|
||||
// you may not use this file except in compliance with the License.
|
||||
// You may obtain a copy of the License at
|
||||
//
|
||||
// https://www.apache.org/licenses/LICENSE-2.0
|
||||
//
|
||||
// Unless required by applicable law or agreed to in writing, software
|
||||
// distributed under the License is distributed on an "AS IS" BASIS,
|
||||
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
// See the License for the specific language governing permissions and
|
||||
// limitations under the License.
|
||||
//===----------------------------------------------------------------------===//
|
||||
|
||||
import ContainerizationError
|
||||
import ContainerizationExtras
|
||||
import ContainerizationOCI
|
||||
import Foundation
|
||||
|
||||
/// An ImageStore handles the mappings between an image's
|
||||
/// reference and the underlying descriptor inside of a content store.
|
||||
public actor ImageStore: Sendable {
|
||||
/// The ImageStore path it was created with.
|
||||
public nonisolated let path: URL
|
||||
|
||||
private let referenceManager: ReferenceManager
|
||||
internal let contentStore: ContentStore
|
||||
internal let lock: AsyncLock = AsyncLock()
|
||||
|
||||
public init(path: URL, contentStore: ContentStore? = nil) throws {
|
||||
try FileManager.default.createDirectory(at: path, withIntermediateDirectories: true)
|
||||
|
||||
if let contentStore {
|
||||
self.contentStore = contentStore
|
||||
} else {
|
||||
self.contentStore = try LocalContentStore(path: path.appendingPathComponent("content"))
|
||||
}
|
||||
|
||||
self.path = path
|
||||
self.referenceManager = try ReferenceManager(path: path)
|
||||
}
|
||||
|
||||
/// Return the default image store for the current user.
|
||||
public static let `default`: ImageStore = {
|
||||
do {
|
||||
let root = try defaultRoot()
|
||||
return try ImageStore(path: root)
|
||||
} catch {
|
||||
fatalError("unable to initialize default ImageStore \(error)")
|
||||
}
|
||||
}()
|
||||
|
||||
private static func defaultRoot() throws -> URL {
|
||||
let root = FileManager.default.urls(
|
||||
for: .applicationSupportDirectory,
|
||||
in: .userDomainMask
|
||||
).first
|
||||
guard let root else {
|
||||
throw ContainerizationError(.notFound, message: "unable to get Application Support directory for current user")
|
||||
}
|
||||
return root.appendingPathComponent("com.apple.containerization")
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
extension ImageStore {
|
||||
/// Get an image from the `ImageStore`.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - reference: Name of the image.
|
||||
/// - pull: Pull the image if it is not found.
|
||||
///
|
||||
/// - Returns: A `Containerization.Image` object whose `reference` matches the given string.
|
||||
/// This method throws a `ContainerizationError(code: .notFound)` if the provided reference does not exist in the `ImageStore`.
|
||||
public func get(reference: String, pull: Bool = false) async throws -> Image {
|
||||
do {
|
||||
let desc = try await self.referenceManager.get(reference: reference)
|
||||
return Image(description: desc, contentStore: self.contentStore)
|
||||
} catch let error as ContainerizationError {
|
||||
if error.code == .notFound && pull {
|
||||
return try await self.pull(reference: reference)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/// Get a list of all images in the `ImageStore`.
|
||||
///
|
||||
/// - Returns: A `[Containerization.Image]` for all the images in the `ImageStore`.
|
||||
public func list() async throws -> [Image] {
|
||||
try await self.referenceManager.list().map { desc in
|
||||
Image(description: desc, contentStore: self.contentStore)
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new image in the `ImageStore`.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - description: The underlying `Image.Description` that contains information about the reference and index descriptor for the image to be created.
|
||||
///
|
||||
/// - Note: It is assumed that the underlying manifests and blob layers for the image already exists in the `ContentStore` that the `ImageStore` was initialized with. This method is invoked when the `pull(...)` , `load(...)` and `tag(...)` methods are used.
|
||||
/// - Returns: A `Containerization.Image`
|
||||
@discardableResult
|
||||
public func create(description: Image.Description) async throws -> Image {
|
||||
try await self.lock.withLock { ctx in
|
||||
try await self._create(description: description, lock: ctx)
|
||||
}
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
internal func _create(description: Image.Description, lock: AsyncLock.Context) async throws -> Image {
|
||||
try await self.referenceManager.create(description: description)
|
||||
return Image(description: description, contentStore: self.contentStore)
|
||||
}
|
||||
|
||||
/// Delete an image from the `ImageStore`.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - reference: Name of the image that is to be deleted.
|
||||
/// - performCleanup: Perform a garbage collection on the `ContentStore`, removing all unreferenced image layers and manifests,
|
||||
public func delete(reference: String, performCleanup: Bool = false) async throws {
|
||||
try await self.lock.withLock { lockCtx in
|
||||
try await self.referenceManager.delete(reference: reference)
|
||||
if performCleanup {
|
||||
try await self._cleanUpOrphanedBlobs(lockCtx)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Clean up orphaned blobs that are no longer referenced by any image.
|
||||
///
|
||||
/// - Returns: Returns a tuple of `(deleted, freed)`.
|
||||
/// `deleted` : A list of the names of the content items that were deleted from the `ContentStore`,
|
||||
/// `freed` : The total size of the items that were deleted.
|
||||
@discardableResult
|
||||
public func cleanUpOrphanedBlobs() async throws -> (deleted: [String], freed: UInt64) {
|
||||
try await self.lock.withLock { lockCtx in
|
||||
try await self._cleanUpOrphanedBlobs(lockCtx)
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate the size of orphaned blobs without deleting them.
|
||||
///
|
||||
/// - Returns: The total size in bytes of blobs that are not referenced by any image.
|
||||
public func calculateOrphanedBlobsSize() async throws -> UInt64 {
|
||||
try await self.lock.withLock { lockCtx in
|
||||
try await self._calculateOrphanedBlobsSize(lockCtx)
|
||||
}
|
||||
}
|
||||
|
||||
@discardableResult
|
||||
private func _cleanUpOrphanedBlobs(_ lock: AsyncLock.Context) async throws -> (deleted: [String], freed: UInt64) {
|
||||
let images = try await self.list()
|
||||
var referenced: [String] = []
|
||||
for image in images {
|
||||
try await referenced.append(contentsOf: image.referencedDigests().uniqued())
|
||||
}
|
||||
let (deleted, size) = try await self.contentStore.delete(keeping: referenced)
|
||||
return (deleted, size)
|
||||
}
|
||||
|
||||
private func _calculateOrphanedBlobsSize(_ lock: AsyncLock.Context) async throws -> UInt64 {
|
||||
let images = try await self.list()
|
||||
var referenced: [String] = []
|
||||
for image in images {
|
||||
try await referenced.append(contentsOf: image.referencedDigests().uniqued())
|
||||
}
|
||||
|
||||
// Calculate size of blobs not in the referenced list
|
||||
let referencedSet = Set(referenced.map { $0.trimmingDigestPrefix })
|
||||
let blobsPath = self.path.appendingPathComponent("content/blobs/sha256")
|
||||
|
||||
let fileManager = FileManager.default
|
||||
let allBlobs = try fileManager.contentsOfDirectory(
|
||||
at: blobsPath,
|
||||
includingPropertiesForKeys: [.fileSizeKey],
|
||||
options: [.skipsHiddenFiles]
|
||||
)
|
||||
|
||||
var orphanedSize: UInt64 = 0
|
||||
for blobURL in allBlobs {
|
||||
let digest = blobURL.lastPathComponent
|
||||
if !referencedSet.contains(digest) {
|
||||
if let resourceValues = try? blobURL.resourceValues(forKeys: [.fileSizeKey]),
|
||||
let size = resourceValues.fileSize
|
||||
{
|
||||
orphanedSize += UInt64(size)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return orphanedSize
|
||||
}
|
||||
|
||||
/// Tag an existing image such that it can be referenced by another name.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - existing: The reference to an image that already exists in the `ImageStore`.
|
||||
/// - new: The new reference by which the image should also be referenced as.
|
||||
/// - Note: The new image created in the `ImageStore` will have the same `Image.Description`
|
||||
/// as that of the image with reference `existing.`
|
||||
/// - Returns: A `Containerization.Image` object to the newly created image.
|
||||
public func tag(existing: String, new: String) async throws -> Image {
|
||||
let old = try await self.get(reference: existing)
|
||||
let descriptor = old.descriptor
|
||||
do {
|
||||
_ = try Reference.parse(new)
|
||||
} catch {
|
||||
throw ContainerizationError(.invalidArgument, message: "invalid reference \(new), error: \(error)")
|
||||
}
|
||||
let newDescription = Image.Description(reference: new, descriptor: descriptor)
|
||||
return try await self.create(description: newDescription)
|
||||
}
|
||||
}
|
||||
|
||||
extension ImageStore {
|
||||
/// Pull an image and its associated manifest and blob layers from a remote registry.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - reference: A string that references an image in a remote registry of the form `<host>[:<port>]/repository:<tag>`
|
||||
/// For example: "docker.io/library/alpine:latest".
|
||||
/// - platform: An optional parameter to indicate the platform to be pulled for the image.
|
||||
/// Defaults to `nil` signifying that layers for all supported platforms by the image will be pulled.
|
||||
/// - insecure: A boolean indicating if the connection to the remote registry should be made via plain-text http or not.
|
||||
/// Defaults to false, meaning the connection to the registry will be over https.
|
||||
/// - auth: An object that implements the `Authentication` protocol,
|
||||
/// used to add any credentials to the HTTP requests that are made to the registry.
|
||||
/// Defaults to `nil` meaning no additional credentials are added to any HTTP requests made to the registry.
|
||||
/// - progress: An optional handler over which progress update events about the pull operation can be received.
|
||||
///
|
||||
/// - Returns: A `Containerization.Image` object to the newly pulled image.
|
||||
public func pull(
|
||||
reference: String, platform: Platform? = nil, insecure: Bool = false,
|
||||
auth: Authentication? = nil, progress: ProgressHandler? = nil, maxConcurrentDownloads: Int = 3
|
||||
) async throws -> Image {
|
||||
|
||||
let matcher = createPlatformMatcher(for: platform)
|
||||
let client = try RegistryClient(reference: reference, insecure: insecure, auth: auth, tlsConfiguration: TLSUtils.makeEnvironmentAwareTLSConfiguration())
|
||||
|
||||
let ref = try Reference.parse(reference)
|
||||
let name = ref.path
|
||||
guard let tag = ref.tag ?? ref.digest else {
|
||||
throw ContainerizationError(.invalidArgument, message: "invalid tag/digest for image reference \(reference)")
|
||||
}
|
||||
|
||||
let rootDescriptor = try await client.resolve(name: name, tag: tag)
|
||||
let (id, tempDir) = try await self.contentStore.newIngestSession()
|
||||
let operation = ImportOperation(
|
||||
name: name, contentStore: self.contentStore, client: client, ingestDir: tempDir, progress: progress, maxConcurrentDownloads: maxConcurrentDownloads)
|
||||
do {
|
||||
let index = try await operation.import(root: rootDescriptor, matcher: matcher)
|
||||
return try await self.lock.withLock { lock in
|
||||
try await self.contentStore.completeIngestSession(id)
|
||||
let description = Image.Description(reference: reference, descriptor: index)
|
||||
let image = try await self._create(description: description, lock: lock)
|
||||
return image
|
||||
}
|
||||
} catch {
|
||||
try? await self.contentStore.cancelIngestSession(id)
|
||||
throw error
|
||||
}
|
||||
}
|
||||
|
||||
/// Push an image and its associated manifest and blob layers to a remote registry.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - reference: A string that references an image in the `ImageStore`. It must be of the form `<host>[:<port>]/repository:<tag>`
|
||||
/// For example: "ghcr.io/foo-bar-baz/image:v1".
|
||||
/// - platform: An optional parameter to indicate the platform to be pushed for the image.
|
||||
/// Defaults to `nil` signifying that layers for all supported platforms by the image will be pushed to the remote registry.
|
||||
/// - insecure: A boolean indicating if the connection to the remote registry should be made via plain-text http or not.
|
||||
/// Defaults to false, meaning the connection to the registry will be over https.
|
||||
/// - auth: An object that implements the `Authentication` protocol,
|
||||
/// used to add any credentials to the HTTP requests that are made to the registry.
|
||||
/// Defaults to `nil` meaning no additional credentials are added to any HTTP requests made to the registry.
|
||||
/// - progress: An optional handler over which progress update events about the push operation can be received.
|
||||
///
|
||||
public func push(reference: String, platform: Platform? = nil, insecure: Bool = false, auth: Authentication? = nil, progress: ProgressHandler? = nil) async throws {
|
||||
let matcher = createPlatformMatcher(for: platform)
|
||||
let client = try RegistryClient(reference: reference, insecure: insecure, auth: auth, tlsConfiguration: TLSUtils.makeEnvironmentAwareTLSConfiguration())
|
||||
try await self.pushSingle(reference: reference, client: client, matcher: matcher, progress: progress)
|
||||
}
|
||||
|
||||
/// Push multiple image references to a remote registry, sharing a single ``RegistryClient``.
|
||||
///
|
||||
/// All references must resolve to the same registry host. Passing references that target
|
||||
/// different hosts throws a ``ContainerizationError`` with code ``invalidArgument``.
|
||||
///
|
||||
/// - Parameters:
|
||||
/// - references: An array of fully qualified image reference strings to push.
|
||||
/// Each must include a host (e.g., `"ghcr.io/myrepo/myimage:v1"`).
|
||||
/// - platform: An optional parameter to indicate the platform to be pushed for each image.
|
||||
/// Defaults to `nil` signifying that layers for all supported platforms will be pushed.
|
||||
/// - insecure: A boolean indicating if the connection to the remote registry should be made via plain-text http or not.
|
||||
/// Defaults to false, meaning the connection to the registry will be over https.
|
||||
/// - auth: An object that implements the `Authentication` protocol,
|
||||
/// used to add any credentials to the HTTP requests that are made to the registry.
|
||||
/// Defaults to `nil` meaning no additional credentials are added to any HTTP requests made to the registry.
|
||||
/// - maxConcurrentUploads: Maximum number of concurrent tag pushes. Defaults to 3.
|
||||
/// - progress: An optional handler over which progress update events about the push operations can be received.
|
||||
///
|
||||
public func push(
|
||||
references: [String], platform: Platform? = nil, insecure: Bool = false,
|
||||
auth: Authentication? = nil, maxConcurrentUploads: Int = 3, progress: ProgressHandler? = nil
|
||||
) async throws {
|
||||
guard let firstReference = references.first else {
|
||||
return
|
||||
}
|
||||
|
||||
// Parse all references upfront: validate hosts and avoid re-parsing inside tasks.
|
||||
let parsed = try references.map { ref in try Reference.parse(ref) }
|
||||
let hosts = parsed.compactMap { $0.resolvedDomain }
|
||||
guard hosts.count == references.count else {
|
||||
throw ContainerizationError(.invalidArgument, message: "all references must include a host")
|
||||
}
|
||||
let uniqueHosts = Set(hosts)
|
||||
guard uniqueHosts.count == 1 else {
|
||||
throw ContainerizationError(
|
||||
.invalidArgument,
|
||||
message: "all references must target the same registry host, got: \(uniqueHosts.sorted().joined(separator: ", "))")
|
||||
}
|
||||
|
||||
let matcher = createPlatformMatcher(for: platform)
|
||||
let client = try RegistryClient(
|
||||
reference: firstReference, insecure: insecure, auth: auth,
|
||||
tlsConfiguration: TLSUtils.makeEnvironmentAwareTLSConfiguration())
|
||||
|
||||
let pushOne: @Sendable (String) async -> (String, String?) = { reference in
|
||||
do {
|
||||
try await self.pushSingle(reference: reference, client: client, matcher: matcher, progress: progress)
|
||||
return (reference, nil)
|
||||
} catch {
|
||||
return (reference, String(describing: error))
|
||||
}
|
||||
}
|
||||
|
||||
var iterator = references.makeIterator()
|
||||
var failures: [(reference: String, message: String)] = []
|
||||
|
||||
await withTaskGroup(of: (String, String?).self) { group in
|
||||
for _ in 0..<maxConcurrentUploads {
|
||||
guard let reference = iterator.next() else { break }
|
||||
group.addTask { await pushOne(reference) }
|
||||
}
|
||||
for await (ref, error) in group {
|
||||
if let error {
|
||||
failures.append((ref, error))
|
||||
}
|
||||
if let reference = iterator.next() {
|
||||
group.addTask { await pushOne(reference) }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !failures.isEmpty {
|
||||
let details = failures.map { "\($0.reference): \($0.message)" }.joined(separator: "\n")
|
||||
throw ContainerizationError(.internalError, message: "failed to push one or more images:\n\(details)")
|
||||
}
|
||||
}
|
||||
|
||||
private func pushSingle(
|
||||
reference: String, client: ContentClient, matcher: @Sendable (Platform) -> Bool, progress: ProgressHandler?
|
||||
) async throws {
|
||||
let allowedMediaTypes = [MediaTypes.dockerManifestList, MediaTypes.index]
|
||||
let img = try await self.get(reference: reference)
|
||||
guard allowedMediaTypes.contains(img.mediaType) else {
|
||||
throw ContainerizationError(.internalError, message: "cannot push image \(reference): unsupported media type \(img.mediaType), expected an index or manifest list")
|
||||
}
|
||||
let ref = try Reference.parse(reference)
|
||||
guard let tag = ref.tag ?? ref.digest else {
|
||||
throw ContainerizationError(.invalidArgument, message: "invalid tag/digest for image reference \(reference)")
|
||||
}
|
||||
let operation = ExportOperation(name: ref.path, tag: tag, contentStore: self.contentStore, client: client, progress: progress)
|
||||
try await operation.export(index: img.descriptor, platforms: matcher)
|
||||
}
|
||||
}
|
||||
|
||||
extension ImageStore {
|
||||
/// Get the image for the init block from the image store.
|
||||
/// If the image does not exist locally, pull the image.
|
||||
public func getInitImage(reference: String, auth: Authentication? = nil, progress: ProgressHandler? = nil) async throws -> InitImage {
|
||||
do {
|
||||
let image = try await self.get(reference: reference)
|
||||
return InitImage(image: image)
|
||||
} catch let error as ContainerizationError {
|
||||
if error.code == .notFound {
|
||||
let image = try await self.pull(reference: reference, auth: auth, progress: progress)
|
||||
return InitImage(image: image)
|
||||
}
|
||||
throw error
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user