chore: import upstream snapshot with attribution
Build containerization / Verify commit signatures (push) Has been skipped
Build containerization / containerization (push) Successful in 0s
Linux build / Linux compile check (push) Has been cancelled
Linux build / Determine Swift version (push) Has been cancelled

This commit is contained in:
wehub-resource-sync
2026-07-13 12:25:30 +08:00
commit 680845cb1c
445 changed files with 103779 additions and 0 deletions
@@ -0,0 +1,56 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
package actor AsyncStore<T> {
private var _value: T?
package init(_ value: T? = nil) {
self._value = value
}
package func get() -> T? {
self._value
}
package func set(_ value: T) {
self._value = value
}
}
package actor AsyncSet<T: Hashable> {
private var buffer: Set<T>
package init<S: Sequence>(_ elements: S) where S.Element == T {
buffer = Set(elements)
}
package var count: Int {
buffer.count
}
package func insert(_ element: T) {
buffer.insert(element)
}
@discardableResult
package func remove(_ element: T) -> T? {
buffer.remove(element)
}
package func contains(_ element: T) -> Bool {
buffer.contains(element)
}
}
@@ -0,0 +1,59 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerizationExtras
import Crypto
import Foundation
import NIOCore
/// Protocol for defining a single OCI content
public protocol Content: Sendable {
/// URL to the content
var path: URL { get }
/// sha256 of content
func digest() throws -> SHA256.Digest
/// Size of content
func size() throws -> UInt64
/// Data representation of entire content
func data() throws -> Data
/// Data representation partial content
func data(offset: UInt64, length: Int) throws -> Data?
/// Decode the content into an object
func decode<T>() throws -> T where T: Decodable
}
/// Protocol defining methods to fetch and push OCI content
public protocol ContentClient: Sendable {
func fetch<T: Codable>(name: String, descriptor: Descriptor) async throws -> T
func fetchBlob(name: String, descriptor: Descriptor, into file: URL, progress: ProgressHandler?) async throws -> (Int64, SHA256Digest)
func fetchData(name: String, descriptor: Descriptor) async throws -> Data
func push<T: Sendable & AsyncSequence>(
name: String,
ref: String,
descriptor: Descriptor,
streamGenerator: () throws -> T,
progress: ProgressHandler?
) async throws where T.Element == ByteBuffer
}
@@ -0,0 +1,67 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import Crypto
import Foundation
/// Protocol for defining a content store where OCI image metadata and layers will be managed
/// and manipulated.
public protocol ContentStore: Sendable {
/// Retrieves a piece of Content based on the digest string.
/// Returns `nil` if the requested digest is not found.
func get(digest: String) async throws -> Content?
/// Retrieves a specific content metadata type based on the digest string.
/// Returns `nil` if the requested digest is not found.
func get<T: Decodable>(digest: String) async throws -> T?
/// Remove a list of digests in the content store.
@discardableResult
func delete(digests: [String]) async throws -> ([String], UInt64)
/// Removes all content from the store except for the digests in the provided list.
@discardableResult
func delete(keeping: [String]) async throws -> ([String], UInt64)
/// Creates a transactional write to the content store.
/// The function takes a closure given a temporary `URL` of the base directory which all contents should be written to.
/// This is transaction write where any failed operation in the closure (caught exception) will result in all contents written
/// in the closure to be deleted.
///
/// If the closure succeeds, then all the content that have been written to the temporary `URL` will be moved into the actual
/// blobs path of the content store.
@discardableResult
func ingest(_ body: @Sendable @escaping (URL) async throws -> Void) async throws -> [String]
/// Creates a new ingest session and returns the session ID and temporary ingest directory corresponding to the session.
/// The contents from the ingest directory are processed and moved into the content store once the session is marked complete.
/// This can be done by invoking the `completeIngestSession` method with the returned session ID.
func newIngestSession() async throws -> (id: String, ingestDir: URL)
/// Completes a previously started ingest session corresponding to `id`.
/// The contents from the ingest directory from the session are moved into the content store atomically.
/// Any failure encountered will result in a transaction failure causing none of the contents to be ingested into the store.
@discardableResult
func completeIngestSession(_ id: String) async throws -> [String]
/// Cancels a previously started ingest session corresponding to `id`.
/// The contents from the ingest directory corresponding to the session are removed.
func cancelIngestSession(_ id: String) async throws
/// Total bytes allocated on disk for the content store, covering
/// committed blobs and any active ingest sessions.
func totalAllocatedSize() async throws -> UInt64
}
@@ -0,0 +1,137 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerizationError
import Crypto
import Foundation
import NIOCore
/// Provides a context to write data into a directory.
public class ContentWriter {
private let base: URL
private let encoder = JSONEncoder()
/// Create a new ContentWriter.
/// - Parameters:
/// - base: The URL to write content to. If this is not a directory a
/// ContainerizationError will be thrown with a code of .internalError.
public init(for base: URL) throws {
self.encoder.outputFormatting = [JSONEncoder.OutputFormatting.sortedKeys]
self.base = base
var isDirectory = ObjCBool(true)
let exists = FileManager.default.fileExists(atPath: base.path, isDirectory: &isDirectory)
guard exists && isDirectory.boolValue else {
throw ContainerizationError(.internalError, message: "cannot create ContentWriter for path \(base.absolutePath()), not a directory")
}
}
/// Writes the data blob to the base URL provided in the constructor.
/// - Parameters:
/// - data: The data blob to write to a file under the base path.
@discardableResult
public func write(_ data: Data) throws -> (size: Int64, digest: SHA256.Digest) {
let digest = SHA256.hash(data: data)
let destination = base.appendingPathComponent(digest.encoded)
try data.write(to: destination)
return (Int64(data.count), digest)
}
/// Reads the data present in the passed in URL and writes it to the base path.
/// - Parameters:
/// - url: The URL to read the data from.
@discardableResult
public func create(from url: URL) throws -> (size: Int64, digest: SHA256.Digest) {
let sourceFD = Foundation.open(url.path, O_RDONLY)
guard sourceFD >= 0 else {
let errCode = POSIXErrorCode(rawValue: errno) ?? .EINVAL
let err = POSIXError(errCode)
throw ContainerizationError(.internalError, message: "failed to open \(url.path) for reading", cause: err)
}
defer { close(sourceFD) }
let tempURL = base.appendingPathComponent(UUID().uuidString)
let destFD = Foundation.open(tempURL.path, O_WRONLY | O_CREAT | O_TRUNC, 0o644)
guard destFD >= 0 else {
let errCode = POSIXErrorCode(rawValue: errno) ?? .EINVAL
let err = POSIXError(errCode)
throw ContainerizationError(.internalError, message: "failed to create temporary file at \(tempURL.absolutePath())", cause: err)
}
let chunkSize = 1024 * 1024 // 1 MiB
let buf = UnsafeMutableRawBufferPointer.allocate(byteCount: chunkSize, alignment: 1)
defer { buf.deallocate() }
guard let baseAddress = buf.baseAddress else {
close(destFD)
try? FileManager.default.removeItem(at: tempURL)
throw ContainerizationError(.internalError, message: "failed to allocate read buffer of size \(chunkSize)")
}
var hasher = SHA256()
var totalSize: Int64 = 0
while true {
let n = read(sourceFD, baseAddress, chunkSize)
if n == 0 { break }
if n < 0 {
close(destFD)
let errCode = POSIXErrorCode(rawValue: errno) ?? .EINVAL
let err = POSIXError(errCode)
try? FileManager.default.removeItem(at: tempURL)
throw ContainerizationError(.internalError, message: "failed to read from \(url.path)", cause: err)
}
hasher.update(data: UnsafeRawBufferPointer(start: baseAddress, count: n))
var written = 0
while written < n {
let w = Foundation.write(destFD, baseAddress.advanced(by: written), n - written)
if w < 0 {
close(destFD)
let errCode = POSIXErrorCode(rawValue: errno) ?? .EINVAL
let err = POSIXError(errCode)
try? FileManager.default.removeItem(at: tempURL)
throw ContainerizationError(.internalError, message: "failed to write to \(tempURL.absolutePath())", cause: err)
}
written += w
}
totalSize += Int64(n)
}
close(destFD)
let digest = hasher.finalize()
let destination = base.appendingPathComponent(digest.encoded)
do {
try FileManager.default.moveItem(at: tempURL, to: destination)
} catch let error as NSError {
guard error.code == NSFileWriteFileExistsError else {
throw error
}
try? FileManager.default.removeItem(at: tempURL)
} catch {
try? FileManager.default.removeItem(at: tempURL)
throw error
}
return (totalSize, digest)
}
/// Encodes the passed in type as a JSON blob and writes it to the base path.
/// - Parameters:
/// - content: The type to convert to JSON.
@discardableResult
public func create<T: Encodable>(from content: T) throws -> (size: Int64, digest: SHA256.Digest) {
let data = try self.encoder.encode(content)
return try self.write(data)
}
}
@@ -0,0 +1,78 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import ContainerizationError
import Crypto
import Foundation
public final class LocalContent: Content {
public let path: URL
private let file: FileHandle
public init(path: URL) throws {
guard FileManager.default.fileExists(atPath: path.path) else {
throw ContainerizationError(.notFound, message: "content at path \(path.absolutePath())")
}
self.file = try FileHandle(forReadingFrom: path)
self.path = path
}
public func digest() throws -> SHA256.Digest {
let bufferSize = 64 * 1024 // 64 KB
var hasher = SHA256()
try self.file.seek(toOffset: 0)
while case let data = file.readData(ofLength: bufferSize), !data.isEmpty {
hasher.update(data: data)
}
let digest = hasher.finalize()
try self.file.seek(toOffset: 0)
return digest
}
public func data(offset: UInt64 = 0, length size: Int = 0) throws -> Data? {
try file.seek(toOffset: offset)
if size == 0 {
return try file.readToEnd()
}
return try file.read(upToCount: size)
}
public func data() throws -> Data {
try Data(contentsOf: self.path)
}
public func size() throws -> UInt64 {
let fileAttrs = try FileManager.default.attributesOfItem(atPath: self.path.absolutePath())
if let size = fileAttrs[FileAttributeKey.size] as? UInt64 {
return size
}
throw ContainerizationError(.internalError, message: "could not determine file size for \(path.absolutePath())")
}
public func decode<T>() throws -> T where T: Decodable {
let json = JSONDecoder()
let data = try Data(contentsOf: self.path)
return try json.decode(T.self, from: data)
}
deinit {
try? self.file.close()
}
}
@@ -0,0 +1,230 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
// swiftlint:disable unused_optional_binding
import ContainerizationError
import ContainerizationExtras
import Crypto
import Foundation
/// A `ContentStore` implementation that stores content on the local filesystem.
public actor LocalContentStore: ContentStore {
private static let encoder = JSONEncoder()
private let _basePath: URL
private let _ingestPath: URL
private let _blobPath: URL
private let _lock: AsyncLock
private var activeIngestSessions: AsyncSet<String> = AsyncSet([])
/// Create a new `LocalContentStore`.
///
/// - Parameters:
/// - path: The path where content should be written under.
public init(path: URL) throws {
let ingestPath = path.appendingPathComponent("ingest")
let blobPath = path.appendingPathComponent("blobs/sha256")
let fileManager = FileManager.default
try fileManager.createDirectory(at: ingestPath, withIntermediateDirectories: true)
try fileManager.createDirectory(at: blobPath, withIntermediateDirectories: true)
self._basePath = path
self._ingestPath = ingestPath
self._blobPath = blobPath
self._lock = AsyncLock()
Self.encoder.outputFormatting = .sortedKeys
}
/// Get a piece of content from the store. Returns nil if not
/// found.
///
/// - Parameters:
/// - digest: The string digest of the content.
public func get(digest: String) throws -> Content? {
let d = digest.trimmingDigestPrefix
let path = self._blobPath.appendingPathComponent(d)
do {
return try LocalContent(path: path)
} catch let err as ContainerizationError {
switch err.code {
case .notFound:
return nil
default:
throw err
}
}
}
/// Get a piece of content from the store and return the decoded version of
/// it.
///
/// - Parameters:
/// - digest: The string digest of the content.
public func get<T: Decodable & Sendable>(digest: String) throws -> T? {
guard let content: Content = try self.get(digest: digest) else {
return nil
}
return try content.decode()
}
/// Delete all content besides a set provided.
///
/// - Parameters:
/// - keeping: The set of string digests to keep.
public func delete(keeping: [String]) async throws -> ([String], UInt64) {
let fileManager = FileManager.default
let all = try fileManager.contentsOfDirectory(at: self._blobPath, includingPropertiesForKeys: nil)
let allDigests = Set(all.map { $0.lastPathComponent })
let toDelete = allDigests.subtracting(keeping)
return try await self.delete(digests: Array(toDelete))
}
/// Delete a specific set of content.
///
/// - Parameters:
/// - digests: Array of strings denoting the digests of the content to delete.
@discardableResult
public func delete(digests: [String]) async throws -> ([String], UInt64) {
let store = AsyncStore<([String], UInt64)>()
try await self._lock.withLock { context in
let fileManager = FileManager.default
var deleted: [String] = []
var deletedBytes: UInt64 = 0
for toDelete in digests {
let p = self._blobPath.appendingPathComponent(toDelete)
guard let content = try? LocalContent(path: p) else {
continue
}
deletedBytes += try content.size()
try fileManager.removeItem(at: p)
deleted.append(toDelete)
}
await store.set((deleted, deletedBytes))
}
return await store.get() ?? ([], 0)
}
/// Creates a transactional write to the content store.
///
/// - Parameters:
/// - body: Closure that is given a temporary `URL` of the base directory which all contents should be written to.
/// This is a transaction write where any failed operation in the closure (caught exception) will result in all contents written
/// in the closure to be deleted. If the closure succeeds, then all the content that have been written to the temporary `URL`
/// will be moved into the actual blobs path of the content store.
@discardableResult
public func ingest(_ body: @Sendable @escaping (URL) async throws -> Void) async throws -> [String] {
let (id, tempPath) = try await self.newIngestSession()
try await body(tempPath)
return try await self.completeIngestSession(id)
}
/// Creates a new ingest session and returns the session ID and temporary ingest directory corresponding to the session.
/// The contents from the ingest directory are processed and moved into the content store once the session is marked complete.
/// This can be done by invoking the `completeIngestSession` method with the returned session ID.
public func newIngestSession() async throws -> (id: String, ingestDir: URL) {
let id = UUID().uuidString
let temporaryPath = self._ingestPath.appendingPathComponent(id)
let fileManager = FileManager.default
try fileManager.createDirectory(atPath: temporaryPath.path, withIntermediateDirectories: true)
await self.activeIngestSessions.insert(id)
return (id, temporaryPath)
}
/// Completes a previously started ingest session corresponding to `id`. The contents from the ingest
/// directory from the session are moved into the content store atomically. Any failure encountered will
/// result in a transaction failure causing none of the contents to be ingested into the store.
/// - Parameters:
/// - id: id of the ingest session to complete.
@discardableResult
public func completeIngestSession(_ id: String) async throws -> [String] {
guard await activeIngestSessions.contains(id) else {
throw ContainerizationError(.internalError, message: "invalid session id \(id)")
}
await activeIngestSessions.remove(id)
let temporaryPath = self._ingestPath.appendingPathComponent(id)
let fileManager = FileManager.default
defer {
try? fileManager.removeItem(at: temporaryPath)
}
let tempDigests: [URL] = try fileManager.contentsOfDirectory(at: temporaryPath, includingPropertiesForKeys: nil)
return try await self._lock.withLock { context in
var moved: [String] = []
let fileManager = FileManager.default
do {
try tempDigests.forEach {
let digest = $0.lastPathComponent
let target = self._blobPath.appendingPathComponent(digest)
// only ingest if not exists
if !fileManager.fileExists(atPath: target.path) {
try fileManager.moveItem(at: $0, to: target)
moved.append(digest)
}
}
} catch {
moved.forEach {
try? fileManager.removeItem(at: self._blobPath.appendingPathComponent($0))
}
throw error
}
return tempDigests.map { $0.lastPathComponent }
}
}
/// Cancels a previously started ingest session corresponding to `id`.
/// The contents from the ingest directory corresponding to the session are removed.
/// - Parameters:
/// - id: id of the ingest session to complete.
public func cancelIngestSession(_ id: String) async throws {
guard let _ = await self.activeIngestSessions.remove(id) else {
return
}
let temporaryPath = self._ingestPath.appendingPathComponent(id)
let fileManager = FileManager.default
try? fileManager.removeItem(at: temporaryPath)
}
/// Total bytes allocated on disk for the content store, covering
/// committed blobs and any active ingest sessions.
public func totalAllocatedSize() throws -> UInt64 {
let fileManager = FileManager.default
guard
let enumerator = fileManager.enumerator(
at: self._basePath,
includingPropertiesForKeys: [.totalFileAllocatedSizeKey, .isRegularFileKey],
options: [.skipsHiddenFiles]
)
else {
throw ContainerizationError(.internalError, message: "failed to enumerate content store at \(self._basePath.path)")
}
var size: UInt64 = 0
for case let fileURL as URL in enumerator {
guard let values = try? fileURL.resourceValues(forKeys: [.totalFileAllocatedSizeKey, .isRegularFileKey]), values.isRegularFile == true,
let fileSize = values.totalFileAllocatedSize
else {
// Skip directories and other non-regular entries. On Linux,
// `.totalFileAllocatedSizeKey` reports block allocation for
// directories, which would otherwise count empty-store
// inode overhead as content.
continue
}
size += UInt64(fileSize)
}
return size
}
}
@@ -0,0 +1,32 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import Crypto
import Foundation
extension SHA256.Digest {
/// Returns the digest as a string.
public var digestString: String {
let parts = self.description.split(separator: ": ")
return "sha256:\(parts[1])"
}
/// Returns the digest without a 'sha256:' prefix.
public var encoded: String {
let parts = self.description.split(separator: ": ")
return String(parts[1])
}
}
@@ -0,0 +1,26 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
extension String {
/// Removes any prefix (sha256:) from a digest string.
public var trimmingDigestPrefix: String {
let split = self.split(separator: ":")
if split.count == 2 {
return String(split[1])
}
return self
}
}
@@ -0,0 +1,36 @@
//===----------------------------------------------------------------------===//
// Copyright © 2025-2026 Apple Inc. and the Containerization project authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//===----------------------------------------------------------------------===//
import Foundation
extension URL {
/// Returns the unescaped absolutePath of a URL joined by separator.
public func absolutePath() -> String {
#if os(macOS)
return self.path(percentEncoded: false)
#else
return self.path
#endif
}
/// Returns the domain name of a registry.
public var domain: String? {
guard let host = self.absoluteString.split(separator: ":").first else {
return nil
}
return String(host)
}
}