#nullable enable using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.Collections.ObjectModel; using System.Diagnostics.CodeAnalysis; using System.IO; using System.Linq; using System.Threading.Tasks; using T3.Core.Compilation; using T3.Core.IO; using T3.Core.Logging; using T3.Core.Operator; using T3.Core.Resource; using T3.Core.Resource.Assets; using T3.Core.Stats; using T3.Core.Settings; // ReSharper disable RedundantNameQualifier namespace T3.Core.Model; /// /// The runtime-loaded Symbol information and base class of essentially what is a read only project. /// /// /// Regarding naming, we consider all t3 operator packages as packages for the sake of consistency with future nuget terminology etc. /// -- only the user's editable "packages" are referred to as projects /// /// Is used to create an EditorSymbolPackage. /// public abstract partial class SymbolPackage : IResourcePackage { public virtual ResourceFileWatcher? FileWatcher => null; public bool IsSharingResources => AssemblyInformation.ShouldShareResources; public readonly bool DoNotIncludedSharedPackages; public virtual bool IsReadOnly => true; public readonly AssemblyInformation AssemblyInformation; /** Primary directory of the package that contains the csproj, home and other other data */ public string Folder { get; } public virtual string DisplayName => AssemblyInformation.Name; /** For readonly packages we search for .t3 files in the Symbols folder */ protected virtual IEnumerable SymbolSearchFiles { get { var dir = Path.Combine(Folder, FileLocations.ReleaseSymbolsSubfolder); if (!Directory.Exists(dir)) return []; return Directory.EnumerateFiles(dir, $"*{SymbolExtension}", SearchOption.AllDirectories); } } protected event Action? SymbolAdded; protected event Action? SymbolUpdated; protected event Action? SymbolRemoved; private static ConcurrentBag _allPackages = []; public static IEnumerable AllPackages => _allPackages; public string AssetsFolder { get; private set; } = null!; public IReadOnlyCollection Dependencies => (ReadOnlyCollection)DependencyDict.Values; protected readonly ConcurrentDictionary DependencyDict = new(); public string RootNamespace => ReleaseInfo.RootNamespace; protected virtual ReleaseInfo ReleaseInfo { get { if (AssemblyInformation.TryGetReleaseInfo(out var releaseInfo)) return releaseInfo; throw new InvalidOperationException($"Failed to get release info for package {AssemblyInformation.Name}"); } } public string Name { get { // 1. If AssemblyInfo already has a name (Debug/Dev), use it. if (!string.IsNullOrWhiteSpace(AssemblyInformation.Name)) return AssemblyInformation.Name; // 2. Fallback to ReleaseInfo's RootNamespace (Release build). // Accessing ReleaseInfo triggers TryGetReleaseInfo() internally. try { return ReleaseInfo.RootNamespace; } catch (Exception) { // 3. Last resort fallback return AssemblyInformation.Name ?? string.Empty; } } } public Guid Id => AssemblyInformation.Id != Guid.Empty ? AssemblyInformation.Id : ReleaseInfo.PackageId; // Use a dedicated PackageId field static SymbolPackage() { // OpUpdateCounter registers itself in its constructor. _ = new OpUpdateCounter(); RegisterTypes(); } protected SymbolPackage(AssemblyInformation assembly, string? mainDirectory, bool initializeResources) { AssemblyInformation = assembly; Folder = mainDirectory ?? assembly.Directory; lock (_allPackages) _allPackages.Add(this); DoNotIncludedSharedPackages = assembly.Name == FileLocations.LibPackageName; // This can be delayed for EditableSymbolPackages and the called directly from there if (initializeResources) { // ReSharper disable once VirtualMemberCallInConstructor InitializeAssets(); } } protected virtual void InitializeAssets() { AssetsFolder = Path.Combine(Folder, FileLocations.AssetsSubfolder); // Force the assembly information to load the JSON metadata now // This ensures Name and Id are valid before registration starts. _ = ReleaseInfo; // Avoid creating resource folder in protected program folder if (!IsReadOnly) { Directory.CreateDirectory(AssetsFolder); } ResourcePackageManager.AddSharedResourceFolder(this, AssemblyInformation.ShouldShareResources); AssetRegistry.RegisterAssetsFromPackage(this); } public virtual void Dispose() { AssetRegistry.UnregisterPackage(Id); ResourcePackageManager.RemoveSharedResourceFolder(this); ClearSymbols(); var currentPackages = _allPackages.ToList(); currentPackages.Remove(this); lock (_allPackages) _allPackages = new ConcurrentBag(currentPackages); AssemblyInformation.Unload(); // Todo - symbol instance destruction...? return; void ClearSymbols() { if (SymbolDict.Count == 0) return; var symbols = SymbolDict.Values.ToArray(); foreach (var symbol in symbols) { var id = symbol.Id; SymbolDict.Remove(id, out _); } } } /// /// Loads symbols from the assembly and locates their symbol .t3/json files /// /// if true, parallel loading is used /// Newly read json files for new types /// All new symbols, including those for which a json file was not found public void LoadSymbols(bool parallel, out List newlyRead, out List allNewSymbols) { Log.Debug($" Loading {AssemblyInformation.Name}..."); if (!AssemblyInformation.TryLoadTypes()) { var error = $"Failed to load types for {AssemblyInformation.Name}"; Log.Error(error); newlyRead = []; allNewSymbols = []; return; } IDictionary newTypes; // Computed by set difference up front: LoadTypes may run in parallel, and removing from a // shared HashSet inside it raced — lost removals made live symbols appear deleted. var removedSymbolIds = new HashSet(); foreach (var id in SymbolDict.Keys) { if (!AssemblyInformation.OperatorTypeInfo.ContainsKey(id)) removedSymbolIds.Add(id); } ConcurrentBag updatedSymbols = new(); if (parallel) { newTypes = new ConcurrentDictionary(); AssemblyInformation.OperatorTypeInfo .AsParallel() .ForAll(kvp => LoadTypes(kvp.Key, kvp.Value.Type, newTypes, updatedSymbols)); } else { newTypes = new Dictionary(); foreach (var (guid, type) in AssemblyInformation.OperatorTypeInfo) { LoadTypes(guid, type.Type, newTypes, updatedSymbols); } } // gather slot changes foreach (var symbol in updatedSymbols) { symbol.UpdateInstanceType(true); } // update symbol instances foreach (var symbol in updatedSymbols) { UpdateSymbolInstances(symbol); SymbolUpdated?.Invoke(symbol); } if (SymbolRemoved != null) { // remaining symbols have been removed from the assembly foreach (var symbolId in removedSymbolIds) { SymbolRemoved.Invoke(symbolId); } } newlyRead = []; allNewSymbols = []; if (newTypes.Count != 0) { _corruptedSymbolFiles.Clear(); var searchFileEnumerator = parallel ? SymbolSearchFiles.AsParallel() : SymbolSearchFiles; var symbolsRead = searchFileEnumerator .Select(TryReadSymbolFile) .Where(result => result != null) .Select(result => { var file = result!; if (!newTypes.TryGetValue(file.Guid, out var type)) return default; return ReadSymbolFromJsonFileResult(file, type); }) .Where(symbolReadResult => symbolReadResult.Result.Symbol is not null) .ToArray(); if (CoreSettings.Config.LogCompilationDetails) Log.Debug($"{AssemblyInformation.Name}: Registering loaded symbols..."); foreach (var readSymbolResult in symbolsRead) { var symbol = readSymbolResult.Result.Symbol; var id = symbol!.Id; if (!SymbolDict.TryAdd(id, symbol)) { Log.Error($"Can't load symbol for [{symbol.Name}]. Registry already contains id {symbol.Id}: [{SymbolDict[symbol.Id].Name}]"); continue; } newlyRead.Add(readSymbolResult.Result); newTypes.Remove(id, out _); allNewSymbols.Add(symbol); SymbolAdded?.Invoke(readSymbolResult.Path, symbol); } } // these do not have a symbol json file but are defined in the assembly foreach (var (guid, newType) in newTypes) { var symbol = CreateSymbol(newType, guid); var id = symbol.Id; if (!SymbolDict.TryAdd(id, symbol)) { Log.Error($"{AssemblyInformation.Name}: Ignoring redefinition symbol {symbol.Name}."); continue; } //Log.Debug($"{AssemblyInformation.Name}: new added symbol: {newType}"); allNewSymbols.Add(symbol); SymbolAdded?.Invoke(null, symbol); } OnSymbolsLoaded(); return; void LoadTypes(Guid guid, Type type, IDictionary newTypesDict, ConcurrentBag updated) { if (SymbolDict.TryGetValue(guid, out var symbol)) { if (symbol == null) // this should never happen?? { Log.Error($"Skipping update of invalid symbol {guid}. Symbol entry was null - this is a bug."); return; } // we already have this symbol, so mark it as updated (but dont actually update it yet) symbol.UpdateTypeWithoutUpdatingDefinitionsOrInstances(type, this); updated.Add(symbol); } else { // it's a new type!! if (!newTypesDict.TryAdd(guid, type)) { Log.Error($"{DisplayName}: Failed to add new type {type} with guid '{guid}' - " + "is there another operator type with the same guid in this project?"); } } } SymbolJsonResult ReadSymbolFromJsonFileResult(JsonFileResult jsonInfo, Type type) { try { var result = SymbolJson.ReadSymbolRoot(jsonInfo.Guid, jsonInfo.JToken, type, this); jsonInfo.Object = result.Symbol; return new SymbolJsonResult(result, jsonInfo.FilePath); } catch (Exception e) { // Skip a structurally-broken symbol instead of aborting the whole load; recorded so the // package won't be saved over. The default result is dropped by the caller's Where. Log.Error($"Skipping corrupted symbol file '{jsonInfo.FilePath}': {e.Message}"); _corruptedSymbolFiles.Add(jsonInfo.FilePath); return default; } } JsonFileResult? TryReadSymbolFile(string filePath) { try { return JsonFileResult.ReadAndCreate(filePath); } catch (Exception e) { // A single corrupt/unreadable .t3 must not abort loading every project. Skip it — the // symbol goes missing (parents referencing it are protected by // Symbol.HasUnresolvedChildren) and the file is recorded for restore-from-backup. Log.Error($"Skipping corrupted symbol file '{filePath}': {e.Message}"); _corruptedSymbolFiles.Add(filePath); return null; } } } protected virtual void OnSymbolsLoaded() { } public static void UpdateSymbolInstances(Symbol symbol, bool forceTypeUpdate = false) { symbol.UpdateInstanceType(forceTypeUpdate); symbol.CreateAnimationUpdateActionsForSymbolInstances(); } protected internal Symbol CreateSymbol(Type instanceType, Guid id) { return new Symbol(instanceType, id, this); } public static void ApplySymbolChildren(List symbolsRead) { Parallel.ForEach(symbolsRead, result => TryReadAndApplyChildren(result)); } protected static bool TryReadAndApplyChildren(SymbolJson.SymbolReadResult result) { if (SymbolJson.TryReadAndApplySymbolChildren(result)) return true; var symbol = result.Symbol; if (symbol == null) { Log.Error($"Problem obtaining children of 'null' with {result.ChildrenJsonArray.Length} children"); return false; } Log.Error($"Problem obtaining children of {symbol.Name ?? "'null'"} ({symbol.Id})"); return false; } public readonly record struct SymbolJsonResult(in SymbolJson.SymbolReadResult Result, string Path); public IReadOnlyDictionary Symbols => SymbolDict; private readonly ConcurrentBag _corruptedSymbolFiles = new(); /// /// Paths of .t3 files that were corrupt/unreadable during the last load and were skipped. When /// non-empty the editor refuses to save this package, so a symbol re-created empty from its type /// can't overwrite a file a backup could still recover. /// public IReadOnlyCollection CorruptedSymbolFilePaths => _corruptedSymbolFiles; /// Cheap per-frame check (unlike .Count) for UI flags. public bool HasCorruptedSymbolFiles => !_corruptedSymbolFiles.IsEmpty; /// Records a corrupt/unreadable symbol or symbol-UI file so the package won't be saved over it. protected void RecordCorruptedSymbolFile(string filePath) => _corruptedSymbolFiles.Add(filePath); protected readonly ConcurrentDictionary SymbolDict = new(); public const string SymbolExtension = ".t3"; public bool ContainsSymbolName(ReadOnlySpan newSymbolName, ReadOnlySpan symbolNamespace) { foreach (var existing in SymbolDict.Values) { var existingNamespace = existing.Namespace.AsSpan(); var existingName = existing.Name.AsSpan(); if (newSymbolName.SequenceEqual(existingName) && symbolNamespace.SequenceEqual(existingNamespace)) return true; } return false; } public void AddResourceDependencyOn(FileResource resource) { if (!TryGetDependencyCounter(resource, out var dependencyCount)) return; dependencyCount.ResourceCount++; } public void RemoveResourceDependencyOn(FileResource fileResource) { if (!TryGetDependencyCounter(fileResource, out var dependency)) return; dependency.ResourceCount--; RemoveIfNoRemainingReferences(dependency); } public void AddDependencyOn(Symbol symbol) { if (symbol.SymbolPackage == this) return; if (!TryGetDependencyCounter(symbol, out var dependency)) return; if (dependency.SymbolChildCount++ == 0) { // this is the first reference to this package DependencyDict.TryAdd((SymbolPackage)dependency.Package, dependency); } } public void RemoveDependencyOn(Symbol symbol) { if (symbol.SymbolPackage == this) return; if (!TryGetDependencyCounter(symbol, out var dependency)) return; dependency.SymbolChildCount--; RemoveIfNoRemainingReferences(dependency); } private void RemoveIfNoRemainingReferences(DependencyCounter dependency) { if (dependency is { SymbolChildCount: 0, ResourceCount: 0 }) { DependencyDict.Remove((SymbolPackage)dependency.Package, out _); } } private bool TryGetDependencyCounter(IResource fileResource, [NotNullWhen(true)] out DependencyCounter? dependencyCounter) { var owningPackage = fileResource.OwningPackage; if (owningPackage is not SymbolPackage symbolPackage) { dependencyCounter = null; return false; } if (DependencyDict.TryGetValue(symbolPackage, out dependencyCounter)) return true; dependencyCounter = new DependencyCounter { Package = symbolPackage }; if (DependencyDict.TryAdd(symbolPackage, dependencyCounter)) { // this is the first reference to this package return true; } // another thread added it in the meantime?? return DependencyDict.TryGetValue(symbolPackage, out dependencyCounter); } public bool OwnsNamespace(string namespaceName) { return namespaceName == RootNamespace || namespaceName.StartsWith(RootNamespace) || AssemblyInformation.Namespaces.Contains(namespaceName) || AssemblyInformation.Namespaces.Any(x => namespaceName.StartsWith(x)); } } public sealed record DependencyCounter { public required IResourcePackage Package { get; init; } internal int SymbolChildCount { get; set; } internal int ResourceCount { get; set; } public override string ToString() { return $"{Package.DisplayName}: Symbol References: {SymbolChildCount}, Resource References: {ResourceCount}"; } }