static CacheEntry GetAssetCacheEntry(IBuildCache cache, GUID asset, bool NonRecursiveDependencies)
{
CacheEntry entry;
entry = cache.GetCacheEntry(asset, NonRecursiveDependencies ? -kVersion : kVersion);
return(entry);
}
static CachedInfo GetCachedInfo(IBuildCache cache, CacheEntry entry, IEnumerable <ResourceFile> resources, BundleDetails details)
{
var info = new CachedInfo();
info.Asset = entry;
info.Dependencies = new CacheEntry[0];
info.Data = new object[] { details };
return(info);
}
public void RunCommand()
{
if (_command == null)
{
return;
}
if (_command is CommonParametersBaseCommand commonParams)
{
if (commonParams.Verbose.Value)
{
_verbose = true;
}
if (commonParams.NoBuildCache.Value)
{
_buildCache = new DummyBuildCache();
}
else
{
_buildCache = new PersistentBuildCache(_tools.Path);
}
}
if (_command is BuildInteractiveCommand)
{
RunInteractive(_command as CommonInteractiveCommand);
}
else if (_command is BuildInteractiveNoUpdateCommand)
{
_forbiddenDependencyUpdate = true;
RunInteractive(_command as CommonInteractiveCommand);
}
else if (_command is BuildCommand bCommand)
{
RunBuild(bCommand);
}
else if (_command is TestCommand testCommand)
{
RunTest(testCommand);
}
else if (_command is TranslationCommand tCommand)
{
RunTranslation(tCommand);
}
else if (_command is PackageManagerInstallCommand installCommand)
{
RunPackageInstall(installCommand);
}
else if (_command is PackageManagerUpgradeCommand upgradeCommand)
{
RunPackageUpgrade(upgradeCommand);
}
else if (_command is PackageManagerAddCommand addCommand)
{
RunPackageAdd(addCommand);
}
}
/// <summary>
/// Creates a cached builder
/// </summary>
/// <param name="wrappedBuilder">The builder instance to be wrapped</param>
/// <param name="cache">The cache implementation to be used</param>
/// <param name="targetDir">The target directory's file system abstraction</param>
public CachedBuilder(IBuilder wrappedBuilder, IBuildCache cache, [TargetRoot] IFileSystemDirectory targetDir)
{
Contract.Requires(wrappedBuilder != null);
Contract.Requires(cache != null);
Contract.Requires(targetDir != null);
this.wrappedBuilder = wrappedBuilder;
this.cache = cache;
this.targetDir = targetDir;
}
/// <summary>
/// Creates a cached builder
/// </summary>
/// <param name="wrappedBuilder">The builder instance to be wrapped</param>
/// <param name="cache">The cache implementation to be used</param>
/// <param name="targetDir">The target directory's file system abstraction</param>
public CachedBuilder(IBuilder wrappedBuilder, IBuildCache cache, [TargetRoot] IFileSystemDirectory targetDir)
{
Contract.Requires(wrappedBuilder != null);
Contract.Requires(cache != null);
Contract.Requires(targetDir != null);
this.wrappedBuilder = wrappedBuilder;
this.cache = cache;
this.targetDir = targetDir;
}
public static PipelineBuilder Start(string name, IDataCache storage, IBuildCache cache, IAssetPackager packager)
{
return(new PipelineBuilder {
_packager = packager,
_storage = storage,
_importers = new List <IAssetImporter>(),
_processors = new Dictionary <Type, IAssetProcessor>(),
_cache = cache
});
}
internal void SetupTaskContext()
{
m_GlobalUsage = m_DependencyData.GlobalUsage;
foreach (var sceneInfo in m_DependencyData.SceneInfo)
{
m_GlobalUsage |= sceneInfo.Value.globalUsage;
}
m_BuildSettings = m_Parameters.GetContentBuildSettings();
m_UseCache = m_Parameters.UseCache ? m_Cache : null;
}
internal BuildPipeline(IDataCache storage, IBuildCache cache,
IAssetPackager packager, List <IAssetImporter> importers,
Dictionary <Type, IAssetProcessor> processors)
{
_packager = packager;
_importers = importers;
_processors = processors;
_cache = cache;
_memoryManager = new AssetMemoryManager(storage);
Identity = CalculateId();
}
public static async Task <IEnumerable <IProxyAsset> > FetchOrCreate(
this IBuildCache cache,
string pipeline, AssetMemoryManager mem, AssetSource source,
Func <AssetSource, Task <IEnumerable <IProxyAsset> > > build)
{
var cacheKey = BuildCache.ComputeCacheKey(pipeline, source);
while (true)
{
var cached = await cache.Fetch(cacheKey);
if (cached.Type == CacheResult.Cached)
{
var cachedAssets = await Task.WhenAll(cached.Assets.Select(mem.DeserializeProxy));
if (cachedAssets.All(a => a != null))
{
return(cachedAssets);
}
}
if (cached.Type == CacheResult.IncompleteKey)
{
cacheKey = BuildCache.ComputeCacheKey(pipeline, source, cached.MissingInputs);
}
else
{
break;
}
}
var assets = (await build(source)).ToList();
var extraFiles = new string[0]; //TODO: record extra input files
var serializedProxies = assets.Select(mem.SerializeProxy).ToList();
if (extraFiles.Length == 0)
{
await cache.Store(cacheKey, CacheRecord.Found(serializedProxies));
}
else
{
await cache.Store(cacheKey, CacheRecord.Incomplete(extraFiles));
await cache.Store(BuildCache.ComputeCacheKey(pipeline, source, extraFiles),
CacheRecord.Found(serializedProxies));
}
return(assets);
}
static CachedInfo GetCachedInfo(IBuildCache cache, GUID asset, AssetLoadInfo assetInfo, BuildUsageTagSet usageTags, SpriteImporterData importerData, ExtendedAssetData assetData)
{
var info = new CachedInfo();
info.Asset = cache.GetCacheEntry(asset, kVersion);
var uniqueTypes = new HashSet <System.Type>();
var objectTypes = new List <KeyValuePair <ObjectIdentifier, System.Type[]> >();
var dependencies = new HashSet <CacheEntry>();
ExtensionMethods.ExtractCommonCacheData(cache, assetInfo.includedObjects, assetInfo.referencedObjects, uniqueTypes, objectTypes, dependencies);
info.Dependencies = dependencies.ToArray();
info.Data = new object[] { assetInfo, usageTags, importerData, assetData, objectTypes };
return(info);
}
/// <summary>
/// Creates a cached builder
/// </summary>
/// <param name="wrappedBuilder">The builder instance to be wrapped</param>
/// <param name="cache">The cache implementation to be used</param>
/// <param name="targetDir">The target directory's file system abstraction</param>
public CachedBuilder(IBuilder wrappedBuilder, IBuildCache cache, [TargetRoot] IFileSystemDirectory targetDir)
{
Contract.Requires(wrappedBuilder != null);
Contract.Requires(cache != null);
Contract.Requires(targetDir != null);
this.wrappedBuilder = wrappedBuilder;
this.cache = cache;
this.targetDir = targetDir;
supportsFallback = wrappedBuilder.BuilderType.GetCustomAttributes(typeof(FallbackToCacheAttribute), false).Any();
var agressiveAttribute = wrappedBuilder.BuilderType.GetCustomAttributes(typeof (AggressiveCacheRestoreAttribute), false).OfType<AggressiveCacheRestoreAttribute>().FirstOrDefault();
aggressive = agressiveAttribute != null;
agressiveModeExceptions = agressiveAttribute != null
? agressiveAttribute.ExceptionExpressions
: new Regex[0];
}
static CachedInfo GetCachedInfo(IBuildCache cache, GUID asset, AssetLoadInfo assetInfo, BuildUsageTagSet usageTags, SpriteImporterData importerData, ExtendedAssetData assetData)
{
var info = new CachedInfo();
info.Asset = cache.GetCacheEntry(asset, kVersion);
var dependencies = new HashSet <CacheEntry>();
foreach (var reference in assetInfo.referencedObjects)
{
dependencies.Add(cache.GetCacheEntry(reference));
}
info.Dependencies = dependencies.ToArray();
info.Data = new object[] { assetInfo, usageTags, importerData, assetData };
return(info);
}
static CachedInfo GetCachedInfo(IBuildCache cache, CacheEntry entry, IEnumerable <ResourceFile> resources, BundleDetails details)
{
var info = new CachedInfo();
info.Asset = entry;
var dependencies = new HashSet <CacheEntry>();
foreach (var resource in resources)
{
dependencies.Add(cache.GetCacheEntry(resource.fileName));
}
info.Dependencies = dependencies.ToArray();
info.Data = new object[] { details };
return(info);
}
/// <summary>
/// Creates a cached builder
/// </summary>
/// <param name="wrappedBuilder">The builder instance to be wrapped</param>
/// <param name="cache">The cache implementation to be used</param>
/// <param name="targetDir">The target directory's file system abstraction</param>
public CachedBuilder(IBuilder wrappedBuilder, IBuildCache cache, [TargetRoot] IFileSystemDirectory targetDir)
{
Contract.Requires(wrappedBuilder != null);
Contract.Requires(cache != null);
Contract.Requires(targetDir != null);
this.wrappedBuilder = wrappedBuilder;
this.cache = cache;
this.targetDir = targetDir;
supportsFallback = wrappedBuilder.BuilderType.GetCustomAttributes(typeof(FallbackToCacheAttribute), false).Any();
var agressiveAttribute = wrappedBuilder.BuilderType.GetCustomAttributes(typeof(AggressiveCacheRestoreAttribute), false).OfType <AggressiveCacheRestoreAttribute>().FirstOrDefault();
aggressive = agressiveAttribute != null;
agressiveModeExceptions = agressiveAttribute != null
? agressiveAttribute.ExceptionExpressions
: new Regex[0];
}
static CacheEntry GetCacheEntry(IBuildCache cache, string bundleName, IEnumerable <ResourceFile> resources, BuildCompression compression)
{
var entry = new CacheEntry();
entry.Type = CacheEntry.EntryType.Data;
entry.Guid = HashingMethods.Calculate("ArchiveAndCompressBundles", bundleName).ToGUID();
List <object> toHash = new List <object> {
kVersion, compression
};
foreach (var resource in resources)
{
toHash.Add(resource.serializedFile);
toHash.Add(resource.fileAlias);
toHash.Add(cache.GetCacheEntry(resource.fileName).Hash);
}
entry.Hash = HashingMethods.Calculate(toHash).ToHash128();
entry.Version = kVersion;
return(entry);
}
public void RunCommand()
{
if (_command == null)
{
return;
}
if (_command is CommonParametersBaseCommand commonParams)
{
if (commonParams.Verbose.Value)
{
_verbose = true;
}
if (commonParams.NoBuildCache.Value)
{
_buildCache = new DummyBuildCache();
}
else
{
_buildCache = new PersistentBuildCache(_tools.Path);
}
}
if (_command is BuildInteractiveCommand iCommand)
{
RunInteractive(iCommand.Port.Value, iCommand.Sprite.Value);
}
else if (_command is BuildInteractiveNoUpdateCommand yCommand)
{
_forbiddenDependencyUpdate = true;
RunInteractive(yCommand.Port.Value, yCommand.Sprite.Value);
}
else if (_command is BuildCommand bCommand)
{
RunBuild(bCommand);
}
else if (_command is TestCommand testCommand)
{
RunTest(testCommand);
}
}
public static void ExtractCommonCacheData(IBuildCache cache, IEnumerable <ObjectIdentifier> includedObjects, IEnumerable <ObjectIdentifier> referencedObjects, HashSet <Type> uniqueTypes, List <ObjectTypes> objectTypes, HashSet <CacheEntry> dependencies)
{
if (includedObjects != null)
{
foreach (var objectId in includedObjects)
{
var types = BuildCacheUtility.GetSortedUniqueTypesForObject(objectId);
objectTypes.Add(new ObjectTypes(objectId, types));
uniqueTypes.UnionWith(types);
}
}
if (referencedObjects != null)
{
foreach (var objectId in referencedObjects)
{
var types = BuildCacheUtility.GetSortedUniqueTypesForObject(objectId);
objectTypes.Add(new ObjectTypes(objectId, types));
uniqueTypes.UnionWith(types);
dependencies.Add(cache.GetCacheEntry(objectId));
}
}
dependencies.UnionWith(uniqueTypes.Select(cache.GetCacheEntry));
}
public CacheServerDownloader(IBuildCache cache, string host, int port = 8126)
{
m_Cache = cache;
m_Client = new Client(host, port);
m_Client.Connect();
}
public static ReturnCode RunCachedOperation <T>(IBuildCache cache, IBuildLogger log, IProgressTracker tracker, List <WorkItem <T> > workItems,
IRunCachedCallbacks <T> cbs
)
{
using (log.ScopedStep(LogLevel.Info, "RunCachedOperation"))
{
List <CacheEntry> cacheEntries = null;
List <WorkItem <T> > nonCachedItems = workItems;
var cachedItems = new List <WorkItem <T> >();
for (int i = 0; i < workItems.Count; i++)
{
workItems[i].Index = i;
}
IList <CachedInfo> cachedInfo = null;
if (cache != null)
{
using (log.ScopedStep(LogLevel.Info, "Creating Cache Entries"))
for (int i = 0; i < workItems.Count; i++)
{
workItems[i].entry = cbs.CreateCacheEntry(workItems[i]);
}
cacheEntries = workItems.Select(i => i.entry).ToList();
using (log.ScopedStep(LogLevel.Info, "Load Cached Data"))
cache.LoadCachedData(cacheEntries, out cachedInfo);
cachedItems = workItems.Where(x => cachedInfo[x.Index] != null).ToList();
nonCachedItems = workItems.Where(x => cachedInfo[x.Index] == null).ToList();
}
using (log.ScopedStep(LogLevel.Info, "Process Entries"))
foreach (WorkItem <T> item in nonCachedItems)
{
if (!tracker.UpdateInfoUnchecked(item.StatusText))
{
return(ReturnCode.Canceled);
}
cbs.ProcessUncached(item);
}
using (log.ScopedStep(LogLevel.Info, "Process Cached Entries"))
foreach (WorkItem <T> item in cachedItems)
{
cbs.ProcessCached(item, cachedInfo[item.Index]);
}
foreach (WorkItem <T> item in workItems)
{
cbs.PostProcess(item);
}
if (cache != null)
{
List <CachedInfo> uncachedInfo;
using (log.ScopedStep(LogLevel.Info, "Saving to Cache"))
{
using (log.ScopedStep(LogLevel.Info, "Creating Cached Infos"))
uncachedInfo = nonCachedItems.Select((item) => cbs.CreateCachedInfo(item)).ToList();
cache.SaveCachedData(uncachedInfo);
}
}
log.AddEntrySafe(LogLevel.Info, $"Total Entries: {workItems.Count}, Processed: {nonCachedItems.Count}, Cached: {cachedItems.Count}");
return(ReturnCode.Success);
}
}
