/// <inheritdoc/> void IRunCachedCallbacks <Item> .ProcessUncached(WorkItem <Item> item) { IWriteOperation op = m_WriteData.WriteOperations[item.Index]; string targetDir = m_UseCache != null?m_UseCache.GetCachedArtifactsDirectory(item.entry) : m_Parameters.TempOutputFolder; Directory.CreateDirectory(targetDir); using (m_Log.ScopedStep(LogLevel.Info, $"Writing {op.GetType().Name}", op.Command.fileName)) { #if UNITY_2020_2_OR_NEWER || ENABLE_DETAILED_PROFILE_CAPTURING using (new ProfileCaptureScope(m_Log, ProfileCaptureOptions.None)) item.Context.Result = op.Write(targetDir, m_BuildSettings, m_GlobalUsage); #else item.Context.Result = op.Write(targetDir, m_BuildSettings, m_GlobalUsage); #endif } item.Context.MetaData = CalculateFileMetadata(ref item.Context.Result); if (ScriptableBuildPipeline.slimWriteResults) { SlimifySerializedObjects(ref item.Context.Result); } }
void IRunCachedCallbacks <Item> .ProcessUncached(WorkItem <Item> item) { IWriteOperation op = m_WriteData.WriteOperations[item.Index]; string targetDir = m_UseCache != null?m_UseCache.GetCachedArtifactsDirectory(item.entry) : m_Parameters.TempOutputFolder; Directory.CreateDirectory(targetDir); using (m_Log.ScopedStep(LogLevel.Info, $"Writing File {op.GetType().Name}")) item.Context.Result = op.Write(targetDir, m_BuildSettings, m_GlobalUsage); item.Context.MetaData = CalculateFileMetadata(ref item.Context.Result); if (ScriptableBuildPipeline.slimWriteResults) { SlimifySerializedObjects(ref item.Context.Result); } }
/// <inheritdoc /> public ReturnCode Run() { BuildUsageTagGlobal globalUsage = m_DependencyData.GlobalUsage; foreach (var sceneInfo in m_DependencyData.SceneInfo) { globalUsage |= sceneInfo.Value.globalUsage; } IList <CacheEntry> entries = m_WriteData.WriteOperations.Select(x => GetCacheEntry(x, m_Parameters.GetContentBuildSettings(), globalUsage)).ToList(); IList <CachedInfo> cachedInfo = null; IList <CachedInfo> uncachedInfo = null; if (m_Parameters.UseCache && m_Cache != null) { m_Cache.LoadCachedData(entries, out cachedInfo); uncachedInfo = new List <CachedInfo>(); } for (int i = 0; i < m_WriteData.WriteOperations.Count; i++) { IWriteOperation op = m_WriteData.WriteOperations[i]; WriteResult result; if (cachedInfo != null && cachedInfo[i] != null) { if (!m_Tracker.UpdateInfoUnchecked(string.Format("{0} (Cached)", op.Command.internalName))) { return(ReturnCode.Canceled); } result = (WriteResult)cachedInfo[i].Data[0]; } else { if (!m_Tracker.UpdateInfoUnchecked(op.Command.internalName)) { return(ReturnCode.Canceled); } var outputFolder = m_Parameters.TempOutputFolder; if (m_Parameters.UseCache && m_Cache != null) { outputFolder = m_Cache.GetCachedArtifactsDirectory(entries[i]); } Directory.CreateDirectory(outputFolder); result = op.Write(outputFolder, m_Parameters.GetContentBuildSettings(), globalUsage); if (uncachedInfo != null) { uncachedInfo.Add(GetCachedInfo(entries[i], op, result)); } } SetOutputInformation(op.Command.internalName, result); } if (m_Parameters.UseCache && m_Cache != null) { m_Cache.SaveCachedData(uncachedInfo); } return(ReturnCode.Success); }
/// <inheritdoc /> public ReturnCode Run() { if (m_Content.Scenes.IsNullOrEmpty()) return ReturnCode.SuccessNotRun; IList<CachedInfo> cachedInfo = null; IList<CachedInfo> uncachedInfo = null; if (m_Parameters.UseCache && m_Cache != null) { IList<CacheEntry> entries = m_Content.Scenes.Select(x => m_Cache.GetCacheEntry(x, Version)).ToList(); m_Cache.LoadCachedData(entries, out cachedInfo); uncachedInfo = new List<CachedInfo>(); } BuildSettings settings = m_Parameters.GetContentBuildSettings(); for (int i = 0; i < m_Content.Scenes.Count; i++) { GUID scene = m_Content.Scenes[i]; string scenePath = AssetDatabase.GUIDToAssetPath(scene.ToString()); SceneDependencyInfo sceneInfo; BuildUsageTagSet usageTags; Hash128 prefabDependency = new Hash128(); if (cachedInfo != null && cachedInfo[i] != null) { if (!m_Tracker.UpdateInfoUnchecked(string.Format("{0} (Cached)", scenePath))) return ReturnCode.Canceled; sceneInfo = (SceneDependencyInfo)cachedInfo[i].Data[0]; usageTags = cachedInfo[i].Data[1] as BuildUsageTagSet; prefabDependency = (Hash128)cachedInfo[i].Data[2]; } else { if (!m_Tracker.UpdateInfoUnchecked(scenePath)) return ReturnCode.Canceled; usageTags = new BuildUsageTagSet(); #if UNITY_2019_3_OR_NEWER sceneInfo = ContentBuildInterface.CalculatePlayerDependenciesForScene(scenePath, settings, usageTags, m_DependencyData.DependencyUsageCache); #else string outputFolder = m_Parameters.TempOutputFolder; if (m_Parameters.UseCache && m_Cache != null) outputFolder = m_Cache.GetCachedArtifactsDirectory(m_Cache.GetCacheEntry(scene, Version)); Directory.CreateDirectory(outputFolder); sceneInfo = ContentBuildInterface.PrepareScene(scenePath, settings, usageTags, m_DependencyData.DependencyUsageCache, outputFolder); #endif if (uncachedInfo != null) { // We only need to gather prefab dependencies and calculate the hash if we are using caching, otherwise we can skip it var prefabEntries = AssetDatabase.GetDependencies(AssetDatabase.GUIDToAssetPath(scene.ToString())).Where(path => path.EndsWith(".prefab")).Select(m_Cache.GetCacheEntry); prefabDependency = HashingMethods.Calculate(prefabEntries).ToHash128(); uncachedInfo.Add(GetCachedInfo(scene, sceneInfo.referencedObjects, sceneInfo, usageTags, prefabEntries, prefabDependency)); } } SetOutputInformation(scene, sceneInfo, usageTags, prefabDependency); } if (m_Parameters.UseCache && m_Cache != null) m_Cache.SaveCachedData(uncachedInfo); return ReturnCode.Success; }
public ReturnCode Run() { Dictionary <string, ulong> fileOffsets = new Dictionary <string, ulong>(); List <KeyValuePair <string, List <ResourceFile> > > bundleResources; { Dictionary <string, List <ResourceFile> > bundleToResources = new Dictionary <string, List <ResourceFile> >(); foreach (var pair in m_Results.WriteResults) { string bundle = m_WriteData.FileToBundle[pair.Key]; List <ResourceFile> resourceFiles; bundleToResources.GetOrAdd(bundle, out resourceFiles); resourceFiles.AddRange(pair.Value.resourceFiles); foreach (ResourceFile serializedFile in pair.Value.resourceFiles) { if (!serializedFile.serializedFile) { continue; } ObjectSerializedInfo firstObject = pair.Value.serializedObjects.First(x => x.header.fileName == serializedFile.fileAlias); fileOffsets[serializedFile.fileName] = firstObject.header.offset; } } bundleResources = bundleToResources.ToList(); } Dictionary <string, HashSet <string> > bundleDependencies = new Dictionary <string, HashSet <string> >(); foreach (var files in m_WriteData.AssetToFiles.Values) { if (files.IsNullOrEmpty()) { continue; } string bundle = m_WriteData.FileToBundle[files.First()]; HashSet <string> dependencies; bundleDependencies.GetOrAdd(bundle, out dependencies); dependencies.UnionWith(files.Select(x => m_WriteData.FileToBundle[x])); dependencies.Remove(bundle); } IList <CacheEntry> entries = bundleResources.Select(x => GetCacheEntry(x.Key, x.Value, m_Parameters.GetCompressionForIdentifier(x.Key))).ToList(); IList <CachedInfo> cachedInfo = null; IList <CachedInfo> uncachedInfo = null; if (m_Parameters.UseCache && m_Cache != null) { m_Cache.LoadCachedData(entries, out cachedInfo); uncachedInfo = new List <CachedInfo>(); } for (int i = 0; i < bundleResources.Count; i++) { string bundleName = bundleResources[i].Key; ResourceFile[] resourceFiles = bundleResources[i].Value.ToArray(); BuildCompression compression = m_Parameters.GetCompressionForIdentifier(bundleName); string writePath; BundleDetails details; if (cachedInfo != null && cachedInfo[i] != null) { if (!m_Tracker.UpdateInfoUnchecked(string.Format("{0} (Cached)", bundleName))) { return(ReturnCode.Canceled); } details = (BundleDetails)cachedInfo[i].Data[0]; details.FileName = string.Format("{0}/{1}", m_Parameters.OutputFolder, bundleName); HashSet <string> dependencies; if (bundleDependencies.TryGetValue(bundleName, out dependencies)) { details.Dependencies = dependencies.ToArray(); } else { details.Dependencies = new string[0]; } writePath = string.Format("{0}/{1}", m_Cache.GetCachedArtifactsDirectory(entries[i]), bundleName); } else { if (!m_Tracker.UpdateInfoUnchecked(bundleName)) { return(ReturnCode.Canceled); } details = new BundleDetails(); writePath = string.Format("{0}/{1}", m_Parameters.TempOutputFolder, bundleName); if (m_Parameters.UseCache && m_Cache != null) { writePath = string.Format("{0}/{1}", m_Cache.GetCachedArtifactsDirectory(entries[i]), bundleName); } Directory.CreateDirectory(Path.GetDirectoryName(writePath)); details.FileName = string.Format("{0}/{1}", m_Parameters.OutputFolder, bundleName); details.Crc = ContentBuildInterface.ArchiveAndCompress(resourceFiles, writePath, compression); details.Hash = CalculateHashVersion(fileOffsets, resourceFiles); HashSet <string> dependencies; if (bundleDependencies.TryGetValue(bundleName, out dependencies)) { details.Dependencies = dependencies.ToArray(); } else { details.Dependencies = new string[0]; } if (uncachedInfo != null) { uncachedInfo.Add(GetCachedInfo(m_Cache, entries[i], resourceFiles, details)); } } SetOutputInformation(writePath, details.FileName, bundleName, details); } if (m_Parameters.UseCache && m_Cache != null) { m_Cache.SaveCachedData(uncachedInfo); } return(ReturnCode.Success); }
// We don't return from this function until all downloads are processed. So it is safe to dispose immediately after. public void DownloadMissing(IList <CacheEntry> entries, IList <CachedInfo> cachedInfos) { Assert.AreEqual(entries.Count, cachedInfos.Count); Directory.CreateDirectory(k_CachePath); m_Semaphore = new Semaphore(0, entries.Count); m_Client.DownloadFinished += ThreadedDownloadFinished; // Queue up downloads for the missing or invalid local data for (var index = 0; index < entries.Count; index++) { // Only download data for cachedInfos that are invalid if (cachedInfos[index] != null) { continue; } var entry = entries[index]; string finalHash = HashingMethods.Calculate(entry.Hash, m_GlobalHash).ToHash128().ToString(); var fileId = FileId.From(entry.Guid.ToString(), finalHash); // Download artifacts before info to ensure both are available when download for info returns var downloadArtifact = new FileDownloadItem(fileId, FileType.Resource, GetCachedArtifactsFile(entry)); m_Client.QueueDownload(downloadArtifact); var downloadInfo = new FileDownloadItem(fileId, FileType.Info, GetCachedInfoFile(entry)); m_Client.QueueDownload(downloadInfo); } // Check downloads to see if it is usable data var formatter = new BinaryFormatter(); for (var index = 0; index < entries.Count; index++) { // find the next invalid cachedInfo while (index < entries.Count && cachedInfos[index] != null) { index++; } // make sure we didn't go out of bounds looking for invalid entries if (index >= entries.Count) { break; } // Wait for info download m_Semaphore.WaitOne(); string tempInfoFile = GetCachedInfoFile(entries[index]); if (!File.Exists(tempInfoFile)) { continue; } try { CachedInfo info; using (var fileStream = new FileStream(tempInfoFile, FileMode.Open, FileAccess.Read)) info = formatter.Deserialize(fileStream) as CachedInfo; if (m_Cache.HasAssetOrDependencyChanged(info)) { continue; } // Not every info file will have artifacts. So just check to see if we downloaded something. // TODO: May want to extend CachedInfo with Artifact knowledge if there is a performance benefit? string tempArtifactFile = GetCachedArtifactsFile(entries[index]); string tempArtifactDir = Path.ChangeExtension(tempArtifactFile, ""); if (File.Exists(tempArtifactFile) && !FileCompressor.Decompress(tempArtifactFile, tempArtifactDir)) { continue; } // All valid, move downloaded data into place cachedInfos[index] = info; string targetInfoFile = m_Cache.GetCachedInfoFile(info.Asset); if (File.Exists(targetInfoFile)) { File.Delete(targetInfoFile); } else { Directory.CreateDirectory(Path.GetDirectoryName(targetInfoFile)); } File.Move(tempInfoFile, targetInfoFile); if (Directory.Exists(tempArtifactDir)) { string targetArtifactDir = m_Cache.GetCachedArtifactsDirectory(info.Asset); if (Directory.Exists(targetArtifactDir)) { Directory.Delete(targetArtifactDir, true); } Directory.Move(tempArtifactDir, targetArtifactDir); } } catch (Exception e) { BuildLogger.LogException(e); } } m_Client.ResetDownloadFinishedEventHandler(); ((IDisposable)m_Semaphore).Dispose(); m_Semaphore = null; Directory.Delete(k_CachePath, true); }
/// <inheritdoc /> public ReturnCode Run() { if (m_Content.Scenes.IsNullOrEmpty()) { return(ReturnCode.SuccessNotRun); } IList <CachedInfo> cachedInfo = null; IList <CachedInfo> uncachedInfo = null; if (m_Parameters.UseCache && m_Cache != null) { IList <CacheEntry> entries = m_Content.Scenes.Select(x => m_Cache.GetCacheEntry(x, Version)).ToList(); m_Cache.LoadCachedData(entries, out cachedInfo); uncachedInfo = new List <CachedInfo>(); } BuildSettings settings = m_Parameters.GetContentBuildSettings(); for (int i = 0; i < m_Content.Scenes.Count; i++) { GUID scene = m_Content.Scenes[i]; string scenePath = AssetDatabase.GUIDToAssetPath(scene.ToString()); SceneDependencyInfo sceneInfo; BuildUsageTagSet usageTags; if (cachedInfo != null && cachedInfo[i] != null) { if (!m_Tracker.UpdateInfoUnchecked(string.Format("{0} (Cached)", scenePath))) { return(ReturnCode.Canceled); } sceneInfo = (SceneDependencyInfo)cachedInfo[i].Data[0]; usageTags = cachedInfo[i].Data[1] as BuildUsageTagSet; } else { if (!m_Tracker.UpdateInfoUnchecked(scenePath)) { return(ReturnCode.Canceled); } usageTags = new BuildUsageTagSet(); #if UNITY_2019_3_OR_NEWER sceneInfo = ContentBuildInterface.CalculatePlayerDependenciesForScene(scenePath, settings, usageTags, m_DependencyData.DependencyUsageCache); #else string outputFolder = m_Parameters.TempOutputFolder; if (m_Parameters.UseCache && m_Cache != null) { outputFolder = m_Cache.GetCachedArtifactsDirectory(m_Cache.GetCacheEntry(scene, Version)); } Directory.CreateDirectory(outputFolder); sceneInfo = ContentBuildInterface.PrepareScene(scenePath, settings, usageTags, m_DependencyData.DependencyUsageCache, outputFolder); #endif if (uncachedInfo != null) { uncachedInfo.Add(GetCachedInfo(scene, sceneInfo.referencedObjects, sceneInfo, usageTags)); } } SetOutputInformation(scene, sceneInfo, usageTags); } if (m_Parameters.UseCache && m_Cache != null) { m_Cache.SaveCachedData(uncachedInfo); } return(ReturnCode.Success); }
/// <inheritdoc /> public ReturnCode Run() { if (m_Content.Scenes.IsNullOrEmpty()) { return(ReturnCode.SuccessNotRun); } IList <CachedInfo> cachedInfo = null; IList <CachedInfo> uncachedInfo = null; if (m_Parameters.UseCache && m_Cache != null) { IList <CacheEntry> entries = m_Content.Scenes.Select(x => GetSceneCacheEntry(x)).ToList(); m_Cache.LoadCachedData(entries, out cachedInfo); uncachedInfo = new List <CachedInfo>(); } BuildSettings settings = m_Parameters.GetContentBuildSettings(); for (int i = 0; i < m_Content.Scenes.Count; i++) { GUID scene = m_Content.Scenes[i]; string scenePath = AssetDatabase.GUIDToAssetPath(scene.ToString()); SceneDependencyInfo sceneInfo; BuildUsageTagSet usageTags; Hash128 prefabDependency = new Hash128(); if (cachedInfo != null && cachedInfo[i] != null) { if (!m_Tracker.UpdateInfoUnchecked(string.Format("{0} (Cached)", scenePath))) { return(ReturnCode.Canceled); } sceneInfo = (SceneDependencyInfo)cachedInfo[i].Data[0]; usageTags = cachedInfo[i].Data[1] as BuildUsageTagSet; prefabDependency = (Hash128)cachedInfo[i].Data[2]; var objectTypes = cachedInfo[i].Data[3] as List <ObjectTypes>; if (objectTypes != null) { BuildCacheUtility.SetTypeForObjects(objectTypes); } } else { if (!m_Tracker.UpdateInfoUnchecked(scenePath)) { return(ReturnCode.Canceled); } usageTags = new BuildUsageTagSet(); #if UNITY_2019_3_OR_NEWER #if NONRECURSIVE_DEPENDENCY_DATA if (m_Parameters.NonRecursiveDependencies) { sceneInfo = ContentBuildInterface.CalculatePlayerDependenciesForScene(scenePath, settings, usageTags, m_DependencyData.DependencyUsageCache, DependencyType.ValidReferences); ObjectIdentifier[] filteredReferences = sceneInfo.referencedObjects.ToArray(); filteredReferences = ExtensionMethods.FilterReferencedObjectIDs(scene, filteredReferences, m_Parameters.Target, m_Parameters.ScriptInfo, new HashSet <GUID>(m_Content.Assets)); sceneInfo.SetReferencedObjects(filteredReferences); } else { sceneInfo = ContentBuildInterface.CalculatePlayerDependenciesForScene(scenePath, settings, usageTags, m_DependencyData.DependencyUsageCache); } #else sceneInfo = ContentBuildInterface.CalculatePlayerDependenciesForScene(scenePath, settings, usageTags, m_DependencyData.DependencyUsageCache); #endif #else string outputFolder = m_Parameters.TempOutputFolder; if (m_Parameters.UseCache && m_Cache != null) { outputFolder = m_Cache.GetCachedArtifactsDirectory(m_Cache.GetCacheEntry(scene, Version)); } Directory.CreateDirectory(outputFolder); sceneInfo = ContentBuildInterface.PrepareScene(scenePath, settings, usageTags, m_DependencyData.DependencyUsageCache, outputFolder); #endif if (uncachedInfo != null) { // We only need to gather prefab dependencies and calculate the hash if we are using caching, otherwise we can skip it var prefabEntries = AssetDatabase.GetDependencies(AssetDatabase.GUIDToAssetPath(scene.ToString())).Where(path => path.EndsWith(".prefab")).Select(m_Cache.GetCacheEntry); prefabDependency = HashingMethods.Calculate(prefabEntries).ToHash128(); uncachedInfo.Add(GetCachedInfo(scene, sceneInfo.referencedObjects, sceneInfo, usageTags, prefabEntries, prefabDependency)); } } SetOutputInformation(scene, sceneInfo, usageTags, prefabDependency); } if (m_Parameters.UseCache && m_Cache != null) { m_Cache.SaveCachedData(uncachedInfo); } return(ReturnCode.Success); }