public void ThreadingTasks_SaveQueue_RunsInParallelWith_UploadQueue() { SemaphoreSlim s1 = new SemaphoreSlim(0); SemaphoreSlim s2 = new SemaphoreSlim(0); SemaphoreSlim s3 = new SemaphoreSlim(0); ConcurrentQueue <int> callOrder = new ConcurrentQueue <int>(); ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.SaveQueue, (_) => { s1.Wait(); callOrder.Enqueue(1); s2.Release(); s3.Wait(); callOrder.Enqueue(2); }, null); ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.UploadQueue, (_) => { s2.Wait(); callOrder.Enqueue(3); s3.Release(); }, null); Thread.Sleep(100); s1.Release(); ThreadingManager.WaitForOutstandingTasks(); Assert.AreEqual(3, callOrder.Count); CollectionAssert.AreEqual(new[] { 1, 3, 2 }, callOrder.ToArray()); }
public void ThreadingTasks_SameQueues_ExecuteSequentially(object queue) { var testQueue = (ThreadingManager.ThreadQueues)queue; SemaphoreSlim s1 = new SemaphoreSlim(0); SemaphoreSlim s2 = new SemaphoreSlim(0); ConcurrentQueue <int> callOrder = new ConcurrentQueue <int>(); ThreadingManager.QueueTask(testQueue, (_) => { s1.Wait(); callOrder.Enqueue(1); Thread.Sleep(50); callOrder.Enqueue(2); }, null); ThreadingManager.QueueTask(testQueue, (_) => { callOrder.Enqueue(3); }, null); Thread.Sleep(100); s1.Release(); ThreadingManager.WaitForOutstandingTasks(); Assert.AreEqual(3, callOrder.Count); CollectionAssert.AreEqual(new[] { 1, 2, 3 }, callOrder.ToArray()); }
public void WhenPruneTaskActive_SaveAndUploadTasksWaitForPruneCompletion() { SemaphoreSlim s1 = new SemaphoreSlim(0); ConcurrentQueue <int> callOrder = new ConcurrentQueue <int>(); ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.PruneQueue, (_) => { s1.Wait(); callOrder.Enqueue(1); }, null); ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.SaveQueue, (_) => { callOrder.Enqueue(2); }, null); ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.UploadQueue, (_) => { callOrder.Enqueue(3); }, null); Thread.Sleep(100); s1.Release(); ThreadingManager.WaitForOutstandingTasks(); Assert.AreEqual(3, callOrder.Count); Assert.IsTrue(callOrder.TryDequeue(out int result)); Assert.AreEqual(1, result); var results = callOrder.ToArray(); CollectionAssert.AreEquivalent(new[] { 2, 3 }, results); }
/// <inheritdoc /> public void SaveCachedData(IList <CachedInfo> infos) { if (infos == null || infos.Count == 0) { return; } using (m_Logger.ScopedStep(LogLevel.Info, "SaveCachedData")) { m_Logger.AddEntrySafe(LogLevel.Info, $"Saving {infos.Count} infos"); // Setup Operations var ops = new FileOperations(infos.Count); using (m_Logger.ScopedStep(LogLevel.Info, "SetupOperations")) { for (int i = 0; i < infos.Count; i++) { var op = ops.data[i]; op.file = GetCachedInfoFile(infos[i].Asset); ops.data[i] = op; } } // Start writing thread ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.SaveQueue, Write, ops); using (m_Logger.ScopedStep(LogLevel.Info, "SerializingCacheInfos")) { // Serialize data as previous data is being written out var formatter = new BinaryFormatter(); for (int index = 0; index < infos.Count; index++, ops.waitLock.Release()) { try { var op = ops.data[index]; var stream = new MemoryStream(); formatter.Serialize(stream, infos[index]); if (stream.Length > 0) { op.bytes = stream; ops.data[index] = op; // If we have a cache server connection, upload the cached data if (m_Uploader != null) { m_Uploader.QueueUpload(infos[index].Asset, GetCachedArtifactsDirectory(infos[index].Asset), new MemoryStream(stream.GetBuffer(), false)); } } } catch (Exception e) { BuildLogger.LogException(e); } } } } }
// We return from this function before all uploads are complete. So we must wait to dispose until all uploads are finished. public void QueueUpload(CacheEntry entry, string artifactsPath, MemoryStream stream) { var item = new WorkItem(); string finalHash = HashingMethods.Calculate(entry.Hash, m_GlobalHash).ToString(); item.fileId = FileId.From(entry.Guid.ToString(), finalHash); item.artifactsPath = artifactsPath; item.stream = stream; ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.UploadQueue, UploadItem, item); }
internal void SyncPendingSaves() { using (m_Logger.ScopedStep(LogLevel.Info, "SyncPendingSaves")) ThreadingManager.WaitForOutstandingTasks(); }
/// <summary> /// Default implementation of generating Asset Bundles using the Scriptable Build Pipeline. /// </summary> /// <param name="parameters">Set of parameters used for building asset bundles.</param> /// <param name="content">Set of content and explicit asset bundle layout to build.</param> /// <param name="result">Results from building the content and explicit asset bundle layout.</param> /// <param name="taskList">Custom task list for building asset bundles.</param> /// <param name="contextObjects">Additional context objects to make available to the build.</param> /// <returns>Return code with status information about success or failure causes.</returns> public static ReturnCode BuildAssetBundles(IBundleBuildParameters parameters, IBundleBuildContent content, out IBundleBuildResults result, IList <IBuildTask> taskList, params IContextObject[] contextObjects) { if (BuildPipeline.isBuildingPlayer) { result = null; BuildLogger.LogException(new InvalidOperationException("Cannot build asset bundles while a build is in progress")); return(ReturnCode.Exception); } // Avoid throwing exceptions in here as we don't want them bubbling up to calling user code if (parameters == null) { result = null; BuildLogger.LogException(new ArgumentNullException("parameters")); return(ReturnCode.Exception); } // Avoid throwing exceptions in here as we don't want them bubbling up to calling user code if (taskList.IsNullOrEmpty()) { result = null; BuildLogger.LogException(new ArgumentException("Argument cannot be null or empty.", "taskList")); return(ReturnCode.Exception); } // Don't run if there are unsaved changes if (ValidationMethods.HasDirtyScenes()) { result = null; return(ReturnCode.UnsavedChanges); } ThreadingManager.WaitForOutstandingTasks(); BuildContext buildContext = new BuildContext(contextObjects); BuildLog buildLog = null; IBuildLogger logger; if (!buildContext.TryGetContextObject <IBuildLogger>(out logger)) { logger = buildLog = new BuildLog(); buildContext.SetContextObject(buildLog); } using (logger.ScopedStep(LogLevel.Info, "AssetDatabase.SaveAssets")) AssetDatabase.SaveAssets(); ReturnCode exitCode; result = new BundleBuildResults(); BuildCacheUtility.ClearCacheHashes(); using (var interfacesWrapper = new BuildInterfacesWrapper()) #if !CI_TESTRUNNER_PROJECT using (new SceneStateCleanup()) using (var progressTracker = new ProgressTracker()) #else using (var progressTracker = new ProgressLoggingTracker()) #endif using (var buildCache = new BuildCache(parameters.CacheServerHost, parameters.CacheServerPort)) { Directory.CreateDirectory(parameters.TempOutputFolder); Directory.CreateDirectory(parameters.ScriptOutputFolder); try { buildContext.SetContextObject(parameters); buildContext.SetContextObject(content); buildContext.SetContextObject(result); buildContext.SetContextObject(interfacesWrapper); buildContext.SetContextObject(progressTracker); buildContext.SetContextObject(buildCache); // If IDeterministicIdentifiers was passed in with contextObjects, don't add the default if (!buildContext.ContainsContextObject(typeof(IDeterministicIdentifiers))) { buildContext.SetContextObject(parameters.ContiguousBundles ? new PrefabPackedIdentifiers() : (IDeterministicIdentifiers) new Unity5PackedIdentifiers()); } buildContext.SetContextObject(new BuildDependencyData()); buildContext.SetContextObject(new BundleWriteData()); buildContext.SetContextObject(BuildCallbacks); buildCache.SetBuildLogger(logger); } catch (Exception e) { // Avoid throwing exceptions in here as we don't want them bubbling up to calling user code result = null; BuildLogger.LogException(e); return(ReturnCode.Exception); } exitCode = BuildTasksRunner.Validate(taskList, buildContext); if (exitCode >= ReturnCode.Success) #if SBP_PROFILER_ENABLE { exitCode = BuildTasksRunner.RunProfiled(taskList, buildContext); } #else { exitCode = BuildTasksRunner.Run(taskList, buildContext); } #endif if (Directory.Exists(parameters.TempOutputFolder)) { Directory.Delete(parameters.TempOutputFolder, true); } if (buildLog != null) { string buildLogPath = parameters.GetOutputFilePathForIdentifier("buildlogtep.json"); Directory.CreateDirectory(Path.GetDirectoryName(buildLogPath)); File.WriteAllText(parameters.GetOutputFilePathForIdentifier("buildlogtep.json"), buildLog.FormatForTraceEventProfiler()); } } long maximumCacheSize = ScriptableBuildPipeline.maximumCacheSize * BuildCache.k_BytesToGigaBytes; BuildCache.PruneCache_Background(maximumCacheSize); return(exitCode); }