public void ThreadingTasks_SaveQueue_RunsInParallelWith_UploadQueue()
{
SemaphoreSlim s1 = new SemaphoreSlim(0);
SemaphoreSlim s2 = new SemaphoreSlim(0);
SemaphoreSlim s3 = new SemaphoreSlim(0);
ConcurrentQueue <int> callOrder = new ConcurrentQueue <int>();
ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.SaveQueue, (_) =>
{
s1.Wait();
callOrder.Enqueue(1);
s2.Release();
s3.Wait();
callOrder.Enqueue(2);
}, null);
ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.UploadQueue, (_) =>
{
s2.Wait();
callOrder.Enqueue(3);
s3.Release();
}, null);
Thread.Sleep(100);
s1.Release();
ThreadingManager.WaitForOutstandingTasks();
Assert.AreEqual(3, callOrder.Count);
CollectionAssert.AreEqual(new[] { 1, 3, 2 }, callOrder.ToArray());
}
public void ThreadingTasks_SameQueues_ExecuteSequentially(object queue)
{
var testQueue = (ThreadingManager.ThreadQueues)queue;
SemaphoreSlim s1 = new SemaphoreSlim(0);
SemaphoreSlim s2 = new SemaphoreSlim(0);
ConcurrentQueue <int> callOrder = new ConcurrentQueue <int>();
ThreadingManager.QueueTask(testQueue, (_) =>
{
s1.Wait();
callOrder.Enqueue(1);
Thread.Sleep(50);
callOrder.Enqueue(2);
}, null);
ThreadingManager.QueueTask(testQueue, (_) =>
{
callOrder.Enqueue(3);
}, null);
Thread.Sleep(100);
s1.Release();
ThreadingManager.WaitForOutstandingTasks();
Assert.AreEqual(3, callOrder.Count);
CollectionAssert.AreEqual(new[] { 1, 2, 3 }, callOrder.ToArray());
}
public void WhenPruneTaskActive_SaveAndUploadTasksWaitForPruneCompletion()
{
SemaphoreSlim s1 = new SemaphoreSlim(0);
ConcurrentQueue <int> callOrder = new ConcurrentQueue <int>();
ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.PruneQueue, (_) =>
{
s1.Wait();
callOrder.Enqueue(1);
}, null);
ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.SaveQueue, (_) =>
{
callOrder.Enqueue(2);
}, null);
ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.UploadQueue, (_) =>
{
callOrder.Enqueue(3);
}, null);
Thread.Sleep(100);
s1.Release();
ThreadingManager.WaitForOutstandingTasks();
Assert.AreEqual(3, callOrder.Count);
Assert.IsTrue(callOrder.TryDequeue(out int result));
Assert.AreEqual(1, result);
var results = callOrder.ToArray();
CollectionAssert.AreEquivalent(new[] { 2, 3 }, results);
}
/// <inheritdoc />
public void SaveCachedData(IList <CachedInfo> infos)
{
if (infos == null || infos.Count == 0)
{
return;
}
using (m_Logger.ScopedStep(LogLevel.Info, "SaveCachedData"))
{
m_Logger.AddEntrySafe(LogLevel.Info, $"Saving {infos.Count} infos");
// Setup Operations
var ops = new FileOperations(infos.Count);
using (m_Logger.ScopedStep(LogLevel.Info, "SetupOperations"))
{
for (int i = 0; i < infos.Count; i++)
{
var op = ops.data[i];
op.file = GetCachedInfoFile(infos[i].Asset);
ops.data[i] = op;
}
}
// Start writing thread
ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.SaveQueue, Write, ops);
using (m_Logger.ScopedStep(LogLevel.Info, "SerializingCacheInfos"))
{
// Serialize data as previous data is being written out
var formatter = new BinaryFormatter();
for (int index = 0; index < infos.Count; index++, ops.waitLock.Release())
{
try
{
var op = ops.data[index];
var stream = new MemoryStream();
formatter.Serialize(stream, infos[index]);
if (stream.Length > 0)
{
op.bytes = stream;
ops.data[index] = op;
// If we have a cache server connection, upload the cached data
if (m_Uploader != null)
{
m_Uploader.QueueUpload(infos[index].Asset, GetCachedArtifactsDirectory(infos[index].Asset), new MemoryStream(stream.GetBuffer(), false));
}
}
}
catch (Exception e)
{
BuildLogger.LogException(e);
}
}
}
}
}
// We return from this function before all uploads are complete. So we must wait to dispose until all uploads are finished.
public void QueueUpload(CacheEntry entry, string artifactsPath, MemoryStream stream)
{
var item = new WorkItem();
string finalHash = HashingMethods.Calculate(entry.Hash, m_GlobalHash).ToString();
item.fileId = FileId.From(entry.Guid.ToString(), finalHash);
item.artifactsPath = artifactsPath;
item.stream = stream;
ThreadingManager.QueueTask(ThreadingManager.ThreadQueues.UploadQueue, UploadItem, item);
}
internal void SyncPendingSaves()
{
using (m_Logger.ScopedStep(LogLevel.Info, "SyncPendingSaves"))
ThreadingManager.WaitForOutstandingTasks();
}
/// <summary>
/// Default implementation of generating Asset Bundles using the Scriptable Build Pipeline.
/// </summary>
/// <param name="parameters">Set of parameters used for building asset bundles.</param>
/// <param name="content">Set of content and explicit asset bundle layout to build.</param>
/// <param name="result">Results from building the content and explicit asset bundle layout.</param>
/// <param name="taskList">Custom task list for building asset bundles.</param>
/// <param name="contextObjects">Additional context objects to make available to the build.</param>
/// <returns>Return code with status information about success or failure causes.</returns>
public static ReturnCode BuildAssetBundles(IBundleBuildParameters parameters, IBundleBuildContent content, out IBundleBuildResults result, IList <IBuildTask> taskList, params IContextObject[] contextObjects)
{
if (BuildPipeline.isBuildingPlayer)
{
result = null;
BuildLogger.LogException(new InvalidOperationException("Cannot build asset bundles while a build is in progress"));
return(ReturnCode.Exception);
}
// Avoid throwing exceptions in here as we don't want them bubbling up to calling user code
if (parameters == null)
{
result = null;
BuildLogger.LogException(new ArgumentNullException("parameters"));
return(ReturnCode.Exception);
}
// Avoid throwing exceptions in here as we don't want them bubbling up to calling user code
if (taskList.IsNullOrEmpty())
{
result = null;
BuildLogger.LogException(new ArgumentException("Argument cannot be null or empty.", "taskList"));
return(ReturnCode.Exception);
}
// Don't run if there are unsaved changes
if (ValidationMethods.HasDirtyScenes())
{
result = null;
return(ReturnCode.UnsavedChanges);
}
ThreadingManager.WaitForOutstandingTasks();
BuildContext buildContext = new BuildContext(contextObjects);
BuildLog buildLog = null;
IBuildLogger logger;
if (!buildContext.TryGetContextObject <IBuildLogger>(out logger))
{
logger = buildLog = new BuildLog();
buildContext.SetContextObject(buildLog);
}
using (logger.ScopedStep(LogLevel.Info, "AssetDatabase.SaveAssets"))
AssetDatabase.SaveAssets();
ReturnCode exitCode;
result = new BundleBuildResults();
BuildCacheUtility.ClearCacheHashes();
using (var interfacesWrapper = new BuildInterfacesWrapper())
#if !CI_TESTRUNNER_PROJECT
using (new SceneStateCleanup())
using (var progressTracker = new ProgressTracker())
#else
using (var progressTracker = new ProgressLoggingTracker())
#endif
using (var buildCache = new BuildCache(parameters.CacheServerHost, parameters.CacheServerPort))
{
Directory.CreateDirectory(parameters.TempOutputFolder);
Directory.CreateDirectory(parameters.ScriptOutputFolder);
try
{
buildContext.SetContextObject(parameters);
buildContext.SetContextObject(content);
buildContext.SetContextObject(result);
buildContext.SetContextObject(interfacesWrapper);
buildContext.SetContextObject(progressTracker);
buildContext.SetContextObject(buildCache);
// If IDeterministicIdentifiers was passed in with contextObjects, don't add the default
if (!buildContext.ContainsContextObject(typeof(IDeterministicIdentifiers)))
{
buildContext.SetContextObject(parameters.ContiguousBundles ? new PrefabPackedIdentifiers() : (IDeterministicIdentifiers) new Unity5PackedIdentifiers());
}
buildContext.SetContextObject(new BuildDependencyData());
buildContext.SetContextObject(new BundleWriteData());
buildContext.SetContextObject(BuildCallbacks);
buildCache.SetBuildLogger(logger);
}
catch (Exception e)
{
// Avoid throwing exceptions in here as we don't want them bubbling up to calling user code
result = null;
BuildLogger.LogException(e);
return(ReturnCode.Exception);
}
exitCode = BuildTasksRunner.Validate(taskList, buildContext);
if (exitCode >= ReturnCode.Success)
#if SBP_PROFILER_ENABLE
{ exitCode = BuildTasksRunner.RunProfiled(taskList, buildContext); }
#else
{ exitCode = BuildTasksRunner.Run(taskList, buildContext); }
#endif
if (Directory.Exists(parameters.TempOutputFolder))
{
Directory.Delete(parameters.TempOutputFolder, true);
}
if (buildLog != null)
{
string buildLogPath = parameters.GetOutputFilePathForIdentifier("buildlogtep.json");
Directory.CreateDirectory(Path.GetDirectoryName(buildLogPath));
File.WriteAllText(parameters.GetOutputFilePathForIdentifier("buildlogtep.json"), buildLog.FormatForTraceEventProfiler());
}
}
long maximumCacheSize = ScriptableBuildPipeline.maximumCacheSize * BuildCache.k_BytesToGigaBytes;
BuildCache.PruneCache_Background(maximumCacheSize);
return(exitCode);
}
