private async Async.Task <(BucketConfig, WorkSet)?> BuildWorkSet(Task[] tasks)
{
var taskIds = tasks.Select(x => x.TaskId).ToHashSet();
var workUnits = new List <WorkUnit>();
BucketConfig?bucketConfig = null;
foreach (var task in tasks)
{
if ((task.Config.PrereqTasks?.Count ?? 0) > 0)
{
// if all of the prereqs are in this bucket, they will be
// scheduled together
if (!taskIds.IsSupersetOf(task.Config.PrereqTasks !))
{
if (!(await _taskOperations.CheckPrereqTasks(task)))
{
continue;
}
}
}
var result = await BuildWorkunit(task);
if (result == null)
{
continue;
}
if (bucketConfig == null)
{
bucketConfig = result.Value.Item1;
}
else if (bucketConfig != result.Value.Item1)
{
throw new Exception($"bucket configs differ: {bucketConfig} VS {result.Value.Item1}");
}
workUnits.Add(result.Value.Item2);
}
if (bucketConfig != null)
{
var setupUrl = await _containers.GetContainerSasUrl(bucketConfig.setupContainer, StorageType.Corpus, BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List) ?? throw new Exception("container not found");
var workSet = new WorkSet(
Reboot: bucketConfig.reboot,
Script: bucketConfig.setupScript != null,
SetupUrl: setupUrl,
WorkUnits: workUnits
);
return(bucketConfig, workSet);
}
return(null);
}
public async Task <bool> ScheduleWorkset(Pool pool, WorkSet workSet)
{
if (pool.State == PoolState.Shutdown || pool.State == PoolState.Halt)
{
return(false);
}
return(await _context.Queue.QueueObject(GetPoolQueue(pool.PoolId), workSet, StorageType.Corpus));
}
static void Main(string[] args)
{
var items = new string[] { "1", "2", "3", "300" };
using (var outfile = File.AppendText("file.txt"))
{
using (var ws = new WorkSet <string>(x => { lock (outfile) outfile.WriteLine(x); }))
foreach (var item in items)
{
ws.Process(item);
}
}
}
private async Async.Task <bool> ScheduleWorkset(WorkSet workSet, Pool pool, long count)
{
if (!PoolStateHelper.Available.Contains(pool.State))
{
_logTracer.Info($"pool not available for work: {pool.Name} state: {pool.State}");
return(false);
}
for (var i = 0L; i < count; i++)
{
if (!await _poolOperations.ScheduleWorkset(pool, workSet))
{
_logTracer.Error($"unable to schedule workset. pool:{pool.Name} workset: {workSet}");
return(false);
}
}
return(true);
}
private static void mainUpdateLoop(Object workSet)
{
WorkSet set = (WorkSet)workSet;
List <Thing> list2 = set.list2;
FieldInfo tickt = set.tickt;
TickList __instance = set.__instance;
Parallel.For(0, list2.Count, new ParallelOptions {
MaxDegreeOfParallelism = ThreadingMod.NUM_THREADS_PER_MAP
}, m =>
{
if (!list2[m].Destroyed)
{
try
{
switch (((TickerType)tickt.GetValue(__instance)))
{
case TickerType.Normal:
list2[m].Tick();
break;
case TickerType.Rare:
list2[m].TickRare();
break;
case TickerType.Long:
list2[m].TickLong();
break;
}
}
catch (Exception ex)
{
if (Prefs.DevMode)
{
Log.Error("Exception ticking " + list2[m].ToString() + ": " + ex.ToString());
}
}
}
});
loopDone = true;
loopPauser.Set();
}
public void Run(IEnumerable <GallioAction> actions)
{
// NOTE: This method has been optimized to minimize the total stack depth of the action
// by inlining blocks on the critical path that had previously been factored out.
if (actions == null)
{
throw new ArgumentNullException("actions");
}
// Copy the queue of actions to process from the enumeration.
var actionQueue = new Queue <GallioAction>();
foreach (GallioAction action in actions)
{
if (action == null)
{
throw new ArgumentNullException("actions");
}
actionQueue.Enqueue(action);
}
// Short-circuit when no actions to run to satisfy our invariant for pending work sets.
if (actionQueue.Count == 0)
{
return;
}
// Add this work set to the list of pending work sets.
WorkSet workSet;
lock (syncRoot)
{
workSet = new WorkSet(actionQueue);
pendingWorkSets.AddLast(workSet);
activeThreads += 1;
}
// Loop until all actions in this work set are finished.
for (; ;)
{
GallioAction nextAction;
lock (syncRoot)
{
// Synchronize and exit if this work set is finished.
if (!workSet.SyncHasPendingActions())
{
activeThreads -= 1;
while (workSet.SyncHasActionsInProgress())
{
Monitor.Wait(syncRoot);
}
return;
}
// Prepare next action from this work set only.
nextAction = workSet.SyncPrepareNextAction();
// Remove the work set from the list of pending work sets if it has no other pending actions.
if (!workSet.SyncHasPendingActions())
{
pendingWorkSets.Remove(workSet);
}
// Spawn more threads if needed for pending work sets.
SyncSpawnBackgroundActionLoopIfNeeded();
}
// Run the next action.
try
{
nextAction();
}
catch (Exception ex)
{
ReportUnhandledException(ex);
}
finally
{
lock (syncRoot)
{
workSet.SyncActionFinished();
Monitor.PulseAll(syncRoot);
}
}
}
}
public void Run(IEnumerable<Action> actions)
{
// NOTE: This method has been optimized to minimize the total stack depth of the action
// by inlining blocks on the critical path that had previously been factored out.
if (actions == null)
throw new ArgumentNullException("actions");
// Copy the queue of actions to process from the enumeration.
var actionQueue = new Queue<Action>();
foreach (Action action in actions)
{
if (action == null)
throw new ArgumentNullException("actions");
actionQueue.Enqueue(action);
}
// Short-circuit when no actions to run to satisfy our invariant for pending work sets.
if (actionQueue.Count == 0)
return;
// Add this work set to the list of pending work sets.
WorkSet workSet;
lock (syncRoot)
{
workSet = new WorkSet(actionQueue);
pendingWorkSets.AddLast(workSet);
activeThreads += 1;
}
// Loop until all actions in this work set are finished.
for (; ; )
{
Action nextAction;
lock (syncRoot)
{
// Synchronize and exit if this work set is finished.
if (!workSet.SyncHasPendingActions())
{
activeThreads -= 1;
while (workSet.SyncHasActionsInProgress())
Monitor.Wait(syncRoot);
return;
}
// Prepare next action from this work set only.
nextAction = workSet.SyncPrepareNextAction();
// Remove the work set from the list of pending work sets if it has no other pending actions.
if (!workSet.SyncHasPendingActions())
pendingWorkSets.Remove(workSet);
// Spawn more threads if needed for pending work sets.
SyncSpawnBackgroundActionLoopIfNeeded();
}
// Run the next action.
try
{
nextAction();
}
catch (Exception ex)
{
ReportUnhandledException(ex);
}
finally
{
lock (syncRoot)
{
workSet.SyncActionFinished();
Monitor.PulseAll(syncRoot);
}
}
}
}
public static TemplateList ReadFile(string templateListFilePath, FilterProcessor contentsFilter)
{
if (string.IsNullOrWhiteSpace(templateListFilePath))
{
return(null);
}
TemplateList listData = new TemplateList();
try
{
using (FileStream fs = new FileStream(templateListFilePath, FileMode.Open, FileAccess.Read))
using (TextReader rd = new StreamReader(fs))
{
WorkSet currentTargetSet = listData.mainWorkSet;
string lineBuffer = rd.ReadLine();
while (lineBuffer != null)
{
if (string.IsNullOrWhiteSpace(lineBuffer) || lineBuffer.StartsWith('#'))
{
lineBuffer = rd.ReadLine();
continue;
}
lineBuffer = contentsFilter.Filter(lineBuffer).Trim();
string[] lineTokens = lineBuffer.Split(':', StringSplitOptions.TrimEntries);
if (lineTokens.Length > 0)
{
if (lineTokens[0].StartsWith('['))
{
// Section Descriptor
string sectionName = _PopSectionName(lineTokens[0]);
if (sectionName != null)
{
sectionName = sectionName.Trim();
if (sectionName == string.Empty)
{
currentTargetSet = listData.mainWorkSet;
}
else
{
if (!listData.conditionalWorkSets.ContainsKey(sectionName))
{
listData.conditionalWorkSets.Add(sectionName, new WorkSet());
}
currentTargetSet = listData.conditionalWorkSets[sectionName];
}
}
}
else
{
switch (lineTokens[0])
{
case "C": // Copy
{
if (lineTokens.Length > 1)
{
PathData srcPathData = PathData.ParsePathString(lineTokens[1]);
PathData targetPathData = null;
if (lineTokens.Length > 2)
{
targetPathData = PathData.ParsePathString(lineTokens[2]);
}
else
{
targetPathData = srcPathData;
}
if (lineTokens[1].EndsWith("/*"))
{
currentTargetSet.copyAllEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData));
}
else
{
currentTargetSet.copyEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData));
}
}
}
break;
case "F": // Filter
{
if (lineTokens.Length > 1)
{
PathData srcPathData = PathData.ParsePathString(lineTokens[1]);
PathData targetPathData = null;
if (lineTokens.Length > 2)
{
targetPathData = PathData.ParsePathString(lineTokens[2]);
}
else
{
targetPathData = srcPathData;
}
currentTargetSet.filterEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData));
}
}
break;
}
}
}
lineBuffer = rd.ReadLine();
}
}
}
catch
{
return(null);
}
return(listData);
}
