示例#1
0
    private async Async.Task <(BucketConfig, WorkSet)?> BuildWorkSet(Task[] tasks)
    {
        var taskIds   = tasks.Select(x => x.TaskId).ToHashSet();
        var workUnits = new List <WorkUnit>();

        BucketConfig?bucketConfig = null;

        foreach (var task in tasks)
        {
            if ((task.Config.PrereqTasks?.Count ?? 0) > 0)
            {
                // if all of the prereqs are in this bucket, they will be
                // scheduled together
                if (!taskIds.IsSupersetOf(task.Config.PrereqTasks !))
                {
                    if (!(await _taskOperations.CheckPrereqTasks(task)))
                    {
                        continue;
                    }
                }
            }

            var result = await BuildWorkunit(task);

            if (result == null)
            {
                continue;
            }

            if (bucketConfig == null)
            {
                bucketConfig = result.Value.Item1;
            }
            else if (bucketConfig != result.Value.Item1)
            {
                throw new Exception($"bucket configs differ: {bucketConfig} VS {result.Value.Item1}");
            }

            workUnits.Add(result.Value.Item2);
        }

        if (bucketConfig != null)
        {
            var setupUrl = await _containers.GetContainerSasUrl(bucketConfig.setupContainer, StorageType.Corpus, BlobContainerSasPermissions.Read | BlobContainerSasPermissions.List) ?? throw new Exception("container not found");

            var workSet = new WorkSet(
                Reboot: bucketConfig.reboot,
                Script: bucketConfig.setupScript != null,
                SetupUrl: setupUrl,
                WorkUnits: workUnits
                );

            return(bucketConfig, workSet);
        }

        return(null);
    }
示例#2
0
    public async Task <bool> ScheduleWorkset(Pool pool, WorkSet workSet)
    {
        if (pool.State == PoolState.Shutdown || pool.State == PoolState.Halt)
        {
            return(false);
        }

        return(await _context.Queue.QueueObject(GetPoolQueue(pool.PoolId), workSet, StorageType.Corpus));
    }
    static void Main(string[] args)
    {
        var items = new string[] { "1", "2", "3", "300" };

        using (var outfile = File.AppendText("file.txt"))
        {
            using (var ws = new WorkSet <string>(x => { lock (outfile) outfile.WriteLine(x); }))
                foreach (var item in items)
                {
                    ws.Process(item);
                }
        }
    }
示例#4
0
    private async Async.Task <bool> ScheduleWorkset(WorkSet workSet, Pool pool, long count)
    {
        if (!PoolStateHelper.Available.Contains(pool.State))
        {
            _logTracer.Info($"pool not available for work: {pool.Name} state: {pool.State}");
            return(false);
        }

        for (var i = 0L; i < count; i++)
        {
            if (!await _poolOperations.ScheduleWorkset(pool, workSet))
            {
                _logTracer.Error($"unable to schedule workset. pool:{pool.Name} workset: {workSet}");
                return(false);
            }
        }
        return(true);
    }
        private static void mainUpdateLoop(Object workSet)
        {
            WorkSet set = (WorkSet)workSet;

            List <Thing> list2      = set.list2;
            FieldInfo    tickt      = set.tickt;
            TickList     __instance = set.__instance;

            Parallel.For(0, list2.Count, new ParallelOptions {
                MaxDegreeOfParallelism = ThreadingMod.NUM_THREADS_PER_MAP
            }, m =>
            {
                if (!list2[m].Destroyed)
                {
                    try
                    {
                        switch (((TickerType)tickt.GetValue(__instance)))
                        {
                        case TickerType.Normal:
                            list2[m].Tick();
                            break;

                        case TickerType.Rare:
                            list2[m].TickRare();
                            break;

                        case TickerType.Long:
                            list2[m].TickLong();
                            break;
                        }
                    }
                    catch (Exception ex)
                    {
                        if (Prefs.DevMode)
                        {
                            Log.Error("Exception ticking " + list2[m].ToString() + ": " + ex.ToString());
                        }
                    }
                }
            });

            loopDone = true;
            loopPauser.Set();
        }
示例#6
0
        public void Run(IEnumerable <GallioAction> actions)
        {
            // NOTE: This method has been optimized to minimize the total stack depth of the action
            //       by inlining blocks on the critical path that had previously been factored out.

            if (actions == null)
            {
                throw new ArgumentNullException("actions");
            }

            // Copy the queue of actions to process from the enumeration.
            var actionQueue = new Queue <GallioAction>();

            foreach (GallioAction action in actions)
            {
                if (action == null)
                {
                    throw new ArgumentNullException("actions");
                }
                actionQueue.Enqueue(action);
            }

            // Short-circuit when no actions to run to satisfy our invariant for pending work sets.
            if (actionQueue.Count == 0)
            {
                return;
            }

            // Add this work set to the list of pending work sets.
            WorkSet workSet;

            lock (syncRoot)
            {
                workSet = new WorkSet(actionQueue);

                pendingWorkSets.AddLast(workSet);
                activeThreads += 1;
            }

            // Loop until all actions in this work set are finished.
            for (; ;)
            {
                GallioAction nextAction;

                lock (syncRoot)
                {
                    // Synchronize and exit if this work set is finished.
                    if (!workSet.SyncHasPendingActions())
                    {
                        activeThreads -= 1;

                        while (workSet.SyncHasActionsInProgress())
                        {
                            Monitor.Wait(syncRoot);
                        }

                        return;
                    }

                    // Prepare next action from this work set only.
                    nextAction = workSet.SyncPrepareNextAction();

                    // Remove the work set from the list of pending work sets if it has no other pending actions.
                    if (!workSet.SyncHasPendingActions())
                    {
                        pendingWorkSets.Remove(workSet);
                    }

                    // Spawn more threads if needed for pending work sets.
                    SyncSpawnBackgroundActionLoopIfNeeded();
                }

                // Run the next action.
                try
                {
                    nextAction();
                }
                catch (Exception ex)
                {
                    ReportUnhandledException(ex);
                }
                finally
                {
                    lock (syncRoot)
                    {
                        workSet.SyncActionFinished();
                        Monitor.PulseAll(syncRoot);
                    }
                }
            }
        }
示例#7
0
        public void Run(IEnumerable<Action> actions)
        {
            // NOTE: This method has been optimized to minimize the total stack depth of the action
            //       by inlining blocks on the critical path that had previously been factored out.

            if (actions == null)
                throw new ArgumentNullException("actions");

            // Copy the queue of actions to process from the enumeration.
            var actionQueue = new Queue<Action>();
            foreach (Action action in actions)
            {
                if (action == null)
                    throw new ArgumentNullException("actions");
                actionQueue.Enqueue(action);
            }

            // Short-circuit when no actions to run to satisfy our invariant for pending work sets.
            if (actionQueue.Count == 0)
                return;

            // Add this work set to the list of pending work sets.
            WorkSet workSet;
            lock (syncRoot)
            {
                workSet = new WorkSet(actionQueue);

                pendingWorkSets.AddLast(workSet);
                activeThreads += 1;
            }

            // Loop until all actions in this work set are finished.
            for (; ; )
            {
                Action nextAction;

                lock (syncRoot)
                {
                    // Synchronize and exit if this work set is finished.
                    if (!workSet.SyncHasPendingActions())
                    {
                        activeThreads -= 1;

                        while (workSet.SyncHasActionsInProgress())
                            Monitor.Wait(syncRoot);

                        return;
                    }

                    // Prepare next action from this work set only.
                    nextAction = workSet.SyncPrepareNextAction();

                    // Remove the work set from the list of pending work sets if it has no other pending actions.
                    if (!workSet.SyncHasPendingActions())
                        pendingWorkSets.Remove(workSet);

                    // Spawn more threads if needed for pending work sets.
                    SyncSpawnBackgroundActionLoopIfNeeded();
                }

                // Run the next action.
                try
                {
                    nextAction();
                }
                catch (Exception ex)
                {
                    ReportUnhandledException(ex);
                }
                finally
                {
                    lock (syncRoot)
                    {
                        workSet.SyncActionFinished();
                        Monitor.PulseAll(syncRoot);
                    }
                }
            }
        }
示例#8
0
        public static TemplateList ReadFile(string templateListFilePath, FilterProcessor contentsFilter)
        {
            if (string.IsNullOrWhiteSpace(templateListFilePath))
            {
                return(null);
            }

            TemplateList listData = new TemplateList();

            try
            {
                using (FileStream fs = new FileStream(templateListFilePath, FileMode.Open, FileAccess.Read))
                    using (TextReader rd = new StreamReader(fs))
                    {
                        WorkSet currentTargetSet = listData.mainWorkSet;

                        string lineBuffer = rd.ReadLine();
                        while (lineBuffer != null)
                        {
                            if (string.IsNullOrWhiteSpace(lineBuffer) || lineBuffer.StartsWith('#'))
                            {
                                lineBuffer = rd.ReadLine();
                                continue;
                            }
                            lineBuffer = contentsFilter.Filter(lineBuffer).Trim();

                            string[] lineTokens = lineBuffer.Split(':', StringSplitOptions.TrimEntries);
                            if (lineTokens.Length > 0)
                            {
                                if (lineTokens[0].StartsWith('['))
                                {
                                    // Section Descriptor
                                    string sectionName = _PopSectionName(lineTokens[0]);
                                    if (sectionName != null)
                                    {
                                        sectionName = sectionName.Trim();
                                        if (sectionName == string.Empty)
                                        {
                                            currentTargetSet = listData.mainWorkSet;
                                        }
                                        else
                                        {
                                            if (!listData.conditionalWorkSets.ContainsKey(sectionName))
                                            {
                                                listData.conditionalWorkSets.Add(sectionName, new WorkSet());
                                            }
                                            currentTargetSet = listData.conditionalWorkSets[sectionName];
                                        }
                                    }
                                }
                                else
                                {
                                    switch (lineTokens[0])
                                    {
                                    case "C":   // Copy
                                    {
                                        if (lineTokens.Length > 1)
                                        {
                                            PathData srcPathData    = PathData.ParsePathString(lineTokens[1]);
                                            PathData targetPathData = null;
                                            if (lineTokens.Length > 2)
                                            {
                                                targetPathData = PathData.ParsePathString(lineTokens[2]);
                                            }
                                            else
                                            {
                                                targetPathData = srcPathData;
                                            }

                                            if (lineTokens[1].EndsWith("/*"))
                                            {
                                                currentTargetSet.copyAllEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData));
                                            }
                                            else
                                            {
                                                currentTargetSet.copyEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData));
                                            }
                                        }
                                    }
                                    break;

                                    case "F":   // Filter
                                    {
                                        if (lineTokens.Length > 1)
                                        {
                                            PathData srcPathData    = PathData.ParsePathString(lineTokens[1]);
                                            PathData targetPathData = null;
                                            if (lineTokens.Length > 2)
                                            {
                                                targetPathData = PathData.ParsePathString(lineTokens[2]);
                                            }
                                            else
                                            {
                                                targetPathData = srcPathData;
                                            }
                                            currentTargetSet.filterEntries.Add(new Tuple <PathData, PathData>(srcPathData, targetPathData));
                                        }
                                    }
                                    break;
                                    }
                                }
                            }

                            lineBuffer = rd.ReadLine();
                        }
                    }
            }
            catch
            {
                return(null);
            }

            return(listData);
        }