Exemplo n.º 1
0
        private List<DistributedThread> InitMapThreads(int startWorkerNo, int workersCount)
        {
            var mapFunc = new Func<string, string, IBluepathCommunicationFramework, IEnumerable<string>>((filePath, mapFileName, bluepath) =>
                {
                    if (!(bluepath.Storage is IExtendedStorage))
                    {
                        throw new Exception("Mapper requires IExtendedStorage features.");
                    }

                    var storage = new BluepathStorage(bluepath.Storage as IExtendedStorage);

                    var mapProvider = Loader.Load<IMapProvider>(mapFileName, storage);
                    var mapper = new Mapper(filePath.ToString(), mapProvider.Map, storage);
                    var mapResult = mapper.PerformMap();

                    var keys = mapResult.Select(r => r.Key).Distinct().OrderBy(k => k);
                    var i = 0;
                    foreach (var res in mapResult)
                    {
                        Log.TraceMessage(Log.Activity.Custom,string.Format("[Map] Storing key '{0}', value '{1}'", res.Key, res.Value));
                        storage.Store(string.Format(Properties.Settings.Default.MapOutputFileName, res.Key, bluepath.ProcessEid, i++), res.Value);
                    }

                    return keys;
                });

            return this.InitThread(startWorkerNo, workersCount, mapFunc);
        }
Exemplo n.º 2
0
        private void PerformMap()
        {
            while (this.taskQueue.Count > 0)
            {
                Tuple<string, string> task;
                if (!this.taskQueue.TryDequeue(out task))
                {
                    continue;
                }

                var mapProvider = Loader.Load<IMapProvider>(task.Item2, this.Storage);
                var mapper = new Mapper(task.Item1, mapProvider.Map, this.Storage);
                var mapResult = mapper.PerformMap();
                foreach (var res in mapResult)
                {
                    this.Storage.Store(
                        string.Format(Properties.Settings.Default.MapOutputFileName, Base64.Encode(res.Key), this.Id, Guid.NewGuid()),
                        res.Value);
                }
            }
        }