private List<DistributedThread> InitMapThreads(int startWorkerNo, int workersCount) { var mapFunc = new Func<string, string, IBluepathCommunicationFramework, IEnumerable<string>>((filePath, mapFileName, bluepath) => { if (!(bluepath.Storage is IExtendedStorage)) { throw new Exception("Mapper requires IExtendedStorage features."); } var storage = new BluepathStorage(bluepath.Storage as IExtendedStorage); var mapProvider = Loader.Load<IMapProvider>(mapFileName, storage); var mapper = new Mapper(filePath.ToString(), mapProvider.Map, storage); var mapResult = mapper.PerformMap(); var keys = mapResult.Select(r => r.Key).Distinct().OrderBy(k => k); var i = 0; foreach (var res in mapResult) { Log.TraceMessage(Log.Activity.Custom,string.Format("[Map] Storing key '{0}', value '{1}'", res.Key, res.Value)); storage.Store(string.Format(Properties.Settings.Default.MapOutputFileName, res.Key, bluepath.ProcessEid, i++), res.Value); } return keys; }); return this.InitThread(startWorkerNo, workersCount, mapFunc); }
private void PerformMap() { while (this.taskQueue.Count > 0) { Tuple<string, string> task; if (!this.taskQueue.TryDequeue(out task)) { continue; } var mapProvider = Loader.Load<IMapProvider>(task.Item2, this.Storage); var mapper = new Mapper(task.Item1, mapProvider.Map, this.Storage); var mapResult = mapper.PerformMap(); foreach (var res in mapResult) { this.Storage.Store( string.Format(Properties.Settings.Default.MapOutputFileName, Base64.Encode(res.Key), this.Id, Guid.NewGuid()), res.Value); } } }