public Mapper(string filePath, Func<string, string, IEnumerable<KeyValuePair<string, string>>> map, IMapReduceStorage storage) { this.Key = filePath; this.Value = storage.Read(filePath); this.map = map; this.storage = storage; }
public static void LoadToStorage(string realPath, FileUri storageFileName, IMapReduceStorage storage) { using (var sr = new StreamReader(realPath)) { storage.Store(storage.GetFileName(storageFileName.Uri), sr.ReadToEnd()); } }
public void CoordinatorWorks() { //Bluepath.Log.ClearXes(); using (var redisStorage = new RedisStorage(Host)) { this.storage = new BluepathStorage(redisStorage); this.storage.Clean(); this.storage.Store("f1", "ala ma kota"); this.storage.Store("f2", "kota alama"); this.storage.Store("f3", "dolan ma"); var filesToRead = this.storage.ListFiles(); var mapperCodeFile = new FileUri("file:///SampleMapper.cs"); var reducerCodeFile = new FileUri("file:///SampleReducer.cs"); TestHelpers.LoadToStorage(@"..\..\SampleMapper.cs", mapperCodeFile, this.storage); TestHelpers.LoadToStorage(@"..\..\SampleReducer.cs", reducerCodeFile, this.storage); var connectionManager = new ConnectionManager(new Dictionary<ServiceUri, PerformanceStatistics>(), null, null, null); var scheduler = new ThreadNumberScheduler(connectionManager); var coordinator = new Coordinator(connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly); coordinator.Start(2, 2, mapperCodeFile, reducerCodeFile, filesToRead.Select(f => new FileUri(f.ToString()))); string result = string.Empty; Debug.WriteLine("Listing files..."); foreach (var file in this.storage.ListFiles()) { var fileName = this.storage.GetFileName(file); Debug.Write(fileName); try { Debug.WriteLine(" -- {0}", (object)Base64Decode(fileName)); } catch { Debug.WriteLine(string.Empty); } } foreach (var uri in this.storage.ListFiles()) { var file = this.storage.GetFileName(uri); if (file.Contains("REDUCE") && file.Contains("kota")) { result = this.storage.Read(file); Debug.WriteLine("File '{0}' contains '{1}'.", file, result); } } result.ShouldBe("2"); this.storage.Clean(); } System.Threading.Thread.Sleep(3000); //Bluepath.Log.SaveXes(@"c:\temp\bluepath.xes", clearListAfterSave: true); //Bluepath.Log.ClearXes(); }
public Reducer(string key, Func<string, IEnumerable<string>, KeyValuePair<string, string>> reduce, IMapReduceStorage storage) { this.key = key; this.reduce = reduce; this.FileFilter = new Regex(string.Format("^" + Settings.Default.MapOutputFileName + "$", this.key, RegexExtensions.GuidRegexString, "[0-9]+")); this.storage = storage; this.Load(); }
public static string[] CreateTwoKeyFileSet(IMapReduceStorage storage) { var keys = new string[] { "k1", "k2" }; foreach (var key in keys) { for (int i = 0; i < 3; i++) { var fileName = string.Format(Bluepath.MapReduce.Properties.Settings.Default.MapOutputFileName, key, Guid.NewGuid(), i); storage.Store(fileName, "1"); } } return keys; }
public void Init() { this.storage = new InMemoryStorage(); }