public Mapper(string filePath, Func<string, string, IEnumerable<KeyValuePair<string, string>>> map, IMapReduceStorage storage)
{
this.Key = filePath;
this.Value = storage.Read(filePath);
this.map = map;
this.storage = storage;
}
public static void LoadToStorage(string realPath, FileUri storageFileName, IMapReduceStorage storage)
{
using (var sr = new StreamReader(realPath))
{
storage.Store(storage.GetFileName(storageFileName.Uri), sr.ReadToEnd());
}
}
public void CoordinatorWorks()
{
//Bluepath.Log.ClearXes();
using (var redisStorage = new RedisStorage(Host))
{
this.storage = new BluepathStorage(redisStorage);
this.storage.Clean();
this.storage.Store("f1", "ala ma kota");
this.storage.Store("f2", "kota alama");
this.storage.Store("f3", "dolan ma");
var filesToRead = this.storage.ListFiles();
var mapperCodeFile = new FileUri("file:///SampleMapper.cs");
var reducerCodeFile = new FileUri("file:///SampleReducer.cs");
TestHelpers.LoadToStorage(@"..\..\SampleMapper.cs", mapperCodeFile, this.storage);
TestHelpers.LoadToStorage(@"..\..\SampleReducer.cs", reducerCodeFile, this.storage);
var connectionManager = new ConnectionManager(new Dictionary<ServiceUri, PerformanceStatistics>(), null, null, null);
var scheduler = new ThreadNumberScheduler(connectionManager);
var coordinator = new Coordinator(connectionManager, scheduler, DistributedThread.ExecutorSelectionMode.LocalOnly);
coordinator.Start(2, 2, mapperCodeFile, reducerCodeFile, filesToRead.Select(f => new FileUri(f.ToString())));
string result = string.Empty;
Debug.WriteLine("Listing files...");
foreach (var file in this.storage.ListFiles())
{
var fileName = this.storage.GetFileName(file);
Debug.Write(fileName);
try
{
Debug.WriteLine(" -- {0}", (object)Base64Decode(fileName));
}
catch
{
Debug.WriteLine(string.Empty);
}
}
foreach (var uri in this.storage.ListFiles())
{
var file = this.storage.GetFileName(uri);
if (file.Contains("REDUCE") && file.Contains("kota"))
{
result = this.storage.Read(file);
Debug.WriteLine("File '{0}' contains '{1}'.", file, result);
}
}
result.ShouldBe("2");
this.storage.Clean();
}
System.Threading.Thread.Sleep(3000);
//Bluepath.Log.SaveXes(@"c:\temp\bluepath.xes", clearListAfterSave: true);
//Bluepath.Log.ClearXes();
}
public Reducer(string key, Func<string, IEnumerable<string>, KeyValuePair<string, string>> reduce, IMapReduceStorage storage)
{
this.key = key;
this.reduce = reduce;
this.FileFilter = new Regex(string.Format("^" + Settings.Default.MapOutputFileName + "$", this.key, RegexExtensions.GuidRegexString, "[0-9]+"));
this.storage = storage;
this.Load();
}
public static string[] CreateTwoKeyFileSet(IMapReduceStorage storage)
{
var keys = new string[] { "k1", "k2" };
foreach (var key in keys)
{
for (int i = 0; i < 3; i++)
{
var fileName = string.Format(Bluepath.MapReduce.Properties.Settings.Default.MapOutputFileName, key, Guid.NewGuid(), i);
storage.Store(fileName, "1");
}
}
return keys;
}
public void Init()
{
this.storage = new InMemoryStorage();
}