private List<DistributedThread> InitMapThreads(int startWorkerNo, int workersCount)
{
var mapFunc = new Func<string, string, IBluepathCommunicationFramework, IEnumerable<string>>((filePath, mapFileName, bluepath) =>
{
if (!(bluepath.Storage is IExtendedStorage))
{
throw new Exception("Mapper requires IExtendedStorage features.");
}
var storage = new BluepathStorage(bluepath.Storage as IExtendedStorage);
var mapProvider = Loader.Load<IMapProvider>(mapFileName, storage);
var mapper = new Mapper(filePath.ToString(), mapProvider.Map, storage);
var mapResult = mapper.PerformMap();
var keys = mapResult.Select(r => r.Key).Distinct().OrderBy(k => k);
var i = 0;
foreach (var res in mapResult)
{
Log.TraceMessage(Log.Activity.Custom,string.Format("[Map] Storing key '{0}', value '{1}'", res.Key, res.Value));
storage.Store(string.Format(Properties.Settings.Default.MapOutputFileName, res.Key, bluepath.ProcessEid, i++), res.Value);
}
return keys;
});
return this.InitThread(startWorkerNo, workersCount, mapFunc);
}
public void FileSystemStorageRemoveDeletesGivenFile()
{
using (var redisStorage = new RedisStorage(Host))
{
var storage = new BluepathStorage(redisStorage, eraseContents: true);
storage.Store("a", "aa");
storage.Store("b", "bb");
var fileToRemove = storage.ListFiles().First(u => u.OriginalString.Contains("a"));
storage.Remove(fileToRemove);
var noOfFiles = storage.ListFiles().Count();
noOfFiles.ShouldBe(1);
storage.ListFiles().First().OriginalString.ShouldContain("b");
}
}
public void FileSystemStoragCleanMethodDeletesAllFiles()
{
using (var redisStorage = new RedisStorage(Host))
{
var storage = new BluepathStorage(redisStorage, eraseContents: true);
storage.Store("a", "aa");
storage.Store("b", "bb");
var noOfFiles1 = storage.ListFiles().Count();
noOfFiles1.ShouldBe(2);
storage.Clean();
var noOfFiles2 = storage.ListFiles().Count();
noOfFiles2.ShouldBe(0);
}
}
private List<DistributedThread> InitReduceThreads(int startWorkerNo, int workersCount)
{
var reduceFunc = new Func<string, string, IBluepathCommunicationFramework, IEnumerable<string>>((filePath, reduceFuncName, bluepath) =>
{
if (!(bluepath.Storage is IExtendedStorage))
{
throw new Exception("Reducer requires IExtendedStorage features.");
}
var storage = new BluepathStorage(bluepath.Storage as IExtendedStorage);
var reduceProvider = Loader.Load<IReduceProvider>(reduceFuncName, storage);
var reducer = new Reducer(filePath.ToString(), reduceProvider.Reduce, storage);
var reduceResult = reducer.PerformReduce();
Log.TraceMessage(Log.Activity.Custom,string.Format("[Reduce] Storing key '{0}', value '{1}'", reduceResult.Key, reduceResult.Value));
storage.Store(string.Format(Properties.Settings.Default.ReduceOutputFileName, reduceResult.Key, bluepath.ProcessEid, 0), reduceResult.Value);
return new List<string>() { reduceResult.Key };
});
return this.InitThread(startWorkerNo, workersCount, reduceFunc);
}