protected override void uploadResult()
{
string username = WorkerConfig.WorkConfig.Username;
string bucketName = WorkerConfig.WorkConfig.AssemblyMetaData.File.BucketName;
S3ObjectMetadata resultS3Object = new S3ObjectMetadata(
bucketName,
String.Format("{0}-{1}-{2}-{3}", username, TaskId, CoordinatorId, WorkerId)
);
Stream stream = null;
try{
stream = File.Open(Reduce.UserCreatedFile, FileMode.Open);
}
catch (Exception e) {
throw new Exception("No data emitted during reduce");
}
resultS3Object.upStream(stream);
Console.WriteLine("ReduceWorkFinished sending");
//Thread.Sleep(WorkerId * 2000);
Coordinator.Tell(new ReduceWorkFinishedMessage(WorkerId, TaskId, Reduce.EmittedKeys, resultS3Object), self);
Console.WriteLine("ReduceWorkFinished sent");
}
protected override void uploadResult()
{
Dictionary <string, string> createdFiles = map.createdFiles;
Dictionary <string, S3ObjectMetadata> mapResult = new Dictionary <string, S3ObjectMetadata> ();
UniqueKeyGenerator keyGenerator = new UniqueKeyGenerator();
foreach (KeyValuePair <string, string> pair in createdFiles)
{
string username = WorkerConfig.WorkConfig.Username;
string bucketName = WorkerConfig.WorkConfig.AssemblyMetaData.File.BucketName;
S3ObjectMetadata resultS3Object = new S3ObjectMetadata(
bucketName,
String.Format("{0}-{1}-{2}-{3}-{4}", username, TaskId, CoordinatorId, WorkerId, keyGenerator.generateKey())
);
resultS3Object.upStream(File.Open(pair.Value, FileMode.Open));
mapResult.Add(pair.Key, resultS3Object);
}
Console.WriteLine("MapWorkFinished sending");
//Thread.Sleep(WorkerId * 2000);
Coordinator.Tell(new MapWorkFinishedMessage(WorkerId, TaskId, mapResult), self);
Console.WriteLine("MapWorkFinished sent");
}
public object Get(ResultDto request)
{
var found = Db.Select <Task>(e => e.OwnerId == GetCurrentAuthUserId() && e.Id == request.TaskId);
if (found.Count != 0)
{
var entity = Db.Select <ResultMetadata>(e => e.TaskId == request.TaskId && e.Id == request.Id).FirstOrDefault();
if (entity != null)
{
var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), entity.Name);
S3Object.downStream().CopyTo(Response.OutputStream);
return(Response);
}
else
{
return(new HttpResult(HttpStatusCode.NotFound, "Not found"));
}
}
else
{
return(new HttpResult(HttpStatusCode.NotFound, "Not found"));
}
}
private static void deleteUnusedFiles(Func <string, bool> matches)
{
var bucketName = Environment.GetEnvironmentVariable("S3_BUCKET_NAME");
var S3Bucket = new S3Bucket(bucketName);
S3Bucket.fetchKeys();
while (S3Bucket.moveNext())
{
var currentKey = S3Bucket.getCurrentKey();
if (matches.Invoke(currentKey))
{
var S3Object = new S3ObjectMetadata(bucketName, currentKey);
S3Object.remove();
}
}
}
private HttpResult deleteMetadata <Entity>(MetadataDto dto, string type) where Entity : Metadata, new()
{
var found = Db.Select <Entity>(e => e.OwnerId == GetCurrentAuthUserId() && e.Id == dto.Id);
//TODO - transaction?
if (found.Count != 0)
{
var inProgressTasks = Db.Select <Task>(entity => entity.Status == "in progres");
if (found[0].CanDelete(inProgressTasks))
{
Db.DeleteById <Entity>(dto.Id);
var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), $"{GetCurrentAuthUserId()}-{type}-{dto.Id}");
S3Object.remove();
}
}
return(new HttpResult(HttpStatusCode.NoContent, "No content"));
}
protected Work loadClientAssembly(string className)
{
S3ObjectMetadata file = AssemblyMetaData.File;
string @namespace = AssemblyMetaData.Namespace;
string asseblyFileName = LocalFileUtils.DirectoryPath + "assembly.dll";
using (Stream stream = file.downStream()) {
using (FileStream fileStream = File.Create(asseblyFileName)){
stream.CopyTo(fileStream);
}
}
Assembly assembly = Assembly.LoadFrom(asseblyFileName);
Type type = assembly.GetType(@namespace + "." + className);
Work work = (Work)Activator.CreateInstance(type);
work.setEmitParams(LocalFileUtils.DirectoryPath);
return(work);
}
private HttpResult put <Entity>(ContentDto request, string type) where Entity : Metadata, new()
{
var entity = Db.Select <Entity>(e => e.OwnerId == GetCurrentAuthUserId() && e.Id == request.Id)
.FirstOrDefault();
if (entity != null)
{
var objectKey = $"{GetCurrentAuthUserId()}-{type}-{entity.Id}";
var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), objectKey);
S3Object.upStreamThroughLocalBuffer(request.RequestStream);
entity.IsUploaded = true;
Db.Save(entity);
return(new HttpResult(HttpStatusCode.OK, "Ok"));
}
else
{
return(new HttpResult(HttpStatusCode.BadRequest, "Resource does not exist or you are not its owner"));
}
}
public List <Dictionary <string, S3ObjectMetadata> > divide()
{
var entry = getNewEntry();
long totalSize = InputFiles.Values.Aggregate(0L, (acc, file) => acc + file.getSize());
Console.WriteLine("totalSize= " + totalSize);
maxFragmentSize = totalSize / M;
foreach (var pair in InputFiles)
{
currentFilename = pair.Key;
using (var fileStreamReader = new StreamReader(pair.Value.downStream()))
{
string line;
while ((line = fileStreamReader.ReadLine()) != null)
{
writeToCurrentFragment(line);
closed = false;
if (shouldCloseFragment())
{
usedSpace = 0;
closeCurrentFragmentWriter();
var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), getCurrentFragmentName());
using (var fragmentStream = File.OpenRead(getCurrentFragmentPath()))
{
S3Object.upStream(fragmentStream);
}
File.Delete(getCurrentFragmentPath());
entry.Add(currentFilename, S3Object);
Response.Add(entry);
entry = getNewEntry();
closed = true;
}
}
if (!closed)
{
var currentFragmentClosed = closeCurrentFragmentWriter();
if (currentFragmentClosed)
{
var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), getCurrentFragmentName());
using (var fragmentStream = File.OpenRead(getCurrentFragmentPath()))
{
S3Object.upStream(fragmentStream);
}
File.Delete(getCurrentFragmentPath());
entry.Add(currentFilename, S3Object);
}
}
}
}
Response.Add(entry);
return(Response);
}
public void addReduceResult(S3ObjectMetadata file, List <string> keys)
{
ReduceResult.Add(new Tuple <S3ObjectMetadata, List <string> >(file, keys));
ReduceFiles.Add(file);
}
