protected override void uploadResult() { string username = WorkerConfig.WorkConfig.Username; string bucketName = WorkerConfig.WorkConfig.AssemblyMetaData.File.BucketName; S3ObjectMetadata resultS3Object = new S3ObjectMetadata( bucketName, String.Format("{0}-{1}-{2}-{3}", username, TaskId, CoordinatorId, WorkerId) ); Stream stream = null; try{ stream = File.Open(Reduce.UserCreatedFile, FileMode.Open); } catch (Exception e) { throw new Exception("No data emitted during reduce"); } resultS3Object.upStream(stream); Console.WriteLine("ReduceWorkFinished sending"); //Thread.Sleep(WorkerId * 2000); Coordinator.Tell(new ReduceWorkFinishedMessage(WorkerId, TaskId, Reduce.EmittedKeys, resultS3Object), self); Console.WriteLine("ReduceWorkFinished sent"); }
protected override void uploadResult() { Dictionary <string, string> createdFiles = map.createdFiles; Dictionary <string, S3ObjectMetadata> mapResult = new Dictionary <string, S3ObjectMetadata> (); UniqueKeyGenerator keyGenerator = new UniqueKeyGenerator(); foreach (KeyValuePair <string, string> pair in createdFiles) { string username = WorkerConfig.WorkConfig.Username; string bucketName = WorkerConfig.WorkConfig.AssemblyMetaData.File.BucketName; S3ObjectMetadata resultS3Object = new S3ObjectMetadata( bucketName, String.Format("{0}-{1}-{2}-{3}-{4}", username, TaskId, CoordinatorId, WorkerId, keyGenerator.generateKey()) ); resultS3Object.upStream(File.Open(pair.Value, FileMode.Open)); mapResult.Add(pair.Key, resultS3Object); } Console.WriteLine("MapWorkFinished sending"); //Thread.Sleep(WorkerId * 2000); Coordinator.Tell(new MapWorkFinishedMessage(WorkerId, TaskId, mapResult), self); Console.WriteLine("MapWorkFinished sent"); }
public object Get(ResultDto request) { var found = Db.Select <Task>(e => e.OwnerId == GetCurrentAuthUserId() && e.Id == request.TaskId); if (found.Count != 0) { var entity = Db.Select <ResultMetadata>(e => e.TaskId == request.TaskId && e.Id == request.Id).FirstOrDefault(); if (entity != null) { var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), entity.Name); S3Object.downStream().CopyTo(Response.OutputStream); return(Response); } else { return(new HttpResult(HttpStatusCode.NotFound, "Not found")); } } else { return(new HttpResult(HttpStatusCode.NotFound, "Not found")); } }
private static void deleteUnusedFiles(Func <string, bool> matches) { var bucketName = Environment.GetEnvironmentVariable("S3_BUCKET_NAME"); var S3Bucket = new S3Bucket(bucketName); S3Bucket.fetchKeys(); while (S3Bucket.moveNext()) { var currentKey = S3Bucket.getCurrentKey(); if (matches.Invoke(currentKey)) { var S3Object = new S3ObjectMetadata(bucketName, currentKey); S3Object.remove(); } } }
private HttpResult deleteMetadata <Entity>(MetadataDto dto, string type) where Entity : Metadata, new() { var found = Db.Select <Entity>(e => e.OwnerId == GetCurrentAuthUserId() && e.Id == dto.Id); //TODO - transaction? if (found.Count != 0) { var inProgressTasks = Db.Select <Task>(entity => entity.Status == "in progres"); if (found[0].CanDelete(inProgressTasks)) { Db.DeleteById <Entity>(dto.Id); var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), $"{GetCurrentAuthUserId()}-{type}-{dto.Id}"); S3Object.remove(); } } return(new HttpResult(HttpStatusCode.NoContent, "No content")); }
protected Work loadClientAssembly(string className) { S3ObjectMetadata file = AssemblyMetaData.File; string @namespace = AssemblyMetaData.Namespace; string asseblyFileName = LocalFileUtils.DirectoryPath + "assembly.dll"; using (Stream stream = file.downStream()) { using (FileStream fileStream = File.Create(asseblyFileName)){ stream.CopyTo(fileStream); } } Assembly assembly = Assembly.LoadFrom(asseblyFileName); Type type = assembly.GetType(@namespace + "." + className); Work work = (Work)Activator.CreateInstance(type); work.setEmitParams(LocalFileUtils.DirectoryPath); return(work); }
private HttpResult put <Entity>(ContentDto request, string type) where Entity : Metadata, new() { var entity = Db.Select <Entity>(e => e.OwnerId == GetCurrentAuthUserId() && e.Id == request.Id) .FirstOrDefault(); if (entity != null) { var objectKey = $"{GetCurrentAuthUserId()}-{type}-{entity.Id}"; var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), objectKey); S3Object.upStreamThroughLocalBuffer(request.RequestStream); entity.IsUploaded = true; Db.Save(entity); return(new HttpResult(HttpStatusCode.OK, "Ok")); } else { return(new HttpResult(HttpStatusCode.BadRequest, "Resource does not exist or you are not its owner")); } }
public List <Dictionary <string, S3ObjectMetadata> > divide() { var entry = getNewEntry(); long totalSize = InputFiles.Values.Aggregate(0L, (acc, file) => acc + file.getSize()); Console.WriteLine("totalSize= " + totalSize); maxFragmentSize = totalSize / M; foreach (var pair in InputFiles) { currentFilename = pair.Key; using (var fileStreamReader = new StreamReader(pair.Value.downStream())) { string line; while ((line = fileStreamReader.ReadLine()) != null) { writeToCurrentFragment(line); closed = false; if (shouldCloseFragment()) { usedSpace = 0; closeCurrentFragmentWriter(); var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), getCurrentFragmentName()); using (var fragmentStream = File.OpenRead(getCurrentFragmentPath())) { S3Object.upStream(fragmentStream); } File.Delete(getCurrentFragmentPath()); entry.Add(currentFilename, S3Object); Response.Add(entry); entry = getNewEntry(); closed = true; } } if (!closed) { var currentFragmentClosed = closeCurrentFragmentWriter(); if (currentFragmentClosed) { var S3Object = new S3ObjectMetadata(Environment.GetEnvironmentVariable("S3_BUCKET_NAME"), getCurrentFragmentName()); using (var fragmentStream = File.OpenRead(getCurrentFragmentPath())) { S3Object.upStream(fragmentStream); } File.Delete(getCurrentFragmentPath()); entry.Add(currentFilename, S3Object); } } } } Response.Add(entry); return(Response); }
public void addReduceResult(S3ObjectMetadata file, List <string> keys) { ReduceResult.Add(new Tuple <S3ObjectMetadata, List <string> >(file, keys)); ReduceFiles.Add(file); }