public IEnumerable<TransferItem> GetNextTransferItems(IDs3Client client, MasterObjectList jobResponse)
{
var clientFactory = client.BuildFactory(jobResponse.Nodes);
return
from chunk in jobResponse.Objects
let transferClient = clientFactory.GetClientForNodeId(chunk.NodeId)
from jobObject in chunk.ObjectsList
where !(bool)jobObject.InCache
select new TransferItem(transferClient, Blob.Convert(jobObject));
}
public IEnumerable<TransferItem> GetNextTransferItems(IDs3Client client, MasterObjectList jobResponse)
{
this._client = client;
this._jobResponse = jobResponse;
lock (this._lock)
{
_allocatedChunks = new Dictionary<Guid, bool>(jobResponse.Objects.Select(chunk => chunk)
.ToDictionary(
chunk => chunk.ChunkId,
chunk => false));
this._streamToChunkDictionary = GetStreamToChunksDictionary();
}
// Flatten all batches into a single enumerable.
return EnumerateTransferItemBatches().SelectMany(it => it);
}
public IEnumerable<TransferItem> GetNextTransferItems(IDs3Client client, MasterObjectList jobResponse)
{
this._client = client;
this._jobResponse = jobResponse;
if (_withAggregation)
{
_jobResponse.Objects = GetObjectsNotInCache();
}
lock (this._chunksRemainingLock)
{
_toAllocateChunks = new HashSet<Guid>(jobResponse.Objects.Select(chunk => chunk.ChunkId));
}
// Flatten all batches into a single enumerable.
return EnumerateTransferItemBatches().SelectMany(it => it);
}
public PutBulkJobSpectraS3Response(MasterObjectList responsePayload)
{
this.ResponsePayload = responsePayload;
}
public ModifyActiveJobSpectraS3Response(MasterObjectList responsePayload)
{
this.ResponsePayload = responsePayload;
}
private void PutObject(IDs3Client client, BulkObject obj, MasterObjectList bulkResult)
{
var fileToPut = File.OpenRead(testDirectoryBigFolderForMaxBlob + obj.Name);
var contentStream = new ObjectRequestStream(fileToPut, obj.Offset, obj.Length);
var putObjectRequest = new PutObjectRequest(
bulkResult.BucketName,
obj.Name,
contentStream
).WithJob(bulkResult.JobId)
.WithOffset(obj.Offset);
client.PutObject(putObjectRequest);
fileToPut.Close();
}
private void AsyncUpload(IDs3Client client, ICollection<Guid> chunkIds, MasterObjectList response,
MasterObjectList bulkResult)
{
Parallel.ForEach(response.Objects,
chunk =>
{
if (!chunkIds.Contains(chunk.ChunkId)) return;
chunkIds.Remove(chunk.ChunkId);
// it is possible that if we start resending a chunk, due to the program crashing, that
// some objects will already be in cache. Check to make sure that they are not, and then
// send the object to Spectra S3
Parallel.ForEach(chunk.ObjectsList,
obj =>
{
if ((bool) obj.InCache) return;
PutObject(client, obj, bulkResult);
});
Console.WriteLine();
});
}
public ReplicatePutJobSpectraS3Response(MasterObjectList responsePayload)
{
this.ResponsePayload = responsePayload;
}