/// <summary>
/// Sets up a temporary data policy with a temporary storage domain and partition
/// for use in integration tests where the BP may not currently have access to a
/// partition.
/// </summary>
/// <param name="fixtureName">Name of fixutre being run under this domain. This is
/// used to prevent race conditions between test setup and teardown</param>
/// <param name="dataPolicyId"></param>
/// <param name="client"></param>
/// <returns>Storage domain and data persistence Ids for use in teardown</returns>
public static TempStorageIds Setup(
string fixtureName,
Guid dataPolicyId,
IDs3Client client)
{
//Create storage domain
PutStorageDomainSpectraS3Response storageDomainResponse = ABMTestHelper.CreateStorageDomain(fixtureName, client);
//Crate pool partition
PutPoolPartitionSpectraS3Response poolPartitionResponse = ABMTestHelper.CreatePoolPartition(fixtureName, PoolType.ONLINE, client);
//Create storage domain member linking pool partition to storage domain
PutPoolStorageDomainMemberSpectraS3Response memberResponse = ABMTestHelper.CreatePoolStorageDomainMember(
storageDomainResponse.ResponsePayload.Id,
poolPartitionResponse.ResponsePayload.Id,
client);
//Create data persistence rule
PutDataPersistenceRuleSpectraS3Response dataPersistenceResponse = ABMTestHelper.CreateDataPersistenceRule(
dataPolicyId,
storageDomainResponse.ResponsePayload.Id,
client);
return new TempStorageIds(memberResponse.ResponsePayload.Id, dataPersistenceResponse.ResponsePayload.Id);
}
public ReadTransferItemSource(
Action<TimeSpan> wait,
IDs3Client client,
JobResponse initialJobResponse)
: this(wait, client, -1, initialJobResponse)
{
}
/// <summary>
/// Creates a pool partition with the specified name and pool type
/// </summary>
/// <param name="storageDomainId"></param>
/// <param name="poolPartitionId"></param>
/// <param name="client"></param>
/// <returns></returns>
public static PutPoolStorageDomainMemberSpectraS3Response CreatePoolStorageDomainMember(
Guid storageDomainId,
Guid poolPartitionId,
IDs3Client client)
{
return client.PutPoolStorageDomainMemberSpectraS3(new PutPoolStorageDomainMemberSpectraS3Request(poolPartitionId, storageDomainId));
}
public void Transfer(IDs3Client client, string bucketName, string objectName, long blobOffset, Guid jobId,
IEnumerable<Range> ranges, Stream stream, IMetadataAccess metadataAccess,
Action<string, IDictionary<string, string>> metadataListener, int objectTransferAttempts)
{
var currentTry = 0;
while (true)
{
var request = new PutObjectRequest(bucketName, objectName, stream)
.WithJob(jobId)
.WithOffset(blobOffset);
if (blobOffset == 0 && metadataAccess != null)
{
request.WithMetadata(MetadataUtils.GetUriEscapeMetadata(metadataAccess.GetMetadataValue(objectName)));
}
try
{
client.PutObject(request);
return;
}
catch (Exception ex)
{
if (ExceptionClassifier.IsRecoverableException(ex))
{
BestEffort.ModifyForRetry(stream, objectTransferAttempts, ref currentTry, request.ObjectName, request.Offset.Value, ex);
}
else
{
throw;
}
}
}
}
public ReadTransferItemSource(
IDs3Client client,
int retryAfter,
JobResponse initialJobResponse)
: this(Thread.Sleep, client, retryAfter, initialJobResponse)
{
}
private void TransferBlob(IDs3Client client, Blob blob)
{
var ranges = this._rangesForRequests[blob];
var getLength = ranges.Sum(r => r.Length);
this._transferrer.Transfer(
client,
this.BucketName,
blob.Context,
blob.Range.Start,
this.JobId,
ranges,
new StreamTranslator <TItem, Blob>(
this._rangeTranslator,
this._resourceStore,
blob,
getLength
)
);
var fullRequestRange = ContextRange.Create(Range.ByLength(0L, getLength), blob);
foreach (var contextRange in this._rangeTranslator.Translate(fullRequestRange))
{
this._itemTracker.CompleteRange(contextRange);
}
}
public void Transfer(IDs3Client client, string bucketName, string objectName, long blobOffset, Guid jobId,
IEnumerable<Range> ranges, Stream stream, IMetadataAccess metadataAccess,
Action<string, IDictionary<string, string>> metadataListener, int objectTransferAttempts)
{
var currentTry = 0;
var transferrer = _transferrer;
var tRanges = ranges;
while (true)
{
try
{
transferrer.Transfer(client, bucketName, objectName, blobOffset, jobId, tRanges, stream,
metadataAccess, metadataListener, objectTransferAttempts);
return;
}
catch (Ds3ContentLengthNotMatch ex)
{
BestEffort.ModifyForRetry(stream, objectTransferAttempts, ref currentTry, objectName, blobOffset, ref tRanges, ref transferrer, ex);
}
catch (Exception ex)
{
if (ExceptionClassifier.IsRecoverableException(ex))
{
BestEffort.ModifyForRetry(stream, _retries, ref currentTry, objectName, blobOffset, ex);
}
else
{
throw;
}
}
}
}
public WriteTransferItemSource(
Action <TimeSpan> wait,
IDs3Client client,
JobResponse jobResponse)
{
this._wait = wait;
this._client = client;
this._jobResponse = jobResponse;
}
public WriteTransferItemSource(
Action<TimeSpan> wait,
IDs3Client client,
JobResponse jobResponse)
{
this._wait = wait;
this._client = client;
this._jobResponse = jobResponse;
}
public void Transfer(IDs3Client client, string bucketName, string objectName, long blobOffset, Guid jobId,
IEnumerable<Range> ranges, Stream stream, IMetadataAccess metadataAccess,
Action<string, IDictionary<string, string>> metadataListener, int objectTransferAttempts)
{
var response = client.GetObject(new GetObjectRequest(bucketName, objectName, stream, jobId, blobOffset));
if (blobOffset == 0)
{
metadataListener?.Invoke(objectName, MetadataUtils.GetUriUnEscapeMetadata(response.Metadata));
}
}
public ReadTransferItemSource(
Action<TimeSpan> wait,
IDs3Client client,
JobResponse initialJobResponse)
{
this._wait = wait;
this._client = client;
this._jobId = initialJobResponse.JobId;
this._blobsRemaining = new HashSet<Blob>(Blob.Convert(initialJobResponse));
}
public ReadTransferItemSource(
Action <TimeSpan> wait,
IDs3Client client,
JobResponse initialJobResponse)
{
this._wait = wait;
this._client = client;
this._jobId = initialJobResponse.JobId;
this._blobsRemaining = new HashSet <Blob>(Blob.Convert(initialJobResponse));
}
public IEnumerable<TransferItem> GetNextTransferItems(IDs3Client client, MasterObjectList jobResponse)
{
var clientFactory = client.BuildFactory(jobResponse.Nodes);
return
from chunk in jobResponse.Objects
let transferClient = clientFactory.GetClientForNodeId(chunk.NodeId)
from jobObject in chunk.ObjectsList
where !(bool)jobObject.InCache
select new TransferItem(transferClient, Blob.Convert(jobObject));
}
/// <summary>
/// Creates a data persistence rule to link the specified data policy and storage domain
/// </summary>
/// <param name="dataPolicyId"></param>
/// <param name="storageDomainId"></param>
/// <param name="client"></param>
/// <returns></returns>
public static PutDataPersistenceRuleSpectraS3Response CreateDataPersistenceRule(
Guid dataPolicyId,
Guid storageDomainId,
IDs3Client client)
{
return client.PutDataPersistenceRuleSpectraS3(new PutDataPersistenceRuleSpectraS3Request(
dataPolicyId,
DataIsolationLevel.STANDARD,
storageDomainId,
DataPersistenceRuleType.PERMANENT));
}
public void Transfer(
IDs3Client client,
string bucketName,
string objectName,
long blobOffset,
Guid jobId,
IEnumerable<Range> ranges,
Stream stream)
{
client.GetObject(new GetObjectRequest(bucketName, objectName, jobId, blobOffset, stream));
}
public void Transfer(
IDs3Client client,
string bucketName,
string objectName,
long blobOffset,
Guid jobId,
IEnumerable <Range> ranges,
Stream stream)
{
client.GetObject(new GetObjectRequest(bucketName, objectName, jobId, blobOffset, stream));
}
public Ds3ExampleClient(string endpoint, Credentials credentials, string proxy)
{
Ds3Builder builder = new Ds3Builder(endpoint, credentials);
if (!string.IsNullOrEmpty(proxy))
{
builder.WithProxy(new Uri(proxy));
}
_client = builder.Build();
// Set up the high-level abstractions.
_helpers = new Ds3ClientHelpers(_client);
}
public static void DeleteBucket(IDs3Client client, string bucketName)
{
if (client.HeadBucket(new HeadBucketRequest(bucketName)).Status == HeadBucketResponse.StatusType.DoesntExist)
{
return;
}
IDs3ClientHelpers helpers = new Ds3ClientHelpers(client);
var objs = helpers.ListObjects(bucketName);
client.DeleteObjectList(new DeleteObjectListRequest(bucketName, objs));
client.DeleteBucket(new DeleteBucketRequest(bucketName));
}
/// <summary>
/// This will get the object and return the name of the temporary file it was written to.
/// It is up to the caller to delete the temporary file
/// </summary>
public static string GetSingleObject(IDs3Client client, string bucketName, string objectName, int retries = 5)
{
string tempFilename = Path.GetTempFileName();
using (Stream fileStream = new FileStream(tempFilename, FileMode.Truncate, FileAccess.Write))
{
IDs3ClientHelpers helper = new Ds3ClientHelpers(client, getObjectRetries: retries);
var job = helper.StartReadJob(bucketName, new List<Ds3Object>{ new Ds3Object(objectName, null)});
job.Transfer(key => fileStream);
return tempFilename;
}
}
public void Setup()
{
try
{
_client = Ds3TestUtils.CreateClient();
Guid dataPolicyId = TempStorageUtil.SetupDataPolicy(FixtureName, false, ChecksumType.Type.MD5, _client);
EnvStorageIds = TempStorageUtil.Setup(FixtureName, dataPolicyId, _client);
}
catch (Exception)
{
// So long as any SetUp method runs without error, the TearDown method is guaranteed to run.
// It will not run if a SetUp method fails or throws an exception.
Teardown();
throw;
}
}
internal static string GetSingleObjectWithRange(IDs3Client client, string bucketName, string objectName, Range range)
{
string tempFilename = Path.GetTempFileName();
using (Stream fileStream = new FileStream(tempFilename, FileMode.Truncate, FileAccess.Write))
{
IDs3ClientHelpers helper = new Ds3ClientHelpers(client);
var job = helper.StartPartialReadJob(bucketName, new List<string>(), new List<Ds3PartialObject> { new Ds3PartialObject(range, objectName) });
job.Transfer(key => fileStream);
return tempFilename;
}
}
private JobObjectList AllocateChunk(IDs3Client client, Guid chunkId)
{
JobObjectList chunk = null;
var chunkGone = false;
while (chunk == null && !chunkGone)
{
// This is an idempotent operation, so we don't care if it's already allocated.
client
.AllocateJobChunk(new AllocateJobChunkRequest(chunkId))
.Match(
allocatedChunk => { chunk = allocatedChunk; },
this._wait,
() => { chunkGone = true; }
);
}
return chunk;
}
public void Startup()
{
try
{
this._client = Ds3Builder.FromEnv().WithProxy(new Uri("http://localhost:9090")).Build();
var dataPolicyId = TempStorageUtil.SetupDataPolicy(FixtureName, false, ChecksumType.Type.MD5, _client);
_envStorageIds = TempStorageUtil.Setup(FixtureName, dataPolicyId, _client);
}
catch (Exception)
{
// So long as any SetUp method runs without error, the TearDown method is guaranteed to run.
// It will not run if a SetUp method fails or throws an exception.
Teardown();
throw;
}
}
public IEnumerable<TransferItem> GetNextTransferItems(IDs3Client client, MasterObjectList jobResponse)
{
this._client = client;
this._jobResponse = jobResponse;
lock (this._lock)
{
_allocatedChunks = new Dictionary<Guid, bool>(jobResponse.Objects.Select(chunk => chunk)
.ToDictionary(
chunk => chunk.ChunkId,
chunk => false));
this._streamToChunkDictionary = GetStreamToChunksDictionary();
}
// Flatten all batches into a single enumerable.
return EnumerateTransferItemBatches().SelectMany(it => it);
}
private JobObjectList AllocateChunk(IDs3Client client, Guid chunkId)
{
JobObjectList chunk = null;
var chunkGone = false;
while (chunk == null && !chunkGone)
{
// This is an idempotent operation, so we don't care if it's already allocated.
client
.AllocateJobChunk(new AllocateJobChunkRequest(chunkId))
.Match(
allocatedChunk => { chunk = allocatedChunk; },
this._wait,
() => { chunkGone = true; }
);
}
return(chunk);
}
public IEnumerable<TransferItem> GetNextTransferItems(IDs3Client client, MasterObjectList jobResponse)
{
this._client = client;
this._jobResponse = jobResponse;
if (_withAggregation)
{
_jobResponse.Objects = GetObjectsNotInCache();
}
lock (this._chunksRemainingLock)
{
_toAllocateChunks = new HashSet<Guid>(jobResponse.Objects.Select(chunk => chunk.ChunkId));
}
// Flatten all batches into a single enumerable.
return EnumerateTransferItemBatches().SelectMany(it => it);
}
public void Transfer(
IDs3Client client,
string bucketName,
string objectName,
long blobOffset,
Guid jobId,
IEnumerable<Range> ranges,
Stream stream)
{
var currentTry = 0;
var transferrer = _transferrer;
var _ranges = ranges;
while (true)
{
try
{
transferrer.Transfer(client, bucketName, objectName, blobOffset, jobId, _ranges, stream);
break;
}
catch (Ds3ContentLengthNotMatch exception)
{
if (_retries != -1 && currentTry >= _retries)
{
throw new Ds3NoMoreRetriesException(Resources.TooManyRetriesForPartialData, exception, currentTry);
}
// Issue a partial get for the remainder of the request
// Seek back one byte to make sure that the connection did not fail part way through a byte
stream.Seek(-1, SeekOrigin.Current);
_ranges = JobsUtil.RetryRanges(_ranges, exception.BytesRead, exception.ContentLength);
transferrer = new PartialReadTransferrer();
currentTry++;
}
}
}
public Ds3ClientHelpers(IDs3Client client)
{
this._client = client;
}
public TransferItem(IDs3Client client, Blob blob)
{
this.Client = client;
this.Blob = blob;
}
public void Setup()
{
Ds3Builder builder = Ds3Builder.FromEnv();
_client = builder.Build();
}
public void Startup()
{
try
{
this._client = Ds3TestUtils.CreateClient(this._copyBufferSize);
this._helpers = new Ds3ClientHelpers(this._client);
var dataPolicyId = TempStorageUtil.SetupDataPolicy(FixtureName, false, ChecksumType.Type.MD5, _client);
_envStorageIds = TempStorageUtil.Setup(FixtureName, dataPolicyId, _client);
}
catch (Exception)
{
// So long as any SetUp method runs without error, the TearDown method is guaranteed to run.
// It will not run if a SetUp method fails or throws an exception.
Teardown();
throw;
}
}
public WriteTransferItemSource(
IDs3Client client,
JobResponse jobResponse)
: this(Thread.Sleep, client, jobResponse)
{
}
/// <summary>
/// Tears down the temporary test environment
/// </summary>
/// <param name="fixtureName"></param>
/// <param name="ids"></param>
/// <param name="client"></param>
public static void TearDown(
string fixtureName,
TempStorageIds ids,
IDs3Client client)
{
//try to delete as much as possible
var exceptionsThrown = new Queue<Exception>();
try
{
ABMTestHelper.DeleteDataPersistenceRule(ids.DataPersistenceRuleId, client);
}
catch (Exception ex)
{
exceptionsThrown.Enqueue(ex);
}
try
{
ABMTestHelper.DeleteDataPolicy(fixtureName, client);
}
catch (Exception ex)
{
exceptionsThrown.Enqueue(ex);
}
try
{
ABMTestHelper.DeleteStorageDomainMember(ids.StorageDomainMemberId, client);
}
catch (Exception ex)
{
exceptionsThrown.Enqueue(ex);
}
try
{
ABMTestHelper.DeleteStorageDomain(fixtureName, client);
}
catch (Exception ex)
{
exceptionsThrown.Enqueue(ex);
}
try
{
ABMTestHelper.DeletePoolPartition(fixtureName, client);
}
catch (Exception ex)
{
exceptionsThrown.Enqueue(ex);
}
if (exceptionsThrown.Count > 0)
{
throw new AggregateException(exceptionsThrown);
}
}
public TransferItem(IDs3Client client, Blob blob)
{
this.Client = client;
this.Blob = blob;
}
public ReadTransferItemSource(
IDs3Client client,
JobResponse initialJobResponse)
: this(Thread.Sleep, client, initialJobResponse)
{
}
public Ds3ClientFactory(IDs3Client client, IEnumerable<JobNode> nodes)
{
this._client = client;
this._nodes = nodes.ToDictionary(node => node.Id);
}
public void Setup()
{
_client = Ds3TestUtils.CreateClient();
}
public static void LoadTestData(IDs3Client client, string bucketName)
{
IDs3ClientHelpers helper = new Ds3ClientHelpers(client);
helper.EnsureBucketExists(bucketName);
var job = helper.StartWriteJob(bucketName, Objects);
job.Transfer(key => ReadResource(key));
}
private Objects AllocateChunk(IDs3Client client, Guid chunkId, IDictionary<Guid, bool> allocatedChunks)
{
Objects chunk = null;
var chunkGone = false;
while (chunk == null && !chunkGone)
{
client
.AllocateJobChunkSpectraS3(new AllocateJobChunkSpectraS3Request(chunkId))
.Match(
allocatedChunk =>
{
allocatedChunks[chunkId] = true; //mark this chunk as allocated
chunk = allocatedChunk;
this.RetryAfer.Reset(); // Reset the number of retries to the initial value
},
this.RetryAfer.RetryAfterFunc,
() =>
{
chunkGone = true;
}
);
}
return chunk;
}
public Ds3ClientFactory(IDs3Client client, IEnumerable <Node> nodes)
{
this._client = client;
this._nodes = nodes.ToDictionary(node => node.Id);
}
