public async Task <bool> ExecuteAsync(CancellationToken ct)
{
ParseConnectionString();
// If the connection string AND AccountKey & AccountName are provided, error out.
if (Log.HasLoggedErrors)
{
return(false);
}
Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
ContainerName, AccountName, DownloadDirectory);
try
{
List <string> blobNames = new List <string>();
if (BlobNames == null)
{
ListAzureBlobs listAzureBlobs = new ListAzureBlobs()
{
AccountName = AccountName,
AccountKey = AccountKey,
ContainerName = ContainerName,
FilterBlobNames = BlobNamePrefix,
BuildEngine = this.BuildEngine,
HostObject = this.HostObject
};
listAzureBlobs.Execute();
blobNames = listAzureBlobs.BlobNames.ToList();
}
else
{
blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>();
if (BlobNamePrefix != null)
{
blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>();
}
}
if (BlobNameExtension != null)
{
blobNames = blobNames.Where(b => Path.GetExtension(b) == BlobNameExtension).ToList <string>();
}
if (!Directory.Exists(DownloadDirectory))
{
Directory.CreateDirectory(DownloadDirectory);
}
using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
using (HttpClient client = new HttpClient())
{
client.Timeout = TimeSpan.FromMinutes(10);
await Task.WhenAll(blobNames.Select(item => DownloadItem(client, ct, item, clientThrottle)));
}
}
catch (Exception e)
{
Log.LogError(e.ToString());
}
return(!Log.HasLoggedErrors);
}
public Facts(ITestOutputHelper output)
{
CloudBlobClient = new Mock <ICloudBlobClient>();
CloudBlobContainer = new Mock <ICloudBlobContainer>();
CloudBlob = new Mock <ISimpleCloudBlob>();
Options = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >();
TelemetryService = new Mock <IAzureSearchTelemetryService>();
Logger = output.GetLogger <PopularityTransferDataClient>();
Config = new AzureSearchJobConfiguration
{
StorageContainer = "unit-test-container",
};
ETag = "\"some-etag\"";
AccessCondition = new Mock <IAccessCondition>();
StringCache = new StringCache();
ReplaceLatestIndexedPopularityTransfersDurationMetric = new Mock <IDisposable>();
Options
.Setup(x => x.Value)
.Returns(() => Config);
CloudBlobClient
.Setup(x => x.GetContainerReference(It.IsAny <string>()))
.Returns(() => CloudBlobContainer.Object);
CloudBlobContainer
.Setup(x => x.GetBlobReference(It.IsAny <string>()))
.Returns(() => CloudBlob.Object)
.Callback <string>(x => BlobNames.Add(x));
CloudBlob
.Setup(x => x.ETag)
.Returns(ETag);
CloudBlob
.Setup(x => x.OpenWriteAsync(It.IsAny <AccessCondition>()))
.ReturnsAsync(() => new RecordingStream(bytes =>
{
SavedBytes.Add(bytes);
SavedStrings.Add(Encoding.UTF8.GetString(bytes));
}));
CloudBlob
.Setup(x => x.Properties)
.Returns(new CloudBlockBlob(new Uri("https://example/blob")).Properties);
TelemetryService
.Setup(x => x.TrackReplaceLatestIndexedPopularityTransfers(It.IsAny <int>()))
.Returns(ReplaceLatestIndexedPopularityTransfersDurationMetric.Object);
Target = new PopularityTransferDataClient(
CloudBlobClient.Object,
Options.Object,
TelemetryService.Object,
Logger);
}
/// <inheritdoc />
public async Task <string> EnqueueAsync(T message, CancellationToken cancellationToken)
{
await _blobContainer.CreateIfNotExistsAsync(cancellationToken);
string blobName = BlobNames.GetConflictFreeDateTimeBasedBlobName();
IStorageBlockBlob blob = _blobContainer.GetBlockBlobReference(blobName);
message.AddMetadata(blob.Metadata);
string messageBody = JsonConvert.SerializeObject(message, JsonSerialization.Settings);
await blob.UploadTextAsync(messageBody, cancellationToken : cancellationToken);
return(blobName);
}
public void Write(IndexerLogEntry entry)
{
if (entry == null)
{
return;
}
string logBlobName = BlobNames.GetConflictFreeDateTimeBasedBlobName(entry.Date);
entry.Id = logBlobName;
string logEntryAsJson = JsonConvert.SerializeObject(entry, JsonSerialization.Settings);
_logsContainer.CreateIfNotExists();
CloudBlockBlob logBlob = _logsContainer.GetBlockBlobReference(_containerDirectory + "/" + logBlobName);
logBlob.Metadata[BlobLogEntryKeys.TitleKey] = entry.Title;
logBlob.Metadata[BlobLogEntryKeys.LogDate] = entry.Date.ToString(CultureInfo.InvariantCulture);
logBlob.UploadText(logEntryAsJson);
}
