示例#1
0
        public async Task <bool> ExecuteAsync(CancellationToken ct)
        {
            ParseConnectionString();
            // If the connection string AND AccountKey & AccountName are provided, error out.
            if (Log.HasLoggedErrors)
            {
                return(false);
            }

            Log.LogMessage(MessageImportance.Normal, "Downloading contents of container {0} from storage account '{1}' to directory {2}.",
                           ContainerName, AccountName, DownloadDirectory);

            try
            {
                List <string> blobNames = new List <string>();
                if (BlobNames == null)
                {
                    ListAzureBlobs listAzureBlobs = new ListAzureBlobs()
                    {
                        AccountName     = AccountName,
                        AccountKey      = AccountKey,
                        ContainerName   = ContainerName,
                        FilterBlobNames = BlobNamePrefix,
                        BuildEngine     = this.BuildEngine,
                        HostObject      = this.HostObject
                    };
                    listAzureBlobs.Execute();
                    blobNames = listAzureBlobs.BlobNames.ToList();
                }
                else
                {
                    blobNames = BlobNames.Select(b => b.ItemSpec).ToList <string>();
                    if (BlobNamePrefix != null)
                    {
                        blobNames = blobNames.Where(b => b.StartsWith(BlobNamePrefix)).ToList <string>();
                    }
                }

                if (BlobNameExtension != null)
                {
                    blobNames = blobNames.Where(b => Path.GetExtension(b) == BlobNameExtension).ToList <string>();
                }

                if (!Directory.Exists(DownloadDirectory))
                {
                    Directory.CreateDirectory(DownloadDirectory);
                }
                using (var clientThrottle = new SemaphoreSlim(this.MaxClients, this.MaxClients))
                    using (HttpClient client = new HttpClient())
                    {
                        client.Timeout = TimeSpan.FromMinutes(10);
                        await Task.WhenAll(blobNames.Select(item => DownloadItem(client, ct, item, clientThrottle)));
                    }
            }
            catch (Exception e)
            {
                Log.LogError(e.ToString());
            }
            return(!Log.HasLoggedErrors);
        }
            public Facts(ITestOutputHelper output)
            {
                CloudBlobClient    = new Mock <ICloudBlobClient>();
                CloudBlobContainer = new Mock <ICloudBlobContainer>();
                CloudBlob          = new Mock <ISimpleCloudBlob>();
                Options            = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >();
                TelemetryService   = new Mock <IAzureSearchTelemetryService>();
                Logger             = output.GetLogger <PopularityTransferDataClient>();
                Config             = new AzureSearchJobConfiguration
                {
                    StorageContainer = "unit-test-container",
                };

                ETag            = "\"some-etag\"";
                AccessCondition = new Mock <IAccessCondition>();
                StringCache     = new StringCache();
                ReplaceLatestIndexedPopularityTransfersDurationMetric = new Mock <IDisposable>();

                Options
                .Setup(x => x.Value)
                .Returns(() => Config);
                CloudBlobClient
                .Setup(x => x.GetContainerReference(It.IsAny <string>()))
                .Returns(() => CloudBlobContainer.Object);
                CloudBlobContainer
                .Setup(x => x.GetBlobReference(It.IsAny <string>()))
                .Returns(() => CloudBlob.Object)
                .Callback <string>(x => BlobNames.Add(x));
                CloudBlob
                .Setup(x => x.ETag)
                .Returns(ETag);
                CloudBlob
                .Setup(x => x.OpenWriteAsync(It.IsAny <AccessCondition>()))
                .ReturnsAsync(() => new RecordingStream(bytes =>
                {
                    SavedBytes.Add(bytes);
                    SavedStrings.Add(Encoding.UTF8.GetString(bytes));
                }));
                CloudBlob
                .Setup(x => x.Properties)
                .Returns(new CloudBlockBlob(new Uri("https://example/blob")).Properties);

                TelemetryService
                .Setup(x => x.TrackReplaceLatestIndexedPopularityTransfers(It.IsAny <int>()))
                .Returns(ReplaceLatestIndexedPopularityTransfersDurationMetric.Object);

                Target = new PopularityTransferDataClient(
                    CloudBlobClient.Object,
                    Options.Object,
                    TelemetryService.Object,
                    Logger);
            }
示例#3
0
        /// <inheritdoc />
        public async Task <string> EnqueueAsync(T message, CancellationToken cancellationToken)
        {
            await _blobContainer.CreateIfNotExistsAsync(cancellationToken);

            string            blobName = BlobNames.GetConflictFreeDateTimeBasedBlobName();
            IStorageBlockBlob blob     = _blobContainer.GetBlockBlobReference(blobName);

            message.AddMetadata(blob.Metadata);
            string messageBody = JsonConvert.SerializeObject(message, JsonSerialization.Settings);
            await blob.UploadTextAsync(messageBody, cancellationToken : cancellationToken);

            return(blobName);
        }
示例#4
0
        public void Write(IndexerLogEntry entry)
        {
            if (entry == null)
            {
                return;
            }

            string logBlobName = BlobNames.GetConflictFreeDateTimeBasedBlobName(entry.Date);

            entry.Id = logBlobName;

            string logEntryAsJson = JsonConvert.SerializeObject(entry, JsonSerialization.Settings);

            _logsContainer.CreateIfNotExists();

            CloudBlockBlob logBlob = _logsContainer.GetBlockBlobReference(_containerDirectory + "/" + logBlobName);

            logBlob.Metadata[BlobLogEntryKeys.TitleKey] = entry.Title;
            logBlob.Metadata[BlobLogEntryKeys.LogDate]  = entry.Date.ToString(CultureInfo.InvariantCulture);

            logBlob.UploadText(logEntryAsJson);
        }