public BlobClientTests()
{
BlobClient =
new StorageAccountFactory().GetStorageAccount("UseInMemoryDevelopmentStorage=true")
.CreateBlobClient();
BlobClientBaseAddress = Constants.DevelopmentStorageBlobClientEndpoint;
}
public BlobContainerTests()
{
BlobClient =
StorageAccountFactory.Create()
.GetStorageAccount(PlatformTestConstants.PlatformTestStorageConnectionString)
.CreateBlobClient();
}
public BlobDirectoryTests()
{
BlobContainer = StorageAccountFactory.Create()
.GetStorageAccount(PlatformTestConstants.PlatformTestStorageConnectionString)
.CreateBlobClient()
.GetBlobContainer("blobcontainertestdirectory");
}
public static async Task UpdateMessageAsync(UpdateCommandData updateCommandData)
{
var storageAccount = StorageAccountFactory.Get(updateCommandData);
// Create the queue client.
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue.
CloudQueue queue = queueClient.GetQueueReference(updateCommandData.Queue);
// Get
var retrievedMessage = await queue.GetMessageAsync();
if (retrievedMessage != null)
{
Console.WriteLine($"Readed: {retrievedMessage.AsString}");
//Update with new text
retrievedMessage.SetMessageContent(updateCommandData.UpdateMessage);
await queue.UpdateMessageAsync(retrievedMessage,
TimeSpan.FromSeconds(60.0), // Make it invisible for another 60 seconds.
MessageUpdateFields.Content | MessageUpdateFields.Visibility);
Console.WriteLine($"Updated: {retrievedMessage.AsString}");
}
else
{
Console.WriteLine("No messages on queue");
}
}
public static async Task DeQueueMessageAsync(BaseCommandData commandData)
{
var storageAccount = StorageAccountFactory.Get(commandData);
// Create the queue client.
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue.
CloudQueue queue = queueClient.GetQueueReference(commandData.Queue);
// Read
var retrievedMessage = await queue.GetMessageAsync();
if (retrievedMessage != null)
{
Console.WriteLine($"Readed: {retrievedMessage.AsString}");
//Do stuff...
await queue.DeleteMessageAsync(retrievedMessage);
Console.WriteLine($"Deleted: {retrievedMessage.AsString}");
}
else
{
Console.WriteLine("No messages on queue");
}
}
public async Task DeQueueMessageAsync(BaseCommandData commandData)
{
var storageAccount = StorageAccountFactory.Get(commandData);
// Create the queue client.
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue.
CloudQueue queue = queueClient.GetQueueReference(commandData.Queue);
// Read
var retrievedMessage = await queue.GetMessageAsync(TimeSpan.FromSeconds(30),
Options,
Context);
Console.WriteLine($"Readed: {retrievedMessage.AsString}");
//Do stuff...
await queue.DeleteMessageAsync(retrievedMessage,
new QueueRequestOptions(),
Context);
Console.WriteLine($"Deleted: {retrievedMessage.AsString}");
}
public void ConfigureServices(IServiceCollection services)
{
services.AddControllers();
var azureStorageConnectionString = Configuration.GetValue <string>(AzureStorageConnectionStringKey);
services
.ConfigureMapper()
.ConfigureServices(Configuration.GetConnectionString(ConnectionStringKey))
.ConfigureAzureStorage(azureStorageConnectionString)
.ConfigureLogger(StorageAccountFactory.Create(azureStorageConnectionString));
ConfigureSwagger(services);
}
public static async Task CountMessagesAsync(BaseCommandData commandData)
{
var storageAccount = StorageAccountFactory.Get(commandData);
// Create the queue client.
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue.
CloudQueue queue = queueClient.GetQueueReference(commandData.Queue);
// Fetch
await queue.FetchAttributesAsync();
// Display number of messages.
Console.WriteLine($"Number of messages in queue: {queue.ApproximateMessageCount}");
}
/// <summary>
/// Clean up container created by test
/// </summary>
private static void TestCleanUp(string containerName, StorageAccountFactory storageAccountFactory)
{
try
{
var container = storageAccountFactory.GetContainer(containerName);
container.DeleteIfExists();
}
catch (Exception ex)
{
Utility.TraceSource.WriteError(
TraceType,
"Failed to delete container {0}: {1}",
containerName,
ex);
}
}
public static async Task InsertMessageAsync(InsertCommandData insertCommandData)
{
var storageAccount = StorageAccountFactory.Get(insertCommandData);
// Create the queue client.
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue.
CloudQueue queue = queueClient.GetQueueReference(insertCommandData.Queue);
// Create the queue if it doesn't already exist.
await queue.CreateIfNotExistsAsync();
// Create a message and add it to the queue.
CloudQueueMessage message = new CloudQueueMessage(insertCommandData.Message);
await queue.AddMessageAsync(message);
Console.WriteLine($"Inserted: {insertCommandData.Message}");
}
public static async Task PeekMessageAsync(BaseCommandData commandData)
{
var storageAccount = StorageAccountFactory.Get(commandData);
// Create the queue client.
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue.
CloudQueue queue = queueClient.GetQueueReference(commandData.Queue);
// Peek
var peekedMessage = await queue.PeekMessageAsync();
if (peekedMessage != null)
{
Console.WriteLine(peekedMessage.AsString);
}
else
{
Console.WriteLine("No messages on queue");
}
}
public static async Task DeQueueMessageAsync(BaseCommandData commandData)
{
var storageAccount = StorageAccountFactory.Get(commandData);
// Create the queue client.
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue.
CloudQueue queue = queueClient.GetQueueReference(commandData.Queue);
// Read batch
var readed = await queue.GetMessagesAsync(5, TimeSpan.FromMinutes(1), new QueueRequestOptions(), new OperationContext());
foreach (CloudQueueMessage retrievedMessage in readed)
{
Console.WriteLine($"Readed: {retrievedMessage.AsString}");
// Process all messages in less than 5 minutes, deleting each message after processing.
await queue.DeleteMessageAsync(retrievedMessage);
//Do stuff...
Console.WriteLine($"Deleted: {retrievedMessage.AsString}");
}
}
public static async Task ReadMessageAsync(BaseCommandData commandData)
{
var storageAccount = StorageAccountFactory.Get(commandData);
// Create the queue client.
CloudQueueClient queueClient = storageAccount.CreateCloudQueueClient();
// Retrieve a reference to a queue.
CloudQueue queue = queueClient.GetQueueReference(commandData.Queue);
// Read (it will make themessage invisible toother for 30 seconds)
// If message is deleted it will come back visible (as is it would be queued again).
// In this case message DequeueCount property is inreased by 1
var retrievedMessage = await queue.GetMessageAsync();
if (retrievedMessage != null)
{
Console.WriteLine($"Readed: {retrievedMessage.AsString}");
}
else
{
Console.WriteLine("No messages on queue");
}
}
public GithubActionsWithAnAzureFunction()
{
var config = new Config();
var azureRegion = config.Require("azureRegion");
var workloadResources = "ghares";
var workloadFunction = "ghafunc";
var resourceGroupForResources = ResourceGroupFactory.Create(GetName("rg", workloadResources, azureRegion));
var resourceGroupForFunction = ResourceGroupFactory.Create(GetName("rg", workloadFunction, azureRegion));
// Storage account is required by Function App.
// Also, we will upload the function code to the same storage account.
var storageAccount = StorageAccountFactory.Create(resourceGroupForResources, $"st{workloadResources}{Deployment.Instance.StackName}{azureRegion}001");
// Export the primary key of the Storage Account
this.PrimaryStorageKey = Output.Tuple(resourceGroupForResources.Name, storageAccount.Name).Apply(names =>
Output.CreateSecret(GetStorageAccountPrimaryKey(names.Item1, names.Item2)));
var functionSku = config.Require("functionSku").Split('/');
// Define a Consumption Plan for the Function App.
// You can change the SKU to Premium or App Service Plan if needed.
var appServicePlanName = GetName("plan", workloadFunction, azureRegion);
var appServicePlan = new AppServicePlan(appServicePlanName, new AppServicePlanArgs
{
Name = appServicePlanName,
ResourceGroupName = resourceGroupForResources.Name,
// Run on Linux
Kind = "Linux",
// Consumption plan SKU
Sku = new SkuDescriptionArgs
{
Tier = functionSku[0],
Name = functionSku[1]
},
// For Linux, you need to change the plan to have Reserved = true property.
Reserved = true
}, new CustomResourceOptions {
DeleteBeforeReplace = true
});
var container = new BlobContainer("zips-container", new BlobContainerArgs
{
AccountName = storageAccount.Name,
PublicAccess = PublicAccess.None,
ResourceGroupName = resourceGroupForResources.Name,
}, new CustomResourceOptions {
DeleteBeforeReplace = true
});
var functionAppPublishFolder = Path.Combine("C:\\dev\\swiftbit\\GithubActionsWithAzureFunction\\src", "GithubActions.AzureFunction", "bin", "Release", "netcoreapp3.1", "publish");
var blob = new Blob("zip", new BlobArgs
{
AccountName = storageAccount.Name,
ContainerName = container.Name,
ResourceGroupName = resourceGroupForResources.Name,
Type = BlobType.Block,
Source = new FileArchive(functionAppPublishFolder)
}, new CustomResourceOptions {
DeleteBeforeReplace = true
});
var codeBlobUrl = SignedBlobReadUrl(blob, container, storageAccount, resourceGroupForResources);
// Application insights
var appInsightsName = GetName("appi", workloadFunction, azureRegion);
var appInsights = new Component(appInsightsName, new ComponentArgs
{
ResourceName = appInsightsName,
ApplicationType = ApplicationType.Web,
Kind = "web",
ResourceGroupName = resourceGroupForResources.Name,
}, new CustomResourceOptions {
DeleteBeforeReplace = true
});
var funcName = GetName("func", workloadFunction, azureRegion);
var app = new WebApp(funcName, new WebAppArgs
{
Name = funcName,
Kind = "FunctionApp",
ResourceGroupName = resourceGroupForFunction.Name,
ServerFarmId = appServicePlan.Id,
SiteConfig = new SiteConfigArgs
{
AppSettings = new[]
{
new NameValuePairArgs {
Name = "AzureWebJobsStorage",
Value = GetConnectionString(resourceGroupForResources.Name, storageAccount.Name),
},
new NameValuePairArgs {
Name = "runtime",
Value = "dotnet",
},
new NameValuePairArgs {
Name = "FUNCTIONS_WORKER_RUNTIME",
Value = "dotnet",
},
new NameValuePairArgs {
Name = "WEBSITE_RUN_FROM_PACKAGE",
Value = codeBlobUrl,
},
new NameValuePairArgs {
Name = "APPLICATIONINSIGHTS_CONNECTION_STRING",
Value = Output.Format($"InstrumentationKey={appInsights.InstrumentationKey}"),
},
new NameValuePairArgs {
Name = "Greeting",
Value = "Hi",
},
},
},
});
this.Endpoint = Output.Format($"https://{app.DefaultHostName}/api/HelloFunction?name=Pulumi");
}
public BlobDirectoryTests()
{
BlobContainer =
new StorageAccountFactory().GetStorageAccount("UseInMemoryDevelopmentStorage=true")
.CreateBlobClient().GetBlobContainer("BlobContainerTestDirectory");
}
public StorageFactoryTests()
{
StorageFactory = new StorageAccountFactory();
}
private static void DownloadBlobs(string downloadedBlobsPath, string containerName, StorageAccountFactory storageAccountFactory)
{
CloudStorageAccount storageAccount = null;
try
{
storageAccount = storageAccountFactory.GetStorageAccount();
// Create the blob client object
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
// Get the container reference
CloudBlobContainer container = blobClient.GetContainerReference(containerName);
CloudBlobDirectory directory = container.GetDirectoryReference(string.Join("/", TestFabricNodeName, "Fabric"));
BlobContinuationToken continuationToken = null;
BlobRequestOptions requestOptions = new BlobRequestOptions();
OperationContext operationContext = new OperationContext();
var blobResultSegment = directory.ListBlobsSegmented(true, BlobListingDetails.All, null, continuationToken, requestOptions, operationContext);
int zipSequence = 1;
foreach (ICloudBlob blob in blobResultSegment.Results)
{
try
{
var localFilePath = Path.Combine(downloadedBlobsPath, string.Concat(zipSequence++, ".dtr.zip"));
blob.DownloadToFile(localFilePath, FileMode.Create);
var outputFilePath = localFilePath.Substring(0, localFilePath.Length - 4);
ZipFile.ExtractToDirectory(localFilePath, outputFilePath);
DirectoryInfo outputDir = new DirectoryInfo(outputFilePath);
foreach (var file in outputDir.GetFiles())
{
File.Copy(file.FullName, Path.Combine(downloadedBlobsPath, file.Name));
File.Delete(localFilePath);
Directory.Delete(outputDir.FullName, true);
}
}
catch (Exception ex)
{
Utility.TraceSource.WriteError(
TraceType,
"Failed to download and unzip blob: {0}",
ex);
}
}
}
finally
{
// Replace real key with a bogus key and then clear out the real key.
if (storageAccount != null)
{
storageAccount.Credentials.UpdateKey(
AzureUtility.DevelopmentStorageAccountWellKnownKey,
storageAccount.Credentials.KeyName);
}
}
}
private static void VerifyEvents(string downloadedBlobsPath, string containerName, StorageAccountFactory storageAccountFactory)
{
DownloadBlobs(downloadedBlobsPath, containerName, storageAccountFactory);
TestUtility.AssertDtrPartsEqual(string.Empty, expectedOutputDataFolderPath, string.Empty, downloadedBlobsPath);
}
private static void CreateEtlInMemoryProducerWithAzureBlobConsumer(
string logDirectory,
ITraceFileEventReaderFactory traceFileEventReaderFactory,
out EtlInMemoryProducer etlInMemoryProducer,
out string containerName,
out StorageAccountFactory storageAccountFactory,
AccessCondition uploadStreamAccessCondition = null,
AccessCondition uploadFileAccessCondition = null)
{
var mockDiskSpaceManager = TestUtility.MockRepository.Create <DiskSpaceManager>();
var etlInMemoryProducerSettings = new EtlInMemoryProducerSettings(
true,
TimeSpan.FromSeconds(1),
TimeSpan.FromDays(3000),
WinFabricEtlType.DefaultEtl,
logDirectory,
TestEtlFilePatterns,
true);
var configReader = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReader>();
configReader.Setup(c => c.GetSettings()).Returns(etlInMemoryProducerSettings);
var mockTraceEventSourceFactory = TestUtility.MockRepository.Create <ITraceEventSourceFactory>();
mockTraceEventSourceFactory
.Setup(tsf => tsf.CreateTraceEventSource(It.IsAny <EventTask>()))
.Returns(new ErrorAndWarningFreeTraceEventSource());
var configStore = TestUtility.MockRepository.Create <IConfigStore>();
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "MaxDiskQuotaInMB"))
.Returns("100");
configStore
.Setup(cs => cs.ReadUnencryptedString("Diagnostics", "DiskFullSafetySpaceInMB"))
.Returns("0");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.EnabledParamName))
.Returns("true");
containerName = string.Format("{0}-{1}", "fabriclogs", Guid.NewGuid());
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.ContainerParamName))
.Returns(containerName);
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.FileSyncIntervalParamName))
.Returns("0.25");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DataDeletionAgeParamName))
.Returns("1");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.TestDataDeletionAgeParamName))
.Returns("0");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.LogFilterParamName))
.Returns("*.*:4");
configStore
.Setup(cs => cs.ReadUnencryptedString(TestConfigSectionName, AzureConstants.DeploymentId))
.Returns(AzureConstants.DefaultDeploymentId);
var accountKey = GetTestStorageAccountKey();
var isEncrypted = false;
var storageConnectionString = string.Format(@"xstore:DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", AccountName, accountKey);
configStore
.Setup(cs => cs.ReadString(TestConfigSectionName, AzureConstants.ConnectionStringParamName, out isEncrypted))
.Returns(storageConnectionString);
Utility.InitializeConfigStore(configStore.Object);
var mockConfigReaderFactory = TestUtility.MockRepository.Create <IEtlInMemoryProducerConfigReaderFactory>();
mockConfigReaderFactory.Setup(f => f.CreateEtlInMemoryProducerConfigReader(It.IsAny <FabricEvents.ExtensionsEvents>(), It.IsAny <string>()))
.Returns(configReader.Object);
// Create the Azure Blob Uploader consumer
ConfigReader.AddAppConfig(Utility.WindowsFabricApplicationInstanceId, null);
var consumerInitParam = new ConsumerInitializationParameters(
Utility.WindowsFabricApplicationInstanceId,
TestConfigSectionName,
TestFabricNodeId,
TestFabricNodeName,
Utility.LogDirectory,
Utility.DcaWorkFolder,
new DiskSpaceManager());
// Save storage connection for clean up
var azureUtility = new AzureUtility(new FabricEvents.ExtensionsEvents(FabricEvents.Tasks.FabricDCA), TraceType);
storageAccountFactory = azureUtility.GetStorageAccountFactory(
new ConfigReader(consumerInitParam.ApplicationInstanceId),
TestConfigSectionName,
AzureConstants.ConnectionStringParamName);
var azureBlobConsumer = new AzureBlobEtwUploader(consumerInitParam, uploadStreamAccessCondition, uploadFileAccessCondition);
var etlToInMemoryBufferWriter = azureBlobConsumer.GetDataSink();
var producerInitParam = new ProducerInitializationParameters
{
ConsumerSinks = new[] { etlToInMemoryBufferWriter }
};
// Create the in-memory producer
etlInMemoryProducer = new EtlInMemoryProducer(
mockDiskSpaceManager.Object,
mockConfigReaderFactory.Object,
traceFileEventReaderFactory,
mockTraceEventSourceFactory.Object,
producerInitParam);
}
public BlobTests()
{
BlobClient =
new StorageAccountFactory().GetStorageAccount("UseInMemoryDevelopmentStorage=true")
.CreateBlobClient();
}
public BlobClientTests()
{
BlobClient = StorageAccountFactory.Create().GetStorageAccount(PlatformTestConstants.PlatformTestStorageConnectionString).CreateBlobClient();
BlobClientBaseAddress = PlatformTestConstants.PlatformTestBlobClientBaseAddress;
}
public StorageFactoryTests()
{
StorageFactory = StorageAccountFactory.Create();
}
public StorageAccountTests()
{
StorageAccount = (StorageAccount)StorageAccountFactory.Create().GetStorageAccount(PlatformTestConstants.PlatformTestStorageConnectionString);
}
