private void SetTestStorageAccount(PowerShell powershell)
{
if (String.IsNullOrEmpty(EnvConnectionStringInPowerShell))
{
PSCommand currentCommand = powershell.Commands.Clone();
string envConnStringScript = string.Format("$env:{0}", Test.Data.Get("EnvContextKey"));
powershell.AddScript(envConnStringScript);
Collection<PSObject> output = powershell.Invoke();
if (output.Count == 1)
{
EnvConnectionStringInPowerShell = output[0].BaseObject.ToString();
powershell.Commands = currentCommand;
}
else
{
Test.AssertFail("Can not find the environment variable 'AZURE_STORAGE_CONNECTION_STRING' in powershell instance");
}
}
if (String.IsNullOrEmpty(EnvConnectionStringInPowerShell))
{
throw new ArgumentException("Please set the StorageConnectionString element of TestData.xml");
}
CommonStorageAccount = CloudStorageAccount.Parse(EnvConnectionStringInPowerShell);
CommonBlobHelper = new CloudBlobHelper(CommonStorageAccount);
}
public static void MyClassInitialize(TestContext testContext)
{
Trace.WriteLine("ClassInit");
Test.FullClassName = testContext.FullyQualifiedTestClassName;
StorageAccount = TestBase.GetCloudStorageAccountFromConfig();
//init the blob helper for blob related operations
BlobHelper = new CloudBlobHelper(StorageAccount);
// import module
string moduleFilePath = Test.Data.Get("ModuleFilePath");
if (moduleFilePath.Length > 0)
PowerShellAgent.ImportModule(moduleFilePath);
// $context = New-AzureStorageContext -ConnectionString ...
PowerShellAgent.SetStorageContext(StorageAccount.ToString(true));
BlockFilePath = Path.Combine(Test.Data.Get("TempDir"), FileUtil.GetSpecialFileName());
PageFilePath = Path.Combine(Test.Data.Get("TempDir"), FileUtil.GetSpecialFileName());
FileUtil.CreateDirIfNotExits(Path.GetDirectoryName(BlockFilePath));
FileUtil.CreateDirIfNotExits(Path.GetDirectoryName(PageFilePath));
// Generate block file and page file which are used for uploading
Helper.GenerateMediumFile(BlockFilePath, 1);
Helper.GenerateMediumFile(PageFilePath, 1);
}
//TODO remove it if it's useless
public CLICommonBVT(CloudStorageAccount StorageAccount, TestContext testContext)
{
CommonStorageAccount = StorageAccount;
testContextInstance = testContext;
//init the blob helper for blob related operations
CommonBlobHelper = new CloudBlobHelper(CommonStorageAccount);
GenerateBvtTempFiles();
}
//TODO remove it if it's useless
public CLICommonBVT(CloudStorageAccount StorageAccount, TestContext testContext)
{
CommonStorageAccount = StorageAccount;
testContextInstance = testContext;
//init the blob helper for blob related operations
CommonBlobHelper = new CloudBlobHelper(CommonStorageAccount);
GenerateBvtTempFiles();
}
public static void MyClassInitialize(TestContext testContext)
{
TestBase.TestClassInitialize(testContext);
BlobHelper = new CloudBlobHelper(StorageAccount);
FileHelper = new CloudFileHelper(StorageAccount);
//set the ConcurrentTaskCount field
PowerShellAgent.ConcurrentTaskCount = Environment.ProcessorCount * 8;
}
public CloudBlob GetCloudBlobReference(FileNode fileNode)
{
var container = this.BlobHelper.BlobClient.GetContainerReference(this.ContainerName);
var blobName = fileNode.GetURLRelativePath();
if (blobName.StartsWith("/"))
{
blobName = blobName.Substring(1, blobName.Length - 1);
}
return(CloudBlobHelper.GetCloudBlobReference(container, blobName, this.BlobType));
}
public static void MyClassInitialize(TestContext testContext)
{
Trace.WriteLine("ClassInit");
Test.FullClassName = testContext.FullyQualifiedTestClassName;
StorageAccount = TestBase.GetCloudStorageAccountFromConfig();
//init the blob helper for blob related operations
BlobHelper = new CloudBlobHelper(StorageAccount);
// import module
string moduleFilePath = Test.Data.Get("ModuleFilePath");
if (moduleFilePath.Length > 0)
PowerShellAgent.ImportModule(moduleFilePath);
// $context = New-AzureStorageContext -ConnectionString ...
PowerShellAgent.SetStorageContext(StorageAccount.ToString(true));
}
public static void CLIBlobFuncClassInit(TestContext testContext)
{
TestBase.TestClassInitialize(testContext);
//init the blob helper for blob related operations
BlobHelper = new CloudBlobHelper(StorageAccount);
BlockFilePath = Path.Combine(Test.Data.Get("TempDir"), FileUtil.GetSpecialFileName());
PageFilePath = Path.Combine(Test.Data.Get("TempDir"), FileUtil.GetSpecialFileName());
AppendFilePath = Path.Combine(Test.Data.Get("TempDir"), FileUtil.GetSpecialFileName());
FileUtil.CreateDirIfNotExits(Path.GetDirectoryName(BlockFilePath));
FileUtil.CreateDirIfNotExits(Path.GetDirectoryName(PageFilePath));
FileUtil.CreateDirIfNotExits(Path.GetDirectoryName(AppendFilePath));
// Generate block file and page file which are used for uploading
FileUtil.GenerateMediumFile(BlockFilePath, Utility.GetRandomTestCount(1, 10));
FileUtil.GenerateMediumFile(PageFilePath, Utility.GetRandomTestCount(1, 10));
FileUtil.GenerateMediumFile(AppendFilePath, Utility.GetRandomTestCount(1, 10));
}
public static void CLICommonBVTInitialize(TestContext testContext)
{
Test.Info(string.Format("{0} Class Initialize", testContext.FullyQualifiedTestClassName));
Test.FullClassName = testContext.FullyQualifiedTestClassName;
EnvKey = Test.Data.Get("EnvContextKey");
SaveAndCleanSubScriptionAndEnvConnectionString();
//init the blob helper for blob related operations
CommonBlobHelper = new CloudBlobHelper(CommonStorageAccount);
//Clean Storage Context
Test.Info("Clean storage context in PowerShell");
PowerShellAgent.CleanStorageContext();
// import module
string moduleFilePath = Test.Data.Get("ModuleFilePath");
PowerShellAgent.ImportModule(moduleFilePath);
GenerateBvtTempFiles();
}
public static void MyClassInitialize(TestContext testContext)
{
TestBase.TestClassInitialize(testContext);
//delete perf files
Helper.DeletePattern("testfile_*");
BlobHelper = new CloudBlobHelper(StorageAccount);
FileHelper = new CloudFileHelper(StorageAccount);
FileName = Test.Data.Get("FileName");
FolderName = Test.Data.Get("FolderName");
// import module
PowerShellAgent.ImportModules(Constants.ServiceModulePaths);
//set the ConcurrentTaskCount field
PowerShellAgent.ConcurrentTaskCount = Environment.ProcessorCount * 8;
Trace.WriteLine("ClassInit");
}
public static void MyClassInitialize(TestContext testContext)
{
Trace.WriteLine("ClassInit");
Test.FullClassName = testContext.FullyQualifiedTestClassName;
StorageAccount = TestBase.GetCloudStorageAccountFromConfig();
//init the blob helper for blob related operations
BlobHelper = new CloudBlobHelper(StorageAccount);
// import module
string moduleFilePath = Test.Data.Get("ModuleFilePath");
if (moduleFilePath.Length > 0)
{
PowerShellAgent.ImportModule(moduleFilePath);
}
// $context = New-AzureStorageContext -ConnectionString ...
PowerShellAgent.SetStorageContext(StorageAccount.ToString(true));
}
public async Task <ActionResult> AddMerchants(string id, MerchantFileType merchantFileType, HttpPostedFileBase file)
{
if (file != null && file.ContentLength > 0)
{
if (file.FileName.EndsWith(".xlsx", StringComparison.InvariantCultureIgnoreCase) || file.FileName.EndsWith(".csv", StringComparison.InvariantCultureIgnoreCase))
{
//string fileName = Path.GetFileName(file.FileName);
string fileName = string.Concat(Path.GetFileNameWithoutExtension(file.FileName),
DateTime.UtcNow.ToString("yyyyMMddHHmmssfff"),
Path.GetExtension(file.FileName));
Log.Info("uploading file '{0}' with filename '{1}'", Path.GetFileName(file.FileName), fileName);
CloudBlobContainer cloudBlobContainer = CloudBlobHelper.GetBlobContainer(ConfigurationManager.AppSettings["IngestionStorageConnectionString"], ConfigurationManager.AppSettings["IngestionStorageContainerName"]);
CloudBlockBlob cloudBlockBlob = cloudBlobContainer.GetBlockBlobReference(fileName);
cloudBlockBlob.Properties.ContentType = file.ContentType;
await cloudBlockBlob.UploadFromStreamAsync(file.InputStream);
string result = null;
using (var client = GetHttpClient())
{
HttpResponseMessage response = await client.PostAsJsonAsync(MerchantApi + "import", new { ProviderId = id, FileName = fileName, MerchantFileType = merchantFileType, Author = User.Identity.Name });
if (response.IsSuccessStatusCode)
{
result = await response.Content.ReadAsAsync <string>();
}
}
if (string.IsNullOrWhiteSpace(result))
{
throw new HttpException((int)HttpStatusCode.InternalServerError, "Scheduling file processor job failed");
}
return(Content("Scheduled JobId: " + result));
}
}
throw new HttpException((int)HttpStatusCode.BadRequest, "Invalid file");
}
public static void SetCalculatedFileMD5(DMLibDataInfo dataInfo, DataAdaptor <DMLibDataInfo> destAdaptor, bool disableMD5Check = false)
{
foreach (FileNode fileNode in dataInfo.EnumerateFileNodes())
{
if (DMLibTestBase.IsCloudBlob(DMLibTestContext.DestType))
{
CloudBlobDataAdaptor cloudBlobDataAdaptor = destAdaptor as CloudBlobDataAdaptor;
CloudBlob cloudBlob = cloudBlobDataAdaptor.GetCloudBlobReference(fileNode);
fileNode.MD5 = CloudBlobHelper.CalculateMD5ByDownloading(cloudBlob, disableMD5Check);
}
else if (DMLibTestContext.DestType == DMLibDataType.CloudFile)
{
CloudFileDataAdaptor cloudFileDataAdaptor = destAdaptor as CloudFileDataAdaptor;
CloudFile cloudFile = cloudFileDataAdaptor.GetCloudFileReference(fileNode);
fileNode.MD5 = CloudFileHelper.CalculateMD5ByDownloading(cloudFile, disableMD5Check);
}
// No need to set md5 for local destination
}
}
private void GenerateDir(DirNode dirNode, CloudBlobDirectory cloudBlobDir)
{
DMLibDataHelper.CreateLocalDirIfNotExists(this.TempFolder);
foreach (var subDir in dirNode.DirNodes)
{
CloudBlobDirectory subCloudBlobDir = cloudBlobDir.GetDirectoryReference(subDir.Name);
this.GenerateDir(subDir, subCloudBlobDir);
}
List <FileNode> snapshotList = new List <FileNode>();
foreach (var file in dirNode.FileNodes)
{
CloudBlob cloudBlob = CloudBlobHelper.GetCloudBlobReference(cloudBlobDir, file.Name, this.BlobType);
this.GenerateFile(file, cloudBlob, snapshotList);
}
foreach (var snapshot in snapshotList)
{
dirNode.AddFileNode(snapshot);
}
}
public CloudBlob GetCloudBlobReference(string rootPath, FileNode fileNode, StorageCredentials credentials = null)
{
var container = this.BlobHelper.BlobClient.GetContainerReference(this.ContainerName);
if (credentials != null)
{
container = new CloudBlobContainer(container.StorageUri, credentials);
}
var blobName = fileNode.GetURLRelativePath();
if (blobName.StartsWith("/"))
{
blobName = blobName.Substring(1, blobName.Length - 1);
}
if (!string.IsNullOrEmpty(rootPath))
{
blobName = rootPath + "/" + blobName;
}
return(CloudBlobHelper.GetCloudBlobReference(container, blobName, this.BlobType));
}
public PageBlobUploadOperation(Agent agent, CloudBlobHelper helper) : base(agent, helper)
{
this.Name = "Upload-page-blob(s)";
this.MaxSize = Constants.MAX_PAGE_BLOB_SIZE;
this.Unit = Constants.PAGE_BLOB_UNIT;
}
public static void CLICommonBVTInitialize(TestContext testContext)
{
Test.Info(string.Format("{0} Class Initialize", testContext.FullyQualifiedTestClassName));
Test.FullClassName = testContext.FullyQualifiedTestClassName;
EnvKey = Test.Data.Get("EnvContextKey");
SaveAndCleanSubScriptionAndEnvConnectionString();
//init the blob helper for blob related operations
CommonBlobHelper = new CloudBlobHelper(CommonStorageAccount);
//Clean Storage Context
Test.Info("Clean storage context in PowerShell");
PowerShellAgent.CleanStorageContext();
// import module
string moduleFilePath = Test.Data.Get("ModuleFilePath");
PowerShellAgent.ImportModule(moduleFilePath);
GenerateBvtTempFiles();
}
public AppendBlobDownloadOperation(Agent agent, CloudBlobHelper blobHelper)
: base(agent, blobHelper)
{
this.Name = "Download-append-blob(s)";
}
public AppendBlobUploadOperation(Agent agent, CloudBlobHelper helper)
: base(agent, helper)
{
this.Name = "Upload-append-blob(s)";
}
