public void ValidateShareReadableWithSasToken(CloudFileShare share, string fileName, string sasToken)
{
Test.Info("Verify share read permission");
CloudFile file = share.GetRootDirectoryReference().GetFileReference(fileName);
if (!file.Exists())
{
PrepareFileInternal(file, null);
}
CloudStorageAccount sasAccount = TestBase.GetStorageAccountWithSasToken(share.ServiceClient.Credentials.AccountName, sasToken);
CloudFileClient sasClient = sasAccount.CreateCloudFileClient();
CloudFileShare sasShare = sasClient.GetShareReference(share.Name);
CloudFile sasFile = sasShare.GetRootDirectoryReference().GetFileReference(fileName);
sasFile.FetchAttributes();
long buffSize = sasFile.Properties.Length;
byte[] buffer = new byte[buffSize];
MemoryStream ms = new MemoryStream(buffer);
sasFile.DownloadRangeToStream(ms, 0, buffSize);
string md5 = Utility.GetBufferMD5(buffer);
TestBase.ExpectEqual(sasFile.Properties.ContentMD5, md5, "content md5");
}
private void BuildDirNode(CloudFileDirectory cloudDir, DirNode dirNode)
{
cloudDir.FetchAttributes(options: HelperConst.DefaultFileOptions);
dirNode.LastWriteTime = cloudDir.Properties.LastWriteTime;
dirNode.CreationTime = cloudDir.Properties.CreationTime;
dirNode.SMBAttributes = cloudDir.Properties.NtfsAttributes;
if (cloudDir.Metadata.Count > 0)
{
dirNode.Metadata = new Dictionary <string, string>(cloudDir.Metadata);
}
foreach (IListFileItem item in cloudDir.ListFilesAndDirectories(HelperConst.DefaultFileOptions))
{
CloudFile cloudFile = item as CloudFile;
CloudFileDirectory subCloudDir = item as CloudFileDirectory;
if (cloudFile != null)
{
// Cannot fetch attributes while listing, so do it for each cloud file.
cloudFile.FetchAttributes(options: HelperConst.DefaultFileOptions);
FileNode fileNode = new FileNode(cloudFile.Name);
this.BuildFileNode(cloudFile, fileNode);
dirNode.AddFileNode(fileNode);
}
else if (subCloudDir != null)
{
DirNode subDirNode = new DirNode(subCloudDir.Name);
this.BuildDirNode(subCloudDir, subDirNode);
dirNode.AddDirNode(subDirNode);
}
}
}
/// <summary>
/// Validate the write permission in the sas token for the the specified file
/// </summary>
internal void ValidateFileWriteableWithSasToken(CloudFile file, string sasToken, bool useHttps = true)
{
Test.Info("Verify file write permission");
Uri fileUri = file.Uri;
if (useHttps)
{
fileUri = new Uri(fileUri.AbsoluteUri.Replace("http://", "https://"));
}
else
{
fileUri = new Uri(fileUri.AbsoluteUri.Replace("https://", "http://"));
}
CloudFile sasFile = new CloudFile(fileUri, new StorageCredentials(sasToken));
DateTimeOffset?lastModifiedTime = sasFile.Properties.LastModified;
long buffSize = 1024 * 1024;
byte[] buffer = new byte[buffSize];
(new Random()).NextBytes(buffer);
MemoryStream ms = new MemoryStream(buffer);
sasFile.UploadFromStream(ms);
file.FetchAttributes();
DateTimeOffset?newModifiedTime = file.Properties.LastModified;
TestBase.ExpectNotEqual(lastModifiedTime.ToString(), newModifiedTime.ToString(), "Last modified time");
}
public static void ValidateCopyResult(CloudBlob srcBlob, CloudFile destFile)
{
destFile.FetchAttributes();
srcBlob.FetchAttributes();
Test.Assert(destFile.Properties.ContentMD5 == srcBlob.Properties.ContentMD5, "MD5 should be the same.");
Test.Assert(destFile.Properties.ContentType == srcBlob.Properties.ContentType, "Content type should be the same.");
}
public FileSync(CloudFileDirectory cloudDirectory, CloudFile cloudFile, DirectoryInfo localDirectory, ILogger logger)
{
_localDirectory = localDirectory;
Directory = cloudDirectory;
File = null;
_logger = logger;
_cloudFile = cloudFile;
_cloudFile.FetchAttributes();
}
public void TestFileShareSnapshotsCopyToBase()
{
int fileCount = 3;
string snapshotFile = "snapshotFile";
DMLibDataInfo sourceDataInfo = new DMLibDataInfo(string.Empty);
sourceDataInfo.IsFileShareSnapshot = true;
DMLibDataHelper.AddMultipleFiles(sourceDataInfo.RootNode, snapshotFile, fileCount, 1024);
SourceAdaptor.Cleanup();
SourceAdaptor.CreateIfNotExists();
SourceAdaptor.GenerateData(sourceDataInfo);
CloudFileDataAdaptor fileAdaptor = (SourceAdaptor as CloudFileDataAdaptor);
CloudFileDirectory SourceObject = SourceAdaptor.GetTransferObject(sourceDataInfo.RootPath, sourceDataInfo.RootNode) as CloudFileDirectory;
CloudFileDirectory DestObject = fileAdaptor.fileHelper.FileClient.GetShareReference(fileAdaptor.ShareName).GetRootDirectoryReference();
// transfer File and finished succssfully
Task <TransferStatus> task = TransferManager.CopyDirectoryAsync(
SourceObject,
DestObject,
DMLibTestContext.IsAsync,
new CopyDirectoryOptions()
{
Recursive = true
},
new DirectoryTransferContext()
{
ShouldOverwriteCallbackAsync = TransferContext.ForceOverwrite
});
Test.Assert(task.Wait(15 * 60 * 100), "Tansfer finished in time.");
Test.Assert(task.Result.NumberOfFilesFailed == 0, "No Failed File.");
Test.Assert(task.Result.NumberOfFilesSkipped == 0, "No Skipped File.");
Test.Assert(task.Result.NumberOfFilesTransferred == fileCount, string.Format("Transferred file :{0} == {1}", task.Result.NumberOfFilesTransferred, fileCount));
// verify that Files in Share Snapshot is transferred
IEnumerable <IListFileItem> sourceFiles = SourceObject.ListFilesAndDirectories(HelperConst.DefaultFileOptions);
foreach (IListFileItem item in sourceFiles)
{
if (item is CloudFile)
{
CloudFile srcFile = item as CloudFile;
CloudFile destFile = DestObject.GetFileReference(srcFile.Name);
srcFile.FetchAttributes();
destFile.FetchAttributes();
Test.Assert(srcFile.Properties.ContentMD5 == destFile.Properties.ContentMD5, string.Format("File {0} MD5 :{1} == {2}", srcFile.Name, srcFile.Properties.ContentMD5, destFile.Properties.ContentMD5));
}
}
}
/// <summary>
/// Validate the Create permission in the sas token for the the specified file
/// </summary>
internal void ValidateFileCreateableWithSasToken(CloudFile file, string sasToken)
{
Test.Info("Verify file Create permission");
file.DeleteIfExists();
CloudFile sasFile = new CloudFile(file.Uri, new StorageCredentials(sasToken));
int fileLength = 10;
sasFile.Create(fileLength);
file.FetchAttributes();
Test.Assert(file.Exists(), "The file should exist after Creating with sas token");
TestBase.ExpectEqual(fileLength, file.Properties.Length, "File Lenth should match.");
}
public static FileSystemEntry CreateDictionaryEntry(CloudFile file)
{
file.FetchAttributes();
FileEntry retFile = new FileEntry();
retFile.ETag = file.Properties.ETag;
retFile.Name = file.Name;
retFile.Length = file.Properties.Length;
retFile.LastAccess = file.Properties.LastModified.HasValue
? file.Properties.LastModified.Value.UtcDateTime
: (DateTime?)null;
return(retFile);
}
public static FileInformation ToFileInformation(this CloudFile cloudFile)
{
cloudFile.FetchAttributes();
return(new FileInformation
{
Length = cloudFile.Properties.Length,
LastAccessTime = cloudFile.Properties.LastModified.Value.LocalDateTime,
LastWriteTime = cloudFile.Properties.LastModified.Value.LocalDateTime,
CreationTime = cloudFile.Properties.LastModified.Value.LocalDateTime,
Attributes = FileAttributes.Normal,
FileName = cloudFile.Name,
});
}
public static void ValidateCopyResult(CloudFile srcFile, CloudFile destFile, bool destExistBefore = false)
{
destFile.FetchAttributes();
srcFile.FetchAttributes();
if (!destExistBefore)
{
Test.Assert(destFile.Metadata.SequenceEqual(srcFile.Metadata), "Destination's metadata should be the same with source's");
}
Test.Assert(destFile.Properties.ContentMD5 == srcFile.Properties.ContentMD5, "MD5 should be the same.");
Test.Assert(destFile.Properties.ContentType == srcFile.Properties.ContentType, "Content type should be the same.");
}
public void GetDocumentAttributes(DocumentFileViewModel doc)
{
CloudFile file = GetFile(doc.Path);
if (file != null && file.Exists())
{
Debug.WriteLine($"Fetching attributes for {doc.Path}");
file.FetchAttributes();
if (file.Properties != null)
{
doc.Size = file.Properties.Length / 1024;
doc.CreatedAt = file.Properties.LastModified.Value.DateTime;
}
}
}
/// <summary>
/// Validate the read permission in the sas token for the the specified file
/// </summary>
public void ValidateFileReadableWithSasToken(CloudFile file, string sasToken)
{
Test.Info("Verify file read permission");
CloudFile sasFile = new CloudFile(file.Uri, new StorageCredentials(sasToken));
sasFile.FetchAttributes();
long buffSize = sasFile.Properties.Length;
byte[] buffer = new byte[buffSize];
MemoryStream ms = new MemoryStream(buffer);
sasFile.DownloadRangeToStream(ms, 0, buffSize);
string md5 = Utility.GetBufferMD5(buffer);
TestBase.ExpectEqual(sasFile.Properties.ContentMD5, md5, "content md5");
}
public void DownloadFileFromShareSnapshot_dir()
{
string shareName = CloudFileUtil.GenerateUniqueFileShareName();
string dirName = CloudFileUtil.GenerateUniqueDirectoryName();
string fileName = CloudFileUtil.GenerateUniqueFileName();
try
{
CloudFileShare share = fileUtil.EnsureFileShareExists(shareName);
CloudFileShare shareSnapshot1 = share.Snapshot();
CloudFileDirectory dir = fileUtil.EnsureDirectoryExists(share, dirName);
CloudFile file = fileUtil.CreateFile(dir, fileName);
CloudFileShare shareSnapshot2 = share.Snapshot();
file.Delete();
dir.Delete();
//Get File content
string StorageConnectionString = Test.Data.Get("StorageConnectionString");
Test.Assert((CommandAgent as PowerShellAgent).InvokePSScript(string.Format(",(New-AzureStorageContext -ConnectionString \"{5}\" | Get-AzureStorageShare -Name {0} -SnapshotTime \"{1}\").GetRootDirectoryReference().GetDirectoryReference(\"{4}\") | Get-AzureStorageFileContent -Path {2} -Destination {3} -Force",
shareName,
shareSnapshot2.SnapshotTime.Value,
fileName,
fileName,
dirName,
StorageConnectionString)),
string.Format("Download File {0} from share snapshot {1}, {2} should success.", dirName + "\\" + fileName, shareName, shareSnapshot2.SnapshotTime.Value));
//validate MD5
CloudFile file2 = shareSnapshot2.GetRootDirectoryReference().GetDirectoryReference(dirName).GetFileReference(fileName);
file2.FetchAttributes();
Test.Assert(file2.Properties.ContentMD5 == FileUtil.GetFileContentMD5(fileName), "Expected MD5: {0}, real MD5: {1}", file2.Properties.ContentMD5, FileUtil.GetFileContentMD5(fileName));
}
finally
{
try
{
fileUtil.DeleteFileShareIfExists(shareName);
}
catch (Exception e)
{
Test.Warn("Unexpected exception when cleanup file share {0}: {1}", shareName, e);
}
}
}
public void ValidateShareWriteableWithSasToken(CloudFileShare share, string sastoken)
{
Test.Info("Verify share write permission");
CloudStorageAccount sasAccount = TestBase.GetStorageAccountWithSasToken(share.ServiceClient.Credentials.AccountName, sastoken);
CloudFileShare sasShare = sasAccount.CreateCloudFileClient().GetShareReference(share.Name);
string sasFileName = Utility.GenNameString("sasFile");
CloudFile sasFile = sasShare.GetRootDirectoryReference().GetFileReference(sasFileName);
long fileSize = 1024 * 1024;
byte[] buffer = new byte[fileSize];
new Random().NextBytes(buffer);
MemoryStream ms = new MemoryStream(buffer);
sasFile.Create(fileSize);
sasFile.UploadFromStream(ms);
CloudFile retrievedFile = share.GetRootDirectoryReference().GetFileReference(sasFileName);
retrievedFile.FetchAttributes();
TestBase.ExpectEqual(fileSize, retrievedFile.Properties.Length, "blob size");
}
public bool ReadFile(string path, out string text, int?cacheTimeSeconds = null)
{
CloudFile file = AzureRootDirectory.GetFileReference(path);
if (file.Exists())
{
bool isExpired = !cacheTimeSeconds.HasValue;
if (cacheTimeSeconds.HasValue)
{
file.FetchAttributes();
var expirationTime = file.Properties.LastModified.Value.LocalDateTime.AddSeconds(cacheTimeSeconds.Value);
isExpired = expirationTime < DateTime.Now;
}
if (!isExpired)
{
text = file.DownloadText();
return(true);
}
}
text = null;
return(false);
}
private void GeneratePropertiesAndMetaData(CloudFile file)
{
file.Properties.ContentEncoding = Utility.GenNameString("encoding");
file.Properties.ContentLanguage = Utility.GenNameString("lang");
int minMetaCount = 1;
int maxMetaCount = 5;
int minMetaValueLength = 10;
int maxMetaValueLength = 20;
int count = rd.Next(minMetaCount, maxMetaCount);
for (int i = 0; i < count; i++)
{
string metaKey = Utility.GenNameString("metatest");
int valueLength = rd.Next(minMetaValueLength, maxMetaValueLength);
string metaValue = Utility.GenNameString("metavalue-", valueLength);
file.Metadata.Add(metaKey, metaValue);
}
file.SetProperties();
file.SetMetadata();
file.FetchAttributes();
}
private void BuildDirNode(CloudFileDirectory cloudDir, DirNode dirNode)
{
foreach (IListFileItem item in cloudDir.ListFilesAndDirectories(HelperConst.DefaultFileOptions))
{
CloudFile cloudFile = item as CloudFile;
CloudFileDirectory subCloudDir = item as CloudFileDirectory;
if (cloudFile != null)
{
// Cannot fetch attributes while listing, so do it for each cloud file.
cloudFile.FetchAttributes(options: HelperConst.DefaultFileOptions);
FileNode fileNode = new FileNode(cloudFile.Name);
this.BuildFileNode(cloudFile, fileNode);
dirNode.AddFileNode(fileNode);
}
else if (subCloudDir != null)
{
DirNode subDirNode = new DirNode(subCloudDir.Name);
this.BuildDirNode(subCloudDir, subDirNode);
dirNode.AddDirNode(subDirNode);
}
}
}
/// <summary>
/// Gets a file reference from the azure cloud file storage if it exists
/// </summary>
protected CloudFile GetCloudFileInfo(string fileDirectory, string fileName)
{
CloudFile cloudFile = null;
try
{
fileDirectory = CleanRelativeCloudDirectoryName(fileDirectory);
CloudStorageAccount cloudStorageAccount = GetCloudStorageAccount();
CloudFileClient fileClient = GetCloudFileClient(cloudStorageAccount);
CloudFileShare share = GetCloudFileShareReference(fileClient);
CloudFileDirectory cloudFileDirectory = GetCloudFileDirectory(share, fileDirectory);
if ((cloudFileDirectory != null) && cloudFileDirectory.Exists())
{
cloudFile = cloudFileDirectory.GetFileReference(fileName);
cloudFile.FetchAttributes();
}
}
catch (Exception oExeption)
{
oExeption.Log($"GetCloudFileInfo [{fileDirectory}\\{fileName}]");
}
return(cloudFile);
}
public JsonResult GetSingleUserTaxReturn(string userId, int taxYear)
{
bool downloadSuccessful = true;
var results = new List <Tuple <string, string, byte[]> >();
using (var db = new WorktripEntities())
{
try
{
Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)");
var user = db.Users.FirstOrDefault(u => u.Id == userId);
var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear);
taxReturn = taxReturn.OrderBy(d => d.Id);
var fileUrls = new List <UserTaxReturn>();
if (taxReturn.Count() != 0)
{
fileUrls.Add(taxReturn.AsEnumerable().Last());
byte[] bytes = new byte[64000];
var parsedFilePaths = new List <Tuple <string, string> >();
foreach (var url in fileUrls)
{
Match match = filePattern.Match(url.Path);
if (match.Success)
{
var newTuple = new Tuple <string, string>(
match.Groups["directory"].Value,
match.Groups["filename"].Value
);
parsedFilePaths.Add(newTuple);
}
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("worktripdocs");
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
CloudFileDirectory userDir = null;
var userDirName = "";
foreach (var parsedPath in parsedFilePaths)
{
if (userDirName != parsedPath.Item1)
{
userDir = rootDir.GetDirectoryReference(parsedPath.Item1);
if (!userDir.Exists())
{
continue;
}
userDirName = parsedPath.Item1;
}
var filename = parsedPath.Item2;
CloudFile file = userDir.GetFileReference(filename);
if (!file.Exists())
{
continue;
}
file.FetchAttributes();
string fileContents = "";
if (file.Properties.ContentType.ToLower() == "application/pdf")
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
bytes = fileStream.ToArray();
fileContents = ConvertStreamToBase64String(fileStream);
}
else
{
fileContents = file.DownloadText();
}
results.Add(
new Tuple <string, string, byte[]>(filename, file.Properties.ContentType, bytes)
);
}
}
}
catch (Exception e)
{
//Do some error logging here..
downloadSuccessful = false;
}
}
if (downloadSuccessful && results.Count > 0)
{
return(Json(new MyJsonResult
{
status = 0,
fileName = results.ElementAtOrDefault(0).Item1,
fileContentType = results.ElementAtOrDefault(0).Item2,
fileContents = results.ElementAtOrDefault(0).Item3
}));
}
else
{
return(Json(new { status = -1, message = "Error in downloading files" }));
}
}
public JsonResult GetUserDocuments(string userId, int taxYear, int?skip, int?amount)
{
bool downloadSuccessful = true;
var results = new List <Tuple <string, string, string> >();
using (var db = new WorktripEntities())
{
try
{
Regex filePattern = new Regex(@"http.*\/.*\/(?<directory>.*)\/(?<filename>.*)");
var user = db.Users.FirstOrDefault(u => u.Id == userId);
var miscDocs = db.UserMiscDocs.Where(d => d.UserId == userId && d.Year == taxYear);
var taxReturn = db.UserTaxReturns.Where(d => d.UserId == userId && d.Year == taxYear);
miscDocs = miscDocs.OrderBy(d => d.Id);
taxReturn = taxReturn.OrderBy(d => d.Id);
if (skip.HasValue)
{
miscDocs = miscDocs.Skip(skip.Value);
taxReturn = taxReturn.Skip(skip.Value);
}
if (amount.HasValue)
{
miscDocs = miscDocs.Take(amount.Value);
taxReturn = taxReturn.Take(amount.Value);
}
var fileUrls = miscDocs.Select(d => d.Path).ToList();
fileUrls.AddRange(taxReturn.Select(d => d.Path).ToList());
var parsedFilePaths = new List <Tuple <string, string> >();
foreach (var url in fileUrls)
{
Match match = filePattern.Match(url);
if (match.Success)
{
var newTuple = new Tuple <string, string>(
match.Groups["directory"].Value,
match.Groups["filename"].Value
);
parsedFilePaths.Add(newTuple);
}
}
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(
CloudConfigurationManager.GetSetting("StorageConnectionString"));
CloudFileClient fileClient = storageAccount.CreateCloudFileClient();
CloudFileShare share = fileClient.GetShareReference("worktripdocs");
CloudFileDirectory rootDir = share.GetRootDirectoryReference();
CloudFileDirectory userDir = null;
var userDirName = "";
foreach (var parsedPath in parsedFilePaths)
{
if (userDirName != parsedPath.Item1)
{
userDir = rootDir.GetDirectoryReference(parsedPath.Item1);
if (!userDir.Exists())
{
continue;
}
userDirName = parsedPath.Item1;
}
var filename = parsedPath.Item2;
CloudFile file = userDir.GetFileReference(filename);
if (!file.Exists())
{
continue;
}
file.FetchAttributes();
string fileContents = "";
if (file.Properties.ContentType != null &&
file.Properties.ContentType.StartsWith("image/"))
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
fileContents = ConvertImageStreamToBase64String(fileStream);
}
else if (file.Properties.ContentType.ToLower() == "application/pdf")
{
MemoryStream fileStream = new MemoryStream();
file.DownloadToStream(fileStream);
fileContents = ConvertStreamToBase64String(fileStream);
}
else
{
fileContents = file.DownloadText();
}
results.Add(
new Tuple <string, string, string>(filename, file.Properties.ContentType, fileContents)
);
}
}
catch (Exception e)
{
//Do some error logging here..
downloadSuccessful = false;
}
}
if (downloadSuccessful)
{
return(Json(new {
status = 0,
files = results
}));
}
else
{
return(Json(new { status = -1, message = "Error in downloading files" }));
}
}
