// Reads the storage files protected async Task <FileManagerResponse> GetFilesAsync(string path, string filter, FileManagerDirectoryContent[] selectedItems) { FileManagerResponse readResponse = new FileManagerResponse(); List <FileManagerDirectoryContent> details = new List <FileManagerDirectoryContent>(); FileManagerDirectoryContent cwd = new FileManagerDirectoryContent(); try { string[] extensions = ((filter.Replace(" ", "")) ?? "*").Split(",|;".ToCharArray(), System.StringSplitOptions.RemoveEmptyEntries); CloudBlobDirectory sampleDirectory = container.GetDirectoryReference(path); cwd.Name = sampleDirectory.Prefix.Split(sampleDirectory.Parent.Prefix)[sampleDirectory.Prefix.Split(sampleDirectory.Parent.Prefix).Length - 1].Replace("/", ""); cwd.Type = "File Folder"; cwd.FilterPath = selectedItems.Length > 0 ? selectedItems[0].FilterPath : ""; cwd.Size = 0; cwd.HasChild = await HasChildDirectory(path); readResponse.CWD = cwd; BlobResultSegment items = await AsyncReadCall(path, "Read"); foreach (IListBlobItem item in items.Results) { bool includeItem = true; if (!(extensions[0].Equals("*.*") || extensions[0].Equals("*")) && item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob file = (CloudBlockBlob)item; if (!(Array.IndexOf(extensions, "*." + (file.Name.ToString().Trim().Split('.'))[(file.Name.ToString().Trim().Split('.')).Count() - 1]) >= 0)) { includeItem = false; } } if (includeItem) { FileManagerDirectoryContent entry = new FileManagerDirectoryContent(); if (item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob file = (CloudBlockBlob)item; entry.Name = file.Name.Replace(path, ""); entry.Type = System.IO.Path.GetExtension(file.Name.Replace(path, "")); entry.IsFile = true; entry.Size = file.Properties.Length; entry.DateModified = file.Properties.LastModified.Value.LocalDateTime; entry.HasChild = false; entry.FilterPath = selectedItems.Length > 0 ? path.Replace(this.rootPath, "") : "/"; details.Add(entry); } else if (item.GetType() == typeof(CloudBlobDirectory)) { CloudBlobDirectory directory = (CloudBlobDirectory)item; entry.Name = directory.Prefix.Replace(path, "").Replace("/", ""); entry.Type = "Directory"; entry.IsFile = false; entry.Size = 0; entry.HasChild = await HasChildDirectory(directory.Prefix); entry.FilterPath = selectedItems.Length > 0 ? path.Replace(this.rootPath, "") : "/"; details.Add(entry); } } } } catch (Exception) { return(readResponse); } readResponse.Files = details; return(readResponse); }
// Renames file(s) or folder(s) protected async Task <FileManagerResponse> RenameAsync(string path, string oldName, string newName, params FileManagerDirectoryContent[] selectedItems) { FileManagerResponse renameResponse = new FileManagerResponse(); List <FileManagerDirectoryContent> details = new List <FileManagerDirectoryContent>(); FileManagerDirectoryContent entry = new FileManagerDirectoryContent(); bool isAlreadyAvailable = false; bool isFile = false; foreach (FileManagerDirectoryContent FileItem in selectedItems) { FileManagerDirectoryContent s_item = FileItem; isFile = s_item.IsFile; if (isFile) { isAlreadyAvailable = await IsFileExists(path + newName); } else { isAlreadyAvailable = await IsFolderExists(path + newName); } entry.Name = newName; entry.Type = s_item.Type; entry.IsFile = isFile; entry.Size = s_item.Size; entry.HasChild = s_item.HasChild; entry.FilterPath = path; details.Add(entry); break; } if (!isAlreadyAvailable) { if (isFile) { CloudBlob existBlob = container.GetBlobReference(path + oldName); await(container.GetBlobReference(path + newName)).StartCopyAsync(existBlob.Uri); await existBlob.DeleteIfExistsAsync(); } else { CloudBlobDirectory sampleDirectory = container.GetDirectoryReference(path + oldName); BlobResultSegment items = await AsyncReadCall(path + oldName, "Rename"); foreach (IListBlobItem item in items.Results) { string name = item.Uri.AbsolutePath.Replace(sampleDirectory.Uri.AbsolutePath, "").Replace("%20", " "); await(container.GetBlobReference(path + newName + "/" + name)).StartCopyAsync(item.Uri); await container.GetBlobReference(path + oldName + "/" + name).DeleteAsync(); } } renameResponse.Files = (IEnumerable <FileManagerDirectoryContent>)details; } else { ErrorDetails er = new ErrorDetails(); er.FileExists = existFiles; er.Code = "400"; er.Message = "File or Folder Already Already Exists"; renameResponse.Error = er; } return(renameResponse); }
private async Task WriteIndexTags(SoftwareIdentity indexJsonTag, SoftwareIdentity indexXmlTag, CloudBlobDirectory sourceDirectory) { var blobJson = sourceDirectory.GetBlockBlobReference("index.json.SoftwareIdentity"); var blobXml = sourceDirectory.GetBlockBlobReference("index.xml.SoftwareIdentity"); var json = indexJsonTag.SwidTagJson; var xml = indexXmlTag.SwidTagXml; await Task.WhenAll( blobJson.UploadTextAsync(json), blobXml.UploadTextAsync(xml) ); blobJson.Properties.CacheControl = "public, max-age=300"; // cache for 5 minutes. blobXml.Properties.CacheControl = "public, max-age=300"; // cache for 5 minutes. blobJson.Properties.ContentType = FearTheCowboy.Iso19770.Schema.MediaType.SwidTagJsonLd; blobXml.Properties.ContentType = FearTheCowboy.Iso19770.Schema.MediaType.SwidTagXml; await Task.WhenAll( blobJson.SetPropertiesAsync(), blobXml.SetPropertiesAsync() ); }
public static HttpResponseMessage Run([HttpTrigger(AuthorizationLevel.AuthLevelValue, "post")] Input input, [Blob("PathValue", FileAccess.Read, Connection = "ConnectionValue")] CloudBlobDirectory blobDirectory, TraceWriter log) #endif { var permissions = SharedAccessBlobPermissions.Read; // default to read permissions // if permission was supplied, check if it is a possible value if (!string.IsNullOrWhiteSpace(input.Permission)) { if (!Enum.TryParse(input.Permission, out permissions)) { return(new HttpResponseMessage(HttpStatusCode.BadRequest) { Content = new StringContent("Invalid value for 'permissions'") }); } } var container = blobDirectory.Container; var sasToken = input.BlobName != null? GetBlobSasToken(container, input.BlobName, permissions) : GetContainerSasToken(container, permissions); return(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StringContent(sasToken) }); }
/// <inheritdoc /> public void CopyDirectory(string directoryName, string[] sourcePath, string[] targetPath) { #region validation if (string.IsNullOrEmpty(directoryName)) { throw new ArgumentNullException(nameof(directoryName)); } if (sourcePath == targetPath) { throw new InvalidOperationException("source can´t be equal target"); } #endregion // Source directory CloudBlobDirectory sourceCloudBlobDirectory = CloudBlobContainer.GetCloudBlobDirectoryReference(directoryName, sourcePath); // Get blobs and directories on current level IEnumerable <IListBlobItem> sourceListBlobItems = sourceCloudBlobDirectory .EnumerateDirectory("*") .Select <string, IListBlobItem>( listBlobItemName => { if (listBlobItemName.EndsWith(DIRECTORY_SEPARATOR_CHAR)) { return(CloudBlobContainer.GetDirectoryReference(listBlobItemName)); } return(CloudBlobContainer.GetBlobReference(listBlobItemName)); }); // Identify sub virtual directories through evaluation of path segments for each blob reference foreach (IListBlobItem listBlobItem in sourceListBlobItems) { string[] subSource = BlobUtilities.GetPath(directoryName, sourcePath).Split(DIRECTORY_SEPARATOR_CHAR); string[] subTarget = BlobUtilities.GetPath(directoryName, targetPath).Split(DIRECTORY_SEPARATOR_CHAR); switch (listBlobItem) { case CloudBlob cloudBlob: // Get the new blob reference as BlockBlob so we can use the upload text method CloudBlockBlob newCloudBlockBlob = CloudBlobContainer.GetCloudBlockBlobReference(cloudBlob.Uri.Segments.Last(), directoryName, targetPath); // Create the blob TaskUtilities.ExecuteSync(newCloudBlockBlob.UploadTextAsync(string.Empty)); // Get the new uri var source = new Uri(cloudBlob.GetTargetUri(directoryName, sourcePath)); // Copy the blob content TaskUtilities.ExecuteSync(newCloudBlockBlob.StartCopyAsync(source)); break; case CloudBlobDirectory cloudBlobDirectory: string lastSegment = cloudBlobDirectory.Uri.Segments.Last(); string subDirectoryName = lastSegment.Remove(lastSegment.LastIndexOf(DIRECTORY_SEPARATOR_CHAR)); CopyDirectory(subDirectoryName, subSource, subTarget); break; } } }
public async Task CopyBlobInSameStorageAccountAsync() { var cloudBlocks = 0; var cloudDirectory = 0; var cloudPages = 0; var noDots = 0; var notePages = 0; try { BlobContinuationToken token = null; do { var watchAllBlobsSelection = Stopwatch.StartNew(); BlobResultSegment resultSegment = await _azureStorage.Container.ListBlobsSegmentedAsync(token); watchAllBlobsSelection.Stop(); var elapsedMs = watchAllBlobsSelection.ElapsedMilliseconds; _logger.LogInformation($"Gettting all blobs elapsed time (ms): {elapsedMs}"); token = resultSegment.ContinuationToken; var watchCopyingBlobs = Stopwatch.StartNew(); foreach (IListBlobItem item in resultSegment.Results) { if (item.GetType() == typeof(CloudBlockBlob)) { cloudBlocks++; CloudBlockBlob blob = (CloudBlockBlob)item; NewFileName FileName = GetBlobDetails(blob); if (FileName.ApplicantId.Contains("NotePage")) { notePages++; continue; } if (!ApplicantIds.Contains(FileName.ApplicantId)) { ApplicantIds.Add(FileName.ApplicantId); Console.WriteLine($"Creating directory: {FileName.ApplicantId}"); _logger.LogInformation($"Directory created for applicant: {FileName.ApplicantId}"); } var previousDocumentLocation = _azureStorage.Container.GetBlockBlobReference($"{blob.Name}"); var newDocumentLocation = _azureStorage.Container.GetBlockBlobReference($"{FileName.ApplicantId}/{FileName.FileName}"); try { await newDocumentLocation.StartCopyAsync(previousDocumentLocation); } catch (Exception e) { _logger.LogError(e.Message); throw; } Log.Logger.Information($"Moved {previousDocumentLocation.Name} to {newDocumentLocation.Name}"); } else if (item.GetType() == typeof(CloudBlobDirectory)) { CloudBlobDirectory directory = (CloudBlobDirectory)item; Console.WriteLine($"Skipping existing directory: {directory.Uri}"); cloudDirectory++; /* Delete blobs in directory * await DeleteBlobsInDirectory(directory); * continue; */ } else if (item.GetType() == typeof(CloudPageBlob)) { CloudPageBlob pageBlob = (CloudPageBlob)item; Console.WriteLine($"We are not using pageBlobs: {pageBlob}"); cloudPages++; } } watchCopyingBlobs.Stop(); /* Only run this if you want to create records to be processed. * await CreateNBlobsOf500KbAsync(1200); */ /* */ _logger.LogInformation($"Copying blobs elapsed time(ms): {watchCopyingBlobs.ElapsedMilliseconds}"); } while (token != null); _logger.LogInformation($"Cloud Blobs: {cloudBlocks}"); _logger.LogInformation($"Cloud Directory: {cloudDirectory}"); _logger.LogInformation($"No Dots: {noDots}"); _logger.LogInformation($"Note Pages: {notePages}"); _logger.LogInformation("Finished with the Script."); } catch (Exception e) { Console.WriteLine(e); throw; } }
public ActionResult ProjectDitail(int id) { _context = new ApplicationDbContext(); Session["id"] = id; Projects project = _context.Projects.FirstOrDefault(gr => gr.Id == id); if (project == null) { return(RedirectToAction("Index", "MyProject")); } Customers customer = _context.Customers.FirstOrDefault(c => c.Id == project.Customer); List <Customers> customersList = _context.Customers.Select(c => c).ToList <Customers>(); List <Groups> groupList = _context.Groups.Select(g => g).ToList <Groups>(); var projectGroup = (from u in _context.Groups.ToList() from gu in _context.ProjectsGroups.ToList() where gu.ProjId == @project.Id && gu.GroupId == u.Id select u).ToList(); /////files CloudBlobContainer container = GetCloudBlobContainer(); string con = container.Name; List <string> blobs = new List <string>(); int sessionData = (int)Session["id"]; string projectID = Convert.ToString(sessionData); foreach (IListBlobItem item in container.ListBlobs(useFlatBlobListing: true)) { //if(item.Parent.Container.Name == "8") if (item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob blob = (CloudBlockBlob)item; string[] namePart = blob.Name.Split('/'); if (namePart != null && namePart.Count() > 0 && namePart[0] == projectID) { blobs.Add(blob.Name); } } else if (item.GetType() == typeof(CloudPageBlob)) { CloudPageBlob blob = (CloudPageBlob)item; blobs.Add(blob.Name); } else if (item.GetType() == typeof(CloudBlobDirectory)) { CloudBlobDirectory dir = (CloudBlobDirectory)item; blobs.Add(dir.Uri.ToString()); } } /////files var viewModel = new ProjectsFormViewModel { project = project, customer = customer, customers = customersList, groups = groupList, groupsInProject = projectGroup, filelist = blobs }; viewModel.SelectedIDArray = viewModel.groupsInProject.Select(u => u.Id.ToString()).ToArray(); return(View(viewModel)); }
internal async static Task VerifyLeaseState(MethodInfo method, SingletonScope scope, string scopeId, LeaseState leaseState, LeaseStatus leaseStatus, CloudBlobDirectory directory = null) { string lockId = FormatLockId(method, scope, scopeId); CloudBlobDirectory lockDirectory = directory ?? _lockDirectory; CloudBlockBlob lockBlob = lockDirectory.GetBlockBlobReference(lockId); await lockBlob.FetchAttributesAsync(); Assert.Equal(leaseState, lockBlob.Properties.LeaseState); Assert.Equal(leaseStatus, lockBlob.Properties.LeaseStatus); }
/// <summary> /// Initializes a new instance of the <see cref="AzureBlobDirectory"/> class. /// </summary> /// <param name="directory">The <see cref="CloudBlobDirectory"/> to wrap.</param> public AzureBlobDirectory(CloudBlobDirectory directory) { _directory = directory; }
public async void trainmodel() { try { // Create an empty person group string personGroupId = "myfriends"; using (FaceServiceClient faceClient = new FaceServiceClient(faceapikey)) { //await faceClient.CreatePersonGroupAsync(personGroupId, "myfriends"); // Define name here CreatePersonResult friend1 = await faceClient.CreatePersonAsync( // Id of the person group that the person belonged to personGroupId, // Name of the person name.Text.ToString() ); // Define Bill and Clare in the same way // Directory contains image files of Anna //const string friend1ImageDir = Server.MapPath("~/images/People/" + name.Text.ToString() + "/"); //foreach (string imagePath in Directory.GetFiles(Server.MapPath("~/images/People/" + name.Text.ToString() + "/"), "*.jpg")) //{ // using (Stream s = File.OpenRead(imagePath)) // { // // Detect faces in the image and add to Anna // await faceClient.AddPersonFaceAsync( // personGroupId, friend1.PersonId, s); // } //} // Retrieve storage account from connection string. CloudStorageAccount storageAccount = CloudStorageAccount.Parse( CloudConfigurationManager.GetSetting("StorageConnectionString")); // Create the blob client. CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); // Retrieve reference to a previously created container. CloudBlobContainer container = blobClient.GetContainerReference("bbpeople"); // Loop over items within the container and output the length and URI. foreach (IListBlobItem item in container.ListBlobs(null, false)) { if (item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob blob = (CloudBlockBlob)item; if (blob.Name.StartsWith(name.Text)) { CloudBlockBlob blockBlob2 = container.GetBlockBlobReference(blob.Name); string text; using (var memoryStream = new MemoryStream()) { //memoryStream.Position = 0; blockBlob2.DownloadToStream(memoryStream); //text = System.Text.Encoding.UTF8.GetString(memoryStream.ToArray(),0,(int)memoryStream.Length); //var mem = new MemoryStream(); //memoryStream.Position = 0; //memoryStream.CopyTo(mem, (int)memoryStream.Length); //MemoryStream mStrm = new MemoryStream(Encoding.UTF8.GetBytes(text)); //text = System.Text.Encoding.UTF8.GetString(memoryStream.ToArray()); // Detect faces in the image and add to Anna //await faceClient.AddPersonFaceAsync(personGroupId, friend1.PersonId, memoryStream); //await faceClient.AddPersonFaceAsync(personGroupId, friend1.PersonId, blob.Name); try { await faceClient.AddPersonFaceAsync(personGroupId, friend1.PersonId, blob.Uri.ToString()); } catch (Exception e) { showerror(e); //throw e; } } } //Console.WriteLine("Block blob of length {0}: {1}", blob.Properties.Length, blob.Uri); } else if (item.GetType() == typeof(CloudPageBlob)) { CloudPageBlob pageBlob = (CloudPageBlob)item; //Console.WriteLine("Page blob of length {0}: {1}", pageBlob.Properties.Length, pageBlob.Uri); } else if (item.GetType() == typeof(CloudBlobDirectory)) { CloudBlobDirectory directory = (CloudBlobDirectory)item; //Console.WriteLine("Directory: {0}", directory.Uri); } } // Do the same for Other Users as well and Clare await faceClient.TrainPersonGroupAsync(personGroupId); TrainingStatus trainingStatus = null; while (true) { trainingStatus = await faceClient.GetPersonGroupTrainingStatusAsync(personGroupId); if (trainingStatus.Status.ToString() != "running") { break; } await Task.Delay(1000); } } } catch (Exception ex) { showerror(ex); //throw ex; } }
/// <summary> /// Copy data between Azure storage. /// 1. Copy a CloudBlobDirectory /// 2. Cancel the transfer before it finishes with a CancellationToken /// 3. Store the transfer checkpoint into a file after transfer being cancelled /// 4. Reload checkpoint from the file /// 4. Resume the transfer with the loaded checkpoint /// </summary> private static async Task BlobDirectoryCopySample() { CloudBlobDirectory sourceBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir"); CloudBlobDirectory destBlobDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir2"); // When source is CloudBlobDirectory: // 1. If recursive is set to true, data movement library matches the source blob name against SearchPattern as prefix. // 2. Otherwise, data movement library matches the blob with the exact name specified by SearchPattern. // // You can also replace the source directory with a CloudFileDirectory instance to copy data from Azure File Storage. If so: // 1. If recursive is set to true, SearchPattern is not supported. Data movement library simply transfer all azure files // under the source CloudFileDirectory and its sub-directories. // 2. Otherwise, data movement library matches the azure file with the exact name specified by SearchPattern. // // In the following case, data movement library will copy all blobs with the prefix "azure" in source blob directory. CopyDirectoryOptions options = new CopyDirectoryOptions() { SearchPattern = "azure", Recursive = true, }; DirectoryTransferContext context = new DirectoryTransferContext(); context.FileTransferred += FileTransferredCallback; context.FileFailed += FileFailedCallback; context.FileSkipped += FileSkippedCallback; // Create CancellationTokenSource used to cancel the transfer CancellationTokenSource cancellationSource = new CancellationTokenSource(); TransferCheckpoint checkpoint = null; TransferStatus trasferStatus = null; try { Task <TransferStatus> task = TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, context, cancellationSource.Token); // Sleep for 1 seconds and cancel the transfer. // It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will be copied after resume. Thread.Sleep(1000); Console.WriteLine("Cancel the transfer."); cancellationSource.Cancel(); trasferStatus = await task; } catch (Exception e) { Console.WriteLine("The transfer is cancelled: {0}", e.Message); } // Store the transfer checkpoint checkpoint = context.LastCheckpoint; // Serialize the checkpoint into a file #if DOTNET5_4 var formatter = new DataContractSerializer(typeof(TransferCheckpoint)); #else IFormatter formatter = new BinaryFormatter(); #endif string tempFileName = Guid.NewGuid().ToString(); using (var stream = new FileStream(tempFileName, FileMode.Create, FileAccess.Write, FileShare.None)) { formatter.Serialize(stream, checkpoint); } // Deserialize the checkpoint from the file using (var stream = new FileStream(tempFileName, FileMode.Open, FileAccess.Read, FileShare.None)) { checkpoint = formatter.Deserialize(stream) as TransferCheckpoint; } File.Delete(tempFileName); // Create a new TransferContext with the store checkpoint DirectoryTransferContext resumeContext = new DirectoryTransferContext(checkpoint); resumeContext.FileTransferred += FileTransferredCallback; resumeContext.FileFailed += FileFailedCallback; resumeContext.FileSkipped += FileSkippedCallback; // Record the overall progress ProgressRecorder recorder = new ProgressRecorder(); resumeContext.ProgressHandler = recorder; // Resume transfer from the stored checkpoint Console.WriteLine("Resume the cancelled transfer."); trasferStatus = await TransferManager.CopyDirectoryAsync(sourceBlobDir, destBlobDir, false /* isServiceCopy */, options, resumeContext); // Print out the final transfer state Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus)); }
/// <summary> /// Upload local pictures to azure storage. /// 1. Upload png files starting with "azure" in the source directory as block blobs, not including the sub-directory /// and store transfer context in a streamed journal. /// 2. Set their content type to "image/png". /// 3. Cancel the transfer before it finishes with a CancellationToken /// 3. Reload the transfer context from the streamed journal. /// 4. Resume the transfer with the loaded transfer context /// </summary> private static async Task BlobDirectoryUploadSample() { string sourceDirPath = "."; CloudBlobDirectory destDir = await Util.GetCloudBlobDirectoryAsync(ContainerName, "blobdir"); // SearchPattern and Recuresive can be used to determine the files to be transferred from the source directory. The behavior of // SearchPattern and Recuresive varies for different source directory types. // See https://azure.github.io/azure-storage-net-data-movement for more details. // // When source is local directory, data movement library matches source files against the SearchPattern as standard wildcards. If // recuresive is set to false, only files directly under the source directory will be matched. Otherwise, all files in the // sub-directory will be matched as well. // // In the following case, data movement library will upload png files starting with "azure" in the source directory as block blobs, // not including the sub-directory. UploadDirectoryOptions options = new UploadDirectoryOptions() { SearchPattern = "azure*.png", Recursive = false, BlobType = BlobType.BlockBlob }; using (MemoryStream journalStream = new MemoryStream()) { // Store the transfer context in a streamed journal. DirectoryTransferContext context = new DirectoryTransferContext(journalStream); // Register for transfer event. context.FileTransferred += FileTransferredCallback; context.FileFailed += FileFailedCallback; context.FileSkipped += FileSkippedCallback; context.SetAttributesCallbackAsync = async(destination) => { CloudBlob destBlob = destination as CloudBlob; destBlob.Properties.ContentType = "image/png"; }; context.ShouldTransferCallbackAsync = async(source, destination) => { // Can add more logic here to evaluate whether really need to transfer the target. return(true); }; // Create CancellationTokenSource used to cancel the transfer CancellationTokenSource cancellationSource = new CancellationTokenSource(); TransferStatus trasferStatus = null; try { // Start the upload Task <TransferStatus> task = TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, context, cancellationSource.Token); // Sleep for 1 seconds and cancel the transfer. // It may fail to cancel the transfer if transfer is done in 1 second. If so, no file will be copied after resume. Thread.Sleep(1000); Console.WriteLine("Cancel the transfer."); cancellationSource.Cancel(); trasferStatus = await task; } catch (Exception e) { Console.WriteLine("The transfer is cancelled: {0}", e.Message); } journalStream.Position = 0; // Deserialize transfer context from the streamed journal. DirectoryTransferContext resumeContext = new DirectoryTransferContext(journalStream); resumeContext.FileTransferred += FileTransferredCallback; resumeContext.FileFailed += FileFailedCallback; resumeContext.FileSkipped += FileSkippedCallback; resumeContext.SetAttributesCallbackAsync = async(destination) => { CloudBlob destBlob = destination as CloudBlob; destBlob.Properties.ContentType = "image/png"; }; resumeContext.ShouldTransferCallbackAsync = async(source, destination) => { // Can add more logic here to evaluate whether really need to transfer the target. return(true); }; // Resume the upload trasferStatus = await TransferManager.UploadDirectoryAsync(sourceDirPath, destDir, options, resumeContext); Console.WriteLine("Final transfer state: {0}", TransferStatusToString(trasferStatus)); Console.WriteLine("Files in directory {0} uploading to {1} is finished.", sourceDirPath, destDir.Uri.ToString()); } }
void CopyDirectory(string sourceKey, string destinationKey, bool deleteSource = false) { CloudBlobDirectory dir = Container.GetDirectoryReference(sourceKey); CopyDirectory(dir, destinationKey, deleteSource); }
void RemoveDirectory(string key) { CloudBlobDirectory dir = Container.GetDirectoryReference(key); RemoveDirectory(dir); }
public override object Read(string path, string filter, IEnumerable <object> selectedItems = null) { AjaxFileExplorerResponse ReadResponse = new AjaxFileExplorerResponse(); List <FileExplorerDirectoryContent> details = new List <FileExplorerDirectoryContent>(); try { filter = filter.Replace(" ", ""); var extensions = (filter ?? "*").Split(",|;".ToCharArray(), System.StringSplitOptions.RemoveEmptyEntries); CloudBlobDirectory sampleDirectory = container.GetDirectoryReference(path); IEnumerable <IListBlobItem> items = sampleDirectory.ListBlobs(false, BlobListingDetails.Metadata); foreach (IListBlobItem item in items) { bool canAdd = false; if (extensions[0].Equals("*.*") || extensions[0].Equals("*")) { canAdd = true; } else if (item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob file = (CloudBlockBlob)item; var names = file.Name.ToString().Trim().Split('.'); if (Array.IndexOf(extensions, "*." + names[names.Count() - 1]) >= 0) { canAdd = true; } else { canAdd = false; } } else { canAdd = true; } if (canAdd) { if (item.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob file = (CloudBlockBlob)item; FileExplorerDirectoryContent entry = new FileExplorerDirectoryContent(); entry.name = file.Name.Replace(path, "").Replace("/", ""); entry.type = file.Properties.ContentType; entry.isFile = true; entry.size = file.Properties.Length; entry.dateModified = file.Properties.LastModified.Value.LocalDateTime.ToString(); entry.hasChild = false; entry.filterPath = ""; details.Add(entry); } else if (item.GetType() == typeof(CloudBlobDirectory)) { CloudBlobDirectory directory = (CloudBlobDirectory)item; FileExplorerDirectoryContent entry = new FileExplorerDirectoryContent(); entry.name = directory.Prefix.Replace(path, "").Replace("/", ""); entry.type = "Directory"; entry.isFile = false; entry.size = 0; //entry.dateModified = directory.Properties.LastModified.ToString(); entry.hasChild = HasChildDirectory(directory.Prefix); entry.filterPath = ""; details.Add(entry); } } } } catch (Exception e) { ReadResponse.error = e.GetType().FullName + ", " + e.Message; return(ReadResponse); } ReadResponse.files = (IEnumerable <FileExplorerDirectoryContent>)details; return(ReadResponse); }
public AzureBlobDirectory(CloudBlobContainer container, CloudBlobDirectory directory) { Condition.Requires(container, "container").IsNotNull(); this.Container = container; this.Directory = directory; }
private AbortRequestLogger(CloudBlobDirectory directory) { _directory = directory; }
internal static async Task VerifyLeaseDoesNotExistAsync(MethodInfo method, SingletonScope scope, string scopeId, CloudBlobDirectory directory = null) { string lockId = FormatLockId(method, scope, scopeId); CloudBlobDirectory lockDirectory = directory ?? _lockDirectory; CloudBlockBlob lockBlob = lockDirectory.GetBlockBlobReference(lockId); Assert.False(await lockBlob.ExistsAsync()); }
public record Request(CloudBlobDirectory BlobDirectory, string outputDirectoryPath);
private BlobRecentInvocationIndexReader(CloudBlobDirectory directory) { _container = directory.Container; _directoryPrefix = directory.Prefix; }
/// <summary> /// Generates a Snapshot of blobs in blob storage, XML config file will include info such as: ///<para> • Account - specific account being monitored </para> ///<para> • Container - name of the container</para> ///<para> • Name - name of the file/blob</para> ///<para> • MonitorThisFile - This setting controls if a file/blob should be monitored</para> ///<para> • BlobProperty_LastModified - METADATA of the blob of when it was last modifed, note metadata can be altered freely in blob storage</para> ///<para> • BlobProperty_Size - METADATA of size in bytes of the file/blob, note metadata can be altered freely in blob storage</para> ///<para> • BlobProperty_MD5 - METADATA of the blob of its md5 checksum, note metadata can be altered freely in blob storage</para> ///<para> • SHA512 - This is calculated when snapshot is created, files/blobs are downloaded and SHA512 is calculated</para> ///<para> • ContentType - METADATA of the blob of data type, note metadata can be altered freely in blob storage</para> ///<para> • DownloadLocation - The url to download the file/blob</para> ///<para> • AnonymousAccessEnabled - This shows it if was possible to anonymously download the blob at the time of snapshot creation</para> /// </summary> /// <param name="accountname"></param> /// <param name="accountkey"></param> public static void ConfigFile_Snapshot(string BlobEndpoint, string accountname, string accountkey, string SnapshotFileName, bool isbaseline, bool ispermissionscheck) { try { List <string> MonitorContainers = new List <string>(); List <string> SnapShotList_Baseline; int BaseLineListCount; //Generate a list of containers from the baseline snapshot to use to check if the container should be monitored if ((isbaseline == false) && (ispermissionscheck == false)) { SnapShotList_Baseline = Compare.ParseSnapshot(Config.Settings.SnapShotFileName_Baseline); BaseLineListCount = SnapShotList_Baseline.Count(); for (int i = 0; i < BaseLineListCount; i++) { string BaselineBlob = SnapShotList_Baseline[i]; //(monitorcontainer]1,_1_[CurrentContainer]2,_2_[blobname]3,_3_[LineNumber) string monitorcontainer = BaselineBlob.Substring(0, BaselineBlob.IndexOf("]1,_1_[")); // true or fase if container should be monitored string BaselineContainer = BaselineBlob.Substring(0, BaselineBlob.IndexOf("]2,_2_[")); BaselineContainer = BaselineContainer.Substring((BaselineContainer.IndexOf("]1,_1_[")) + 7); if ((monitorcontainer.ToLower()).Contains("false")) { MonitorContainers.Add(BaselineContainer); } } } if (DoNoOverwrite_Check(SnapshotFileName) == false) { CloudStorageAccount storageAccount = CloudStorageAccount.Parse("BlobEndpoint=https://" + accountname + "." + BlobEndpoint + ";DefaultEndpointsProtocol=https;AccountName=" + accountname + ";AccountKey=" + accountkey); CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient(); IEnumerable <CloudBlobContainer> containers = blobClient.ListContainers(); List <CloudBlob> blobsArray = new List <CloudBlob>(); XmlWriterSettings settings = new XmlWriterSettings(); //this will auto indent as well as put everything on a new line settings.Indent = true; using (XmlWriter writer = XmlWriter.Create(SnapshotFileName, settings)) { //Inital information for the general config file writer.WriteStartDocument(); writer.WriteStartElement("Shapshot"); DateTime now = DateTime.Now; writer.WriteAttributeString("DoNotOverwrite", "False"); writer.WriteAttributeString("Created", Convert.ToString(now)); //Acount information writer.WriteStartElement("Account"); writer.WriteAttributeString("Name", accountname); foreach (CloudBlobContainer container in containers) { /* * IEnumerable<Microsoft.WindowsAzure.StorageClient.IListBlobItem> blobs = container.ListBlobs(new BlobRequestOptions() * { * BlobListingDetails = BlobListingDetails.All, * UseFlatBlobListing = true, * }).Cast<Microsoft.WindowsAzure.StorageClient.CloudBlockBlob>(); */ //Container Information writer.WriteStartElement("Container"); writer.WriteAttributeString("Name", BlobStorage.Metadata_BlockBlob.GetContainer_Name(container)); //monitor container (true or false) writer.WriteElementString("MonitorThisContainer", "True"); //do a check if the current container is marked as true or false for monitoring in the baseline snapshot bool monitorthiscontainer = true; if (isbaseline == false) { for (int x = 0; x < MonitorContainers.Count; x++) { if (MonitorContainers[x].Contains(BlobStorage.Metadata_BlockBlob.GetContainer_Name(container))) { monitorthiscontainer = false; } } } //iterate over all the blobs in the container if (monitorthiscontainer == true) { foreach (IListBlobItem listBlobItem in container.ListBlobs(new BlobRequestOptions() { BlobListingDetails = BlobListingDetails.All, UseFlatBlobListing = true, }) ) { /* * The storage service offers three types of blobs, block blobs, append blobs, and page blobs. * You specify the blob type when you create the blob. Once the blob has been created, its type cannot be changed, * and it can be updated only by using operations appropriate for that blob type, * i.e., writing a block or list of blocks to a block blob, appending blocks to a append blob, and writing pages to a page blob. */ //Block Blob if (listBlobItem.GetType() == typeof(CloudBlockBlob)) { CloudBlockBlob blob = (CloudBlockBlob)listBlobItem; string blobname = BlobStorage.Metadata_BlockBlob.GetBlob_Name(blob); string BlobSize = BlobStorage.Metadata_BlockBlob.GetBlob_Size(blob); string downloadpath = BlobStorage.Metadata_BlockBlob.GetBlob_DownloadLocation(blob); writer.WriteStartElement("Blob"); writer.WriteAttributeString("Name", blobname); writer.WriteElementString("Name", blobname); writer.WriteElementString("MonitorThisFile", "True"); writer.WriteElementString("BlobProperty_LastModified", BlobStorage.Metadata_BlockBlob.GetBlob_LastModifiedUtc(blob)); writer.WriteElementString("BlobProperty_Size", BlobSize); writer.WriteElementString("BlobProperty_MD5", BlobStorage.Metadata_BlockBlob.GetBlob_MD5(blob)); writer.WriteElementString("SHA512", BlobStorage.Signature.CalculateBlockBlob_SHA512(blob, (ConvertToInt64(XmlConvert.DecodeName(BlobSize))))); writer.WriteElementString("ContentType", BlobStorage.Metadata_BlockBlob.GetBlob_ContentType(blob)); writer.WriteElementString("DownloadLocation", downloadpath); writer.WriteElementString("AnonymousAccessEnabled", Convert.ToString(BlobStorage.CheckFileAccess.Anonymous_BlobStorageFile(XmlConvert.DecodeName(downloadpath)))); writer.WriteEndElement(); // blob //try to find a way to read only certain bytes from a blob to try to do a signature check on those... problem //is however that something like this has to be done in the blob storage itself which is not possibe. To check //certain bytes you would have to download the whole file over the wire anyway... //for big files... it would have to be a very superficial check based on metadata properties or something of the sort... } else if (listBlobItem.GetType() == typeof(CloudPageBlob)) { CloudPageBlob pageBlob = (CloudPageBlob)listBlobItem; string blobname = BlobStorage.Metadata_PageBlob.GetBlob_Name(pageBlob); string BlobSize = BlobStorage.Metadata_PageBlob.GetBlob_Size(pageBlob); string downloadpath = BlobStorage.Metadata_PageBlob.GetBlob_DownloadLocation(pageBlob); writer.WriteStartElement("Blob"); writer.WriteAttributeString("Name", blobname); writer.WriteElementString("Name", blobname); writer.WriteElementString("MonitorThisFile", "True"); writer.WriteElementString("BlobProperty_LastModified", BlobStorage.Metadata_PageBlob.GetBlob_LastModifiedUtc(pageBlob)); writer.WriteElementString("BlobProperty_Size", BlobSize); writer.WriteElementString("BlobProperty_MD5", BlobStorage.Metadata_PageBlob.GetBlob_MD5(pageBlob)); writer.WriteElementString("SHA512", BlobStorage.Signature.CalculatePageBlob_SHA512(pageBlob, (ConvertToInt64(XmlConvert.DecodeName(BlobSize))))); writer.WriteElementString("ContentType", BlobStorage.Metadata_PageBlob.GetBlob_ContentType(pageBlob)); writer.WriteElementString("DownloadLocation", downloadpath); writer.WriteElementString("AnonymousAccessEnabled", Convert.ToString(BlobStorage.CheckFileAccess.Anonymous_BlobStorageFile(XmlConvert.DecodeName(downloadpath)))); writer.WriteEndElement(); Console.WriteLine("Page blob of length {0}: {1}", pageBlob.Properties.Length, pageBlob.Uri); } else if (listBlobItem.GetType() == typeof(CloudBlobDirectory)) { CloudBlobDirectory directory = (CloudBlobDirectory)listBlobItem; Alerting.ErrorLogging.WriteTo_Log("ConfigFile_Snapshot - Could Not Read CloudBlobDirectory as a Blob: ", directory.ToString()); Console.WriteLine("Directory: {0}", directory.Uri); } } } writer.WriteEndElement(); // container } writer.WriteEndElement(); // account writer.WriteEndElement(); // snapshot writer.WriteEndDocument(); writer.Flush(); writer.Close(); } } } catch (Exception e) { Alerting.ErrorLogging.WriteTo_Log("Failed ConfigFile_Snapshot", e.ToString()); } }
private async Task ProcessTransactions(string sourceAzid, CloudBlobDirectory sourceDirectory, IEnumerable <TagTransactionEntity> tagTransactions, SoftwareIdentity indexJsonTag, SoftwareIdentity indexXmlTag) { var storage = this.Connection.ConnectToTagStorage(); var tagsTable = new TagTable(this.Connection); //var tagsBatch = new AzureBatch(tagsTable.Table); var redirectsTable = new DownloadRedirectsTable(this.Connection); var batchTasks = new List <AzureBatch>(); var redirectsTasks = new List <Task>(); var addedTags = new List <AddedTagResult>(); var updatedTags = new List <UpdatedTagResult>(); var deleteTags = new List <DeleteTagResult>(); foreach (var tagTx in tagTransactions) { var op = (TagTransactionOperation)Enum.Parse(typeof(TagTransactionOperation), tagTx.Operation); switch (op) { case TagTransactionOperation.Create: { var add = await this.CreateTag(sourceAzid, sourceDirectory, tagTx); addedTags.Add(add); var batch = new AzureBatch(tagsTable.Table); batch.Create(add.Tag); batch.Upsert(add.PrimaryTag); batchTasks.Add(batch); var tasks = CreateRedirects(redirectsTable.Table, add.Redirects); redirectsTasks.AddRange(tasks); } break; case TagTransactionOperation.Update: { var update = await this.UpdateTag(sourceAzid, sourceDirectory, tagsTable, tagTx); updatedTags.Add(update); var batch = new AzureBatch(tagsTable.Table); batch.Create(update.Tag); batch.Upsert(update.PrimaryTag); batchTasks.Add(batch); var tasks = CreateRedirects(redirectsTable.Table, update.Redirects); redirectsTasks.AddRange(tasks); } break; case TagTransactionOperation.Delete: { var delete = DeleteTag(sourceAzid, tagsTable, tagTx); deleteTags.Add(delete); var batch = new AzureBatch(tagsTable.Table); batch.Update(delete.Tag); batchTasks.Add(batch); } break; } } foreach (var batch in batchTasks) { await batch.WhenAll(); } await Task.WhenAll(redirectsTasks); if (this.UpdateIndexTags(indexJsonTag, indexXmlTag, addedTags, updatedTags, deleteTags)) { await this.WriteIndexTags(indexJsonTag, indexXmlTag, sourceDirectory); } var deleteTasks = new List <Task>(deleteTags.Count + 1); foreach (var deleteResult in deleteTags) { var blobJson = new CloudBlockBlob(deleteResult.DeleteSourceUris.JsonUri, storage.Credentials); var blobXml = new CloudBlockBlob(deleteResult.DeleteSourceUris.XmlUri, storage.Credentials); deleteTasks.Add(blobJson.DeleteIfExistsAsync()); deleteTasks.Add(blobXml.DeleteIfExistsAsync()); } await Task.WhenAll(deleteTasks); }
private void ArchiveOldReport(string reportPath, string reportName, string fileExt, CloudBlockBlob report) { if (!report.Exists()) { return; } report.FetchAttributes(); string archiveDirString = (archiveBlobPath + ( (archiveBlobPath.EndsWith("/") && reportPath.StartsWith("/")) ? reportPath.Substring(1) : reportPath )).ToLower(); if (!archiveDirString.EndsWith("/")) { archiveDirString = archiveDirString + "/"; } CloudBlobDirectory archiveDir = container.GetDirectoryReference(archiveDirString); //if the archive path exists, see if we need to delete the oldest blob in the path IEnumerable <IListBlobItem> blobs = archiveDir.ListBlobs(true); Dictionary <DateTimeOffset, string> oldReports = new Dictionary <DateTimeOffset, string>(); DateTimeOffset oldest = DateTimeOffset.MaxValue; ICloudBlob oldestBlob = null; int count = 0; foreach (ICloudBlob blob in blobs) { ++count; if (blob.Properties.LastModified.Value < oldest) { oldest = blob.Properties.LastModified.Value; oldestBlob = blob; } } if (count >= maxNumOfReportsArchived) { oldestBlob.Delete(); } //copy report to archive string newUri = (archiveDirString + reportName + "_" + report.Properties.LastModified.Value.ToString("u") + fileExt).ToLower(); CloudBlockBlob newReport = container.GetBlockBlobReference(newUri); newReport.StartCopyFromBlob(report); int retries = 3; while (retries > 0) { if (newReport.CopyState.Status == CopyStatus.Success) { Trace.TraceInformation("Blob archiving succeeded: " + newUri); break; } else if (newReport.CopyState.Status == CopyStatus.Aborted || newReport.CopyState.Status == CopyStatus.Failed || newReport.CopyState.Status == CopyStatus.Invalid) { Trace.TraceError("Blob archiving failed: " + newUri); break; } else { Thread.Sleep(1000); --retries; if (retries == 0) { Trace.TraceError("Blob archiving timed out: " + newUri); } } } }
protected bool DirectoryExists(CloudBlobDirectory directory) { return(directory.ListBlobsSegmented(true, BlobListingDetails.None, 1, new BlobContinuationToken(), null, null).Results.Any()); }
private static Task DeleteDirectory(CloudBlobDirectory directory) { return(directory.GetBlockBlobReference("blob").DeleteAsync()); }
public FifthweekCloudBlobDirectory(CloudBlobDirectory directory) { this.directory = directory; }
private Task <CloudBlobDirectory> CreateNonEmptyDirectory(CloudBlobDirectory parent, string directoryName) { return(CreateNonEmptyDir((dynamic)parent, directoryName)); }
// Gets the details protected async Task <FileManagerResponse> GetDetailsAsync(string path, string[] names, IEnumerable <object> selectedItems = null) { bool isVariousFolders = false; string previousPath = ""; string previousName = ""; FileManagerResponse detailsResponse = new FileManagerResponse(); try { bool isFile = false; bool namesAvailable = names.Length > 0 ? true : false; if (names.Length == 0 && selectedItems != null) { List <string> values = new List <string>(); foreach (FileManagerDirectoryContent item in selectedItems) { values.Add(item.Name); } names = values.ToArray(); } FileDetails fileDetails = new FileDetails(); long multipleSize = 0; if (selectedItems != null) { foreach (FileManagerDirectoryContent fileItem in selectedItems) { if (names.Length == 1) { if (fileItem.IsFile) { var blob = container.GetBlockBlobReference(rootPath + fileItem.FilterPath + fileItem.Name); isFile = fileItem.IsFile; fileDetails.IsFile = isFile; await blob.FetchAttributesAsync(); fileDetails.Name = fileItem.Name; fileDetails.Location = ((namesAvailable ? (rootPath + fileItem.FilterPath + fileItem.Name) : path)).Replace("/", @"\"); fileDetails.Size = byteConversion(blob.Properties.Length); fileDetails.Modified = blob.Properties.LastModified.Value.LocalDateTime; detailsResponse.Details = fileDetails; } else { CloudBlobDirectory sampleDirectory = container.GetDirectoryReference(rootPath + fileItem.FilterPath + fileItem.Name); long sizeValue = getSizeValue((namesAvailable ? rootPath + fileItem.FilterPath + fileItem.Name : "")).Result; isFile = false; fileDetails.Name = fileItem.Name; fileDetails.Location = ((namesAvailable ? rootPath + fileItem.FilterPath + fileItem.Name : path.Substring(0, path.Length - 1))).Replace("/", @"\"); fileDetails.Size = byteConversion(sizeValue); fileDetails.Modified = fileItem.DateModified; detailsResponse.Details = fileDetails; } } else { multipleSize = multipleSize + (fileItem.IsFile ? fileItem.Size : getSizeValue((namesAvailable ? rootPath + fileItem.FilterPath + fileItem.Name : path)).Result); size = 0; fileDetails.Name = previousName == "" ? previousName = fileItem.Name : previousName + ", " + fileItem.Name; previousPath = previousPath == "" ? rootPath + fileItem.FilterPath : previousPath; if (previousPath == rootPath + fileItem.FilterPath && !isVariousFolders) { previousPath = rootPath + fileItem.FilterPath; fileDetails.Location = ((rootPath + fileItem.FilterPath).Replace("/", @"\")).Substring(0, ((rootPath + fileItem.FilterPath).Replace("/", @"\")).Length - 1); } else { isVariousFolders = true; fileDetails.Location = "Various Folders"; } fileDetails.Size = byteConversion(multipleSize); fileDetails.MultipleFiles = true; detailsResponse.Details = fileDetails; } } } return(await Task.Run(() => { size = 0; return detailsResponse; })); } catch (Exception ex) { throw ex; } }
public BlobConcurrentTextStore(CloudBlobDirectory directory) { _directory = directory; }
public static IEnumerable<CloudBlobDirectory> ListSubdirectories(CloudBlobDirectory cloudBlobDirectory) { throw new NotImplementedException(); var l = cloudBlobDirectory.Uri.AbsolutePath.Length; return (from blob in cloudBlobDirectory.ListBlobs().Select(each => each.Uri.AbsolutePath.Substring(l + 1)) let i = blob.IndexOf('/') where i > -1 select blob.Substring(0, i)).Distinct().Select(cloudBlobDirectory.GetSubdirectoryReference); }
public static async Task <IEnumerable <DownloadFile> > DownloadDirectoryFiles(this CloudBlobDirectory directory, CancellationToken cancellationToken) { var fileBlobs = directory.ListBlobs(useFlatBlobListing: true).ToList(); var downloadFiles = new List <DownloadFile>(fileBlobs.Count); foreach (var blob in fileBlobs.OfType <CloudBlob>()) { var downloadFile = await DownloadFileFromBlob(blob, cancellationToken); downloadFiles.Add(downloadFile); } return(downloadFiles); }