protected override GetBackupFolderFilesResult GetBackupFilesInFolder(string folder, DateTime startDateOfRetentionRange) { var backupFiles = new GetBackupFolderFilesResult(); string blobsNextMarker = null; bool firstFileSet = false; do { var blobs = _client.ListBlobs(folder, delimiter: null, listFolders: false, marker: blobsNextMarker); foreach (var blob in blobs.List) { if (firstFileSet == false) { backupFiles.FirstFile = blob.Name; firstFileSet = true; } if (RestorePointsBase.TryExtractDateFromFileName(blob.Name, out var lastModified) && lastModified > startDateOfRetentionRange) { backupFiles.LastFile = blob.Name; return(backupFiles); } } blobsNextMarker = blobs.NextMarker; CancellationToken.ThrowIfCancellationRequested(); } while (blobsNextMarker != null); return(backupFiles); }
private static bool IsAnyBackupFile(string filePath) { if (RestorePointsBase.IsBackupOrSnapshot(filePath)) { return(true); } var extension = Path.GetExtension(filePath); return(InProgressExtension.Equals(extension, StringComparison.OrdinalIgnoreCase)); }
private static void FillResult(RavenStorageClient.ListBlobResult result, Page <BlobHierarchyItem> page, bool listFolders) { result.List = page.Values .Where(x => listFolders || x.IsBlob) .Select(x => listFolders ? RestorePointsBase.GetDirectoryName(x.IsPrefix ? x.Prefix : x.Blob.Name) : x.Blob.Name) .Distinct() .Select(x => new RavenStorageClient.BlobProperties { Name = x }) .ToList(); if (string.IsNullOrWhiteSpace(page.ContinuationToken) == false) { result.ContinuationToken = page.ContinuationToken; } }
private bool GotFreshIncrementalBackup(GetBackupFolderFilesResult backupFiles, DateTime now) { if (backupFiles.LastFile == null) { return(false); } if (backupFiles.FirstFile.Equals(backupFiles.LastFile)) { return(false); } if (RestorePointsBase.TryExtractDateFromFileName(backupFiles.LastFile, out var lastModified) == false) { lastModified = File.GetLastWriteTimeUtc(backupFiles.LastFile).ToLocalTime(); } return(now - lastModified < _retentionPolicy.MinimumBackupAgeToKeep); }
public async Task <ListBlobResult> ListBlobsAsync(string prefix, string delimiter, bool listFolders, int?maxResult = null, string marker = null) { var url = GetUrl(_serverUrlForContainer, "restype=container&comp=list"); if (prefix != null) { url += $"&prefix={Uri.EscapeDataString(prefix)}"; } if (delimiter != null) { url += $"&delimiter={delimiter}"; } if (maxResult != null) { url += $"&maxresults={maxResult}"; } if (marker != null) { url += $"&marker={marker}"; } var requestMessage = new HttpRequestMessage(HttpMethods.Get, url) { Headers = { { "x-ms-date", SystemTime.UtcNow.ToString("R") }, { "x-ms-version", AzureStorageVersion } } }; var client = GetClient(); SetAuthorizationHeader(client, HttpMethods.Get, url, requestMessage.Headers); var response = await client.SendAsync(requestMessage, CancellationToken).ConfigureAwait(false); if (response.StatusCode == HttpStatusCode.NotFound) { return new ListBlobResult { List = new List <BlobProperties>() } } ; if (response.IsSuccessStatusCode == false) { throw StorageException.FromResponseMessage(response); } var responseStream = await response.Content.ReadAsStreamAsync(); var listBlobsResult = XDocument.Load(responseStream); var result = GetResult(); var nextMarker = listBlobsResult.Root.Element("NextMarker")?.Value; return(new ListBlobResult { List = result, NextMarker = nextMarker == "true" ? listBlobsResult.Root.Element("NextMarker")?.Value : null }); IEnumerable <BlobProperties> GetResult() { if (listFolders) { return(listBlobsResult .Descendants("Blobs") .Descendants("Name") .Select(x => RestorePointsBase.GetDirectoryName(x.Value)) .Distinct() .Select(x => new BlobProperties { Name = x })); } return(listBlobsResult .Descendants("Blob") .Select(x => new BlobProperties { Name = x.Element("Name")?.Value, })); } }
private bool UpdateFoldersToDelete(GetFoldersResult folders, DateTime now, List <string> foldersToDelete) { var firstDateInRetentionRange = now - _retentionPolicy.MinimumBackupAgeToKeep.Value; foreach (var folder in folders.List) { CancellationToken.ThrowIfCancellationRequested(); var folderName = GetFolderName(folder); var folderDetails = RestorePointsBase.ParseFolderName(folderName); if (folderDetails.BackupTimeAsString == null) { if (Logger.IsInfoEnabled) { Logger.Info($"Failed to get backup date time for folder: {folder}"); } continue; } if (DateTime.TryParseExact( folderDetails.BackupTimeAsString, BackupTask.GetDateTimeFormat(folderDetails.BackupTimeAsString), CultureInfo.InvariantCulture, DateTimeStyles.None, out var backupTime) == false) { if (Logger.IsInfoEnabled) { Logger.Info($"Failed to parse backup date time for folder: {folder}"); } continue; } if (now - backupTime < _retentionPolicy.MinimumBackupAgeToKeep) { // all backups are sorted by date return(false); } if (string.Equals(folderDetails.DatabaseName, _databaseName, StringComparison.OrdinalIgnoreCase) == false) { continue; // a backup for a different database } var backupFiles = GetBackupFilesInFolder(folder, firstDateInRetentionRange); if (backupFiles == null) { continue; // folder is empty } var hasFullBackupOrSnapshot = BackupUtils.IsFullBackupOrSnapshot(backupFiles.FirstFile); if (hasFullBackupOrSnapshot == false) { continue; // no snapshot or full backup } if (GotFreshIncrementalBackup(backupFiles, now)) { continue; } foldersToDelete.Add(folder); } return(true); }