protected override GetBackupFolderFilesResult GetBackupFilesInFolder(string folder, DateTime startDateOfRetentionRange)
{
var backupFiles = new GetBackupFolderFilesResult();
string blobsNextMarker = null;
bool firstFileSet = false;
do
{
var blobs = _client.ListBlobs(folder, delimiter: null, listFolders: false, marker: blobsNextMarker);
foreach (var blob in blobs.List)
{
if (firstFileSet == false)
{
backupFiles.FirstFile = blob.Name;
firstFileSet = true;
}
if (RestorePointsBase.TryExtractDateFromFileName(blob.Name, out var lastModified) && lastModified > startDateOfRetentionRange)
{
backupFiles.LastFile = blob.Name;
return(backupFiles);
}
}
blobsNextMarker = blobs.NextMarker;
CancellationToken.ThrowIfCancellationRequested();
} while (blobsNextMarker != null);
return(backupFiles);
}
private static bool IsAnyBackupFile(string filePath)
{
if (RestorePointsBase.IsBackupOrSnapshot(filePath))
{
return(true);
}
var extension = Path.GetExtension(filePath);
return(InProgressExtension.Equals(extension, StringComparison.OrdinalIgnoreCase));
}
private static void FillResult(RavenStorageClient.ListBlobResult result, Page <BlobHierarchyItem> page, bool listFolders)
{
result.List = page.Values
.Where(x => listFolders || x.IsBlob)
.Select(x => listFolders ? RestorePointsBase.GetDirectoryName(x.IsPrefix ? x.Prefix : x.Blob.Name) : x.Blob.Name)
.Distinct()
.Select(x => new RavenStorageClient.BlobProperties {
Name = x
})
.ToList();
if (string.IsNullOrWhiteSpace(page.ContinuationToken) == false)
{
result.ContinuationToken = page.ContinuationToken;
}
}
private bool GotFreshIncrementalBackup(GetBackupFolderFilesResult backupFiles, DateTime now)
{
if (backupFiles.LastFile == null)
{
return(false);
}
if (backupFiles.FirstFile.Equals(backupFiles.LastFile))
{
return(false);
}
if (RestorePointsBase.TryExtractDateFromFileName(backupFiles.LastFile, out var lastModified) == false)
{
lastModified = File.GetLastWriteTimeUtc(backupFiles.LastFile).ToLocalTime();
}
return(now - lastModified < _retentionPolicy.MinimumBackupAgeToKeep);
}
public async Task <ListBlobResult> ListBlobsAsync(string prefix, string delimiter, bool listFolders, int?maxResult = null, string marker = null)
{
var url = GetUrl(_serverUrlForContainer, "restype=container&comp=list");
if (prefix != null)
{
url += $"&prefix={Uri.EscapeDataString(prefix)}";
}
if (delimiter != null)
{
url += $"&delimiter={delimiter}";
}
if (maxResult != null)
{
url += $"&maxresults={maxResult}";
}
if (marker != null)
{
url += $"&marker={marker}";
}
var requestMessage = new HttpRequestMessage(HttpMethods.Get, url)
{
Headers =
{
{ "x-ms-date", SystemTime.UtcNow.ToString("R") },
{ "x-ms-version", AzureStorageVersion }
}
};
var client = GetClient();
SetAuthorizationHeader(client, HttpMethods.Get, url, requestMessage.Headers);
var response = await client.SendAsync(requestMessage, CancellationToken).ConfigureAwait(false);
if (response.StatusCode == HttpStatusCode.NotFound)
{
return new ListBlobResult
{
List = new List <BlobProperties>()
}
}
;
if (response.IsSuccessStatusCode == false)
{
throw StorageException.FromResponseMessage(response);
}
var responseStream = await response.Content.ReadAsStreamAsync();
var listBlobsResult = XDocument.Load(responseStream);
var result = GetResult();
var nextMarker = listBlobsResult.Root.Element("NextMarker")?.Value;
return(new ListBlobResult
{
List = result,
NextMarker = nextMarker == "true" ? listBlobsResult.Root.Element("NextMarker")?.Value : null
});
IEnumerable <BlobProperties> GetResult()
{
if (listFolders)
{
return(listBlobsResult
.Descendants("Blobs")
.Descendants("Name")
.Select(x => RestorePointsBase.GetDirectoryName(x.Value))
.Distinct()
.Select(x => new BlobProperties
{
Name = x
}));
}
return(listBlobsResult
.Descendants("Blob")
.Select(x => new BlobProperties
{
Name = x.Element("Name")?.Value,
}));
}
}
private bool UpdateFoldersToDelete(GetFoldersResult folders, DateTime now, List <string> foldersToDelete)
{
var firstDateInRetentionRange = now - _retentionPolicy.MinimumBackupAgeToKeep.Value;
foreach (var folder in folders.List)
{
CancellationToken.ThrowIfCancellationRequested();
var folderName = GetFolderName(folder);
var folderDetails = RestorePointsBase.ParseFolderName(folderName);
if (folderDetails.BackupTimeAsString == null)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to get backup date time for folder: {folder}");
}
continue;
}
if (DateTime.TryParseExact(
folderDetails.BackupTimeAsString,
BackupTask.GetDateTimeFormat(folderDetails.BackupTimeAsString),
CultureInfo.InvariantCulture,
DateTimeStyles.None,
out var backupTime) == false)
{
if (Logger.IsInfoEnabled)
{
Logger.Info($"Failed to parse backup date time for folder: {folder}");
}
continue;
}
if (now - backupTime < _retentionPolicy.MinimumBackupAgeToKeep)
{
// all backups are sorted by date
return(false);
}
if (string.Equals(folderDetails.DatabaseName, _databaseName, StringComparison.OrdinalIgnoreCase) == false)
{
continue; // a backup for a different database
}
var backupFiles = GetBackupFilesInFolder(folder, firstDateInRetentionRange);
if (backupFiles == null)
{
continue; // folder is empty
}
var hasFullBackupOrSnapshot = BackupUtils.IsFullBackupOrSnapshot(backupFiles.FirstFile);
if (hasFullBackupOrSnapshot == false)
{
continue; // no snapshot or full backup
}
if (GotFreshIncrementalBackup(backupFiles, now))
{
continue;
}
foldersToDelete.Add(folder);
}
return(true);
}
