protected override GetFoldersResult GetSortedFolders()
{
var prefix = string.IsNullOrWhiteSpace(_client.RemoteFolderName) ? string.Empty : $"{_client.RemoteFolderName}{Delimiter}";
var result = _client.ListBlobs(prefix, Delimiter, listFolders: true, marker: _nextMarker);
_nextMarker = result.NextMarker;
return(new GetFoldersResult
{
List = result.List.Select(x => x.Name).OrderBy(x => x).ToList(),
HasMore = result.NextMarker != null
});
}
protected override async Task <List <string> > GetFilesForRestore()
{
var prefix = string.IsNullOrEmpty(_remoteFolderName) ? "" : _remoteFolderName;
var allObjects = await _client.ListBlobs(prefix, string.Empty, false);
return(allObjects.ListBlob.Select(x => new string(x.Name)).ToList());
}
public override async Task FetchRestorePoints(string path)
{
var objects = await _client.ListBlobs(path, "/", true);
var folders = objects.ListBlob.ToList();
if (folders.Count == 0)
{
await FetchRestorePointsForPath(path, assertLegacyBackups : true);
}
else
{
foreach (var folder in folders)
{
await FetchRestorePointsForPath(folder.Name, assertLegacyBackups : true);
}
}
}
public async Task GetFolderPathOptions()
{
PeriodicBackupConnectionType connectionType;
var type = GetStringValuesQueryString("type", false).FirstOrDefault();
if (type == null)
{
//Backward compatibility
connectionType = PeriodicBackupConnectionType.Local;
}
else if (Enum.TryParse(type, out connectionType) == false)
{
throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var folderPathOptions = new FolderPathOptions();;
switch (connectionType)
{
case PeriodicBackupConnectionType.Local:
var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
var path = GetStringQueryString("path", required: false);
folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
break;
case PeriodicBackupConnectionType.S3:
var json = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && json == null)
{
throw new BadRequestException("No JSON was posted.");
}
var s3Settings = JsonDeserializationServer.S3Settings(json);
if (s3Settings == null)
{
throw new BadRequestException("No S3Settings were found.");
}
if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) ||
string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) ||
string.IsNullOrWhiteSpace(s3Settings.BucketName) ||
string.IsNullOrWhiteSpace(s3Settings.AwsRegionName))
{
break;
}
using (var client = new RavenAwsS3Client(s3Settings))
{
// fetching only the first 64 results for the auto complete
var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, 64);
if (folders != null)
{
foreach (var folder in folders.FileInfoDetails)
{
var fullPath = folder.FullPath;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
}
break;
case PeriodicBackupConnectionType.Azure:
var azureJson = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var azureSettings = JsonDeserializationServer.AzureSettings(azureJson);
if (azureSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(azureSettings.AccountName) ||
string.IsNullOrWhiteSpace(azureSettings.AccountKey) ||
string.IsNullOrWhiteSpace(azureSettings.StorageContainer))
{
break;
}
using (var client = new RavenAzureClient(azureSettings))
{
var folders = (await client.ListBlobs(azureSettings.RemoteFolderName, "/", true));
foreach (var folder in folders.ListBlob)
{
var fullPath = folder.Name;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
break;
case PeriodicBackupConnectionType.GoogleCloud:
var googleCloudJson = context.ReadForMemory(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson);
if (googleCloudSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) ||
string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson))
{
break;
}
using (var client = new RavenGoogleCloudClient(googleCloudSettings))
{
var folders = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName));
var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length;
foreach (var folder in folders)
{
const char separator = '/';
var splitted = folder.Name.Split(separator);
var result = string.Join(separator, splitted.Take(requestedPathLength)) + separator;
if (string.IsNullOrWhiteSpace(result))
{
continue;
}
folderPathOptions.List.Add(result);
}
}
break;
case PeriodicBackupConnectionType.FTP:
case PeriodicBackupConnectionType.Glacier:
throw new NotSupportedException();
default:
throw new ArgumentOutOfRangeException();
}
using (var writer = new BlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
});
}
}
}
private static void PutBlobs(int blobsCount, bool useSasToken)
{
var containerName = Guid.NewGuid().ToString();
var blobNamesToPut = new List <string>();
for (var i = 0; i < blobsCount; i++)
{
blobNamesToPut.Add($"azure/{Guid.NewGuid()}/{i}");
}
var sasToken = useSasToken ? GetSasTokenAndCreateTheContainer(containerName) : null;
using (var client = new RavenAzureClient(GetAzureSettings(containerName, sasToken)))
{
try
{
if (useSasToken == false)
{
client.DeleteContainer();
client.PutContainer();
}
for (var i = 0; i < blobsCount; i++)
{
client.PutBlob(blobNamesToPut[i], new MemoryStream(Encoding.UTF8.GetBytes("123")),
new Dictionary <string, string> {
{ "property1", "value1" }, { "property2", "value2" }
});
}
for (var i = 0; i < blobsCount; i++)
{
var blob = client.GetBlob(blobNamesToPut[i]);
Assert.NotNull(blob);
using (var reader = new StreamReader(blob.Data))
Assert.Equal("123", reader.ReadToEnd());
var property1 = blob.Metadata.Keys.Single(x => x.Contains("property1"));
var property2 = blob.Metadata.Keys.Single(x => x.Contains("property2"));
Assert.Equal("value1", blob.Metadata[property1]);
Assert.Equal("value2", blob.Metadata[property2]);
}
var listBlobs = client.ListBlobs("azure", null, listFolders: false);
var blobNames = listBlobs.List.Select(b => b.Name).ToList();
Assert.Equal(blobsCount, blobNames.Count);
// delete all blobs
client.DeleteMultipleBlobs(blobNames);
listBlobs = client.ListBlobs("azure", null, listFolders: false);
blobNames = listBlobs.List.Select(b => b.Name).ToList();
Assert.Equal(0, blobNames.Count);
for (var i = 0; i < blobsCount; i++)
{
var blob = client.GetBlob(blobNamesToPut[i]);
Assert.Null(blob);
}
}
finally
{
client.DeleteContainer();
}
}
}
