private UploaderSettings GenerateUploaderSetting(Config.Categories.BackupConfiguration configuration)
{
var s3Settings = BackupTask.GetBackupConfigurationFromScript(Configuration.Connection.S3Settings, x => JsonDeserializationServer.S3Settings(x),
Database, updateServerWideSettingsFunc: null, serverWide: false);
var azureSettings = BackupTask.GetBackupConfigurationFromScript(Configuration.Connection.AzureSettings, x => JsonDeserializationServer.AzureSettings(x),
Database, updateServerWideSettingsFunc: null, serverWide: false);
var glacierSettings = BackupTask.GetBackupConfigurationFromScript(Configuration.Connection.GlacierSettings, x => JsonDeserializationServer.GlacierSettings(x),
Database, updateServerWideSettingsFunc: null, serverWide: false);
var googleCloudSettings = BackupTask.GetBackupConfigurationFromScript(Configuration.Connection.GoogleCloudSettings, x => JsonDeserializationServer.GoogleCloudSettings(x),
Database, updateServerWideSettingsFunc: null, serverWide: false);
var ftpSettings = BackupTask.GetBackupConfigurationFromScript(Configuration.Connection.FtpSettings, x => JsonDeserializationServer.FtpSettings(x),
Database, updateServerWideSettingsFunc: null, serverWide: false);
return(new UploaderSettings(configuration)
{
S3Settings = s3Settings,
AzureSettings = azureSettings,
GlacierSettings = glacierSettings,
GoogleCloudSettings = googleCloudSettings,
FtpSettings = ftpSettings,
DatabaseName = Database.Name,
TaskName = Name
});
}
public async Task GetFolderPathOptions()
{
PeriodicBackupConnectionType connectionType;
var type = GetStringValuesQueryString("type", false).FirstOrDefault();
if (type == null)
{
//Backward compatibility
connectionType = PeriodicBackupConnectionType.Local;
}
else if (Enum.TryParse(type, out connectionType) == false)
{
throw new ArgumentException($"Query string '{type}' was not recognized as valid type");
}
using (ServerStore.ContextPool.AllocateOperationContext(out JsonOperationContext context))
{
var folderPathOptions = new FolderPathOptions();
;
switch (connectionType)
{
case PeriodicBackupConnectionType.Local:
var isBackupFolder = GetBoolValueQueryString("backupFolder", required: false) ?? false;
var path = GetStringQueryString("path", required: false);
folderPathOptions = FolderPath.GetOptions(path, isBackupFolder, ServerStore.Configuration);
break;
case PeriodicBackupConnectionType.S3:
var json = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && json == null)
{
throw new BadRequestException("No JSON was posted.");
}
var s3Settings = JsonDeserializationServer.S3Settings(json);
if (s3Settings == null)
{
throw new BadRequestException("No S3Settings were found.");
}
if (string.IsNullOrWhiteSpace(s3Settings.AwsAccessKey) ||
string.IsNullOrWhiteSpace(s3Settings.AwsSecretKey) ||
string.IsNullOrWhiteSpace(s3Settings.BucketName) ||
string.IsNullOrWhiteSpace(s3Settings.AwsRegionName))
{
break;
}
using (var client = new RavenAwsS3Client(s3Settings, ServerStore.Configuration.Backup))
{
// fetching only the first 64 results for the auto complete
var folders = await client.ListObjectsAsync(s3Settings.RemoteFolderName, "/", true, take : 64);
if (folders != null)
{
foreach (var folder in folders.FileInfoDetails)
{
var fullPath = folder.FullPath;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
}
break;
case PeriodicBackupConnectionType.Azure:
var azureJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && azureJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var azureSettings = JsonDeserializationServer.AzureSettings(azureJson);
if (azureSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(azureSettings.AccountName) ||
string.IsNullOrWhiteSpace(azureSettings.AccountKey) ||
string.IsNullOrWhiteSpace(azureSettings.StorageContainer))
{
break;
}
using (var client = RavenAzureClient.Create(azureSettings, ServerStore.Configuration.Backup))
{
var folders = (await client.ListBlobsAsync(azureSettings.RemoteFolderName, "/", true));
foreach (var folder in folders.List)
{
var fullPath = folder.Name;
if (string.IsNullOrWhiteSpace(fullPath))
{
continue;
}
folderPathOptions.List.Add(fullPath);
}
}
break;
case PeriodicBackupConnectionType.GoogleCloud:
var googleCloudJson = await context.ReadForMemoryAsync(RequestBodyStream(), "studio-tasks/format");
if (connectionType != PeriodicBackupConnectionType.Local && googleCloudJson == null)
{
throw new BadRequestException("No JSON was posted.");
}
var googleCloudSettings = JsonDeserializationServer.GoogleCloudSettings(googleCloudJson);
if (googleCloudSettings == null)
{
throw new BadRequestException("No AzureSettings were found.");
}
if (string.IsNullOrWhiteSpace(googleCloudSettings.BucketName) ||
string.IsNullOrWhiteSpace(googleCloudSettings.GoogleCredentialsJson))
{
break;
}
using (var client = new RavenGoogleCloudClient(googleCloudSettings, ServerStore.Configuration.Backup))
{
var folders = (await client.ListObjectsAsync(googleCloudSettings.RemoteFolderName));
var requestedPathLength = googleCloudSettings.RemoteFolderName.Split('/').Length;
foreach (var folder in folders)
{
const char separator = '/';
var splitted = folder.Name.Split(separator);
var result = string.Join(separator, splitted.Take(requestedPathLength)) + separator;
if (string.IsNullOrWhiteSpace(result))
{
continue;
}
folderPathOptions.List.Add(result);
}
}
break;
case PeriodicBackupConnectionType.FTP:
case PeriodicBackupConnectionType.Glacier:
throw new NotSupportedException();
default:
throw new ArgumentOutOfRangeException();
}
await using (var writer = new AsyncBlittableJsonTextWriter(context, ResponseBodyStream()))
{
context.Write(writer, new DynamicJsonValue
{
[nameof(FolderPathOptions.List)] = TypeConverter.ToBlittableSupportedType(folderPathOptions.List)
});
}
}
}
private void UploadToServer(string backupPath, string folderName, string fileName)
{
var s3Settings = GetBackupConfigurationFromScript(_configuration.S3Settings, x => JsonDeserializationServer.S3Settings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForS3(settings, _database.Name));
var glacierSettings = GetBackupConfigurationFromScript(_configuration.GlacierSettings, x => JsonDeserializationServer.GlacierSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGlacier(settings, _database.Name));
var azureSettings = GetBackupConfigurationFromScript(_configuration.AzureSettings, x => JsonDeserializationServer.AzureSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForAzure(settings, _database.Name));
var googleCloudSettings = GetBackupConfigurationFromScript(_configuration.GoogleCloudSettings, x => JsonDeserializationServer.GoogleCloudSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForGoogleCloud(settings, _database.Name));
var ftpSettings = GetBackupConfigurationFromScript(_configuration.FtpSettings, x => JsonDeserializationServer.FtpSettings(x),
settings => PutServerWideBackupConfigurationCommand.UpdateSettingsForFtp(settings, _database.Name));
TaskCancelToken.Token.ThrowIfCancellationRequested();
var uploaderSettings = new BackupUploaderSettings(_database.Configuration.Backup)
{
S3Settings = s3Settings,
GlacierSettings = glacierSettings,
AzureSettings = azureSettings,
GoogleCloudSettings = googleCloudSettings,
FtpSettings = ftpSettings,
BackupPath = backupPath,
FolderName = folderName,
FileName = fileName,
DatabaseName = _database.Name,
TaskName = _taskName,
BackupType = _configuration.BackupType
};
var backupUploader = new BackupUploader(uploaderSettings, _retentionPolicyParameters, _logger, _backupResult, _onProgress, TaskCancelToken);
backupUploader.Execute();
}