public static void UpdateSettingsForGoogleCloud(GoogleCloudSettings googleCloudSettings, string databaseName)
{
if (googleCloudSettings != null)
{
googleCloudSettings.RemoteFolderName = GetUpdatedPath(googleCloudSettings.RemoteFolderName, databaseName);
}
}
public RavenGoogleCloudClient(GoogleCloudSettings settings, Progress progress = null, CancellationToken?cancellationToken = null)
{
if (string.IsNullOrWhiteSpace(settings.BucketName))
{
throw new ArgumentException("Google Cloud Bucket name cannot be null or empty");
}
if (string.IsNullOrWhiteSpace(settings.GoogleCredentialsJson))
{
throw new ArgumentException("Google Credentials JSON cannot be null or empty");
}
try
{
_client = StorageClient.Create(GoogleCredential.FromJson(settings.GoogleCredentialsJson));
}
catch (Exception e)
{
throw new ArgumentException("Wrong format for Google Credentials.", e);
}
var credentialJsonType = JObject.Parse(settings.GoogleCredentialsJson);
if (credentialJsonType.TryGetValue(ProjectIdPropertyName, StringComparison.OrdinalIgnoreCase, out var value))
{
_projectId = value.Value <string>();
}
_bucketName = settings.BucketName;
CancellationToken = cancellationToken ?? CancellationToken.None;
_progress = progress;
}
private void SetupGoogleCloudOlapEtl(DocumentStore store, string script, GoogleCloudSettings settings, string customPartition = null, string transformationName = null)
{
var connectionStringName = $"{store.Database} to GoogleCloud";
var configuration = new OlapEtlConfiguration
{
Name = "olap-gcp-test",
ConnectionStringName = connectionStringName,
RunFrequency = LocalTests.DefaultFrequency,
CustomPartitionValue = customPartition,
Transforms =
{
new Transformation
{
Name = transformationName ?? "MonthlyOrders",
Collections = new List <string>{
"Orders"
},
Script = script
}
}
};
AddEtl(store, configuration, new OlapConnectionString
{
Name = connectionStringName,
GoogleCloudSettings = settings
});
}
static GoogleCloudFact()
{
GoogleCloudSettings = new GoogleCloudSettings
{
BucketName = Environment.GetEnvironmentVariable(BucketNameEnvironmentVariable, EnvironmentVariableTarget.User),
GoogleCredentialsJson = Environment.GetEnvironmentVariable(GoogleCloudCredentialEnvironmentVariable, EnvironmentVariableTarget.User)
};
}
public StorageService(GoogleCloudSettings googleCloudSettings)
{
var json = JsonConvert.SerializeObject(googleCloudSettings);
var credential = GoogleCredential.FromJson(json);
_client = StorageClient.Create(credential);
_googleCloudSettings = googleCloudSettings;
}
static GoogleCloudFactAttribute()
{
GoogleCloudSettings = new GoogleCloudSettings
{
BucketName = Environment.GetEnvironmentVariable(BucketNameEnvironmentVariable),
GoogleCredentialsJson = Environment.GetEnvironmentVariable(GoogleCloudCredentialEnvironmentVariable)
};
}
public PeriodicBackupConfiguration CreateBackupConfiguration(string backupPath = null, BackupType backupType = BackupType.Backup, bool disabled = false, string fullBackupFrequency = "0 0 1 1 *",
string incrementalBackupFrequency = null, long?taskId = null, string mentorNode = null, BackupEncryptionSettings backupEncryptionSettings = null, AzureSettings azureSettings = null,
GoogleCloudSettings googleCloudSettings = null, S3Settings s3Settings = null, RetentionPolicy retentionPolicy = null, string name = null)
{
var config = new PeriodicBackupConfiguration()
{
BackupType = backupType,
FullBackupFrequency = fullBackupFrequency,
Disabled = disabled
};
if (taskId.HasValue)
{
config.TaskId = taskId.Value;
}
if (string.IsNullOrEmpty(mentorNode) == false)
{
config.MentorNode = mentorNode;
}
if (string.IsNullOrEmpty(name) == false)
{
config.Name = name;
}
if (string.IsNullOrEmpty(incrementalBackupFrequency) == false)
{
config.IncrementalBackupFrequency = incrementalBackupFrequency;
}
if (string.IsNullOrEmpty(backupPath) == false)
{
config.LocalSettings = new LocalSettings {
FolderPath = backupPath
}
}
;
if (backupEncryptionSettings != null)
{
config.BackupEncryptionSettings = backupEncryptionSettings;
}
if (azureSettings != null)
{
config.AzureSettings = azureSettings;
}
if (googleCloudSettings != null)
{
config.GoogleCloudSettings = googleCloudSettings;
}
if (s3Settings != null)
{
config.S3Settings = s3Settings;
}
if (retentionPolicy != null)
{
config.RetentionPolicy = retentionPolicy;
}
return(config);
}
private void SetupGoogleCloudOlapEtl(DocumentStore store, GoogleCloudSettings settings, OlapEtlConfiguration configuration)
{
var connectionStringName = $"{store.Database} to GCP";
AddEtl(store, configuration, new OlapConnectionString
{
Name = connectionStringName,
GoogleCloudSettings = settings
});
}
public override DynamicJsonValue ToJson()
{
var json = base.ToJson();
json[nameof(LocalSettings)] = LocalSettings?.ToJson();
json[nameof(S3Settings)] = S3Settings?.ToJson();
json[nameof(AzureSettings)] = AzureSettings?.ToJson();
json[nameof(GlacierSettings)] = GlacierSettings?.ToJson();
json[nameof(GoogleCloudSettings)] = GoogleCloudSettings?.ToJson();
json[nameof(FtpSettings)] = FtpSettings?.ToJson();
return(json);
}
private void DeleteFromGoogleCloud(GoogleCloudSettings settings)
{
using (var client = new RavenGoogleCloudClient(settings, _settings.Configuration, progress: null, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.DeleteObject(key);
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportDeletion(GoogleCloudName)} storage bucket: {settings.BucketName}");
}
}
}
protected override void ValidateImpl(ref List <string> errors)
{
if (S3Settings != null)
{
if (S3Settings.HasSettings() == false)
{
errors.Add($"{nameof(S3Settings)} has no valid setting. '{nameof(S3Settings.BucketName)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (AzureSettings != null)
{
if (AzureSettings.HasSettings() == false)
{
errors.Add($"{nameof(AzureSettings)} has no valid setting. '{nameof(AzureSettings.StorageContainer)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (GlacierSettings != null)
{
if (GlacierSettings.HasSettings() == false)
{
errors.Add($"{nameof(GlacierSettings)} has no valid setting. '{nameof(GlacierSettings.VaultName)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (GoogleCloudSettings != null)
{
if (GoogleCloudSettings.HasSettings() == false)
{
errors.Add($"{nameof(GoogleCloudSettings)} has no valid setting. '{nameof(GoogleCloudSettings.BucketName)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (FtpSettings != null)
{
if (FtpSettings.HasSettings() == false)
{
errors.Add($"{nameof(FtpSettings)} has no valid setting. '{nameof(FtpSettings.Port)}' is 0 and '{nameof(FtpSettings.Url)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
if (LocalSettings != null)
{
if (LocalSettings.HasSettings() == false)
{
errors.Add($"{nameof(LocalSettings)} has no valid setting. '{nameof(LocalSettings.FolderPath)}' and '{nameof(GetBackupConfigurationScript)}' are both null");
}
}
}
private void UploadToGoogleCloud(GoogleCloudSettings settings, Stream stream, Progress progress)
{
using (var client = new RavenGoogleCloudClient(settings, _settings.Configuration, progress, TaskCancelToken.Token))
{
var key = CombinePathAndKey(settings.RemoteFolderName);
client.UploadObject(key, stream, new Dictionary <string, string>
{
{ "Description", GetArchiveDescription() }
});
if (_logger.IsInfoEnabled)
{
_logger.Info($"{ReportSuccess(GoogleCloudName)} storage bucket: {settings.BucketName}");
}
var runner = new GoogleCloudRetentionPolicyRunner(_retentionPolicyParameters, client);
runner.Execute();
}
}
private static async Task DeleteObjects(GoogleCloudSettings settings)
{
if (settings == null)
{
return;
}
try
{
using (var client = new RavenGoogleCloudClient(settings, DefaultBackupConfiguration))
{
var all = await client.ListObjectsAsync(prefix : settings.RemoteFolderName);
foreach (var obj in all)
{
await client.DeleteObjectAsync(obj.Name);
}
}
}
catch (Exception)
{
// ignored
}
}
public GoogleCloudRestorePoints(SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, GoogleCloudSettings client) : base(sortedList, context)
{
_client = new RavenGoogleCloudClient(client);
}
public GoogleApiTimezoneLookup(GoogleCloudSettings settings)
{
_settings = settings;
}
public GoogleCloudStorageService(IOptionsSnapshot <GoogleCloudSettings> googleCloudSettings, IStorageClientWrapper storageClientWrapper)
{
this.googleCloudSettings = googleCloudSettings.Value;
this.storageClient = storageClientWrapper.GetStorageClient(googleCloudSettings.Value);
}
public StorageClient GetStorageClient(GoogleCloudSettings googleCloudSettings)
{
var googleCredentials = GoogleCredential.FromJson(JsonSerializer.Serialize(googleCloudSettings.GoogleCloudKey));
return(Create(googleCredentials));
}
public GoogleCloudRestorePoints(Config.Categories.BackupConfiguration configuration, SortedList <DateTime, RestorePoint> sortedList, TransactionOperationContext context, GoogleCloudSettings client) : base(sortedList, context)
{
_configuration = configuration;
_client = new RavenGoogleCloudClient(client, configuration);
}
