public static async Task <HttpResponseMessage> QueueUpBackupPartitions(
[Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
[Property(Name = StorageSettingCopyFromPropertyName)] string storageSettingCopyFrom,
[Property(Name = StorageSettingCopyToPropertyName)] string storageSettingCopyTo,
[Resource] RepositoryBackup repositoryBackup,
AzureApplication application,
RequestMessage <TableBackup> requestQuery,
HttpRequestMessage request,
EastFive.Analytics.ILogger logger,
MultipartResponseAsync <InvocationMessage> onQueued,
AlreadyExistsResponse onAlreadyExists)
{
logger.Trace($"Cleaning backup results");
var repo = AzureTableDriverDynamic.FromStorageString(storageSettingCopyTo);
await DeleteAllAsync(GetRepository(storageSettingCopyTo));
CloudStorageAccount account = CloudStorageAccount
.Parse(storageSettingCopyFrom);
CloudTableClient tableClient =
new CloudTableClient(account.TableEndpoint, account.Credentials);
return(await await repositoryBackup.StorageCreateAsync(
(discard) =>
{
var resourceInfoToProcess = tableClient
.ListTables()
.Distinct()
.Select(
async cloudTable =>
{
var tableBackup = new TableBackup()
{
tableBackupRef = Ref <TableBackup> .NewRef(),
backup = repositoryBackupRef,
tableName = cloudTable.Name,
when = DateTime.UtcNow,
};
var invocationMessage = await requestQuery
.HttpPost(tableBackup)
.CompileRequest(request)
.FunctionAsync();
logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
return invocationMessage;
})
.AsyncEnumerable();
return onQueued(resourceInfoToProcess);
},
() => onAlreadyExists().AsTask()));
}
public static async Task QueueUpBackupContainers(string serviceBusQueueName, string sourceConnectionString, string destConnectionString,
AzureApplication application, EastFive.Analytics.ILogger logger)
{
var containerResources = DiscoverAllContainers();
var blocksSent = await containerResources
.Where(info => !ignoreContainerNames.Contains(info.name.ToLower()))
.Select(
async(info) =>
{
var sw = new Stopwatch();
sw.Start();
logger.Trace($"Queuing backup messages for container {info.name}");
var backupMessages = GeneratePrefixInformation(info.prefixGenerator)
.Select(
(prefix) =>
{
return(new ContainerMessage
{
sourceConnectionString = sourceConnectionString,
destConnectionString = destConnectionString,
name = info.name,
prefix = prefix,
});
})
.Select(message => JsonConvert.SerializeObject(message))
.Select(message => Encoding.UTF8.GetBytes(message))
.ToArray();
await application.SendServiceBusMessageAsync(serviceBusQueueName, backupMessages);
sw.Stop();
logger.Trace($"{backupMessages.Length} messages queued for {info.name} in {sw.Elapsed.TotalSeconds} seconds");
GC.Collect();
return(backupMessages.Length);
})
.AsyncEnumerable() //.Throttle(desiredRunCount: 2)
.ToArrayAsync();
var totalMessages = blocksSent.Sum();
logger.Trace($"Total of {totalMessages} messages queued for all containers");
await ThrowIfContainerIsMissingStorageResourceAttribute(sourceConnectionString, containerResources);
}
public static async Task <IHttpResponse> QueueUpBackupPartitions(
[Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
[Property(Name = StorageSettingCopyFromPropertyName)] string storageSettingCopyFrom,
[Property(Name = StorageSettingCopyToPropertyName)] string storageSettingCopyTo,
[Resource] RepositoryBackup repositoryBackup,
AzureApplication application,
EastFive.Api.Security security,
RequestMessage <TableBackup> requestQuery,
IHttpRequest request,
EastFive.Analytics.ILogger logger,
MultipartAsyncResponse <InvocationMessage> onQueued,
AlreadyExistsResponse onAlreadyExists)
{
CloudStorageAccount account = CloudStorageAccount
.Parse(storageSettingCopyFrom);
CloudTableClient tableClient =
new CloudTableClient(account.TableEndpoint, account.Credentials);
return(await repositoryBackup.StorageCreateAsync(
(discard) =>
{
var includedTables = BackupFunction.DiscoverStorageResources()
.Where(x => x.message.Any())
.Select(x => x.tableName)
.ToArray();
var resourceInfoToProcess = tableClient.GetTables()
.Where(table => includedTables.Contains(table.Name, StringComparer.OrdinalIgnoreCase))
.Distinct()
.Select(
async cloudTable =>
{
var tableBackup = new TableBackup()
{
tableBackupRef = Ref <TableBackup> .NewRef(),
backup = repositoryBackupRef,
tableName = cloudTable.Name,
when = DateTime.UtcNow,
};
var invocationMessage = await requestQuery
.HttpPost(tableBackup)
.CompileRequest(request)
.FunctionAsync();
logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
return invocationMessage;
})
.Await(readAhead: 10);
return onQueued(resourceInfoToProcess);
},
() => onAlreadyExists()));
}
public static async Task ProcessBackupPartitionsAsync(TableMessage message, EastFive.Analytics.ILogger logger)
{
logger.Trace($"Starting backup for table {message.tableName}");
var sw = new Stopwatch();
sw.Start();
var table = message.tableName;
var sourceRepo = GetRepository(message.sourceConnectionString);
var destRepo = GetRepository(message.destConnectionString);
var filter = message.where.FormatWhereInformation()
.Join(" and ");
await Backup.CreateOrUpdateAsync(destRepo, table, filter);
var completedRows = await sourceRepo.Copy(filter, table, destRepo)
.ToArrayAsync();
sw.Stop();
await Backup.CreateOrUpdateAsync(destRepo, table, filter, completedRows.Length, sw.Elapsed.TotalSeconds);
logger.Trace($"copied {completedRows.Length} rows in {sw.Elapsed.TotalSeconds} seconds for {filter}");
}
public static async Task <IHttpResponse> QueueUpBackupPartitions(
[Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
[Property(Name = WhenPropertyName)] DateTime when,
RequestMessage <TableBackup> requestQuery,
IHttpRequest request,
EastFive.Analytics.ILogger logger,
MultipartAsyncResponse <InvocationMessage> onQueued,
NoContentResponse onTooEarly,
NotFoundResponse onNotFound)
{
return(await repositoryBackupRef.StorageUpdateAsync(
async (repoBack, saveAsync) =>
{
var needsToRun = NCrontab.CrontabSchedule.TryParse(repoBack.frequency,
chronSchedule =>
{
var next = chronSchedule.GetNextOccurrence(repoBack.when);
if (when > next)
{
return true;
}
return false;
},
ex =>
{
return false;
});
if (!needsToRun)
{
return onTooEarly();
}
var includedTables = BackupFunction.DiscoverStorageResources()
.Where(x => x.message.Any())
.Select(x => x.tableName)
.ToArray();
CloudStorageAccount account = CloudStorageAccount
.Parse(repoBack.storageSettingCopyFrom);
CloudTableClient tableClient =
new CloudTableClient(account.TableEndpoint, account.Credentials);
var tables = tableClient.GetTables();
var resourceInfoToProcess = tables
.Where(table => includedTables.Contains(table.Name, StringComparer.OrdinalIgnoreCase))
.Distinct()
.Select(
async cloudTable =>
{
var tableBackup = new TableBackup()
{
tableBackupRef = Ref <TableBackup> .NewRef(),
backup = repositoryBackupRef,
tableName = cloudTable.Name,
when = DateTime.UtcNow,
};
var invocationMessage = await requestQuery
.HttpPost(tableBackup)
.CompileRequest(request)
.FunctionAsync();
logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
return invocationMessage;
})
.Await(readAhead: 10);
repoBack.when = when;
await saveAsync(repoBack);
return onQueued(resourceInfoToProcess);
},
() => onNotFound()));
}
