public static IEnumerableAsync <T> OnComplete <T>(this IEnumerableAsync <T> enumerableAsync,
Action <T[]> onComplete,
EastFive.Analytics.ILogger logger = default(ILogger))
{
var enumerator = enumerableAsync.GetEnumerator();
var stack = new Stack <T>();
return(EnumerableAsync.Yield <T>(
async(next, last) =>
{
//if (!tag.IsNullOrWhiteSpace())
// Console.WriteLine($"Join[{tag}] MoveNextAsync.");
if (await enumerator.MoveNextAsync())
{
var current = enumerator.Current;
//if (!tag.IsNullOrWhiteSpace())
// Console.WriteLine($"Join[{tag}] Passthrough on value.");
stack.Push(current);
return next(current);
}
var allValues = stack.ToArray();
if (!logger.IsDefaultOrNull())
{
Console.WriteLine($"OnComplete Accumulated `{allValues.Length}` Values.");
}
onComplete(allValues);
if (!logger.IsDefaultOrNull())
{
Console.WriteLine($"OnComplete Complete.");
}
return last;
}));
}
public static async Task <T[]> ToArrayAsync <T>(this IEnumerableAsync <T> enumerableAsync,
EastFive.Analytics.ILogger logger = default(Analytics.ILogger))
{
var items = await enumerableAsync.Async(logger);
return(items.ToArray());
}
public static IEnumerableAsync <T> JoinTask <T>(this IEnumerableAsync <T> enumerableAsync,
Task task,
EastFive.Analytics.ILogger logger = default(ILogger))
{
var scopedLogger = logger.CreateScope($"Join[{task.Id}]");
var enumerator = enumerableAsync.GetEnumerator();
return(EnumerableAsync.Yield <T>(
async(next, last) =>
{
//if (!tag.IsNullOrWhiteSpace())
// Console.WriteLine($"Join[{tag}] MoveNextAsync.");
if (await enumerator.MoveNextAsync())
{
//if (!tag.IsNullOrWhiteSpace())
// Console.WriteLine($"Join[{tag}] Passthrough on value.");
return next(enumerator.Current);
}
scopedLogger.Trace($"Joining Task.");
await task;
scopedLogger.Trace($"Complete.");
return last;
}));
}
public static IEnumerableAsync <T> JoinTask <T>(this IEnumerableAsync <T> enumerableAsync,
Func <Task> task,
EastFive.Analytics.ILogger logger = default(ILogger))
{
enumerableAsync
.JoinTask(
Task.Run(task),
logger);
return(enumerableAsync);
}
public static async Task <IHttpResponse> QueueUpBackupPartitions(
[Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
[Property(Name = StorageSettingCopyFromPropertyName)] string storageSettingCopyFrom,
[Property(Name = StorageSettingCopyToPropertyName)] string storageSettingCopyTo,
[Resource] RepositoryBackup repositoryBackup,
AzureApplication application,
EastFive.Api.Security security,
RequestMessage <TableBackup> requestQuery,
IHttpRequest request,
EastFive.Analytics.ILogger logger,
MultipartAsyncResponse <InvocationMessage> onQueued,
AlreadyExistsResponse onAlreadyExists)
{
CloudStorageAccount account = CloudStorageAccount
.Parse(storageSettingCopyFrom);
CloudTableClient tableClient =
new CloudTableClient(account.TableEndpoint, account.Credentials);
return(await repositoryBackup.StorageCreateAsync(
(discard) =>
{
var includedTables = BackupFunction.DiscoverStorageResources()
.Where(x => x.message.Any())
.Select(x => x.tableName)
.ToArray();
var resourceInfoToProcess = tableClient.GetTables()
.Where(table => includedTables.Contains(table.Name, StringComparer.OrdinalIgnoreCase))
.Distinct()
.Select(
async cloudTable =>
{
var tableBackup = new TableBackup()
{
tableBackupRef = Ref <TableBackup> .NewRef(),
backup = repositoryBackupRef,
tableName = cloudTable.Name,
when = DateTime.UtcNow,
};
var invocationMessage = await requestQuery
.HttpPost(tableBackup)
.CompileRequest(request)
.FunctionAsync();
logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
return invocationMessage;
})
.Await(readAhead: 10);
return onQueued(resourceInfoToProcess);
},
() => onAlreadyExists()));
}
public static async Task <HttpResponseMessage> RunAsync(
[UpdateId] IRefs <InvocationMessage> invocationMessageRefs,
[HeaderLog] EastFive.Analytics.ILogger analyticsLog,
InvokeApplicationDirect invokeApplication,
MultipartResponseAsync onRun)
{
var messages = await invocationMessageRefs.refs
.Select(invocationMessageRef => InvokeAsync(invocationMessageRef, invokeApplication, logging: analyticsLog))
.AsyncEnumerable()
.ToArrayAsync();
return(await onRun(messages));
}
public static async Task <HttpResponseMessage> QueueUpBackupPartitions(
[Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
[Property(Name = StorageSettingCopyFromPropertyName)] string storageSettingCopyFrom,
[Property(Name = StorageSettingCopyToPropertyName)] string storageSettingCopyTo,
[Resource] RepositoryBackup repositoryBackup,
AzureApplication application,
RequestMessage <TableBackup> requestQuery,
HttpRequestMessage request,
EastFive.Analytics.ILogger logger,
MultipartResponseAsync <InvocationMessage> onQueued,
AlreadyExistsResponse onAlreadyExists)
{
logger.Trace($"Cleaning backup results");
var repo = AzureTableDriverDynamic.FromStorageString(storageSettingCopyTo);
await DeleteAllAsync(GetRepository(storageSettingCopyTo));
CloudStorageAccount account = CloudStorageAccount
.Parse(storageSettingCopyFrom);
CloudTableClient tableClient =
new CloudTableClient(account.TableEndpoint, account.Credentials);
return(await await repositoryBackup.StorageCreateAsync(
(discard) =>
{
var resourceInfoToProcess = tableClient
.ListTables()
.Distinct()
.Select(
async cloudTable =>
{
var tableBackup = new TableBackup()
{
tableBackupRef = Ref <TableBackup> .NewRef(),
backup = repositoryBackupRef,
tableName = cloudTable.Name,
when = DateTime.UtcNow,
};
var invocationMessage = await requestQuery
.HttpPost(tableBackup)
.CompileRequest(request)
.FunctionAsync();
logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
return invocationMessage;
})
.AsyncEnumerable();
return onQueued(resourceInfoToProcess);
},
() => onAlreadyExists().AsTask()));
}
public static Task <HttpResponseMessage> ListAsync(
[QueryParameter(Name = "last_modified")] DateTime day,
[HeaderLog] EastFive.Analytics.ILogger analyticsLog,
InvokeApplicationDirect invokeApplication,
MultipartResponseAsync <InvocationMessage> onRun)
{
Expression <Func <InvocationMessage, bool> > expr = (im) => true;
var messages = expr
.StorageQuery()
.Where(msg => DateTime.UtcNow - msg.lastModified < TimeSpan.FromDays(3.0));
return(onRun(messages));
}
public static IHttpResponse ListAsync(
[QueryParameter(Name = "start_time")] DateTime startTime,
[QueryParameter(Name = "end_time")] DateTime endTime,
[HeaderLog] EastFive.Analytics.ILogger analyticsLog,
MultipartAsyncResponse <InvocationMessage> onRun)
{
Expression <Func <InvocationMessage, bool> > allQuery = (im) => true;
var messages = allQuery
.StorageQuery()
.Where(msg => msg.lastModified >= startTime)
.Where(msg => msg.lastModified <= endTime);
return(onRun(messages));
}
public static IEnumerableAsync <HttpResponseMessage> InvokeAsync(
byte [] invocationMessageIdsBytes,
IApplication application,
IInvokeApplication invokeApplication,
EastFive.Analytics.ILogger analyticsLog = default)
{
return(invocationMessageIdsBytes
.Split(index => 16)
.Select(
invocationMessageIdBytes =>
{
var idBytes = invocationMessageIdBytes.ToArray();
var invocationMessageId = new Guid(idBytes);
var invocationMessageRef = invocationMessageId.AsRef <InvocationMessage>();
return InvokeAsync(invocationMessageRef, invokeApplication, analyticsLog);
})
.Parallel());
}
public static async Task <HttpResponseMessage> CreateAsync(
[Property(Name = IdPropertyName)] IRef <TableBackup> tableBackupRef,
[Property(Name = WhenPropertyName)] DateTime when,
[Property(Name = TableNamePropertyName)] string tableName,
[Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
[Resource] TableBackup tableBackup,
RequestMessage <TableBackup> requestQuery,
HttpRequestMessage request,
EastFive.Analytics.ILogger logger,
CreatedBodyResponse <InvocationMessage> onCreated,
AlreadyExistsResponse onAlreadyExists)
{
return(await await tableBackup.StorageCreateAsync(
async (entity) =>
{
var invocationMessage = await requestQuery
.ById(tableBackupRef)
.HttpPatch(default)
public static async Task <IHttpResponse> CreateAsync(
[Property(Name = IdPropertyName)] IRef <TableBackupOperation> tableBackupOperationRef,
[Property(Name = WhenPropertyName)] DateTime when,
[Property(Name = TableBackupPropertyName)] IRef <TableBackup> tableBackupRef,
[Property(Name = OperationSetPropertyName)] string operationSet,
[Resource] TableBackupOperation tableBackup,
RequestMessage <TableBackupOperation> requestQuery,
IHttpRequest request,
EastFive.Api.Security security,
EastFive.Analytics.ILogger logger,
CreatedBodyResponse <InvocationMessage> onCreated,
AlreadyExistsResponse onAlreadyExists)
{
return(await await tableBackup.StorageCreateAsync(
async (discard) =>
{
var invocationMessage = await requestQuery
.ById(tableBackupOperationRef)
.HttpPatch(default)
public static async Task ProcessBackupPartitionsAsync(TableMessage message, EastFive.Analytics.ILogger logger)
{
logger.Trace($"Starting backup for table {message.tableName}");
var sw = new Stopwatch();
sw.Start();
var table = message.tableName;
var sourceRepo = GetRepository(message.sourceConnectionString);
var destRepo = GetRepository(message.destConnectionString);
var filter = message.where.FormatWhereInformation()
.Join(" and ");
await Backup.CreateOrUpdateAsync(destRepo, table, filter);
var completedRows = await sourceRepo.Copy(filter, table, destRepo)
.ToArrayAsync();
sw.Stop();
await Backup.CreateOrUpdateAsync(destRepo, table, filter, completedRows.Length, sw.Elapsed.TotalSeconds);
logger.Trace($"copied {completedRows.Length} rows in {sw.Elapsed.TotalSeconds} seconds for {filter}");
}
public static async Task <IHttpResponse> QueueUpBackupPartitions(
[Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
[Property(Name = WhenPropertyName)] DateTime when,
RequestMessage <TableBackup> requestQuery,
IHttpRequest request,
EastFive.Analytics.ILogger logger,
MultipartAsyncResponse <InvocationMessage> onQueued,
NoContentResponse onTooEarly,
NotFoundResponse onNotFound)
{
return(await repositoryBackupRef.StorageUpdateAsync(
async (repoBack, saveAsync) =>
{
var needsToRun = NCrontab.CrontabSchedule.TryParse(repoBack.frequency,
chronSchedule =>
{
var next = chronSchedule.GetNextOccurrence(repoBack.when);
if (when > next)
{
return true;
}
return false;
},
ex =>
{
return false;
});
if (!needsToRun)
{
return onTooEarly();
}
var includedTables = BackupFunction.DiscoverStorageResources()
.Where(x => x.message.Any())
.Select(x => x.tableName)
.ToArray();
CloudStorageAccount account = CloudStorageAccount
.Parse(repoBack.storageSettingCopyFrom);
CloudTableClient tableClient =
new CloudTableClient(account.TableEndpoint, account.Credentials);
var tables = tableClient.GetTables();
var resourceInfoToProcess = tables
.Where(table => includedTables.Contains(table.Name, StringComparer.OrdinalIgnoreCase))
.Distinct()
.Select(
async cloudTable =>
{
var tableBackup = new TableBackup()
{
tableBackupRef = Ref <TableBackup> .NewRef(),
backup = repositoryBackupRef,
tableName = cloudTable.Name,
when = DateTime.UtcNow,
};
var invocationMessage = await requestQuery
.HttpPost(tableBackup)
.CompileRequest(request)
.FunctionAsync();
logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
return invocationMessage;
})
.Await(readAhead: 10);
repoBack.when = when;
await saveAsync(repoBack);
return onQueued(resourceInfoToProcess);
},
() => onNotFound()));
}
public static async Task QueueUpBackupContainers(string serviceBusQueueName, string sourceConnectionString, string destConnectionString,
AzureApplication application, EastFive.Analytics.ILogger logger)
{
var containerResources = DiscoverAllContainers();
var blocksSent = await containerResources
.Where(info => !ignoreContainerNames.Contains(info.name.ToLower()))
.Select(
async(info) =>
{
var sw = new Stopwatch();
sw.Start();
logger.Trace($"Queuing backup messages for container {info.name}");
var backupMessages = GeneratePrefixInformation(info.prefixGenerator)
.Select(
(prefix) =>
{
return(new ContainerMessage
{
sourceConnectionString = sourceConnectionString,
destConnectionString = destConnectionString,
name = info.name,
prefix = prefix,
});
})
.Select(message => JsonConvert.SerializeObject(message))
.Select(message => Encoding.UTF8.GetBytes(message))
.ToArray();
await application.SendServiceBusMessageAsync(serviceBusQueueName, backupMessages);
sw.Stop();
logger.Trace($"{backupMessages.Length} messages queued for {info.name} in {sw.Elapsed.TotalSeconds} seconds");
GC.Collect();
return(backupMessages.Length);
})
.AsyncEnumerable() //.Throttle(desiredRunCount: 2)
.ToArrayAsync();
var totalMessages = blocksSent.Sum();
logger.Trace($"Total of {totalMessages} messages queued for all containers");
await ThrowIfContainerIsMissingStorageResourceAttribute(sourceConnectionString, containerResources);
}
