Beispiel #1
0
        public static IEnumerableAsync <T> OnComplete <T>(this IEnumerableAsync <T> enumerableAsync,
                                                          Action <T[]> onComplete,
                                                          EastFive.Analytics.ILogger logger = default(ILogger))
        {
            var enumerator = enumerableAsync.GetEnumerator();
            var stack      = new Stack <T>();

            return(EnumerableAsync.Yield <T>(
                       async(next, last) =>
            {
                //if (!tag.IsNullOrWhiteSpace())
                //    Console.WriteLine($"Join[{tag}] MoveNextAsync.");
                if (await enumerator.MoveNextAsync())
                {
                    var current = enumerator.Current;
                    //if (!tag.IsNullOrWhiteSpace())
                    //    Console.WriteLine($"Join[{tag}] Passthrough on value.");
                    stack.Push(current);
                    return next(current);
                }

                var allValues = stack.ToArray();
                if (!logger.IsDefaultOrNull())
                {
                    Console.WriteLine($"OnComplete Accumulated `{allValues.Length}` Values.");
                }
                onComplete(allValues);
                if (!logger.IsDefaultOrNull())
                {
                    Console.WriteLine($"OnComplete Complete.");
                }

                return last;
            }));
        }
Beispiel #2
0
        public static async Task <T[]> ToArrayAsync <T>(this IEnumerableAsync <T> enumerableAsync,
                                                        EastFive.Analytics.ILogger logger = default(Analytics.ILogger))
        {
            var items = await enumerableAsync.Async(logger);

            return(items.ToArray());
        }
Beispiel #3
0
        public static IEnumerableAsync <T> JoinTask <T>(this IEnumerableAsync <T> enumerableAsync,
                                                        Task task,
                                                        EastFive.Analytics.ILogger logger = default(ILogger))
        {
            var scopedLogger = logger.CreateScope($"Join[{task.Id}]");
            var enumerator   = enumerableAsync.GetEnumerator();

            return(EnumerableAsync.Yield <T>(
                       async(next, last) =>
            {
                //if (!tag.IsNullOrWhiteSpace())
                //    Console.WriteLine($"Join[{tag}] MoveNextAsync.");
                if (await enumerator.MoveNextAsync())
                {
                    //if (!tag.IsNullOrWhiteSpace())
                    //    Console.WriteLine($"Join[{tag}] Passthrough on value.");
                    return next(enumerator.Current);
                }

                scopedLogger.Trace($"Joining Task.");
                await task;
                scopedLogger.Trace($"Complete.");
                return last;
            }));
        }
Beispiel #4
0
 public static IEnumerableAsync <T> JoinTask <T>(this IEnumerableAsync <T> enumerableAsync,
                                                 Func <Task> task,
                                                 EastFive.Analytics.ILogger logger = default(ILogger))
 {
     enumerableAsync
     .JoinTask(
         Task.Run(task),
         logger);
     return(enumerableAsync);
 }
        public static async Task <IHttpResponse> QueueUpBackupPartitions(
            [Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
            [Property(Name = StorageSettingCopyFromPropertyName)] string storageSettingCopyFrom,
            [Property(Name = StorageSettingCopyToPropertyName)] string storageSettingCopyTo,
            [Resource] RepositoryBackup repositoryBackup,
            AzureApplication application,
            EastFive.Api.Security security,
            RequestMessage <TableBackup> requestQuery,
            IHttpRequest request,
            EastFive.Analytics.ILogger logger,
            MultipartAsyncResponse <InvocationMessage> onQueued,
            AlreadyExistsResponse onAlreadyExists)
        {
            CloudStorageAccount account = CloudStorageAccount
                                          .Parse(storageSettingCopyFrom);
            CloudTableClient tableClient =
                new CloudTableClient(account.TableEndpoint, account.Credentials);

            return(await repositoryBackup.StorageCreateAsync(
                       (discard) =>
            {
                var includedTables = BackupFunction.DiscoverStorageResources()
                                     .Where(x => x.message.Any())
                                     .Select(x => x.tableName)
                                     .ToArray();

                var resourceInfoToProcess = tableClient.GetTables()
                                            .Where(table => includedTables.Contains(table.Name, StringComparer.OrdinalIgnoreCase))
                                            .Distinct()
                                            .Select(
                    async cloudTable =>
                {
                    var tableBackup = new TableBackup()
                    {
                        tableBackupRef = Ref <TableBackup> .NewRef(),
                        backup = repositoryBackupRef,
                        tableName = cloudTable.Name,
                        when = DateTime.UtcNow,
                    };
                    var invocationMessage = await requestQuery
                                            .HttpPost(tableBackup)
                                            .CompileRequest(request)
                                            .FunctionAsync();

                    logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
                    return invocationMessage;
                })
                                            .Await(readAhead: 10);
                return onQueued(resourceInfoToProcess);
            },
                       () => onAlreadyExists()));
        }
        public static async Task <HttpResponseMessage> RunAsync(
            [UpdateId] IRefs <InvocationMessage> invocationMessageRefs,
            [HeaderLog] EastFive.Analytics.ILogger analyticsLog,
            InvokeApplicationDirect invokeApplication,
            MultipartResponseAsync onRun)
        {
            var messages = await invocationMessageRefs.refs
                           .Select(invocationMessageRef => InvokeAsync(invocationMessageRef, invokeApplication, logging: analyticsLog))
                           .AsyncEnumerable()
                           .ToArrayAsync();

            return(await onRun(messages));
        }
Beispiel #7
0
        public static async Task <HttpResponseMessage> QueueUpBackupPartitions(
            [Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
            [Property(Name = StorageSettingCopyFromPropertyName)] string storageSettingCopyFrom,
            [Property(Name = StorageSettingCopyToPropertyName)] string storageSettingCopyTo,
            [Resource] RepositoryBackup repositoryBackup,
            AzureApplication application,
            RequestMessage <TableBackup> requestQuery,
            HttpRequestMessage request,
            EastFive.Analytics.ILogger logger,
            MultipartResponseAsync <InvocationMessage> onQueued,
            AlreadyExistsResponse onAlreadyExists)
        {
            logger.Trace($"Cleaning backup results");
            var repo = AzureTableDriverDynamic.FromStorageString(storageSettingCopyTo);

            await DeleteAllAsync(GetRepository(storageSettingCopyTo));

            CloudStorageAccount account = CloudStorageAccount
                                          .Parse(storageSettingCopyFrom);
            CloudTableClient tableClient =
                new CloudTableClient(account.TableEndpoint, account.Credentials);

            return(await await repositoryBackup.StorageCreateAsync(
                       (discard) =>
            {
                var resourceInfoToProcess = tableClient
                                            .ListTables()
                                            .Distinct()
                                            .Select(
                    async cloudTable =>
                {
                    var tableBackup = new TableBackup()
                    {
                        tableBackupRef = Ref <TableBackup> .NewRef(),
                        backup = repositoryBackupRef,
                        tableName = cloudTable.Name,
                        when = DateTime.UtcNow,
                    };
                    var invocationMessage = await requestQuery
                                            .HttpPost(tableBackup)
                                            .CompileRequest(request)
                                            .FunctionAsync();

                    logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
                    return invocationMessage;
                })
                                            .AsyncEnumerable();
                return onQueued(resourceInfoToProcess);
            },
                       () => onAlreadyExists().AsTask()));
        }
        public static Task <HttpResponseMessage> ListAsync(
            [QueryParameter(Name = "last_modified")] DateTime day,
            [HeaderLog] EastFive.Analytics.ILogger analyticsLog,
            InvokeApplicationDirect invokeApplication,
            MultipartResponseAsync <InvocationMessage> onRun)
        {
            Expression <Func <InvocationMessage, bool> > expr = (im) => true;

            var messages = expr
                           .StorageQuery()
                           .Where(msg => DateTime.UtcNow - msg.lastModified < TimeSpan.FromDays(3.0));

            return(onRun(messages));
        }
        public static IHttpResponse ListAsync(
            [QueryParameter(Name = "start_time")] DateTime startTime,
            [QueryParameter(Name = "end_time")] DateTime endTime,
            [HeaderLog] EastFive.Analytics.ILogger analyticsLog,
            MultipartAsyncResponse <InvocationMessage> onRun)
        {
            Expression <Func <InvocationMessage, bool> > allQuery = (im) => true;

            var messages = allQuery
                           .StorageQuery()
                           .Where(msg => msg.lastModified >= startTime)
                           .Where(msg => msg.lastModified <= endTime);

            return(onRun(messages));
        }
 public static IEnumerableAsync <HttpResponseMessage> InvokeAsync(
     byte [] invocationMessageIdsBytes,
     IApplication application,
     IInvokeApplication invokeApplication,
     EastFive.Analytics.ILogger analyticsLog = default)
 {
     return(invocationMessageIdsBytes
            .Split(index => 16)
            .Select(
                invocationMessageIdBytes =>
     {
         var idBytes = invocationMessageIdBytes.ToArray();
         var invocationMessageId = new Guid(idBytes);
         var invocationMessageRef = invocationMessageId.AsRef <InvocationMessage>();
         return InvokeAsync(invocationMessageRef, invokeApplication, analyticsLog);
     })
            .Parallel());
 }
Beispiel #11
0
 public static async Task <HttpResponseMessage> CreateAsync(
     [Property(Name = IdPropertyName)] IRef <TableBackup> tableBackupRef,
     [Property(Name = WhenPropertyName)] DateTime when,
     [Property(Name = TableNamePropertyName)] string tableName,
     [Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
     [Resource] TableBackup tableBackup,
     RequestMessage <TableBackup> requestQuery,
     HttpRequestMessage request,
     EastFive.Analytics.ILogger logger,
     CreatedBodyResponse <InvocationMessage> onCreated,
     AlreadyExistsResponse onAlreadyExists)
 {
     return(await await tableBackup.StorageCreateAsync(
                async (entity) =>
     {
         var invocationMessage = await requestQuery
                                 .ById(tableBackupRef)
                                 .HttpPatch(default)
Beispiel #12
0
 public static async Task <IHttpResponse> CreateAsync(
     [Property(Name = IdPropertyName)] IRef <TableBackupOperation> tableBackupOperationRef,
     [Property(Name = WhenPropertyName)] DateTime when,
     [Property(Name = TableBackupPropertyName)] IRef <TableBackup> tableBackupRef,
     [Property(Name = OperationSetPropertyName)] string operationSet,
     [Resource] TableBackupOperation tableBackup,
     RequestMessage <TableBackupOperation> requestQuery,
     IHttpRequest request,
     EastFive.Api.Security security,
     EastFive.Analytics.ILogger logger,
     CreatedBodyResponse <InvocationMessage> onCreated,
     AlreadyExistsResponse onAlreadyExists)
 {
     return(await await tableBackup.StorageCreateAsync(
                async (discard) =>
     {
         var invocationMessage = await requestQuery
                                 .ById(tableBackupOperationRef)
                                 .HttpPatch(default)
        public static async Task ProcessBackupPartitionsAsync(TableMessage message, EastFive.Analytics.ILogger logger)
        {
            logger.Trace($"Starting backup for table {message.tableName}");
            var sw = new Stopwatch();

            sw.Start();

            var table      = message.tableName;
            var sourceRepo = GetRepository(message.sourceConnectionString);
            var destRepo   = GetRepository(message.destConnectionString);
            var filter     = message.where.FormatWhereInformation()
                             .Join(" and ");
            await Backup.CreateOrUpdateAsync(destRepo, table, filter);

            var completedRows = await sourceRepo.Copy(filter, table, destRepo)
                                .ToArrayAsync();

            sw.Stop();

            await Backup.CreateOrUpdateAsync(destRepo, table, filter, completedRows.Length, sw.Elapsed.TotalSeconds);

            logger.Trace($"copied {completedRows.Length} rows in {sw.Elapsed.TotalSeconds} seconds for {filter}");
        }
        public static async Task <IHttpResponse> QueueUpBackupPartitions(
            [Property(Name = IdPropertyName)] IRef <RepositoryBackup> repositoryBackupRef,
            [Property(Name = WhenPropertyName)] DateTime when,
            RequestMessage <TableBackup> requestQuery,
            IHttpRequest request,
            EastFive.Analytics.ILogger logger,
            MultipartAsyncResponse <InvocationMessage> onQueued,
            NoContentResponse onTooEarly,
            NotFoundResponse onNotFound)
        {
            return(await repositoryBackupRef.StorageUpdateAsync(
                       async (repoBack, saveAsync) =>
            {
                var needsToRun = NCrontab.CrontabSchedule.TryParse(repoBack.frequency,
                                                                   chronSchedule =>
                {
                    var next = chronSchedule.GetNextOccurrence(repoBack.when);
                    if (when > next)
                    {
                        return true;
                    }
                    return false;
                },
                                                                   ex =>
                {
                    return false;
                });
                if (!needsToRun)
                {
                    return onTooEarly();
                }

                var includedTables = BackupFunction.DiscoverStorageResources()
                                     .Where(x => x.message.Any())
                                     .Select(x => x.tableName)
                                     .ToArray();

                CloudStorageAccount account = CloudStorageAccount
                                              .Parse(repoBack.storageSettingCopyFrom);
                CloudTableClient tableClient =
                    new CloudTableClient(account.TableEndpoint, account.Credentials);

                var tables = tableClient.GetTables();
                var resourceInfoToProcess = tables
                                            .Where(table => includedTables.Contains(table.Name, StringComparer.OrdinalIgnoreCase))
                                            .Distinct()
                                            .Select(
                    async cloudTable =>
                {
                    var tableBackup = new TableBackup()
                    {
                        tableBackupRef = Ref <TableBackup> .NewRef(),
                        backup = repositoryBackupRef,
                        tableName = cloudTable.Name,
                        when = DateTime.UtcNow,
                    };
                    var invocationMessage = await requestQuery
                                            .HttpPost(tableBackup)
                                            .CompileRequest(request)
                                            .FunctionAsync();

                    logger.Trace($"Invocation[{invocationMessage.id}] will backup table `{tableBackup.tableName}`.");
                    return invocationMessage;
                })
                                            .Await(readAhead: 10);
                repoBack.when = when;
                await saveAsync(repoBack);

                return onQueued(resourceInfoToProcess);
            },
                       () => onNotFound()));
        }
Beispiel #15
0
        public static async Task QueueUpBackupContainers(string serviceBusQueueName, string sourceConnectionString, string destConnectionString,
                                                         AzureApplication application, EastFive.Analytics.ILogger logger)
        {
            var containerResources = DiscoverAllContainers();

            var blocksSent = await containerResources
                             .Where(info => !ignoreContainerNames.Contains(info.name.ToLower()))
                             .Select(
                async(info) =>
            {
                var sw = new Stopwatch();
                sw.Start();
                logger.Trace($"Queuing backup messages for container {info.name}");
                var backupMessages = GeneratePrefixInformation(info.prefixGenerator)
                                     .Select(
                    (prefix) =>
                {
                    return(new ContainerMessage
                    {
                        sourceConnectionString = sourceConnectionString,
                        destConnectionString = destConnectionString,
                        name = info.name,
                        prefix = prefix,
                    });
                })
                                     .Select(message => JsonConvert.SerializeObject(message))
                                     .Select(message => Encoding.UTF8.GetBytes(message))
                                     .ToArray();
                await application.SendServiceBusMessageAsync(serviceBusQueueName, backupMessages);
                sw.Stop();
                logger.Trace($"{backupMessages.Length} messages queued for {info.name} in {sw.Elapsed.TotalSeconds} seconds");
                GC.Collect();
                return(backupMessages.Length);
            })
                             .AsyncEnumerable() //.Throttle(desiredRunCount: 2)
                             .ToArrayAsync();

            var totalMessages = blocksSent.Sum();

            logger.Trace($"Total of {totalMessages} messages queued for all containers");

            await ThrowIfContainerIsMissingStorageResourceAttribute(sourceConnectionString, containerResources);
        }