async Task PurgeOrchestrationHistorySegmentAsync(
            TableQuerySegment<OrchestrationStateEntity> orchestrationStateEntitySegment)
        {
            var stateEntitiesToDelete = new List<OrchestrationStateEntity>(orchestrationStateEntitySegment.Results);

            var historyEntitiesToDelete = new ConcurrentBag<IEnumerable<OrchestrationHistoryEventEntity>>();
            await Task.WhenAll(orchestrationStateEntitySegment.Results.Select(
                entity => Task.Run(async () =>
                {
                    IEnumerable<OrchestrationHistoryEventEntity> historyEntities =
                        await
                            tableClient.ReadOrchestrationHistoryEventsAsync(
                                entity.State.OrchestrationInstance.InstanceId,
                                entity.State.OrchestrationInstance.ExecutionId).ConfigureAwait(false);

                    historyEntitiesToDelete.Add(historyEntities);
                })));

            List<Task> historyDeleteTasks = historyEntitiesToDelete.Select(
                historyEventList => tableClient.DeleteEntitesAsync(historyEventList)).Cast<Task>().ToList();

            // need to serialize history deletes before the state deletes so we dont leave orphaned history events
            await Task.WhenAll(historyDeleteTasks).ConfigureAwait(false);
            await Task.WhenAll(tableClient.DeleteEntitesAsync(stateEntitiesToDelete)).ConfigureAwait(false);
        }
Exemple #2
0
        public async Task Execute(params string[] args)
        {
            var stopWatch = Stopwatch.StartNew();

            var parsedArguments = TableCopyArguments.Parse(args);

            if (!parsedArguments.Valid)
            {
                Console.WriteLine("Must give source and target, both in a form environmentName:TableName.");
                return;
            }

            Console.Out.WriteLine("Copying {0} from {1} to {2} at {3}.",
                                  parsedArguments.Source.TableName,
                                  parsedArguments.Source.Environment,
                                  parsedArguments.Target.TableName,
                                  parsedArguments.Target.Environment);

            var sourceTable = new EnvironmentClient(parsedArguments.Source.Environment).GetTable(parsedArguments.Source.TableName);
            var targetTable = new EnvironmentClient(parsedArguments.Target.Environment).GetTable(parsedArguments.Target.TableName);

            if (!await sourceTable.ExistsAsync())
            {
                Console.Out.WriteLine("Source table does not exist, exiting.");
                return;
            }

            await targetTable.CreateIfNotExistsAsync();

            var allQuery = new TableQuery <DynamicTableEntity>();
            TableQuerySegment <DynamicTableEntity> querySegment = null;
            var sourceEntities = new List <DynamicTableEntity>();

            while (querySegment == null || querySegment.ContinuationToken != null)
            {
                querySegment = await sourceTable.ExecuteQuerySegmentedAsync(
                    allQuery, querySegment != null?querySegment.ContinuationToken : null);

                sourceEntities.AddRange(querySegment);
            }

            var batches = new List <TableBatchOperation>();
            var groupedByPartitionKey = sourceEntities.GroupBy(
                p => p.PartitionKey,
                e => e,
                (key, list) => new KeyValuePair <string, IEnumerable <DynamicTableEntity> >(key, list));

            foreach (var g in groupedByPartitionKey)
            {
                const int BatchSize = 100;
                int       i         = 1;

                // If batch size is more than 100, chunk it as it is the max sixe for azure storage batch operation
                foreach (var chunk in g.Value.Chunk(BatchSize))
                {
                    Console.Out.WriteLine(g.Key + " - Batch " + i);
                    var batch = new TableBatchOperation();

                    foreach (var entity in chunk)
                    {
                        batch.Insert(entity);
                    }

                    batches.Add(batch);
                    i++;
                }
            }

            Console.Out.WriteLine("This might take a while...");
            var waits = batches.Select(targetTable.ExecuteBatchAsync).Cast <Task>().ToArray();

            Task.WaitAll(waits, TimeSpan.FromHours(1));

            Console.Out.WriteLine("\r\nCopying successful, copied {0:n0} rows in {1:n1} seconds.",
                                  sourceEntities.Count,
                                  stopWatch.Elapsed.TotalSeconds);
            return;
        }
        public static dynamic GetAzureStorageListingsCore(HttpRequest req, Logging logging)
        {
            string  requestBody     = new System.IO.StreamReader(req.Body).ReadToEndAsync().Result;
            dynamic taskInformation = JsonConvert.DeserializeObject(requestBody);
            string  _TaskInstanceId = taskInformation["TaskInstanceId"].ToString();
            string  _ExecutionUid   = taskInformation["ExecutionUid"].ToString();

            try
            {
                string _storageAccountName = taskInformation["Source"]["StorageAccountName"];
                //The name is actually the base url so we need to parse it to get the name only
                _storageAccountName = _storageAccountName.Split('.')[0].Replace("https://", "");
                string _storageAccountToken = taskInformation["Source"]["StorageAccountToken"];
                Int64  _SourceSystemId      = taskInformation["Source"]["SystemId"];

                TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
                using SqlConnection _con = TMD.GetSqlConnection();

                var res = _con.QueryWithRetry(string.Format("Select Max(PartitionKey) MaxPartitionKey from AzureStorageListing where SystemId = {0}", _SourceSystemId.ToString()));

                string MaxPartitionKey = DateTime.UtcNow.AddDays(-1).ToString("yyyy-MM-dd hh:mm");

                foreach (var r in res)
                {
                    if (r.MaxPartitionKey != null)
                    {
                        MaxPartitionKey = DateTime.Parse(r.MaxPartitionKey).AddMinutes(-1).ToString("yyyy-MM-dd hh:mm");
                    }
                }

                using (HttpClient SourceClient = new HttpClient())
                {
                    //Now use the SAS URI to connect rather than the MSI / Service Principal as AD Based Auth not yet avail for tables
                    var _storageCredentials  = new StorageCredentials(_storageAccountToken);
                    var SourceStorageAccount = new CloudStorageAccount(storageCredentials: _storageCredentials, accountName: _storageAccountName, endpointSuffix: "core.windows.net", useHttps: true);
                    var client = SourceStorageAccount.CreateCloudTableClient();

                    CloudTable table = client.GetTableReference("Filelist");

                    TableQuery <DynamicTableEntity> query = new TableQuery <DynamicTableEntity>().Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThan, MaxPartitionKey.ToString()));

                    DataTable  dt = new DataTable();
                    DataColumn dc = new DataColumn();
                    dc.ColumnName = "PartitionKey";
                    dc.DataType   = typeof(string);
                    dt.Columns.Add(dc);
                    DataColumn dc1 = new DataColumn();
                    dc1.ColumnName = "RowKey";
                    dc1.DataType   = typeof(string);
                    dt.Columns.Add(dc1);
                    DataColumn dc2 = new DataColumn();
                    dc2.ColumnName = "SystemId";
                    dc2.DataType   = typeof(Int64);
                    dt.Columns.Add(dc2);
                    DataColumn dc3 = new DataColumn();
                    dc3.ColumnName = "FilePath";
                    dc3.DataType   = typeof(string);
                    dt.Columns.Add(dc3);

                    string Filelist = "";
                    TableContinuationToken token = null;
                    do
                    {
                        TableQuerySegment <DynamicTableEntity> resultSegment = table.ExecuteQuerySegmentedAsync(query, token).Result;
                        token = resultSegment.ContinuationToken;

                        //load into data table
                        foreach (var entity in resultSegment.Results)
                        {
                            DataRow dr = dt.NewRow();
                            dr["PartitionKey"] = entity.PartitionKey;
                            dr["RowKey"]       = entity.RowKey;
                            dr["SystemId"]     = _SourceSystemId;
                            dr["FilePath"]     = entity.Properties["FilePath"].StringValue;
                            dt.Rows.Add(dr);
                            Filelist += entity.Properties["FilePath"].StringValue + System.Environment.NewLine;
                        }
                    } while (token != null);


                    if (dt.Rows.Count > 0)
                    {
                        Table t = new Table();
                        t.Schema = "dbo";
                        string TableGuid = Guid.NewGuid().ToString();
                        t.Name = $"#AzureStorageListing{TableGuid}";

                        TMD.BulkInsert(dt, t, true, _con);
                        Dictionary <string, string> SqlParams = new Dictionary <string, string>
                        {
                            { "TempTable", t.QuotedSchemaAndName() },
                            { "SourceSystemId", _SourceSystemId.ToString() }
                        };

                        string MergeSQL = GenerateSQLStatementTemplates.GetSQL(Shared.GlobalConfigs.GetStringConfig("SQLTemplateLocation"), "MergeIntoAzureStorageListing", SqlParams);
                        _con.ExecuteWithRetry(MergeSQL, 120);
                        if ((JArray)taskInformation["Alerts"] != null)
                        {
                            foreach (JObject Alert in (JArray)taskInformation["Alerts"])
                            {
                                //Only Send out for Operator Level Alerts
                                if (Alert["AlertCategory"].ToString() == "Task Specific Operator Alert")
                                {
                                    AlertOperator(_SourceSystemId, Alert["AlertEmail"].ToString(), "", Filelist);
                                }
                            }
                        }
                    }

                    _con.Close();
                    _con.Dispose();
                    TMD.LogTaskInstanceCompletion(System.Convert.ToInt64(_TaskInstanceId), System.Guid.Parse(_ExecutionUid), TaskMetaData.BaseTasks.TaskStatus.Complete, Guid.Empty, "");
                }
            }

            catch (Exception e)
            {
                logging.LogErrors(e);
                TaskMetaDataDatabase TMD = new TaskMetaDataDatabase();
                TMD.LogTaskInstanceCompletion(System.Convert.ToInt64(_TaskInstanceId), System.Guid.Parse(_ExecutionUid), TaskMetaData.BaseTasks.TaskStatus.FailedRetry, Guid.Empty, "Failed when trying to Generate Sas URI and Send Email");

                JObject Root = new JObject
                {
                    ["Result"] = "Failed"
                };

                return(Root);
            }
            return(new { });
        }
Exemple #4
0
        public static async Task <bool> MakeQueueAsync(ILogger logger)
        {
            Settings            settings    = Settings.Load();
            CloudStorageAccount account     = CloudStorageAccount.Parse(settings.AzureWebJobsStorage);
            CloudTableClient    tableClient = account.CreateCloudTableClient();
            CloudQueueClient    queueClient = account.CreateCloudQueueClient();
            CloudQueue          jobQueue    = queueClient.GetQueueReference(settings.JobQueueName);

            _ = await jobQueue.CreateIfNotExistsAsync();

            CloudTable jobTable = tableClient.GetTableReference(settings.JobTableName);

            CloudTable jobHistoryTable = tableClient.GetTableReference(settings.JobHistoryTableName);

            _ = await jobHistoryTable.CreateIfNotExistsAsync();

            JobEntity plan = null;
            {
                TableQuery <JobEntity>        query    = new TableQuery <JobEntity>().Take(8);
                TableQuerySegment <JobEntity> segments = await jobTable.ExecuteQuerySegmentedAsync(query, null);

                var jobEntities = segments.Results;
                var ordered     = jobEntities.OrderByDescending(i => i.CreateAt);

                if (ordered.Count() == 0)
                {
                    return(true);
                }

                plan = ordered.First();
            }

            int step = int.Parse(settings.Step);

            if (step <= 0)
            {
                step = DEFAULT_STEP;
            }

            {
                string             targetPath    = settings.TargetPath;
                Uri                uri           = new Uri(targetPath);
                CloudBlobContainer blobContainer = new CloudBlobContainer(uri, account.Credentials);

                int    section   = plan.Current / step;
                string path      = $"{plan.Version}/{plan.PartitionKey}/{section * step}";
                long   fileCount = await StorageHelper.GetBlobFileCountAsync(blobContainer, path, int.Parse(settings.MaxResult));

                if (fileCount <= int.Parse(settings.StepThreshold) && fileCount > 0)
                {
                    logger.LogInformation($"file count {fileCount} less than step threshold");
                    return(true);
                }
            }


            if (plan.Current >= plan.Amount)
            {
                TableOperation delete = TableOperation.Delete(plan);
                _ = await jobTable.ExecuteAsync(delete);

                plan.UpdateAt = DateTime.Now;
                TableOperation insertOrMerge = TableOperation.InsertOrMerge(plan);
                _ = await jobHistoryTable.ExecuteAsync(insertOrMerge);

                return(true);
            }
            else
            {
                int start   = plan.Current;
                int current = plan.Current + step;
                int count   = step;
                if (current > plan.Amount)
                {
                    count        = plan.Amount - plan.Current;
                    plan.Current = plan.Amount;
                }
                else
                {
                    plan.Current += step;
                }
                plan.UpdateAt = DateTime.Now;
                TableOperation insertOrMerge = TableOperation.InsertOrMerge(plan);
                _ = await jobTable.ExecuteAsync(insertOrMerge);

                MakeQueueMessageAsync(jobQueue, plan, start, count, settings.SourcePath, settings.TargetPath);
            }
            return(true);
        }
        public async Task RemoveFromAllRolesAsync(T user)
        {
            bool error = false;
            List <TableUserRole> roles = new List <TableUserRole>();
            string partitionKeyQuery   = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, user.Id);
            TableQuery <TableUserRole>        query        = new TableQuery <TableUserRole>().Where(partitionKeyQuery);
            TableQuerySegment <TableUserRole> querySegment = null;

            while (querySegment == null || querySegment.ContinuationToken != null)
            {
                querySegment = await _rolesTable.ExecuteQuerySegmentedAsync(query, querySegment != null?querySegment.ContinuationToken : null);

                roles.AddRange(querySegment.Results);
            }

            TableBatchOperation batch = new TableBatchOperation();

            foreach (TableUserRole role in roles)
            {
                role.ETag = "*"; //Delete even if it has changed
                batch.Add(TableOperation.Delete(role));
                if (batch.Count >= 100)
                {
                    try
                    {
                        //Try executing as a batch
                        await _rolesTable.ExecuteBatchAsync(batch);

                        batch.Clear();
                    }
                    catch { }

                    //If a batch wont work, try individually
                    foreach (TableOperation op in batch)
                    {
                        try
                        {
                            await _rolesTable.ExecuteAsync(op);
                        }
                        catch
                        {
                            error = true;
                        }
                    }

                    batch.Clear();
                }
            }
            if (batch.Count > 0)
            {
                try
                {
                    //Try executing as a batch
                    await _rolesTable.ExecuteBatchAsync(batch);

                    batch.Clear();
                }
                catch { }

                //If a batch wont work, try individually
                foreach (TableOperation op in batch)
                {
                    try
                    {
                        await _rolesTable.ExecuteAsync(op);
                    }
                    catch
                    {
                        error = true;
                    }
                }
            }
            if (error)
            {
                throw new Exception();
            }
        }
        public async Task RemoveAllLoginsAsync(T user)
        {
            bool error = false;
            List <TableUserLogin> Logins                    = new List <TableUserLogin>();
            string partitionKeyQuery                        = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, user.Id);
            TableQuery <TableUserLogin>        query        = new TableQuery <TableUserLogin>().Where(partitionKeyQuery);
            TableQuerySegment <TableUserLogin> querySegment = null;

            while (querySegment == null || querySegment.ContinuationToken != null)
            {
                querySegment = await _loginTable.ExecuteQuerySegmentedAsync(query, querySegment != null?querySegment.ContinuationToken : null);

                Logins.AddRange(querySegment.Results);
            }

            TableBatchOperation batch      = new TableBatchOperation();
            TableBatchOperation batchIndex = new TableBatchOperation();

            foreach (TableUserLogin login in Logins)
            {
                login.ETag = "*"; //Delete even if it has changed
                batch.Add(TableOperation.Delete(login));
                TableUserLoginProviderKeyIndex providerKeyIndex = new TableUserLoginProviderKeyIndex(user.Id, login.ProviderKey, login.LoginProvider);
                providerKeyIndex.ETag = "*";
                batchIndex.Add(TableOperation.Delete(providerKeyIndex));

                if (batch.Count >= 100 || batchIndex.Count >= 100)
                {
                    try
                    {
                        //Try executing as a batch
                        await _loginTable.ExecuteBatchAsync(batch);

                        batch.Clear();
                    }
                    catch { }

                    //If a batch wont work, try individually
                    foreach (TableOperation op in batch)
                    {
                        try
                        {
                            await _loginTable.ExecuteAsync(op);
                        }
                        catch
                        {
                            error = true;
                        }
                    }

                    //Delete the index individually becase of the partition keys
                    foreach (TableOperation op in batchIndex)
                    {
                        try
                        {
                            await _loginProviderKeyIndexTable.ExecuteAsync(op);
                        }
                        catch
                        {
                            error = true;
                        }
                    }

                    batch.Clear();
                    batchIndex.Clear();
                }
            }
            if (batch.Count > 0 || batchIndex.Count > 0)
            {
                try
                {
                    //Try executing as a batch
                    await _loginTable.ExecuteBatchAsync(batch);

                    batch.Clear();
                }
                // ReSharper disable once EmptyGeneralCatchClause
                catch { }

                //If a batch wont work, try individually
                foreach (TableOperation op in batch)
                {
                    try
                    {
                        await _loginTable.ExecuteAsync(op);
                    }
                    catch
                    {
                        error = true;
                    }
                }

                //Delete the index individually becase of the partition keys
                foreach (TableOperation op in batchIndex)
                {
                    try
                    {
                        await _loginProviderKeyIndexTable.ExecuteAsync(op);
                    }
                    catch
                    {
                        error = true;
                    }
                }
            }

            if (error)
            {
                throw new Exception();
            }
        }
Exemple #7
0
 public AzureUtils GetSearchResult(string rowPropertyName, string expectedPropertyValue)
 {
     tableSegment = cloudTable.ExecuteQuerySegmentedAsync(
         new TableQuery().Where(TableQuery.GenerateFilterCondition(rowPropertyName, QueryComparisons.Equal, expectedPropertyValue)), null).Result;
     return(this);
 }
 private List <CsvTableRecord> FormatRecordResults(CloudTable cloudTable, TableQuerySegment tableSegment)