예제 #1
0
        async public static Task <bool> SaveAsync(CloudTable table, IEnumerable <BaseEntity> items)
        {
            TableBatchOperation batch = new TableBatchOperation();

            foreach (var item in items)
            {
                item.SetKeys();
                batch.InsertOrMerge(item);
            }

            bool retry       = true;
            bool createTable = false;

            do
            {
                try
                {
                    if (createTable)
                    {
                        // Try to create the table
                        await table.CreateIfNotExistsAsync();

                        createTable = false;
                    }

                    // Execute the batch
                    var result = await table.ExecuteBatchAsync(batch);

                    // Check the status codes (success is 200-299)
                    return(result.All(o => o.HttpStatusCode >= 200 && o.HttpStatusCode < 300));
                }
                catch (AggregateException ex)
                {
                    if (ex.InnerExceptions.Any(o => o is StorageException))
                    {
                        // Table doesn't exist
                        // Set our value to loop around and try again, creating the table first
                        // Don't do it if we were already trying to create the table
                        if (retry)
                        {
                            createTable = true;
                            retry       = false;
                        }
                        else
                        {
                            // We must have generated an exception after trying to create the table,
                            // so don't try again.
                            createTable = false;
                            retry       = false;
                        }
                    }
                }
                catch (StorageException ex)
                {
                    // Check the status code
                    System.Diagnostics.Debug.WriteLine(ex);

                    // Table doesn't exist
                    // Set our value to loop around and try again, creating the table first
                    // Don't do it if we were already trying to create the table
                    if (retry)
                    {
                        createTable = true;
                        retry       = false;
                    }
                    else
                    {
                        // We must have generated an exception after trying to create the table,
                        // so don't try again.
                        createTable = false;
                        retry       = false;
                    }
                }
            } while (createTable);

            return(false);
        }
예제 #2
0
        /// <summary>
        /// Removes a batch of items from the table.
        /// </summary>
        /// <param name="table">CloudTable instance.</param>
        /// <param name="tbo">TableBatchOperations instance.</param>
        internal async Task <int> RemoveItemsAsync(CloudTable table, TableBatchOperation tbo)
        {
            if (null == table)
            {
                throw new ArgumentNullException("Argument is null", nameof(table));
            }
            if (null == tbo)
            {
                throw new ArgumentNullException("Argument is null", nameof(tbo));
            }

            int       count         = 0;
            const int maxRetryCount = 5;
            int       retry         = maxRetryCount;

            do
            {
                // Reset count in case the while was retried.
                count = 0;
                try
                {
                    TableRequestOptions tro = new TableRequestOptions()
                    {
                        MaximumExecutionTime = TimeSpan.FromSeconds(60),
                        ServerTimeout        = TimeSpan.FromSeconds(5),
                        RetryPolicy          = new ExponentialRetry(TimeSpan.FromSeconds(1), 3)
                    };

                    // Ensure that the batch isn't empty, if it is, return the count.
                    if (0 == tbo.Count)
                    {
                        break;
                    }

                    // Execute the batch operations.
                    IList <TableResult> results = results = await table.ExecuteBatchAsync(tbo, tro, null, this._token);

                    if ((null != results) && (results.Count > 0))
                    {
                        int itemCount = 0, failureCount = 0;
                        foreach (TableResult result in results)
                        {
                            itemCount++;
                            if (false == ((HttpStatusCode)result.HttpStatusCode).IsSuccessCode())
                            {
                                failureCount++;
                            }
                        }

                        ServiceEventSource.Current.Trace($"Removed {itemCount - failureCount} of {itemCount} items from {table.Name}.");
                        count = itemCount - failureCount;
                    }
                }
                catch (StorageException ex)
                {
                    // ResourceNotFound is returned when one of the batch items isn't found. Need to remove it and try again.
                    if (ex.RequestInformation?.ExtendedErrorInformation?.ErrorCode.Contains("ResourceNotFound") ?? false)
                    {
                        // Get the index of the item within the batch.
                        if (false == int.TryParse(
                                ex.RequestInformation?.ExtendedErrorInformation?.ErrorMessage.Split(':')[0],
                                out int
                                index))
                        {
                            ServiceEventSource.Current.Trace("Unknown index, setting to 0", table.Name);
                            index = 0;
                        }

                        if (index < tbo.Count)
                        {
                            ServiceEventSource.Current.Trace($"StorageException: ResourceNotFound for item {index}", table.Name);
                            await Task.Delay(500);

                            tbo.RemoveAt(index);
                            retry--;
                        }
                        else
                        {
                            ServiceEventSource.Current.Trace("Abandoning batch.", table.Name);
                            break;
                        }
                    }
                    else
                    {
                        ServiceEventSource.Current.Exception(ex.Message, ex.GetType().Name, ex.StackTrace);
                        break;
                    }
                }
            } while ((retry > 0) && (retry < maxRetryCount)); // Only retry if we hit a retryable exception or run out of retries.

            return(count);
        }
예제 #3
0
        public async Task CreateAsync(T user)
        {
            if (user == null)
            {
                throw new ArgumentNullException("user");
            }

            user.SetPartitionAndRowKey();

            var            userNameIndex  = new UserNameIndexEntity(user.UserName.ToSha256(), user.Id);
            TableOperation indexOperation = TableOperation.Insert(userNameIndex);

            try
            {
                await _userNameIndexTable.ExecuteAsync(indexOperation);
            }
            catch (StorageException ex)
            {
                if (ex.RequestInformation.HttpStatusCode == 409)
                {
                    throw new DuplicateUsernameException();
                }

                throw;
            }

            if (!String.IsNullOrWhiteSpace(user.Email))
            {
                var            userEmailIndexEntity = new UserEmailIndexEntity(user.Email.ToSha256(), user.Id);
                TableOperation emailIndexOperation  = TableOperation.Insert(userEmailIndexEntity);
                try
                {
                    await _userEmailIndexTable.ExecuteAsync(emailIndexOperation);
                }
                catch (StorageException ex)
                {
                    try
                    {
                        userNameIndex.ETag = "*";
                        TableOperation deleteOperation = TableOperation.Delete(userNameIndex);
                        _userNameIndexTable.ExecuteAsync(deleteOperation).Wait();
                    }
                    catch (Exception)
                    {
                        // if we can't delete the index item throw out the exception below
                    }

                    if (ex.RequestInformation.HttpStatusCode == 409)
                    {
                        throw new DuplicateEmailException();
                    }
                    throw;
                }
            }

            try
            {
                if (user.LockoutEndDate < _minTableStoreDate)
                {
                    user.LockoutEndDate = _minTableStoreDate;
                }

                TableOperation operation = TableOperation.InsertOrReplace(user);
                await _userTable.ExecuteAsync(operation);

                if (user.Logins.Any())
                {
                    var batch           = new TableBatchOperation();
                    var loginIndexItems = new List <UserLoginProviderKeyIndexEntity>();
                    foreach (UserLoginEntity login in user.Logins)
                    {
                        login.UserId = user.Id;
                        login.SetPartitionKeyRowKey();
                        batch.InsertOrReplace(login);

                        var loginIndexItem = new UserLoginProviderKeyIndexEntity(user.Id, login.ProviderKey, login.LoginProvider);
                        loginIndexItems.Add(loginIndexItem);
                    }
                    await _userLoginTable.ExecuteBatchAsync(batch);

                    // can't batch the index items as different primary keys
                    foreach (UserLoginProviderKeyIndexEntity loginIndexItem in loginIndexItems)
                    {
                        await _userLoginProviderKeyIndexTable.ExecuteAsync(TableOperation.InsertOrReplace(loginIndexItem));
                    }
                }
            }
            catch (Exception)
            {
                // attempt to delete the index item - needs work
                userNameIndex.ETag = "*";
                TableOperation deleteOperation = TableOperation.Delete(userNameIndex);
                _userNameIndexTable.ExecuteAsync(deleteOperation).Wait();
                throw;
            }
        }
예제 #4
0
        public static async Task Run(
            [TimerTrigger("0 0 0 * * *")] TimerInfo timer,
            [Table("%Storage:ContractTable%", Connection = "Storage:Connection")] CloudTable contractTable,
            [Table("%Storage:StatTable%", Connection = "Storage:Connection")] CloudTable statTable,
            ILogger log,
            ExecutionContext context)
        {
            IConfigurationRoot config = new ConfigurationBuilder()
                                        .SetBasePath(context.FunctionAppDirectory)
                                        .AddJsonFile("local.settings.json", optional: true, reloadOnChange: true)
                                        .AddEnvironmentVariables()
                                        .Build();

            var stat = new Dictionary <ProcessingStatFields, long>
            {
                { ProcessingStatFields.Processed, 0 },
                { ProcessingStatFields.Failed, 0 },
                { ProcessingStatFields.Finished, 0 },
                { ProcessingStatFields.LowSeverity, 0 },
                { ProcessingStatFields.MediumSeverity, 0 },
                { ProcessingStatFields.HighSeverity, 0 },
                { ProcessingStatFields.NoIssues, 0 },
            };

            TableContinuationToken token = null;

            do
            {
                var query = new TableQuery <ContractEntity> {
                    TakeCount = 100
                };
                var segment = contractTable.ExecuteQuerySegmentedAsync(query, token).Result;

                foreach (var entry in segment.Results)
                {
                    stat[ProcessingStatFields.Processed] += 1;
                    if (entry.AnalysisStatus == "Error")
                    {
                        stat[ProcessingStatFields.Failed] += 1;
                    }
                    else
                    {
                        stat[ProcessingStatFields.Finished] += 1;
                    }

                    switch (entry.Severity)
                    {
                    case "Low":
                        stat[ProcessingStatFields.LowSeverity] += 1;
                        break;

                    case "Medium":
                        stat[ProcessingStatFields.MediumSeverity] += 1;
                        break;

                    case "High":
                        stat[ProcessingStatFields.HighSeverity] += 1;
                        break;

                    default:
                        stat[ProcessingStatFields.NoIssues] += 1;
                        break;
                    }
                }

                token = segment.ContinuationToken;
            } while (token != null);

            var batchOperation = new TableBatchOperation();

            foreach (var key in stat.Keys)
            {
                var entry = new StatEntity
                {
                    PartitionKey = "ProcessingStat",
                    RowKey       = key.ToString(),
                    Count        = stat[key]
                };
                batchOperation.InsertOrReplace(entry);
            }

            await statTable.ExecuteBatchAsync(batchOperation);
        }
        public static async Task <IActionResult> createWordBulk(
            [HttpTrigger(AuthorizationLevel.Function, "get", "post", Route = null)] HttpRequest req,
            //Offenbar bekommt die Funktion automatisch ein korrekt eingerichtetes Objekt für die Tabelle mitgegeben
            [Table("categorization")] CloudTable tabCats,
            ILogger log)
        {
            List <bulkWordData> allBW;

            //Bulk data - array of objects with name, langu, desc as attributes
            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();

            if (requestBody == null || requestBody == "")
            {
                return(null);
            }

            log.LogInformation("dummy");

            allBW = JsonConvert.DeserializeObject <List <bulkWordData> >(requestBody);

            mainWordData   MWD    = new mainWordData();
            TableOperation query  = TableOperation.Retrieve <mainWordData>("mainWordData", "count");
            TableResult    tabRes = await tabCats.ExecuteAsync(query);

            if (tabRes.Result != null)
            {
                MWD = (mainWordData)tabRes.Result;
            }

            MWD.wordCount++;
            lastBulkWord = MWD.wordCount;

            bulkTO  = new TableBatchOperation();
            bulkTO2 = new TableBatchOperation();

            tmpAllW2NI = await intGetAllWordNames(tabCats);

            int bCount = 0;
            int xcount = 0;
            List <Task <IList <TableResult> > > batchTasks = new List <Task <IList <TableResult> > >();

            foreach (bulkWordData oneWord in allBW)
            {
                xcount++;
                if (oneWord.n == null || oneWord.n == "" || bCount == 54)
                {
                    var x = 0;
                }
                bCount++;
                await intCreateWord(tabCats, oneWord.n, true);

                if (bCount == 99)
                {
                    if (bulkTO.Count > 0)
                    {
                        batchTasks.Add(tabCats.ExecuteBatchAsync(bulkTO));
                    }
                    if (bulkTO2.Count > 0)
                    {
                        batchTasks.Add(tabCats.ExecuteBatchAsync(bulkTO2));
                    }
                    bCount = 0;
                    bulkTO.Clear();
                    bulkTO2.Clear();
                }
            }

            MWD.wordCount = lastBulkWord;
            TableOperation writeBack = TableOperation.InsertOrReplace(MWD);
            var            dummy     = tabCats.ExecuteAsync(writeBack);

            if (bulkTO.Count > 0)
            {
                batchTasks.Add(tabCats.ExecuteBatchAsync(bulkTO));
            }
            if (bulkTO2.Count > 0)
            {
                batchTasks.Add(tabCats.ExecuteBatchAsync(bulkTO2));
            }

            try
            {
                //Task.WaitAll(batchTasks.ToArray());
            }
            catch (System.Exception ex)
            {
                log.LogInformation(ex.Message);
                throw;
            }


            return(new OkObjectResult(""));
        }
        public async Task StoreAsync <T>(nStoreOperation storaeOperationType, IEnumerable <T> models) where T : new()
        {
            try
            {
                // notify delegate
                if (_delegate != null)
                {
                    _delegate.OnStoring(typeof(T), storaeOperationType);
                }

                // Retrieve a reference to the table.
                CloudTable table = GetTableReference(GetTableName <T>());

                // Create the batch operation.
                List <TableBatchOperation> batchOperations = new List <TableBatchOperation>();

                // Create the first batch
                var currentBatch = new TableBatchOperation();
                batchOperations.Add(currentBatch);

                // lookup the entitymapper
                var entityMapper = _entityMapperRegistry[typeof(T)];

                // define the modelcounter
                int modelCounter = 0;

                // Add all items
                foreach (var model in models)
                {
                    switch (storaeOperationType)
                    {
                    case nStoreOperation.insertOperation:
                        currentBatch.Insert(new DynamicTableEntity <T>(model, entityMapper));
                        break;

                    case nStoreOperation.insertOrReplaceOperation:
                        currentBatch.InsertOrReplace(new DynamicTableEntity <T>(model, entityMapper));
                        break;

                    case nStoreOperation.mergeOperation:
                        currentBatch.Merge(new DynamicTableEntity <T>(model, entityMapper));
                        break;

                    case nStoreOperation.mergeOrInserOperation:
                        currentBatch.InsertOrMerge(new DynamicTableEntity <T>(model, entityMapper));
                        break;

                    case nStoreOperation.delete:
                        currentBatch.Delete(new DynamicTableEntity <T>(model, entityMapper));
                        break;
                    }

                    modelCounter++;

                    if (modelCounter % 100 == 0)
                    {
                        currentBatch = new TableBatchOperation();
                        batchOperations.Add(currentBatch);
                    }
                }

                // execute
                foreach (var createdBatch in batchOperations)
                {
                    if (createdBatch.Count() > 0)
                    {
                        await table.ExecuteBatchAsync(createdBatch);

                        // notify delegate
                        if (_delegate != null)
                        {
                            _delegate.OnStored(typeof(T), storaeOperationType, createdBatch.Count(), null);
                        }
                    }
                }
            }
            catch (StorageException ex)
            {
                // check the exception
                if (_autoCreateTable && ex.Message.StartsWith("0:The table specified does not exist", StringComparison.CurrentCulture))
                {
                    // try to create the table
                    await CreateTableAsync <T>();

                    // retry
                    await StoreAsync <T>(storaeOperationType, models);
                }
                else
                {
                    // notify delegate
                    if (_delegate != null)
                    {
                        _delegate.OnStored(typeof(T), storaeOperationType, 0, ex);
                    }

                    throw ex;
                }
            }
        }
예제 #7
0
 public static async Task InsertOrReplaceAsync <T>(this CloudTable table, IEnumerable <T> entitiesToReplace) where T : ITableEntity, new()
 {
     await table.ExecuteBatchAsync(entitiesToReplace, (batch, entity) => batch.InsertOrReplace(entity));
 }
        public async Task <ActionResult> Edit(string partitionKey, string rowKey, string listName, string emailAddress, Subscriber editedSubscriber)
        {
            // Since MailingList and UpdateModel are in the view as EditorFor fields,
            // and since they refer to the same properties as PartitionKey and RowKey,
            // exclude PartitionKey and RowKey from model binding when calling UpdateModel.
            var excludeProperties = new string[] { "PartitionKey", "RowKey" };

            if (ModelState.IsValid)
            {
                try
                {
                    UpdateModel(editedSubscriber, string.Empty, null, excludeProperties);
                    if (editedSubscriber.PartitionKey == partitionKey && editedSubscriber.RowKey == rowKey)
                    {
                        //Keys didn't change -- Update the row
                        var replaceOperation = TableOperation.Replace(editedSubscriber);
                        await mailingListTable.ExecuteAsync(replaceOperation);
                    }
                    else
                    {
                        // Keys changed, delete the old record and insert the new one.
                        if (editedSubscriber.PartitionKey != partitionKey)
                        {
                            // PartitionKey changed, can't do delete/insert in a batch.
                            var deleteOperation = TableOperation.Delete(new Subscriber {
                                PartitionKey = partitionKey, RowKey = rowKey, ETag = editedSubscriber.ETag
                            });
                            await mailingListTable.ExecuteAsync(deleteOperation);

                            var insertOperation = TableOperation.Insert(editedSubscriber);
                            await mailingListTable.ExecuteAsync(insertOperation);
                        }
                        else
                        {
                            // RowKey changed, do delete/insert in a batch.
                            var batchOperation = new TableBatchOperation();
                            batchOperation.Delete(new Subscriber {
                                PartitionKey = partitionKey, RowKey = rowKey, ETag = editedSubscriber.ETag
                            });
                            batchOperation.Insert(editedSubscriber);
                            await mailingListTable.ExecuteBatchAsync(batchOperation);
                        }
                    }
                    return(RedirectToAction("Index"));
                }
                catch (StorageException ex)
                {
                    if (ex.RequestInformation.HttpStatusCode == 412)
                    {
                        // Concurrency error.
                        // Only catching concurrency errors for non-key fields.  If someone
                        // changes a key field we'll get a 404 and we have no way to know
                        // what they changed it to.
                        var currentSubscriber = FindRow(partitionKey, rowKey);
                        if (currentSubscriber.Verified != editedSubscriber.Verified)
                        {
                            ModelState.AddModelError("Verified", "Current value: " + currentSubscriber.Verified);
                        }
                        ModelState.AddModelError(string.Empty, "The record you attempted to edit "
                                                 + "was modified by another user after you got the original value. The "
                                                 + "edit operation was canceled and the current values in the database "
                                                 + "have been displayed. If you still want to edit this record, click "
                                                 + "the Save button again. Otherwise click the Back to List hyperlink.");
                        ModelState.SetModelValue("ETag", new ValueProviderResult(currentSubscriber.ETag, currentSubscriber.ETag, null));
                    }
                    else
                    {
                        throw;
                    }
                }
            }

            var lists = await GetListNamesAsync();

            ViewBag.ListName = new SelectList(lists, "ListName", "Description", listName);
            return(View(editedSubscriber));
        }
예제 #9
0
        public async Task CommitChanges()
        {
            await context.ExecuteBatchAsync(operations);

            operations = new TableBatchOperation();
        }
예제 #10
0
        public static async Task Run(
            [QueueTrigger("aladin-newbooks")] CloudQueueMessage message,
            [Table("BookEntity")] CloudTable bookTable,
            [Table("LineAccount")] CloudTable lineAccountTable,
            [Table("Credentials", "Twitter")] CloudTable credentialsTable,
            ILogger log,
            CancellationToken cancellationToken)
        {
            try
            {
                var    queueItem  = JsonConvert.DeserializeObject <QueueItem>(message.AsString);
                string categoryId = queueItem.CategoryId;

                var itemList       = new List <ItemLookUpResult.Item>();
                var batchOperation = new TableBatchOperation();

                foreach (int itemId in queueItem.ItemList)
                {
                    var lookUpResult = await LookUpItemAsync(itemId);

                    foreach (var item in lookUpResult.item)
                    {
                        cancellationToken.ThrowIfCancellationRequested();

                        var bookEntity = new BookEntity();
                        bookEntity.PartitionKey = categoryId;
                        bookEntity.RowKey       = itemId.ToString();
                        bookEntity.Name         = item.title;

                        batchOperation.InsertOrReplace(bookEntity);
                        itemList.Add(item);
                    }
                }

                if (itemList.Count > 0 && !cancellationToken.IsCancellationRequested)
                {
                    var tokens = await Twitter.Utils.CreateTokenAsync(credentialsTable, queueItem.CategoryId);

                    var tweetTask = tokens != null?TweetItemsAsync(tokens, itemList, cancellationToken) : Task.CompletedTask;

                    // 배치처리는 파티션 키가 동일해야하고, 100개까지 가능하다는데...
                    // 일단 파티션 키는 전부 동일하게 넘어올테고, 100개 넘을일은 없겠...지?
                    var tableTask = bookTable.ExecuteBatchAsync(batchOperation);

                    // 트위터와 테이블 쓰기 작업을 먼저한다.
                    await Task.WhenAll(tweetTask, tableTask);

                    // 라인 메시지를 보낼때 리미트가 걸려 예외가 발생하므로 라인만 따로 한다.
                    if (queueItem.CategoryId == Aladin.Const.CategoryID.Comics)
                    {
                        await SendLineMessageAsync(lineAccountTable, itemList, log);
                    }
                }
            }
            catch (Exception e)
            {
                log.LogError(e.Message);

                if (e is LineResponseException lineEx)
                {
                    if (lineEx.StatusCode != HttpStatusCode.TooManyRequests)
                    {
                        // 에러 발생시 내 계정으로 예외 정보를 보냄.
                        string adminLineId = Environment.GetEnvironmentVariable("LINE_ADMIN_USER_ID");
                        var    error       = new
                        {
                            Type       = e.GetType().ToString(),
                            Message    = e.Message,
                            StackTrace = e.StackTrace
                        };
                        string json = JsonConvert.SerializeObject(error, Formatting.Indented);
                        await lineMessagingClient.PushMessageAsync(adminLineId, json);
                    }
                }
            }
        }
예제 #11
0
        public static long DeleteFileProcessStats(DataConfig providerConfig, POCO.System system)
        {
            long numEntriesDeleted = 0;

            string systemKey = system.PartitionKey;

            if (!systemKey.EndsWith("|"))
            {
                systemKey += "|";
            }

            // Create a filter for the system
            List <Filter> filters = new List <Filter>();
            Filter        pkfilt  = new Filter("PartitionKey", systemKey, "eq");

            filters.Add(pkfilt);

            // Check the data provider to use
            switch (providerConfig.ProviderType)
            {
            case "azure.tableservice":
                string combinedFilter = Utils.GenerateAzureFilter(filters);

                CloudTable table = Utils.GetCloudTable(providerConfig, AzureTableNames.FileProcessStats);

                TableQuery <TableEntity> query = new TableQuery <TableEntity>().Where(combinedFilter);

                List <TableEntity> recordsToDelete = new List <TableEntity>();

                TableContinuationToken token = null;

                var runningQuery = new TableQuery <TableEntity>()
                {
                    FilterString  = query.FilterString,
                    SelectColumns = query.SelectColumns
                };

                do
                {
                    runningQuery.TakeCount = query.TakeCount - recordsToDelete.Count;

                    // Add the entries to the recordsToDelete list
                    Task <TableQuerySegment <TableEntity> > tSeg = table.ExecuteQuerySegmentedAsync <TableEntity>(runningQuery, token);
                    tSeg.Wait();
                    token = tSeg.Result.ContinuationToken;
                    recordsToDelete.AddRange(tSeg.Result);

                    // Create a batch of records to delete
                    TableBatchOperation batch = new TableBatchOperation();
                    foreach (TableEntity entity in recordsToDelete)
                    {
                        batch.Add(TableOperation.Delete(entity));

                        if (batch.Count == 100)
                        {
                            numEntriesDeleted += batch.Count;
                            Task tBatchDelete = table.ExecuteBatchAsync(batch);
                            tBatchDelete.Wait();
                            batch = new TableBatchOperation();
                        }
                    }
                    if (batch.Count > 0)
                    {
                        numEntriesDeleted += batch.Count;
                        Task tBatchDelete = table.ExecuteBatchAsync(batch);
                        tBatchDelete.Wait();
                    }
                } while (token != null && (query.TakeCount == null || recordsToDelete.Count < query.TakeCount.Value));        //!ct.IsCancellationRequested &&

                break;

            case "internal.mongodb":
                IMongoCollection <BsonDocument> collection = Utils.GetMongoCollection <BsonDocument>(providerConfig, MongoTableNames.FileProcessStats);

                // Create the delete filter
                FilterDefinition <BsonDocument> filter = Utils.GenerateMongoFilter <BsonDocument>(filters);

                // Replace current document
                DeleteResult result = collection.DeleteMany(filter);

                numEntriesDeleted = result.DeletedCount;
                break;

            default:
                throw new ApplicationException("Data provider not recognised: " + providerConfig.ProviderType);
            }

            return(numEntriesDeleted);
        }
예제 #12
0
 /// <summary>
 ///     Executes a batch of operations on a table asynchronously.
 /// </summary>
 /// <param name="tableBatchOperation">
 ///     The <see cref="T:Microsoft.WindowsAzure.Storage.Table.TableBatchOperation" /> object representing the operations to execute on the table.
 /// </param>
 /// <param name="cancellationToken">Cancalltion token.</param>
 /// <returns>
 ///     An enumerable collection of <see cref="T:Microsoft.WindowsAzure.Storage.Table.TableResult" /> objects that contains the results, in order, of each operation in the
 ///     <see cref="T:Microsoft.WindowsAzure.Storage.Table.TableBatchOperation" />
 ///     on the table.
 /// </returns>
 public Task <IList <TableResult> > ExecuteBatchAsync(TableBatchOperation tableBatchOperation, CancellationToken cancellationToken)
 {
     return(_cloudTable.ExecuteBatchAsync(tableBatchOperation, cancellationToken));
 }
예제 #13
0
        private static async Task <String> InsertAllTheThings(CloudTable _table)
        {
            var taskCount     = 0;
            var taskThreshold = 200; // Seems to be a good value to start with
            var stopwatch     = new Stopwatch();

            stopwatch.Start();

            var items = new List <MyTableEntity>();
            // Create a new customer entity.

            List <String> firstnames = new List <String>();
            List <String> lastnames  = new List <String>();

            Console.WriteLine("Reading CSV files...");
            using (var reader = new StreamReader(firstNamesPath))
            {
                var line   = reader.ReadLine();
                var values = line.Split(',');
                firstnames = values.ToList <String>();
            }

            using (var reader = new StreamReader(lastNamesPath))
            {
                var line   = reader.ReadLine();
                var values = line.Split(',');
                lastnames = values.ToList <String>();
            }

            Console.WriteLine("Reading firstnames and lastnames finished by: " + stopwatch.Elapsed.ToString());

            Console.WriteLine("Creating random lists from firstnames and lastnames...");
            while (items.Count < soManyItems)
            {
                var partitionKey = "placeholder";

                var rnd       = new Random();
                var firstname = firstnames[rnd.Next(0, firstnames.Count - 1)];
                var lastname  = lastnames[rnd.Next(0, lastnames.Count - 1)];

                MyTableEntity entity1 = new MyTableEntity(partitionKey);
                entity1.Email       = String.Format("{0}@{1}.com", firstname, lastname);
                entity1.PhoneNumber = String.Format("{0}-{1}-{2}", rnd.Next(1, 999), rnd.Next(1, 999), rnd.Next(1, 9999));
                entity1.Firstname   = firstname;
                entity1.Lastname    = lastname;

                items.Add(entity1);
            }

            Console.WriteLine("Creating random lists finished by: " + stopwatch.Elapsed.ToString());

            Console.WriteLine("Starting batch operations for " + soManyItems + " items...");
            Console.WriteLine("200 tasks with 100 inserts running parallel...");
            Console.WriteLine("for a total of 20.000 inserts per operation...");

            var batchTasks = new List <Task <TableBatchResult> >();

            var count       = 1;
            var maxParallel = 100; // 100 is max

            for (var i = 0; i < items.Count; i += maxParallel)
            {
                taskCount++;
                var partitionkey = "" + i + maxParallel;
                var batchItems   = items.Skip(i).Take(maxParallel).ToList();

                var batch = new TableBatchOperation();

                foreach (var item in batchItems)
                {
                    item.PartitionKey = "partition" + count;
                    batch.InsertOrMerge(item);
                }

                var task = _table.ExecuteBatchAsync(batch);
                batchTasks.Add(task);

                if (taskCount >= taskThreshold)
                {
                    await Task.WhenAll(batchTasks);

                    Console.WriteLine("Finished batch " + count + " by: " + stopwatch.Elapsed.ToString());
                    count++;
                    taskCount = 0;
                }
            }
            //Console.WriteLine("Finished by: " + stopwatch.Elapsed.ToString());

            //Task.WhenAll(batchTasks);

            //Console.WriteLine("Finished all by: " + stopwatch.Elapsed.ToString());
            stopwatch.Stop();

            return("Finished " + soManyItems + " inserts by " + stopwatch.Elapsed.ToString());
        }
예제 #14
0
        public AzureDatabase(string storageAccountString, string tableName)
        {
            CloudStorageAccount account = CloudStorageAccount.Parse(storageAccountString);

            qclient = account.CreateCloudQueueClient();
            client  = account.CreateCloudTableClient();
            client.DefaultRequestOptions.PayloadFormat = TablePayloadFormat.JsonNoMetadata;
            table   = client.GetTableReference(tableName);
            mthread = new Thread(async delegate() {
                await table.CreateIfNotExistsAsync();
                while (running)
                {
                    evt.WaitOne();
                    evt.Reset();
                    Dictionary <ulong, List <AzureOperationHandle> > ops;
                    lock (evt)
                    {
                        ops = pendingOperations;
                        pendingOperations = new Dictionary <ulong, List <AzureOperationHandle> >();
                    }

                    List <Task> runningTasks = new List <Task>();
                    foreach (var shard in ops)
                    {
                        TableBatchOperation upserts   = new TableBatchOperation();
                        TableBatchOperation deletions = new TableBatchOperation();
                        Dictionary <ScalableEntity, List <AzureOperationHandle> > retrieves = new Dictionary <ScalableEntity, List <AzureOperationHandle> >(EntityComparer.instance);
                        Dictionary <ByteRange, List <AzureOperationHandle> > rangeRetrieves = new Dictionary <ByteRange, List <AzureOperationHandle> >();
                        foreach (var op in shard.Value.Where(m => m.Type == OpType.Upsert || m.Type == OpType.Delete))
                        {
                            switch (op.Type)
                            {
                            case OpType.Upsert:
                                upserts.Add(TableOperation.InsertOrReplace(new AzureEntity()
                                {
                                    PartitionKey = op.Entity.Partition.ToString(), RowKey = Uri.EscapeDataString(Convert.ToBase64String(op.Entity.Key)), Value = op.Entity.Value
                                }));
                                if (upserts.Count == 100)
                                {
                                    runningTasks.Add(table.ExecuteBatchAsync(upserts));
                                    upserts = new TableBatchOperation();
                                }
                                break;

                            case OpType.Delete:
                                deletions.Add(TableOperation.Delete(new AzureEntity()
                                {
                                    PartitionKey = op.Entity.Partition.ToString(), RowKey = Uri.EscapeDataString(Convert.ToBase64String(op.Entity.Key)), Value = op.Entity.Value
                                }));
                                if (deletions.Count == 100)
                                {
                                    runningTasks.Add(table.ExecuteBatchAsync(deletions));
                                    deletions = new TableBatchOperation();
                                }
                                break;
                            }
                        }
                        Func <IEnumerable <string>, string> and = (q) => {
                            string query = null;
                            foreach (string er in q.Where(m => m != null))
                            {
                                if (query == null)
                                {
                                    query = er;
                                }
                                else
                                {
                                    query = TableQuery.CombineFilters(query, TableOperators.And, er);
                                }
                            }
                            return(query);
                        };
                        Func <IEnumerable <string>, string> or = (q) => {
                            string query = null;
                            foreach (string er in q.Where(m => m != null))
                            {
                                if (query == null)
                                {
                                    query = er;
                                }
                                else
                                {
                                    query = TableQuery.CombineFilters(query, TableOperators.Or, er);
                                }
                            }
                            return(query);
                        };
                        Func <Dictionary <ScalableEntity, List <AzureOperationHandle> >, Dictionary <ByteRange, List <AzureOperationHandle> >, TableContinuationToken, TableQuery <AzureEntity>, Task> runSegmentedQuery = null;
                        runSegmentedQuery = async(tableops, rangeops, token, compiledQuery) => {
                            if (compiledQuery == null)
                            {
                                string query = null;
                                foreach (var iable in tableops.Values.SelectMany(m => m).Where(m => m.Type == OpType.Retrieve))
                                {
                                    if (query == null)
                                    {
                                        query = TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, iable.Entity.Partition.ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, Uri.EscapeDataString(Convert.ToBase64String(iable.Entity.Key))));
                                    }
                                    else
                                    {
                                        query = TableQuery.CombineFilters(query, TableOperators.Or, TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, iable.Entity.Partition.ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, Uri.EscapeDataString(Convert.ToBase64String(iable.Entity.Key)))));
                                    }
                                }

                                foreach (var iable in rangeops.Values.SelectMany(m => m).Where(m => m.Type == OpType.RangeRetrieve))
                                {
                                    string startQuery = null;
                                    string endQuery   = null;
                                    if (iable.StartRange != null)
                                    {
                                        startQuery = TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThan, iable.StartRange.LinearHash().ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThan, Uri.EscapeDataString(Convert.ToBase64String(iable.StartRange))));
                                    }
                                    if (iable.EndRange != null)
                                    {
                                        endQuery = TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.LessThan, iable.EndRange.LinearHash().ToString()), TableOperators.And, TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, Uri.EscapeDataString(Convert.ToBase64String(iable.EndRange))));
                                    }
                                    query = or(new string[] { query, and(new string[] { startQuery, endQuery }) });
                                }
                                compiledQuery = new TableQuery <AzureEntity>().Where(query);
                            }
                            var segment = await table.ExecuteQuerySegmentedAsync(compiledQuery, token);
                            token       = segment.ContinuationToken;
                            List <AzureOperationHandle> finished = new List <AzureOperationHandle>();
                            foreach (var iable in segment)
                            {
                                var ent = new ScalableEntity(Convert.FromBase64String(Uri.UnescapeDataString(iable.RowKey)), iable.Value);
                                if (tableops.ContainsKey(ent))
                                {
                                    finished.AddRange(tableops[ent].Select(m => m.SetValue(ent.Value)));
                                }
                                if (rangeops.Any())
                                {
                                    var range = new ByteRange(iable.Value, iable.Value);
                                    if (rangeops.ContainsKey(range))
                                    {
                                        finished.AddRange(rangeops[range].Select(m => m.AddValue(new ScalableEntity(Convert.FromBase64String(Uri.UnescapeDataString(iable.RowKey)), iable.Value))));
                                    }
                                }
                            }
                            //Combine callbacks for finished queries
                            var callbacks = finished.ToLookup(m => m.callback);
                            foreach (var iable in callbacks)
                            {
                                if (!iable.Key(iable.Where(m => m.Entity != null).Select(m => m.Entity).Union(iable.SelectMany(m => m.values), EntityComparer.instance)))
                                {
                                    iable.AsParallel().ForAll(m => m.Type = OpType.Nop);
                                    compiledQuery = null; //Re-compile query
                                }
                            }
                            if (token != null)
                            {
                                await runSegmentedQuery(tableops, rangeops, token, compiledQuery);
                            }
                        };
                        foreach (var op in shard.Value.Where(m => m.Type == OpType.Retrieve || m.Type == OpType.RangeRetrieve))
                        {
                            switch (op.Type)
                            {
                            case OpType.Retrieve:
                                if (!retrieves.ContainsKey(op.Entity))
                                {
                                    retrieves.Add(op.Entity, new List <AzureOperationHandle>());
                                }
                                retrieves[op.Entity].Add(op);
                                if (retrieves.Count + rangeRetrieves.Count == 100)
                                {
                                    runningTasks.Add(runSegmentedQuery(retrieves, rangeRetrieves, null, null));
                                    retrieves      = new Dictionary <ScalableEntity, List <AzureOperationHandle> >(EntityComparer.instance);
                                    rangeRetrieves = new Dictionary <ByteRange, List <AzureOperationHandle> >();
                                }
                                break;

                            case OpType.RangeRetrieve:
                                var ranger = new ByteRange(op.StartRange, op.EndRange);
                                if (!rangeRetrieves.ContainsKey(ranger))
                                {
                                    rangeRetrieves.Add(ranger, new List <AzureOperationHandle>());
                                }
                                rangeRetrieves[ranger].Add(op);
                                if (retrieves.Count + rangeRetrieves.Count == 100)
                                {
                                    runningTasks.Add(runSegmentedQuery(retrieves, rangeRetrieves, null, null));
                                    retrieves      = new Dictionary <ScalableEntity, List <AzureOperationHandle> >(EntityComparer.instance);
                                    rangeRetrieves = new Dictionary <ByteRange, List <AzureOperationHandle> >();
                                }
                                break;
                            }
                        }
                        foreach (var op in shard.Value.Where(m => m.Type == OpType.Nop))
                        {
                            op.Task.SetResult(op.Entity);
                        }
                        if (upserts.Any())
                        {
                            runningTasks.Add(table.ExecuteBatchAsync(upserts));
                        }
                        if (retrieves.Any() || rangeRetrieves.Any())
                        {
                            runningTasks.Add(runSegmentedQuery(retrieves, rangeRetrieves, null, null));
                        }
                    }
                    await Task.WhenAll(runningTasks);
                    ops.SelectMany(m => m.Value).AsParallel().ForAll(m => m.Task.SetResult(m.Entity));
                }
            });
            mthread.Name = "AzureDB-webrunner";
            mthread.Start();
        }
예제 #15
0
        public async Task DoBatchAsync(TableBatchOperation batch)
        {
            CloudTable table = await GetTable();

            await table.ExecuteBatchAsync(batch);
        }
        public async Task TableFilterTest()
        {
            // Reinitialize the name resolver to avoid conflicts
            _resolver = new RandomNameResolver();

            IHost host = new HostBuilder()
                         .ConfigureDefaultTestHost <AzureStorageEndToEndTests>(b =>
            {
                b.AddAzureStorage();
            })
                         .ConfigureServices(services =>
            {
                services.AddSingleton <INameResolver>(_resolver);
            })
                         .Build();

            // write test entities
            string           testTableName = _resolver.ResolveInString(TableName);
            CloudTableClient tableClient   = _storageAccount.CreateCloudTableClient();
            CloudTable       table         = tableClient.GetTableReference(testTableName);
            await table.CreateIfNotExistsAsync();

            var operation = new TableBatchOperation();

            operation.Insert(new Person
            {
                PartitionKey = "1",
                RowKey       = "1",
                Name         = "Lary",
                Age          = 20,
                Location     = "Seattle"
            });
            operation.Insert(new Person
            {
                PartitionKey = "1",
                RowKey       = "2",
                Name         = "Moe",
                Age          = 35,
                Location     = "Seattle"
            });
            operation.Insert(new Person
            {
                PartitionKey = "1",
                RowKey       = "3",
                Name         = "Curly",
                Age          = 45,
                Location     = "Texas"
            });
            operation.Insert(new Person
            {
                PartitionKey = "1",
                RowKey       = "4",
                Name         = "Bill",
                Age          = 28,
                Location     = "Tam O'Shanter"
            });

            await table.ExecuteBatchAsync(operation);

            JobHost jobHost    = host.GetJobHost();
            var     methodInfo = GetType().GetMethod(nameof(TableWithFilter));
            var     input      = new Person {
                Age = 25, Location = "Seattle"
            };
            string json      = JsonConvert.SerializeObject(input);
            var    arguments = new { person = json };
            await jobHost.CallAsync(methodInfo, arguments);

            // wait for test results to appear
            await TestHelpers.Await(() => testResult != null);

            JArray results = (JArray)testResult;

            Assert.Single(results);

            input = new Person {
                Age = 25, Location = "Tam O'Shanter"
            };
            json      = JsonConvert.SerializeObject(input);
            arguments = new { person = json };
            await jobHost.CallAsync(methodInfo, arguments);

            await TestHelpers.Await(() => testResult != null);

            results = (JArray)testResult;
            Assert.Single(results);
            Assert.Equal("Bill", (string)results[0]["Name"]);
        }
예제 #17
0
 public async Task AddBatch(TableBatchOperation tableOperation)
 {
     await table.ExecuteBatchAsync(tableOperation);
 }
        public override async Task ExecuteAsync(CloudTable cloudTable = null)
        {
            // Collection of ids
            List <string> moviesType1 = new List <string>();
            List <string> moviesType2 = new List <string>();

            // Prepare batch operation
            TableBatchOperation batchOperation1 = new TableBatchOperation();
            TableBatchOperation batchOperation2 = new TableBatchOperation();
            TableBatchOperation batchOperation3 = new TableBatchOperation();

            // Generate some data
            Console.WriteLine("...Generate test data...");
            for (int i = 1; i <= 10; i++)
            {
                var movie = new Movie(type: "Comedy", title: $"Movie {i}")
                {
                    Id          = i,
                    Title       = $"Movie {i}",
                    Description = "Sample description",
                    Type        = "Comedy",
                    Premiere    = 2011,
                    Timestamp   = new DateTimeOffset(DateTime.UtcNow)
                };

                moviesType1.Add(movie.Title);
                batchOperation1.Insert(movie);
            }

            // Generate some data
            for (int i = 1; i <= 10; i++)
            {
                var movie = new Movie(type: "Horror", title: $"Other Movie {i}")
                {
                    Id          = i,
                    Title       = $"Other Movie {i}",
                    Description = "Sample description",
                    Type        = "Horror",
                    Premiere    = 2001,
                    Timestamp   = new DateTimeOffset(DateTime.UtcNow)
                };

                moviesType2.Add(movie.Title);
                batchOperation2.Insert(movie);
            }

            for (int i = 1; i <= 10; i++)
            {
                var movie = new Movie(type: "Horror", title: $"Other Horror Movie {i}")
                {
                    Id          = i,
                    Title       = $"Other Horror Movie {i}",
                    Description = "Sample description",
                    Type        = "Horror",
                    Premiere    = 2002,
                    Timestamp   = new DateTimeOffset(DateTime.UtcNow)
                };

                batchOperation3.Insert(movie);
            }

            // Execute batch
            Console.WriteLine("...Execute batch operations...");
            await cloudTable.ExecuteBatchAsync(batchOperation1);

            await cloudTable.ExecuteBatchAsync(batchOperation2);

            await cloudTable.ExecuteBatchAsync(batchOperation3);

            // Insert Index Entities
            var index1 = new Index(partitionKey: "Comedy", rowKey: "2011")
            {
                IndexEntities = string.Join(',', moviesType1.ToArray())
            };
            var index2 = new Index(partitionKey: "Horror", rowKey: "2001")
            {
                IndexEntities = string.Join(',', moviesType2.ToArray())
            };

            TableOperation insert1 = TableOperation.Insert(index1);
            TableOperation insert2 = TableOperation.Insert(index2);

            // Execute insert operations
            Console.WriteLine("...Execute insert operations...");
            await cloudTable.ExecuteAsync(insert1);

            await cloudTable.ExecuteAsync(insert2);


            // Retrive index
            Console.WriteLine("...Retrive index...");
            TableQuery <Index> rangeQuery1 = new TableQuery <Index>()
                                             .Where(TableQuery.CombineFilters(
                                                        TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "Comedy")
                                                        , TableOperators.And
                                                        , TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, "2011")));

            var result1 = await cloudTable.ExecuteQuerySegmentedAsync(rangeQuery1, null);

            string[] movieTitles = { };
            foreach (var index in result1)
            {
                Console.WriteLine($"{nameof(index.PartitionKey)}: {index.PartitionKey}, {nameof(index.RowKey)}: {index.RowKey}, {nameof(index.IndexEntities)}: {index.IndexEntities}");
                movieTitles = index.IndexEntities.Split(',');
            }

            // Retrive entities
            Console.WriteLine("...Retrive entities - using index...");
            TableQuery <Movie> query = new TableQuery <Movie>();

            query.Where(
                TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "Comedy"),
                                          TableOperators.And,
                                          TableQuery.CombineFilters(TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, "Movie 1"),
                                                                    TableOperators.Or,
                                                                    TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThanOrEqual, "Movie 10")))
                );

            var movies = await cloudTable.ExecuteQuerySegmentedAsync(query, null);

            foreach (var movie in movies)
            {
                Console.WriteLine($"Movie Type: {movie.Type}, Premiere: {movie.Premiere}, Title: {movie.Title}");
            }
        }
예제 #19
0
        public static async Task <IActionResult> Run(
            [HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "ip")] HttpRequest req,
            [Table(HostIpDataEntity.TableName, Connection = "AzureWebJobsStorage")] CloudTable cloudTable,
            ClaimsPrincipal principal, ILogger log)
        {
            var userIdClaim = principal?.FindFirst(Claims.ObjectId);
            var userId      = userIdClaim?.Value;

            if (string.IsNullOrWhiteSpace(userId))
            {
                log.LogError("request has invalid userId {UserId}", userId);
                return(new UnauthorizedResult());
            }

            log.LogTrace("IP update request received");

            string requestBody = await new StreamReader(req.Body).ReadToEndAsync();

            UpdateIPRequestModel model;

            try
            {
                model = JsonConvert.DeserializeObject <UpdateIPRequestModel>(requestBody);
            }
            catch (JsonException)
            {
                log.LogError("IP update request has invalid payload");
                return(new BadRequestResult());
            }

            var  validationResult = new List <ValidationResult>();
            bool isValid          = Validator.TryValidateObject(model, new ValidationContext(model), validationResult);

            if (!isValid)
            {
                log.LogError("IP update request has invalid model");
                return(new BadRequestObjectResult(validationResult));
            }

            try
            {
                TableBatchOperation batchOperations = new TableBatchOperation();

                foreach (var nicInfo in model.NicIp)
                {
                    var            entity   = new HostIpDataEntity(userId, model.ComputerName, nicInfo.Id, nicInfo.Name, nicInfo.Addresses);
                    TableOperation insertOp = TableOperation.InsertOrMerge(entity);
                    batchOperations.Add(insertOp);
                }
                TableBatchResult result = await cloudTable.ExecuteBatchAsync(batchOperations);
            }
            catch (StorageException e)
            {
                log.LogError(e.Message);
                throw;
            }

            log.LogTrace("successfully updated IP for computer {ComputerName}, {Entries} entries updated", model.ComputerName, model.NicIp.Count);

            return(new NoContentResult());
        }
        public async Task DoBatchAsync(TableBatchOperation batch)
        {
            CloudTable table = await GetTable();

            await table.ExecuteBatchAsync(batch, GetRequestOptions(), null);
        }
예제 #21
0
 public static async Task RemoveAsync <T>(this CloudTable table, IEnumerable <T> entitiesToRemove) where T : ITableEntity, new()
 {
     await table.ExecuteBatchAsync(entitiesToRemove, (batch, entity) => batch.Delete(entity));
 }
        public static async Task Run([TimerTrigger("0 */1 * * * *")] TimerInfo myTimer, ILogger log, [Table("dselbtable")] CloudTable loadbalancerconfigTable)
        {
            log.LogInformation($"C# Timer trigger function executed at: {DateTime.Now}");


            // query all rows
            var tableQuery = new TableQuery <UpdateLoadBalancerEntity>();
            var result     = await loadbalancerconfigTable.ExecuteQuerySegmentedAsync(tableQuery, null);

            // Create the batch operation.
            TableBatchOperation batchDeleteOperation = new TableBatchOperation();

            foreach (var row in result)
            {
                batchDeleteOperation.Delete(row);
            }

            // Execute the batch operation.
            await loadbalancerconfigTable.ExecuteBatchAsync(batchDeleteOperation);

            //Get Token
            string token = AuthHelper.GetTokenAsync().Result;

            log.LogInformation($"Token Received: {token}");

            string        subscriptionsUri = "https://management.azure.com/subscriptions?api-version=2016-06-01";
            Subscriptions subscriptions    = await ResilientRestClient.GetAsync <Subscriptions>(subscriptionsUri, token);

            log.LogInformation($"Subs Received");

            //Query SubscriptionIDs which has Gateway Connections
            string query = "where type =~ 'microsoft.network/connections'| distinct subscriptionId ";
            Dictionary <string, int> options = new Dictionary <string, int>();

            options["$skip"] = 0;
            Dictionary <string, object> requestBodyObj = new Dictionary <string, object>();
            List <string> subscriptionIds = subscriptions.value.Select(subs => subs.subscriptionId).ToList();

            requestBodyObj.Add("subscriptions", subscriptionIds);
            requestBodyObj.Add("query", query);
            requestBodyObj.Add("options", options);
            string        resourceGraphUri = "https://management.azure.com/providers/Microsoft.ResourceGraph/resources?api-version=2018-09-01-preview";
            List <string> expressRouteConnectedSubscriptions = new List <string>();
            //TEST CASE
            string subId = Environment.GetEnvironmentVariable("TestSubscriptionID", EnvironmentVariableTarget.Process);

            expressRouteConnectedSubscriptions.Add(subId);
            //Commenting for test
            // ResourceGraphResponse resourcesubs = ResilientRestClient.PostAsync<ResourceGraphResponse>(resourceGraphUri, token, requestBodyObj).Result;
            // foreach (List<string> row in resourcesubs.data.rows)
            // {
            //     expressRouteConnectedSubscriptions.Add(row[0]);
            // }
            // log.LogInformation($"The number of subs are {expressRouteConnectedSubscriptions.Count}");

            //TODO ADD Logic to clear the table or after automation job completion trigger an update to remove the upgraded LB from table

            //Query for Load Balancers
            string lbquery = "where type =~ 'Microsoft.Network/loadbalancers'| where tostring(sku.name) =='Basic' | project id, subscriptionId, resourceGroup, name, location";
            Dictionary <string, int> lboptions = new Dictionary <string, int>();

            options["$skip"] = 0;
            Dictionary <string, object> lbrequestBodyObj = new Dictionary <string, object>();

            lbrequestBodyObj.Add("subscriptions", expressRouteConnectedSubscriptions);
            lbrequestBodyObj.Add("query", lbquery);
            lbrequestBodyObj.Add("options", lboptions);

            List <LoadBalancerObj> loadBalancers = new List <LoadBalancerObj>();

            ResourceGraphResponse lbs = ResilientRestClient.PostAsync <ResourceGraphResponse>(resourceGraphUri, token, lbrequestBodyObj).Result;
            int i = 1;

            foreach (List <string> row in lbs.data.rows)
            {
                LoadBalancerObj lb = new LoadBalancerObj
                {
                    ResourceId     = row[0],
                    SubscriptionId = row[1],
                    ResourceGroup  = row[2],
                    Name           = row[3],
                    Location       = row[4]
                };
                loadBalancers.Add(lb);
            }
            while (lbs.skipToken != null)
            {
                log.LogInformation(i.ToString());
                Dictionary <string, int> nextSkip = new Dictionary <string, int>();
                nextSkip["$skip"] = i * 1000;
                Dictionary <string, object> updatedrequestBodyObj = new Dictionary <string, object>();
                updatedrequestBodyObj["subscriptions"] = expressRouteConnectedSubscriptions;
                updatedrequestBodyObj["query"]         = lbquery;
                updatedrequestBodyObj["$skipToken"]    = lbs.skipToken;
                updatedrequestBodyObj["options"]       = nextSkip;
                lbs = ResilientRestClient.PostAsync <ResourceGraphResponse>(resourceGraphUri, token, updatedrequestBodyObj).Result;
                foreach (List <string> row in lbs.data.rows)
                {
                    LoadBalancerObj lb = new LoadBalancerObj
                    {
                        ResourceId     = row[0],
                        SubscriptionId = row[1],
                        ResourceGroup  = row[2],
                        Name           = row[3],
                        Location       = row[4]
                    };
                    loadBalancers.Add(lb);
                }
                i++;
            }

            Parallel.ForEach(loadBalancers, async lb =>
            {
                UpdateLoadBalancerEntity loadBalancerEntity = new UpdateLoadBalancerEntity
                {
                    PartitionKey  = lb.SubscriptionId,
                    RowKey        = lb.Name,
                    Location      = lb.Location,
                    ResourceGroup = lb.ResourceGroup,
                    ResourceId    = lb.ResourceId
                };
                TableOperation insertOperation = TableOperation.InsertOrMerge(loadBalancerEntity);
                await loadbalancerconfigTable.ExecuteAsync(insertOperation);
            });
        }
예제 #23
0
        // Processes a page of members (including guests) or owners for one group and stores owners and guests to Azure table storage
        private static async Task <GroupMemberPageResult> ProcessGroupMemberPageAsync(string getMembersUrl, JToken group, HttpClient httpClient,
                                                                                      CloudTable groupGuestsTable, CloudTable groupOwnersTable, bool isOwners)
        {
            var result = new GroupMemberPageResult();
            HttpResponseMessage groupMembersJson;

            groupMembersJson = await httpClient.GetAsync(getMembersUrl).ConfigureAwait(false);

            var groupMemberJsonObject = JObject.Parse(groupMembersJson.Content.ReadAsStringAsync().Result);

            var groupGuestsBatchOperation = new TableBatchOperation();
            var groupOwnersBatchOperation = new TableBatchOperation();

            if (groupMemberJsonObject["value"] is JArray groupMembers)
            {
                foreach (var groupMember in groupMembers)
                {
                    var groupMemberTableEntity = new GroupMemberTableEntity()
                    {
                        PartitionKey      = group["id"]?.ToString(),
                        RowKey            = Guid.NewGuid().ToString(),
                        GroupId           = group["id"]?.ToString(),
                        GroupDisplayName  = group["displayName"]?.ToString(),
                        GroupMailNickname = group["mailNickname"]?.ToString(),
                        Id             = groupMember["id"]?.ToString(),
                        UPN            = groupMember["userPrincipalName"]?.ToString(),
                        DisplayName    = groupMember["displayName"]?.ToString(),
                        AccountEnabled = groupMember["accountEnabled"]?.ToString()
                    };
                    if (isOwners)
                    {
                        result.OwnerCount++;
                        groupOwnersBatchOperation.Add(TableOperation.InsertOrReplace(groupMemberTableEntity));
                    }
                    else
                    {
                        result.MemberCount++;
                        //if member is a guest
                        if (groupMember["userType"] != null &&
                            groupMember["userType"].ToString().ToLower().Equals("guest"))
                        {
                            result.GuestCount++;
                            groupGuestsBatchOperation.Add(TableOperation.InsertOrReplace(groupMemberTableEntity));
                        }
                    }
                }
            }

            if (isOwners && groupOwnersBatchOperation.Count > 0)
            {
                await groupOwnersTable.ExecuteBatchAsync(groupOwnersBatchOperation)
                .ConfigureAwait(false);
            }
            else if (groupGuestsBatchOperation.Count > 0)
            {
                await groupGuestsTable.ExecuteBatchAsync(groupGuestsBatchOperation)
                .ConfigureAwait(false);
            }

            result.NextLink = groupMemberJsonObject["@odata.nextLink"]?.ToString();
            return(result);
        }
예제 #24
0
 /// <summary>
 ///     Executes a batch operation on a table as an atomic operation.
 /// </summary>
 /// <param name="table">Target table.</param>
 /// <param name="batch">
 ///     The <see cref="T:Microsoft.WindowsAzure.Storage.Table.TableBatchOperation" /> object representing the operations to execute on the table.
 /// </param>
 /// <returns>
 ///     An enumerable collection of <see cref="T:Microsoft.WindowsAzure.Storage.Table.TableResult" /> objects that contains the results, in order, of each operation in the
 ///     <see
 ///         cref="T:Microsoft.WindowsAzure.Storage.Table.TableBatchOperation" />
 ///     on the table.
 /// </returns>
 public static IList <TableResult> ExecuteBatch(this CloudTable table, TableBatchOperation batch)
 {
     return(table.ExecuteBatchAsync(batch).ExecuteSynchronously());
 }
예제 #25
0
        private async Task UploadBufferAsync()
        {
            if (m_Buffer.Count > 1 && m_Buffer[0].UseBatches)
            {
                // Batch upload
                var table   = MapTable(m_Buffer[0]);
                var uploads = new TableBatchOperation();
                var links   = new TableBatchOperation();
                var id      = m_Buffer[0].Controller;

                foreach (var data in m_Buffer)
                {
                    var entity = data.ToEntity(data.ID ?? m_RowKeyBase + "-" + m_Seq++);
                    uploads.Insert(entity);
                    if (data.ID != null)
                    {
                        links.Insert(new Link(table.Name, data.ID, entity.PartitionKey, entity.RowKey));
                    }
                }

                OnDebug?.Invoke($"Batch uploading {uploads.Count} records to Azure table storage {table.Name} for controller [{id}]...");

                try {
                    if (links.Count > 0)
                    {
                        await m_LinksTable.ExecuteBatchAsync(links);
                    }

                    var r = await table.ExecuteBatchAsync(uploads);

                    // Check for errors
                    var errors = m_Buffer
                                 .Where((entry, x) => r.Count <= x || (r[x].HttpStatusCode != 201 && r[x].HttpStatusCode != 204))
                                 .ToList();

                    var successes = m_Buffer
                                    .Where((entry, x) => r.Count > x && (r[x].HttpStatusCode == 201 || r[x].HttpStatusCode == 204))
                                    .ToList();

                    OnUploadSuccess?.Invoke(201, successes, $"{m_Buffer.Count - errors.Count} record(s) out of {m_Buffer.Count} for controller [{id}] successfully uploaded to Azure table storage {table.Name}.");

                    m_Buffer.Clear();

                    if (errors.Count > 0)
                    {
                        m_Buffer.AddRange(errors);
                        OnUploadError?.Invoke(0, errors, $"{errors.Count} record(s) for controller [{id}] failed to upload to Azure table storage {table.Name}.");
                    }
                } catch (StorageException ex) {
                    var status = ex.RequestInformation.HttpStatusCode;
                    var errmsg = ex.RequestInformation.ExtendedErrorInformation?.ErrorMessage ?? ex.RequestInformation.HttpStatusMessage ?? ex.Message;

                    switch (status)
                    {
                    case 0: {
                        OnError?.Invoke(ex, $"Azure table storage batch upload to {table.Name} for controller [{id}] failed.");
                        break;
                    }

                    case 401:
                    case 403: {
                        OnUploadError?.Invoke(status, m_Buffer, $"Azure table storage batch upload to {table.Name} for controller [{id}] forbidden: {errmsg}");
                        break;
                    }

                    default: {
                        OnUploadError?.Invoke(status, m_Buffer, $"Azure table storage batch upload to {table.Name} for controller [{id}] failed: {errmsg}");
                        break;
                    }
                    }
                } catch (Exception ex) {
                    OnError?.Invoke(ex, $"Azure table storage batch upload to {table.Name} for controller [{id}] failed.");
                }
            }
            else if (m_Buffer.Count > 0)
            {
                // Single upload
                var data   = m_Buffer[0];
                var id     = data.Controller;
                var table  = MapTable(data);
                var entity = data.ToEntity(data.ID ?? m_RowKeyBase + "-" + m_Seq++);
                var insert = TableOperation.Insert(entity);
                var link   = (data.ID != null) ? TableOperation.Insert(new Link(table.Name, data.ID, entity.PartitionKey, entity.RowKey)) : null;

                OnDebug?.Invoke($"Uploading record to Azure table storage {table.Name} for controller [{id}]...");

                try {
                    TableResult r;

                    if (link != null)
                    {
                        r = await m_LinksTable.ExecuteAsync(link);
                    }

                    r = await table.ExecuteAsync(insert);

                    OnUploadSuccess?.Invoke(r.HttpStatusCode, new[] { data }, $"Azure table storage upload to {table.Name} for controller [{id}] succeeded, result = {r.HttpStatusCode}.");
                    if (m_Buffer.Count <= 1)
                    {
                        m_Buffer.Clear();
                    }
                    else
                    {
                        m_Buffer.RemoveAt(0);
                    }
                } catch (StorageException ex) {
                    var status = ex.RequestInformation.HttpStatusCode;
                    var errmsg = ex.RequestInformation.ExtendedErrorInformation?.ErrorMessage ?? ex.RequestInformation.HttpStatusMessage ?? ex.Message;

                    switch (status)
                    {
                    case 0: {
                        OnError?.Invoke(ex, $"Azure table storage upload to {table.Name} for controller [{id}] failed.");
                        break;
                    }

                    case 401:
                    case 403: {
                        OnUploadError?.Invoke(status, new[] { data }, $"Azure table storage upload to {table.Name} for controller [{id}] forbidden: {errmsg}");
                        break;
                    }

                    default: {
                        OnUploadError?.Invoke(status, new[] { data }, $"Azure table storage upload to {table.Name} for controller [{id}] failed: {errmsg}");
                        break;
                    }
                    }
                } catch (Exception ex) {
                    OnError?.Invoke(ex, $"Azure table storage upload to {table.Name} for controller [{id}] failed.");
                }
            }
        }
예제 #26
0
            public async Task ProcessEventsAsync(PartitionContext context, IEnumerable <EventData> messages)
            {
                EventProcInfo[id] = TimeStampThis("Processing");
                var parsedData = messages.Select(eventData => Encoding.UTF8.GetString(eventData.GetBytes())).Select(JsonConvert.DeserializeObject <SystemEvent>).ToList();

                if (Engine.EngineIsRunning)
                {
                    await Engine.AddToMainQueue(parsedData);

                    EventProcInfo[id] = TimeStampThis(parsedData.Count + " events added to engine");
                }
                else
                {
                    EventProcInfo[id] = TimeStampThis("Engine is not running. Cannot add events. Aborting.");
                    return;
                }
                try
                {
                    var        storageMan = new AzureStorageManager(CloudConfigurationManager.GetSetting("AzureStorageConnectionString"));
                    CloudTable Table      = storageMan.GetTableReference(ConfigurationManager.AppSettings["OperationStorageTable"]);

                    var       batches    = new Dictionary <string, TableBatchOperation>();
                    var       batchNames = new Dictionary <string, string>();
                    const int maxOps     = Microsoft.WindowsAzure.Storage.Table.Protocol.TableConstants.TableServiceBatchMaximumOperations;
                    foreach (var operationResult in parsedData)
                    {
                        string batchName;
                        if (!batchNames.TryGetValue(operationResult.PartitionKey, out batchName))
                        {
                            batchName = operationResult.PartitionKey;
                        }
                        TableBatchOperation batchOperation;
                        if (!batches.ContainsKey(batchName))
                        {
                            batchOperation = new TableBatchOperation();
                            batches.Add(batchName, batchOperation);
                        }
                        else
                        {
                            batches.TryGetValue(batchName, out batchOperation);
                        }
                        Debug.Assert(batchOperation != null, "Could not find batchOperation in Dictionary.");

                        if (batchOperation.Count == maxOps)
                        {
                            batchOperation = new TableBatchOperation();
                            batches.Add(GetNewBatchName(operationResult.PartitionKey, batchNames), batchOperation);
                        }
                        batchOperation.Insert(operationResult);
                    }
                    EventProcInfo[id] = TimeStampThis("Running batches");
                    foreach (var batch in batches)
                    {
                        await Table.ExecuteBatchAsync(batch.Value);
                    }
                }
                catch (Exception ex) when(ex.Message.Contains("The specified entity already exists"))
                {
                    Logger.AddRow("Duplicate entry tried to be added to table storage.");
                }
                catch (Exception ex)
                {
                    EventProcInfo[id] = TimeStampThis("!ERROR! " + ex.Message);
                    Logger.AddRow("!ERROR! In event processor");
                    Logger.AddRow(ex.ToString());
                }
                finally
                {
                    EventProcInfo[id] = TimeStampThis("Setting Checkpoint");
                    await context.CheckpointAsync();

                    EventProcInfo[id] = TimeStampThis("Checkpoint set");
                }
            }
예제 #27
0
 public Task <IList <TableResult> > ExecuteBatchAsync(TableBatchOperation batchOperation)
 {
     return(table.ExecuteBatchAsync(batchOperation));
 }
        public static async Task <HttpResponseMessage> PutOrchestratorStatus(
            [HttpTrigger(AuthorizationLevel.Anonymous, "get", "post")] HttpRequestMessage req,
            [Table("OrchestratorStatus", Connection = "StatusTableStorage")] CloudTable status,
            TraceWriter log)
        {
            log.Info("C# HttpTrigger for EventGrid.");

            // parse query parameter
            var requestBody = await req.Content.ReadAsStringAsync();

            log.Info($"Body => {requestBody}");

            if (string.IsNullOrEmpty(requestBody))
            {
                return(req.CreateResponse(HttpStatusCode.OK));
            }

            var messages = JsonConvert.DeserializeObject <JArray>(requestBody);

            //If the request is for subscription validation, send back the validation code.
            if (messages.Count > 0 && string.Equals((string)messages[0]["eventType"],
                                                    "Microsoft.EventGrid.SubscriptionValidationEvent",
                                                    System.StringComparison.OrdinalIgnoreCase))
            {
                log.Info("Validate request received");
                return(req.CreateResponse(HttpStatusCode.OK,
                                          new
                {
                    validationResponse = messages[0]["data"]["validationCode"]
                }));
            }

            var batchOperation = new TableBatchOperation();

            foreach (JObject message in messages)
            {
                log.Info($"EventGridEvent: {message}");

                var data = message["Data"].ToObject <EventGridEventData>();

                switch (data.EventType)
                {
                case (int)OrchestrationRuntimeStatus.Running:
                    batchOperation.InsertOrMerge(new OrchestratorStatus()
                    {
                        PartitionKey = data.InstanceId,
                        RowKey       = Guid.NewGuid().ToString(),
                        HubName      = data.HubName
                    });
                    break;

                case (int)OrchestrationRuntimeStatus.Completed:
                    var query = new TableQuery <OrchestratorStatus>()
                                .Where(TableQuery.GenerateFilterCondition(nameof(OrchestratorStatus.PartitionKey), QueryComparisons.Equal, data.InstanceId));

                    var segment = await status.ExecuteQuerySegmentedAsync(query, null);

                    foreach (var tableEntity in segment)
                    {
                        batchOperation.Delete(tableEntity);
                    }
                    break;

                default:
                    break;
                }
            }

            await status.ExecuteBatchAsync(batchOperation, new TableRequestOptions
            {
                RetryPolicy = new LinearRetry(TimeSpan.FromMilliseconds(10), 5)
            }, null);

            return(req.CreateResponse(HttpStatusCode.OK));
        }
예제 #29
0
        public async Task RemoveFromAllRolesAsync(T user)
        {
            bool   error             = false;
            var    Roles             = new List <UserRoleEntity>();
            string partitionKeyQuery = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, user.Id);
            TableQuery <UserRoleEntity>        query        = new TableQuery <UserRoleEntity>().Where(partitionKeyQuery);
            TableQuerySegment <UserRoleEntity> querySegment = null;

            while (querySegment == null || querySegment.ContinuationToken != null)
            {
                querySegment =
                    await _userRoleTable.ExecuteQuerySegmentedAsync(query, querySegment != null?querySegment.ContinuationToken : null);

                Roles.AddRange(querySegment.Results);
            }

            var batch = new TableBatchOperation();

            foreach (UserRoleEntity role in Roles)
            {
                role.ETag = "*"; //Delete even if it has changed
                batch.Add(TableOperation.Delete(role));
                if (batch.Count >= 100)
                {
                    try
                    {
                        //Try executing as a batch
                        await _userRoleTable.ExecuteBatchAsync(batch);

                        batch.Clear();
                    }
                    catch
                    {
                    }

                    //If a batch wont work, try individually
                    foreach (TableOperation op in batch)
                    {
                        try
                        {
                            await _userRoleTable.ExecuteAsync(op);
                        }
                        catch
                        {
                            error = true;
                        }
                    }

                    batch.Clear();
                }
            }
            if (batch.Count > 0)
            {
                try
                {
                    //Try executing as a batch
                    await _userRoleTable.ExecuteBatchAsync(batch);

                    batch.Clear();
                }
                catch
                {
                }

                //If a batch wont work, try individually
                foreach (TableOperation op in batch)
                {
                    try
                    {
                        await _userRoleTable.ExecuteAsync(op);
                    }
                    catch
                    {
                        error = true;
                    }
                }
            }
            if (error)
            {
                throw new Exception();
            }
        }
        public async Task TableFilterTest()
        {
            // Reinitialize the name resolver to avoid conflicts
            _resolver = new RandomNameResolver();

            JobHostConfiguration hostConfig = new JobHostConfiguration()
            {
                NameResolver = _resolver,
                TypeLocator  = new FakeTypeLocator(
                    this.GetType(),
                    typeof(BlobToCustomObjectBinder))
            };

            hostConfig.AddService <IWebJobsExceptionHandler>(new TestExceptionHandler());

            // write test entities
            string           testTableName = _resolver.ResolveInString(TableName);
            CloudTableClient tableClient   = _storageAccount.CreateCloudTableClient();
            CloudTable       table         = tableClient.GetTableReference(testTableName);
            await table.CreateIfNotExistsAsync();

            var operation = new TableBatchOperation();

            operation.Insert(new Person
            {
                PartitionKey = "1",
                RowKey       = "1",
                Name         = "Lary",
                Age          = 20,
                Location     = "Seattle"
            });
            operation.Insert(new Person
            {
                PartitionKey = "1",
                RowKey       = "2",
                Name         = "Moe",
                Age          = 35,
                Location     = "Seattle"
            });
            operation.Insert(new Person
            {
                PartitionKey = "1",
                RowKey       = "3",
                Name         = "Curly",
                Age          = 45,
                Location     = "Texas"
            });
            operation.Insert(new Person
            {
                PartitionKey = "1",
                RowKey       = "4",
                Name         = "Bill",
                Age          = 28,
                Location     = "Tam O'Shanter"
            });
            await table.ExecuteBatchAsync(operation);

            JobHost host       = new JobHost(hostConfig);
            var     methodInfo = this.GetType().GetMethod("TableWithFilter", BindingFlags.Public | BindingFlags.Static);
            var     input      = new Person {
                Age = 25, Location = "Seattle"
            };
            string json      = JsonConvert.SerializeObject(input);
            var    arguments = new { person = json };
            await host.CallAsync(methodInfo, arguments);

            // wait for test results to appear
            await TestHelpers.Await(() => testResult != null);

            JArray results = (JArray)testResult;

            Assert.Single(results);

            input = new Person {
                Age = 25, Location = "Tam O'Shanter"
            };
            json      = JsonConvert.SerializeObject(input);
            arguments = new { person = json };
            await host.CallAsync(methodInfo, arguments);

            await TestHelpers.Await(() => testResult != null);

            results = (JArray)testResult;
            Assert.Single(results);
            Assert.Equal("Bill", (string)results[0]["Name"]);
        }