/// <summary> /// Delete entities in batches /// </summary> /// <param name="table">instance of <see cref="CloudTable"/></param> /// <param name="segment">instance of <see cref="TableQuerySegment"/></param> /// <returns>List of batch execution tasks</returns> private List <Task> DeleteBatch(CloudTable table, TableQuerySegment <TableEntity> segment) { if (segment.Count() == 0) { return(new List <Task>()); } // sort entities by partition key var list = segment.Results.GroupBy(x => x.PartitionKey); int batchSize = 100; List <Task> tasks = new List <Task>(); // delete in batches by partition key foreach (var group in list) { int batchCount = 0; var items = group.Skip(batchCount * batchSize).Take(batchSize); do { var batchOps = new TableBatchOperation(); foreach (var item in items) { batchOps.Delete(item); } Trace.WriteLine($"deleting {items.Count()} rows"); tasks.Add(table.ExecuteBatchAsync(batchOps)); batchCount++; items = group.Skip(batchCount * batchSize).Take(batchSize); } while (items.Count() > 0); } return(tasks); }
public async Task <IList <string> > GetRolesAsync(TUser user, CancellationToken cancellationToken) { TableQuerySegment Segment = await _db.RoleData.ExecuteQuerySegmentedAsync(new TableQuery().Where($"PartitionKey eq 'UserRole_{user.RowKey}'"), null); //if (Segment.Count() > 0) { IList<string> x = Segment.Select(role => role.RowKey).ToList(); } return(Segment.Count() > 0 ? Segment.Select(role => role.RowKey).ToList() : null); }
public async Task <TUser> FindByEmailAsync(string normalizedEmail, CancellationToken cancellationToken) { TableQuerySegment <TUser> Segment = await _db.UserData.ExecuteQuerySegmentedAsync(new TableQuery <TUser>().Where($"Email eq '{normalizedEmail}'"), null); if (Segment.Count() > 0) { return(Segment.FirstOrDefault()); } return(null); }
public async Task <AzureTableUser> FindByEmailAsync(string email) { AzureTableUser User = null; TableQuerySegment <UserDataEntity> Segment = await UserTable.ExecuteQuerySegmentedAsync(new TableQuery <UserDataEntity>().Where($"Email eq '{email}'").Take(1), null); if (Segment.Count() > 0) { User = Segment.FirstOrDefault().ToUser(); } return(User); }
public async Task <long> CountAsync(TableQuery query, CancellationToken cancelationToken = default(CancellationToken)) { TableContinuationToken token = null; var oc = new OperationContext(); var ro = new TableRequestOptions(); long result = 0; do { TableQuerySegment seg = await Table.ExecuteQuerySegmentedAsync(query, token, ro, oc, cancelationToken); result += seg.Count(); token = seg.ContinuationToken; } while (token != null && !cancelationToken.IsCancellationRequested); return(result); }
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken) { try { if (!_tableResult.Any()) { return(null); } if (_currentReadRow >= _tableResult.Count()) { if (_token == null) { return(null); } _tableResult = await _tableReference.ExecuteQuerySegmentedAsync(_tableQuery, _token); if (!_tableResult.Any()) { return(null); } _token = _tableResult.ContinuationToken; _currentReadRow = 0; } var currentEntity = _tableResult.ElementAt(_currentReadRow); var row = GetRow(currentEntity); _currentReadRow++; return(row); } catch (Exception ex) { throw new ConnectionException("The azure storage table reader failed due to the following error: " + ex.Message, ex); } }
public async Task <IList <Claim> > GetClaimsAsync(TUser user, CancellationToken cancellationToken) { TableQuerySegment <TUserClaim> Segment = await _db.UserClaimData.ExecuteQuerySegmentedAsync(new TableQuery <TUserClaim>().Where($"PartitionKey eq '{user.Id}'"), null); return(Segment.Count() > 0 ? Segment.Select(uc => uc.ToClaim()).ToList() : null); }
public async Task <IList <UserLoginInfo> > GetLoginsAsync(TUser user, CancellationToken cancellationToken) { TableQuerySegment <TUserLogin> Segment = await _db.UserLoginData.ExecuteQuerySegmentedAsync(new TableQuery <TUserLogin>().Where($"UserId eq '{user.Id}'"), null); return(Segment.Count() > 0 ? Segment.Select(ul => new UserLoginInfo(ul.LoginProvider, ul.ProviderKey, ul.ProviderDisplayName)).ToList() : null); }
public async Task <IList <TUser> > GetUsersInRoleAsync(string roleName, CancellationToken cancellationToken) { TableQuerySegment <TUser> Segment = await _db.RoleData.ExecuteQuerySegmentedAsync(new TableQuery <TUser>().Where($"PartitionKey eq 'UserRole_{roleName}'"), null); return(Segment.Count() > 0 ? (IList <TUser>)Segment : null); }
protected void LogStartingToProcessPage(TableQuerySegment <DynamicTableEntity> page, DateTime firstResultTimestamp) { Logger.Information($"Got {page.Count()} results starting at timestamp {firstResultTimestamp}"); }
public static async Task <HttpResponseMessage> HttpStart( [HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequestMessage req, [Table("RunningInstances")] CloudTable tableInput, [Table("RunningInstances")] IAsyncCollector <InstanceTableEntity> tableOutput, [OrchestrationClient] DurableOrchestrationClient starter, ILogger log) { string requestBody = await req.Content.ReadAsStringAsync(); try { Update update = JsonConvert.DeserializeObject <Update>(requestBody); string chatId = update.GetChatId(); TableQuery <InstanceTableEntity> sessionQuery = new TableQuery <InstanceTableEntity>().Where( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, chatId)); TableQuerySegment <InstanceTableEntity> sessions = await tableInput.ExecuteQuerySegmentedAsync(sessionQuery, null); if ((update.Message?.Entities?.Length ?? 0) != 0 && (update.Message?.Entities?.First().Type ?? MessageEntityType.Unknown) == MessageEntityType.BotCommand && (update.Message?.EntityValues?.First()?.ToLowerInvariant().Equals("/cancel") ?? false)) { //Cancel current user session and reset all state! foreach (var entity in sessions) { await tableInput.ExecuteAsync(TableOperation.Delete(entity)); } await botClient.SendTextMessageAsync( chatId : chatId, text : $"All running ({sessions.Count()}) sessions have now been purged"); } else { bool didContinueASession = false; if (sessions.Count() > 0) { foreach (InstanceTableEntity session in sessions) { OrchestrationRuntimeStatus status = (await starter.GetStatusAsync(session.RowKey)).RuntimeStatus; if (status == OrchestrationRuntimeStatus.Failed || status == OrchestrationRuntimeStatus.Canceled || status == OrchestrationRuntimeStatus.Completed || status == OrchestrationRuntimeStatus.Terminated || status == OrchestrationRuntimeStatus.Unknown) { log.LogInformation($"Removing session with ID = '{session.RowKey}' because it was terminated"); await tableInput.ExecuteAsync(TableOperation.Delete(session)); await botClient.SendTextMessageAsync( chatId : chatId, text : $"FYI: I purged a old session with ID {session.RowKey} that had status: {status.ToString()} (Was started at UTC: {session.Timestamp.UtcDateTime.ToShortDateString()})"); } else { if (update.CallbackQuery != null) { //One should not have more than 1 running session, but I will not crash because of it. So just continue all the session until they get cleaned up. await starter.RaiseEventAsync(session.RowKey, "Callback", update); log.LogInformation($"Continuing on session with ID = '{session.RowKey}'"); didContinueASession = true; } } } } if (!didContinueASession) { var newInstance = new InstanceTableEntity { PartitionKey = chatId, RowKey = await starter.StartNewAsync("BotBrainFunction", update) }; await tableOutput.AddAsync(newInstance); log.LogInformation($"Started orchestration with ID = '{newInstance.RowKey}'."); } } } catch (Exception ex) { log.LogError(ex, $"Could not parse update data. {ex.Message}"); log.LogInformation(requestBody); } // Function input comes from the request content. //string instanceId = await starter.StartNewAsync("Function2", null); //log.LogInformation($"Started orchestration with ID = '{instanceId}'."); //return starter.CreateCheckStatusResponse(req, instanceId); HttpResponseMessage response = req.CreateResponse(HttpStatusCode.OK); //await Task.Delay(1); return(response); }
public static async Task <IActionResult> Run( [HttpTrigger(AuthorizationLevel.Function, "get", Route = FunctionName.ScrapeProduct + "/{userId}/{productId}")] HttpRequest req, [Table(TableName.ProductInfo, "{userId}")] CloudTable productInfoTable, [Table(TableName.ScrapeConfig)] CloudTable scrapeConfigTable, [Queue(QueueName.AddProductHistory)] IAsyncCollector <ProductInfo> addProductHistoryMessageQueue, [Queue(QueueName.EmailsToSend)] IAsyncCollector <SendGridMessage> emailMessageQueue, string userId, string productId, ILogger log) { log.LogInformation($"Request to scrape product {productId}"); TableQuery <ProductInfo> productQuery = new TableQuery <ProductInfo>().Where( TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, productId)); TableQuerySegment <ProductInfo> products = await productInfoTable.ExecuteQuerySegmentedAsync(productQuery, null); if (products != null && products.Count() == 1) { //Find matching criteria ProductInfo product = products.First(); TableQuery <ScrapeConfig> configQuery = new TableQuery <ScrapeConfig>().Where( TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, product.URL.ToCoreUrl())); TableQuerySegment <ScrapeConfig> configs = await scrapeConfigTable.ExecuteQuerySegmentedAsync(configQuery, null); if (!configs.Any()) { //Notify the admin SendGridMessage message = new SendGridMessage(); message.SetFrom(new EmailAddress("*****@*****.**", "Product scraper")); message.AddTo(CommonName.AdminEmail); message.SetSubject("Missing configuration"); message.AddContent("text/plain", $"There was request for scraping products from {product.URL.ToCoreUrl()}, consider adding configuration soon. Product Url {product.URL}"); await emailMessageQueue.AddAsync(message); } else if (configs.Count() > 1) { //Notify the admin SendGridMessage message = new SendGridMessage(); message.SetFrom(new EmailAddress("*****@*****.**", "Product scraper")); message.AddTo(CommonName.AdminEmail); message.SetSubject("Multiple configurations"); message.AddContent("text/plain", $"There are more than one configuration for: {product.URL.ToCoreUrl()}, consider deleting one."); await emailMessageQueue.AddAsync(message); } else if (configs.Count() == 1) { Utils utils = new Utils(); await utils.Scrape(configs.First(), product, log); //Update product in db TableOperation operation = TableOperation.InsertOrReplace(product); await productInfoTable.ExecuteAsync(operation); //Add to history queue await addProductHistoryMessageQueue.AddAsync(product); } } else { log.LogInformation($"Multiple products matches the criteria userId={userId} productId={productId}"); } string responseMessage = string.IsNullOrEmpty(productId) ? "Please provide productId in the path" : $"Product {productId} has been scraped"; return(new OkObjectResult(responseMessage)); }