/// <summary>
/// Delete entities in batches
/// </summary>
/// <param name="table">instance of <see cref="CloudTable"/></param>
/// <param name="segment">instance of <see cref="TableQuerySegment"/></param>
/// <returns>List of batch execution tasks</returns>
private List <Task> DeleteBatch(CloudTable table, TableQuerySegment <TableEntity> segment)
{
if (segment.Count() == 0)
{
return(new List <Task>());
}
// sort entities by partition key
var list = segment.Results.GroupBy(x => x.PartitionKey);
int batchSize = 100;
List <Task> tasks = new List <Task>();
// delete in batches by partition key
foreach (var group in list)
{
int batchCount = 0;
var items = group.Skip(batchCount * batchSize).Take(batchSize);
do
{
var batchOps = new TableBatchOperation();
foreach (var item in items)
{
batchOps.Delete(item);
}
Trace.WriteLine($"deleting {items.Count()} rows");
tasks.Add(table.ExecuteBatchAsync(batchOps));
batchCount++;
items = group.Skip(batchCount * batchSize).Take(batchSize);
} while (items.Count() > 0);
}
return(tasks);
}
public async Task <IList <string> > GetRolesAsync(TUser user, CancellationToken cancellationToken)
{
TableQuerySegment Segment = await _db.RoleData.ExecuteQuerySegmentedAsync(new TableQuery().Where($"PartitionKey eq 'UserRole_{user.RowKey}'"), null);
//if (Segment.Count() > 0) { IList<string> x = Segment.Select(role => role.RowKey).ToList(); }
return(Segment.Count() > 0 ? Segment.Select(role => role.RowKey).ToList() : null);
}
public async Task <TUser> FindByEmailAsync(string normalizedEmail, CancellationToken cancellationToken)
{
TableQuerySegment <TUser> Segment = await _db.UserData.ExecuteQuerySegmentedAsync(new TableQuery <TUser>().Where($"Email eq '{normalizedEmail}'"), null);
if (Segment.Count() > 0)
{
return(Segment.FirstOrDefault());
}
return(null);
}
public async Task <AzureTableUser> FindByEmailAsync(string email)
{
AzureTableUser User = null;
TableQuerySegment <UserDataEntity> Segment = await UserTable.ExecuteQuerySegmentedAsync(new TableQuery <UserDataEntity>().Where($"Email eq '{email}'").Take(1), null);
if (Segment.Count() > 0)
{
User = Segment.FirstOrDefault().ToUser();
}
return(User);
}
public async Task <long> CountAsync(TableQuery query, CancellationToken cancelationToken = default(CancellationToken))
{
TableContinuationToken token = null;
var oc = new OperationContext();
var ro = new TableRequestOptions();
long result = 0;
do
{
TableQuerySegment seg = await Table.ExecuteQuerySegmentedAsync(query, token, ro, oc, cancelationToken);
result += seg.Count();
token = seg.ContinuationToken;
} while (token != null && !cancelationToken.IsCancellationRequested);
return(result);
}
protected override async Task <object[]> ReadRecord(CancellationToken cancellationToken)
{
try
{
if (!_tableResult.Any())
{
return(null);
}
if (_currentReadRow >= _tableResult.Count())
{
if (_token == null)
{
return(null);
}
_tableResult = await _tableReference.ExecuteQuerySegmentedAsync(_tableQuery, _token);
if (!_tableResult.Any())
{
return(null);
}
_token = _tableResult.ContinuationToken;
_currentReadRow = 0;
}
var currentEntity = _tableResult.ElementAt(_currentReadRow);
var row = GetRow(currentEntity);
_currentReadRow++;
return(row);
}
catch (Exception ex)
{
throw new ConnectionException("The azure storage table reader failed due to the following error: " + ex.Message, ex);
}
}
public async Task <IList <Claim> > GetClaimsAsync(TUser user, CancellationToken cancellationToken)
{
TableQuerySegment <TUserClaim> Segment = await _db.UserClaimData.ExecuteQuerySegmentedAsync(new TableQuery <TUserClaim>().Where($"PartitionKey eq '{user.Id}'"), null);
return(Segment.Count() > 0 ? Segment.Select(uc => uc.ToClaim()).ToList() : null);
}
public async Task <IList <UserLoginInfo> > GetLoginsAsync(TUser user, CancellationToken cancellationToken)
{
TableQuerySegment <TUserLogin> Segment = await _db.UserLoginData.ExecuteQuerySegmentedAsync(new TableQuery <TUserLogin>().Where($"UserId eq '{user.Id}'"), null);
return(Segment.Count() > 0 ? Segment.Select(ul => new UserLoginInfo(ul.LoginProvider, ul.ProviderKey, ul.ProviderDisplayName)).ToList() : null);
}
public async Task <IList <TUser> > GetUsersInRoleAsync(string roleName, CancellationToken cancellationToken)
{
TableQuerySegment <TUser> Segment = await _db.RoleData.ExecuteQuerySegmentedAsync(new TableQuery <TUser>().Where($"PartitionKey eq 'UserRole_{roleName}'"), null);
return(Segment.Count() > 0 ? (IList <TUser>)Segment : null);
}
protected void LogStartingToProcessPage(TableQuerySegment <DynamicTableEntity> page, DateTime firstResultTimestamp)
{
Logger.Information($"Got {page.Count()} results starting at timestamp {firstResultTimestamp}");
}
public static async Task <HttpResponseMessage> HttpStart(
[HttpTrigger(AuthorizationLevel.Function, "post", Route = null)] HttpRequestMessage req,
[Table("RunningInstances")] CloudTable tableInput,
[Table("RunningInstances")] IAsyncCollector <InstanceTableEntity> tableOutput,
[OrchestrationClient] DurableOrchestrationClient starter,
ILogger log)
{
string requestBody = await req.Content.ReadAsStringAsync();
try
{
Update update = JsonConvert.DeserializeObject <Update>(requestBody);
string chatId = update.GetChatId();
TableQuery <InstanceTableEntity> sessionQuery = new TableQuery <InstanceTableEntity>().Where(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, chatId));
TableQuerySegment <InstanceTableEntity> sessions = await tableInput.ExecuteQuerySegmentedAsync(sessionQuery, null);
if ((update.Message?.Entities?.Length ?? 0) != 0 &&
(update.Message?.Entities?.First().Type ?? MessageEntityType.Unknown) == MessageEntityType.BotCommand &&
(update.Message?.EntityValues?.First()?.ToLowerInvariant().Equals("/cancel") ?? false))
{
//Cancel current user session and reset all state!
foreach (var entity in sessions)
{
await tableInput.ExecuteAsync(TableOperation.Delete(entity));
}
await botClient.SendTextMessageAsync(
chatId : chatId,
text : $"All running ({sessions.Count()}) sessions have now been purged");
}
else
{
bool didContinueASession = false;
if (sessions.Count() > 0)
{
foreach (InstanceTableEntity session in sessions)
{
OrchestrationRuntimeStatus status = (await starter.GetStatusAsync(session.RowKey)).RuntimeStatus;
if (status == OrchestrationRuntimeStatus.Failed ||
status == OrchestrationRuntimeStatus.Canceled ||
status == OrchestrationRuntimeStatus.Completed ||
status == OrchestrationRuntimeStatus.Terminated ||
status == OrchestrationRuntimeStatus.Unknown)
{
log.LogInformation($"Removing session with ID = '{session.RowKey}' because it was terminated");
await tableInput.ExecuteAsync(TableOperation.Delete(session));
await botClient.SendTextMessageAsync(
chatId : chatId,
text : $"FYI: I purged a old session with ID {session.RowKey} that had status: {status.ToString()} (Was started at UTC: {session.Timestamp.UtcDateTime.ToShortDateString()})");
}
else
{
if (update.CallbackQuery != null)
{
//One should not have more than 1 running session, but I will not crash because of it. So just continue all the session until they get cleaned up.
await starter.RaiseEventAsync(session.RowKey, "Callback", update);
log.LogInformation($"Continuing on session with ID = '{session.RowKey}'");
didContinueASession = true;
}
}
}
}
if (!didContinueASession)
{
var newInstance = new InstanceTableEntity
{
PartitionKey = chatId,
RowKey = await starter.StartNewAsync("BotBrainFunction", update)
};
await tableOutput.AddAsync(newInstance);
log.LogInformation($"Started orchestration with ID = '{newInstance.RowKey}'.");
}
}
}
catch (Exception ex)
{
log.LogError(ex, $"Could not parse update data. {ex.Message}");
log.LogInformation(requestBody);
}
// Function input comes from the request content.
//string instanceId = await starter.StartNewAsync("Function2", null);
//log.LogInformation($"Started orchestration with ID = '{instanceId}'.");
//return starter.CreateCheckStatusResponse(req, instanceId);
HttpResponseMessage response = req.CreateResponse(HttpStatusCode.OK);
//await Task.Delay(1);
return(response);
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Function, "get", Route = FunctionName.ScrapeProduct + "/{userId}/{productId}")] HttpRequest req,
[Table(TableName.ProductInfo, "{userId}")] CloudTable productInfoTable,
[Table(TableName.ScrapeConfig)] CloudTable scrapeConfigTable,
[Queue(QueueName.AddProductHistory)] IAsyncCollector <ProductInfo> addProductHistoryMessageQueue,
[Queue(QueueName.EmailsToSend)] IAsyncCollector <SendGridMessage> emailMessageQueue,
string userId,
string productId,
ILogger log)
{
log.LogInformation($"Request to scrape product {productId}");
TableQuery <ProductInfo> productQuery = new TableQuery <ProductInfo>().Where(
TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.Equal, productId));
TableQuerySegment <ProductInfo> products = await productInfoTable.ExecuteQuerySegmentedAsync(productQuery, null);
if (products != null && products.Count() == 1)
{
//Find matching criteria
ProductInfo product = products.First();
TableQuery <ScrapeConfig> configQuery = new TableQuery <ScrapeConfig>().Where(
TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, product.URL.ToCoreUrl()));
TableQuerySegment <ScrapeConfig> configs = await scrapeConfigTable.ExecuteQuerySegmentedAsync(configQuery, null);
if (!configs.Any())
{
//Notify the admin
SendGridMessage message = new SendGridMessage();
message.SetFrom(new EmailAddress("*****@*****.**", "Product scraper"));
message.AddTo(CommonName.AdminEmail);
message.SetSubject("Missing configuration");
message.AddContent("text/plain", $"There was request for scraping products from {product.URL.ToCoreUrl()}, consider adding configuration soon. Product Url {product.URL}");
await emailMessageQueue.AddAsync(message);
}
else if (configs.Count() > 1)
{
//Notify the admin
SendGridMessage message = new SendGridMessage();
message.SetFrom(new EmailAddress("*****@*****.**", "Product scraper"));
message.AddTo(CommonName.AdminEmail);
message.SetSubject("Multiple configurations");
message.AddContent("text/plain", $"There are more than one configuration for: {product.URL.ToCoreUrl()}, consider deleting one.");
await emailMessageQueue.AddAsync(message);
}
else if (configs.Count() == 1)
{
Utils utils = new Utils();
await utils.Scrape(configs.First(), product, log);
//Update product in db
TableOperation operation = TableOperation.InsertOrReplace(product);
await productInfoTable.ExecuteAsync(operation);
//Add to history queue
await addProductHistoryMessageQueue.AddAsync(product);
}
}
else
{
log.LogInformation($"Multiple products matches the criteria userId={userId} productId={productId}");
}
string responseMessage = string.IsNullOrEmpty(productId)
? "Please provide productId in the path" : $"Product {productId} has been scraped";
return(new OkObjectResult(responseMessage));
}