private async Task CreateRatingsTableIfNotExists()
{
await ratingsTable.CreateIfNotExistsAsync();
}
public Task CreateTableIfNotExistsAsync(CancellationToken token)
{
return(_table.CreateIfNotExistsAsync(token));
}
public Task CreateIfNotExistsAsync()
{
return(_table.CreateIfNotExistsAsync());
}
public async Task EnsureExistsAsync()
{
await table.CreateIfNotExistsAsync();
}
public async Task Open(IWorkContext context)
{
_client = _account.CreateCloudTableClient();
_table = _client.GetTableReference(_tableName);
await _table.CreateIfNotExistsAsync();
}
public static async Task <IActionResult> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "post", Route = "slack")] HttpRequest req,
[Table(
TableConstants.UserTable,
Connection = Constants.StorageConnection
)] CloudTable userTable,
ILogger log)
{
await userTable.CreateIfNotExistsAsync();
var bodyString = await req.ReadAsStringAsync();
var slackPayload = HttpUtility.ParseQueryString(bodyString);
var token = slackPayload["token"];
if (!string.Equals(token, Config.SlackToken))
{
throw new System.Exception("Not allowed");
}
var args = slackPayload["text"]?.Trim();
if (string.Equals("list", args))
{
var repo = new UserRepository(userTable);
var entities = await repo.GetUsersAsync();
var nonHiddenUserIds = entities
.Where(e => e.Active)
.Select(e => e.RowKey)
.ToList();
log.LogInformation("Found these users: {Users}", string.Join(", ", nonHiddenUserIds));
var listenerTracks = await SpotifyHelper.GetListenerTracks(nonHiddenUserIds, log);
return(new JsonResult(new
{
blocks = new
{
type = "section",
block_id = "listeners",
fields = listenerTracks.Select(lt => new { type = "mrkdwn", text = $"- {lt.userId}: {lt.currentTrack}" }).ToArray <object>()
}
}));
}
return(new JsonResult(new
{
blocks = new object[] {
new {
type = "section",
block_id = "help_text_commands",
fields = new [] {
new {
type = "mrkdwn",
text = "*Hey! Welcome to now playing* \n Press the button below to go to the website, or register with Now Playing:"
}
}
},
new { type = "actions", elements = new object[] {
new {
type = "button",
action_id = "dashboard_nowplaying",
url = Config.HostUrl,
text = new {
type = "plain_text",
text = "Dashboard website",
}
},
new {
type = "button",
action_id = "register_nowplaying",
url = $"{Config.HostUrl}api/authorize",
text = new {
type = "plain_text",
text = "Register with Now Playing",
}
}
} }
}
}));
}
public ActivityAzureLogger(CloudStorageAccount account)
{
tableClient = account.CreateCloudTableClient();
activityTable = tableClient.GetTableReference("ClientActivity");
activityTable.CreateIfNotExistsAsync().Wait();
}
public async void TableStorageTest()
{
ClearOutput();
WriteLine("-- Testing Table Storage --");
WriteLine("0. Creating table client");
CloudTableClient tableClient = StorageAccount.CreateCloudTableClient();
WriteLine("1. Create a Table for the demo");
// Create a table client for interacting with the table service
CloudTable table = tableClient.GetTableReference(TableName);
try
{
if (await table.CreateIfNotExistsAsync())
{
WriteLine(string.Format("Created Table named: {0}", TableName));
}
else
{
WriteLine(string.Format("Table {0} already exists", TableName));
}
}
catch (StorageException)
{
WriteLine("If you are running with the default configuration please make sure you have started the storage emulator. Press the Windows key and type Azure Storage to select and run it from the list of applications - then restart the sample.");
throw;
}
// Create an instance of a customer entity. See the Model\CustomerEntity.cs for a description of the entity.
CustomerEntity customer = new CustomerEntity("Harp", "Walter")
{
Email = "*****@*****.**",
PhoneNumber = "425-555-0101"
};
// Demonstrate how to Update the entity by changing the phone number
WriteLine("2. Update an existing Entity using the InsertOrMerge Operation.");
customer.PhoneNumber = "425-555-0105";
await InsertOrMergeEntityAsync(table, customer);
// Demonstrate how to Read the updated entity using a point query
WriteLine("3. Reading the updated Entity.");
customer = await RetrieveEntityUsingPointQueryAsync(table, "Harp", "Walter");
// Demonstrate how to Delete an entity
WriteLine("4. Delete the entity.");
await DeleteEntityAsync(table, customer);
// Demonstrate upsert and batch table operations
WriteLine("5. Inserting a batch of entities.");
await BatchInsertOfCustomerEntitiesAsync(table);
// Query a range of data within a partition
WriteLine("6. Retrieving entities with surname of Smith and first names >= 1 and <= 5");
await PartitionRangeQueryAsync(table, "Smith", "0001", "0005");
// Query for all the data within a partition
WriteLine("7. Retrieve entities with surname of Smith.");
await PartitionScanAsync(table, "Smith");
WriteLine("-- Test Complete --");
}
public async Task Initialize(Logger logger)
{
if (table == null)
{
await _syncLock.WaitAsync();
try
{
if (table == null)
{
// Initialize the table
var storageConnectionString = this.storageConnectionString;
var storageAccount = CloudStorageAccount.Parse(storageConnectionString);
// var storageAccount = CloudStorageAccount.DevelopmentStorageAccount;
var tableClient = storageAccount.CreateCloudTableClient();
table = tableClient.GetTableReference(tableName);
await table.CreateIfNotExistsAsync();
try
{
// And initialize a lock-free lookup for our action types, for deserialization
var runtimeId = RuntimeEnvironment.GetRuntimeIdentifier();
IEnumerable <TypeInfo> actionTypes =
DependencyContext.Default.GetRuntimeAssemblyNames(runtimeId)
.Where(a => !a.FullName.StartsWith("Microsoft") && !a.FullName.StartsWith("System") && !a.FullName.StartsWith("xunit"))
.Select(name => {
try
{
return(Assembly.Load(name));
}
catch (Exception e)
{
if (logger != null)
{
logger.Log(1004, Severity.Warning, $"Warn: error loading assembly '{name}'", null, e);
}
return(null);
}
})
.Where(assembly => assembly != null)
.SelectMany(a =>
{
try
{
return(a.GetTypes());
}
catch (Exception e)
{
if (logger != null)
{
logger.Log(1004, Severity.Warning, $"Warn: error getting types from assembly '{a.FullName}'", null, e);
}
return(Enumerable.Empty <Type>());
}
})
.Where(t => t.GetTypeInfo().IsPublic)
.Where(t => t.GetInterfaces().Contains(typeof(IAction)))
.Select(t => t.GetTypeInfo())
.GroupBy(ti => ti.Name)
.Select(g => g.First());
actionTypeLookup = actionTypes.ToDictionary(t => t.Name).ToImmutableDictionary();
}
catch (Exception e)
{
throw new Exception("Reflection failed! ");
}
//var test = AppDomain.CurrentDomain.GetAssemblies()
// .Where(a => !a.FullName.StartsWith("Microsoft") && !a.FullName.StartsWith("System") && !a.FullName.StartsWith("xunit"))
// .SelectMany(a => a.GetTypes())
// .Where(t => t.GetInterfaces().Contains(typeof(IAction)))
// .Select(t => t.GetTypeInfo())
// .OrderBy(ti => ti.Name)
// .GroupBy(ti => ti.Assembly.FullName).ToList();
}
}
catch (Exception e)
{
logger.Error(104, e.Message, e);
throw;
}
finally
{
_syncLock.Release();
}
}
}
public async Task <bool> CreateIfNotExistsAsync()
{
return(await table.CreateIfNotExistsAsync());
}
public TableRepository(CloudStorageAccount storageAccount)
{
_client = storageAccount.CreateCloudTableClient();
_table = _client.GetTableReference(typeof(T).Name);
_table.CreateIfNotExistsAsync().Wait();
}
internal static async Task ProcessResults()
{
//CurrentBenchmarkFileId helyett: VerifierBenchmark.ToString()
var filename = $"Result_{CurrentBenchmarkId}.xlsx";
var fullfilename = Path.Combine(OutputDirectory.ToString(), filename);
if (Program.Offline)
{
Console.WriteLine($"{DateTime.Now}: Writing results to disk...");
CurrentBenchmark.Dump(fullfilename, CurrentBenchmark.Parameters);
}
else
{
//TODO: szinten excelezni json results helyett
Console.WriteLine($"{DateTime.Now}: Writing results to disk...");
SerializationHelper.JsonSerializeToFile <BenchmarkResults>(CurrentResults, fullfilename);
Console.WriteLine($"{DateTime.Now}: Uploading results to Azure Blob store...");
var blob = Container.GetBlockBlobReference($"Results/{filename}");
await blob.DeleteIfExistsAsync();
await blob.UploadFromFileAsync(fullfilename);
Console.WriteLine($"{DateTime.Now}: Writing results to Azure Table store...");
CloudTableClient tableClient = Program.Account.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference(Program.Experiment + "FinalResults");
await table.CreateIfNotExistsAsync();
var finalResultEntity = new DynamicTableEntity();
finalResultEntity.PartitionKey = filename;
finalResultEntity.RowKey = CurrentResultType;
finalResultEntity.Properties.Add("Machine", EntityProperty.CreateEntityPropertyFromObject(System.Environment.MachineName));
finalResultEntity.Properties.Add("Frr", EntityProperty.CreateEntityPropertyFromObject(CurrentResults.FinalResult.Frr));
finalResultEntity.Properties.Add("Far", EntityProperty.CreateEntityPropertyFromObject(CurrentResults.FinalResult.Far));
finalResultEntity.Properties.Add("Aer", EntityProperty.CreateEntityPropertyFromObject(CurrentResults.FinalResult.Aer));
var tableOperation = TableOperation.InsertOrReplace(finalResultEntity);
await table.ExecuteAsync(tableOperation);
table = tableClient.GetTableReference(Program.Experiment + "SignerResults");
await table.CreateIfNotExistsAsync();
var signerResultEntity = new DynamicTableEntity();
signerResultEntity.PartitionKey = filename;
foreach (var signerResult in CurrentResults.SignerResults)
{
signerResultEntity.RowKey = signerResult.Signer;
signerResultEntity.Properties["Frr"] = EntityProperty.CreateEntityPropertyFromObject(signerResult.Frr);
signerResultEntity.Properties["Far"] = EntityProperty.CreateEntityPropertyFromObject(signerResult.Far);
signerResultEntity.Properties["Aer"] = EntityProperty.CreateEntityPropertyFromObject(signerResult.Aer);
tableOperation = TableOperation.InsertOrReplace(signerResultEntity);
await table.ExecuteAsync(tableOperation);
}
Console.WriteLine($"{DateTime.Now}: Writing results to MongoDB...");
var document = BsonSerializer.Deserialize <BsonDocument>(SerializationHelper.JsonSerialize <BenchmarkResults>(CurrentResults));
document.Add("Benchmark", filename);
document.Add("Machine", System.Environment.MachineName);
document.Add("Time", DateTime.Now);
document.Add("ResultType", CurrentResultType);
var client = new MongoClient("mongodb+srv://sigstat:[email protected]/test?retryWrites=true");
var database = client.GetDatabase("benchmarks");
var collection = database.GetCollection <BsonDocument>("results");
await collection.InsertOneAsync(document);
}
}
/// <summary>
/// Method for Crawling and parsing Html page
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
private async Task Crawl(string url)
{
if (string.IsNullOrEmpty(url) || string.IsNullOrWhiteSpace(url)) // if url is empty, exit immediately
{
return;
}
WebsitePage MainWebsitePageObject = new WebsitePage(); // instantiate Main Website Page Object
currentWorkingUrl = url;
try
{
htmlDoc = webGet.Load(url); // download html page
}
catch (WebException)
{
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
try
{
htmlDoc = webGet.Load(url);
}
catch (Exception ex)
{
MainWebsitePageObject.Url = url;
MainWebsitePageObject.ErrorTag = "REQUEST ERROR";
MainWebsitePageObject.PartitionKey = "REQUEST ERROR";
MainWebsitePageObject.RowKey = Generate256HashCode(url);
MainWebsitePageObject.Title = "REQUEST ERROR";
MainWebsitePageObject.ErrorDetails = ex.Message;
await PushErrorPageObject(MainWebsitePageObject);
// exit method immediately
return;
}
}
// PARSE HTML FOR CONTENT
uri = new Uri(url); // get domain name
string urlDomain = domainParser.Get(uri.Authority).Domain; // get domain name
// parse page title
string title = "Title";
HtmlNode titleNode;
titleNode = htmlDoc.DocumentNode.SelectSingleNode("//title");
if (titleNode != null && !string.IsNullOrEmpty(titleNode.InnerText))
{
title = HtmlEntity.DeEntitize(titleNode.InnerText.Trim());
}
else
{
titleNode = htmlDoc.DocumentNode.SelectSingleNode("//h1");
if (titleNode != null && !string.IsNullOrEmpty(titleNode.InnerText))
{
title = HtmlEntity.DeEntitize(titleNode.InnerText.Trim());
}
}
MainWebsitePageObject.Title = title;
MainWebsitePageObject.Url = url;
MainWebsitePageObject.PartitionKey = urlDomain;
MainWebsitePageObject.RowKey = Generate256HashCode(url);
// parse body content
string content = "Content Preview Not Available";
try
{
IEnumerable <string> words = htmlDoc.DocumentNode?
.SelectNodes("//body//p//text()")?
.Select(x => HtmlEntity.DeEntitize(x.InnerText.Trim()))
.Where(x => !string.IsNullOrWhiteSpace(x))
.Take(255);
content = string.Join(" ", words.Where(x => !string.IsNullOrWhiteSpace(x)));
string[] contentArray = content.Split(' ');
content = string.Empty;
foreach (string c in contentArray.Where(x => !string.IsNullOrWhiteSpace(x)))
{
content += c;
content += " ";
}
content = content.TrimEnd(' ');
}
catch (Exception)
{ /* retain default value content = "Content"*/ }
MainWebsitePageObject.Content = content;
DateTime current = DateTime.UtcNow;
DateTime?publishDate; // check publish date (in meta data tags if available)
try
{
// get the publish stored in meta data eg. (cnn.com, espn.com)
string pubDate = htmlDoc.DocumentNode
.SelectSingleNode("//meta[@name='pubdate']")
.GetAttributeValue("content", string.Empty);
publishDate = Convert.ToDateTime(pubDate);
if (!(publishDate.Value.Year == current.Year && (current.Month - publishDate.Value.Month <= 3)))
{
Trace.TraceInformation("Article is too old: " + url); // article is too old, it will not be added to the database
return;
}
}
catch (Exception)
{
publishDate = null;
}
MainWebsitePageObject.PublishDate = publishDate;
if (urlDomain.Contains("bleacherreport")) // check if site domain is 'bleacherreport'
{
string keywords = htmlDoc.DocumentNode
.SelectSingleNode("//meta[@name='keywords']")
.GetAttributeValue("content", string.Empty);
if (!keywords.ToLower().Contains("nba")) // check if page is not about nba
{
return; // exit method immediately this will not be added to the database
}
}
// check for any errors
string errorTag = string.Empty;
string errorDetails = string.Empty;
if (htmlDoc.ParseErrors.Any())
{
errorTag = "HTML PARSING ERROR"; // set ErrorTag
string errors = string.Empty;
foreach (var error in htmlDoc.ParseErrors)
{
errors += error.Reason + ";";
}
MainWebsitePageObject.ErrorTag = errorTag;
MainWebsitePageObject.ErrorDetails = errorDetails;
await PushErrorPageObject(MainWebsitePageObject); // push to ErrorTable
}
// if all tests passed,
if (!domainDictionary.ContainsKey(urlDomain)) // add domain name to the 'domain table', and 'domainDictionary'
{
domainDictionary.Add(urlDomain, urlDomain);
DomainObject domainObject = new DomainObject(urlDomain);
TableOperation insertOrMerge = TableOperation.InsertOrMerge(domainObject); // Table operation insertOrMerge
await domainTable.ExecuteAsync(insertOrMerge);
}
CloudTable table = tableClient.GetTableReference(urlDomain); // create a table using url domain as a name
await table.CreateIfNotExistsAsync(); // create if does not exist
var titleKeywords = GetValidKeywords(title);
foreach (string word in titleKeywords) // create website page object, use keywords as partition key and add to container
{
WebsitePage page = new WebsitePage(word, url) // keyword = partition key, url = rowkey,(hashed)
{
Title = title,
};
container.Add(page); // add page to the container
}
bool exists = await Exists(websitePageMasterTable, MainWebsitePageObject.PartitionKey, MainWebsitePageObject.RowKey);
if (!exists) // push if does not exists
{
TableOperation insertOperation = TableOperation.InsertOrMerge(MainWebsitePageObject); // insert main website page object to the database
await websitePageMasterTable.ExecuteAsync(insertOperation); // try insert to the database
await UpdateIndexedCount();
}
await BatchInsertToDatabase(minimumWebSiteCount); // batch insert operation
}
public static Task <TResult> ResolveCreate <TResult>(this StorageException exception,
CloudTable table,
Func <TResult> retry,
Func <ExtendedErrorInformationCodes, string, TResult> onFailure = default,
Func <TResult> onAlreadyExists = default,
AzureStorageDriver.RetryDelegate onTimeout = default)
{
return(exception.ParseStorageException(
onEntityAlreadyExists: (msg) =>
{
if (!onAlreadyExists.IsDefaultOrNull())
{
return onAlreadyExists().AsTask();
}
throw new ExtendedErrorInformationException(ExtendedErrorInformationCodes.EntityAlreadyExists, msg);
},
onNotFound: async(msg) => // IsProblemTableDoesNotExist
{
try
{
await table.CreateIfNotExistsAsync();
return retry();
}
catch (StorageException createEx)
{
// Catch bug with azure storage table client library where
// if two resources attempt to create the table at the same
// time one gets a precondtion failed error.
System.Threading.Thread.Sleep(1000);
return retry();
}
catch (Exception) { throw; };
},
onTimeout: async(msg) => // IsProblemTimeout
{
if (onTimeout.IsDefaultOrNull())
{
onTimeout = AzureStorageDriver.GetRetryDelegate();
}
bool shouldRetry = false;
await onTimeout(exception.RequestInformation.HttpStatusCode, exception,
() =>
{
shouldRetry = true;
return 1.AsTask();
});
if (shouldRetry)
{
return retry();
}
throw exception;
},
onDefaultCallback:
(error, msg) =>
{
if (!onFailure.IsDefaultOrNull())
{
return onFailure(error, msg).AsTask();
}
throw new ExtendedErrorInformationException(error, msg);
}));
}
public static async Task <HttpResponseMessage> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req,
ILogger log, ExecutionContext context)
{
log.LogInformation("C# HTTP trigger function processed a request.");
string connectionsJson = File.ReadAllText(Path.Combine(context.FunctionAppDirectory, "Connections.json"));
JObject ConnectionsObject = JObject.Parse(connectionsJson);
string connectionString = ConnectionsObject["AZURE_STORAGE_URL"].ToString();
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(connectionString);
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("virtualroomconfig");
await table.CreateIfNotExistsAsync();
var room = req.Headers["room"];
if (string.IsNullOrEmpty(room))
{
room = req.Query["room"];
}
if (string.IsNullOrEmpty(room))
{
return(new HttpResponseMessage(HttpStatusCode.BadRequest)
{
Content = new StringContent("Please pass a room name on the query string or in the header")
});
}
var partitionKey = "Demo";
var rowKey = room;
try
{
// get the room from the table
var getRoom = TableOperation.Retrieve <VirtualRoomConfig>(partitionKey, rowKey);
var query = await table.ExecuteAsync(getRoom);
var currRoomConfig = (VirtualRoomConfig)query.Result;
// if room not exist, create a record using default config
if (currRoomConfig == null)
{
var defaultRoom = new VirtualRoomConfig(partitionKey, rowKey);
var createRoom = TableOperation.Insert(defaultRoom);
await table.ExecuteAsync(createRoom);
currRoomConfig = (VirtualRoomConfig)(await table.ExecuteAsync(getRoom)).Result;
}
var operation = req.Query["operation"].ToString().ToLower();
var updated = false;
if (!string.IsNullOrEmpty(operation))
{
if (operation.Equals("reset"))
{
currRoomConfig.LoadDefaultConfig();
updated = true;
}
else if (operation.Equals("turn"))
{
var item = req.Query["item"].ToString().ToLower();
var instance = req.Query["instance"].ToString().ToLower();
var value = req.Query["value"].ToString().ToLower();
bool?valueBool = (value.Equals("on") || value.Equals("open")) ? true : ((value.Equals("off") || value.Equals("close")) ? (bool?)false : null);
if (valueBool == null)
{
updated = false;
}
else if (item.Equals("lights"))
{
if (instance.Equals("all"))
{
if (currRoomConfig.Lights_bathroom == (bool)valueBool && currRoomConfig.Lights_room == (bool)valueBool)
{
currRoomConfig.Message = $"All lights already {value}";
}
else
{
currRoomConfig.Lights_bathroom = (bool)valueBool;
currRoomConfig.Lights_room = (bool)valueBool;
currRoomConfig.Message = $"Ok, turning all the lights {value}";
}
updated = true;
}
else if (instance.Equals("room"))
{
if (currRoomConfig.Lights_room == (bool)valueBool)
{
currRoomConfig.Message = $"Room light already {value}";
}
else
{
currRoomConfig.Lights_room = (bool)valueBool;
currRoomConfig.Message = $"Ok, turning {value} the room light";
}
updated = true;
}
else if (instance.Equals("bathroom"))
{
if (currRoomConfig.Lights_bathroom == (bool)valueBool)
{
currRoomConfig.Message = $"Bathroom light already {value}";
}
else
{
currRoomConfig.Lights_bathroom = (bool)valueBool;
currRoomConfig.Message = $"Ok, turning {value} the bathroom light";
}
updated = true;
}
}
else if (item.Equals("tv"))
{
if (currRoomConfig.Television == (bool)valueBool)
{
currRoomConfig.Message = $"TV already {value}";
}
else
{
currRoomConfig.Television = (bool)valueBool;
currRoomConfig.Message = $"Ok, turning the TV {value}";
}
updated = true;
}
else if (item.Equals("blinds"))
{
if (currRoomConfig.Blinds == (bool)valueBool)
{
currRoomConfig.Message = (bool)valueBool ? "Blinds already opened" : "Blinds already closed";
}
else
{
currRoomConfig.Blinds = (bool)valueBool;
currRoomConfig.Message = (bool)valueBool ? "All right, opening the blinds" : "All right, closing the blinds";
}
updated = true;
}
else if (item.Equals("ac"))
{
if (currRoomConfig.AC == (bool)valueBool)
{
currRoomConfig.Message = $"AC already {value}";
}
else
{
currRoomConfig.AC = (bool)valueBool;
currRoomConfig.Message = $"Ok, turning the AC {value}";
}
updated = true;
}
}
else if (operation.Equals("settemperature"))
{
currRoomConfig.Temperature = int.Parse(req.Query["value"]);
currRoomConfig.Message = "set temperature to " + req.Query["value"];
updated = true;
}
else if (operation.Equals("increasetemperature"))
{
currRoomConfig.Temperature += int.Parse(req.Query["value"]);
currRoomConfig.Message = "raised temperature by " + req.Query["value"] + " degrees";
updated = true;
}
else if (operation.Equals("decreasetemperature"))
{
currRoomConfig.Temperature -= int.Parse(req.Query["value"]);
currRoomConfig.Message = "decreased temperature by " + req.Query["value"] + " degrees";
updated = true;
}
}
if (updated)
{
var updateRoom = TableOperation.Replace(currRoomConfig as VirtualRoomConfig);
await table.ExecuteAsync(updateRoom);
log.LogInformation("successfully updated the record");
}
return(new HttpResponseMessage(HttpStatusCode.OK)
{
Content = new StringContent(JsonConvert.SerializeObject(currRoomConfig, Formatting.Indented), Encoding.UTF8, "application/json")
});
}
catch (Exception e)
{
log.LogError(e.Message);
return(new HttpResponseMessage(HttpStatusCode.BadRequest)
{
Content = new StringContent("Failed to process request")
});
}
}
public static async Task <string> Run(
[HttpTrigger(AuthorizationLevel.Anonymous, "get", "post", Route = null)] HttpRequest req,
ILogger log)
{
string code = req.Query["code"];
string username = req.Query["user"];
string grant_type = "authorization_code";
string url = @"https://login.microsoftonline.com/{teant}/oauth2/v2.0/token";
HttpClient client = new HttpClient();
CloudStorageAccount storageAccount =
CloudStorageAccount.Parse("XXXX");
CloudTableClient tableClient = storageAccount.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("appUsers");
TableOperation retrieveOperation = TableOperation.Retrieve <User>(username, username);
TableResult retrievedResult = await table.ExecuteAsync(retrieveOperation);
User deleteEntity = (User)retrievedResult.Result;
if (deleteEntity != null)
{
TableOperation deleteOperation = TableOperation.Delete(deleteEntity);
await table.ExecuteAsync(deleteOperation);
}
var values = new Dictionary <string, string>
{
{ "grant_type", "authorization_code" },
{ "client_id", "a02f6ab7-1acd-4bfc-97d5-992acc1301b1" },
{ "scope", "https://graph.microsoft.com/User.Read offline_access" },
{ "redirect_uri", "https://graphsharepoint.azurewebsites.net/api/GetToken" },
{ "client_secret", "XXX" },
{ "code", code }
};
var content = new FormUrlEncodedContent(values);
var response = await client.PostAsync(url, content);
dynamic responseString = await response.Content.ReadAsAsync <object>();
await table.CreateIfNotExistsAsync();
User user = new User
{
RowKey = username,
name = username,
accessToken = responseString.access_token,
refreshToken = responseString.refresh_token,
PartitionKey = username
};
TableOperation insertOperation = TableOperation.Insert(user);
await table.ExecuteAsync(insertOperation);
return(responseString.access_token);
}
public async Task CreateIfNotExistsAsync()
{
await cloudTable.CreateIfNotExistsAsync();
}
/// <summary>
/// Create the table
/// </summary>
public async Task CreateTableAsync()
{
await _cloudTable.CreateIfNotExistsAsync().ConfigureAwait(false);
}
public Task Initialize() => _table.CreateIfNotExistsAsync();
public async static void Storage()
{
CloudStorageAccount storageAccount = CloudStorageAccount.Parse("UseDevelopmentStorage=true");
CloudTableClient cloudTableClient = storageAccount.CreateCloudTableClient();
CloudTable table = cloudTableClient.GetTableReference("StartStopVMs");
await table.CreateIfNotExistsAsync();
/*
* for (int i = 0; i < 12; i++)
* {
* DateTime x = DateTime.Now.Subtract(TimeSpan.FromHours(i*6));
* string partitionKey = "partition";
* var entry = new StartStopVMEntry(partitionKey, String.Format("{0:d21}{1}{2}", DateTimeOffset.MaxValue.UtcDateTime.Ticks - new DateTimeOffset(x).UtcDateTime.Ticks, "-", Guid.NewGuid().ToString()));
* entry.VMName = "vm";
* entry.VMType = "b";
* entry.VMStartedOrStopped = "c";
* entry.VMdatetime = x;
* await table.ExecuteAsync(TableOperation.Insert(entry));
* }
*/
//Find all entries from now to before
var fromTime = DateTime.UtcNow;
var fromTicks = String.Format("{0:d21}", DateTimeOffset.MaxValue.UtcDateTime.Ticks - new DateTimeOffset(fromTime).UtcDateTime.Ticks);
var toTime = DateTime.UtcNow.Subtract(TimeSpan.FromDays(30));
var toTicks = String.Format("{0:d21}", DateTimeOffset.MaxValue.UtcDateTime.Ticks - new DateTimeOffset(toTime).UtcDateTime.Ticks);
string pkFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "partition");
string rkLowerFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.GreaterThanOrEqual, fromTicks);
string rkUpperFilter = TableQuery.GenerateFilterCondition("RowKey", QueryComparisons.LessThan, toTicks);
// Note CombineFilters has the effect of “([Expression1]) Operator (Expression2]), as such passing in a complex expression will result in a logical grouping.
string combinedRowKeyFilter = TableQuery.CombineFilters(rkLowerFilter, TableOperators.And, rkUpperFilter);
string combinedFilter = TableQuery.CombineFilters(pkFilter, TableOperators.And, combinedRowKeyFilter);
TableQuery <StartStopVMEntry> query = new TableQuery <StartStopVMEntry>().Where(combinedFilter);
TableContinuationToken token = null;
query.TakeCount = 50;
int segmentNumber = 0;
do
{
// Execute the query, passing in the continuation token.
// The first time this method is called, the continuation token is null. If there are more results, the call
// populates the continuation token for use in the next call.
TableQuerySegment <StartStopVMEntry> segment = await table.ExecuteQuerySegmentedAsync(query, token);
// Indicate which segment is being displayed
if (segment.Results.Count > 0)
{
segmentNumber++;
Console.WriteLine();
Console.WriteLine("Segment {0}", segmentNumber);
}
// Save the continuation token for the next call to ExecuteQuerySegmentedAsync
token = segment.ContinuationToken;
// Write out the properties for each entity returned.
foreach (StartStopVMEntry entity in segment)
{
Console.WriteLine("\t Customer: {0},{1},{2},{3}", entity.PartitionKey, entity.RowKey, entity.VMdatetime.ToShortDateString(), entity.Timestamp);
}
Console.WriteLine();
}while (token != null);
}
public static async Task Run(
[ServiceBusTrigger(QueueNames.StatusLoop, Connection = "ServiceBusConnectionString")] Message msg,
[ServiceBus(QueueNames.StatusLoop, Connection = "ServiceBusConnectionString")] IAsyncCollector <Message> statusLoop,
[ServiceBus(QueueNames.StatusMessage, Connection = "ServiceBusConnectionString")] IAsyncCollector <Message> statusMessages,
[Table(TableNames.Status, PartitionKeys.Status, "{CorrelationId}")] ImportStatusEntity status,
[Table(TableNames.Status)] CloudTable table,
[Table(TableNames.StatusProcess)] CloudTable statusProcTable,
string correlationId,
ILogger log)
{
await statusProcTable.CreateIfNotExistsAsync();
await table.CreateIfNotExistsAsync();
if (string.IsNullOrWhiteSpace(correlationId))
{
correlationId = Guid.NewGuid().ToString();
log.LogWarning("CorrelationID is empty, generating a new one ({correlationId})", correlationId);
return;
}
if (status == null)
{
log.LogWarning("Oops, the status entry of import ({correlationId}) could not be found", correlationId);
return;
}
var statusUpdateCommand = msg.Convert <ImportStatusReportCommand>();
var updateCounts = await GetProcessingCountsAsync(correlationId, statusProcTable);
status.TotalSucceeded += updateCounts.Item1;
status.TotalFailed += updateCounts.Item2;
status.LastModificationOn = DateTimeOffset.UtcNow;
if (!status.CompletedOn.HasValue && status.TotalEntries == status.TotalFailed + status.TotalSucceeded)
{
status.CompletedOn = DateTimeOffset.UtcNow;
}
var operation = TableOperation.Replace(status);
await table.ExecuteAsync(operation);
try
{
var importStatus = new ImportStatusMessageDto
{
TotalEntries = status.TotalEntries,
ErrorMessage = status.ErrorMessage,
CompletedOn = status.CompletedOn,
CorrelationId = correlationId,
Failed = status.TotalFailed,
StartedOn = status.CreatedOn,
Succeeded = status.TotalSucceeded
};
await statusMessages.AddAsync(importStatus.Convert(correlationId));
if (updateCounts.Item1 > 0 || updateCounts.Item2 > 0)
{
statusUpdateCommand.StopReportingAt = DateTimeOffset.UtcNow.AddMinutes(30);
}
if (!status.CompletedOn.HasValue && statusUpdateCommand.StopReportingAt.CompareTo(DateTimeOffset.UtcNow) > 0)
{
await statusLoop.AddAsync(statusUpdateCommand.Convert(correlationId, 1));
}
}
catch (Exception ex)
{
log.LogError(ex, "Failed to send message to status update queueu");
}
await statusMessages.FlushAsync();
await statusLoop.FlushAsync();
log.LogInformation($"C# ServiceBus queue trigger function processed message: {msg}");
}
public IActionResult GetByFilter([FromHeader] string spfilefilter)
{
string entityAsJson = "";
try
{
_logger.LogInformation("CPAPI: Get");
// Validate the filter
if (spfilefilter == null || spfilefilter.Length == 0)
{
_logger.LogDebug("Filter not provided, returning blank");
return(new ObjectResult("[]"));
}
// Deserialize the filter
SPFileFilter oFilter = new SPFileFilter();
if (spfilefilter != null && spfilefilter.Length > 0)
{
_logger.LogDebug("Deserializing filter of length: " + spfilefilter.Length);
oFilter = JsonConvert.DeserializeObject <SPFileFilter>(spfilefilter);
}
string storageAccountConnectionString = Utils.GetSecretOrEnvVar(ConfigurationProperties.AzureStorageAccountConnectionString, Configuration, _logger).Trim();
// validate tika base address
if (storageAccountConnectionString == "")
{
_logger.LogWarning("Azure storage account connection string not set");
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
else
{
_logger.LogDebug("Azure storage account connection string loaded");
}
// Process the records
CloudStorageAccount account = CloudStorageAccount.Parse(storageAccountConnectionString);
// Create the table client.
//log.Info("Creating cloud table client");
CloudTableClient tableClient = account.CreateCloudTableClient();
// Create the table if it doesn't exist.
//log.Info("Getting table reference");
CloudTable table = tableClient.GetTableReference("stlpo365spfiles");
Task tCreate = table.CreateIfNotExistsAsync();
tCreate.Wait();
// Create a default query
TableQuery <SPFileProcessingStatusEntity> query = new TableQuery <SPFileProcessingStatusEntity>();
if (oFilter.spfiles.Count > 0)
{
string combinedFilter = "";
foreach (SPFileFilterEntry filterentry in oFilter.spfiles)
{
string cleanFilterPKey = Utils.CleanTableKey(filterentry.spfileabsoluteuri);
string pkquery = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, cleanFilterPKey);
combinedFilter = pkquery;
}
// Create final combined query
query = new TableQuery <SPFileProcessingStatusEntity>().Where(combinedFilter);
}
List <SPFileProcessingStatusEntity> spfileEntities = new List <SPFileProcessingStatusEntity>();
TableContinuationToken token = null;
var runningQuery = new TableQuery <SPFileProcessingStatusEntity>()
{
FilterString = query.FilterString,
SelectColumns = query.SelectColumns
};
do
{
runningQuery.TakeCount = query.TakeCount - spfileEntities.Count;
Task <TableQuerySegment <SPFileProcessingStatusEntity> > tSeg = table.ExecuteQuerySegmentedAsync <SPFileProcessingStatusEntity>(runningQuery, token);
tSeg.Wait();
token = tSeg.Result.ContinuationToken;
spfileEntities.AddRange(tSeg.Result);
} while (token != null && (query.TakeCount == null || spfileEntities.Count < query.TakeCount.Value)); //!ct.IsCancellationRequested &&
//no sorting
//spfileEntities.Sort((x, y) => String.Compare(x.Label, y.Label));
entityAsJson = JsonConvert.SerializeObject(spfileEntities, Formatting.Indented);
}
catch (Exception ex)
{
string exceptionMsg = "SPFile batch status GET exception: " + ex.Message;
//log.Info("Exception occurred extracting text from uploaded file \r\nError: " + ex.Message);
if (ex.InnerException != null)
{
exceptionMsg = exceptionMsg + "[" + ex.InnerException.Message + "]";
}
_logger.LogError(exceptionMsg);
return(StatusCode((int)System.Net.HttpStatusCode.InternalServerError));
}
ObjectResult result = new ObjectResult(entityAsJson);
return(result);
}
public IActionResult Index(string Category)
{
if (!String.IsNullOrEmpty(Category))
{
var userid = _userManager.GetUserId(HttpContext.User);
CloudStorageAccount storage = connectionstrg();
CloudTableClient tableClient = storage.CreateCloudTableClient();
CloudTable table = tableClient.GetTableReference("HaulTable");
TableQuery <Haul> query = new TableQuery <Haul>();
List <Haul> hauls = new List <Haul>();
TableContinuationToken token = null;
do
{
TableQuerySegment <Haul> resultSegment = table.ExecuteQuerySegmentedAsync(query, token).Result;
token = resultSegment.ContinuationToken;
foreach (Haul haul in resultSegment.Results)
{
hauls.Add(haul);
}
}while (token != null);
var data = from m in hauls select m;
if (!String.IsNullOrEmpty(userid))
{
data = data.Where(m => m.UserId.Contains(userid) && m.PartitionKey.Contains(Category));
}
return(View(data));
}
else
{
var userid = _userManager.GetUserId(HttpContext.User);
CloudStorageAccount storage = connectionstrg();
CloudTableClient tableclient = storage.CreateCloudTableClient();
CloudTable table = tableclient.GetTableReference("HaulTable");
ViewBag.result = table.CreateIfNotExistsAsync().Result;
ViewBag.TableName = table.Name;
TableQuery <Haul> query = new TableQuery <Haul>();
List <Haul> hauls = new List <Haul>();
TableContinuationToken token = null;
do
{
TableQuerySegment <Haul> resultSegment = table.ExecuteQuerySegmentedAsync(query, token).Result;
token = resultSegment.ContinuationToken;
foreach (Haul haul in resultSegment.Results)
{
hauls.Add(haul);
}
}while (token != null);
var data = from m in hauls select m;
if (!String.IsNullOrEmpty(userid))
{
data = data.Where(m => m.UserId.Contains(userid));
}
return(View(data));
}
}
public void CreateIfNotExists()
{
table.CreateIfNotExistsAsync().Wait();
}
public GenericRepository(CloudTableClient tableClient, string tableName)
{
this.tableClient = tableClient;
table = this.tableClient.GetTableReference(tableName);
table.CreateIfNotExistsAsync();
}
public async Task CreateTableAsync()
{
CloudTable table = tableClient.GetTableReference(typeof(T).Name);
await table.CreateIfNotExistsAsync();
}
/// <summary>
/// Initiates an asynchronous operation to create a table if it does not already exist.
/// </summary>
/// <param name="cancellationToken">
/// A <see cref="T:System.Threading.CancellationToken" /> to observe while waiting for a
/// task to complete.
/// </param>
/// <returns>
/// A <see cref="T:System.Threading.Tasks.Task`1" /> object of type <c>bool</c> that represents the asynchronous
/// operation.
/// </returns>
public Task <bool> CreateIfNotExistsAsync(CancellationToken cancellationToken = default(CancellationToken))
{
return(_cloudTable.CreateIfNotExistsAsync(null, null, cancellationToken));
}
public async Task CreateTablesIfNotCreated()
{
await snapshotsTable.CreateIfNotExistsAsync();
}
protected async Task InitializeAsync(string tableName)
{
_table = _tableClient.GetTableReference(tableName);
await _table.CreateIfNotExistsAsync();
}
public string GetAggregationStats(string ontologyUri = "")
{
string storageAccountConnectionString = Configuration.GetConnectionString("AzureCloudStorage"); //Utils.GetStorageAccountConnectionString();
CloudStorageAccount account = CloudStorageAccount.Parse(storageAccountConnectionString);
// Create the table client.
//log.Info("Creating cloud table client");
CloudTableClient tableClient = account.CreateCloudTableClient();
// Create the table if it doesn't exist.
//log.Info("Getting table reference");
CloudTable table = tableClient.GetTableReference("stlpstats");
Task tCreate = table.CreateIfNotExistsAsync();
tCreate.Wait();
string combinedFilter = "";
string pkFilter = TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.Equal, "CountByAggregation");
if (ontologyUri != "")
{
combinedFilter = pkFilter;
}
else
{
// Single filter on partition key
combinedFilter = pkFilter;
}
TableQuery <RecordStats> query = new TableQuery <RecordStats>().Where(combinedFilter);
RecordStatsJson allStats = new RecordStatsJson();
TableContinuationToken token = null;
var runningQuery = new TableQuery <RecordStats>()
{
FilterString = query.FilterString,
SelectColumns = query.SelectColumns
};
do
{
runningQuery.TakeCount = query.TakeCount - allStats.stats.Count;
Task <TableQuerySegment <RecordStats> > tSeg = table.ExecuteQuerySegmentedAsync <RecordStats>(runningQuery, token);
tSeg.Wait();
token = tSeg.Result.ContinuationToken;
allStats.stats.AddRange(tSeg.Result);
} while (token != null && (query.TakeCount == null || allStats.stats.Count < query.TakeCount.Value)); //!ct.IsCancellationRequested &&
// Top 10 order by descending
var itemList = from t in allStats.stats
orderby t.StatsCount descending
select t;
allStats.stats = itemList.Take(10).ToList();
string jsonStats = JsonConvert.SerializeObject(allStats);
return(jsonStats);
}