private AnalyticsEntry ConvertToEntry(AnalyticsEntryViewModel model) { var entry = new AnalyticsEntry { Url = model.Url, UserIdentifier = GetUserIdentifier(), VisitDateUtc = DateTime.UtcNow }; var contentItem = model.ContentItemId.HasValue ? _contentManager.Get(model.ContentItemId.Value, VersionOptions.Published) : null; if (contentItem == null) { return(entry); } entry.ContentItemId = contentItem.Id; var tagsPart = contentItem.As <TagsPart>(); if (tagsPart == null) { return(entry); } entry.Tags = string.Join(",", tagsPart.CurrentTags); return(entry); }
public static void SeedDatabase(SqlServerAnalyticsContext context) { var random = new Random(32019); var users = new[] { "designer", "Designer", "*****@*****.**", null, "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", "*****@*****.**", }; var versions = new[] { "1.0.1", "1.1.5", "1.0.0-beta", }; var areas = new[] { "links", "magazines", "sellers", "authentication", }; var ips = new[] { "::1", "139.82.87.23", "173.0.82.126", "179.109.36.19", "179.109.39.199", "186.205.89.240", "189.122.142.54", "50.31.134.17" }; var actions = new[] { new SampleRequest("GET /activities", 50, 250, 50, 4), new SampleRequest("GET /app", 60, 600, 75, 7), new SampleRequest("POST /auth/login", 55, 50, 150, 3), new SampleRequest("POST /auth/refresh-token", 75, 200, 150, 3), new SampleRequest("GET /categories", 100, 150, 250, 3), new SampleRequest("PUT /categories", 200, 350, 50, 4), new SampleRequest("POST /categories", 50, 5000, 25, 2), new SampleRequest("GET /images", 50, 200, 75, 1), new SampleRequest("PUT /images", 20, 100, 150, 1), new SampleRequest("POST /images", 15, 150, 500, 1), new SampleRequest("GET /links", 30, 175, 250, 2), new SampleRequest("PUT /links", 60, 200, 250, 5), new SampleRequest("POST /links", 80, 50, 150, 3), new SampleRequest("GET /magazines", 50, 75, 100, 2), new SampleRequest("PUT /magazines", 25, 20, 50, 3), new SampleRequest("POST /magazines", 75, 10, 25, 4), new SampleRequest("POST /paypal", 50, 1000, 25, 4), new SampleRequest("POST /plans", 30, 500, 50, 4), new SampleRequest("PUT /plans", 75, 100, 100, 2), new SampleRequest("GET /plans", 10, 50, 250, 2), new SampleRequest("GET /sellers", 5, 75, 250, 1), new SampleRequest("POST /sellers", 60, 150, 100, 2), new SampleRequest("PUT /sellers", 10, 100, 150, 9), new SampleRequest("POST /templates", 20, 50, 150, 2), new SampleRequest("PUT /templates", 25, 50, 300, 3), new SampleRequest("GET /templates", 10, 50, 3500, 5), new SampleRequest("POST /templates/thumbnail", 20, 250, 1500, 1), new SampleRequest("GET /users/client-quota", 50, 300, 200, 4), new SampleRequest("GET /users/profile", 20, 200, 50, 2), }; var agents = new[] { "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36", "PayPal/AUHR-214.0-54656248", }; var responses = new[] { 401, 400, 500, }; var now = new DateTime(DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day); var startDate = now.AddDays(-30); var currentData = startDate; var probability = random.NextDouble(); while (currentData < DateTime.UtcNow) { var iterations = random.Next(3, 10); for (int i = 0; i < iterations; i++) { var path = actions[random.Next(0, actions.Length)]; var data = new AnalyticsEntry(); data.Version = versions[random.Next(0, versions.Length)]; data.Area = areas[random.Next(0, areas.Length)]; data.Username = users[random.Next(0, users.Length)]; data.IpAddress = ips[random.Next(0, ips.Length)]; data.UserAgent = agents[random.Next(0, agents.Length)]; data.Path = path.Path; data.Method = path.Path.Split()[0]; data.Action = path.Path; data.Response = random.NextDouble() > 0.95 ? responses[random.Next(0, responses.Length)] : 200; data.Duration = path.GetDuration(random); data.RequestSize = path.GetRequestSize(random); data.ResponseSize = path.GetResponseSize(random); data.WasCached = random.NextDouble() > 0.9; data.TransactionCount = path.Requests; data.TotalTransactionTime = (int)(path.GetDuration(random) * 0.75); data.Timestamp = currentData; context.Add(data); } currentData = currentData.AddHours(1); } context.SaveChanges(); }
public static void ProcessQueue([QueueTrigger(queueName: Utility.QUEUE)] string request, [CosmosDB(Utility.URL_TRACKING, Utility.URL_STATS, CreateIfNotExists = true, ConnectionStringSetting = "CosmosDb")] out dynamic doc, ILogger log) { try { AnalyticsEntry parsed = Utility.ParseQueuePayload(request); var page = parsed.LongUrl.AsPage(HttpUtility.ParseQueryString); telemetry.TrackPageView(page); log.LogInformation($"Tracked page view {page}"); var analytics = parsed.LongUrl.ExtractCampaignAndMedium(HttpUtility.ParseQueryString); var campaign = analytics.Item1; var medium = analytics.Item2; if (!string.IsNullOrWhiteSpace(medium)) { telemetry.TrackEvent(medium); log.LogInformation($"Tracked custom event: {medium}"); } // cosmos DB var normalize = new[] { '/' }; doc = new ExpandoObject(); doc.id = Guid.NewGuid().ToString(); doc.page = page.TrimEnd(normalize); if (!string.IsNullOrWhiteSpace(parsed.ShortUrl)) { doc.shortUrl = parsed.ShortUrl; } if (!string.IsNullOrWhiteSpace(campaign)) { doc.campaign = campaign; } if (parsed.Referrer != null) { doc.referrerUrl = parsed.Referrer.AsPage(HttpUtility.ParseQueryString); doc.referrerHost = parsed.Referrer.DnsSafeHost; } if (!string.IsNullOrWhiteSpace(parsed.Agent)) { doc.agent = parsed.Agent; try { var parser = UAParser.Parser.GetDefault(); var client = parser.Parse(parsed.Agent); { var browser = client.UserAgent.Family; var version = client.UserAgent.Major; var browserVersion = $"{browser} {version}"; doc.browser = browser; doc.browserVersion = version; doc.browserWithVersion = browserVersion; } if (client.Device.IsSpider) { doc.crawler = 1; } if (parsed.Agent.ToLowerInvariant().Contains("mobile")) { doc.mobile = 1; var manufacturer = client.Device.Brand; doc.mobileManufacturer = manufacturer; var model = client.Device.Model; doc.mobileModel = model; doc.mobileDevice = $"{manufacturer} {model}"; } else { doc.desktop = 1; } if (!string.IsNullOrWhiteSpace(client.OS.Family)) { doc.platform = client.OS.Family; doc.platformVersion = client.OS.Major; doc.platformWithVersion = $"{client.OS.Family} {client.OS.Major}"; } } catch (Exception ex) { log.LogError(ex, $"Error parsing user agent [{parsed.Agent}]"); } } doc.count = 1; doc.timestamp = parsed.TimeStamp; doc.host = parsed.LongUrl.DnsSafeHost; if (!string.IsNullOrWhiteSpace(medium)) { ((IDictionary <string, object>)doc).Add(medium, 1); } log.LogInformation($"CosmosDB: {doc.id}|{doc.page}|{parsed.ShortUrl}|{campaign}|{medium}"); } catch (Exception ex) { log.LogError(ex, "An unexpected error occurred."); throw; } }
public static void ProcessQueue([QueueTrigger(queueName: Utility.QUEUE)] string request, [DocumentDB(Utility.URL_TRACKING, Utility.URL_STATS, CreateIfNotExists = true, ConnectionStringSetting = "CosmosDb")] out dynamic doc, TraceWriter log) { try { AnalyticsEntry parsed = Utility.ParseQueuePayload(request); var page = parsed.LongUrl.AsPage(HttpUtility.ParseQueryString); telemetry.TrackPageView(page); log.Info($"Tracked page view {page}"); var analytics = parsed.LongUrl.ExtractCampaignAndMedium(HttpUtility.ParseQueryString); var campaign = analytics.Item1; var medium = analytics.Item2; if (!string.IsNullOrWhiteSpace(medium)) { telemetry.TrackEvent(medium); log.Info($"Tracked custom event: {medium}"); } // cosmos DB var normalize = new[] { '/' }; doc = new ExpandoObject(); doc.id = Guid.NewGuid().ToString(); doc.page = page.TrimEnd(normalize); if (!string.IsNullOrWhiteSpace(parsed.ShortUrl)) { doc.shortUrl = parsed.ShortUrl; } if (!string.IsNullOrWhiteSpace(campaign)) { doc.campaign = campaign; } if (parsed.Referrer != null) { doc.referrerUrl = parsed.Referrer.AsPage(HttpUtility.ParseQueryString); doc.referrerHost = parsed.Referrer.DnsSafeHost; } if (!string.IsNullOrWhiteSpace(parsed.Agent)) { doc.agent = parsed.Agent; try { var capabilities = new HttpBrowserCapabilities() { Capabilities = new Hashtable { { string.Empty, parsed.Agent } } }; var factory = new BrowserCapabilitiesFactory(); factory.ConfigureBrowserCapabilities(new NameValueCollection(), capabilities); factory.ConfigureCustomCapabilities(new NameValueCollection(), capabilities); if (!string.IsNullOrEmpty(capabilities.Browser)) { var browser = capabilities.Browser; var version = capabilities.MajorVersion; var browserVersion = $"{browser} {capabilities.MajorVersion}"; doc.browser = browser; doc.browserVersion = version; doc.browserWithVersion = browserVersion; } if (capabilities.Crawler) { doc.crawler = 1; } if (capabilities.IsMobileDevice) { doc.mobile = 1; var manufacturer = capabilities.MobileDeviceManufacturer; var model = capabilities.MobileDeviceModel; doc.mobileManufacturer = manufacturer; doc.mobileModel = model; doc.mobileDevice = $"{manufacturer} {model}"; } else { doc.desktop = 1; } if (!string.IsNullOrWhiteSpace(capabilities.Platform)) { doc.platform = capabilities.Platform; } } catch (Exception ex) { log.Error($"Error parsing user agent [{parsed.Agent}]", ex); } } doc.count = 1; doc.timestamp = parsed.TimeStamp; doc.host = parsed.LongUrl.DnsSafeHost; if (!string.IsNullOrWhiteSpace(medium)) { ((IDictionary <string, object>)doc).Add(medium, 1); } log.Info($"CosmosDB: {doc.id}|{doc.page}|{parsed.ShortUrl}|{campaign}|{medium}"); } catch (Exception ex) { log.Error("An unexpected error occurred", ex); throw; } }
public void StoreData(AnalyticsEntry request) { _context.Data.Add(request); _context.SaveChanges(); }