private AnalyticsEntry ConvertToEntry(AnalyticsEntryViewModel model)
{
var entry = new AnalyticsEntry
{
Url = model.Url,
UserIdentifier = GetUserIdentifier(),
VisitDateUtc = DateTime.UtcNow
};
var contentItem = model.ContentItemId.HasValue ? _contentManager.Get(model.ContentItemId.Value, VersionOptions.Published) : null;
if (contentItem == null)
{
return(entry);
}
entry.ContentItemId = contentItem.Id;
var tagsPart = contentItem.As <TagsPart>();
if (tagsPart == null)
{
return(entry);
}
entry.Tags = string.Join(",", tagsPart.CurrentTags);
return(entry);
}
public static void SeedDatabase(SqlServerAnalyticsContext context)
{
var random = new Random(32019);
var users = new[] {
"designer",
"Designer",
"*****@*****.**",
null,
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
"*****@*****.**",
};
var versions = new[] {
"1.0.1",
"1.1.5",
"1.0.0-beta",
};
var areas = new[] {
"links",
"magazines",
"sellers",
"authentication",
};
var ips = new[] {
"::1",
"139.82.87.23",
"173.0.82.126",
"179.109.36.19",
"179.109.39.199",
"186.205.89.240",
"189.122.142.54",
"50.31.134.17"
};
var actions = new[] {
new SampleRequest("GET /activities", 50, 250, 50, 4),
new SampleRequest("GET /app", 60, 600, 75, 7),
new SampleRequest("POST /auth/login", 55, 50, 150, 3),
new SampleRequest("POST /auth/refresh-token", 75, 200, 150, 3),
new SampleRequest("GET /categories", 100, 150, 250, 3),
new SampleRequest("PUT /categories", 200, 350, 50, 4),
new SampleRequest("POST /categories", 50, 5000, 25, 2),
new SampleRequest("GET /images", 50, 200, 75, 1),
new SampleRequest("PUT /images", 20, 100, 150, 1),
new SampleRequest("POST /images", 15, 150, 500, 1),
new SampleRequest("GET /links", 30, 175, 250, 2),
new SampleRequest("PUT /links", 60, 200, 250, 5),
new SampleRequest("POST /links", 80, 50, 150, 3),
new SampleRequest("GET /magazines", 50, 75, 100, 2),
new SampleRequest("PUT /magazines", 25, 20, 50, 3),
new SampleRequest("POST /magazines", 75, 10, 25, 4),
new SampleRequest("POST /paypal", 50, 1000, 25, 4),
new SampleRequest("POST /plans", 30, 500, 50, 4),
new SampleRequest("PUT /plans", 75, 100, 100, 2),
new SampleRequest("GET /plans", 10, 50, 250, 2),
new SampleRequest("GET /sellers", 5, 75, 250, 1),
new SampleRequest("POST /sellers", 60, 150, 100, 2),
new SampleRequest("PUT /sellers", 10, 100, 150, 9),
new SampleRequest("POST /templates", 20, 50, 150, 2),
new SampleRequest("PUT /templates", 25, 50, 300, 3),
new SampleRequest("GET /templates", 10, 50, 3500, 5),
new SampleRequest("POST /templates/thumbnail", 20, 250, 1500, 1),
new SampleRequest("GET /users/client-quota", 50, 300, 200, 4),
new SampleRequest("GET /users/profile", 20, 200, 50, 2),
};
var agents = new[] {
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_5) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:78.0) Gecko/20100101 Firefox/78.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/83.0.4103.116 Safari/537.36",
"PayPal/AUHR-214.0-54656248",
};
var responses = new[] {
401,
400,
500,
};
var now = new DateTime(DateTime.UtcNow.Year, DateTime.UtcNow.Month, DateTime.UtcNow.Day);
var startDate = now.AddDays(-30);
var currentData = startDate;
var probability = random.NextDouble();
while (currentData < DateTime.UtcNow)
{
var iterations = random.Next(3, 10);
for (int i = 0; i < iterations; i++)
{
var path = actions[random.Next(0, actions.Length)];
var data = new AnalyticsEntry();
data.Version = versions[random.Next(0, versions.Length)];
data.Area = areas[random.Next(0, areas.Length)];
data.Username = users[random.Next(0, users.Length)];
data.IpAddress = ips[random.Next(0, ips.Length)];
data.UserAgent = agents[random.Next(0, agents.Length)];
data.Path = path.Path;
data.Method = path.Path.Split()[0];
data.Action = path.Path;
data.Response = random.NextDouble() > 0.95 ? responses[random.Next(0, responses.Length)] : 200;
data.Duration = path.GetDuration(random);
data.RequestSize = path.GetRequestSize(random);
data.ResponseSize = path.GetResponseSize(random);
data.WasCached = random.NextDouble() > 0.9;
data.TransactionCount = path.Requests;
data.TotalTransactionTime = (int)(path.GetDuration(random) * 0.75);
data.Timestamp = currentData;
context.Add(data);
}
currentData = currentData.AddHours(1);
}
context.SaveChanges();
}
public static void ProcessQueue([QueueTrigger(queueName: Utility.QUEUE)] string request,
[CosmosDB(Utility.URL_TRACKING, Utility.URL_STATS, CreateIfNotExists = true, ConnectionStringSetting = "CosmosDb")] out dynamic doc,
ILogger log)
{
try
{
AnalyticsEntry parsed = Utility.ParseQueuePayload(request);
var page = parsed.LongUrl.AsPage(HttpUtility.ParseQueryString);
telemetry.TrackPageView(page);
log.LogInformation($"Tracked page view {page}");
var analytics = parsed.LongUrl.ExtractCampaignAndMedium(HttpUtility.ParseQueryString);
var campaign = analytics.Item1;
var medium = analytics.Item2;
if (!string.IsNullOrWhiteSpace(medium))
{
telemetry.TrackEvent(medium);
log.LogInformation($"Tracked custom event: {medium}");
}
// cosmos DB
var normalize = new[] { '/' };
doc = new ExpandoObject();
doc.id = Guid.NewGuid().ToString();
doc.page = page.TrimEnd(normalize);
if (!string.IsNullOrWhiteSpace(parsed.ShortUrl))
{
doc.shortUrl = parsed.ShortUrl;
}
if (!string.IsNullOrWhiteSpace(campaign))
{
doc.campaign = campaign;
}
if (parsed.Referrer != null)
{
doc.referrerUrl = parsed.Referrer.AsPage(HttpUtility.ParseQueryString);
doc.referrerHost = parsed.Referrer.DnsSafeHost;
}
if (!string.IsNullOrWhiteSpace(parsed.Agent))
{
doc.agent = parsed.Agent;
try
{
var parser = UAParser.Parser.GetDefault();
var client = parser.Parse(parsed.Agent);
{
var browser = client.UserAgent.Family;
var version = client.UserAgent.Major;
var browserVersion = $"{browser} {version}";
doc.browser = browser;
doc.browserVersion = version;
doc.browserWithVersion = browserVersion;
}
if (client.Device.IsSpider)
{
doc.crawler = 1;
}
if (parsed.Agent.ToLowerInvariant().Contains("mobile"))
{
doc.mobile = 1;
var manufacturer = client.Device.Brand;
doc.mobileManufacturer = manufacturer;
var model = client.Device.Model;
doc.mobileModel = model;
doc.mobileDevice = $"{manufacturer} {model}";
}
else
{
doc.desktop = 1;
}
if (!string.IsNullOrWhiteSpace(client.OS.Family))
{
doc.platform = client.OS.Family;
doc.platformVersion = client.OS.Major;
doc.platformWithVersion = $"{client.OS.Family} {client.OS.Major}";
}
}
catch (Exception ex)
{
log.LogError(ex, $"Error parsing user agent [{parsed.Agent}]");
}
}
doc.count = 1;
doc.timestamp = parsed.TimeStamp;
doc.host = parsed.LongUrl.DnsSafeHost;
if (!string.IsNullOrWhiteSpace(medium))
{
((IDictionary <string, object>)doc).Add(medium, 1);
}
log.LogInformation($"CosmosDB: {doc.id}|{doc.page}|{parsed.ShortUrl}|{campaign}|{medium}");
}
catch (Exception ex)
{
log.LogError(ex, "An unexpected error occurred.");
throw;
}
}
public static void ProcessQueue([QueueTrigger(queueName: Utility.QUEUE)] string request,
[DocumentDB(Utility.URL_TRACKING, Utility.URL_STATS, CreateIfNotExists = true, ConnectionStringSetting = "CosmosDb")] out dynamic doc,
TraceWriter log)
{
try
{
AnalyticsEntry parsed = Utility.ParseQueuePayload(request);
var page = parsed.LongUrl.AsPage(HttpUtility.ParseQueryString);
telemetry.TrackPageView(page);
log.Info($"Tracked page view {page}");
var analytics = parsed.LongUrl.ExtractCampaignAndMedium(HttpUtility.ParseQueryString);
var campaign = analytics.Item1;
var medium = analytics.Item2;
if (!string.IsNullOrWhiteSpace(medium))
{
telemetry.TrackEvent(medium);
log.Info($"Tracked custom event: {medium}");
}
// cosmos DB
var normalize = new[] { '/' };
doc = new ExpandoObject();
doc.id = Guid.NewGuid().ToString();
doc.page = page.TrimEnd(normalize);
if (!string.IsNullOrWhiteSpace(parsed.ShortUrl))
{
doc.shortUrl = parsed.ShortUrl;
}
if (!string.IsNullOrWhiteSpace(campaign))
{
doc.campaign = campaign;
}
if (parsed.Referrer != null)
{
doc.referrerUrl = parsed.Referrer.AsPage(HttpUtility.ParseQueryString);
doc.referrerHost = parsed.Referrer.DnsSafeHost;
}
if (!string.IsNullOrWhiteSpace(parsed.Agent))
{
doc.agent = parsed.Agent;
try
{
var capabilities = new HttpBrowserCapabilities()
{
Capabilities = new Hashtable {
{ string.Empty, parsed.Agent }
}
};
var factory = new BrowserCapabilitiesFactory();
factory.ConfigureBrowserCapabilities(new NameValueCollection(), capabilities);
factory.ConfigureCustomCapabilities(new NameValueCollection(), capabilities);
if (!string.IsNullOrEmpty(capabilities.Browser))
{
var browser = capabilities.Browser;
var version = capabilities.MajorVersion;
var browserVersion = $"{browser} {capabilities.MajorVersion}";
doc.browser = browser;
doc.browserVersion = version;
doc.browserWithVersion = browserVersion;
}
if (capabilities.Crawler)
{
doc.crawler = 1;
}
if (capabilities.IsMobileDevice)
{
doc.mobile = 1;
var manufacturer = capabilities.MobileDeviceManufacturer;
var model = capabilities.MobileDeviceModel;
doc.mobileManufacturer = manufacturer;
doc.mobileModel = model;
doc.mobileDevice = $"{manufacturer} {model}";
}
else
{
doc.desktop = 1;
}
if (!string.IsNullOrWhiteSpace(capabilities.Platform))
{
doc.platform = capabilities.Platform;
}
}
catch (Exception ex)
{
log.Error($"Error parsing user agent [{parsed.Agent}]", ex);
}
}
doc.count = 1;
doc.timestamp = parsed.TimeStamp;
doc.host = parsed.LongUrl.DnsSafeHost;
if (!string.IsNullOrWhiteSpace(medium))
{
((IDictionary <string, object>)doc).Add(medium, 1);
}
log.Info($"CosmosDB: {doc.id}|{doc.page}|{parsed.ShortUrl}|{campaign}|{medium}");
}
catch (Exception ex)
{
log.Error("An unexpected error occurred", ex);
throw;
}
}
public void StoreData(AnalyticsEntry request)
{
_context.Data.Add(request);
_context.SaveChanges();
}
