static void DownloadTelemetry(string monthsBack) { var table = CreateTelemetryTable(); int monthsBackNum; int.TryParse(monthsBack ?? "", out monthsBackNum); var query = new TableQuery <Telemetry.AzureStorageEntry>(); if (monthsBackNum != 0) { query = query.Where(TableQuery.GenerateFilterCondition("PartitionKey", QueryComparisons.GreaterThan, DateTime.UtcNow.AddMonths(-monthsBackNum).ToString("yyyy-MM"))); } var dataSet = new Telemetry.RawDataSet(); int processedEntriesCount = 0; Console.WriteLine(); Console.Write("Processed entries: 0"); Action printProcessedEntries = () => { Console.Write("\b\b\b\b"); Console.Write("{0,4}", processedEntriesCount); }; foreach (var entry in table.ExecuteQuery(query)) { dataSet.HandleTelemetryEntry(entry); processedEntriesCount++; if (processedEntriesCount % 100 == 0) { printProcessedEntries(); } } printProcessedEntries(); Console.WriteLine(); Telemetry.Analytics.AnalizeFeaturesUse(dataSet); Console.WriteLine(); }
public static void AnalizeFeaturesUse(RawDataSet dataSet) { var subFeaturesUseFrequencies = FeatureMetadata.AllFeatures.SelectMany(f => f.SubFeatures.Values).ToDictionary( sf => sf, sf => new UseFrequencyInfo()); foreach (var dateGroup in dataSet .Sessions .Select(s => new { Session = s, Date = s.Started.Date }) .OrderBy(x => x.Date) .GroupBy(x => x.Date)) { var totalNrOfLogs = dateGroup.Sum(x => x.Session.TotalNrOfLogs); if (totalNrOfLogs == 0) { continue; // no logs were open this day, ignore this day } var date = dateGroup.First().Date; foreach (var featureGroup in dateGroup.SelectMany(d => d.Session.Features).GroupBy(f => f.Meta)) { var featureMeta = featureGroup.First().Meta; // Feature with meta featureMeta was used this day (date). // Let's count how often each sub-feature was used this day. var counters = featureMeta.SubFeatures.ToDictionary(sf => sf.Key, sf => 0); foreach (var sf in featureGroup.SelectMany(f => f.SubFeatures)) { counters[sf.Meta.Id] += sf.UseCount; } foreach (var sf in featureMeta.SubFeatures) { subFeaturesUseFrequencies[sf.Value].AddDataPoint(date, (double)counters[sf.Key] / (double)totalNrOfLogs); } } } var frequencesXmlDoc = new XDocument(new XElement("root")); foreach (var sf in subFeaturesUseFrequencies) { DateTime firstDate = new DateTime(); Trendline trend = null; var isDeadSubFeature = false; if (sf.Value.weeklyAverageUseFrequencies.Count == 0) { isDeadSubFeature = true; } else { firstDate = sf.Value.weeklyAverageUseFrequencies.First().date; trend = CalcTend(sf.Value.weeklyAverageUseFrequencies.Select( f => new DataPoint() { X = (f.date - firstDate).Days, Y = f.value }).ToArray()); } if (trend != null && Math.Abs(trend.slope) < 1e-6 && Math.Abs(trend.offset) < 1e-6) { isDeadSubFeature = true; } else if (trend != null && trend.slope < 0) { var predictedUnuseDate = firstDate.AddDays(-trend.offset / trend.slope); if (predictedUnuseDate < DateTime.Now) { isDeadSubFeature = true; } } if (isDeadSubFeature) { var prefix = ""; if (sf.Key.IsKnownObsoleteCodepath) { prefix += "Known "; } if (sf.Key.IsRareCodepath) { prefix += "Rare "; } if (prefix == "") { prefix = " "; } Console.Write(prefix); Console.Write("dead feature: {0} trend: ", sf.Key.Name); if (trend != null) { Console.WriteLine("{0}*x + {1}", trend.slope, trend.offset); } else { Console.WriteLine("N/A"); } } foreach (var freq in sf.Value.weeklyAverageUseFrequencies) { frequencesXmlDoc.Root.Add(new XElement( "subFeatureUseFrequency", new XElement("featureId", sf.Key.Feature.Id), new XElement("subFeatureId", sf.Key.Id), new XElement("subFeatureName", sf.Key.Name), new XElement("year", freq.date.Year), new XElement("month", freq.date.Month), new XElement("dayOfYear", freq.date.DayOfYear), new XElement("week", freq.date.DayOfYear / 7), new XElement("freq", freq.value) )); } } frequencesXmlDoc.Save(Environment.ExpandEnvironmentVariables("%USERPROFILE%\\lj.telem.sub-features.freqs.xml")); }