public LinearProgression Smoothed(TimeSpan windowSize)
{
// make a List of the cumulative time spent
DateTime minDate = this.Owner.DiscoveryDate;
IEnumerable <ListItemStats <DateTime, ParticipationAndSummary> > items = this.searchHelper.AllItems;
LinearProgression cumulatives = new LinearProgression();
// first sort the starts and ends
StatList <DateTime, double> deltaIntensities = new StatList <DateTime, double>(new DateComparer(), new FloatAdder());
foreach (ListItemStats <DateTime, ParticipationAndSummary> item in items)
{
deltaIntensities.Add(item.Value.Participation.StartDate, 1);
deltaIntensities.Add(item.Value.Participation.EndDate, -1);
}
if (deltaIntensities.NumItems < 1)
{
return(new LinearProgression());
}
DateTime maxDate = deltaIntensities.GetLastValue().Key;
// now add up the (possibly overlapping) values
double intensity = 0;
DateTime prevDate = minDate;
double sum = 0;
foreach (ListItemStats <DateTime, double> deltaIntensity in deltaIntensities.AllItems)
{
double duration = deltaIntensity.Key.Subtract(prevDate).TotalSeconds;
sum += intensity * duration;
intensity += deltaIntensity.Value;
cumulatives.Add(deltaIntensity.Key, sum);
prevDate = deltaIntensity.Key;
}
// find what's in the sliding window by subtracting the cumulative from the shifted cumulative
LinearProgression shiftedCumulatives = cumulatives.Shifted(windowSize);
if (windowSize.TotalSeconds > 0)
{
cumulatives.Add(maxDate.Add(windowSize), sum);
LinearProgression result = cumulatives.Minus(shiftedCumulatives);
return(result);
}
else
{
shiftedCumulatives.Add(maxDate, sum);
LinearProgression result = shiftedCumulatives.Minus(cumulatives);
return(result);
}
}
private List <Datapoint> getDatapoints(IEnumerable <Activity> activities)
{
StatList <DateTime, int> discoveredCounts = new StatList <DateTime, int>(new DateComparer(), new IntAdder());
foreach (Activity activity in activities)
{
discoveredCounts.Add(activity.DiscoveryDate, 1);
}
DateTime minPossibleDate = TimeProgression.AbsoluteTime.StartDate;
bool isFirst = false;
List <Datapoint> datapoints = new List <Datapoint>();
// compute cumulative values
foreach (DateTime when in discoveredCounts.Keys)
{
if (isFirst)
{
isFirst = false;
}
TimeSpan duration = when.Subtract(minPossibleDate);
int cumulative = discoveredCounts.CombineBeforeKey(when, true);
datapoints.Add(new Datapoint(duration.TotalSeconds, cumulative, 1));
}
return(datapoints);
}
public SampleItem()
{
foreach (var stat in Helpers.List)
{
StatList.Add(new AdditiveStatClass((float)_rng.NextDouble()));
}
}
public List <ProtoActivity> TextSearch(string query, int count)
{
if (query == null || query == "" || count < 1)
{
return(new List <ProtoActivity>(0));
}
StatList <double, ProtoActivity> sortedItems = new StatList <double, ProtoActivity>(new ReverseDoubleComparer(), new NoopCombiner <ProtoActivity>());
foreach (ProtoActivity protoActivity in this.ProtoActivities)
{
double textQuality = this.stringQueryMatcher.StringScore(protoActivity.Text, query);
if (textQuality > 0)
{
double matchQuality = textQuality + protoActivity.IntrinsicInterest;
sortedItems.Add(matchQuality, protoActivity);
}
}
count = Math.Min(count, sortedItems.NumItems);
List <ProtoActivity> top = new List <ProtoActivity>(count);
for (int i = 0; i < count; i++)
{
top.Add(sortedItems.GetValueAtIndex(i).Value);
}
return(top);
}
private IEnumerable <int> nChooseK(int maxExclusive, int count)
{
StatList <int, bool> choices = new StatList <int, bool>(new IntComparerer(), new NoopCombiner <bool>());
if (count > maxExclusive)
{
count = maxExclusive;
}
for (int i = 0; i < count; i++)
{
int randInt = this.generator.Next(maxExclusive - i);
int index = randInt;
while (true)
{
int smallerCount = choices.CountBeforeKey(index, true);
if (index != randInt + smallerCount)
{
index = randInt + smallerCount;
}
else
{
break;
}
}
choices.Add(index, true);
}
return(choices.Keys);
}
public void AddTotalSaleValue()
{
StatList.Clear();
StatList.Add(new Statistics(SellingTotal(), "Indkomst"));
StatList.Add(new Statistics(BuyingTotal(), "Udgifter"));
StatList.Add(new Statistics((SellingTotal() - BuyingTotal()), "Profit"));
StatList.Add(new Statistics(0, ""));
}
private StatList <DateTime, double> GetEfficiencies()
{
// Suppose efficiency[day[i]] = i
StatList <DateTime, double> efficiencies = new StatList <DateTime, double>(new DateComparer(), new FloatAdder());
for (int i = 1; i < 365; i++)
{
efficiencies.Add(new DateTime(2000, 1, 1).AddDays(i), Math.Pow(1.01, i));
}
return(efficiencies);
}
public void SetValue(StatType type, object value)
{
if (!StatList.ContainsKey(type))
{
StatList.Add(type, value);
}
else
{
StatList[type] = value;
}
RaiseEvent();
}
public void ExtractOriginals()
{
foreach (var tile in TileCodes)
{
var path = $@"D:\Data\DEM\NZ\{tile}.asc";
var file = NzReader.Read(path);
Assert.AreEqual(8192, file.Columns);
Assert.AreEqual(8192, file.Rows);
Assert.AreEqual(-999, file.NoDataValue);
var list = new StatList();
var arr = file.Data;
foreach (var index in arr.Indices())
{
var value = arr[index];
if (value == file.NoDataValue)
{
foreach (var neighbour in arr.Neighbours(index))
{
var neighbourValue = arr[neighbour];
if (neighbourValue != file.NoDataValue)
{
list.Add(neighbourValue);
}
}
}
}
if (list.Any())
{
file.ReplaceNoDataValue(list.Average());
}
var outputPath = Path.Combine(OutputPath, $"{tile}.npy");
np.save(outputPath, file.Data);
using (var stream = File.Create(Path.Combine(OutputPath, $"{tile}.png")))
{
new ImageConverter().ToPng(file.Data, stream, HeightToGrayscale.FullRange8(file.Data));
}
}
}
// Find the top few matching activities
public List <Activity> FindBestMatches(ActivityDescriptor descriptor, int count)
{
if (count < 1)
{
return(new List <Activity>(0));
}
if (count == 1)
{
Activity best = this.ResolveDescriptor(descriptor);
if (best != null)
{
return new List <Activity>()
{
best
}
}
;
return(new List <Activity>()
{
});
}
IEnumerable <Activity> activities = this.GetCandidateMatches(descriptor);
StatList <double, Activity> sortedItems = new StatList <double, Activity>(new ReverseDoubleComparer(), new NoopCombiner <Activity>());
foreach (Activity activity in activities)
{
double quality = this.MatchQuality(descriptor, activity);
if (quality > 0)
{
sortedItems.Add(quality, activity);
}
}
count = Math.Min(count, sortedItems.NumItems);
List <Activity> top = new List <Activity>(count);
for (int i = 0; i < count; i++)
{
top.Add(sortedItems.GetValueAtIndex(i).Value);
}
return(top);
}
public void LoadFile()
{
foreach (var tile in new[] { "BJ", "UB", "UC" })
{
var path = $@"D:\Data\DEM\NZ\{tile}.asc";
var file = NzReader.Read(path);
Assert.AreEqual(8192, file.Columns);
Assert.AreEqual(8192, file.Rows);
Assert.AreEqual(-999, file.NoDataValue);
var list = new StatList();
var arr = file.Data;
foreach (var index in arr.Indices())
{
var value = arr[index];
if (value == file.NoDataValue)
{
foreach (var neighbour in arr.Neighbours(index))
{
var neighbourValue = arr[neighbour];
if (neighbourValue != file.NoDataValue)
{
list.Add(neighbourValue);
}
}
}
}
file.ReplaceNoDataValue(list.Average());
using (var stream = File.Create($@"D:\Data\DEM\NZ\{tile}.png"))
{
new ImageConverter().ToPng(file.Data, stream, HeightToGrayscale.FullRange8(file.Data));
}
}
}
public override void Finish()
{
this.log("Grouping participations by category");
foreach (KeyValuePair <String, Dictionary <Activity, int> > entry in this.feedbacks)
{
this.log("");
this.log("Instances of feedback " + entry.Key + ":");
// sort
StatList <int, Activity> sorted = new StatList <int, Activity>(new IntComparerer(), null);
foreach (KeyValuePair <Activity, int> e2 in entry.Value)
{
sorted.Add(e2.Value, e2.Key);
}
// Find the activities that were done often enough to be noteworthy
int max = sorted.GetLastValue().Key;
int cumulative = 0;
int minInterestingKey = 0;
List <ListItemStats <int, Activity> > interesting = new List <ListItemStats <int, Activity> >();
foreach (ListItemStats <int, Activity> stats in sorted.AllItems)
{
cumulative += stats.Key;
// Some activities were only done a very small number of times. We skip displaying those because they might be distracting
if (cumulative >= max)
{
interesting.Add(stats);
}
}
// sort the more commonly done activities to the top
interesting.Reverse();
foreach (ListItemStats <int, Activity> stats in interesting)
{
this.log(stats.Value + " : " + stats.Key + " times");
}
}
base.Finish();
}
public Participation_BinComparison_View(Engine engine, IEnumerable <Activity> activitiesToPredictFrom, Activity activityToPredict, TimeSpan windowSize)
{
DateTime now = DateTime.Now;
engine.EnsureRatingsAreAssociated();
LinearProgression progressionToPredict = activityToPredict.ParticipationsSmoothed((new TimeSpan()).Subtract(windowSize));
StatList <NeighborhoodInterpolation, Activity> results = new StatList <NeighborhoodInterpolation, Activity>(new Neighborhood_MiddleOutputMean_Comparer(), new NoopCombiner <Activity>());
foreach (Activity activity in activitiesToPredictFrom)
{
System.Diagnostics.Debug.WriteLine("comparing " + activity + " and " + activityToPredict.Name);
List <Datapoint> datapoints = activity.compareParticipations(windowSize, progressionToPredict, now.Subtract(windowSize));
// put data into the interpolator
FloatRange inputRange = new FloatRange(0, true, windowSize.TotalSeconds, true);
AdaptiveLinearInterpolator <Distribution> interpolator = new AdaptiveLinearInterpolator <Distribution>(new HyperBox <Distribution>(new FloatRange[] { inputRange }), new DistributionAdder());
foreach (Datapoint datapoint in datapoints)
{
interpolator.AddDatapoint(new AdaptiveInterpolation.Datapoint <Distribution>(datapoint.Input, Distribution.MakeDistribution(datapoint.Output, 0, datapoint.Weight)));
}
// ask the interpolator which input has the highest average output
IEnumerable <double[]> representativePoints = interpolator.FindRepresentativePoints();
if (representativePoints.Count() > 0)
{
double[] bestInput = new double[1];
AdaptiveInterpolation.Distribution bestOutput = null;
foreach (double[] coordinates in representativePoints)
{
AdaptiveInterpolation.Distribution output = interpolator.Interpolate(coordinates);
if (bestOutput == null || output.Mean > bestOutput.Mean)
{
bestInput = coordinates;
bestOutput = output;
}
}
// Check nearby regions for their outputs too
HyperBox <Distribution> inputNeighborhood = interpolator.FindNeighborhoodCoordinates(bestInput);
double inputNeighborhoodWidth = inputNeighborhood.Coordinates[0].Width;
double lowerInput = Math.Max(inputNeighborhood.Coordinates[0].LowCoordinate - inputNeighborhoodWidth, inputRange.LowCoordinate);
double higherInput = Math.Min(inputNeighborhood.Coordinates[0].HighCoordinate + inputNeighborhoodWidth, inputRange.HighCoordinate);
NeighborhoodInterpolation result = new NeighborhoodInterpolation();
result.Middle = new Interpolation(bestInput[0], bestOutput);
result.Left = new Interpolation(lowerInput, interpolator.Interpolate(new double[] { lowerInput }));
result.Right = new Interpolation(higherInput, interpolator.Interpolate(new double[] { higherInput }));
results.Add(result, activity);
}
}
IEnumerable <ListItemStats <NeighborhoodInterpolation, Activity> > resultList = results.AllItems;
if (resultList.Count() <= 0)
{
// This shouldn't be able to happen unless we disallow predicting the activity from itself
this.SubLayout = new TextblockLayout("No activities found!");
}
else
{
int numWindowDays = (int)windowSize.TotalDays;
GridLayout_Builder layoutBuilder = new Vertical_GridLayout_Builder().Uniform();
if (resultList.Count() > 1)
{
string title = "Activities that when done most strongly predict future participations in " + activityToPredict.Name;
layoutBuilder.AddLayout(new TextblockLayout(title));
}
else
{
string title = "Predicting future participations in " + activityToPredict.Name;
layoutBuilder.AddLayout(new TextblockLayout(title));
}
string explanation = "A block of the form a->{b,c,d} means that when you have spent an average of <a> min/day on the given activity over the last " + numWindowDays + " days, " +
"you have spent on average <c> min/day on " + activityToPredict.Name + " over the next " + numWindowDays + " days, with <b> and <d> marking -1 and +1 standard deviation, respectively";
layoutBuilder.AddLayout(new TextblockLayout(explanation));
int count = 0;
foreach (ListItemStats <NeighborhoodInterpolation, Activity> result in resultList.Reverse())
{
count++;
if (count > 3)
{
break;
}
Activity activity = result.Value;
NeighborhoodInterpolation neighborhood = result.Key;
layoutBuilder.AddLayout(new TextblockLayout("After " + activity.Name + ":"));
foreach (Interpolation interpolation in neighborhood.Items)
{
double inputSecondsPerWindow = interpolation.Input;
double inputMinutesPerDay = inputSecondsPerWindow / windowSize.TotalDays / 60;
double avgOutputSecondsPerWindow = interpolation.Output.Mean;
double avgOutputMinutesPerDay = avgOutputSecondsPerWindow / windowSize.TotalDays / 60;
double stddevOutputSecondsPerWindow = interpolation.Output.StdDev;
double stddevOutputMinutesPerDay = stddevOutputSecondsPerWindow / windowSize.TotalDays / 60;
string itemLine = Math.Round(inputMinutesPerDay, 1) + " -> {" +
Math.Round((avgOutputMinutesPerDay - stddevOutputMinutesPerDay), 1) + "," +
Math.Round(avgOutputMinutesPerDay, 1) + "," +
Math.Round((avgOutputMinutesPerDay + stddevOutputMinutesPerDay), 1) + "}";
layoutBuilder.AddLayout(new TextblockLayout(itemLine));
}
}
this.SubLayout = layoutBuilder.Build();
}
}
public ParticipationCorrelationView(Engine engine, IEnumerable <Activity> activitiesToPredictFrom, Activity activityToPredict, TimeSpan windowSize)
{
DateTime now = DateTime.Now;
engine.EnsureRatingsAreAssociated();
LinearProgression progressionToPredict = activityToPredict.ParticipationsSmoothed((new TimeSpan()).Subtract(windowSize));
StatList <double, Activity> results = new StatList <double, Activity>(new DoubleComparer(), new NoopCombiner <Activity>());
foreach (Activity activity in activitiesToPredictFrom)
{
System.Diagnostics.Debug.WriteLine("comparing " + activity + " and " + activityToPredict.Name);
List <Datapoint> datapoints = activity.compareParticipations(windowSize, progressionToPredict, now.Subtract(windowSize));
// now compute the value of the formula
Correlator correlator = new Correlator();
foreach (Datapoint datapoint in datapoints)
{
correlator.Add(datapoint);
}
double outputIncreasePerInputIncrease = correlator.Slope;
if (!double.IsNaN(outputIncreasePerInputIncrease))
{
results.Add(outputIncreasePerInputIncrease, activity);
}
}
IEnumerable <ListItemStats <double, Activity> > resultList = results.AllItems;
GridLayout_Builder layoutBuilder = new Vertical_GridLayout_Builder().Uniform();
List <ListItemStats <double, Activity> > mostPositivelyCorrelated = new List <ListItemStats <double, Activity> >();
List <ListItemStats <double, Activity> > mostNegativelyCorrelated = new List <ListItemStats <double, Activity> >();
int i = 0;
int numPositives = Math.Min(4, resultList.Count());
foreach (ListItemStats <double, Activity> result in resultList.Reverse())
{
mostPositivelyCorrelated.Add(result);
i++;
if (i > numPositives)
{
break;
}
}
i = 0;
int numNegatives = Math.Min(4, resultList.Count() - numPositives);
foreach (ListItemStats <double, Activity> result in resultList)
{
mostNegativelyCorrelated.Add(result);
i++;
if (i > numNegatives)
{
break;
}
}
if (resultList.Count() <= 0)
{
// This shouldn't be able to happen unless we disallow predicting the activity from itself
this.SubLayout = new TextblockLayout("No activities found!");
}
else
{
string title = "Things you do that are correlated with doing more or less of " + activityToPredict.Name + " over the following " +
Math.Round(windowSize.TotalDays, 0) + " days";
layoutBuilder.AddLayout(new TextblockLayout(title));
if (numPositives > 0)
{
if (numPositives > 1)
{
layoutBuilder.AddLayout(new TextblockLayout("Doing one minute of these activities adds this many minutes:"));
}
else
{
layoutBuilder.AddLayout(new TextblockLayout("Doing one minute of this activity adds this many minutes:"));
}
foreach (ListItemStats <double, Activity> result in mostPositivelyCorrelated)
{
double correlation = result.Key;
Activity activity = result.Value;
String message = activity.Name + ": " + Math.Round(correlation, 5);
layoutBuilder.AddLayout(new TextblockLayout(message));
}
}
if (numNegatives > 0)
{
if (numNegatives > 1)
{
layoutBuilder.AddLayout(new TextblockLayout("Doing one minute of these activities subtracts this many minutes:"));
}
else
{
layoutBuilder.AddLayout(new TextblockLayout("Doing one minute of this activity subtracts this many minutes:"));
}
foreach (ListItemStats <double, Activity> result in mostNegativelyCorrelated)
{
double correlation = result.Key;
Activity activity = result.Value;
String message = activity.Name + ": " + Math.Round(correlation, 5);
layoutBuilder.AddLayout(new TextblockLayout(message));
}
}
this.SubLayout = layoutBuilder.Build();
}
}