public static NodeServer <NodeType> Create(
Vector[] initVectors,
int numOfNodes,
int vectorLength,
GlobalVectorType globalVectorType,
ApproximationType approximation,
MonitoredFunction monitoredFunction,
ResolveNodesFunction <NodeType> resolveNodes,
Func <Vector, ConvexBound, int, int, NodeType> createNode)
{
initVectors = initVectors.Map(v => v.Clone());
var globalVector = globalVectorType.GetValue(initVectors);
var(lowerBound, upperBound) = approximation.Calc(monitoredFunction.Function(globalVector));
var upperConvexBound = monitoredFunction.UpperBound(globalVector, upperBound);
var lowerConvexBound = monitoredFunction.LowerBound(globalVector, lowerBound);
var upperNodes = ArrayUtils.Init(numOfNodes, i => createNode(globalVector.Clone(), upperConvexBound, i, vectorLength));
var lowerNodes = ArrayUtils.Init(numOfNodes, i => createNode(globalVector.Clone(), lowerConvexBound, i, vectorLength));
NodeServer <NodeType> ReCreate(Vector[] newInitVectors)
=> Create(newInitVectors, numOfNodes, vectorLength, globalVectorType, approximation, monitoredFunction, resolveNodes, createNode);
return(new NodeServer <NodeType>(initVectors, numOfNodes, vectorLength, globalVectorType, upperBound, lowerBound, monitoredFunction.Function,
approximation, upperNodes, lowerNodes, resolveNodes, ReCreate));
}
public ChartProperties(ThickeningType t, double v, ApproximationType at, Color c)
{
thickeningType = t;
thickeningValue = v;
approximationType = at;
color = c;
}
public static void RunRandomly(Random rnd, int numOfNodes, ApproximationType approximation, int vectorLength, int iterations,
string resultDir)
{
var windowSize = vectorLength * 2;
var stepSize = windowSize / 5;
var resultPath =
PathBuilder.Create(resultDir, "InnerProduct")
.AddProperty("Dataset", "Random")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Window", windowSize.ToString())
.AddProperty("Iterations", iterations.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
{
var innerProduct = new InnerProductFunction(vectorLength);
var initVectors = ArrayUtils.Init(numOfNodes, _ => ArrayUtils.Init(vectorLength, __ => (rnd.NextDouble() - 0.5) * numOfNodes).ToVector());
var multiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength,
approximation, innerProduct.MonitoredFunction);
for (int i = 0; i < iterations; i++)
{
var changes = ArrayUtils.Init(numOfNodes, index => index != 0 ? new Vector() : ArrayUtils.Init(vectorLength, _ => rnd.NextDouble() * numOfNodes / 10).ToVector());
multiRunner.Run(changes, rnd, true)
.Select(r => r.AsCsvString())
.ForEach((Action <string>)resultCsvFile.WriteLine);
}
}
}
public OracleServer(Vector[] nodesVectors, int numOfNodes, int vectorLength,
GlobalVectorType globalVectorType, double upperBound, double lowerBound,
Func <Vector, double> function, ApproximationType approximation)
: base(nodesVectors, numOfNodes, vectorLength, globalVectorType, upperBound, lowerBound, function,
approximation)
{
Init();
}
private void YInterpolationCheckBox_CheckedChanged(object sender, EventArgs e)
{
if (YInterpolationCheckBox.Checked == true)
{
approximationType = ApproximationType.Y_INTERPOLATION;
XInterpolationCheckBox.Checked = false;
YInterpolationCheckBox.Checked = true;
ZeroApprocsimationCheckBox.Checked = false;
}
}
public SimpleIteration(double Xo,
double Xn,
double Yo,
double Yn,
uint N,
uint M,
ApproximationType approximationType)
: base(Xo, Xn, Yo, Yn, N, M, approximationType)
{
}
private void ZeroApprocsimationCheckBox_CheckedChanged(object sender, EventArgs e)
{
if (ZeroApprocsimationCheckBox.Checked == true)
{
approximationType = ApproximationType.ZERO_APPROXIMATION;
XInterpolationCheckBox.Checked = false;
YInterpolationCheckBox.Checked = false;
ZeroApprocsimationCheckBox.Checked = true;
}
}
protected AbstractServer(Vector[] nodesVectors, int numOfNodes, int vectorLength, GlobalVectorType globalVectorType, double upperBound, double lowerBound, Func <Vector, double> function, ApproximationType approximation)
{
NodesVectors = nodesVectors;
NumOfNodes = numOfNodes;
VectorLength = vectorLength;
GlobalVectorType = globalVectorType;
UpperBound = upperBound;
LowerBound = lowerBound;
Function = function;
Approximation = approximation;
}
public ConjGradient(
double Xo,
double Xn,
double Yo,
double Yn,
uint N,
uint M,
ApproximationType approximationType)
: base(Xo, Xn, Yo, Yn, N, M, approximationType)
{
this.hvector = new double[N + 1u, M + 1u];
}
public AccumaltedResult(Communication communication, int numberOfFullSyncs, double functionValue, double upperBound, double lowerBound, double[] nodesFunctionValues, int loopIndex, ApproximationType approximation, int numOfNodes, int vectorLength, MonitoringScheme monitoringScheme)
{
Communication = communication;
NumberOfFullSyncs = numberOfFullSyncs;
FunctionValue = functionValue;
UpperBound = upperBound;
LowerBound = lowerBound;
NodesFunctionValues = nodesFunctionValues;
LoopIndex = loopIndex;
Approximation = approximation;
NumOfNodes = numOfNodes;
VectorLength = vectorLength;
MonitoringScheme = monitoringScheme;
}
public static OracleServer Create(
Vector[] initVectors,
int numOfNodes,
int vectorLength,
GlobalVectorType globalVectorType,
ApproximationType approximation,
MonitoredFunction monitoredFunction)
{
initVectors = initVectors.Map(v => v.Clone());
var globalVector = globalVectorType.GetValue(initVectors);
var(lowerBound, upperBound) = approximation.Calc(monitoredFunction.Function(globalVector));
return(new OracleServer(initVectors, numOfNodes, vectorLength, globalVectorType, upperBound, lowerBound, monitoredFunction.Function, approximation));
}
public NodeServer(Vector[] nodesVectors,
int numOfNodes, int vectorLength,
GlobalVectorType globalVectorType,
double upperBound, double lowerBound,
Func <Vector, double> function,
ApproximationType approximation, NodeType[] upperNodes,
NodeType[] lowerNodes,
ResolveNodesFunction <NodeType> resolveNodes,
Func <Vector[], NodeServer <NodeType> > reCreate)
: base(nodesVectors, numOfNodes, vectorLength, globalVectorType, upperBound, lowerBound, function,
approximation)
{
UpperNodes = upperNodes;
LowerNodes = lowerNodes;
ResolveNodes = resolveNodes;
ReCreate = reCreate;
}
public static void RunCTU(Random rnd, int maxIterations, int numOfNodes, int window, int collapseDimension,
ApproximationType approximation, string ctuBinaryPath, string resultDir)
{
var resultPath =
PathBuilder.Create(resultDir, "EntropySketch")
.AddProperty("Dataset", "CTU")
.AddProperty("SketchDimension", collapseDimension.ToString())
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("Window", window.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
var entropySketch = new EntropySketchFunction(collapseDimension);
var header = string.Join(",", Enumerable.Range(1, collapseDimension).Select(i => "y" + i)) + "," +
string.Join(",", Enumerable.Range(1, numOfNodes).Select(i => "server_max_" + i));
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes) + ",Entropy," + header))
using (var ctuProbabilityWindow = CtuProbabilityWindow.Init(ctuBinaryPath, numOfNodes, window))
{
var initProbabilityVectors = ctuProbabilityWindow.CurrentProbabilityVector().Map(entropySketch.CollapseProbabilityVector);
var multiRunner = MultiRunner.InitAll(initProbabilityVectors, numOfNodes, collapseDimension,
approximation, entropySketch.MonitoredFunction);
//multiRunner.OnlySchemes(new MonitoringScheme.Oracle());
int i = 0;
while (ctuProbabilityWindow.MoveNext() && (i++ < maxIterations))
{
var entropy = Vector.AverageVector(ctuProbabilityWindow.CurrentProbabilityVector())
.IndexedValues.Select(p => p.Value).Sum(v => - v * Math.Log(v));
var changeProbabilityVectors = ctuProbabilityWindow.CurrentChangeProbabilityVector()
.Map(entropySketch.CollapseProbabilityVector);
multiRunner.Run(changeProbabilityVectors, rnd, false)
.Select(r =>
{
return(r.AsCsvString());
//var oracle = multiRunner.Runners.Values.OfType<MonitoringRunner<OracleServer>>().First().Server;
//var nodesMax = string.Join(",", oracle.NodesVectors.Select(v => v[v.MaximumIndex()]));
//var res = string.Join(",",oracle.GlobalVector.Enumerate(collapseDimension).Select(y => y.ToString()));
//return r.AsCsvString() + "," + entropy.ToString() + "," + res + "," + nodesMax;
})
.ForEach(resultCsvFile.WriteLine);
}
}
}
public static void RunMilanoPhoneActivity(Random rnd, int numOfNodes, int window,
ApproximationType approximation, int width,
int height, GeographicalDistributing distributingMethod,
string phoneActivityDir, string resultDir)
{
var vectorLength = width * height;
var hashFunction = FourwiseIndepandantFunction.Init(rnd);
var hashFunctionsTable = HashFunctionTable.Init(numOfNodes, vectorLength, hashFunction);
var secondMomentFunction = new SecondMoment(width, height);
var resultPath =
PathBuilder.Create(resultDir, "AMS_F2")
.AddProperty("Dataset", "MilanoPhoneActivity")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Width", width.ToString())
.AddProperty("Height", height.ToString())
.AddProperty("Window", window.ToString())
.AddProperty("Distributing", distributingMethod.Name)
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes) + ",F2"))
{
var phonesActivityDataParser = PhonesActivityDataParser.Create(phoneActivityDir);
var phonesActivityWindowManger = PhonesActivityWindowManger.Init(window, numOfNodes, vectorLength, hashFunctionsTable, phonesActivityDataParser, distributingMethod);
var initVectors = phonesActivityWindowManger.GetCurrentVectors();
var multiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength, approximation, secondMomentFunction.MonitoredFunction);
while (phonesActivityWindowManger.TakeStep())
{
var shouldEnd = new StrongBox <bool>(false);
var changeVectors = phonesActivityWindowManger.GetChangeVector();
// var sumVector = Vector.SumVector(phonesActivityWindowManger.Window.Value.CurrentNodesCountVectors());
//var f2Value = sumVector.IndexedValues.Values.Sum(x => x * x);
multiRunner.Run(changeVectors, rnd, false)
// .SideEffect(a => shouldEnd.Value = shouldEnd.Value || (a.MonitoringScheme is MonitoringScheme.Oracle && a.NumberOfFullSyncs > 0))
.Select(r => r.AsCsvString()) //+ "," + f2Value)
.ForEach(resultCsvFile.WriteLine);
//if (shouldEnd.Value)
// break;
}
}
}
public static void RunDatabaseAccesses(Random rnd, int numOfNodes,
int window,
ApproximationType approximation, int vectorLength,
UsersDistributing distributing,
string databaseAccessesPath, string resultDir)
{
var resultPath =
PathBuilder.Create(resultDir, "Entropy")
.AddProperty("Dataset", "DatabaseAccesses")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("Window", window.ToString())
.AddProperty("Approximation", approximation.AsString())
.AddProperty("DistributingMethod", distributing.Name)
.ToPath("csv");
var hashUser = new Func <int, int>(userId => userId % numOfNodes);
var entropy = new EntropyFunction(vectorLength);
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
using (var databaseAccessesStatistics = DatabaseAccessesStatistics.Init(databaseAccessesPath, numOfNodes, window, distributing.DistributeFunc))
{
var initProbabilityVectors = databaseAccessesStatistics.InitProbabilityVectors();
if (!initProbabilityVectors.All(v => v.Sum().AlmostEqual(1.0, 0.000001)))
{
throw new Exception();
}
var multiRunner = MultiRunner.InitAll(initProbabilityVectors, numOfNodes, vectorLength,
approximation, entropy.MonitoredFunction);
while (databaseAccessesStatistics.TakeStep())
{
var changeProbabilityVectors = databaseAccessesStatistics.GetChangeProbabilityVectors();
if (!changeProbabilityVectors.All(v => v.Sum().AlmostEqual(0.0, 0.000001)))
{
throw new Exception();
}
multiRunner.Run(changeProbabilityVectors, rnd, true)
.Select(r => r.AsCsvString())
.ForEach(resultCsvFile.WriteLine);
}
}
}
public static void RunStocks(Random rnd, MonitoredFunction function, string cbName, int iterations, Tree <long> closestValueQuery, int numOfNodes, int window, DateTime startingDateTime,
int minAmountAtDay, ApproximationType approximation,
string stocksDirPath, string resultDir)
{
var vectorLength = closestValueQuery.Data.Length;
var resultPath =
PathBuilder.Create(resultDir, "Entropy_" + cbName)
.AddProperty("Dataset", "Stocks")
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("Window", window.ToString())
.AddProperty("StartingTime", startingDateTime.ToShortDateString().Replace('/', '-'))
.AddProperty("MinAmountAtDay", minAmountAtDay.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
using (var stocksProbabilityWindow = StocksProbabilityWindow.Init(stocksDirPath, startingDateTime, minAmountAtDay, numOfNodes, window, closestValueQuery))
{
var initProbabilityVectors = stocksProbabilityWindow.CurrentProbabilityVector();
if (!initProbabilityVectors.All(v => v.Sum().AlmostEqual(1.0, 0.000001)))
{
throw new Exception();
}
var multiRunner = MultiRunner.InitAll(initProbabilityVectors, numOfNodes, vectorLength,
approximation, function);
int i = 0;
while (stocksProbabilityWindow.MoveNext() && (i++ < iterations))
{
var changeProbabilityVectors = stocksProbabilityWindow.CurrentChangeProbabilityVector();
if (!changeProbabilityVectors.All(v => v.Sum().AlmostEqual(0.0, 0.000001)))
{
throw new Exception();
}
multiRunner.Run(changeProbabilityVectors, rnd, false)
.Select(r => r.AsCsvString())
.ForEach(resultCsvFile.WriteLine);
}
}
}
public static void RunBagOfWords(Random rnd, int vectorLength, string wordsPath,
ApproximationType approximation,
string resultDir,
Func <int, bool> isLeft, string[] textFilesPathes)
{
var numOfNodes = textFilesPathes.Length;
var windowSize = 20000;
var amountOfIterations = 2000;
var stepSize = 1000;
var halfVectorLength = vectorLength / 2;
var optionalWords = File.ReadLines(wordsPath).Take(halfVectorLength).ToArray();
var optionalStrings = new SortedSet <string>(optionalWords, StringComparer.OrdinalIgnoreCase);
var resultPath =
PathBuilder.Create(resultDir, "InnerProduct")
.AddProperty("Dataset", "BagOfWords")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Window", windowSize.ToString())
.AddProperty("Iterations", amountOfIterations.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
using (var stringDataParser =
TextParser <string> .Init(StreamReaderUtils.EnumarateWords, windowSize, optionalStrings,
textFilesPathes))
{
var innerProduct = new InnerProductFunction(vectorLength);
var initVectors = stringDataParser.Histograms.Map(h => h.CountVector())
.PadWithZeros(halfVectorLength, isLeft);
var multiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength,
approximation, innerProduct.MonitoredFunction);
var changes = stringDataParser.AllCountVectors(stepSize)
.Select(ch => ch.PadWithZeros(halfVectorLength, isLeft))
.Take(amountOfIterations);
multiRunner.RunAll(changes, rnd, true)
.Select(r => r.AsCsvString())
.ForEach((Action <string>)resultCsvFile.WriteLine);
}
}
public void Init(
double Xo, double Xn, double Yo, double Yn,
uint N, uint M, ApproximationType approximationType)
{
this.data = new double[N + 1u, M + 1u];
this.dataPrev = new double[N + 1u, M + 1u];
this.function = new double[N + 1u, M + 1u];
this.residual = new double[N + 1, M + 1];
this.Xo = Xo;
this.Yo = Yo;
this.Xn = Xn;
this.Yn = Yn;
this.N = N;
this.M = M;
this.h = (this.Xn - this.Xo) / N;
this.k = (this.Yn - this.Yo) / M;
this.h2 = -Math.Pow(N / (Xn - Xo), 2);
this.k2 = -Math.Pow(M / (Yn - Yo), 2);
this.a2 = -2.0 * (h2 + k2);
this.approximationType = approximationType;
InitMethod();
}
public static void RunDatabaseAccesses(Random rnd, int numOfNodes, int window,
ApproximationType approximation, int width,
int height, UsersDistributing distributing,
string databaseAccessesPath, string resultDir)
{
var vectorLength = width * height;
var hashFunction = FourwiseIndepandantFunction.Init(rnd);
var hashFunctionsTable = HashFunctionTable.Init(numOfNodes, vectorLength, hashFunction);
var secondMomentFunction = new SecondMoment(width, height);
var resultPath =
PathBuilder.Create(resultDir, "AMS_F2")
.AddProperty("Dataset", "DatabaseAccesses")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Width", width.ToString())
.AddProperty("Height", height.ToString())
.AddProperty("Window", window.ToString())
.AddProperty("Distributing", distributing.Name)
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
using (var databaseAccessesStatistics = DatabaseAccessesStatistics.Init(databaseAccessesPath, numOfNodes, window, distributing.DistributeFunc))
{
var initCountVectors = databaseAccessesStatistics.InitCountVectors();
var initVectors = hashFunction.TransformToAMSSketch(initCountVectors, vectorLength, hashFunctionsTable);
var multiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength,
approximation, secondMomentFunction.MonitoredFunction);
while (databaseAccessesStatistics.TakeStep())
{
var changeCountVectors = databaseAccessesStatistics.GetChangeCountVectors();
var changeVectors = hashFunction.TransformToAMSSketch(changeCountVectors, vectorLength, hashFunctionsTable);
multiRunner.Run(changeVectors, rnd, true)
.Select(r => r.AsCsvString())
.ForEach(resultCsvFile.WriteLine);
}
}
}
public static void RunStocks(Random rnd, int iterations, Tree <long> closestValueQuery, int numOfNodes, int window, int collapseDimension, DateTime startingDateTime,
int minAmountAtDay, ApproximationType approximation,
string stocksDirPath, string resultDir)
{
var vectorLength = closestValueQuery.Data.Length;
var resultPath =
PathBuilder.Create(resultDir, "EntropySketch")
.AddProperty("Dataset", "Stocks")
.AddProperty("BucketLength", vectorLength.ToString())
.AddProperty("SketchDimension", collapseDimension.ToString())
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("Window", window.ToString())
.AddProperty("StartingTime", startingDateTime.ToShortDateString().Replace('/', '-'))
.AddProperty("MinAmountAtDay", minAmountAtDay.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
var entropySketch = new EntropySketchFunction(collapseDimension);
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes) + ",Entropy"))
using (var stocksProbabilityWindow = StocksProbabilityWindow.Init(stocksDirPath, startingDateTime, minAmountAtDay, numOfNodes, window, closestValueQuery))
{
var initProbabilityVectors = stocksProbabilityWindow.CurrentProbabilityVector().Map(entropySketch.CollapseProbabilityVector);
var multiRunner = MultiRunner.InitAll(initProbabilityVectors, numOfNodes, collapseDimension,
approximation, entropySketch.MonitoredFunction);
int i = 0;
while (stocksProbabilityWindow.MoveNext() && (i++ < iterations))
{
var changeProbabilityVectors = stocksProbabilityWindow.CurrentChangeProbabilityVector()
.Map(entropySketch.CollapseProbabilityVector);
var entropy = Vector.AverageVector(stocksProbabilityWindow.CurrentProbabilityVector())
.IndexedValues.Values.Sum(p => - p * Math.Log(p));
multiRunner.Run(changeProbabilityVectors, rnd, false)
.Select(r => r.AsCsvString() + "," + entropy)
.ForEach(resultCsvFile.WriteLine);
}
}
}
public StatisticsOption(ApproximationType at, ThickeningType tt, Double v)
{
ApproximationType = at;
ThickeningType = tt;
ThickeningValue = v;
}
public StatisticsOption(ApproximationType at, ThickeningType tt, int v)
{
ApproximationType = at;
ThickeningType = tt;
ThickeningValue = v;
}
public static void Run(Random rnd, int numOfNodes, int vectorLength, int iterations, ApproximationType approximation, string resultDir)
{
var resultPath =
PathBuilder.Create(resultDir, "Sphere")
.AddProperty("Data", "Random")
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Iterations", iterations.ToString())
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
Vector[] GetChange()
{
Vector GenerateChange()
{
return(ArrayUtils.Init(vectorLength, i => (rnd.NextDouble() - 0.5) / 5).ToVector());
}
// return ArrayUtils.Init(numOfNodes, GenerateChange() : ArrayUtils.Init(vectorLength, _ => 0.0).ToVector());
return(ArrayUtils.Init(numOfNodes, _ => GenerateChange()));
}
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
{
var sphereFunction = new SphereFunction(vectorLength);
var initVectors = Vector.Init(numOfNodes);
var multiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength,
approximation, sphereFunction.MonitoredFunction);
multiRunner.OnlySchemes(new MonitoringScheme.Value(), new MonitoringScheme.Distance(2), new MonitoringScheme.Oracle(), new MonitoringScheme.Vector());
for (int i = 0; i < iterations; i++)
{
multiRunner.Run(GetChange(), rnd, false)
.Select(r => r.AsCsvString())
.ForEach((Action <string>)resultCsvFile.WriteLine);
}
}
}
public static void RunTaxiTrips(Random random, int iterations, int sqrtNumOfNodes, int hoursInWindow, ApproximationType approximation, int sqrtVectorLength, DataSplitter <TaxiTripEntry> splitter, CityRegion cityRegion, string taxiBinDataPath, string resultDir)
{
var vectorLength = 2 * sqrtVectorLength * sqrtVectorLength;
var numOfNodes = sqrtNumOfNodes * sqrtNumOfNodes;
var resultPath =
PathBuilder.Create(resultDir, "InnerProduct")
.AddProperty("Dataset", "TaxiTrips")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Window", hoursInWindow.ToString())
.AddProperty("Iterations", iterations.ToString())
.AddProperty("Approximation", approximation.AsString())
.AddProperty("DataSplit", splitter.Name)
.ToPath("csv");
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
using (var binaryReader = new BinaryReader(File.OpenRead(taxiBinDataPath)))
{
var innerProductFunction = new InnerProductFunction(vectorLength);
var taxiTrips = TaxiTripEntry.FromBinary(binaryReader);
var windowManager = TaxiTripsWindowManger.Init(hoursInWindow, sqrtNumOfNodes, sqrtVectorLength, splitter, cityRegion, taxiTrips);
var initVectors = windowManager.GetCurrentVectors();
var multiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength, approximation, innerProductFunction.MonitoredFunction);
for (int i = 0; i < iterations; i++)
{
if (!windowManager.TakeStep())
{
break;
}
var changeVectors = windowManager.GetChangeVector();
multiRunner.Run(changeVectors, random, false)
.Select(r => r.AsCsvString())
.ForEach(resultCsvFile.WriteLine);
}
}
}
public static AccumaltedResult Init(ApproximationType approximation, int numOfNodes, int vectorLength, MonitoringScheme monitoringScheme)
=> new AccumaltedResult(Communication.Zero, 0, 0, 0, 0, new[] { 0.0 }, 0, approximation, numOfNodes, vectorLength, monitoringScheme);
public static void RunRandomly(Random rnd, int numOfNodes, ApproximationType approximation, int vectorLength, int collapeDimension, int iterations, bool oneChanges,
string resultDir)
{
var entropyResultPath =
PathBuilder.Create(resultDir, "Entropy")
.AddProperty("Dataset", "Random")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("OneChanges", oneChanges.ToString())
.AddProperty("Iterations", iterations.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
var sketchEntropyResultPath =
PathBuilder.Create(resultDir, "SketchEntropy")
.AddProperty("Dataset", "Random")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", collapeDimension.ToString())
.AddProperty("OneChanges", oneChanges.ToString())
.AddProperty("Iterations", iterations.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
using (var entropyResultCsvFile = AutoFlushedTextFile.Create(entropyResultPath, AccumaltedResult.Header(numOfNodes)))
using (var sketchEntropyResultCsvFile = AutoFlushedTextFile.Create(sketchEntropyResultPath, AccumaltedResult.Header(numOfNodes)))
{
var sqrt = (int)Math.Sqrt(vectorLength);
var entropyFunction = new EntropyFunction(vectorLength);
var entropySketchFunction = new EntropySketchFunction(collapeDimension);
Func <int, Vector> genVector = node => ArrayUtils.Init(vectorLength, rnd.NextDouble())
.ToVector().NormalizeInPlaceToSum(1.0);
var initVectors = ArrayUtils.Init(numOfNodes, genVector);
var sketchInitVectors = initVectors.Map(entropySketchFunction.CollapseProbabilityVector);
var globalVector = Vector.AverageVector(initVectors);
Vector getChangeVector(int node)
{
if (oneChanges && node >= 1)
{
return(new Vector());
}
var someVector = genVector(node);
Vector changeVector = someVector - globalVector;
if (!oneChanges)
{
changeVector.DivideInPlace(numOfNodes);
}
globalVector.AddInPlace(changeVector / numOfNodes);
return(changeVector);
}
var entropyMultiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength,
approximation, entropyFunction.MonitoredFunction);
var sketchEntropyMultiRunner = MultiRunner.InitAll(sketchInitVectors, numOfNodes, vectorLength,
approximation, entropySketchFunction.MonitoredFunction);
var shouldStop = new StrongBox <bool>(false);
for (int i = 0; i < iterations; i++)
{
var changes = ArrayUtils.Init(numOfNodes, getChangeVector);
var sketchChanges = changes.Map(entropySketchFunction.CollapseProbabilityVector);
//entropyMultiRunner.Run(changes, rnd, false)
//.SideEffect(r => shouldStop.Value = shouldStop.Value || (r.MonitoringScheme is MonitoringScheme.Oracle && r.NumberOfFullSyncs >= 8))
//.Select(r => r.AsCsvString())
//.ForEach((Action<string>)entropyResultCsvFile.WriteLine);
sketchEntropyMultiRunner.Run(sketchChanges, rnd, false)
.SideEffect(r => shouldStop.Value = shouldStop.Value || (r.MonitoringScheme is MonitoringScheme.Oracle && r.NumberOfFullSyncs >= 8))
.Select(r => r.AsCsvString())
.ForEach((Action <string>)sketchEntropyResultCsvFile.WriteLine);
if (shouldStop.Value)
{
break;
}
}
}
}
public static HashSet <Location> GenerateAreaMask(float firstSize, float secondSize, AreaMaskShapeType areaMaskShapeType, ApproximationType approximationType, float roundThreshold = 0.5f)
{
HashSet <Location> areaMask = new HashSet <Location>();
switch (areaMaskShapeType)
{
case AreaMaskShapeType.Vertical:
(firstSize, secondSize - 1).For(i => areaMask.Add(new Location(0, i)));
break;
case AreaMaskShapeType.Horizontal:
(firstSize, secondSize - 1).For(i => areaMask.Add(new Location(i, 0)));
break;
case AreaMaskShapeType.Cross:
(-firstSize, firstSize).For(i => areaMask.Add(new Location(0, i)));
(-secondSize, secondSize).For(i => areaMask.Add(new Location(i, 0)));
break;
case AreaMaskShapeType.Rectangle:
(-firstSize, firstSize)
.For(x =>
secondSize
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
break;
case AreaMaskShapeType.Square:
(-firstSize, firstSize).For(x =>
(-secondSize, secondSize).For(y =>
{
areaMask.Add(new Location(x, y));
}));
break;
case AreaMaskShapeType.Rhombus:
switch (approximationType)
{
case ApproximationType.Round:
(-firstSize, firstSize)
.For(x =>
(-(secondSize * (1f - Mathf.Abs(x) / firstSize)).Round(roundThreshold),
(secondSize * (1f - (Mathf.Abs(x) / firstSize))).Round(roundThreshold))
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(-secondSize, secondSize)
.For(y =>
(-(firstSize * (1f - Mathf.Abs(y) / secondSize)).Round(roundThreshold),
(firstSize * (1f - (Mathf.Abs(y) / secondSize))).Round(roundThreshold))
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
case ApproximationType.Convex:
(-firstSize, firstSize)
.For(x =>
(-(secondSize * (1f - Mathf.Abs(x) / firstSize)).Ceil(),
(secondSize * (1f - (Mathf.Abs(x) / firstSize))).Ceil())
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(-secondSize, secondSize)
.For(y =>
(-(firstSize * (1f - Mathf.Abs(y) / secondSize)).Ceil(),
(firstSize * (1f - (Mathf.Abs(y) / secondSize))).Ceil())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
case ApproximationType.Concave:
(-firstSize, firstSize)
.For(x =>
(-(secondSize * (1f - Mathf.Abs(x) / firstSize)).Floor(),
(secondSize * (1f - (Mathf.Abs(x) / firstSize))).Floor())
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(-secondSize, secondSize)
.For(y =>
(-(firstSize * (1f - Mathf.Abs(y) / secondSize)).Floor(),
(firstSize * (1f - (Mathf.Abs(y) / secondSize))).Floor())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
}
break;
case AreaMaskShapeType.Circle:
switch (approximationType)
{
case ApproximationType.Round:
(-firstSize, firstSize)
.For(x =>
(-(secondSize * (1f - (x / firstSize).Square()).Sqrt()).Round(roundThreshold),
(secondSize * (1f - (x / firstSize).Square()).Sqrt()).Round(roundThreshold))
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(-secondSize, secondSize)
.For(y =>
(-(firstSize * (1f - (y / secondSize).Square()).Sqrt()).Round(roundThreshold),
(firstSize * (1f - (y / secondSize).Square()).Sqrt()).Round(roundThreshold))
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
case ApproximationType.Convex:
(-firstSize, firstSize)
.For(x =>
(-(secondSize * (1f - (x / firstSize).Square()).Sqrt()).Ceil(),
(secondSize * (1f - (x / firstSize).Square()).Sqrt()).Ceil())
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(-secondSize, secondSize)
.For(y =>
(-(firstSize * (1f - (y / secondSize).Square()).Sqrt()).Ceil(),
(firstSize * (1f - (y / secondSize).Square()).Sqrt()).Ceil())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
case ApproximationType.Concave:
(-firstSize, firstSize)
.For(x =>
(-(secondSize * (1f - (x / firstSize).Square()).Sqrt()).Floor(),
(secondSize * (1f - (x / firstSize).Square()).Sqrt()).Floor())
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(-secondSize, secondSize)
.For(y =>
(-(firstSize * (1f - (y / secondSize).Square()).Sqrt()).Floor(),
(firstSize * (1f - (y / secondSize).Square()).Sqrt()).Floor())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
}
break;
case AreaMaskShapeType.Arc:
switch (approximationType)
{
case ApproximationType.Round:
(-firstSize, firstSize)
.For(x =>
(((secondSize.Square() - firstSize.Square()).Sqrt() / firstSize * x.Abs()).Round(1f - roundThreshold),
(secondSize.Square() - x.Square()).Sqrt().Round(roundThreshold))
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(secondSize.Square() - firstSize.Square()).Sqrt()
.For(y =>
(-(firstSize / secondSize * y).Round(roundThreshold),
(firstSize / secondSize * y).Round(roundThreshold))
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
((secondSize.Square() - firstSize.Square()).Sqrt(), secondSize)
.For(y =>
(-(secondSize.Square() - y.Square()).Sqrt().Round(roundThreshold),
(secondSize.Square() - y.Square()).Sqrt().Round(roundThreshold))
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
case ApproximationType.Convex:
(-firstSize, firstSize)
.For(x =>
(((secondSize.Square() - firstSize.Square()).Sqrt() / firstSize * x.Abs()).Floor(),
(secondSize.Square() - x.Square()).Sqrt().Ceil())
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(secondSize.Square() - firstSize.Square()).Sqrt()
.For(y =>
(-(firstSize / secondSize * y).Ceil(),
(firstSize / secondSize * y).Ceil())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
((secondSize.Square() - firstSize.Square()).Sqrt(), secondSize)
.For(y =>
(-(secondSize.Square() - y.Square()).Sqrt().Ceil(),
(secondSize.Square() - y.Square()).Sqrt().Ceil())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
case ApproximationType.Concave:
(-firstSize, firstSize)
.For(x =>
(((secondSize.Square() - firstSize.Square()).Sqrt() / firstSize * x.Abs()).Ceil(),
(secondSize.Square() - x.Square()).Sqrt().Floor())
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
(secondSize.Square() - firstSize.Square()).Sqrt()
.For(y =>
(-(firstSize / secondSize * y).Floor(),
(firstSize / secondSize * y).Floor())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
((secondSize.Square() - firstSize.Square()).Sqrt(), secondSize)
.For(y =>
(-(secondSize.Square() - y.Square()).Sqrt().Floor(),
(secondSize.Square() - y.Square()).Sqrt().Floor())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
}
break;
case AreaMaskShapeType.Cone:
switch (approximationType)
{
case ApproximationType.Round:
(-firstSize, firstSize)
.For(x =>
((secondSize / firstSize * x.Abs()).Round(roundThreshold), secondSize - 1)
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
secondSize
.For(y =>
(-(firstSize / secondSize * y).Round(roundThreshold),
(firstSize / secondSize * y).Round(roundThreshold))
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
case ApproximationType.Convex:
(-firstSize, firstSize)
.For(x =>
((secondSize / firstSize * x.Abs()).Floor(), secondSize - 1)
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
secondSize
.For(y =>
(-(firstSize / secondSize * y).Ceil(),
(firstSize / secondSize * y).Ceil())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
case ApproximationType.Concave:
(-firstSize, firstSize)
.For(x =>
((secondSize / firstSize * x.Abs()).Ceil(), secondSize - 1)
.For(y =>
{
areaMask.Add(new Location(x, y));
}));
secondSize
.For(y =>
(-(firstSize / secondSize * y).Floor(),
(firstSize / secondSize * y).Floor())
.For(x =>
{
areaMask.Add(new Location(x, y));
}));
break;
}
break;
}
return(areaMask);
}
public Method(
double Xo, double Xn, double Yo, double Yn,
uint N, uint M, ApproximationType approximationType)
{
Init(Xo, Xn, Yo, Yn, N, M, approximationType);
}
public static void RunRandomly(Random rnd, int width, int height, int numOfNodes, int iterations, ApproximationType approximation, bool oneChanges,
string resultDir)
{
var vectorLength = width * height;
var resultPath =
PathBuilder.Create(resultDir, "AMS_F2")
.AddProperty("Dataset", "Random")
.AddProperty("OneChanges", oneChanges.ToString())
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Width", width.ToString())
.AddProperty("Height", height.ToString())
.AddProperty("Iterations", iterations.ToString())
.AddProperty("Approximation", approximation.AsString())
.ToPath("csv");
var secondMomentFunction = new SecondMoment(width, height);
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
{
var initVectors = ArrayUtils.Init(numOfNodes, _ => ArrayUtils.Init(vectorLength, __ => (double)rnd.Next(-4, 5)).ToVector());
var multiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength, approximation, secondMomentFunction.MonitoredFunction);
// multiRunner.OnlySchemes(new MonitoringScheme.Value(), new MonitoringScheme.FunctionMonitoring(), new MonitoringScheme.Oracle());
const int OracleFullSyncs = 2;
Func <int, Vector> ChangeGenerator() => nodeIndex =>
{
if (oneChanges)
{
if (nodeIndex != 0)
{
return(new Vector());
}
return(ArrayUtils.Init(vectorLength, _ => (rnd.NextDouble() - 0.5) * numOfNodes / 5).ToVector());
}
else
{
return(ArrayUtils.Init(vectorLength, _ => (rnd.NextDouble() - 0.5) / 5).ToVector());
}
};
for (int i = 0; i < iterations; i++)
{
var changes = ArrayUtils.Init(numOfNodes, ChangeGenerator());
var stop = new StrongBox <bool>(false);
multiRunner.Run(changes, rnd, false)
.SideEffect(r => stop.Value = stop.Value || (r.MonitoringScheme.Equals(new MonitoringScheme.Oracle()) && r.NumberOfFullSyncs > OracleFullSyncs))
.Select(r => r.AsCsvString())
.ForEach(resultCsvFile.WriteLine);
if (stop.Value)
{
break;
}
}
}
}
