public void TestBinary()
{
using (var memmoryStream = new MemoryStream(new byte[1024], true))
{
var taxiTrip = new TaxiTripEntry(1f, 2f, 3f, 4f, 5, DateTime.Now, DateTime.UtcNow, PaymentType.Credit,
TaxiVendor.CMT, 22f, 33f, 44f, 55f, 66f);
using (var binaryWriter = new BinaryWriter(memmoryStream))
using (var binaryReader = new BinaryReader(memmoryStream))
{
taxiTrip.ToBinary(binaryWriter);
binaryWriter.Flush();
memmoryStream.Position = 0;
var deserialized = TaxiTripEntry.FromBinary(binaryReader).First();
Assert.AreEqual(taxiTrip.PaymentType, deserialized.PaymentType);
Assert.AreEqual(taxiTrip.PickupTime, deserialized.PickupTime);
Assert.AreEqual(taxiTrip.DropoffLatitude, deserialized.DropoffLatitude);
Assert.AreEqual(taxiTrip.DropoffLongtitude, deserialized.DropoffLongtitude);
Assert.AreEqual(taxiTrip.DropoffTime, deserialized.DropoffTime);
Assert.AreEqual(taxiTrip.FareAmount, deserialized.FareAmount);
Assert.AreEqual(taxiTrip.NumOfPassangers, deserialized.NumOfPassangers);
Assert.AreEqual(taxiTrip.PickupLatitude, deserialized.PickupLatitude);
Assert.AreEqual(taxiTrip.PickupLongtitude, deserialized.PickupLongtitude);
Assert.AreEqual(taxiTrip.TaxiVendor, deserialized.TaxiVendor);
Assert.AreEqual(taxiTrip.Tip, deserialized.Tip);
Assert.AreEqual(taxiTrip.TotalPayment, deserialized.TotalPayment);
}
}
}
public void TestStringParsing()
{
var dataCsvLine = "2013000002,2013000002,VTS,1,,2013-01-01 00:00:00,2013-01-01 00:06:00,5,360,.98,-73.978325,40.778091,-73.981834,40.768639";
var fareCsvLine = "2013000002,2013000002,VTS,2013-01-01 00:00:00,CSH,6,0.5,0.5,0,0,7";
var result = TaxiTripEntry.TryParse(dataCsvLine, fareCsvLine);
Assert.IsTrue(result.IsSome(out TaxiTripEntry taxiTrip));
Assert.AreEqual(taxiTrip.PaymentType, PaymentType.Cash);
Assert.AreEqual(taxiTrip.PickupTime, DateTime.Parse("2013-01-01 00:00:00"));
Assert.AreEqual(taxiTrip.DropoffLatitude, 40.768639f, 0.000001f);
Assert.AreEqual(taxiTrip.DropoffLongtitude, -73.9818340f, 0.000001f);
Assert.AreEqual(taxiTrip.DropoffTime, DateTime.Parse("2013-01-01 00:06:00"));
Assert.AreEqual(taxiTrip.FareAmount, 6);
Assert.AreEqual(taxiTrip.NumOfPassangers, 5);
Assert.AreEqual(taxiTrip.PickupLatitude, 40.778091f, 0.000001f);
Assert.AreEqual(taxiTrip.PickupLongtitude, -73.978325f, 0.000001f);
Assert.AreEqual(taxiTrip.TaxiVendor, TaxiVendor.VTS);
Assert.AreEqual(taxiTrip.Tip, 0);
Assert.AreEqual(taxiTrip.TotalPayment, 7);
}
public static void Main(string[] args)
{
const int cacheSize = 1000;
var baseDir = @"C:\Users\Yuval\Desktop\Data\Taxi Data\Good Data\FOIL2013";
var outputFile = @"C:\Users\Yuval\Desktop\Data\Taxi Data\Good Data\FOIL2013\result.bin";
/*using (var read = new BinaryReader(File.OpenRead(outputFile)))
* {
* foreach (var taxiTripEntry in TaxiTripEntry.FromBinary(read))
* {
* Console.WriteLine(taxiTripEntry.DropoffTime.ToString("F"));
* }
* }
* return;*/
using (var binOutputFile = File.Create(outputFile, 4096, FileOptions.SequentialScan))
using (var binaryWriter = new BinaryWriter(binOutputFile))
using (var cachedWriter = CachedMinWriter <TaxiTripEntry> .Create(cacheSize, t => t.ToBinary(binaryWriter)))
foreach (var innerDir in Directory.EnumerateDirectories(baseDir).OrderBy(x => x))
{
var files = Directory.EnumerateFiles(innerDir).ToArray();
var dataCsv = files.First(p => Path.GetFileNameWithoutExtension(p).Contains("data"));
var fareCsv = files.First(p => Path.GetFileNameWithoutExtension(p).Contains("fare"));
using (var dataCsvFile = File.OpenText(dataCsv))
using (var fareCsvFile = File.OpenText(fareCsv))
{
while (true)
{
if (dataCsvFile.EndOfStream || fareCsvFile.EndOfStream)
{
break;
}
TaxiTripEntry.TryParse(dataCsvFile.ReadLine(), fareCsvFile.ReadLine())
.Iter(cachedWriter.WriteItem);
}
}
}
}
public static void RunTaxiTrips(Random random, int iterations, int sqrtNumOfNodes, int hoursInWindow, ApproximationType approximation, int sqrtVectorLength, DataSplitter <TaxiTripEntry> splitter, CityRegion cityRegion, string taxiBinDataPath, string resultDir)
{
var vectorLength = 2 * sqrtVectorLength * sqrtVectorLength;
var numOfNodes = sqrtNumOfNodes * sqrtNumOfNodes;
var resultPath =
PathBuilder.Create(resultDir, "InnerProduct")
.AddProperty("Dataset", "TaxiTrips")
.AddProperty("Nodes", numOfNodes.ToString())
.AddProperty("VectorLength", vectorLength.ToString())
.AddProperty("Window", hoursInWindow.ToString())
.AddProperty("Iterations", iterations.ToString())
.AddProperty("Approximation", approximation.AsString())
.AddProperty("DataSplit", splitter.Name)
.ToPath("csv");
using (var resultCsvFile = AutoFlushedTextFile.Create(resultPath, AccumaltedResult.Header(numOfNodes)))
using (var binaryReader = new BinaryReader(File.OpenRead(taxiBinDataPath)))
{
var innerProductFunction = new InnerProductFunction(vectorLength);
var taxiTrips = TaxiTripEntry.FromBinary(binaryReader);
var windowManager = TaxiTripsWindowManger.Init(hoursInWindow, sqrtNumOfNodes, sqrtVectorLength, splitter, cityRegion, taxiTrips);
var initVectors = windowManager.GetCurrentVectors();
var multiRunner = MultiRunner.InitAll(initVectors, numOfNodes, vectorLength, approximation, innerProductFunction.MonitoredFunction);
for (int i = 0; i < iterations; i++)
{
if (!windowManager.TakeStep())
{
break;
}
var changeVectors = windowManager.GetChangeVector();
multiRunner.Run(changeVectors, random, false)
.Select(r => r.AsCsvString())
.ForEach(resultCsvFile.WriteLine);
}
}
}