//List<Task> tasks = new List<Task>(); public void BatchRouting() { int _BatchSize = 1000; int _MaxCount = context.TripData.Count(); ThreadPool.GetMinThreads(out int _PoolSize, out int minPorts); ThreadPool.SetMaxThreads(_PoolSize, _PoolSize); int TaskCount = 0; int numberOfBatches = (int)((double)_MaxCount / (double)_BatchSize + 1.0f); Console.WriteLine("No. of Trips: {0}\t" + "No. of Batchs: {1} of size {2}", _MaxCount, numberOfBatches, _BatchSize); Util.Progress progress = new Util.Progress(5000, _MaxCount); progress.Start(); int BatchCount = 0; while (BatchCount < numberOfBatches) { ThreadPool.QueueUserWorkItem((obj) => { Batch((int)obj, _BatchSize, progress); }, BatchCount++); Thread.Sleep(500); } }
public static void GetPlacesData(City nyc) { GooglePlacesAPI api = new GooglePlacesAPI(); string file = Path.Combine(NYCConst.Places_Dir, "nycplaces.json"); var tracts = nyc.Select(t => new { id = t.UID, lng = t.Longitude, lat = t.Latitude, radius = t.Length / 6.562 }); List <RatingData> tractPopularity = new List <RatingData>(); Progress progress = new Progress(1000, tracts.Count()); progress.Start(); do { Parallel.ForEach(tracts, tract => { GooglePlacesAPI.Place[] places = null; Task.Run(async() => { places = await api.GetPlacesAsync(tract.lat, tract.lng, tract.radius); }).Wait(); RatingData data = new RatingData(); float ratingTotal = 0; int userTotal = 0; int placesCount = 0; foreach (var place in places) { if (place.User_ratings_total > 0) { ratingTotal += place.Rating * place.User_ratings_total; userTotal += place.User_ratings_total; placesCount++; } } data.id = tract.id; data.Rating = (userTotal > 0) ? ratingTotal / userTotal : 0; data.TotalUsers = userTotal; data.PlacesCount = placesCount; tractPopularity.Add(data); progress.inc(); }); tracts = tracts.Where(x => !tractPopularity.Select(i => i.id).Contains(x.id)); }while (tracts.Count() > 0); progress.Stop(); Console.WriteLine("Processed {0} places", tractPopularity.Count()); File.WriteAllTextAsync(file, JsonConvert.SerializeObject(tractPopularity)); }
public static Dictionary <string, float> ComputeAttraction(City city, TripRecordContext context) { Progress progress = new Progress(1000, context.Count); Dictionary <string, float> attraction = new Dictionary <string, float>(); //Process Console.WriteLine(">Processing Attraction Heat Map<"); progress.Start(); foreach (var date in context) { var trips = context.Get(date); foreach (var trip in trips) { var region = city.FindRegion(trip.Pickup_Latitude, trip.Pickup_Longitude)?.UID; if (region != null) { attraction[region] = attraction.GetOrCreate(region) + 1; } region = city.FindRegion(trip.Dropoff_Latitude, trip.Dropoff_Longitude)?.UID; if (region != null) { attraction[region] = attraction.GetOrCreate(region) + 1; } } progress.inc(); } progress.Stop(); //Save Console.WriteLine(">Saving Attraction Heat Map<"); File.WriteAllText(Path.Combine(NYCConst.Base_Dir, "Attraction.json"), JsonConvert.SerializeObject(attraction)); return(attraction); }
public static void ExportNYCTrips_ByDay(DateTime from, DateTime to, DayOfWeek day) { Stopwatch stopwatch = new Stopwatch(); SqlContext context = new SqlContext(); City use; //use.Deserialize(@"C:\Users\seetam\Documents\TaxiData\Zoning\nyzd.shp"); #region DataRetrival Console.WriteLine(">Data Retrival<"); stopwatch.Restart(); var Rows = context.TripData.AsNoTracking() .Where(t => t.Trip_Date >= from && t.Trip_Date < to && t.Trip_Day == day.ToString()) .Select(t => new { t.Pickup_Latitude, t.Pickup_Longitude, t.Dropoff_Latitude, t.Dropoff_Longitude, t.Passenger_Count, t.Trip_Date, t.Trip_Hour, t.Trip_Minute }); Console.WriteLine("Row Count: {0}", Rows.Count()); Console.WriteLine("Execution Time: {0} Seconds\n", (float)stopwatch.ElapsedMilliseconds / 1000); #endregion #region DataProcessing Console.WriteLine(">Populating Trip List<"); stopwatch.Restart(); List <TripStr> Trips = new List <TripStr>(Rows.Count()); Progress progress = new Progress(3000, Rows.Count()); progress.Start(); Rows.ForEachAsync(t => { //int pZone = use.FindZone(t.Pickup_Latitude, t.Pickup_Longitude); //int dZone = use.FindZone(t.Dropoff_Latitude, t.Dropoff_Longitude); //if (pZone != -1 && dZone != -1) // Trips.Add(new TripStr // { // PickupZone = pZone, // DropoffZone = dZone, // PassengerCount = t.Passenger_Count, // Date = t.Trip_Date, // Hour = t.Trip_Hour, // Minute = t.Trip_Minute // }); progress.inc(); }).Wait(); progress.Stop(); Console.WriteLine("Time Elapsed: {0} Seconds\n", (float)stopwatch.ElapsedMilliseconds / 1000); #endregion #region SavingData Console.WriteLine(">Saving Trips to binary file<"); stopwatch.Restart(); //TODO //ToBinary.CsvToBinary(Trips, string.Format(@".\{0}Trips-2015.dat", day.ToString())); Console.WriteLine("Time Elapsed: {0} Seconds\n", (float)stopwatch.ElapsedMilliseconds / 1000); #endregion }