void ChadStart() { RefreshFileData(); var buckets = ComputeNode.Catalogs.Values.Cast <ICatalog>().Where(c => c.CatalogName.Equals(CatalogName)).First().Buckets; var bucketMods = buckets.Select(b => b.Value.BucketMod).ToList(); var MyFiles = SaturationFiles.Where(f => bucketMods.Contains(Math.Abs(f.GetHashCode()) % ComputeNode.GlobalBucketCount)).OrderBy(f => f.Name).ToList(); MyFiles.AsParallel().ForAll(f => SaturateFile(new SaturateFileData() { eventDone = null, Data = f })); }
/// <summary> /// thread to saturate from a file /// </summary> /// <param name="obj"></param> /// private void SaturateFile(object obj) private void SaturateFile(object obj) { SaturateFileData data = (SaturateFileData)obj; System.Diagnostics.Trace.WriteLine(SaturationFiles.Where(f => f.Saturated == false).Count()); try { ICatalog catalog = ComputeNode.Catalogs.Values.Cast <ICatalog>().Where(c => c.CatalogName.Equals(CatalogName)).First(); long mod = Math.Abs(data.Data.GetHashCode()) % ComputeNode.GlobalBucketCount; var bucket = catalog.Buckets.Values.FirstOrDefault(b => b.BucketMod == mod); ITableMetadata tableMeta = catalog.CatalogMetadata.Tables[TableName]; // create a copy of the FileProcessor by reflection using (var proc = (IFileProcessor)FileProcessor.GetType().GetConstructor(new Type[] { }).Invoke(new object[] { })) { // create a copy of the LineProcessor by reflection proc.LineProcessor = (ILineProcessor)FileProcessor.LineProcessor.GetType().GetConstructor(new Type[] { }).Invoke(new object[] { }); if (!proc.OpenFile(data.Data.Name, tableMeta)) { throw new Exception("Error Opening file: " + data.Data.Name); } // columns metadata ordered by ordinal position var columns = tableMeta.ColumnsMetadata.Values.ToList().OrderBy(f => f.OrdinalPosition).ToList(); if (tableMeta.IsFixedWidth) { while (proc.NextLine()) { // must fix this.... var item = new WeatherDataItem(); var line = proc.GetLine(); // iterate through the columns in order by ordinal position columns.ForEach(column => { var field = line.Substring(column.FixedWidthStartIndex, column.FixedWidthLength); // set the field by reflection object result; if (proc.LineProcessor.ProcessColumn(column, field, out result)) { item.GetType().GetField(column.ColumnName).SetValue(item, result); } }); // and add to bucket bucket.BucketDataTables[TableName].AddItem(item); } } else { throw new NotImplementedException(); //columns are delimited...untested code //while (proc.NextLine()) //{ // var item = new WeatherDataItem(); // var line = proc.GetLine(); // var fields = line.Split(tableMeta.ColumnDelimiters, StringSplitOptions.None); // tableMeta.ColumnsMetadata.Values.ToList().ForEach(column => // { // var field = fields[column.OrdinalPosition]; // object result; // if (proc.ProcessColumn(column, field, out result)) // { // item.GetType().GetField(column.ColumnName).SetValue(item, result); // } // }); // bucket.BucketDataTables["Weather"].AddItem(item); //} } data.Data.Saturated = true; } } catch (Exception ex) { System.Diagnostics.Trace.WriteLine(ex.ToString()); } finally { //data.table.LastSaturation = DateTime.UtcNow; data.Data.Saturating = false; if (data.eventDone != null) { data.eventDone.Set(); } } }
/// <summary> /// feed the thread pool for saturation /// </summary> private void Saturate() { //the currently executing threads and thier events Thread[] threads = new Thread[SaturationThreadCount]; ManualResetEvent[] events = new ManualResetEvent[SaturationThreadCount]; for (int x = 0; x < SaturationThreadCount; x++) { events[x] = new ManualResetEvent(true); } //saturator is paused bool Paused = false; //main loop while (true) { try { //check the memory usage var memStatus = SystemInfo.GetMemoryStatusEx(); if (ComputeNode.Purging) { //mark as paused Paused = true; //report memory usage System.Diagnostics.Trace.WriteLine(string.Format("Saturator has reach maximum memory threshold - TotalMemory:{0} MB, AvailableMemory:{1} MB, AvailablePercent:{2} %", memStatus.ullTotalPhys / 1024 / 1024, memStatus.ullAvailPhys / 1024 / 1024, (double)memStatus.ullAvailPhys / (double)memStatus.ullTotalPhys * 100.0)); //sleep for 5 minutes if (eventStop.WaitOne(1 * 60 * 1000)) { return; } } else { //report is we are unpausing if (Paused) { //report memory usage System.Diagnostics.Trace.WriteLine(string.Format("Saturator is starting after pause - TotalMemory:{0} MB, AvailableMemory:{1} MB, AvailablePercent:{2} %", memStatus.ullTotalPhys / 1024 / 1024, memStatus.ullAvailPhys / 1024 / 1024, (double)memStatus.ullAvailPhys / (double)memStatus.ullTotalPhys * 100.0)); } //mark as not paused Paused = false; //check to refresh catalogs if ((DateTime.UtcNow - LastFileDataRefresh).TotalMinutes > 5) { RefreshFileData(); LastFileDataRefresh = DateTime.UtcNow; //PrintReport(); } //wait at least on signaled event int index = WaitHandle.WaitAny(events, 5000); //check the return if (index != WaitHandle.WaitTimeout) { //get the next file to service FileData file = SaturationFiles.Where(f => f.Saturating == false && f.Saturated == false).OrderBy(f => f.ModifiedDate).FirstOrDefault(); //System.Diagnostics.Trace.WriteLine(SaturationFiles.Where(f => f.Saturated == false ).Count()); //make sure we got one if (file != null) { //create the thread file.Saturating = true; events[index].Reset(); threads[index] = new Thread(new ParameterizedThreadStart(SaturateFile)); threads[index].Start(new SaturateFileData() { eventDone = events[index], Data = file }); } else { //sleep for little while if (eventStop.WaitOne(5000)) { return; } } } } } catch (Exception ex) { System.Diagnostics.Trace.WriteLine(ex.ToString()); } if (eventStop.WaitOne(0)) { return; } } }