/// <summary>
/// refresh the list of files
/// </summary>
private void RefreshFileData()
{
//List<string> files = FileProcessor.GetFileNames();
List <object> files = FileProcessor.GetFileObjects();
//get the new files
//var new_files = files.Where(f => !SaturationFiles.Any(sf => sf.Name.Equals(f, StringComparison.InvariantCultureIgnoreCase))).ToList();
var buckets = ComputeNode.Catalogs.Values.Cast <ICatalog>().Where(c => c.CatalogName.Equals(CatalogName)).First().Buckets;
var bucketMods = buckets.Select(b => b.Value.BucketMod).ToList();
//var MyFiles = files.Where(f => bucketMods.Contains(Math.Abs(f.GetHashCode()) % ComputeNode.GlobalBucketCount)).ToList();
//add to the
//new_files.ForEach(f =>
files.ForEach(f =>
{
var fd = new FileData()
{
Name = f,
ModifiedDate = DateTime.UtcNow,
Saturated = false,
Saturating = false,
};
if (bucketMods.Contains(Math.Abs(fd.GetHashCode()) % ComputeNode.GlobalBucketCount))
{
SaturationFiles.Add(fd);
}
});
}
void ChadStart()
{
RefreshFileData();
var buckets = ComputeNode.Catalogs.Values.Cast <ICatalog>().Where(c => c.CatalogName.Equals(CatalogName)).First().Buckets;
var bucketMods = buckets.Select(b => b.Value.BucketMod).ToList();
var MyFiles = SaturationFiles.Where(f => bucketMods.Contains(Math.Abs(f.GetHashCode()) % ComputeNode.GlobalBucketCount)).OrderBy(f => f.Name).ToList();
MyFiles.AsParallel().ForAll(f =>
SaturateFile(new SaturateFileData()
{
eventDone = null,
Data = f
}));
}
/// <summary>
/// thread to saturate from a file
/// </summary>
/// <param name="obj"></param>
/// private void SaturateFile(object obj)
private void SaturateFile(object obj)
{
SaturateFileData data = (SaturateFileData)obj;
System.Diagnostics.Trace.WriteLine(SaturationFiles.Where(f => f.Saturated == false).Count());
try
{
ICatalog catalog = ComputeNode.Catalogs.Values.Cast <ICatalog>().Where(c => c.CatalogName.Equals(CatalogName)).First();
long mod = Math.Abs(data.Data.GetHashCode()) % ComputeNode.GlobalBucketCount;
var bucket = catalog.Buckets.Values.FirstOrDefault(b => b.BucketMod == mod);
ITableMetadata tableMeta = catalog.CatalogMetadata.Tables[TableName];
// create a copy of the FileProcessor by reflection
using (var proc = (IFileProcessor)FileProcessor.GetType().GetConstructor(new Type[] { }).Invoke(new object[] { }))
{
// create a copy of the LineProcessor by reflection
proc.LineProcessor = (ILineProcessor)FileProcessor.LineProcessor.GetType().GetConstructor(new Type[] { }).Invoke(new object[] { });
if (!proc.OpenFile(data.Data.Name, tableMeta))
{
throw new Exception("Error Opening file: " + data.Data.Name);
}
// columns metadata ordered by ordinal position
var columns = tableMeta.ColumnsMetadata.Values.ToList().OrderBy(f => f.OrdinalPosition).ToList();
if (tableMeta.IsFixedWidth)
{
while (proc.NextLine())
{
// must fix this....
var item = new WeatherDataItem();
var line = proc.GetLine();
// iterate through the columns in order by ordinal position
columns.ForEach(column =>
{
var field = line.Substring(column.FixedWidthStartIndex, column.FixedWidthLength);
// set the field by reflection
object result;
if (proc.LineProcessor.ProcessColumn(column, field, out result))
{
item.GetType().GetField(column.ColumnName).SetValue(item, result);
}
});
// and add to bucket
bucket.BucketDataTables[TableName].AddItem(item);
}
}
else
{
throw new NotImplementedException();
//columns are delimited...untested code
//while (proc.NextLine())
//{
// var item = new WeatherDataItem();
// var line = proc.GetLine();
// var fields = line.Split(tableMeta.ColumnDelimiters, StringSplitOptions.None);
// tableMeta.ColumnsMetadata.Values.ToList().ForEach(column =>
// {
// var field = fields[column.OrdinalPosition];
// object result;
// if (proc.ProcessColumn(column, field, out result))
// {
// item.GetType().GetField(column.ColumnName).SetValue(item, result);
// }
// });
// bucket.BucketDataTables["Weather"].AddItem(item);
//}
}
data.Data.Saturated = true;
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine(ex.ToString());
}
finally
{
//data.table.LastSaturation = DateTime.UtcNow;
data.Data.Saturating = false;
if (data.eventDone != null)
{
data.eventDone.Set();
}
}
}
/// <summary>
/// feed the thread pool for saturation
/// </summary>
private void Saturate()
{
//the currently executing threads and thier events
Thread[] threads = new Thread[SaturationThreadCount];
ManualResetEvent[] events = new ManualResetEvent[SaturationThreadCount];
for (int x = 0; x < SaturationThreadCount; x++)
{
events[x] = new ManualResetEvent(true);
}
//saturator is paused
bool Paused = false;
//main loop
while (true)
{
try
{
//check the memory usage
var memStatus = SystemInfo.GetMemoryStatusEx();
if (ComputeNode.Purging)
{
//mark as paused
Paused = true;
//report memory usage
System.Diagnostics.Trace.WriteLine(string.Format("Saturator has reach maximum memory threshold - TotalMemory:{0} MB, AvailableMemory:{1} MB, AvailablePercent:{2} %",
memStatus.ullTotalPhys / 1024 / 1024, memStatus.ullAvailPhys / 1024 / 1024, (double)memStatus.ullAvailPhys / (double)memStatus.ullTotalPhys * 100.0));
//sleep for 5 minutes
if (eventStop.WaitOne(1 * 60 * 1000))
{
return;
}
}
else
{
//report is we are unpausing
if (Paused)
{
//report memory usage
System.Diagnostics.Trace.WriteLine(string.Format("Saturator is starting after pause - TotalMemory:{0} MB, AvailableMemory:{1} MB, AvailablePercent:{2} %",
memStatus.ullTotalPhys / 1024 / 1024, memStatus.ullAvailPhys / 1024 / 1024, (double)memStatus.ullAvailPhys / (double)memStatus.ullTotalPhys * 100.0));
}
//mark as not paused
Paused = false;
//check to refresh catalogs
if ((DateTime.UtcNow - LastFileDataRefresh).TotalMinutes > 5)
{
RefreshFileData();
LastFileDataRefresh = DateTime.UtcNow;
//PrintReport();
}
//wait at least on signaled event
int index = WaitHandle.WaitAny(events, 5000);
//check the return
if (index != WaitHandle.WaitTimeout)
{
//get the next file to service
FileData file = SaturationFiles.Where(f => f.Saturating == false && f.Saturated == false).OrderBy(f => f.ModifiedDate).FirstOrDefault();
//System.Diagnostics.Trace.WriteLine(SaturationFiles.Where(f => f.Saturated == false ).Count());
//make sure we got one
if (file != null)
{
//create the thread
file.Saturating = true;
events[index].Reset();
threads[index] = new Thread(new ParameterizedThreadStart(SaturateFile));
threads[index].Start(new SaturateFileData()
{
eventDone = events[index],
Data = file
});
}
else
{
//sleep for little while
if (eventStop.WaitOne(5000))
{
return;
}
}
}
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine(ex.ToString());
}
if (eventStop.WaitOne(0))
{
return;
}
}
}
