public IEnumerable<IDataItem> ParseFile(string[] allLines)
{
var res = new ConcurrentBag<IDataItem>();
// header row
//List<string> allLines = new List<string>();
//string currLine = reader.ReadLine();
//while ((currLine = reader.ReadLine()) != null)
//{
// allLines.Add(currLine);
//}
allLines.Skip(1).ToList().ForEach( line =>
{
var item = new WeatherDataItem();
try
{
//item.Id = line.GetHashCode();
item.Station = line.Substring(0, 6);
if (!line.Substring(7, 5).Equals("99999"))
item.Wban = line.Substring(7, 5);
item.Date = new DateTime(int.Parse(line.Substring(14, 4)),
int.Parse(line.Substring(18, 2)),
int.Parse(line.Substring(20, 2)));
if (!line.Substring(24, 6).Equals("9999.9"))
item.MeanTemperature = float.Parse(line.Substring(24, 6));
item.MeanTemperatureCount = int.Parse(line.Substring(31, 2));
if (!line.Substring(35, 6).Equals("9999.9"))
item.MeanDewpoint = float.Parse(line.Substring(35, 6));
item.MeanDewpointCount = int.Parse(line.Substring(42, 2));
if (!line.Substring(46, 6).Equals("9999.9"))
item.MeanSealevelPressure = float.Parse(line.Substring(46, 6));
item.MeanSealevelPressureCount = int.Parse(line.Substring(53, 2));
if (!line.Substring(57, 6).Equals("9999.9"))
item.MeanStationPressure = float.Parse(line.Substring(57, 6));
item.MeanStationPressureCount = int.Parse(line.Substring(64, 2));
if (!line.Substring(68, 5).Equals("999.9"))
item.MeanVisibility = float.Parse(line.Substring(68, 5));
item.MeanVisibilityCount = int.Parse(line.Substring(74, 2));
if (!line.Substring(78, 5).Equals("999.9"))
item.MeanWindSpeed = float.Parse(line.Substring(78, 5));
item.MeanWindSpeedCount = int.Parse(line.Substring(84, 2));
if (!line.Substring(88, 5).Equals("999.9"))
item.MaximumSustainedWindSpeed = float.Parse(line.Substring(88, 5));
if (!line.Substring(95, 5).Equals("999.9"))
item.MaximumGust = float.Parse(line.Substring(95, 5));
if (!line.Substring(102, 6).Equals("9999.9"))
item.MaximumTemperature = float.Parse(line.Substring(102, 6));
item.MaximumTemperatureFlag = line.Substring(108, 1).ToCharArray().First();
if (!line.Substring(110, 6).Equals("9999.9"))
item.MinimumTemperature = float.Parse(line.Substring(110, 6));
item.MinimumTemperatureFlag = line.Substring(116, 1).ToCharArray().First();
if (!line.Substring(118, 5).Equals("99.99"))
item.Precipitation = float.Parse(line.Substring(118, 5));
item.PrecipitationFlag = line.Substring(123, 1).ToCharArray().First();
if (!line.Substring(125, 5).Equals("999.9"))
item.SnowDepth = float.Parse(line.Substring(125, 5));
item.IsFog = (line.Substring(132, 1).Equals("1"));
item.IsRain = (line.Substring(133, 1).Equals("1"));
item.IsSnow = (line.Substring(134, 1).Equals("1"));
item.IsHail = (line.Substring(135, 1).Equals("1"));
item.IsThunder = (line.Substring(136, 1).Equals("1"));
item.IsTornado = (line.Substring(137, 1).Equals("1"));
var epochDays = item.Date.Subtract(new DateTime(1929, 1, 1)).TotalDays;
item.Id = long.Parse(epochDays + item.Station + line.Substring(7, 5));
res.Add(item);
}
catch (FormatException ex)
{
System.Diagnostics.Trace.WriteLine(ex.ToString());
System.Diagnostics.Trace.WriteLine("Parse error on line: {0}", line );
}
});
return res.ToList();
}
/// <summary>
/// thread to saturate from a file
/// </summary>
/// <param name="obj"></param>
/// private void SaturateFile(object obj)
private void SaturateFile(object obj)
{
SaturateFileData data = (SaturateFileData)obj;
System.Diagnostics.Trace.WriteLine(SaturationFiles.Where(f => f.Saturated == false).Count());
try
{
ICatalog catalog = ComputeNode.Catalogs.Values.Cast<ICatalog>().Where(c => c.CatalogName.Equals(CatalogName)).First();
long mod = Math.Abs(data.Data.GetHashCode()) % ComputeNode.GlobalBucketCount;
var bucket = catalog.Buckets.Values.FirstOrDefault(b => b.BucketMod == mod);
ITableMetadata tableMeta = catalog.CatalogMetadata.Tables[TableName];
// create a copy of the FileProcessor by reflection
using (var proc = (IFileProcessor)FileProcessor.GetType().GetConstructor(new Type[] { }).Invoke(new object[] { }))
{
// create a copy of the LineProcessor by reflection
proc.LineProcessor = (ILineProcessor)FileProcessor.LineProcessor.GetType().GetConstructor(new Type[] { }).Invoke(new object[] { });
if (!proc.OpenFile(data.Data.Name, tableMeta))
throw new Exception("Error Opening file: " + data.Data.Name);
// columns metadata ordered by ordinal position
var columns = tableMeta.ColumnsMetadata.Values.ToList().OrderBy(f => f.OrdinalPosition).ToList();
if (tableMeta.IsFixedWidth)
{
while (proc.NextLine())
{
// must fix this....
var item = new WeatherDataItem();
var line = proc.GetLine();
// iterate through the columns in order by ordinal position
columns.ForEach(column =>
{
var field = line.Substring(column.FixedWidthStartIndex, column.FixedWidthLength);
// set the field by reflection
object result;
if (proc.LineProcessor.ProcessColumn(column, field, out result))
{
item.GetType().GetField(column.ColumnName).SetValue(item, result);
}
});
// and add to bucket
bucket.BucketDataTables[TableName].AddItem(item);
}
}
else
{
throw new NotImplementedException();
//columns are delimited...untested code
//while (proc.NextLine())
//{
// var item = new WeatherDataItem();
// var line = proc.GetLine();
// var fields = line.Split(tableMeta.ColumnDelimiters, StringSplitOptions.None);
// tableMeta.ColumnsMetadata.Values.ToList().ForEach(column =>
// {
// var field = fields[column.OrdinalPosition];
// object result;
// if (proc.ProcessColumn(column, field, out result))
// {
// item.GetType().GetField(column.ColumnName).SetValue(item, result);
// }
// });
// bucket.BucketDataTables["Weather"].AddItem(item);
//}
}
data.Data.Saturated = true;
}
}
catch (Exception ex)
{
System.Diagnostics.Trace.WriteLine(ex.ToString());
}
finally
{
//data.table.LastSaturation = DateTime.UtcNow;
data.Data.Saturating = false;
if(data.eventDone != null)
data.eventDone.Set();
}
}
