/// <summary>
/// we want a consistent line of buckets with difference of BucketSize. Use this method when you identify a DateTime that is (after
/// rounding to the BucketSize, not in Buckets. This method will add additional buckets up until the key has been reached (either
/// adding new higher buckets if the key is > than the current maximum or adding additional lower buckets if it is lower.
/// </summary>
/// <param name="key"></param>
private void ExtendBucketsToEncompas(DateTime key)
{
int addCount = 0;
int somethingHasGoneWrongIfAddedThisManyBuckets = 100000;
if (Buckets.Keys.Max() < key)
{
//extend upwards
DateTime toAdd = Buckets.Keys.Max();
while (toAdd != key && addCount < somethingHasGoneWrongIfAddedThisManyBuckets)
{
toAdd = ExtractionTimeTimeCoverageAggregatorBucket.IncreaseDateTimeBy(toAdd, BucketSize);
Buckets.Add(toAdd, new ExtractionTimeTimeCoverageAggregatorBucket(toAdd));
addCount++;
}
}
else
if (Buckets.Keys.Min() > key)
{
//extend downwards
DateTime toAdd = Buckets.Keys.Min();
while (toAdd != key && addCount < somethingHasGoneWrongIfAddedThisManyBuckets)
{
toAdd = ExtractionTimeTimeCoverageAggregatorBucket.DecreaseDateTimeBy(toAdd, BucketSize);
Buckets.Add(toAdd, new ExtractionTimeTimeCoverageAggregatorBucket(toAdd));
addCount++;
}
}
else
{
throw new Exception("Could not work out how to extend agregation buckets where dictionary of datetimes had a max of " + Buckets.Keys.Max() + " and a min of" + Buckets.Keys.Min() + " which could not be extended in either direction to encompas" + key);
}
if (addCount == somethingHasGoneWrongIfAddedThisManyBuckets)
{
throw new Exception("Added " + addCount + " buckets without reaching the key... oops");
}
}
public void ProcessRow(DataRow row)
{
//there is no configured time period field for the dataset, we have already thrown so just ignore calls
if (string.IsNullOrWhiteSpace(_expectedTimeFieldInOutputBuffer))
{
return;
}
//check that the listed extraction field that has timeliness information in it is actually included in the column collection
if (!haveCheckedTimeFieldExists)
{
if (!row.Table.Columns.Contains(_expectedTimeFieldInOutputBuffer))
{
throw new MissingFieldException("Catalogue " + _catalogue.Name + " specifies that the time periodicity field of the dataset is called " + _expectedTimeFieldInOutputBuffer + " but the pipeline did not contain a field with this name, the fileds in the pipeline are (" + string.Join(",", row.Table.Columns.Cast <DataColumn>().Select(c => c.ColumnName)) + ")");
}
else
{
haveCheckedTimeFieldExists = true;
}
}
object value = row[_expectedTimeFieldInOutputBuffer];
if (value == DBNull.Value)
{
countOfNullsSeen++;
return;
}
DateTime key;
object identifier = null;
try
{
if (_expectedExtractionIdentifierInOutputBuffer != null)
{
identifier = row[_expectedExtractionIdentifierInOutputBuffer];
}
}
catch (Exception)
{
//could not find the identifier in the output buffer, could be that there are multiple CHI columns e.g. CHI_Baby1, CHI_Baby2 or something
//only swallow this exception (and abandon counting of distinct release identifiers) if it is the first output row.
if (firstRow)
{
_expectedExtractionIdentifierInOutputBuffer = null;//give up trying to work out the extraction identifier
}
else
{
throw;
}
}
firstRow = false;
try
{
if (value is string)
{
if (string.IsNullOrWhiteSpace((string)value))
{
countOfNullsSeen++;
return;
}
else
{
string valueAsString = (string)value;
//trim off times
if (Regex.IsMatch(valueAsString, "[0-2][0-9]:[0-5][0-9]:[0-5][0-9]"))
{
valueAsString = valueAsString.Substring(0, valueAsString.Length - "00:00:00".Length).Trim();
}
key = DateTime.ParseExact(valueAsString, "dd/MM/yyyy", null);
}
}
else if (value is DateTime)
{
key = (DateTime)value;
}
else
{
key = Convert.ToDateTime(value);
}
}
catch (Exception)
{
countOfBrokenDates++;
return;
}
//drop the time part
key = ExtractionTimeTimeCoverageAggregatorBucket.RoundDateTimeDownToNearestBucketFloor(key, BucketSize);
if (!Buckets.Any())
{
//no buckets yet, create a bucket here
Buckets.Add(key, new ExtractionTimeTimeCoverageAggregatorBucket(key));
Buckets[key].SawIdentifier(identifier);
}
else
{
if (!Buckets.ContainsKey(key))
{
ExtendBucketsToEncompas(key);
}
Buckets[key].SawIdentifier(identifier);
}
}