public void IncrementAndLogAdd()
{
if (MaxAdds >= 0 && Added >= MaxAdds)
{
ImportLog.Log("MAX ADDS EXCEEDED, {0}", GetStats());
throw Exceeded = new MaxAddsExceededException(Added, "adds");
}
switch ((++Added) % LogAdds)
{
case 0: ImportLog.Log(_LogType.ltTimer, "Added {0} records (emitted: {1}, errors: {2})", Added, Emitted, Errors); break;
case 1: if (Added != 1)
{
break;
}
ImportLog.Log(_LogType.ltTimerStart, "Added 1 record"); break;
}
}
public void IncrementEmitted()
{
if (MaxEmits >= 0 && Emitted >= MaxEmits)
{
ImportLog.Log("MAX EMITS EXCEEDED, {0}", GetStats());
throw Exceeded = new MaxAddsExceededException(Emitted, "emits");
}
switch ((++Emitted) % LogAdds)
{
case 0:
if ((this.ImportFlags & _ImportFlags.LogEmits) == 0)
{
if (Added != 0)
{
break;
}
}
ImportFlags |= _ImportFlags.LogEmits;
ImportLog.Log(_LogType.ltTimer, "Emitted {0} records", Emitted); break;
}
}
public bool HandleException(Exception err, String prefix = "record", bool exceptIfNotHandled = true)
{
MaxAddsExceededException mae = err as MaxAddsExceededException;
if (mae != null)
{
throw new MaxAddsExceededException(mae);
}
Errors++;
String pfx = String.IsNullOrEmpty(prefix) ? "_error" : prefix + "/_error";
Pipeline.HandleValue(this, pfx, err);
if ((ActionFlags & _ActionFlags.Handled) != 0)
{
return(true);
}
if (exceptIfNotHandled)
{
throw new BMException(err, err.Message);
}
return(false);
}
public MaxAddsExceededException(MaxAddsExceededException other)
: base(other.Message, other)
{
Limit = other.Limit;
}
public void Import(PipelineContext ctx)
{
Logger importLog = ctx.ImportLog;
Logger errorLog = ctx.ErrorLog;
bool stopNeeded = false;
importLog.Log(_LogType.ltProgress | _LogType.ltTimerStart, "[{0}]: starting import with pipeline {1}, default endpoint={2}, maxadds={3} ", Name, Pipeline.Name, Pipeline.DefaultEndpoint, ctx.MaxAdds);
Pipeline.Start(ctx);
Pipeline.HandleValue(ctx, "_datasource/_start", this);
try
{
Datasource.Import(ctx, Pipeline);
importLog.Log(_LogType.ltProgress | _LogType.ltTimerStop, "[{0}]: raw import ended. Partial stats={1}.", Name, ctx.GetStats());
stopNeeded = true;
}
catch (Exception e)
{
ctx.LastError = e;
if (MaxAddsExceededException.ContainsMaxAddsExceededException(e))
{
stopNeeded = true;
ctx.ErrorState |= _ErrorState.Limited;
importLog.Log(_LogType.ltWarning | _LogType.ltTimerStop, "[{0}]: {1}", Name, e.Message);
importLog.Log("-- " + ctx.GetStats());
if ((ctx.ImportFlags & _ImportFlags.IgnoreLimited) != 0)
{
importLog.Log(_LogType.ltWarning, "Limited ignored due to importFlags [{0}].", ctx.ImportFlags);
}
stopNeeded = true;
}
else
{
ctx.ErrorState |= _ErrorState.Error;
String msg = String.Format("[{0}]: crashed err={1}", Name, e.Message);
importLog.Log(_LogType.ltError | _LogType.ltTimerStop, msg);
importLog.Log("-- " + ctx.GetStats());
errorLog.Log(_LogType.ltError, msg);
ctx.ErrorLog.Log(e);
if ((ctx.ImportFlags & _ImportFlags.IgnoreErrors) == 0)
{
throw new BMException(e, "{0}\r\nDatasource={1}.", e.Message, Name);
}
msg = String.Format("Exception ignored due to importFlags [{0}].", ctx.ImportFlags);
importLog.Log(_LogType.ltWarning, msg);
errorLog.Log(_LogType.ltWarning, msg);
stopNeeded = true;
}
}
finally
{
try
{
if (stopNeeded)
{
ctx.OptSendItemStop();
Pipeline.HandleValue(ctx, "_datasource/_stop", this);
}
}
finally
{
Pipeline.Stop(ctx);
}
}
importLog.Log(_LogType.ltProgress | _LogType.ltTimerStop, "[{0}]: import ended. {1}.", Name, ctx.GetStats());
}
public Object HandleValue(PipelineContext ctx, String key, Object value)
{
Object orgValue = value;
Object ret = null;
Object lastAction = null;
try
{
ctx.ActionFlags = 0;
if ((ctx.ImportFlags & _ImportFlags.TraceValues) != 0)
{
logger.Log("HandleValue ({0}, {1} [{2}]", key, value, value == null ? "null" : value.GetType().Name);
}
if (key == null)
{
goto EXIT_RTN;
}
String lcKey = key.ToLowerInvariant();
int keyLen = lcKey.Length;
if (ctx.SkipUntilKey != null)
{
ctx.ActionFlags |= _ActionFlags.Skipped;
if (ctx.SkipUntilKey.Length == keyLen && lcKey.Equals(ctx.SkipUntilKey, StringComparison.OrdinalIgnoreCase))
{
ctx.SkipUntilKey = null;
}
goto EXIT_RTN;
}
int ixStart = findAction(lcKey);
if (ixStart < 0)
{
if (templates.Count == 0 || !checkTemplates(ctx, key, ref lastAction)) //templates==0: otherwise checkTemplates() inserts a NOP action...
{
missed[lcKey] = null;
goto EXIT_RTN;
}
ixStart = findAction(lcKey);
if (ixStart < 0)
{
goto EXIT_RTN; //Should not happen, just to be sure!
}
}
for (int i = ixStart; i < actions.Count; i++)
{
ActionAdmin a = actions[i];
if (i > ixStart && !a.EqualToPrev)
{
break;
}
lastAction = ctx.SetAction(a.Action);
Object tmp = a.Action.HandleValue(ctx, key, value);
ClearVariables(a.Action.VarsToClear);
if (tmp != null)
{
ret = tmp;
}
if ((ctx.ActionFlags & (_ActionFlags.SkipRest | _ActionFlags.ConditionMatched)) != 0)
{
if ((ctx.ActionFlags & _ActionFlags.ConditionMatched) != 0)
{
if (!a.IsCondition)
{
throw new BMException("Action [{0}] is not a condition.", a.Key);
}
i += a.ActionsToSkipIfCond;
continue;
}
break;
}
}
//Make sure the skipUntil can also be set from the last action in a chain...
if (ctx.SkipUntilKey != null && ctx.SkipUntilKey.Length == keyLen && lcKey.Equals(ctx.SkipUntilKey, StringComparison.OrdinalIgnoreCase))
{
ctx.SkipUntilKey = null;
}
EXIT_RTN : return(ret);
}
catch (Exception e)
{
String type;
if (orgValue == value)
{
type = String.Format("[{0}]", getType(orgValue));
}
else
{
type = String.Format("[{0}] (was [{1}])", getType(value), getType(orgValue));
}
ctx.ErrorLog.Log("Exception while handling event. Key={0}, value type={1}, action={2}", key, type, lastAction);
ctx.ErrorLog.Log("-- value={0}", value);
if (orgValue != value)
{
ctx.ErrorLog.Log("-- orgvalue={0}", orgValue);
}
ctx.ErrorLog.Log(e);
PipelineAction act = lastAction as PipelineAction;
if (act == null)
{
ctx.ErrorLog.Log("Cannot dump accu: no current action found.");
}
else
{
var accu = (JObject)act.Endpoint.GetFieldAsToken(null);
ctx.ErrorLog.Log("Dumping content of current accu: fieldcount={0}", accu.Count);
String content = accu.ToString();
if (content != null && content.Length > 1000)
{
content = content.Substring(0, 1000) + "...";
}
ctx.ErrorLog.Log(content);
}
if (MaxAddsExceededException.ContainsMaxAddsExceededException(e))
{
throw;
}
throw new BMException(e, "{0}\r\nKey={1}, valueType={2}.", e.Message, key, type);
}
}