protected virtual void ImportUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt)
{
int orgEmitted = ctx.Emitted;
if (addEmitted)
{
ctx.IncrementEmitted();
}
DateTime dtFile = elt.LastModified;
ctx.SendItemStart(elt);
//TODO if ((ctx.ActionFlags & _ActionFlags.Skip) != 0
//Check if we need to import this file
if ((ctx.ImportFlags & _ImportFlags.ImportFull) == 0) //Not a full import
{
if (!CheckNeedImport(ctx, sink, elt))
{
goto SKIPPED;
}
}
if (ctx.SkipUntilKey == "record")
{
goto SKIPPED;
}
using (Stream fs = _CreateStream(ctx, elt))
{
ImportStream(ctx, sink, elt, fs);
}
if (!addEmitted && orgEmitted == ctx.Emitted)
{
ctx.IncrementEmitted();
}
ctx.OptSendItemStop();
return;
SKIPPED:
ctx.Skipped++;
if (!addEmitted && orgEmitted == ctx.Emitted)
{
ctx.IncrementEmitted();
}
if (logSkips)
{
ctx.DebugLog.Log("Skipped: {0}. Date={1}", elt.FullName, elt.LastModified);
}
}
private void importUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt)
{
int splitUntil = elt.ContextNode.ReadInt("@splituntil", this.splitUntil);
bool objectPerLine = elt.ContextNode.ReadBool("@objectperline", this.objectPerLine);
ctx.SendItemStart(elt);
if ((ctx.ActionFlags & _ActionFlags.Skip) != 0)
{
return;
}
ExistState existState = ExistState.NotExist;
if ((ctx.ImportFlags & _ImportFlags.ImportFull) == 0) //Not a full import
{
existState = toExistState(sink.HandleValue(ctx, "record/_checkexist", null));
}
//Check if we need to convert this file
if ((existState & (ExistState.ExistSame | ExistState.ExistNewer | ExistState.Exist)) != 0)
{
ctx.Skipped++;
ctx.ImportLog.Log("Skipped: {0}. Date={1}", elt, 0);// dtFile);
return;
}
List <String> keys = new List <string>();
List <String> values = new List <String>();
Stream fs = null;
try
{
fs = elt.CreateStream(ctx);
if (!this.objectPerLine)
{
importRecord(ctx, sink, fs, splitUntil);
}
else
{
byte[] buf = new byte[4096];
int offset = 0;
MemoryStream tmp = new MemoryStream();
while (true)
{
int len = offset + fs.Read(buf, offset, buf.Length - offset);
if (len == offset)
{
break;
}
int i = offset;
for (; i < len; i++)
{
if (buf[i] == '\n')
{
break;
}
}
tmp.Write(buf, offset, i - offset);
if (i == offset)
{
offset = 0;
continue;
}
if (tmp.Position > 0)
{
tmp.Position = 0;
importRecord(ctx, sink, tmp, splitUntil);
tmp.Position = 0;
}
if (i + 1 < offset)
{
tmp.Write(buf, i + 1, len - i - 1);
}
}
if (offset > 0)
{
tmp.Write(buf, 0, offset);
}
if (tmp.Position > 0)
{
tmp.Position = 0;
importRecord(ctx, sink, tmp, splitUntil);
}
}
ctx.OptSendItemStop();
}
catch (Exception e)
{
ctx.HandleException(e);
}
}
public void Import(PipelineContext ctx)
{
Logger importLog = ctx.ImportLog;
Logger errorLog = ctx.ErrorLog;
bool stopNeeded = false;
importLog.Log(_LogType.ltProgress | _LogType.ltTimerStart, "[{0}]: starting import with pipeline {1}, default endpoint={2}, maxadds={3} ", Name, Pipeline.Name, Pipeline.DefaultEndpoint, ctx.MaxAdds);
Pipeline.Start(ctx);
Pipeline.HandleValue(ctx, "_datasource/_start", this);
try
{
Datasource.Import(ctx, Pipeline);
importLog.Log(_LogType.ltProgress | _LogType.ltTimerStop, "[{0}]: raw import ended. Partial stats={1}.", Name, ctx.GetStats());
stopNeeded = true;
}
catch (Exception e)
{
ctx.LastError = e;
if (MaxAddsExceededException.ContainsMaxAddsExceededException(e))
{
stopNeeded = true;
ctx.ErrorState |= _ErrorState.Limited;
importLog.Log(_LogType.ltWarning | _LogType.ltTimerStop, "[{0}]: {1}", Name, e.Message);
importLog.Log("-- " + ctx.GetStats());
if ((ctx.ImportFlags & _ImportFlags.IgnoreLimited) != 0)
{
importLog.Log(_LogType.ltWarning, "Limited ignored due to importFlags [{0}].", ctx.ImportFlags);
}
stopNeeded = true;
}
else
{
ctx.ErrorState |= _ErrorState.Error;
String msg = String.Format("[{0}]: crashed err={1}", Name, e.Message);
importLog.Log(_LogType.ltError | _LogType.ltTimerStop, msg);
importLog.Log("-- " + ctx.GetStats());
errorLog.Log(_LogType.ltError, msg);
ctx.ErrorLog.Log(e);
if ((ctx.ImportFlags & _ImportFlags.IgnoreErrors) == 0)
{
throw new BMException(e, "{0}\r\nDatasource={1}.", e.Message, Name);
}
msg = String.Format("Exception ignored due to importFlags [{0}].", ctx.ImportFlags);
importLog.Log(_LogType.ltWarning, msg);
errorLog.Log(_LogType.ltWarning, msg);
stopNeeded = true;
}
}
finally
{
try
{
if (stopNeeded)
{
ctx.OptSendItemStop();
Pipeline.HandleValue(ctx, "_datasource/_stop", this);
}
}
finally
{
Pipeline.Stop(ctx);
}
}
importLog.Log(_LogType.ltProgress | _LogType.ltTimerStop, "[{0}]: import ended. {1}.", Name, ctx.GetStats());
}