private void importUrl(PipelineContext ctx, IDatasourceSink sink, IStreamProvider elt) { int splitUntil = elt.ContextNode.ReadInt("@splituntil", this.splitUntil); bool objectPerLine = elt.ContextNode.ReadBool("@objectperline", this.objectPerLine); ctx.SendItemStart(elt); if ((ctx.ActionFlags & _ActionFlags.Skip) != 0) { return; } ExistState existState = ExistState.NotExist; if ((ctx.ImportFlags & _ImportFlags.ImportFull) == 0) //Not a full import { existState = toExistState(sink.HandleValue(ctx, "record/_checkexist", null)); } //Check if we need to convert this file if ((existState & (ExistState.ExistSame | ExistState.ExistNewer | ExistState.Exist)) != 0) { ctx.Skipped++; ctx.ImportLog.Log("Skipped: {0}. Date={1}", elt, 0);// dtFile); return; } List <String> keys = new List <string>(); List <String> values = new List <String>(); Stream fs = null; try { fs = elt.CreateStream(ctx); if (!this.objectPerLine) { importRecord(ctx, sink, fs, splitUntil); } else { byte[] buf = new byte[4096]; int offset = 0; MemoryStream tmp = new MemoryStream(); while (true) { int len = offset + fs.Read(buf, offset, buf.Length - offset); if (len == offset) { break; } int i = offset; for (; i < len; i++) { if (buf[i] == '\n') { break; } } tmp.Write(buf, offset, i - offset); if (i == offset) { offset = 0; continue; } if (tmp.Position > 0) { tmp.Position = 0; importRecord(ctx, sink, tmp, splitUntil); tmp.Position = 0; } if (i + 1 < offset) { tmp.Write(buf, i + 1, len - i - 1); } } if (offset > 0) { tmp.Write(buf, 0, offset); } if (tmp.Position > 0) { tmp.Position = 0; importRecord(ctx, sink, tmp, splitUntil); } } ctx.OptSendItemStop(); } catch (Exception e) { ctx.HandleException(e); } }
protected virtual Stream _CreateStream(PipelineContext ctx, IStreamProvider elt) { return(elt.CreateStream(ctx)); }