public virtual void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts) { var colnames = from c in queue.GetRowFormat.Columns select c.ColumnName; bool autoinc = queue.GetRowFormat.FindAutoIncrementColumn() != null; if (autoinc) { m_dmp.AllowIdentityInsert(table.FullName, true); } try { while (!queue.IsEof) { IBedRecord row = queue.GetRecord(); m_dmp.PutCmd("^insert ^into %f (%,i) ^values (%,v)", table, colnames, row); } } finally { queue.CloseReading(); } if (autoinc) { m_dmp.AllowIdentityInsert(table.FullName, false); } }
public static void WriteQueueToXml(IDataQueue queue, Stream fw, string dbName, string tableName) { DataTable table = ConnTools.DataTableFromStructure(queue.GetRowFormat); table.TableName = tableName; List <string> fldnames = new List <string>(); foreach (IColumnStructure col in queue.GetRowFormat.Columns) { fldnames.Add(col.ColumnName); } XmlWriter xw = XmlTextWriter.Create(fw, new XmlWriterSettings { Encoding = Encoding.UTF8, CheckCharacters = false }); //XmlTextWriter xw = new XmlTextWriter(fw, Encoding.UTF8); //xw.Settings = new XmlWriterSettings { CheckCharacters = false }; xw.WriteStartDocument(); xw.WriteStartElement(dbName); // mono has bug in writing schema if (!Core.IsMono) { table.WriteXmlSchema(xw); } List <string> ids = new List <string>(); foreach (IColumnStructure col in queue.GetRowFormat.Columns) { ids.Add(XmlTool.NormalizeIdentifier(col.ColumnName)); } try { while (!queue.IsEof) { IBedRecord row = queue.GetRecord(); xw.WriteStartElement(tableName); for (int i = 0; i < row.FieldCount; i++) { row.ReadValue(i); var type = row.GetFieldType(); if (type == TypeStorage.Null) { continue; } xw.WriteElementString(ids[i], XmlTool.ObjectToString(row.GetValue(i))); } xw.WriteEndElement(); } } finally { queue.CloseReading(); } xw.WriteEndElement(); xw.WriteEndDocument(); xw.Flush(); }
public override void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts) { var colnames = from c in queue.GetRowFormat.Columns select c.ColumnName; bool autoinc = queue.GetRowFormat.FindAutoIncrementColumn() != null; if (autoinc) { m_dmp.AllowIdentityInsert(table.FullName, true); } try { if (Cfg != null && Cfg.JoinInserts) { int rowsInBatch = 0; while (!queue.IsEof) { if (Cfg.BatchLimit > 0 && rowsInBatch >= Cfg.BatchLimit) { m_dmp.EndCommand(); rowsInBatch = 0; } if (rowsInBatch > 0) { m_dmp.Put(";\n"); } IBedRecord row = queue.GetRecord(); m_dmp.Put("^insert ^into %f (%,i) ^values (%,v)", table, colnames, row); rowsInBatch++; } if (rowsInBatch > 0) { m_dmp.EndCommand(); } } else { while (!queue.IsEof) { IBedRecord row = queue.GetRecord(); m_dmp.PutCmd("^insert ^into %f (%,i) ^values (%,v)", table, colnames, row); } } } finally { queue.CloseReading(); } if (autoinc) { m_dmp.AllowIdentityInsert(table.FullName, false); } }
public static void WriteQueueToStream(IDataQueue queue, Stream fw, BedDataStats stats) { BinaryWriter bw = new BinaryWriter(fw); bw.Write(SIGNATURE); bw.Write((int)2); // version var ts = queue.GetRowFormat; bw.Write7BitEncodedInt(ts.Columns.Count); var pk = ts.FindConstraint <IPrimaryKey>(); foreach (IColumnStructure col in ts.Columns) { bw.Write(col.ColumnName); bw.Write((byte)col.DataType.DefaultStorage); ColFlags flags = 0; if (pk != null && pk.Columns.IndexOfIf(c => c.ColumnName == col.ColumnName) >= 0) { flags |= ColFlags.ISPK; } bw.Write((byte)flags); } MemoryStream rowdata = new MemoryStream(); BinaryWriter bwrow = new BinaryWriter(rowdata); try { while (!queue.IsEof) { rowdata.Position = 0; rowdata.SetLength(0); IBedRecord row = queue.GetRecord(); BedTool.SaveRecord(ts.Columns.Count, row, bwrow); bw.Write7BitEncodedInt((int)rowdata.Length); rowdata.WriteTo(bw.BaseStream); if (stats != null) { stats.Rows++; stats.Bytes += (int)rowdata.Length; stats.Bytes += 4; } } } finally { queue.CloseReading(); } // write EOF mark bw.Write7BitEncodedInt(-1); }
private void DoFillOnBackground(IDataQueue queue) { try { m_state = TabularDataViewState.Loading; //DataTable curbuf = null; Chunk curchunk = null; try { while (!queue.IsEof) { if (curchunk == null) { curchunk = new Chunk(); } IBedRecord rec = queue.GetRecord(); curchunk.SaveRecord(rec); if (curchunk.Count >= BUFFER_SIZE) { FlushChunk(curchunk); curchunk = null; } } } finally { queue.CloseReading(); } if (curchunk != null) { FlushChunk(curchunk); } m_state = TabularDataViewState.Prepared; } catch (Exception e) { Errors.Report(e); m_state = TabularDataViewState.Error; queue.PutError(e); } finally { queue.CloseWriting(); } if (LoadedNextData != null) { LoadedNextData(this, new LoadedNextDataArgs(m_serializedRows)); } }
public static IEnumerable <IBedRecord> EnumRows(this IDataQueue queue) { try { while (!queue.IsEof) { yield return(queue.GetRecord()); } } finally { queue.CloseReading(); } }
public override void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts) { var colnames = from c in queue.GetRowFormat.Columns select c.ColumnName; try { if (Wcfg != null && Wcfg.ExtendedInserts) { int rowsInBatch = 0; while (!queue.IsEof) { if (Wcfg.BatchLimit > 0 && rowsInBatch >= Wcfg.BatchLimit) { m_dmp.EndCommand(); rowsInBatch = 0; } if (rowsInBatch == 0) { m_dmp.Put("^insert ^into %f (%,i) ^values\n", table, colnames); } else { m_dmp.Put(",\n"); } IBedRecord row = queue.GetRecord(); m_dmp.Put("(%,v)", row); rowsInBatch++; } if (rowsInBatch > 0) { m_dmp.EndCommand(); } } else { while (!queue.IsEof) { IBedRecord row = queue.GetRecord(); m_dmp.PutCmd("^insert ^into %f (%,i) ^values (%,v)", table, colnames, row); } } } finally { queue.CloseReading(); } }
protected virtual void DoWrite(IDataQueue queue) { var fmt = new BedValueFormatter(new DataFormatSettings()); try { int index = 0; while (!queue.IsEof) { var record = queue.GetRecord(); record.ReadValue(0); fmt.ReadFrom(record); string name = OutputFileNameTemplate; name = name.Replace("#FILE#", fmt.GetText()); name = name.Replace("#INDEX#", (index + 1).ToString()); string fn = Path.Combine(OutputDirectory, name); record.ReadValue(1); switch (record.GetFieldType()) { case TypeStorage.String: using (var tw = new StreamWriter(fn, false, m_encoding)) { tw.Write(record.GetString()); } break; case TypeStorage.ByteArray: using (var fw = new FileInfo(fn).OpenWrite()) { var data = record.GetByteArray(); fw.Write(data, 0, data.Length); } break; } index++; } } finally { queue.CloseReading(); } }
public override void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts) { var fmt = new BedValueFormatter(FormatSettings); ExcelXmlDataStore.WriteHeaders(m_fw, table, table.FullName.ToString()); try { while (!queue.IsEof) { var record = queue.GetRecord(); ExcelXmlDataStore.WriteRow(m_fw, table, record, fmt); } } finally { queue.CloseReading(); } m_fw.Write("</ss:Table>\n"); m_fw.Write("</Worksheet>\n"); }
public virtual void Run(IDataQueue queue) { BeforeRun(); try { if (CopyOptions.AllowBulkCopy) { RunBulkCopy(queue); } else { RunInserts(queue); } } finally { queue.CloseReading(); } AfterRun(); }
protected virtual void DoWrite(IDataQueue queue) { var api = StreamApi; if (api == null) { throw new NotImplementedError("DAE-00100"); } StreamWriter fw = GetOutputStream(); if (fw == null) { throw new NotImplementedError("DAE-00101"); } try { ITableStructure ts = queue.GetRowFormat; object manager = null; api.WriteStart(fw, ts, ref manager); try { int index = 0; while (!queue.IsEof) { var record = queue.GetRecord(); api.WriteRecord(fw, ts, record, index, manager); index++; } } finally { queue.CloseReading(); } api.WriteEnd(fw, ts, manager); } finally { fw.Close(); } FinalizeBulkCopy(); }
public override void FillTable(ITableStructure table, IDataQueue queue, TableCopyOptions opts) { ITableStructure ts = queue.GetRowFormat; string fn = FileNameTemplate; fn = fn.Replace("#TABLE#", table.FullName.Name); fn = fn.Replace("#SCHEMA#", table.FullName.Schema ?? ""); string dir = Path.GetDirectoryName(fn); if (!Directory.Exists(dir)) { Directory.CreateDirectory(dir); } var api = DataStore.StreamApi; using (StreamWriter fw = new StreamWriter(fn)) { object manager = null; api.WriteStart(fw, ts, ref manager); int index = 0; try { while (!queue.IsEof) { var record = queue.GetRecord(); api.WriteRecord(fw, ts, record, index, manager); index++; } } finally { queue.CloseReading(); } api.WriteEnd(fw, ts, manager); } }
/// <summary> /// writes content of data queue into output DBF file /// </summary> /// <param name="queue"></param> protected override void DoWrite(IDataQueue queue) { var dbf = new DbfFile(); var formatter = new BedValueFormatter(FormatSettings); try { if (File.Exists(GetWorkingFileName())) { File.Delete(GetWorkingFileName()); } dbf.Create(GetWorkingFileName()); ITableStructure ts = queue.GetRowFormat; foreach (var col in ts.Columns) { DbfColumn.DbfColumnType type; int len = 0, scale = 0; switch (col.DataType.Code) { case DbTypeCode.Array: case DbTypeCode.Generic: case DbTypeCode.Text: case DbTypeCode.Xml: type = DbfColumn.DbfColumnType.Memo; break; case DbTypeCode.Blob: type = DbfColumn.DbfColumnType.Binary; break; case DbTypeCode.Datetime: var dtype = (DbTypeDatetime)col.DataType; if (dtype.SubType == DbDatetimeSubType.Date) { type = DbfColumn.DbfColumnType.Date; } else { type = DbfColumn.DbfColumnType.Character; len = DateTime.UtcNow.ToString("s").Length; } break; case DbTypeCode.Float: type = DbfColumn.DbfColumnType.Number; len = 18; scale = DefaultNumericScale; break; case DbTypeCode.Int: if (AllowFoxProInteger) { type = DbfColumn.DbfColumnType.Integer; } else { type = DbfColumn.DbfColumnType.Number; len = 18; } break; case DbTypeCode.Logical: type = DbfColumn.DbfColumnType.Boolean; break; case DbTypeCode.Numeric: type = DbfColumn.DbfColumnType.Number; len = 18; scale = ((DbTypeNumeric)col.DataType).Scale; break; case DbTypeCode.String: var stype = (DbTypeString)col.DataType; if (stype.IsBinary) { type = DbfColumn.DbfColumnType.Binary; } else if (stype.Length <= 254) { type = DbfColumn.DbfColumnType.Character; len = stype.Length; if (len <= 0) { len = DefaultStringLength; } } else { type = DbfColumn.DbfColumnType.Memo; } break; default: type = DbfColumn.DbfColumnType.Character; len = DefaultStringLength; break; } dbf.Header.AddColumn(col.ColumnName, type, len, scale); } var orec = new DbfRecord(dbf.Header); while (!queue.IsEof) { var record = queue.GetRecord(); orec.Clear(); for (int i = 0; i < ts.Columns.Count; i++) { record.ReadValue(i); formatter.ReadFrom(record); orec[i] = formatter.GetText(); } dbf.Write(orec); } } finally { dbf.Close(); queue.CloseReading(); } FinalizeBulkCopy(); }
public void Close() { m_queue.CloseReading(); m_isClosed = true; }