protected override void DoRead(IDataQueue queue)
{
try
{
using (CsvReader cr = CreateReader())
{
ITableStructure format = GetStructure(cr);
IEnumerable <string[]> reader = cr;
foreach (string[] row in reader)
{
queue.PutRecord(new ArrayDataRecord(format, row));
}
queue.PutEof();
}
}
catch (Exception e)
{
ProgressInfo.LogError(e);
queue.PutError(e);
}
finally
{
queue.CloseWriting();
}
FinalizeBulkCopy();
}
private void DoRead(IDataQueue queue)
{
try
{
int page = 0;
do
{
while (page < m_directory.Count)
{
lock (m_directory)
{
BinaryReader br = new BinaryReader(m_cache);
m_cache.Seek(m_directory[page], SeekOrigin.Begin);
ChunkInfo info = ChunkInfo.LoadInfo(br);
for (int i = 0; i < info.Count; i++)
{
queue.PutRecord(BedTool.LoadRecord(br, m_table));
}
}
page++;
}
if (State == TabularDataViewState.Loading)
{
System.Threading.Thread.Sleep(100);
}
} while (State == TabularDataViewState.Loading);
queue.PutEof();
}
finally
{
queue.CloseWriting();
}
}
private void DoRead(IDataQueue queue)
{
WantTable();
foreach (var row in m_table.Rows)
{
queue.PutRecord(row);
}
queue.PutEof();
}
protected override void DoRead(IDataQueue queue)
{
if (m_table.m_table.FixedData != null)
{
foreach (var row in m_table.m_table.FixedData.Rows)
{
queue.PutRecord(row);
}
}
queue.PutEof();
}
private void DoRead(IDataQueue queue)
{
try
{
foreach (var row in m_table.Rows)
{
queue.PutRecord(row);
}
queue.PutEof();
}
finally
{
queue.CloseWriting();
}
}
private void DoReadTable(NameWithSchema table, IDataQueue queue)
{
try
{
string fnbase = XmlTool.NormalizeIdentifier(table.ToString());
ZipEntry xmlEntry;
try { xmlEntry = m_zip[fnbase + ".xml"]; }
catch { xmlEntry = null; }
ZipEntry drsEntry;
try { drsEntry = m_zip[fnbase + ".drs"]; }
catch { drsEntry = null; }
if (drsEntry == null && xmlEntry == null)
{
var dbs = GetStructure();
if (dbs.Tables.GetIndex(table) < 0)
{
throw new InternalError("DAE-00019 Table not found in data archive:" + table.ToString());
}
// table is empty, only has no drs nor xml file
queue.PutEof();
return;
}
using (Stream fr = (drsEntry ?? xmlEntry).OpenReader())
{
if (drsEntry != null)
{
BedTool.LoadQueue(fr, queue);
}
else if (xmlEntry != null)
{
using (XmlReader reader = XmlReader.Create(fr, new XmlReaderSettings {
CheckCharacters = false
}))
{
XmlDataTool.ReadXmlToQueue(reader, queue, "DataRow");
}
}
}
}
finally
{
queue.CloseWriting();
}
}
/// <summary>
/// reads content of input file into given data queue
/// </summary>
/// <param name="queue"></param>
protected override void DoRead(IDataQueue queue)
{
DbfFile dbf = null;
try
{
dbf = OpenReader();
ITableStructure format = GetStructure(dbf);
DbfRecord irec = new DbfRecord(dbf.Header);
// for each record in input DBF
while (dbf.ReadNext(irec))
{
if (irec.IsDeleted)
{
continue;
}
object[] vals = new object[format.Columns.Count];
for (int i = 0; i < format.Columns.Count; i++)
{
vals[i] = irec[i];
}
var orec = new ArrayDataRecord(format, vals);
queue.PutRecord(new ArrayDataRecord(format, vals));
}
queue.PutEof();
}
catch (Exception e)
{
ProgressInfo.LogError(e);
queue.PutError(e);
}
finally
{
if (dbf != null)
{
dbf.Close();
}
queue.CloseWriting();
}
FinalizeBulkCopy();
}
void DoRead(IDataQueue queue)
{
try
{
using (DbCommand cmd = m_conn.DbFactory.CreateCommand())
{
cmd.CommandText = m_query;
cmd.Connection = m_conn.SystemConnection;
using (IBedReader reader = m_conn.GetAnyDDA().AdaptReader(cmd.ExecuteReader()))
{
while (reader.Read())
{
queue.PutRecord(reader);
}
}
}
}
finally
{
queue.PutEof();
}
}
public static void LoadQueue(Stream fr, IDataQueue queue)
{
try
{
BinaryReader br = new BinaryReader(fr);
int sgn = br.ReadInt32();
if (sgn != SIGNATURE)
{
throw new InternalError("DAE-00021 Bad BED file");
}
int ver = br.ReadInt32();
if (ver != 1 && ver != 2)
{
throw new InternalError("DAE-00022 Bad BED file");
}
int colcnt = br.Read7BitEncodedInt();
TableStructure ts = new TableStructure();
PrimaryKey pk = new PrimaryKey();
for (int i = 0; i < colcnt; i++)
{
ColumnStructure col = new ColumnStructure();
col.ColumnName = br.ReadString();
if (ver >= 2)
{
var type = (TypeStorage)br.ReadByte();
col.DataType = type.GetDatAdminType();
var flags = (ColFlags)br.ReadByte();
if ((flags & ColFlags.ISPK) != 0)
{
pk.Columns.Add(new ColumnReference(col.ColumnName));
}
}
ts._Columns.Add(col);
}
if (pk.Columns.Count > 0)
{
ts._Constraints.Add(pk);
}
for (; ;)
{
int len = br.Read7BitEncodedInt();
if (len < 0)
{
break;
}
var rec = LoadRecord(br, ts);
queue.PutRecord(rec);
}
queue.PutEof();
}
catch (Exception e)
{
Errors.Report(e);
queue.PutError(e);
}
finally
{
queue.CloseWriting();
}
}
void DoRead(IDataQueue queue)
{
try
{
m_conn.SystemConnection.SafeChangeDatabase(m_dbname);
using (DbCommand cmd = m_conn.DbFactory.CreateCommand())
{
var fname = new NameWithSchema(m_schema, m_tblname);
string qfullname = m_conn.Dialect.QuoteFullName(fname);
cmd.CommandText = "SELECT * FROM " + qfullname;
cmd.Connection = m_conn.SystemConnection;
try
{
using (IBedReader reader = m_conn.GetAnyDDA().AdaptReader(cmd.ExecuteReader()))
{
while (reader.Read())
{
queue.PutRecord(reader);
}
}
}
catch (Exception err)
{
if (!m_conn.Dialect.DialectCaps.RangeSelect)
{
throw;
}
if (err is ThreadAbortException)
{
throw;
}
// try to load in more packets
int tblsize = Int32.Parse(m_conn.SystemConnection.ExecuteScalar("SELECT COUNT(*) FROM " + qfullname).ToString());
int ofs = 0;
int maxsize = 200;
while (ofs < tblsize)
{
if (ProgressInfo != null)
{
ProgressInfo.SetCurWork(String.Format("{0}: {1}/{2}", fname, ofs, tblsize));
}
int pacsize = tblsize - ofs;
if (pacsize > maxsize)
{
pacsize = maxsize;
}
using (DbCommand cmd2 = m_conn.DbFactory.CreateCommand())
{
cmd.CommandText = m_conn.Dialect.GetRangeSelect("SELECT * FROM " + qfullname, ofs, pacsize);
cmd.Connection = m_conn.SystemConnection;
using (IBedReader reader = m_conn.GetAnyDDA().AdaptReader(cmd.ExecuteReader()))
{
while (reader.Read())
{
queue.PutRecord(reader);
}
}
}
ofs += pacsize;
}
}
}
}
finally
{
queue.PutEof();
if (ProgressInfo != null)
{
ProgressInfo.SetCurWork("");
}
}
}
public static DataTable ReadXmlToQueue(XmlReader reader, IDataQueue queue, string tableTagName)
{
try
{
reader.MoveToContent();
reader.Read();
reader.MoveToContent();
DataTable structTable = null;
if (reader.LocalName != tableTagName)
{
structTable = new DataTable();
structTable.ReadXmlSchema(reader);
}
Dictionary <string, int> colPos = new Dictionary <string, int>();
ITableStructure rowFormat;
if (structTable != null)
{
foreach (DataColumn col in structTable.Columns)
{
colPos[XmlTool.NormalizeIdentifier(col.ColumnName)] = col.Ordinal;
}
rowFormat = structTable.Columns.GetTableStructure(null);
}
else
{
rowFormat = queue.PutRowFormat;
int index = 0;
foreach (var col in rowFormat.Columns)
{
colPos[XmlTool.NormalizeIdentifier(col.ColumnName)] = index;
index++;
}
}
while (reader.NodeType == XmlNodeType.Element)
{
if (reader.LocalName != tableTagName)
{
throw new XmlFormatError(String.Format("DAE-00301 Bad xml, expected tag {0}, found {1}", tableTagName, reader.LocalName));
}
reader.Read();
reader.MoveToContent();
object[] values = new object[rowFormat.Columns.Count];
for (int i = 0; i < values.Length; i++)
{
values[i] = DBNull.Value;
}
while (reader.NodeType == XmlNodeType.Element)
{
string colname = reader.LocalName;
int pos = colPos[colname];
bool wasdata = false;
Type dataType = structTable != null ? structTable.Columns[pos].DataType : rowFormat.Columns[pos].DataType.DotNetType;
reader.Read();
if (reader.NodeType == XmlNodeType.Text)
{
string data = reader.Value;
values[pos] = XmlTool.StringToObject_DataXml(dataType, data);
reader.Read();
wasdata = true;
}
if (reader.NodeType == XmlNodeType.EndElement && reader.LocalName == colname)
{ // skip end of element
reader.Read();
}
else
{
// error, do not throw, it is error if .NET parser
}
//if (reader.NodeType != XmlNodeType.EndElement) throw new XmlFormatError("Bad XML, expected end of tag");
//if (reader.LocalName != colname) throw new XmlFormatError(String.Format("Bad xml, expected tag {0}, found {1}", colname, reader.LocalName));
//reader.Read();
if (!wasdata)
{
values[pos] = XmlTool.StringToObject_DataXml(dataType, "");
}
}
queue.PutRecord(new ArrayDataRecord(rowFormat, values));
if (reader.NodeType == XmlNodeType.EndElement)
{
reader.Read();
}
}
return(structTable);
}
finally
{
queue.PutEof();
}
}
protected override void DoRead(IDataQueue queue)
{
try
{
TableStructure s = CreateStructure();
Dictionary <string, int> colPos = new Dictionary <string, int>();
for (int i = 0; i < m_columnNames.Count; i++)
{
colPos[m_columnNames[i]] = i;
colPos[XmlTool.NormalizeIdentifier(m_columnNames[i])] = i;
}
using (XmlReader xr = new XmlTextReader(GetWorkingFileName()))
{
xr.MoveToContent();
if (xr.Name != m_rootElementName)
{
Logging.Warning("Root element has different name");
}
xr.Read();
while (xr.NodeType == XmlNodeType.Element)
{
// process one row
object[] values = new object[m_columnNames.Count];
for (int i = 0; i < values.Length; i++)
{
values[i] = DBNull.Value;
}
if (xr.Name == m_rowElementName)
{
switch (m_storageType)
{
case XmlDataStorageType.Attribute:
for (int i = 0; i < xr.AttributeCount; i++)
{
xr.MoveToAttribute(i);
string name = xr.Name;
if (colPos.ContainsKey(name))
{
values[colPos[name]] = xr.Value;
}
}
xr.MoveToElement();
xr.Read();
break;
case XmlDataStorageType.ColumnNamedElement:
xr.Read();
xr.MoveToContent();
while (xr.NodeType == XmlNodeType.Element)
{
string name = xr.Name;
string value = xr.ReadElementContentAs(typeof(string), null).ToString();
if (colPos.ContainsKey(name))
{
values[colPos[name]] = value;
}
}
xr.MoveToContent();
if (xr.NodeType == XmlNodeType.EndElement)
{
xr.Read();
}
break;
case XmlDataStorageType.InvariantNamedElement:
xr.Read();
xr.MoveToContent();
while (xr.NodeType == XmlNodeType.Element)
{
string name = xr.GetAttribute(m_columnAttributeName);
string value = xr.ReadElementContentAs(typeof(string), null).ToString();
if (colPos.ContainsKey(name))
{
values[colPos[name]] = value;
}
}
xr.MoveToContent();
if (xr.NodeType == XmlNodeType.EndElement)
{
xr.Read();
}
break;
}
}
else
{
xr.Skip();
}
queue.PutRecord(new ArrayDataRecord(s, values));
}
}
}
finally
{
queue.PutEof();
}
FinalizeBulkCopy();
}