protected override void DoRead(IDataQueue queue) { try { using (CsvReader cr = CreateReader()) { ITableStructure format = GetStructure(cr); IEnumerable <string[]> reader = cr; foreach (string[] row in reader) { queue.PutRecord(new ArrayDataRecord(format, row)); } queue.PutEof(); } } catch (Exception e) { ProgressInfo.LogError(e); queue.PutError(e); } finally { queue.CloseWriting(); } FinalizeBulkCopy(); }
private void DoRead(IDataQueue queue) { try { int page = 0; do { while (page < m_directory.Count) { lock (m_directory) { BinaryReader br = new BinaryReader(m_cache); m_cache.Seek(m_directory[page], SeekOrigin.Begin); ChunkInfo info = ChunkInfo.LoadInfo(br); for (int i = 0; i < info.Count; i++) { queue.PutRecord(BedTool.LoadRecord(br, m_table)); } } page++; } if (State == TabularDataViewState.Loading) { System.Threading.Thread.Sleep(100); } } while (State == TabularDataViewState.Loading); queue.PutEof(); } finally { queue.CloseWriting(); } }
private void DoFillOnBackground(IDataQueue queue) { try { m_state = TabularDataViewState.Loading; //DataTable curbuf = null; Chunk curchunk = null; try { while (!queue.IsEof) { if (curchunk == null) { curchunk = new Chunk(); } IBedRecord rec = queue.GetRecord(); curchunk.SaveRecord(rec); if (curchunk.Count >= BUFFER_SIZE) { FlushChunk(curchunk); curchunk = null; } } } finally { queue.CloseReading(); } if (curchunk != null) { FlushChunk(curchunk); } m_state = TabularDataViewState.Prepared; } catch (Exception e) { Errors.Report(e); m_state = TabularDataViewState.Error; queue.PutError(e); } finally { queue.CloseWriting(); } if (LoadedNextData != null) { LoadedNextData(this, new LoadedNextDataArgs(m_serializedRows)); } }
private void DoRead(IDataQueue queue) { try { foreach (var row in m_table.Rows) { queue.PutRecord(row); } queue.PutEof(); } finally { queue.CloseWriting(); } }
private void DoReadTable(NameWithSchema table, IDataQueue queue) { try { string fnbase = XmlTool.NormalizeIdentifier(table.ToString()); ZipEntry xmlEntry; try { xmlEntry = m_zip[fnbase + ".xml"]; } catch { xmlEntry = null; } ZipEntry drsEntry; try { drsEntry = m_zip[fnbase + ".drs"]; } catch { drsEntry = null; } if (drsEntry == null && xmlEntry == null) { var dbs = GetStructure(); if (dbs.Tables.GetIndex(table) < 0) { throw new InternalError("DAE-00019 Table not found in data archive:" + table.ToString()); } // table is empty, only has no drs nor xml file queue.PutEof(); return; } using (Stream fr = (drsEntry ?? xmlEntry).OpenReader()) { if (drsEntry != null) { BedTool.LoadQueue(fr, queue); } else if (xmlEntry != null) { using (XmlReader reader = XmlReader.Create(fr, new XmlReaderSettings { CheckCharacters = false })) { XmlDataTool.ReadXmlToQueue(reader, queue, "DataRow"); } } } } finally { queue.CloseWriting(); } }
/// <summary> /// reads content of input file into given data queue /// </summary> /// <param name="queue"></param> protected override void DoRead(IDataQueue queue) { DbfFile dbf = null; try { dbf = OpenReader(); ITableStructure format = GetStructure(dbf); DbfRecord irec = new DbfRecord(dbf.Header); // for each record in input DBF while (dbf.ReadNext(irec)) { if (irec.IsDeleted) { continue; } object[] vals = new object[format.Columns.Count]; for (int i = 0; i < format.Columns.Count; i++) { vals[i] = irec[i]; } var orec = new ArrayDataRecord(format, vals); queue.PutRecord(new ArrayDataRecord(format, vals)); } queue.PutEof(); } catch (Exception e) { ProgressInfo.LogError(e); queue.PutError(e); } finally { if (dbf != null) { dbf.Close(); } queue.CloseWriting(); } FinalizeBulkCopy(); }
public static void LoadQueue(Stream fr, IDataQueue queue) { try { BinaryReader br = new BinaryReader(fr); int sgn = br.ReadInt32(); if (sgn != SIGNATURE) { throw new InternalError("DAE-00021 Bad BED file"); } int ver = br.ReadInt32(); if (ver != 1 && ver != 2) { throw new InternalError("DAE-00022 Bad BED file"); } int colcnt = br.Read7BitEncodedInt(); TableStructure ts = new TableStructure(); PrimaryKey pk = new PrimaryKey(); for (int i = 0; i < colcnt; i++) { ColumnStructure col = new ColumnStructure(); col.ColumnName = br.ReadString(); if (ver >= 2) { var type = (TypeStorage)br.ReadByte(); col.DataType = type.GetDatAdminType(); var flags = (ColFlags)br.ReadByte(); if ((flags & ColFlags.ISPK) != 0) { pk.Columns.Add(new ColumnReference(col.ColumnName)); } } ts._Columns.Add(col); } if (pk.Columns.Count > 0) { ts._Constraints.Add(pk); } for (; ;) { int len = br.Read7BitEncodedInt(); if (len < 0) { break; } var rec = LoadRecord(br, ts); queue.PutRecord(rec); } queue.PutEof(); } catch (Exception e) { Errors.Report(e); queue.PutError(e); } finally { queue.CloseWriting(); } }