ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat) { string file = GetName(context); file = context.ResolveFile(file, ResolveFileMode.Output); context.OutputMessage("Writing file " + Path.GetFullPath(file)); return new CdlFileWriter(file, rowFormat); }
public ICdlWriter CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat) { string file = context.ResolveFile(context.Replace(File), ResolveFileMode.Output); var fw = new StreamWriter(file); var provider = GetConnectionProvider(context); return new SqlFileWriter(fw, provider.Factory); }
public TableWriter(IShellContext context, IConnectionProvider connection, NameWithSchema name, TableInfo inputRowFormat, CopyTableTargetOptions options, TableInfo destinationTableOverride = null, LinkedDatabaseInfo linkedInfo = null, DataFormatSettings sourceDataFormat = null) { _connectionProvider = connection; _linkedInfo = linkedInfo; _name = name; _inputRowFormat = inputRowFormat; _queue = new CdlDataQueue(inputRowFormat); _context = context; _inserter = connection.Factory.CreateBulkInserter(); _inserter.SourceDataFormat = sourceDataFormat; _connection = _connectionProvider.Connect(); _inserter.Connection = _connection; _inserter.Factory = connection.Factory; _inserter.LinkedInfo = _linkedInfo; var db = context.GetDatabaseStructure(connection.ProviderString); _inserter.DestinationTable = destinationTableOverride ?? db.FindTableLike(_name.Schema, _name.Name); _inserter.CopyOptions = options; _inserter.Log += _inserter_Log; _thread = new Thread(Run); _thread.Start(); }
public BulkInserterBase(ILogger <BulkInserterBase> logger) { _logger = logger; BatchSize = 100; CopyOptions = new CopyTableTargetOptions(); }
public BulkInserterBase() { BatchSize = 100; CopyOptions = new CopyTableTargetOptions(); }
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat) { string file = context.ResolveFile(context.Replace(Name), ResolveFileMode.Output); context.OutputMessage("Writing file " + Path.GetFullPath(file)); var dbf = new SocialExplorer.IO.FastDBF.DbfFile(Encoding); if (File.Exists(file)) File.Delete(file); dbf.Create(file); foreach (var col in rowFormat.Columns) { DbfColumn.DbfColumnType type; int len = 0, scale = 0; switch (col.CommonType.Code) { case DbTypeCode.Array: case DbTypeCode.Generic: case DbTypeCode.Text: case DbTypeCode.Xml: type = DbfColumn.DbfColumnType.Memo; break; case DbTypeCode.Blob: type = DbfColumn.DbfColumnType.Binary; break; case DbTypeCode.Datetime: var dtype = (DbTypeDatetime) col.CommonType; if (dtype.SubType == DbDatetimeSubType.Date) { type = DbfColumn.DbfColumnType.Date; } else { type = DbfColumn.DbfColumnType.Character; len = DateTime.UtcNow.ToString("s").Length; } break; case DbTypeCode.Float: type = DbfColumn.DbfColumnType.Number; len = 18; scale = DefaultNumericScale; break; case DbTypeCode.Int: if (AllowFoxProInteger) { type = DbfColumn.DbfColumnType.Integer; } else { type = DbfColumn.DbfColumnType.Number; len = 18; } break; case DbTypeCode.Logical: type = DbfColumn.DbfColumnType.Boolean; break; case DbTypeCode.Numeric: type = DbfColumn.DbfColumnType.Number; len = 18; scale = ((DbTypeNumeric) col.CommonType).Scale; break; case DbTypeCode.String: var stype = (DbTypeString) col.CommonType; if (stype.IsBinary) { type = DbfColumn.DbfColumnType.Binary; } else if (stype.Length <= 254) { type = DbfColumn.DbfColumnType.Character; len = stype.Length; if (len <= 0) len = DefaultStringLength; } else { type = DbfColumn.DbfColumnType.Memo; } break; default: type = DbfColumn.DbfColumnType.Character; len = DefaultStringLength; break; } dbf.Header.AddColumn(col.Name, type, len, scale); } return new DbfWriter(dbf, DataFormat); }
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat) { return CreateTarget(context).CreateWriter(rowFormat, options, context, sourceDataFormat); }
protected override void DoRun(IShellContext context) { ITabularDataSource source; ITabularDataTarget target; if (Source != null && SourceExpression != null) throw new Exception("DBSH-00087 CopyTable: Both Source and SourceExpression are set"); if (Source == null && SourceExpression == null) throw new Exception("DBSH-00088 CopyTable: None Source and SourceExpression are set"); if (Target != null && TargetExpression != null) throw new Exception("DBSH-00089 CopyTable: Both Target and TargetExpression are set"); if (Target == null && TargetExpression == null) throw new Exception("DBSH-00090 CopyTable: None Target and TargetExpression are set"); if (SourceExpression != null) { source = (ITabularDataSource) context.Evaluate(SourceExpression); } else { source = Source; } if (TargetExpression != null) { target = (ITabularDataTarget) context.Evaluate(TargetExpression); } else { target = Target; } var options = new CopyTableTargetOptions { TruncateBeforeCopy = CleanTarget, TargetMapMode = TargetMapMode, AllowBulkCopy = AllowBulkCopy, }; var table = source.GetRowFormat(context); _log.InfoFormat("Copy table data {0}=>{1}", Source.ToStringCtx(context), Target.ToStringCtx(context)); context.OutputMessage(String.Format("Copy table data {0}=>{1}", Source.ToStringCtx(context), Target.ToStringCtx(context))); var transformedInputTable = table; var counts = new List<int>(); if (ColumnMap.Count > 0) { transformedInputTable = new TableInfo(null); foreach (var mapItem in ColumnMap) { var newCols = mapItem.GetOutputColumns(table, context); counts.Add(newCols.Length); transformedInputTable.Columns.AddRange(newCols); } } using (var reader = source.CreateReader(context)) { using (var writer = target.CreateWriter(transformedInputTable, options, context, source.GetSourceFormat(context))) { int rowNumber = 0; while (reader.Read()) { if (ColumnMap.Count > 0) { var outputRecord = new ArrayDataRecord(transformedInputTable); int columnIndex = 0; for (int i = 0; i < ColumnMap.Count; i++) { var map = ColumnMap[i]; int count = counts[i]; for (int j = 0; j < count; j++, columnIndex++) { outputRecord.SeekValue(columnIndex); map.ProcessMapping(j, rowNumber, reader, outputRecord, context); } } writer.Write(outputRecord); } else { writer.Write(reader); } rowNumber++; } } } }
public ICdlWriter CreateWriter(TableInfo inputRowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat) { var connection = GetConnectionProvider(context); using (var conn = connection.Connect()) { var db = new DatabaseInfo(); db.LinkedInfo = LinkedInfo; var tbl = inputRowFormat.CloneTable(db); tbl.FullName = GetFullName(context); foreach (var col in tbl.Columns) col.AutoIncrement = false; tbl.ForeignKeys.Clear(); if (tbl.PrimaryKey != null) tbl.PrimaryKey.ConstraintName = null; tbl.AfterLoadLink(); if (IdentityColumn != null) { var col = new ColumnInfo(tbl); col.Name = IdentityColumn; col.DataType = "int"; col.AutoIncrement = true; col.NotNull = true; var pk = new PrimaryKeyInfo(tbl); pk.Columns.Add(new ColumnReference {RefColumn = col}); pk.ConstraintName = "PK_" + tbl.Name; tbl.PrimaryKey = pk; tbl.Columns.Insert(0, col); } //var sw = new StringWriter(); var so = new ConnectionSqlOutputStream(conn, null, connection.Factory.CreateDialect()); var dmp = connection.Factory.CreateDumper(so, new SqlFormatProperties()); if (DropIfExists) dmp.DropTable(tbl, true); bool useExistingTable = false; if (UseIfExists) { var ts = context.GetDatabaseStructure(connection.ProviderString); useExistingTable = ts.FindTableLike(tbl.FullName.Schema, tbl.FullName.Name) != null; } if (!useExistingTable) { tbl.Columns.ForEach(x => x.EnsureDataType(connection.Factory.CreateSqlTypeProvider())); dmp.CreateTable(tbl); } //using (var cmd = conn.CreateCommand()) //{ // cmd.CommandText = sw.ToString(); // cmd.ExecuteNonQuery(); //} return new TableWriter(context, connection, GetFullName(context), inputRowFormat, options, useExistingTable ? null : tbl, LinkedInfo, sourceDataFormat); } }
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat) { string file = context.ResolveFile(GetName(context), ResolveFileMode.Output); context.OutputMessage("Writing file " + Path.GetFullPath(file)); //var fs = System.IO.File.OpenWrite(file); var fw = new StreamWriter(file, false, Encoding); var writer = new CsvWriter(fw, Delimiter, Quote, Escape, Comment, QuotingMode, EndOfLine, DataFormat); if (HasHeaders) { writer.WriteRow(rowFormat.Columns.Select(c => c.Name)); } return writer; }
ICdlWriter ITabularDataTarget.CreateWriter(TableInfo inputRowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat) { return new TableWriter(context, GetConnectionProvider(context), GetFullName(context), inputRowFormat, options, StructureOverride, LinkedInfo, sourceDataFormat); }
public ICdlWriter CreateWriter(TableInfo rowFormat, CopyTableTargetOptions options, IShellContext context, DataFormatSettings sourceDataFormat) { return GetModel(context).CreateWriter(rowFormat, context.Replace(SheetName)); }