internal override void CreateDataSchema(IntegrationContext context)
{
DataTable dt = context.State["datatable"] as DataTable;
if (dt == null)
{
throw new ArgumentException("context.State[\"datatable\"]");
}
//if (dt.Rows.Count == 0)
// throw new ArgumentException("rows cannot be empty.");
DataSchema dataSchema = new DataSchema();
dataSchema.TransferMode = DataFixMode.Skip;
List <DataSchemaField> fields = new List <DataSchemaField>();
for (int i = 0; i < dt.Columns.Count; i++)
{
DataSchemaField field = new DataSchemaField();
field.Name = field.Source = field.Destination = dt.Columns[i].ColumnName;
field.Type = dt.Columns[i].DataType.ToString();
if (!dt.Columns[i].AllowDBNull)
{
field.SetAnyAttributeValue("required", "true");
}
field.Index = dt.Columns[i].Ordinal;
fields.Add(field);
}
dataSchema.Fields = fields.ToArray();
context.Schema = dataSchema;
}
internal override void CreateDataSchema(IntegrationContext context)
{
string connectionstring = context.State[ContextState.DatabaseConnectionString] as string;
if (string.IsNullOrEmpty(connectionstring))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.DatabaseConnectionString));
}
string tableName = context.State[ContextState.DatabaseTableName] as string;
if (string.IsNullOrEmpty(tableName))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.DatabaseTableName));
}
DataTable dt = new DataTable(tableName);
using (SqlConnection conn = new SqlConnection(connectionstring))
{
conn.Open();
SqlCommand cmd2 = new SqlCommand(string.Format("SELECT * FROM {0} WHERE 1<>1;", tableName), conn);
SqlDataAdapter adapter = new SqlDataAdapter(cmd2);
adapter.Fill(dt);
}
context.Schema = DataTableStorage.GetDataSchema(dt);
}
public bool Run(IDictionary state)
{
using (IntegrationContext context = new IntegrationContext(state))
{
try
{
// TODO: by alex hu, finished the status updating
//while (context.Status == IntegrationStatus.Running)
//{
foreach (IProvider provider in _providers)
{
provider.Run(context);
}
//}
}
catch (Exception ex)
{
Logger.Instance.Error(this, ex);
throw;
}
}
//bool result = false;
//switch(context.Status)
//{
// case IntegrationStatus.Completed:
// case IntegrationStatus.Success:
// result = true;
// break;
//}
return(true);
}
internal virtual void GetDataSchema(IntegrationContext context)
{
string targetPath = context.State[ContextState.SchemaFilePath] as string;
context.Schema = ValidationUtils.GetDataSchemaFromFile(targetPath);
if (context.Schema == null)
{
throw new Exception(string.Format("can not parsing schema based on [{0}]", targetPath));
}
}
internal virtual void SaveDataSchema(IntegrationContext context)
{
string targetPath = context.State[ContextState.SchemaFilePath] as string;
if (string.IsNullOrEmpty(targetPath))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.SchemaFilePath));
}
using (TextWriter writer = new StreamWriter(targetPath))
{
XmlSerializer serializer = new XmlSerializer(typeof(DataSchema));
serializer.Serialize(writer, context.Schema);
writer.Close();
}
}
internal override void TransferData(IntegrationContext context)
{
if (null == context.Schema)
{
throw new ArgumentException("没有找到可用的数据结构!");
}
if (context.Data.Count == 0)
{
throw new ArgumentException("没有找到可用的数据!");
}
DataTable dt = GetEmptyDataTable(context.Schema);
FillDataTable(dt, context.Data);
context.State["datatable"] = dt;
}
internal override void TransferData(IntegrationContext context)
{
string desDirectory = context.State["desDirectory"] as string;
if (string.IsNullOrEmpty(desDirectory))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", desDirectory));
}
if (!Directory.Exists(desDirectory))
{
throw new FileNotFoundException(string.Format("Directory [{0}] does not exist!", desDirectory));
}
string desPrefix = context.State["desPrefix"] as string;
if (null == context.Schema)
{
throw new ArgumentException("没有找到可用的数据结构!");
}
//if (context.Data.Count == 0)
// throw new ArgumentException("没有找到可用的数据!");
string file;
lock (_syncRoot)
{
file = string.Concat(desPrefix, "_", DateTime.Now.ToString("yyyyMMddHHmmssfff"), ".csv");
}
string filepath = Path.Combine(desDirectory, file);
using (StreamWriter writer = new StreamWriter(new FileStream(filepath, FileMode.Create), Encoding.Default))
{
// write column header
WriteHeaderLine(context.Schema, writer);
// write content
foreach (DataItem item in context.Data)
{
WriteItemLine(item, context.Schema, writer);
}
writer.Close();
}
context.State["desFile"] = file;
}
internal override void TransferData(IntegrationContext context)
{
string connectionstring = context.State[ContextState.DatabaseConnectionString] as string;
if (string.IsNullOrEmpty(connectionstring))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.DatabaseConnectionString));
}
if (null == context.Schema)
{
throw new Exception("data schema missed!");
}
if (context.SuccessCount == 0)
{
throw new Exception("data missed!");
}
string tableName = context.State[ContextState.DatabaseTableName] as string;
if (string.IsNullOrEmpty(tableName))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.DatabaseTableName));
}
bool isTruncate = (bool)context.State[ContextState.DatabaseTruncateTable];
if (isTruncate)
{
TruncateData(connectionstring, tableName);
}
using (SqlConnection conn = new SqlConnection(connectionstring))
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn))
{
conn.Open();
DataTable dt = new DataTable(tableName);
SqlCommand cmd2 = new SqlCommand(string.Format("SELECT * FROM {0} WHERE 1<>1;", tableName), conn);
SqlDataAdapter adapter = new SqlDataAdapter(cmd2);
adapter.Fill(dt);
MappingColumns(bulkCopy, dt, context.Schema);
DataTableStorage.FillDestinationDataItems(dt, context.Data);
bulkCopy.DestinationTableName = tableName;
bulkCopy.BatchSize = dt.Rows.Count;
bulkCopy.WriteToServer(dt);
}
}
internal static void GetDataFromSourceDataTable(DataTable dt, IntegrationContext context)
{
foreach (DataRow dr in dt.Rows)
{
DataItem item = GetDataItemFromSource(dr, context.Schema);
if (item != null)
{
if (item.IsValid)
{
context.AddData(item);
}
else
{
context.AddSkippedData(item);
}
}
}
}
protected void ValidateData(IntegrationContext context)
{
foreach (DataItem item in context.Data)
{
if (OnValidating != null)
{
DataItemValidationHandler handler = (DataItemValidationHandler)OnValidating.Clone();
handler.Invoke(item, new DataItemValidationArgs(item.ValidationResults));
if (!item.IsValid)
{
context.SkippedData.Add(item);
context.Data.Remove(item);
}
}
}
// TODO: release the event handler
}
internal override void GetData(IntegrationContext context)
{
string sourcePath = context.State[ContextState.SourcePath] as string;
if (string.IsNullOrEmpty(sourcePath))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.SourcePath));
}
if (!File.Exists(sourcePath))
{
throw new FileNotFoundException(string.Format("File [{0}] not found!", sourcePath));
}
if (null == context.Schema)
{
throw new ArgumentException("data schema missed!");
}
string tableName = context.State["tableName"] as string;
using (ExcelReader reader = new ExcelReader(sourcePath))
{
string[] tables = reader.GetExcelTableNames();
// it is vary possibly that csv data but filed end with xls. we need parse in csv format
if (tables == null)
{
throw new InvalidDataException(string.Format("[{0}]不是有效的excel文件", sourcePath));
}
if (tables.Length == 0)
{
throw new ArgumentException("无法找到excel页", sourcePath);
}
if (tableName == null ||
!Array.Exists(tables,
delegate(string t) { return(t.Equals(tableName, StringComparison.InvariantCultureIgnoreCase)); }))
{
tableName = tables[0];
}
DataTable dt = reader.GetTable(tableName);
DataTableStorage.GetDataFromSourceDataTable(dt, context);
}
}
public virtual void Run(IContext context)
{
IntegrationContext context1 = (IntegrationContext)context;
IntegrationMode mode;
try
{
mode = (IntegrationMode)context.State[name];
}
catch (Exception ex)
{
throw new Exception(string.Format("cannot get transfer action for provider [{0}]", name), ex);
}
if ((mode & IntegrationMode.CreateSchema) == IntegrationMode.CreateSchema)
{
CreateDataSchema(context1);
}
if ((mode & IntegrationMode.SaveSchema) == IntegrationMode.SaveSchema)
{
SaveDataSchema(context1);
}
if ((mode & IntegrationMode.GetSchema) == IntegrationMode.GetSchema)
{
GetDataSchema(context1);
}
if ((mode & IntegrationMode.GetData) == IntegrationMode.GetData)
{
GetData(context1);
}
//ValidateData(context1);
if ((mode & IntegrationMode.TransferData) == IntegrationMode.TransferData &&
((context1.HaltTransferWhenHasSkippedData && context1.SkippedCount == 0) ||
!context1.HaltTransferWhenHasSkippedData))
{
TransferData(context1);
}
if (PerTransferCount <= 0)
{
context1.Status = IntegrationStatus.Success;
}
}
internal override void GetData(IntegrationContext context)
{
DataTable dt = context.State["datatable"] as DataTable;
if (dt == null)
{
throw new ArgumentException("context.State[\"datatable\"]");
}
//if (dt.Rows.Count == 0)
// throw new ArgumentException("rows cannot be empty.");
if (null == context.Schema)
{
throw new ArgumentException("没有找到可用的数据结构!");
}
GetDataFromSourceDataTable(dt, context);
context.State["datatable"] = dt;
}
internal override void TransferData(IntegrationContext context)
{
string desDirectory = context.State["desDirectory"] as string;
if (string.IsNullOrEmpty(desDirectory))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", desDirectory));
}
if (!Directory.Exists(desDirectory))
{
throw new FileNotFoundException(string.Format("Directory [{0}] does not exist!", desDirectory));
}
if (null == context.Schema)
{
throw new ArgumentException("没有找到可用的数据结构!");
}
if (context.Data.Count == 0)
{
throw new ArgumentException("没有找到可用的数据!");
}
DataTable dt = DataTableStorage.GetEmptyDataTable(context.Schema);
foreach (DataItem item in context.Data)
{
DataTableStorage.PrepareDataForDestination(item, dt);
}
string file, message;
if (!ExcelReader.SaveData(desDirectory, dt, out file, out message))
{
// TODO: fails handle here
context.Status = IntegrationStatus.Failure;
}
else
{
context.State["desFile"] = file;
}
}
internal override void GetData(IntegrationContext context)
{
string sourcePath = context.State[ContextState.SourcePath] as string;
if (string.IsNullOrEmpty(sourcePath))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.SourcePath));
}
if (!File.Exists(sourcePath))
{
throw new FileNotFoundException(string.Format("File [{0}] not found!", sourcePath));
}
bool skipfirstline = (bool)context.State[ContextState.FlatFileSkipFirstLine];
bool skiplastline = (bool)context.State[ContextState.FlatFileSkipLastLine];
encoding = StringUtil.GetFileEncoding(sourcePath);
using (
StreamReader reader =
new StreamReader(new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), encoding))
{
if (skipfirstline)
{
GetLine(reader);
}
string row = null;
while (!reader.EndOfStream)
{
if (skiplastline)
{
if (row == null)
{
row = GetLine(reader);
continue;
}
}
else
{
row = GetLine(reader);
}
if (row.Trim().Length == 0)
{
continue;
}
DataItem item = GetDataItem(row, context.Schema, true);
if (item != null)
{
if (item.IsValid)
{
context.AddData(item);
}
else
{
context.AddSkippedData(item);
}
}
if (skiplastline)
{
row = GetLine(reader);
}
}
}
}
internal virtual void TransferData(IntegrationContext context)
{
throw new NotImplementedException();
}
internal override void CreateDataSchema(IntegrationContext context)
{
string sourcePath = context.State[ContextState.SourcePath] as string;
if (string.IsNullOrEmpty(sourcePath))
{
throw new ArgumentNullException(string.Format("缺少参数[{0}]!", ContextState.SourcePath));
}
if (!File.Exists(sourcePath))
{
throw new FileNotFoundException(string.Format("文件[{0}]不存在!", sourcePath));
}
string transfermode = context.State[ContextState.TransferMode] as string;
if (string.IsNullOrEmpty(transfermode))
{
transfermode = DataFixMode.Skip;
}
string tableName = context.State["tableName"] as string;
if (tableName != null && !tableName.EndsWith("$"))
{
tableName += "$";
}
using (ExcelReader reader = new ExcelReader(sourcePath))
{
string[] tables = reader.GetExcelTableNames();
// it is vary possibly that csv data but filed end with xls. we need parse in csv format
if (tables == null)
{
throw new InvalidDataException(string.Format("[{0}]不是有效的excel文件", sourcePath));
}
if (tables.Length == 0)
{
throw new ArgumentException("无法找到excel页", sourcePath);
}
if (tableName == null || !Array.Exists(tables, delegate(string t)
{
if (
t.Equals(tableName, StringComparison.InvariantCultureIgnoreCase))
{
return(true);
}
else if (
t.Equals(tableName + "'",
StringComparison.InvariantCultureIgnoreCase))
{
// incase the escape char '\''
tableName = t;
return(true);
}
else
{
return(false);
}
}))
{
tableName = tables[0];
}
DataTable dt = reader.GetTableSchema(tableName);
DataSchema dataSchema = new DataSchema();
dataSchema.TransferMode = transfermode;
List <DataSchemaField> fields = new List <DataSchemaField>();
for (int i = 0; i < dt.Rows.Count; i++)
{
DataSchemaField field = new DataSchemaField();
field.Name = field.Source = field.Destination = dt.Rows[i][3].ToString(); // COLUMN_NAME
// TODO: by alex hu 11 DATA_TYPE
field.Type = typeof(string).ToString();
field.Index = i;
fields.Add(field);
}
dataSchema.Fields = fields.ToArray();
context.Schema = dataSchema;
}
}
internal virtual void CreateDataSchema(IntegrationContext context)
{
throw new NotImplementedException();
}
internal override void CreateDataSchema(IntegrationContext context)
{
string sourcePath = context.State[ContextState.SourcePath] as string;
if (string.IsNullOrEmpty(sourcePath))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.SourcePath));
}
if (!File.Exists(sourcePath))
{
throw new FileNotFoundException(string.Format("File [{0}] not found!", sourcePath));
}
string delimiterString = context.State[ContextState.FlatFileDelimiter] as string;
if (string.IsNullOrEmpty(delimiterString))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.FlatFileDelimiter));
}
char delimiter = delimiterString[0];
string transfermode = context.State[ContextState.TransferMode] as string;
if (string.IsNullOrEmpty(transfermode))
{
transfermode = DataFixMode.Fix;
}
DataSchema dataSchema = new DataSchema();
dataSchema.SetAnyAttributeValue("delimiter", delimiter.ToString());
dataSchema.TransferMode = transfermode;
encoding = StringUtil.GetFileEncoding(sourcePath);
using (StreamReader reader = new StreamReader(sourcePath, encoding))
{
// maybe this line is included by quote, thus need seek the closed quote
string line = GetLine(reader);
string[] columns = GetLineColumns(line, delimiter);
List <DataSchemaField> fields = new List <DataSchemaField>();
for (int i = 0; i < columns.Length; i++)
{
// skip the empty column
if (columns[i].Trim().Length == 0)
{
continue;
}
DataSchemaField field = new DataSchemaField();
field.Name = field.Source = field.Destination = columns[i];
field.Type = typeof(string).ToString();
field.Index = i;
fields.Add(field);
}
dataSchema.Fields = fields.ToArray();
reader.Close();
}
context.Schema = dataSchema;
}
