public bool Run(IDictionary state)
{
IntegrationContext context = new IntegrationContext(state);
try
{
// TODO: by alex hu, finished the status updating
//while (context.Status == IntegrationStatus.Running)
//{
foreach (IProvider provider in _providers)
{
provider.Run(context);
}
//}
}
catch (System.Exception ex)
{
Logger.Instance.Error(this, ex);
throw;
}
//bool result = false;
//switch(context.Status)
//{
// case IntegrationStatus.Completed:
// case IntegrationStatus.Success:
// result = true;
// break;
//}
return(true);
}
internal override void CreateDataSchema(IntegrationContext context)
{
DataTable dt = context.State["datatable"] as DataTable;
if (dt == null)
{
throw new ArgumentException("context.State[\"datatable\"]");
}
//if (dt.Rows.Count == 0)
// throw new ArgumentException("rows cannot be empty.");
DataSchema dataSchema = new DataSchema();
dataSchema.TransferMode = DataFixMode.Skip;
List <DataSchemaField> fields = new List <DataSchemaField>();
for (int i = 0; i < dt.Columns.Count; i++)
{
DataSchemaField field = new DataSchemaField();
field.Name = field.Source = field.Destination = dt.Columns[i].ColumnName;
field.Type = dt.Columns[i].DataType.ToString();
if (!dt.Columns[i].AllowDBNull)
{
field.SetAnyAttributeValue("required", "true");
}
field.Index = dt.Columns[i].Ordinal;
fields.Add(field);
}
dataSchema.Fields = fields.ToArray();
context.Schema = dataSchema;
}
internal override void CreateDataSchema(IntegrationContext context)
{
string connectionstring = context.State[ContextState.DatabaseConnectionString] as string;
if (string.IsNullOrEmpty(connectionstring))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.DatabaseConnectionString));
}
string tableName = context.State[ContextState.DatabaseTableName] as string;
if (string.IsNullOrEmpty(tableName))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.DatabaseTableName));
}
DataTable dt = new DataTable(tableName);
using (SqlConnection conn = new SqlConnection(connectionstring))
{
conn.Open();
SqlCommand cmd2 = new SqlCommand(string.Format("SELECT * FROM {0} WHERE 1<>1;", tableName), conn);
SqlDataAdapter adapter = new SqlDataAdapter(cmd2);
adapter.Fill(dt);
}
context.Schema = DataTableStorage.GetDataSchema(dt);
}
internal virtual void GetDataSchema(IntegrationContext context)
{
string targetPath = context.State[ContextState.SchemaFilePath] as string;
context.Schema = ValidationUtils.GetDataSchemaFromFile(targetPath);
if (context.Schema == null)
{
throw new System.Exception(string.Format("can not parsing schema based on [{0}]", targetPath));
}
}
internal override void TransferData(IntegrationContext context)
{
if (null == context.Schema)
{
throw new ArgumentException("没有找到可用的数据结构!");
}
if (context.Data.Count == 0)
{
throw new ArgumentException("没有找到可用的数据!");
}
DataTable dt = GetEmptyDataTable(context.Schema);
FillDataTable(dt, context.Data);
context.State["datatable"] = dt;
}
internal virtual void SaveDataSchema(IntegrationContext context)
{
string targetPath = context.State[ContextState.SchemaFilePath] as string;
if (string.IsNullOrEmpty(targetPath))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.SchemaFilePath));
}
using (TextWriter writer = new StreamWriter(targetPath))
{
XmlSerializer serializer = new XmlSerializer(typeof(DataSchema));
serializer.Serialize(writer, context.Schema);
writer.Flush();
}
}
internal override void TransferData(IntegrationContext context)
{
string connectionstring = context.State[ContextState.DatabaseConnectionString] as string;
if (string.IsNullOrEmpty(connectionstring))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.DatabaseConnectionString));
}
if (null == context.Schema)
{
throw new System.Exception("data schema missed!");
}
if (context.SuccessCount == 0)
{
throw new System.Exception("data missed!");
}
string tableName = context.State[ContextState.DatabaseTableName] as string;
if (string.IsNullOrEmpty(tableName))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.DatabaseTableName));
}
bool isTruncate = (bool)context.State[ContextState.DatabaseTruncateTable];
if (isTruncate)
{
TruncateData(connectionstring, tableName);
}
using (SqlConnection conn = new SqlConnection(connectionstring))
using (SqlBulkCopy bulkCopy = new SqlBulkCopy(conn))
{
conn.Open();
DataTable dt = new DataTable(tableName);
SqlCommand cmd2 = new SqlCommand(string.Format("SELECT * FROM {0} WHERE 1<>1;", tableName), conn);
SqlDataAdapter adapter = new SqlDataAdapter(cmd2);
adapter.Fill(dt);
MappingColumns(bulkCopy, dt, context.Schema);
DataTableStorage.FillDestinationDataItems(dt, context.Data);
bulkCopy.DestinationTableName = tableName;
bulkCopy.BatchSize = dt.Rows.Count;
bulkCopy.WriteToServer(dt);
}
}
internal static void GetDataFromSourceDataTable(DataTable dt, IntegrationContext context)
{
foreach (DataRow dr in dt.Rows)
{
DataItem item = GetDataItemFromSource(dr, context.Schema);
if (item != null)
{
if (item.IsValid)
{
context.AddData(item);
}
else
{
context.AddSkippedData(item);
}
}
}
}
protected void ValidateData(IntegrationContext context)
{
foreach (DataItem item in context.Data)
{
if (OnValidating != null)
{
DataItemValidationHandler handler = (DataItemValidationHandler)OnValidating.Clone();
handler.Invoke(item, new DataItemValidationArgs(item.ValidationResults));
if (!item.IsValid)
{
context.SkippedData.Add(item);
context.Data.Remove(item);
}
}
}
// TODO: release the event handler
}
public virtual void Run(IContext context)
{
IntegrationContext context1 = (IntegrationContext)context;
IntegrationMode mode;
try
{
mode = (IntegrationMode)context.State[name];
}
catch (System.Exception ex)
{
throw new System.Exception(string.Format("cannot get transfer action for provider [{0}]", name), ex);
}
if ((mode & IntegrationMode.CreateSchema) == IntegrationMode.CreateSchema)
{
CreateDataSchema(context1);
}
if ((mode & IntegrationMode.SaveSchema) == IntegrationMode.SaveSchema)
{
SaveDataSchema(context1);
}
if ((mode & IntegrationMode.GetSchema) == IntegrationMode.GetSchema)
{
GetDataSchema(context1);
}
if ((mode & IntegrationMode.GetData) == IntegrationMode.GetData)
{
GetData(context1);
}
//ValidateData(context1);
if ((mode & IntegrationMode.TransferData) == IntegrationMode.TransferData &&
((context1.HaltTransferWhenHasSkippedData && context1.SkippedCount == 0) ||
!context1.HaltTransferWhenHasSkippedData))
{
TransferData(context1);
}
if (PerTransferCount <= 0)
{
context1.Status = IntegrationStatus.Success;
}
}
internal override void GetData(IntegrationContext context)
{
DataTable dt = context.State["datatable"] as DataTable;
if (dt == null)
{
throw new ArgumentException("context.State[\"datatable\"]");
}
//if (dt.Rows.Count == 0)
// throw new ArgumentException("rows cannot be empty.");
if (null == context.Schema)
{
throw new ArgumentException("没有找到可用的数据结构!");
}
GetDataFromSourceDataTable(dt, context);
context.State["datatable"] = dt;
}
internal virtual void CreateDataSchema(IntegrationContext context)
{
throw new NotImplementedException();
}
internal virtual void TransferData(IntegrationContext context)
{
throw new NotImplementedException();
}
internal override void GetData(IntegrationContext context)
{
string sourcePath = context.State[ContextState.SourcePath] as string;
if (string.IsNullOrEmpty(sourcePath))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.SourcePath));
}
if (!File.Exists(sourcePath))
{
throw new FileNotFoundException(string.Format("File [{0}] not found!", sourcePath));
}
bool skipfirstline = (bool)context.State[ContextState.FlatFileSkipFirstLine];
bool skiplastline = (bool)context.State[ContextState.FlatFileSkipLastLine];
encoding = StringUtil.GetFileEncoding(sourcePath);
using (
StreamReader reader =
new StreamReader(new FileStream(sourcePath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite), encoding))
{
if (skipfirstline)
{
GetLine(reader);
}
string row = null;
while (!reader.EndOfStream)
{
if (skiplastline)
{
if (row == null)
{
row = GetLine(reader);
continue;
}
}
else
{
row = GetLine(reader);
}
if (row.Trim().Length == 0)
{
continue;
}
DataItem item = GetDataItem(row, context.Schema, true);
if (item != null)
{
if (item.IsValid)
{
context.AddData(item);
}
else
{
context.AddSkippedData(item);
}
}
if (skiplastline)
{
row = GetLine(reader);
}
}
}
}
internal override void CreateDataSchema(IntegrationContext context)
{
string sourcePath = context.State[ContextState.SourcePath] as string;
if (string.IsNullOrEmpty(sourcePath))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.SourcePath));
}
if (!File.Exists(sourcePath))
{
throw new FileNotFoundException(string.Format("File [{0}] not found!", sourcePath));
}
string delimiterString = context.State[ContextState.FlatFileDelimiter] as string;
if (string.IsNullOrEmpty(delimiterString))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", ContextState.FlatFileDelimiter));
}
char delimiter = delimiterString[0];
string transfermode = context.State[ContextState.TransferMode] as string;
if (string.IsNullOrEmpty(transfermode))
{
transfermode = DataFixMode.Fix;
}
DataSchema dataSchema = new DataSchema();
dataSchema.SetAnyAttributeValue("delimiter", delimiter.ToString());
dataSchema.TransferMode = transfermode;
encoding = StringUtil.GetFileEncoding(sourcePath);
using (StreamReader reader = new StreamReader(sourcePath, encoding))
{
// maybe this line is included by quote, thus need seek the closed quote
string line = GetLine(reader);
string[] columns = GetLineColumns(line, delimiter);
List <DataSchemaField> fields = new List <DataSchemaField>();
for (int i = 0; i < columns.Length; i++)
{
// skip the empty column
if (columns[i].Trim().Length == 0)
{
continue;
}
DataSchemaField field = new DataSchemaField();
field.Name = field.Source = field.Destination = columns[i];
field.Type = typeof(string).ToString();
field.Index = i;
fields.Add(field);
}
dataSchema.Fields = fields.ToArray();
reader.Close();
}
context.Schema = dataSchema;
}
internal override void TransferData(IntegrationContext context)
{
string desDirectory = context.State["desDirectory"] as string;
if (string.IsNullOrEmpty(desDirectory))
{
throw new ArgumentNullException(string.Format("argument [{0}] missed!", desDirectory));
}
if (!Directory.Exists(desDirectory))
{
throw new FileNotFoundException(string.Format("Directory [{0}] does not exist!", desDirectory));
}
string desPrefix = context.State["desPrefix"] as string;
if (null == context.Schema)
{
throw new ArgumentException("没有找到可用的数据结构!");
}
//if (context.Data.Count == 0)
// throw new ArgumentException("没有找到可用的数据!");
string file;
lock (_syncRoot)
{
file = string.Concat(desPrefix, "_", DateTime.Now.ToString("yyyyMMddHHmmssfff"), ".csv");
}
string filepath = Path.Combine(desDirectory, file);
FileStream fs = null;
try
{
fs = new FileStream(filepath, FileMode.Create);
using (StreamWriter writer = new StreamWriter(fs, Encoding.Default))
{
fs = null;
// write column header
WriteHeaderLine(context.Schema, writer);
// write content
foreach (DataItem item in context.Data)
{
WriteItemLine(item, context.Schema, writer);
}
writer.Flush();
}
}
finally
{
if (fs != null)
{
fs.Dispose();
}
}
context.State["desFile"] = file;
}
