protected override Task <DataFlowResult> Store(DataFlowContext context)
{
foreach (var item in context.GetParseItems())
{
var tableMetadata = (TableMetadata)context[item.Key];
var file = Path.Combine(Framework.BaseDirectory, "mysql-files",
$"{GenerateFileName(tableMetadata)}.sql");
switch (MySqlFileType)
{
case MySqlFileType.LoadFile:
{
WriteLoadFile(file, tableMetadata, item.Value);
break;
}
case MySqlFileType.InsertSql:
{
WriteInsertFile(file, tableMetadata, item.Value);
break;
}
}
}
return(Task.FromResult(DataFlowResult.Success));
}
protected override async Task <DataFlowResult> Store(DataFlowContext context)
{
foreach (var item in context.GetParseItems())
{
var tableMetadata = (TableMetadata)context[item.Key];
StreamWriter writer = CreateOrOpen(context, tableMetadata, "json");
await writer.WriteLineAsync(JsonConvert.SerializeObject(item.Value));
}
return(DataFlowResult.Success);
}
protected override async Task <DataFlowResult> Store(DataFlowContext context)
{
foreach (var item in context.GetParseItems())
{
var tableMetadata = (TableMetadata)context[item.Key];
var file = Path.Combine(Framework.BaseDirectory, "json",
$"{GenerateFileName(tableMetadata)}.json");
StreamWriter writer = CreateOrOpen(file);
await writer.WriteLineAsync(JsonConvert.SerializeObject(item.Value));
}
return(DataFlowResult.Success);
}
protected override Task <DataFlowResult> Store(DataFlowContext context)
{
foreach (var item in context.GetParseItems())
{
var tableMetadata = (TableMetadata)context[item.Key];
switch (MySqlFileType)
{
case MySqlFileType.LoadFile:
{
WriteLoadFile(context, tableMetadata, item.Value);
break;
}
case MySqlFileType.InsertSql:
{
WriteInsertFile(context, tableMetadata, item.Value);
break;
}
}
}
return(Task.FromResult(DataFlowResult.Success));
}
protected override async Task <DataFlowResult> Store(DataFlowContext context)
{
IDbConnection conn = TryCreateDbConnection(context);
using (conn)
{
foreach (var item in context.GetParseItems())
{
var tableMetadata = (TableMetadata)context[item.Key];
SqlStatements sqlStatements = GetSqlStatements(tableMetadata);
lock (this)
{
EnsureDatabaseAndTableCreated(conn, sqlStatements);
}
for (int i = 0; i < RetryTimes; ++i)
{
IDbTransaction transaction = null;
try
{
if (UseTransaction)
{
transaction = conn.BeginTransaction();
}
var list = item.Value;
switch (StorageType)
{
case StorageType.Insert:
{
await conn.ExecuteAsync(sqlStatements.InsertSql, list, transaction);
break;
}
case StorageType.InsertIgnoreDuplicate:
{
await conn.ExecuteAsync(sqlStatements.InsertIgnoreDuplicateSql, list, transaction);
break;
}
case StorageType.Update:
{
if (string.IsNullOrWhiteSpace(sqlStatements.UpdateSql))
{
throw new SpiderException("未能生成更新 SQL");
}
await conn.ExecuteAsync(sqlStatements.UpdateSql, list, transaction);
break;
}
case StorageType.InsertAndUpdate:
{
await conn.ExecuteAsync(sqlStatements.InsertAndUpdateSql, list, transaction);
break;
}
}
transaction?.Commit();
break;
}
catch (Exception ex)
{
Logger?.LogError($"尝试插入数据失败: {ex}");
// 网络异常需要重试，并且不需要 Rollback
var endOfStreamException = ex.InnerException as EndOfStreamException;
if (endOfStreamException == null)
{
try
{
transaction?.Rollback();
}
catch (Exception e)
{
Logger?.LogError($"数据库回滚失败: {e}");
}
break;
}
}
finally
{
transaction?.Dispose();
}
}
}
}
return(DataFlowResult.Success);
}