/// <summary>
/// Gets the data.
/// </summary>
/// <param name="columns">The columns.</param>
/// <param name="previousHighWatermark">The previous high watermark.</param>
/// <returns>
/// A streaming reader for the data.
/// </returns>
public IDataReader GetData(IEnumerable <string> columns, Watermark previousHighWatermark)
{
try
{
IEnumerable <IFileInfo> files = this.FindFiles(previousHighWatermark);
if (files.Count() == 0)
{
return(null);
}
EventPublisher.Raise(
new SourceFileFoundEvent
{
OperationId = this.Parent.OperationId,
SourceName = this.name,
ConnectionName = this.ConnectionName,
FoundFiles = files
});
return(this.fileSystem.CreateDataReader(files, this.config));
}
catch (IOException ex)
{
if (ex.IsFileLocked())
{
EventPublisher.Raise(
new SourceFileLockedEvent
{
OperationId = this.Parent.OperationId,
SourceName = this.Name,
ConnectionName = this.ConnectionName,
Exception = ex
});
// Back off and wait till next time.
return(null);
}
EventPublisher.Raise(
new SourceErrorEvent
{
OperationId = this.Parent.OperationId,
SourceName = this.Name,
ConnectionName = this.ConnectionName,
Exception = ex
});
throw;
}
}
/// <summary>
/// Updates the high watermark.
/// </summary>
/// <param name="table">The table.</param>
/// <param name="newValue">The new value.</param>
public void UpdateHighWatermark(string table, Watermark newValue)
{
var watermarkFile = this.FormatWatermarkFileName(table);
if (!File.Exists(watermarkFile))
{
File.Delete(watermarkFile);
}
using (var file = new StreamWriter(watermarkFile))
{
var json = JsonConvert.SerializeObject(newValue);
file.Write(json);
}
}
/// <summary>
/// Gets the command text.
/// </summary>
/// <param name="columns">The target columns.</param>
/// <param name="previousHighWatermark">The previous high watermark.</param>
/// <returns>
/// A SQL query.
/// </returns>
protected override string GetCommandText(IEnumerable <string> columns, Watermark previousHighWatermark)
{
var query = new StringBuilder();
query
.Append("select ")
.Append(columns.Aggregate(string.Empty, (a, v) => a + (a.Length == 0 ? v : ", " + v)))
.Append(columns.Count() == 0 ? "*" : string.Empty)
.AppendFormat(CultureInfo.InvariantCulture, " from {0}", this.tableName);
if (previousHighWatermark != null)
{
query.AppendFormat(CultureInfo.InvariantCulture, " where {0} > @highWatermark", previousHighWatermark.WatermarkColumn);
}
return(query.ToString());
}
/// <summary>
/// Gets the columns.
/// </summary>
/// <param name="previousHighWatermark">The previous high watermark.</param>
/// <returns>
/// A sequence of columns.
/// </returns>
public IEnumerable <string> GetColumns(Watermark previousHighWatermark)
{
// The CSV source is a little complex because we derive the column list from real
// source data. If there are files we need to use the actual one that is going to be
// imported. If not then any file will do - even ones below the watermark.
using (var latestData = this.GetData(Enumerable.Empty <string>(), previousHighWatermark))
using (var sampleData = this.GetData(Enumerable.Empty <string>(), null))
{
var columnData = latestData ?? sampleData;
if (columnData == null)
{
yield break;
}
for (int i = 0; i < columnData.FieldCount; ++i)
{
yield return(columnData.GetName(i));
}
}
}
/// <summary>
/// Updates the high watermark using the current value from the staging table.
/// </summary>
/// <returns>
/// A watermark instance.
/// </returns>
public Watermark UpdateHighWatermark()
{
if (this.highWatermarkColumn == null)
{
return(null);
}
var sql = string.Format(CultureInfo.CurrentCulture, "select max({0}) from {1};", this.highWatermarkColumn, this.tableName);
var result = this.connectionFactory.ExecuteScalar(sql, this.OperationId);
if (result == null || result == DBNull.Value)
{
return(null);
}
var newHighwaterMark = new Watermark(this.highWatermarkColumn, (DateTime)result);
this.watermarkService.UpdateHighWatermark(this.tableName, newHighwaterMark);
return(newHighwaterMark);
}
public IDataReader GetData(IEnumerable <string> columns, Watermark previousHighWatermark)
{
try
{
var connection = this.connectionFactory.CreateConnection();
connection.Open();
var command = connection.CreateCommand();
command.CommandText = this.GetCommandText(columns, previousHighWatermark);
command.CommandTimeout = 0;
var highWatermarkParameterName = this.connectionFactory.MakeParameterName("highWatermark");
if (command.CommandText.Contains(highWatermarkParameterName))
{
var watermarkParameter = command.CreateParameter();
watermarkParameter.ParameterName = highWatermarkParameterName;
watermarkParameter.Value = previousHighWatermark == null ? Settings.Default.DefaultHighWatermark : previousHighWatermark.WatermarkValue;
command.Parameters.Add(watermarkParameter);
}
DittoEventSource.Log.DatabaseQuery(this.connectionFactory.Name, command.CommandText, this.OperationId);
return(command.ExecuteReader(CommandBehavior.CloseConnection));
}
catch (DbException ex)
{
EventPublisher.Raise(
new SourceErrorEvent
{
OperationId = this.Parent.OperationId,
SourceName = this.Name,
ConnectionName = this.connectionFactory.Name,
Exception = ex
});
throw;
}
}
/// <summary>
/// Finds the files.
/// </summary>
/// <param name="previousHighWatermark">The previous high watermark.</param>
/// <returns>A sequence of files to import.</returns>
private IEnumerable <IFileInfo> FindFiles(Watermark previousHighWatermark)
{
var allFiles = this.fileSystem.EnumerateFiles(this.connection.ConnectionString, this.filePattern)
.Where(f => previousHighWatermark == null || f.LastWriteTimeUtc > previousHighWatermark.WatermarkValue);
IEnumerable <IFileInfo> importFiles = null;
switch (this.mode)
{
case FolderMode.MultipleFile:
importFiles = allFiles.OrderBy(f => f.LastWriteTimeUtc);
break;
case FolderMode.SingleFile:
importFiles = allFiles.OrderBy(f => f.LastWriteTimeUtc).Take(1);
break;
case FolderMode.LastFileOnly:
importFiles = allFiles.OrderByDescending(f => f.LastWriteTimeUtc).Take(1);
break;
}
return(importFiles);
}
/// <summary>
/// Determines whether the specified previous high watermark has data.
/// </summary>
/// <param name="previousHighWatermark">The previous high watermark.</param>
/// <returns>
/// True if there is data.
/// </returns>
/// <remarks>
/// This allows a shortcut for sources that may not exist e.g. files.
/// </remarks>
public bool HasData(Watermark previousHighWatermark)
{
return(this.FindFiles(previousHighWatermark).Any());
}
/// <summary>
/// Gets the command text.
/// </summary>
/// <param name="columns">The target columns.</param>
/// <param name="previousHighWatermark">The previous high watermark.</param>
/// <returns>
/// A SQL query.
/// </returns>
protected override string GetCommandText(IEnumerable<string> columns, Watermark previousHighWatermark)
{
return this.query;
}
/// <summary> /// Gets the command text. /// </summary> /// <param name="columns">The target columns.</param> /// <param name="previousHighWatermark">The previous high watermark.</param> /// <returns> /// A SQL query. /// </returns> protected abstract string GetCommandText(IEnumerable <string> columns, Watermark previousHighWatermark);
/// <summary>
/// Determines whether the specified previous high watermark has data.
/// </summary>
/// <param name="previousHighWatermark">The previous high watermark.</param>
/// <returns>
/// True if there is data.
/// </returns>
/// <remarks>
/// This allows a shortcut for sources that may not exist e.g. files.
/// </remarks>
public bool HasData(Watermark previousHighWatermark)
{
// Databases always "have" data even if there are no rows.
return(true);
}