public IEnumerable <Row> Read()
{
var batches = _process.Entities.Select(e => e.BatchId).ToArray();
var minBatchId = batches.Min();
var maxBatchId = batches.Max();
_output.Info("Batch Range: {0} to {1}.", minBatchId, maxBatchId);
var sql = $"SELECT {string.Join(",", _output.Entity.Fields.Select(f => "[" + f.Alias + "]"))} FROM [{_output.Process.Star}] WITH (NOLOCK) WHERE [TflBatchId] BETWEEN @MinBatchId AND @MaxBatchId;";
_output.Debug(sql);
using (var cn = new SqlConnection(_output.Connection.GetConnectionString())) {
cn.Open();
var cmd = cn.CreateCommand();
cmd.CommandTimeout = 0;
cmd.CommandType = CommandType.Text;
cmd.CommandText = sql;
cmd.Parameters.AddWithValue("@MinBatchId", minBatchId);
cmd.Parameters.AddWithValue("@MaxBatchId", maxBatchId);
var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess);
var rowCount = 0;
var fieldArray = _output.Entity.Fields.ToArray();
while (reader.Read())
{
rowCount++;
yield return(_rowCreator.Create(reader, fieldArray.Length + _output.Entity.CalculatedFields.Count, fieldArray));
}
_output.Info("{0} from {1}", rowCount, _output.Connection.Name);
}
}
public ActionResponse Execute()
{
var response = new ActionResponse();
try {
if (_output.Entity == null)
{
var directoryInfo = new DirectoryInfo(_output.Connection.Folder);
if (directoryInfo.Exists)
{
_output.Info("Deleting lucene index at {0}.", directoryInfo.FullName);
directoryInfo.Delete(true);
}
}
else
{
var directoryInfo = new DirectoryInfo(Path.Combine(_output.Connection.Folder, _output.Entity.Name));
if (directoryInfo.Exists)
{
_output.Info("Deleting lucene index at {0}.", directoryInfo.FullName);
directoryInfo.Delete(true);
}
}
} catch (Exception ex) {
response.Code = 500;
response.Message = $"Could not delete {_output.Connection.Folder}. {ex.Message}";
}
return(response);
}
int GetBatchId(IDbConnection cn)
{
var batchId = cn.Query <int>(_context.SqlControlLastBatchId()).FirstOrDefault() + 1;
_context.Info("Batch " + batchId);
return(batchId);
}
public async Task <int> RunDeployment()
{
ConsoleHelpers.PrintLogo();
var(configuration, rawConfigContents) = GetConfiguration();
var currentAppVersion = await applicationsClient.GetVersionAsync(
new ApplicationVersionQuery { ApplicationName = configuration.Name }
);
var deploymentVersion = (currentAppVersion.Version ?? 0) + 1;
var services = await ProcessServices(configuration, deploymentVersion);
output.Info("Contacting the clud server to kick off the deployment...");
var response = await deploymentsClient.DeployApplicationAsync(new DeployCommand
{
Name = configuration.Name.ToLowerInvariant(),
Owner = configuration.Owner,
Description = configuration.Description,
Repository = configuration.Repository,
EntryPoint = configuration.EntryPoint,
ConfigurationYaml = rawConfigContents,
Version = deploymentVersion,
CommitHash = await GitHelpers.CommitHash(Directory.GetParent(args.Config).FullName),
Services = { services },
});
LogSuccessfulDeployment(response);
return(0);
}
public void Write(IEnumerable <IRow> rows)
{
var tflHashCode = _output.Entity.TflHashCode();
var tflDeleted = _output.Entity.TflDeleted();
foreach (var part in rows.Partition(_output.Entity.InsertSize))
{
var inserts = new List <IRow>(_output.Entity.InsertSize);
var updates = new List <IRow>(_output.Entity.InsertSize);
var batchCount = 0;
if (_output.Entity.IsFirstRun)
{
foreach (var row in part)
{
inserts.Add(row);
batchCount++;
}
}
else
{
var batch = part.ToArray();
var matching = _matcher.Read(batch).ToArray();
for (int i = 0, batchLength = batch.Length; i < batchLength; i++)
{
var row = batch[i];
var match = matching.FirstOrDefault(f => f.Match(_keys, row));
if (match == null)
{
inserts.Add(row);
}
else
{
if (match[tflDeleted].Equals(true) || !match[tflHashCode].Equals(row[tflHashCode]))
{
updates.Add(row);
}
}
batchCount++;
}
}
if (inserts.Any())
{
_inserter.Write(inserts);
}
if (updates.Any())
{
_updater.Write(updates);
}
_output.Increment(batchCount);
}
_output.Info("{0} inserts into {1}", _output.Entity.Inserts, _output.Connection.Name);
_output.Info("{0} updates to {1}", _output.Entity.Updates, _output.Connection.Name);
}
public void Write(IEnumerable <IRow> rows)
{
if (!_output.Entity.NeedsUpdate())
{
return;
}
var ids = new SSASIdentifiers(_input, _output);
using (var server = new Server()) {
server.Connect($"Data Source={_output.Connection.Server};Catalog=;");
if (server.Databases.Contains(ids.DatabaseId))
{
var database = server.Databases.Find(ids.DatabaseId);
if (_output.Process.Mode == "init")
{
_output.Info($"Processing OLAP database {ids.DatabaseId}");
SSAS.Process(database, ProcessType.ProcessFull, _output);
}
else
{
if (database.Dimensions.Contains(_output.Entity.Alias))
{
_output.Info($"Updating dimension {_output.Entity.Alias}");
var dimension = database.Dimensions.Find(_output.Entity.Alias);
SSAS.Process(dimension, ProcessType.ProcessUpdate, _output);
}
else
{
_output.Error($"{_output.Entity.Alias} dimension does not exist!");
}
if (database.Cubes.Contains(ids.CubeId))
{
var cube = database.Cubes.Find(ids.CubeId);
ProcessPartition(cube, ids.NormalMeasureGroupId);
ProcessPartition(cube, ids.DistinctMeasureGroupId);
}
else
{
_output.Error($"{ids.CubeId} cube does not exist!");
}
}
}
else
{
_output.Error($"{ids.DatabaseId} OLAP database does not exist!");
}
server.Disconnect();
}
}
public IEnumerable <IRow> Read()
{
var batches = _parent.Entities.Select(e => e.BatchId).ToArray();
var minBatchId = batches.Min();
var maxBatchId = batches.Max();
_output.Info("Batch Range: {0} to {1}.", minBatchId, maxBatchId);
var sql = $"SELECT {string.Join(",", _output.Entity.Fields.Where(f=>f.Output).Select(f => _cf.Enclose(f.Alias)))} FROM {_cf.Enclose(_output.Process.Star)} {(_cf.AdoProvider == AdoProvider.SqlServer ? "WITH (NOLOCK)" : string.Empty)} WHERE {_cf.Enclose(Constants.TflBatchId)} BETWEEN @MinBatchId AND @MaxBatchId;";
_output.Debug(() => sql);
using (var cn = _cf.GetConnection()) {
cn.Open();
var cmd = cn.CreateCommand();
cmd.CommandTimeout = 0;
cmd.CommandType = CommandType.Text;
cmd.CommandText = sql;
var min = cmd.CreateParameter();
min.ParameterName = "@MinBatchId";
min.Value = minBatchId;
min.Direction = ParameterDirection.Input;
min.DbType = DbType.Int32;
var max = cmd.CreateParameter();
max.ParameterName = "@MaxBatchId";
max.Value = maxBatchId;
max.Direction = ParameterDirection.Input;
max.DbType = DbType.Int32;
cmd.Parameters.Add(min);
cmd.Parameters.Add(max);
var reader = cmd.ExecuteReader(CommandBehavior.SequentialAccess);
var rowCount = 0;
var fieldArray = _output.Entity.Fields.ToArray();
while (reader.Read())
{
rowCount++;
_output.Increment();
yield return(_rowCreator.Create(reader, fieldArray));
}
_output.Info("{0} from {1}", rowCount, _output.Connection.Name);
}
}
public void Update()
{
if (_master == null)
{
_output.Error("The master isn't set, which indicates your arrangement has errors.");
return;
}
var status = _output.GetEntityStatus();
if (!status.NeedsUpdate())
{
return;
}
using (var cn = _cf.GetConnection()) {
cn.Open();
var sql = _queryWriter.Write(status);
try {
var rowCount = cn.Execute(sql, new {
TflBatchId = _output.Entity.BatchId,
MasterTflBatchId = _master.BatchId
}, null, 0, System.Data.CommandType.Text);
_output.Info(rowCount + " updates to master");
} catch (DbException ex) {
_output.Error("error executing: {0}", sql);
_output.Error(ex, ex.Message);
}
}
}
public void Write(IEnumerable <Row> rows)
{
var sql = _context.SqlUpdateCalculatedFields(_original);
var temp = new List <Field> {
_context.Entity.Fields.First(f => f.Name == Constants.TflKey)
};
temp.AddRange(_context.Entity.CalculatedFields.Where(f => f.Output && f.Name != Constants.TflHashCode));
var fields = temp.ToArray();
var count = 0;
using (var cn = new SqlConnection(_context.Connection.GetConnectionString())) {
cn.Open();
foreach (var batch in rows.Partition(_context.Entity.UpdateSize))
{
var trans = cn.BeginTransaction();
var batchCount = cn.Execute(
sql,
batch.Select(r => r.ToExpandoObject(fields)),
trans,
_context.Connection.Timeout,
CommandType.Text
);
trans.Commit();
count += batchCount;
_context.Increment(batchCount);
}
_context.Info("{0} to {1}", count, _context.Connection.Name);
}
_context.Entity.Updates += count;
}
public void Write(IEnumerable <IRow> rows)
{
foreach (var row in rows)
{
var message = (string)row[_message] ?? string.Empty;
switch (row[_level].ToString().ToLower())
{
case "warn":
case "warning":
_context.Warn(message);
break;
case "error":
_context.Error(message);
break;
case "debug":
_context.Debug(() => message);
break;
default:
_context.Info(message);
break;
}
}
}
public ActionResponse Execute()
{
var open = _cf.AdoProvider == AdoProvider.Access ? "((" : string.Empty;
var close = _cf.AdoProvider == AdoProvider.Access ? ")" : string.Empty;
var command = $@"
INSERT INTO {_model.Flat}({string.Join(",", _model.Aliases)})
SELECT s.{string.Join(",s.", _model.Aliases)}
FROM {open}{_model.Master} m
LEFT OUTER JOIN {_model.Flat} f ON (f.{_model.EnclosedKeyLongName} = m.{_model.EnclosedKeyShortName}){close}
INNER JOIN {_model.Star} s ON (s.{_model.EnclosedKeyLongName} = m.{_model.EnclosedKeyShortName}){close}
WHERE f.{_model.EnclosedKeyLongName} IS NULL
AND m.{_model.Batch} > @Threshold;";
if (_cn.State != ConnectionState.Open)
{
_cn.Open();
}
try {
_output.Debug(() => command);
var count = _model.Threshold > 0 ? _cn.Execute(command, new { _model.Threshold }, commandTimeout: 0, transaction: _trans) : _cn.Execute(command, commandTimeout: 0, transaction: _trans);
_output.Info($"{count} record{count.Plural()} inserted into flat");
} catch (DbException ex) {
return(new ActionResponse(500, ex.Message));
}
return(new ActionResponse(200, "Ok"));
}
public void Delete(IEnumerable <IRow> rows)
{
var criteria = string.Join(" AND ", _output.Entity.GetPrimaryKey().Select(f => f.FieldName()).Select(n => _cf.Enclose(n) + " = @" + n));
var sql = $"UPDATE {_cf.Enclose(_output.Entity.OutputTableName(_output.Process.Name))} SET {_output.Entity.TflDeleted().FieldName()} = CAST(1 AS BIT), {_output.Entity.TflBatchId().FieldName()} = {_output.Entity.BatchId} WHERE {criteria}";
_output.Debug(() => sql);
var count = (uint)0;
using (var cn = _cf.GetConnection()) {
cn.Open();
foreach (var batch in rows.Partition(_output.Entity.DeleteSize))
{
var trans = cn.BeginTransaction();
var batchCount = Convert.ToUInt32(cn.Execute(
sql,
batch.Select(r => r.ToExpandoObject(_fields)),
trans,
0,
CommandType.Text
));
trans.Commit();
count += batchCount;
}
_output.Entity.Deletes += count;
if (_output.Entity.Deletes > 0)
{
_output.Info("{0} deletes from {1}", _output.Entity.Deletes, _output.Connection.Name);
}
}
}
public void Delete(IEnumerable <Row> rows)
{
var sql = _output.SqlDeleteOutput(_output.Entity.BatchId);
var count = 0;
using (var cn = new SqlConnection(_output.Connection.GetConnectionString())) {
cn.Open();
foreach (var batch in rows.Partition(_output.Entity.DeleteSize))
{
var trans = cn.BeginTransaction();
var batchCount = cn.Execute(
sql,
batch.Select(r => r.ToExpandoObject(_output.Entity.GetPrimaryKey())),
trans,
_output.Connection.Timeout,
CommandType.Text
);
trans.Commit();
count += batchCount;
_output.Increment(batchCount);
}
_output.Entity.Deletes += count;
_output.Info("{0} deletes from {1}", _output.Entity.Deletes, _output.Connection.Name);
}
}
public void Write(IEnumerable <IRow> rows)
{
var fullCount = 0;
var batchCount = (uint)0;
foreach (var part in rows.Partition(_context.Entity.InsertSize))
{
var docs = new List <Dictionary <string, object> >();
foreach (var row in part)
{
batchCount++;
fullCount++;
docs.Add(_fields.ToDictionary(field => field.Alias.ToLower(), field => row[field]));
}
var response = _solr.AddRange(docs);
_context.Increment(@by: batchCount);
if (response.Status == 0)
{
_context.Debug(() => $"{batchCount} to output");
}
else
{
_context.Error("ah!");
}
batchCount = 0;
}
_solr.Commit();
if (fullCount > 0)
{
_context.Info($"{fullCount} to output");
}
}
public ActionResponse Execute()
{
var updates = string.Join(", ", _model.Aliases.Where(f => f != _model.EnclosedKeyLongName).Select(f => $"f.{f} = s.{f}"));
var command = $@"
UPDATE f
SET {updates}
FROM {_model.Flat} f
INNER JOIN {_model.Master} m ON (m.{_model.EnclosedKeyShortName} = f.{_model.EnclosedKeyLongName})
INNER JOIN {_model.Star} s ON (f.{_model.EnclosedKeyLongName} = s.{_model.EnclosedKeyLongName})
WHERE m.{_model.Batch} > @Threshold;";
using (var cn = _cf.GetConnection()) {
cn.Open();
var trans = cn.BeginTransaction();
try {
_output.Debug(() => command);
var count = cn.Execute(command, new { _model.Threshold }, commandTimeout: 0, transaction: trans);
_output.Info($"{count} record{count.Plural()} updated in flat");
trans.Commit();
} catch (DbException ex) {
trans.Rollback();
return(new ActionResponse(500, ex.Message)
{
Action = new Action {
Type = "internal", Description = "Flatten Action", ErrorMode = "abort"
}
});
}
}
return(new ActionResponse(200, "Ok"));
}
public void Write(IEnumerable <IRow> rows)
{
_output.Entity.UpdateCommand = _output.SqlUpdateOutput(_cf);
var count = (uint)0;
using (var cn = _cf.GetConnection()) {
cn.Open();
_output.Debug(() => "begin transaction");
var trans = cn.BeginTransaction();
try {
foreach (var batch in rows.Partition(_output.Entity.UpdateSize))
{
var batchCount = Convert.ToUInt32(cn.Execute(
_output.Entity.UpdateCommand,
batch.Select(r => r.ToExpandoObject(_output.GetUpdateFields().ToArray())),
trans,
0,
CommandType.Text
));
count += batchCount;
}
_output.Debug(() => "commit transaction");
trans.Commit();
_output.Entity.Updates += count;
_output.Info("{0} to {1}", count, _output.Connection.Name);
} catch (Exception ex) {
_output.Error(ex.Message);
_output.Warn("rollback transaction");
trans.Rollback();
}
}
}
public ActionResponse Execute()
{
var message = "Ok";
var sqlInsert = _cf.AdoProvider == AdoProvider.SqlCe ?
$"INSERT INTO {_model.Flat}({string.Join(",", _model.Aliases)}) {_context.SqlSelectStar(_cf)}{_cf.Terminator}" :
$"INSERT INTO {_model.Flat}({string.Join(",", _model.Aliases)}) SELECT {string.Join(",", _model.Aliases)} FROM {_model.Star}{_cf.Terminator}";
using (var cn = _cf.GetConnection()) {
cn.Open();
var trans = cn.BeginTransaction();
try {
_context.Debug(() => sqlInsert);
var count = cn.Execute(sqlInsert, commandTimeout: 0, transaction: trans);
message = $"{count} record{count.Plural()} inserted into flat";
_context.Info(message);
trans.Commit();
} catch (DbException ex) {
trans.Rollback();
return(new ActionResponse(500, ex.Message)
{
Action = new Action {
Type = "internal",
Description = "Flatten Action",
ErrorMode = "abort"
}
});
}
}
return(new ActionResponse(200, message));
}
public void Run(Action <InputContext, OutputContext> action, bool wait = true)
{
var cout = new OutputContext(IsTestMode);
var cin = new InputContext(IsTestMode, _testFileName, cout, _readInputLimit);
Stopwatch sw = Stopwatch.StartNew();
cout.Info("Start task");
action(cin, cout);
cout.Info("End task. Elapsed: " + sw.Elapsed);
if (IsTestMode && wait)
{
Console.ReadKey();
}
}
public ActionResponse Execute()
{
var command = $@"
INSERT INTO {_model.Flat}({string.Join(",", _model.Aliases)})
SELECT s.{string.Join(",s.", _model.Aliases)}
FROM {_model.Master} m
LEFT OUTER JOIN {_model.Flat} f ON (f.{_model.EnclosedKeyLongName} = m.{_model.EnclosedKeyShortName})
INNER JOIN {_model.Star} s ON (s.{_model.EnclosedKeyLongName} = m.{_model.EnclosedKeyShortName})
WHERE f.{_model.EnclosedKeyLongName} IS NULL
AND m.{_model.Batch} > @Threshold;";
using (var cn = _cf.GetConnection()) {
cn.Open();
var trans = cn.BeginTransaction();
try {
_output.Debug(() => command);
var count = _model.Threshold > 0 ? cn.Execute(command, new { _model.Threshold }, commandTimeout: 0, transaction: trans) : cn.Execute(command, commandTimeout: 0, transaction: trans);
_output.Info($"{count} record{count.Plural()} inserted into flat");
trans.Commit();
} catch (DbException ex) {
trans.Rollback();
return(new ActionResponse(500, ex.Message));
}
}
return(new ActionResponse(200, "Ok"));
}
public ActionResponse Execute()
{
var masterAlias = Utility.GetExcelName(_model.MasterEntity.Index);
var builder = new StringBuilder();
builder.AppendLine($"INSERT INTO {_model.Flat}({string.Join(",", _model.Aliases)})");
builder.AppendLine($"SELECT {_output.SqlStarFields(_cf)}");
var close = _cf.AdoProvider != AdoProvider.Access ? string.Empty : ")";
foreach (var from in _output.SqlStarFroms(_cf))
{
builder.AppendLine(@from);
}
builder.AppendLine($"LEFT OUTER JOIN {_cf.Enclose(_output.Process.Name + _output.Process.FlatSuffix)} flat ON (flat.{_model.EnclosedKeyLongName} = {masterAlias}.{_model.EnclosedKeyShortName}){close}");
builder.AppendLine($" WHERE flat.{_model.EnclosedKeyLongName} IS NULL AND {masterAlias}.{_model.Batch} > @Threshold; ");
var command = builder.ToString();
if (_cn.State != ConnectionState.Open)
{
_cn.Open();
}
try {
_output.Debug(() => command);
var count = _model.Threshold > 0 ? _cn.Execute(command, new { _model.Threshold }, commandTimeout: 0, transaction: _trans) : _cn.Execute(command, commandTimeout: 0, transaction: _trans);
_output.Info($"{count} record{count.Plural()} inserted into flat");
} catch (DbException ex) {
return(new ActionResponse(500, ex.Message));
}
return(new ActionResponse(200, "Ok"));
}
public object GetMaxVersion()
{
if (string.IsNullOrEmpty(_output.Entity.Version))
{
return(null);
}
var version = _output.Entity.GetVersionField();
var versionName = version.Alias;
var deletedName = _output.Entity.TflDeleted().Alias;
var database = _output.Connection.Database;
var table = _output.Entity.Alias;
var t = Query.Db(database).Table <Dictionary <string, object> >(table);
var result = _output.Entity.Delete ?
_cn.Run(t.Filter(x => !(bool)x[deletedName]).Max(x => x[versionName])) :
_cn.Run(t.Max(x => x[versionName]));
var value = result[versionName];
if (value != null && value.GetType() != Constants.TypeSystem()[version.Type])
{
value = version.Convert(value);
}
_output.Info($"Found Version {value ?? "null"}");
return(value);
}
public void Write(IEnumerable <Row> rows)
{
var query = _output.SqlUpdateOutput(_output.Entity.BatchId);
var count = 0;
using (var cn = new SqlConnection(_output.Connection.GetConnectionString())) {
cn.Open();
foreach (var batch in rows.Partition(_output.Entity.UpdateSize))
{
var trans = cn.BeginTransaction();
var batchCount = cn.Execute(
query,
batch.Select(r => r.ToExpandoObject(_output.OutputFields)),
trans,
_output.Connection.Timeout,
CommandType.Text
);
trans.Commit();
count += batchCount;
_output.Increment(batchCount);
}
_output.Info("{0} to {1}", count, _output.Connection.Name);
}
_output.Entity.Updates += count;
}
public void Write(IEnumerable <IRow> rows)
{
var builder = new StringBuilder();
var fullCount = 0;
var batchCount = (uint)0;
foreach (var part in rows.Partition(_context.Entity.InsertSize))
{
foreach (var row in part)
{
batchCount++;
fullCount++;
foreach (var field in _context.OutputFields)
{
switch (field.Type)
{
case "guid":
row[field] = ((Guid)row[field]).ToString();
break;
case "datetime":
row[field] = ((DateTime)row[field]).ToString("o");
break;
}
if (field.SearchType == "geo_point")
{
row[field] = new Dictionary <string, string> {
{ "text", row[field].ToString() },
{ "location", row[field].ToString() }
};
}
}
builder.Append(_prefix);
foreach (var key in _context.OutputFields.Where(f => f.PrimaryKey))
{
builder.Append(row[key]);
}
builder.AppendLine("\"}}");
builder.AppendLine(JsonConvert.SerializeObject(_context.OutputFields.ToDictionary(f => f.Alias.ToLower(), f => row[f])));
}
var response = _client.Bulk <DynamicResponse>(builder.ToString(), nv => nv
.AddQueryString("refresh", @"true")
);
_context.Increment(@by: batchCount);
if (response.Success)
{
_context.Debug(() => $"{batchCount} to output");
}
else
{
_context.Error(response.OriginalException.Message);
}
builder.Clear();
batchCount = 0;
}
_context.Info($"{fullCount} to output");
}
public ActionResponse Execute()
{
var response = new ActionResponse();
var database = _output.Connection.Database;
var table = _output.Entity.Alias;
var conn = _factory.Get();
DmlResponse res;
if (!conn.Run(Query.DbList()).Contains(database))
{
res = conn.Run(Query.DbCreate(database));
_output.Info($"Created {database}");
}
IDatabaseQuery db = Query.Db(database);
if (conn.Run(db.TableList()).Contains(table))
{
res = conn.Run(db.TableDrop(table));
_output.Info($"Dropped {table}");
}
var keys = _output.Entity.GetPrimaryKey();
if (keys.Count() > 1)
{
_output.Error("You can't create a composite primary key in RethinkDB.");
}
else
{
if (keys.Any())
{
conn.Run(db.TableCreate(table, primaryKey: keys.First().Alias));
}
else
{
conn.Run(db.TableCreate(table));
}
_output.Info($"Created {table}");
}
return(response);
}
public void Update()
{
if (_output.Entity.ShouldUpdateMaster())
{
using (var cn = new SqlConnection(_output.Connection.GetConnectionString())) {
cn.Open();
var sql = _output.SqlUpdateMaster();
var rowCount = cn.Execute(sql, new { TflBatchId = _output.Entity.BatchId, MasterTflBatchId = _master.BatchId }, null, _output.Connection.Timeout, System.Data.CommandType.Text);
_output.Info(rowCount + " updates to master");
}
}
}
public void Write(IEnumerable <IRow> rows)
{
_originalConnectionLimit = ServicePointManager.DefaultConnectionLimit;
ServicePointManager.DefaultConnectionLimit = _context.Connection.MaxDegreeOfParallelism * 2;
try {
Parallel.ForEach(rows.Partition(_context.Entity.InsertSize), _options, part => {
var docs = new List <Dictionary <string, object> >();
foreach (var row in part)
{
Interlocked.Increment(ref _count);
docs.Add(_fields.ToDictionary(field => field.Alias.ToLower(), field => row[field]));
}
var response = _solr.AddRange(docs);
if (response.Status != 0)
{
_context.Error($"Couldn't add range of {docs.Count} document{docs.Count.Plural()} to SOLR.");
}
});
} catch (AggregateException ex) {
foreach (var exception in ex.InnerExceptions)
{
_context.Error(exception.Message);
_context.Error(exception.StackTrace);
}
return;
}
ServicePointManager.DefaultConnectionLimit = _originalConnectionLimit;
if (_count > 0)
{
try {
var commit = _solr.Commit();
if (commit.Status == 0)
{
_context.Entity.Inserts += Convert.ToUInt32(_count);
_context.Info($"Committed {_count} documents in {TimeSpan.FromMilliseconds(commit.QTime)}");
}
else
{
_context.Error($"Failed to commit {_count} documents. SOLR returned status {commit.Status}.");
}
} catch (SolrNetException ex) {
_context.Error($"Failed to commit {_count} documents. {ex.Message}");
}
}
}
public ActionResponse Execute()
{
var prefix = _model.AdoProvider == AdoProvider.PostgreSql ? string.Empty : "f.";
var updates = string.Join(", ", _model.Aliases.Where(f => f != _model.EnclosedKeyLongName).Select(f => $"{prefix}{f} = s.{f}"));
string command;
if (_model.AdoProvider == AdoProvider.PostgreSql)
{
command = $@"
UPDATE {_model.Flat} f
SET {updates}
FROM {_model.Master} m, {_model.Star} s
WHERE m.{_model.EnclosedKeyShortName} = f.{_model.EnclosedKeyLongName}
AND f.{_model.EnclosedKeyLongName} = s.{_model.EnclosedKeyLongName}
AND m.{_model.Batch} > @Threshold;
";
}
else
{
command = $@"
UPDATE f
SET {updates}
FROM {_model.Flat} f
INNER JOIN {_model.Master} m ON (m.{_model.EnclosedKeyShortName} = f.{_model.EnclosedKeyLongName})
INNER JOIN {_model.Star} s ON (f.{_model.EnclosedKeyLongName} = s.{_model.EnclosedKeyLongName})
WHERE m.{_model.Batch} > @Threshold;
";
}
if (_cn.State != ConnectionState.Open)
{
_cn.Open();
}
try {
_output.Debug(() => command);
var count = _cn.Execute(command, new { _model.Threshold }, commandTimeout: 0, transaction: _trans);
_output.Info($"{count} record{count.Plural()} updated in flat");
} catch (DbException ex) {
return(new ActionResponse(500, ex.Message)
{
Action = new Action {
Type = "internal", Description = "Flatten Action", ErrorMode = "abort"
}
});
}
return(new ActionResponse(200, "Ok"));
}
public string ReadLine()
{
if (IsTestMode)
{
if (_readCount == _readLimit)
{
_cout.Error("Read limit exeed!");
}
string result = null;
result = _reader.EndOfStream ? null : _reader.ReadLine();
if (_readCount == 0)
{
_cout.Info("First read from StdIn");
}
_readCount++;
if (_readCount == _readLimit)
{
_cout.Info("Last read from StdIn");
}
if (string.IsNullOrEmpty(result))
{
_cout.Warning("Empty result from StdIn");
}
return(result);
}
else
{
return(Console.ReadLine());
}
}
public void Write(IEnumerable <IRow> rows)
{
var fullCount = 0;
foreach (var part in rows.Partition(_context.Entity.InsertSize))
{
var batchCount = 0;
foreach (var row in part)
{
var doc = _documents[batchCount];
foreach (var field in _fields)
{
doc[field.Alias] = row[field];
}
batchCount++;
fullCount++;
}
var response = _solr.AddRange(_documents.Take(batchCount));
if (response.Status == 0)
{
var count = batchCount;
_context.Debug(() => $"{count} to output");
}
else
{
_context.Error($"Couldn't add range of documents to SOLR.");
}
}
if (fullCount > 0)
{
try {
var commit = _solr.Commit();
if (commit.Status == 0)
{
_context.Entity.Inserts += System.Convert.ToUInt32(fullCount);
_context.Info($"Committed {fullCount} documents in {TimeSpan.FromMilliseconds(commit.QTime)}");
}
else
{
_context.Error($"Failed to commit {fullCount} documents. SOLR returned status {commit.Status}.");
}
} catch (SolrNetException ex) {
_context.Error($"Failed to commit {fullCount} documents. {ex.Message}");
}
}
}
public ActionResponse Execute()
{
var response = new ActionResponse();
_output.Info("Deleting lucene index at {0}.", _output.Connection.Folder);
try {
var directoryInfo = new DirectoryInfo(_output.Connection.Folder);
if (directoryInfo.Exists)
{
directoryInfo.Delete(true);
}
} catch (Exception ex) {
response.Code = 500;
response.Message = $"Could not delete {_output.Connection.Folder}. {ex.Message}";
}
return(response);
}
