public override bool Validate(out string message)
{
try
{
var ok = true;
//IsLoading = true;
var task = Task.Run(async() =>
{
var allAttributes = attributeRepository.GetAll();
foreach (var attr in allAttributes)
{
var initialized = true;
var entity = entityRepository.GetById(attr.EntityId.ToString());
var options = attributeRepository.LoadOptions(attr.Id.ToString());
var connection = connectionRepository.GetById(attr.SourceConnectionId.ToString());
var puller = pullers.FirstOrDefault(p => p.IsImplemented(attr.SourceProcessorId, entity.SourceProcessorId, connection.ProviderId));
var indexer = indexers.FirstOrDefault(p => p.IsImplemented(attr.SourceProcessorId, entity.SourceProcessorId, connection.ProviderId));
puller.SetIndex(attr);
puller.SetOptions(options.Select(o => new OptionItem {
Name = o.Key, Value = o.Value
}));
initialized = initialized && puller.Initialized();
indexer.SetIndex(attr);
indexer.SetOptions(options.Select(o => new OptionItem {
Name = o.Key, Value = o.Value
}));
initialized = initialized && entityRepository.Initialized(attr);
ok = ok && initialized;
if (!ok)
{
logger.Information($@"Index ""{entity.Name}"" is not initialized.");
break;
}
}
});
task.Wait();
message = "All attributes has been initialized.";
logger.Information(message);
return(true);
}
catch (Exception ex)
{
errorLogger.Error(ex, ex.Message);
throw;
}
finally
{
//IsLoading = false;
}
return(true);
}
public override PullResult Preview()
{
var options = AttributeRepository.LoadOptions(AttributeModel.Id.ToString());
var limit = options.GetValue("puller_page_limit", 100);
var offset = 0;
var sqlScript = GetSqlScript(options, false); // should call raw SQL instead of calling view
var sets = adapter.Query(sqlScript, new
{
Limit = limit,
Offset = offset
});
var set = sets.FirstOrDefault();
var results = set?.Rows?.Select(r => {
var jObj = JObject.FromObject(r);
jObj.Remove("RowNum");
return(jObj.ToObject(typeof(object)));
});
return(new PullResult
{
Status = results?.Count() > 0 ? PullState.HasData : PullState.Invalid,
LastToken = new
{
Limit = limit,
Offset = offset
},
Data = results
});
}
private string GetAttributeMessage(QueueItemModel item, out IIndexModel indexModel, out IndexItemModel itemModel)
{
var attributeModel = attributeRepository.GetById(item.TargetEntityId.ToString());
var entityModel = entityRepository.GetById(attributeModel.EntityId.ToString());
var entityOptions = entityRepository.LoadOptions(entityModel.Id.ToString(), new List <string> {
"Indexer"
});
var attributeOptions = attributeRepository.LoadOptions(attributeModel.Id.ToString(), new List <string> {
"Indexer"
});
var attributeItemModel = attributeRepository.GetIndexedItemById(attributeModel, item.TargetItemId);
var entityItemModel = entityRepository.GetIndexedItemBySourceId(entityModel, attributeItemModel.GetSourceId());
itemModel = attributeItemModel;
var entityReporterMappingOption = entityOptions.FirstOrDefault(o => o.Key == "indexer_reporter_columns");
var entityReporterMappingColumns = !string.IsNullOrWhiteSpace(entityReporterMappingOption?.Value)
? JsonConvert.DeserializeObject <List <ReporterColumnMapping> >(entityReporterMappingOption.Value)
: new List <ReporterColumnMapping> {
new ReporterColumnMapping {
SourceName = "Value",
MappingName = "Name",
Key = true,
Value = true
}
};
var attributeReporterMappingOption = attributeOptions.FirstOrDefault(o => o.Key == "indexer_reporter_columns");
var attributeReporterMappingColumns = !string.IsNullOrWhiteSpace(attributeReporterMappingOption?.Value)
? JsonConvert.DeserializeObject <List <ReporterColumnMapping> >(attributeReporterMappingOption.Value)
: new List <ReporterColumnMapping> {
new ReporterColumnMapping {
SourceName = "Value",
MappingName = "Name",
Key = true,
Value = true
}
};
var keys = entityReporterMappingColumns.Where(r => r.Key)
.Select(r => $@"_{r.MappingName}_: {entityItemModel.GetValue(r.SourceName)?.ToString() ?? "(empty)"}");
var vals = attributeReporterMappingColumns.Where(r => r.Value)
.Select(r => $@"_{r.MappingName}_: {attributeItemModel.GetValue(r.SourceName)?.ToString() ?? "(empty)"}");
if (attributeReporterMappingColumns.Count == 1)
{
vals = attributeReporterMappingColumns.Where(r => r.Value)
.Select(r => $@"{attributeItemModel.GetValue(r.SourceName)?.ToString() ?? "(empty)"}");
}
var executed = item.ExecutedAt.UnixTimeToTime().ToString("G");
var executedIn = item.ExecutedAt - item.ExecuteAt;
indexModel = attributeModel;
return($@"*{indexModel.Name}* ({string.Join(", ", keys)}): {vals} {executed} in {executedIn} second(s)");
}
public override PullResult PullNext(object lastToken = null)
{
var options = AttributeRepository.LoadOptions(AttributeModel.Id.ToString());
var totalCount = GetCount(options, true);
var limit = options.GetValue("puller_page_limit", 100);
var offset = 0;
if (lastToken != null)
{
var jToken = JObject.FromObject(lastToken);
if (jToken != null && jToken.ContainsKey("Limit") && jToken.ContainsKey("Offset"))
{
limit = int.Parse(jToken.GetValue("Limit").ToString());
offset = int.Parse(jToken.GetValue("Offset").ToString());
offset = offset + limit;
}
}
if (offset > totalCount) // AccessDb never knows how to stop
{
return(new PullResult
{
Status = PullState.Invalid,
LastToken = new
{
Limit = limit,
Offset = offset
},
Data = null
});
}
var sqlScript = GetSqlScript(options, limit, offset + limit, true);
var sets = adapter.Query(sqlScript, new
{
Limit = limit,
Offset = offset
});
var set = sets.FirstOrDefault();
var results = set?.Rows;
return(new PullResult
{
Status = results?.Count() > 0 ? PullState.HasData : PullState.Invalid,
LastToken = new
{
Limit = limit,
Offset = offset
},
Data = results
});
}
public IActionResult GetOptions(Guid id, [FromBody] AttributeTemplateOptionRequestViewModel model)
{
var attribute = attributeRepository.GetById(id.ToString());
var entityModel = entityRepository.GetById(model.EntityId);
var options = attributeRepository.LoadOptions(id.ToString());
var instanceOptions = options.Select(o => new OptionItem
{
Name = o.Key,
Value = o.Value
}).ToList();
ConnectionModel sourceConnection = null;
ConnectionModel destinationConnection = null;
IAttributePuller puller = null;
IAttributePusher pusher = null;
IIndexer indexer;
if (!string.IsNullOrWhiteSpace(model.SourceConnectionId))
{
sourceConnection = connectionRepository.GetById(model.SourceConnectionId);
puller = pullers.FirstOrDefault(p => p.IsImplemented(model.SourceProcessorId, entityModel.SourceProcessorId, sourceConnection.ProviderId));
puller.SetOptions(instanceOptions);
}
if (!string.IsNullOrWhiteSpace(model.DestinationConnectionId))
{
destinationConnection = connectionRepository.GetById(model.DestinationConnectionId);
pusher = pushers.FirstOrDefault(p => p.IsImplemented(model.DestinationProcessorId, entityModel.DestinationProcessorId, destinationConnection.ProviderId));
pusher.SetOptions(instanceOptions);
}
indexer = indexers.FirstOrDefault(i => i.IsImplemented(model.SourceConnectionId, entityModel.SourceConnectionId.ToString(), model.SourceProcessorId));
indexer?.SetOptions(instanceOptions);
return(Ok(new
{
Puller = puller?.Options ?? new List <OptionItem>(),
Indexer = indexer?.Options ?? new List <OptionItem>(),
Pusher = pusher?.Options ?? new List <OptionItem>(),
}));
}
public override IPuller Init()
{
var options = AttributeRepository.LoadOptions(AttributeModel.Id.ToString());
var sqlScript = options.GetValue("puller_sql_script");
var viewExists = adapter.GetView(AttributeModel.SourceViewName);
if (viewExists != null)
{
adapter.DropView(AttributeModel.SourceViewName);
}
var createViewSQL = $@"
CREATE VIEW {AttributeModel.SourceViewName}
AS
{sqlScript}";
adapter.Execute(createViewSQL);
return(this);
}
public override PullResult PullNext(object lastToken = null)
{
var options = AttributeRepository.LoadOptions(AttributeModel.Id.ToString());
var limit = options.GetValue("puller_page_limit", 100);
var offset = 0;
if (lastToken != null)
{
var jToken = JObject.FromObject(lastToken);
if (jToken != null && jToken.ContainsKey("Limit") && jToken.ContainsKey("Offset"))
{
limit = int.Parse(jToken.GetValue("Limit").ToString());
offset = int.Parse(jToken.GetValue("Offset").ToString());
offset = offset + limit;
}
}
var sqlScript = GetSqlScript(options, true);
var sets = adapter.Query(sqlScript, new
{
Limit = limit,
Offset = offset
});
var set = sets.FirstOrDefault();
var results = set?.Rows?.Select(r => {
var jObj = JObject.FromObject(r);
jObj.Remove("RowNum");
return(jObj.ToObject(typeof(object)));
});
return(new PullResult
{
Status = results?.Count() > 0 ? PullState.HasData : PullState.Invalid,
LastToken = new
{
Limit = limit,
Offset = offset
},
Data = results
});
}
public override IPuller Init()
{
var options = AttributeRepository.LoadOptions(AttributeModel.Id.ToString());
var sqlScript = options.GetValue("puller_sql_script");
var truncateSQL = $@"
IF EXISTS (
SELECT * FROM sys.views
WHERE name = N'{AttributeModel.SourceViewName}'
)
BEGIN
DROP VIEW [{AttributeModel.SourceViewName}];
END
";
adapter.Execute(truncateSQL);
var createViewSQL = $@"
CREATE VIEW [{AttributeModel.SourceViewName}]
AS
{sqlScript}";
adapter.Execute(createViewSQL);
return(this);
}
public override async Task Invoke(IStepExecutionContext context = null)
{
var executeAt = DateTime.Now.ToUnixTimestamp();
var firstQueuedItem = entityRepository.GetCurrentQueuedItems();
if (firstQueuedItem == null)
{
return;
}
IIndexModel indexModel = null;
IndexItemModel itemModel = null;
IIndexer indexer = null;
IPusher pusher = null;
IEnumerable <OptionItem> options = null;
if (firstQueuedItem.TargetEntityType == EntityType.Entity)
{
indexModel = entityRepository.GetById(firstQueuedItem.TargetEntityId.ToString());
options = entityRepository.LoadOptions(indexModel.Id.ToString()).Select(o => new OptionItem {
Name = o.Key, Value = o.Value
});
var sourceConnection = connectionRepository.GetById(indexModel.SourceConnectionId.ToString());
var destinationConnection = connectionRepository.GetById(indexModel.DestinationConnectionId.ToString());
indexer = entityIndexers.FirstOrDefault(i => i.IsImplemented(indexModel.SourceProcessorId, sourceConnection.ProviderId));
pusher = entityPushers.FirstOrDefault(p => p.IsImplemented(indexModel.DestinationProcessorId, destinationConnection.ProviderId));
}
else
{
var attributeModel = attributeRepository.GetById(firstQueuedItem.TargetEntityId.ToString());
indexModel = attributeModel;
var entityModel = entityRepository.GetById(attributeModel.EntityId.ToString());
options = attributeRepository.LoadOptions(attributeModel.Id.ToString()).Select(o => new OptionItem {
Name = o.Key, Value = o.Value
});
var sourceConnection = connectionRepository.GetById(attributeModel.SourceConnectionId.ToString());
var destinationConnection = connectionRepository.GetById(attributeModel.DestinationConnectionId.ToString());
indexer = attributeIndexers.FirstOrDefault(i => i.IsImplemented(attributeModel.SourceProcessorId, entityModel.SourceProcessorId, sourceConnection.ProviderId));
pusher = attributePushers.FirstOrDefault(p => p.IsImplemented(attributeModel.DestinationProcessorId, entityModel.DestinationProcessorId, destinationConnection.ProviderId));
}
indexer.SetIndex(indexModel);
indexer.SetOptions(options);
pusher.SetIndex(indexModel);
pusher.SetOptions(options);
pusherManager.SetIndex(indexModel);
pusherManager.OnReport(s => Logger.Information(s));
pusherManager.SetIndexer(indexer);
pusherManager.SetPusher(pusher);
try
{
itemModel = entityRepository.GetIndexedItemById(indexModel, firstQueuedItem.TargetItemId.ToString());
var pushState = await pusherManager.PushItem(itemModel);
var queueItemStatus = firstQueuedItem.Status == PushState.None ? PushState.Success : firstQueuedItem.Status;
var messageId = messageRepository.Create(new
{
Message = string.Join("\n", pusherManager.GetReportMessages()),
CreatedAt = DateTime.Now.ToUnixTimestamp(),
MessageType = MessageType.Information,
Status = MessageStatus.None
});
queueItemStatus = queueItemStatus & pushState;
if ((pushState & PushState.Success) <= 0)
{
queueItemStatus = (queueItemStatus | PushState.Success) ^ (PushState.Success);
}
queueItemRepository.Update(firstQueuedItem.Id.ToString(), new
{
UpdatedAt = DateTime.Now.ToUnixTimestamp(),
ExecuteAt = executeAt,
ExecutedAt = DateTime.Now.ToUnixTimestamp(),
MessageId = messageId,
Status = queueItemStatus
});
}
catch (Exception ex)
{
var messages = $@"Queue item (Id: {firstQueuedItem.Id}) failed to run.
Addtional information:
```{JsonConvert.SerializeObject(indexModel, Formatting.Indented)}```
Progress:
```{string.Join("\n - ", pusherManager.GetReportMessages())}```
Exception:
```{ex}```";
var messageId = messageRepository.Create(new
{
Message = messages,
CreatedAt = DateTime.Now.ToUnixTimestamp(),
MessageType = MessageType.Error,
Status = MessageStatus.None
});
queueItemRepository.Update(firstQueuedItem.Id.ToString(), new
{
UpdatedAt = DateTime.Now.ToUnixTimestamp(),
ExecuteAt = executeAt,
ExecutedAt = DateTime.Now.ToUnixTimestamp(),
MessageId = messageId,
Status = (firstQueuedItem.Status | PushState.UnexpectedError | PushState.Failed | PushState.Success) ^ PushState.Success, // remove success
});
throw;
}
}