public IActionResult PullEntityData(string id, [FromBody] object nextToken = null)
{
var entity = entityRepository.GetById(id);
var sourceConnection = connectionRepository.GetById(entity.SourceConnectionId.ToString());
var puller = _entityPullers.FirstOrDefault(p => p.IsImplemented(entity.SourceProcessorId, sourceConnection.ProviderId));
puller.SetIndex(entity);
var data = puller.PullNext(nextToken);
return(Ok(data));
}
public override bool Validate(out string message)
{
try
{
var ok = true;
//IsLoading = true;
var task = Task.Run(async() =>
{
var allAttributes = attributeRepository.GetAll();
foreach (var attr in allAttributes)
{
var initialized = true;
var entity = entityRepository.GetById(attr.EntityId.ToString());
var options = attributeRepository.LoadOptions(attr.Id.ToString());
var connection = connectionRepository.GetById(attr.SourceConnectionId.ToString());
var puller = pullers.FirstOrDefault(p => p.IsImplemented(attr.SourceProcessorId, entity.SourceProcessorId, connection.ProviderId));
var indexer = indexers.FirstOrDefault(p => p.IsImplemented(attr.SourceProcessorId, entity.SourceProcessorId, connection.ProviderId));
puller.SetIndex(attr);
puller.SetOptions(options.Select(o => new OptionItem {
Name = o.Key, Value = o.Value
}));
initialized = initialized && puller.Initialized();
indexer.SetIndex(attr);
indexer.SetOptions(options.Select(o => new OptionItem {
Name = o.Key, Value = o.Value
}));
initialized = initialized && entityRepository.Initialized(attr);
ok = ok && initialized;
if (!ok)
{
logger.Information($@"Index ""{entity.Name}"" is not initialized.");
break;
}
}
});
task.Wait();
message = "All attributes has been initialized.";
logger.Information(message);
return(true);
}
catch (Exception ex)
{
errorLogger.Error(ex, ex.Message);
throw;
}
finally
{
//IsLoading = false;
}
return(true);
}
private IEnumerable <OptionItem> GetPullerTemplateOptions(EntityModel e)
{
if (string.IsNullOrWhiteSpace(e.SourceConnectionId.ToString()) || e.SourceConnectionId == Guid.Empty)
{
return(new List <OptionItem>());
}
var conn = connectionRepository.GetById(e.SourceConnectionId.ToString());
var puller = pullers.FirstOrDefault(p => p.IsImplemented(e.SourceProcessorId, conn.ProviderId));
return(puller?.Options ?? new List <OptionItem>());
}
protected virtual IMapper SpreadOptions()
{
ConnectionModel = ConnectionRepository.GetById(EntityModel.DestinationConnectionId.ToString());
var connectionOptions = ConnectionRepository.LoadOptions(ConnectionModel.Id.ToString());
var connectionOptionItems = connectionOptions.Select(c => new OptionItem {
Name = c.Key, Value = c.Value
});
Adapter.SetOptions(connectionOptionItems);
Provider.SetOptions(connectionOptionItems);
return(this);
}
protected virtual IIndexer SpreadOptions()
{
ConnectionModel = ConnectionRepository.GetById(GetIndexModel().SourceConnectionId.ToString());
var connectionOptions = ConnectionRepository.LoadOptions(ConnectionModel.Id.ToString());
var connectionOptionItems = connectionOptions.Select(c => new OptionItem {
Name = c.Key, Value = c.Value
});
Adapter.SetOptions(connectionOptionItems);
Provider.SetOptions(connectionOptionItems);
return(this);
}
virtual IPusher SpreadOptions()
{
var indexModel = GetIndexModel();
ConnectionModel = ConnectionRepository.GetById(indexModel.DestinationConnectionId.ToString());
var connectionOptions = ConnectionRepository.LoadOptions(ConnectionModel.Id.ToString());
var connectionOptionItems = connectionOptions.Select(c => new OptionItem {
Name = c.Key, Value = c.Value
});
Adapter.SetOptions(connectionOptionItems);
Provider.SetOptions(connectionOptionItems);
return(this);
}
public override async Task Invoke(IStepExecutionContext context = null)
{
var executeAt = DateTime.Now.ToUnixTimestamp();
var firstQueuedItem = entityRepository.GetCurrentQueuedItems();
if (firstQueuedItem == null)
{
return;
}
IIndexModel indexModel = null;
IndexItemModel itemModel = null;
IIndexer indexer = null;
IPusher pusher = null;
IEnumerable <OptionItem> options = null;
if (firstQueuedItem.TargetEntityType == EntityType.Entity)
{
indexModel = entityRepository.GetById(firstQueuedItem.TargetEntityId.ToString());
options = entityRepository.LoadOptions(indexModel.Id.ToString()).Select(o => new OptionItem {
Name = o.Key, Value = o.Value
});
var sourceConnection = connectionRepository.GetById(indexModel.SourceConnectionId.ToString());
var destinationConnection = connectionRepository.GetById(indexModel.DestinationConnectionId.ToString());
indexer = entityIndexers.FirstOrDefault(i => i.IsImplemented(indexModel.SourceProcessorId, sourceConnection.ProviderId));
pusher = entityPushers.FirstOrDefault(p => p.IsImplemented(indexModel.DestinationProcessorId, destinationConnection.ProviderId));
}
else
{
var attributeModel = attributeRepository.GetById(firstQueuedItem.TargetEntityId.ToString());
indexModel = attributeModel;
var entityModel = entityRepository.GetById(attributeModel.EntityId.ToString());
options = attributeRepository.LoadOptions(attributeModel.Id.ToString()).Select(o => new OptionItem {
Name = o.Key, Value = o.Value
});
var sourceConnection = connectionRepository.GetById(attributeModel.SourceConnectionId.ToString());
var destinationConnection = connectionRepository.GetById(attributeModel.DestinationConnectionId.ToString());
indexer = attributeIndexers.FirstOrDefault(i => i.IsImplemented(attributeModel.SourceProcessorId, entityModel.SourceProcessorId, sourceConnection.ProviderId));
pusher = attributePushers.FirstOrDefault(p => p.IsImplemented(attributeModel.DestinationProcessorId, entityModel.DestinationProcessorId, destinationConnection.ProviderId));
}
indexer.SetIndex(indexModel);
indexer.SetOptions(options);
pusher.SetIndex(indexModel);
pusher.SetOptions(options);
pusherManager.SetIndex(indexModel);
pusherManager.OnReport(s => Logger.Information(s));
pusherManager.SetIndexer(indexer);
pusherManager.SetPusher(pusher);
try
{
itemModel = entityRepository.GetIndexedItemById(indexModel, firstQueuedItem.TargetItemId.ToString());
var pushState = await pusherManager.PushItem(itemModel);
var queueItemStatus = firstQueuedItem.Status == PushState.None ? PushState.Success : firstQueuedItem.Status;
var messageId = messageRepository.Create(new
{
Message = string.Join("\n", pusherManager.GetReportMessages()),
CreatedAt = DateTime.Now.ToUnixTimestamp(),
MessageType = MessageType.Information,
Status = MessageStatus.None
});
queueItemStatus = queueItemStatus & pushState;
if ((pushState & PushState.Success) <= 0)
{
queueItemStatus = (queueItemStatus | PushState.Success) ^ (PushState.Success);
}
queueItemRepository.Update(firstQueuedItem.Id.ToString(), new
{
UpdatedAt = DateTime.Now.ToUnixTimestamp(),
ExecuteAt = executeAt,
ExecutedAt = DateTime.Now.ToUnixTimestamp(),
MessageId = messageId,
Status = queueItemStatus
});
}
catch (Exception ex)
{
var messages = $@"Queue item (Id: {firstQueuedItem.Id}) failed to run.
Addtional information:
```{JsonConvert.SerializeObject(indexModel, Formatting.Indented)}```
Progress:
```{string.Join("\n - ", pusherManager.GetReportMessages())}```
Exception:
```{ex}```";
var messageId = messageRepository.Create(new
{
Message = messages,
CreatedAt = DateTime.Now.ToUnixTimestamp(),
MessageType = MessageType.Error,
Status = MessageStatus.None
});
queueItemRepository.Update(firstQueuedItem.Id.ToString(), new
{
UpdatedAt = DateTime.Now.ToUnixTimestamp(),
ExecuteAt = executeAt,
ExecutedAt = DateTime.Now.ToUnixTimestamp(),
MessageId = messageId,
Status = (firstQueuedItem.Status | PushState.UnexpectedError | PushState.Failed | PushState.Success) ^ PushState.Success, // remove success
});
throw;
}
}
public override async Task Invoke(IStepExecutionContext context = null)
{
try
{
Logger.Information($@"Updating index changes of {IndexModel.Name}/{IndexModel.Id}...");
var sourceConnection = connectionRepository.GetById(IndexModel.SourceConnectionId.ToString());
//var destConnection = connectionRepository.GetById(IndexModel.DestinationConnectionId.ToString());
IPuller puller = null;
IIndexer indexer = null;
if (IndexModel.EntityType == EntityType.Entity)
{
puller = pullers.Where(p => typeof(IEntityPuller).IsAssignableFrom(p.GetType()))
.Select(p => (IEntityPuller)p)
.FirstOrDefault(p => p.IsImplemented(
IndexModel?.SourceProcessorId,
sourceConnection?.ProviderId));
indexer = indexers.Where(p => typeof(IEntityIndexer).IsAssignableFrom(p.GetType()))
.Select(p => (IEntityIndexer)p)
.FirstOrDefault(p => p.IsImplemented(
IndexModel?.SourceProcessorId,
sourceConnection?.ProviderId));
}
else
{
var entity = entityRepository.GetById((IndexModel as AttributeModel).EntityId.ToString());
puller = pullers.Where(p => typeof(IAttributePuller).IsAssignableFrom(p.GetType()))
.Select(p => (IAttributePuller)p)
.FirstOrDefault(p => p.IsImplemented(
IndexModel.SourceProcessorId,
entity?.SourceProcessorId,
sourceConnection?.ProviderId));
indexer = indexers.Where(p => typeof(IAttributeIndexer).IsAssignableFrom(p.GetType()))
.Select(p => (IAttributeIndexer)p)
.FirstOrDefault(p => p.IsImplemented(
IndexModel.SourceProcessorId,
entity?.SourceProcessorId,
sourceConnection?.ProviderId));
}
var options = entityRepository.LoadOptions(IndexModel.Id.ToString(), IndexModel.EntityType)
.Select(o => new OptionItem {
Name = o.Key, Value = o.Value
});
puller.SetIndex(IndexModel);
puller.SetOptions(options);
indexer.SetIndex(IndexModel);
indexer.SetOptions(options);
indexerManager.SetIndex(IndexModel);
indexerManager.SetPuller(puller);
indexerManager.SetIndexer(indexer);
indexerManager.OnReport(s =>
{
Logger.Information(s);
});
await indexerManager.PullNext();
Logger.Information($@"Updated index changes of {IndexModel.Name}/{IndexModel.Id}");
}
catch (Exception ex)
{
ErrorLogger.Error(ex, ex.Message);
throw;
}
}