public bool GetDependsOnItem(string valueTableName, DependencyItemModel dependence, IndexItemModel item, out IndexItemModel dependsOnItem)
{
var referenceKeyParams = dependence.ReferenceKeysArr;
var foreignKeyParams = dependence.ForeignKeysArr;
var @params = foreignKeyParams
.Select((fk, i) => new { fk, i })
.Select(fk => new KeyValuePair <string, string>(referenceKeyParams[fk.i], item[fk.fk].ToString()))
.ToDictionary(fk => fk.Key, fk => fk.Value);
var dynamicParams = new DynamicParameters();
foreach (var p in @params)
{
if (string.IsNullOrWhiteSpace(p.Value)) // An item has dependency with NULL value is not depends on anything
{
dependsOnItem = null;
return(false);
}
dynamicParams.Add(p.Key, p.Value);
}
var sql = $@"
SELECT TOP 1 * FROM {valueTableName}
WHERE {string.Join(" AND ", @params.Select(p => $@"[{p.Key}] = @{p.Key}"))}
";
dependsOnItem = _connection.Query <object>(sql, param: dynamicParams, transaction: _transaction)
.Select(d => IndexItemModel.FromJObject(JObject.FromObject(d))).FirstOrDefault();
return(true);
}
public IEnumerable <IndexItemModel> GetIndexedItemsBySourceId(IIndexModel model, string id)
{
var items = _connection.Query($@"
SELECT * FROM [{model.ValueTableName}]
WHERE [SourceId] = @Id", param: new { Id = id }, transaction: _transaction) as IEnumerable <IDictionary <string, object> >;
return(items?.Select(i => IndexItemModel.FromJObject(JObject.FromObject(i))));
}
public IndexItemModel GetIndexedItemDestinationId(IIndexModel model, string id)
{
var items = _connection.Query($@"
SELECT * FROM [{model.ValueTableName}]
WHERE [DestinationId] = @Id", param: new { Id = id }, transaction: _transaction) as IEnumerable <IDictionary <string, object> >;
var item = items?.FirstOrDefault();
return(item != null?IndexItemModel.FromJObject(JObject.FromObject(item)) : null);
}
public IEnumerable <IndexItemModel> GetIndexChangedItems(
IIndexModel model,
int limit,
int offset,
out int totalCount)
{
var @params = new DynamicParameters();
@params.Add("Limit", limit > 0 ? limit : 100);
@params.Add("Offset", offset);
@params.Add("ChangedState", ItemState.Changed);
/**
* Do not ignore RelatedItemNotFound and RelatedItemNotSync,
* These items should be checked in code logic
* RelatedItemNotFound: it might be found later
* RelatedItemNotSynced: it will be synced later
*/
@params.Add("ExcludeStates", ItemState.Invalid);
// TODO: RETRY COUNT?
var sql = $@"
SELECT v.*
FROM {model.ValueTableName} v
WHERE
v.[DestinationId] IS NULL
OR v.[State] IS NULL
OR v.[State] = 0
OR (
(v.[State] & @ChangedState) > 0
AND (v.[State] & @ExcludeStates) = 0
)
ORDER BY v.[RetryCount], v.[Id]
OFFSET @Offset ROWS
FETCH NEXT @Limit ROWS ONLY;
";
var countSql = $@"
SELECT COUNT(v.Id)
FROM {model.ValueTableName} v
WHERE
v.[DestinationId] IS NULL
OR v.[State] IS NULL
OR v.[State] = 0
OR (
(v.[State] & @ChangedState) > 0
AND (v.[State] & @ExcludeStates) = 0
)
";
totalCount = _connection
.Query <int>(countSql, param: @params, transaction: _transaction)
.FirstOrDefault();
var result = _connection
.Query(sql, param: @params, transaction: _transaction) as IEnumerable <IDictionary <string, object> >;
return(result.Select(d => IndexItemModel.FromJObject(JObject.FromObject(d))));
}
public virtual IMapper Map(IEnumerable <object> data)
{
if (data == null)
{
return(this);
}
var destinationIdKey = Options?.FirstOrDefault(o => o.Name == "mapper_id_key").Value;
var foreignKeyStr = Options?.FirstOrDefault(o => o.Name == "mapper_foreign_keys").Value;
var foreignKeys = Regex.Split(foreignKeyStr, "[,;|]", RegexOptions.Multiline | RegexOptions.IgnoreCase);
var referenceKeyStr = Options?.FirstOrDefault(o => o.Name == "mapper_reference_keys").Value;
var referenceKeys = Regex.Split(referenceKeyStr, "[,;|]", RegexOptions.Multiline | RegexOptions.IgnoreCase);
var affectedRows = 0;
foreach (var item in data)
{
var jItem = JObject.FromObject(item);
var destinationId = jItem.GetValue(destinationIdKey).ToString();
var queryParams = new DynamicParameters();
var conditions = new List <string>();
// !!! NEVER CHECK FOR DEPENDENCIES. An entity cannot be mapped if its values are required to be check SOURCE v.s DEST with dependencies
for (var i = 0; i < foreignKeys.Length; i++)
{
var foreignValue = jItem.GetValue(foreignKeys[i]);
// Foreign values should NEVER BE NULL
conditions.Add($@"[{referenceKeys[i]}] = @{referenceKeys[i]}");
queryParams.Add(referenceKeys[i], foreignValue?.ToString());
}
var indexedItem = Connection
.Query <object>($@"
SELECT * FROM [{EntityModel.ValueTableName}]
WHERE {string.Join(" AND ", conditions)}
", queryParams)
.Select(i => IndexItemModel.FromJObject(JObject.FromObject(i)))
.FirstOrDefault();
if (indexedItem != null)
{
affectedRows += Connection
.Execute($@"
UPDATE [{EntityModel.ValueTableName}]
SET [DestinationId] = @DestinationId
WHERE [Id] = @Id
",
new {
Id = indexedItem.GetId(),
DestinationId = jItem.GetValue(destinationIdKey).ToString()
});
}
}
Report($@"Mapped {affectedRows} item(s).");
return(this);
}
public IEnumerable <IndexItemModel> GetIndexedItems(
IIndexModel model,
IEnumerable <FilterArgument> filters,
int limit,
int offset,
out int totalCount)
{
var @params = new DynamicParameters();
@params.Add("Limit", limit > 0 ? limit : 100);
@params.Add("Offset", offset);
var filterStrs = new List <string>();
var @where = string.Empty;
if (filters != null && filters.Count() > 0)
{
foreach (var filter in filters)
{
var paramName = StringExtensions.StringExtensions.Random(10);
filterStrs.Add($@"[{filter.Field}] {filter.Op} @Param_{paramName}");
@params.Add($@"Param_{paramName}", filter.Target);
}
//var condition = string.Join(" AND ", filters.Select(f => $@"[{f.Field}] {f.Op} {}"));
@where = string.Join(" AND ", filterStrs);
if (!string.IsNullOrWhiteSpace(@where))
{
@where = $"WHERE {@where}";
}
}
var sql = $@"
SELECT * FROM {model.ValueTableName}
{@where}
ORDER BY [Id]
OFFSET @Offset ROWS
FETCH NEXT @Limit ROWS ONLY;
";
var countSql = $@"
SELECT COUNT(*) FROM {model.ValueTableName}
{@where}
";
totalCount = _connection
.Query <int>(countSql, param: @params, transaction: _transaction)
.FirstOrDefault();
var result = _connection
.Query(sql, param: @params, transaction: _transaction) as IEnumerable <IDictionary <string, object> >;
return(result.Select(d => IndexItemModel.FromJObject(JObject.FromObject(d))));
}
private void UpdateDependencies(IndexItemModel item, string destinationId)
{
using (var entityRepository = ResolverFactory.Resolve <EntityRepository>())
using (var attributeRepository = ResolverFactory.Resolve <AttributeRepository>())
{
if (_indexerModel.EntityType == EntityType.Entity)
{
// Update Attribute Mapping (DestinationId) if _indexerModel is Entity
var attrs = attributeRepository.GetByEntityId(_indexerModel.Id.ToString());
foreach (var attr in attrs)
{
Report($"Updating destination ID for attribute {attr.Name}.");
if (!attributeRepository.Initialized(attr))
{
Report($"Attribute {attr.Name} is not initialized.");
continue;
}
if (attr.HasState(EntityState.Disabled))
{
Report($"Attribute {attr.Name} is Disabled.");
continue;
}
attributeRepository.UpdateItemDestinationId(attr, item.GetSourceId(), destinationId);
Report($"Done updating destination ID for attribute {attr.Name}.");
}
}
var dependencies = entityRepository.GetDependenciesOn(_indexerModel.Id, _indexerModel.EntityType);
foreach (var dependency in dependencies)
{
if (dependency.HasDependOnStep(IntegrationStep.Pushing | IntegrationStep.Pushing))
{
if (dependency.HasExecutionStep(IntegrationStep.Indexing | IntegrationStep.Indexed))
{
// Only add the signal to tell that the dependant entity should be pull (via PullNext) based on this item
entityRepository.AddPullDependency(
dependency.EntityId,
dependency.EntityType,
_indexerModel.Id,
_indexerModel.EntityType,
item.GetId());
}
}
}
}
}
/// <summary>
/// 获取积分明细
/// </summary>
/// <returns>积分列表</returns>
public async Task <IndexModel> Get()
{
var id = (await GetCurrentUser()).Id;
IList <Credit> credits = await _context.Credit.Where(x => x.UserId == id).OrderByDescending(x => x.CreateTime).ToListAsync();
var list = new List <IndexItemModel>();
foreach (var credit in credits)
{
var indexItemModel = new IndexItemModel();
indexItemModel.Balance = credit.Balance;
indexItemModel.Count = credit.Count;
indexItemModel.Description = credit.Description;
indexItemModel.SetProperty("CreateTime", credit.CreateTime);
list.Add(indexItemModel);
}
return(new IndexModel {
Items = list
});
}
public IndexItemModel GetDependsOnItem(string valueTableName, DependencyItemModel dependence, IndexItemModel item)
{
var referenceKeyParams = Regex.Split(dependence.ReferenceKeys, "[,;|]", RegexOptions.Multiline | RegexOptions.IgnoreCase);
var foreignKeyParams = Regex.Split(dependence.ForeignKeys, "[,;|]", RegexOptions.Multiline | RegexOptions.IgnoreCase);
var @params = foreignKeyParams
.Select((fk, i) => new { fk, i })
.Select(fk => new KeyValuePair <string, string>(referenceKeyParams[fk.i], item[fk.fk].ToString()))
.ToDictionary(fk => fk.Key, fk => fk.Value);
var dynamicParams = new DynamicParameters();
foreach (var p in @params)
{
dynamicParams.Add(p.Key, p.Value);
}
var sql = $@"
SELECT TOP 1 * FROM {valueTableName}
WHERE {string.Join(" AND ", @params.Select(p => $@"[{p.Key}] = @{p.Key}"))}
";
var dependsOnItem = _connection.Query <object>(sql, param: dynamicParams, transaction: _transaction)
.Select(d => IndexItemModel.FromJObject(JObject.FromObject(d))).FirstOrDefault();
return(dependsOnItem);
}
private string GetEntityMessage(QueueItemModel item, out IIndexModel indexModel, out IndexItemModel itemModel)
{
indexModel = entityRepository.GetById(item.TargetEntityId.ToString());
var options = entityRepository.LoadOptions(indexModel.Id.ToString(), new List <string> {
"Indexer"
});
var reporterMappingColumnOption = options.FirstOrDefault(o => o.Key == "indexer_reporter_columns");
var reporterMappingColumns = !string.IsNullOrWhiteSpace(reporterMappingColumnOption?.Value)
? JsonConvert.DeserializeObject <List <ReporterColumnMapping> >(reporterMappingColumnOption.Value)
: new List <ReporterColumnMapping> {
new ReporterColumnMapping {
SourceName = "Value",
MappingName = "Name",
Key = true,
Value = true
}
};
var entityModel = entityRepository.GetIndexedItemById(indexModel, item.TargetItemId);
itemModel = entityModel;
var keys = reporterMappingColumns.Where(r => r.Key)
.Select(r => $@"_{r.MappingName}_: {entityModel.GetValue(r.SourceName)?.ToString() ?? "(empty)"}");
var vals = reporterMappingColumns.Where(r => r.Value)
.Select(r => $@"_{r.MappingName}_: {entityModel.GetValue(r.SourceName)?.ToString() ?? "(empty)"}");
if (reporterMappingColumns.Count == 1)
{
vals = reporterMappingColumns.Where(r => r.Value)
.Select(r => $@"{entityModel.GetValue(r.SourceName)?.ToString() ?? "(empty)"}");
}
var executed = item.ExecutedAt.UnixTimeToTime().ToString("G");
var executedIn = item.ExecutedAt - item.ExecuteAt;
return($@"*{indexModel.Name}* ({string.Join(", ", keys)}): {vals} {executed} in {executedIn} second(s)");
}
public async Task <PushState> PushItem(IndexItemModel item)
{
Report($@"
---------------------------------------------------------------------------------
Begin synchronizing item {JsonConvert.SerializeObject(item, Formatting.Indented)}...");
var synchronizerFactory = ResolverFactory.Resolve <SynchronizerFactory>();
var pusher = synchronizerFactory.CreatePusher(_indexerModel);
pusher.OnReport(m => Report(m));
var result = await Task.Run(() =>
{
var entityRepository = ResolverFactory.Resolve <EntityRepository>();
var attributeRepository = ResolverFactory.Resolve <AttributeRepository>();
var messageRepository = ResolverFactory.Resolve <MessageRepository>();
try
{
entityRepository.BeginTransaction();
attributeRepository.BeginTransaction();
messageRepository.BeginTransaction();
var pushState = PushState.Success;
var destinationId = item.GetDestinationId();
if (!string.IsNullOrWhiteSpace(destinationId))
{
if (item.HasState(ItemState.Removed))
{
pushState = pusher.Remove();
}
else
{
pushState = pusher.Update();
}
}
else
{
destinationId = pusher.GetDestinationId();
if (!string.IsNullOrWhiteSpace(destinationId))
{
if (item.HasState(ItemState.Removed))
{
pushState = pusher.Remove(destinationId);
}
else
{
pushState = pusher.Update(destinationId);
}
}
else // still cannot find a destinationId, which means the entity/attribute does not exists
{
pushState = pusher.Create(out destinationId);
}
}
if (!string.IsNullOrWhiteSpace(destinationId) && (pushState & PushState.Success) > 0)
{
entityRepository.UpdateItemDestinationId(_indexerModel, item.GetSourceId(), destinationId);
// Detect dependencies that depends on "Synced" step
UpdateDependencies(item, destinationId);
}
else
{
// Signal to tell that the item is success or not
entityRepository.Retry(_indexerModel, item.GetId(), pushState);
}
entityRepository.Commit();
attributeRepository.Commit();
return(pushState);
}
catch
{
entityRepository.RollBack();
attributeRepository.RollBack();
// Update invalid item
// Increate retry count
// Next time when queue items, it will be put behind because of the retry count
entityRepository.Retry(_indexerModel, item.GetId(), PushState.UnexpectedError);
throw;
}
finally
{
entityRepository?.Dispose();
attributeRepository?.Dispose();
messageRepository?.Dispose();
pusher?.Dispose();
Report($@"
Ended synchronizing...
---------------------------------------------------------------------------------
");
}
});
return(result);
}
public override async Task Invoke(IStepExecutionContext context = null)
{
var executeAt = DateTime.Now.ToUnixTimestamp();
var firstQueuedItem = entityRepository.GetCurrentQueuedItems();
if (firstQueuedItem == null)
{
return;
}
IIndexModel indexModel = null;
IndexItemModel itemModel = null;
IIndexer indexer = null;
IPusher pusher = null;
IEnumerable <OptionItem> options = null;
if (firstQueuedItem.TargetEntityType == EntityType.Entity)
{
indexModel = entityRepository.GetById(firstQueuedItem.TargetEntityId.ToString());
options = entityRepository.LoadOptions(indexModel.Id.ToString()).Select(o => new OptionItem {
Name = o.Key, Value = o.Value
});
var sourceConnection = connectionRepository.GetById(indexModel.SourceConnectionId.ToString());
var destinationConnection = connectionRepository.GetById(indexModel.DestinationConnectionId.ToString());
indexer = entityIndexers.FirstOrDefault(i => i.IsImplemented(indexModel.SourceProcessorId, sourceConnection.ProviderId));
pusher = entityPushers.FirstOrDefault(p => p.IsImplemented(indexModel.DestinationProcessorId, destinationConnection.ProviderId));
}
else
{
var attributeModel = attributeRepository.GetById(firstQueuedItem.TargetEntityId.ToString());
indexModel = attributeModel;
var entityModel = entityRepository.GetById(attributeModel.EntityId.ToString());
options = attributeRepository.LoadOptions(attributeModel.Id.ToString()).Select(o => new OptionItem {
Name = o.Key, Value = o.Value
});
var sourceConnection = connectionRepository.GetById(attributeModel.SourceConnectionId.ToString());
var destinationConnection = connectionRepository.GetById(attributeModel.DestinationConnectionId.ToString());
indexer = attributeIndexers.FirstOrDefault(i => i.IsImplemented(attributeModel.SourceProcessorId, entityModel.SourceProcessorId, sourceConnection.ProviderId));
pusher = attributePushers.FirstOrDefault(p => p.IsImplemented(attributeModel.DestinationProcessorId, entityModel.DestinationProcessorId, destinationConnection.ProviderId));
}
indexer.SetIndex(indexModel);
indexer.SetOptions(options);
pusher.SetIndex(indexModel);
pusher.SetOptions(options);
pusherManager.SetIndex(indexModel);
pusherManager.OnReport(s => Logger.Information(s));
pusherManager.SetIndexer(indexer);
pusherManager.SetPusher(pusher);
try
{
itemModel = entityRepository.GetIndexedItemById(indexModel, firstQueuedItem.TargetItemId.ToString());
var pushState = await pusherManager.PushItem(itemModel);
var queueItemStatus = firstQueuedItem.Status == PushState.None ? PushState.Success : firstQueuedItem.Status;
var messageId = messageRepository.Create(new
{
Message = string.Join("\n", pusherManager.GetReportMessages()),
CreatedAt = DateTime.Now.ToUnixTimestamp(),
MessageType = MessageType.Information,
Status = MessageStatus.None
});
queueItemStatus = queueItemStatus & pushState;
if ((pushState & PushState.Success) <= 0)
{
queueItemStatus = (queueItemStatus | PushState.Success) ^ (PushState.Success);
}
queueItemRepository.Update(firstQueuedItem.Id.ToString(), new
{
UpdatedAt = DateTime.Now.ToUnixTimestamp(),
ExecuteAt = executeAt,
ExecutedAt = DateTime.Now.ToUnixTimestamp(),
MessageId = messageId,
Status = queueItemStatus
});
}
catch (Exception ex)
{
var messages = $@"Queue item (Id: {firstQueuedItem.Id}) failed to run.
Addtional information:
```{JsonConvert.SerializeObject(indexModel, Formatting.Indented)}```
Progress:
```{string.Join("\n - ", pusherManager.GetReportMessages())}```
Exception:
```{ex}```";
var messageId = messageRepository.Create(new
{
Message = messages,
CreatedAt = DateTime.Now.ToUnixTimestamp(),
MessageType = MessageType.Error,
Status = MessageStatus.None
});
queueItemRepository.Update(firstQueuedItem.Id.ToString(), new
{
UpdatedAt = DateTime.Now.ToUnixTimestamp(),
ExecuteAt = executeAt,
ExecutedAt = DateTime.Now.ToUnixTimestamp(),
MessageId = messageId,
Status = (firstQueuedItem.Status | PushState.UnexpectedError | PushState.Failed | PushState.Success) ^ PushState.Success, // remove success
});
throw;
}
}
public override async Task Invoke(IStepExecutionContext context = null)
{
var entityRepository = ResolverFactory.Resolve <EntityRepository>();
var attributeRepository = ResolverFactory.Resolve <AttributeRepository>();
var messageRepository = ResolverFactory.Resolve <MessageRepository>();
var queueItemRepository = ResolverFactory.Resolve <QueueItemRepository>();
IIndexModel indexModel = null;
IndexItemModel itemModel = null;
var logger = ResolverFactory.Resolve <ILogger>("SyncService");
var errorLogger = ResolverFactory.Resolve <ILogger>("Error");
var pusherManager = ResolverFactory.Resolve <PusherManager>();
try
{
var executeAt = DateTime.Now.ToUnixTimestamp();
var firstQueuedItem = entityRepository.GetCurrentQueuedItems();
if (firstQueuedItem == null)
{
return;
}
if (firstQueuedItem.TargetEntityType == EntityType.Entity)
{
indexModel = entityRepository.GetById(firstQueuedItem.TargetEntityId.ToString());
}
else
{
indexModel = attributeRepository.GetById(firstQueuedItem.TargetEntityId.ToString());
}
pusherManager.SetIndex(indexModel);
pusherManager.OnReport(s => logger.Information(s));
try
{
itemModel = entityRepository.GetIndexedItemById(indexModel, firstQueuedItem.TargetItemId.ToString());
var pushState = await pusherManager.PushItem(itemModel);
var messageId = messageRepository.Create(new
{
Message = string.Join("\n", pusherManager.GetReportMessages()),
CreatedAt = DateTime.Now.ToUnixTimestamp(),
MessageType = MessageType.Information,
Status = MessageStatus.None
});
queueItemRepository.Update(firstQueuedItem.Id.ToString(), new
{
UpdatedAt = DateTime.Now.ToUnixTimestamp(),
ExecuteAt = executeAt,
ExecutedAt = DateTime.Now.ToUnixTimestamp(),
MessageId = messageId,
Status = pushState
});
}
catch (Exception ex)
{
var messages = $@"Queue item (Id: {firstQueuedItem.Id}) failed to run.
Addtional information:
```{JsonConvert.SerializeObject(indexModel, Formatting.Indented)}```
Progress:
```{string.Join("\n - ", pusherManager.GetReportMessages())}```
Exception:
```{ex}```";
var messageId = messageRepository.Create(new
{
Message = messages,
CreatedAt = DateTime.Now.ToUnixTimestamp(),
MessageType = MessageType.Error,
Status = MessageStatus.None
});
queueItemRepository.Update(firstQueuedItem.Id.ToString(), new
{
UpdatedAt = DateTime.Now.ToUnixTimestamp(),
ExecuteAt = executeAt,
ExecutedAt = DateTime.Now.ToUnixTimestamp(),
MessageId = messageId,
Status = PushState.UnexpectedError
});
throw;
}
}
catch (Exception ex)
{
errorLogger.Error(ex, ex.Message);
throw;
}
finally
{
entityRepository?.Dispose();
attributeRepository?.Dispose();
messageRepository?.Dispose();
queueItemRepository?.Dispose();
ResolverFactory.Release(logger);
ResolverFactory.Release(errorLogger);
logger = null;
errorLogger = null;
}
}
protected virtual Dictionary <string, string> GetNormalizedValuesByDependencies(IndexItemModel indexedItem = null)
{
using (var entityRepository = ResolverFactory.Resolve <EntityRepository>())
using (var attributeRepository = ResolverFactory.Resolve <AttributeRepository>())
{
var indexedModel = GetIndexModel();
indexedItem = indexedItem ?? IndexedItem;
var dependencies = entityRepository.GetDependencies(indexedModel.Id, indexedModel.EntityType);
var normalizedValues = new Dictionary <string, string>();
foreach (var d in dependencies)
{
var foreignKeys = d.ForeignKeysArr;
var referenceKeys = d.ReferenceKeysArr;
IIndexModel dependsOnModel = null;
if (d.TargetEntityType == EntityType.Attribute)
{
dependsOnModel = attributeRepository.GetById(d.TargetEntityId.ToString());
}
else
{
dependsOnModel = entityRepository.GetById(d.TargetEntityId.ToString());
}
var hasDependencies = entityRepository.GetDependsOnItem(dependsOnModel.ValueTableName, d, indexedItem, out IndexItemModel referencedItem);
if (!hasDependencies)
{
return(normalizedValues);
}
for (var i = 0; i < foreignKeys.Length; i++)
{
var foreignKey = foreignKeys[i];
var referenceKey = referenceKeys[i];
if (!IndexedItem.Properties().Select(p => p.Name).Contains(foreignKey))
{
continue;
}
if (normalizedValues.ContainsKey(foreignKey))
{
continue;
}
normalizedValues.Add(foreignKey, referencedItem?.Value <string>(referenceKey));
}
}
return(normalizedValues);
}
}
public IPusher SetItem(IndexItemModel item)
{
IndexedItem = item;
return(this);
}
