public static CsvObject Create(SystemObjectEntity systemObject, AttributesObject attrObject)
{
return(new CsvObject
{
Id = systemObject.Id,
ParentId = systemObject.ParentId,
SystemId = systemObject.SystemId,
ViewId = systemObject.ViewId,
Descriptor = systemObject.Descriptor,
UnitDescriptor = systemObject.UnitDescriptor,
Designation = systemObject.Designation,
Name = systemObject.Name,
SystemName = systemObject.SystemName,
ObjectId = systemObject.ObjectId,
DefaultProperty = attrObject.DefaultProperty,
DisciplineDescriptor = attrObject.DisciplineDescriptor,
DisciplineId = attrObject.DisciplineId,
FunctionDefaultProperty = attrObject.FunctionDefaultProperty,
FunctionName = attrObject.FunctionName,
ManagedType = attrObject.ManagedType,
ManagedTypeName = attrObject.ManagedTypeName,
SubDisciplineDescriptor = attrObject.SubDisciplineDescriptor,
SubDisciplineId = attrObject.SubDisciplineId,
SubTypeDescriptor = attrObject.SubTypeDescriptor,
SubTypeId = attrObject.SubTypeId,
TypeDescriptor = attrObject.TypeDescriptor,
TypeId = attrObject.TypeId
});
}
public async Task CreateSystemObject(bool bMultiThread, SystemObjectEntity SystemObject)
{
if (bMultiThread)
{
using (var context = _scope.CreateScope().ServiceProvider.GetService <SiemensDbContext>())
{
await Create(context, SystemObject);
}
}
else
{
await Create(_dbContext, SystemObject);
}
}
private async Task Create(SiemensDbContext context, SystemObjectEntity SystemObject)
{
var existing = context.SystemObjects.FirstOrDefault(x => x.SiteId == SystemObject.SiteId && x.ObjectId == SystemObject.ObjectId && x.Name == SystemObject.Name && x.Designation == SystemObject.Designation);
if (existing != null)
{
SystemObject.Id = existing.Id;
context.Entry(SystemObject).State = EntityState.Modified;
}
else
{
context.SystemObjects.Add(SystemObject);
}
await context.SaveChangesAsync();
}
private List <SystemObjectEntity> GetDbItemsRecursively(DataItem item, SystemObjectEntity parent = null)
{
var sObject = Map(item);
if (parent != null)
{
sObject.Parent = parent;
}
var childrenObjects = new List <SystemObjectEntity>();
foreach (var child in item.ChildrenItems)
{
childrenObjects.AddRange(GetDbItemsRecursively(child, sObject));
}
childrenObjects.Insert(0, sObject);
return(childrenObjects);
}
private List <DataItem> ImportRecursive(HttpClient client, string url, LinkType linkType, int?parentSystemObjectId, SystemObjectEntity parentSystemObject, int MaxThreads, ScanRequest scanRequest)
{
var data = client.GetAsync(url).Result;
if (!data.IsSuccessStatusCode)
{
_logger.LogWarning("Failed to invoke {statusCode} {url}", data.StatusCode, url);
throw new Exception($"Failed to invoke {data.StatusCode} {url}");
//return new List<DataItem>();
}
var strData = data.Content.ReadAsStringAsync().Result;
strData = strData.Trim();
_logger.LogInformation("URL: {url}", url);
_logger.LogInformation("Response: {response}", strData);
var items = linkType == LinkType.Systembrowser
? JsonConvert.DeserializeObject <List <DataItem> >(strData)
: new List <DataItem> {
JsonConvert.DeserializeObject <DataItem>(strData)
};
if (linkType == LinkType.Properties && parentSystemObject != null)
{
var dataItem = items.First();
parentSystemObject.Properties = dataItem.Properties?.ToString();
parentSystemObject.FunctionProperties = dataItem.FunctionProperties?.ToString();
return(new List <DataItem>());
}
Parallel.ForEach(items, new ParallelOptions {
MaxDegreeOfParallelism = MaxThreads
}, dataItem =>
{
var dbEntity = new SystemObjectEntity
{
ParentId = parentSystemObjectId,
Name = dataItem.Name,
Descriptor = dataItem.Descriptor,
Designation = dataItem.Designation,
ObjectId = dataItem.ObjectId,
SystemId = dataItem.SystemId,
ViewId = dataItem.ViewId,
SystemName = dataItem.SystemName,
Attributes = dataItem.Attributes?.ToString(),
Properties = dataItem.Properties?.ToString(),
FunctionProperties = dataItem.FunctionProperties?.ToString(),
SiteId = SiteId
};
_systemObjectService.CreateSystemObject(true, dbEntity).Wait();
foreach (var dataItemLink in dataItem.Links)
{
var lt = dataItemLink.Rel.Trim().ToLower() == "systembrowser"
? LinkType.Systembrowser
: LinkType.Properties;
dataItem.ChildrenItems.AddRange(ImportRecursive(client, "api/" + dataItemLink.Href, lt, dbEntity.Id, dbEntity, 1, scanRequest));
}
ProcessingCount++;
if (MaxThreads > 1)
{
scanRequest.NumberOfPoints = ProcessingCount;
UpdateScanRequestInMultiThread(scanRequest).Wait();
_logger.LogInformation($"Processing Count : {ProcessingCount}");
}
});
return(items);
}
private async Task <List <DataItem> > StartScan(HttpClient client, string url, LinkType linkType, SiteConfiguration siteConfiguration, ScanRequest scanRequest)
{
var data = await client.GetAsync(url);
_logger.LogInformation("Started");
if (!data.IsSuccessStatusCode)
{
_logger.LogWarning("Failed to invoke {statusCode} {url}", data.StatusCode, url);
throw new Exception($"Failed to invoke {data.StatusCode} {url}");
//return new List<DataItem>();
}
var strData = await data.Content.ReadAsStringAsync();
strData = strData.Trim();
_logger.LogInformation("URL: {url}", url);
_logger.LogInformation("Response: {response}", strData);
var items = linkType == LinkType.Systembrowser
? JsonConvert.DeserializeObject <List <DataItem> >(strData)
: new List <DataItem> {
JsonConvert.DeserializeObject <DataItem>(strData)
};
// ------- for local testing ----------
//var items = new List<DataItem>();
//for(int i=0; i<100; i++)
//{
// items.Add(new DataItem() { SystemId = i });
//}
//---------------------------------
foreach (var dataItem in items)
{
var dbEntity = new SystemObjectEntity
{
ParentId = null,
Name = dataItem.Name,
Descriptor = dataItem.Descriptor,
Designation = dataItem.Designation,
ObjectId = dataItem.ObjectId,
SystemId = dataItem.SystemId,
ViewId = dataItem.ViewId,
SystemName = dataItem.SystemName,
Attributes = dataItem.Attributes?.ToString(),
Properties = dataItem.Properties?.ToString(),
FunctionProperties = dataItem.FunctionProperties?.ToString(),
SiteId = SiteId
};
_systemObjectService.CreateSystemObject(true, dbEntity).Wait();
foreach (var dataItemLink in dataItem.Links)
{
var lt = dataItemLink.Rel.Trim().ToLower() == "systembrowser"
? LinkType.Systembrowser
: LinkType.Properties;
dataItem.ChildrenItems.AddRange(ImportRecursive(client, "api/" + dataItemLink.Href, lt, dbEntity.Id, dbEntity, siteConfiguration.MaxThreads, scanRequest));
}
ProcessingCount++;
scanRequest.NumberOfPoints = ProcessingCount;
UpdateScanRequestInMultiThread(scanRequest).Wait();
_logger.LogInformation($"Processing Count : {ProcessingCount}");
}
scanRequest.Status = ScanRequestStatus.Completed;
scanRequest.EndTime = DateTime.UtcNow;
scanRequest.NumberOfPoints = ProcessingCount;
await UpdateScanRequestInMultiThread(scanRequest);
_logger.LogInformation($"Completed. Total Processed Count : {ProcessingCount}");
return(items);
}
private async Task <List <StoreItemTask> > ScanItemRecursive(HttpClient client, Guid siteId, ScanItemTask scanItemTask)
{
Interlocked.Increment(ref _scansInProcess);
try
{
_logger.LogInformation($"Scanning url: " + scanItemTask.Url);
var data = await client.GetAsync(scanItemTask.Url);
if (!data.IsSuccessStatusCode)
{
_logger.LogWarning("Failed to invoke {statusCode} {url}", data.StatusCode, scanItemTask.Url);
return(new List <StoreItemTask>());
}
var strData = (await data.Content.ReadAsStringAsync()).Trim();
var items = JsonConvert.DeserializeObject <List <DataItem> >(strData);
var dbEntities = new Dictionary <SystemObjectEntity, TaskCompletionSource <bool> >();
foreach (var dataItem in items)
{
var dbEntity = new SystemObjectEntity
{
Name = dataItem.Name,
Descriptor = dataItem.Descriptor,
Designation = dataItem.Designation,
ObjectId = dataItem.ObjectId,
SystemId = dataItem.SystemId,
ViewId = dataItem.ViewId,
SystemName = dataItem.SystemName,
Attributes = dataItem.Attributes?.ToString(),
Properties = dataItem.Properties?.ToString(),
FunctionProperties = dataItem.FunctionProperties?.ToString(),
SiteId = siteId
};
var propertyLink = dataItem.Links.FirstOrDefault(l => l.Rel.ToLowerInvariant() != "systembrowser");
if (propertyLink != null)
{
var propertyData = await client.GetAsync("api/" + propertyLink.Href);
if (propertyData.IsSuccessStatusCode)
{
var propertyStrData = (await propertyData.Content.ReadAsStringAsync()).Trim();
var propertyItem = JsonConvert.DeserializeObject <DataItem>(propertyStrData);
dbEntity.Properties = propertyItem?.Properties?.ToString();
dbEntity.FunctionProperties = propertyItem?.FunctionProperties?.ToString();
if (!string.IsNullOrEmpty(dbEntity.Attributes))
{
var attributesObject = JsonConvert.DeserializeObject <AttributesObject>(dbEntity.Attributes);
var defaultPropertyName = attributesObject.DefaultProperty;
if (!string.IsNullOrEmpty(defaultPropertyName))
{
var url = "API/API/properties/" + dbEntity.ObjectId;
var response = await client.GetAsync(url);
if (response.IsSuccessStatusCode)
{
var strResponse = await response.Content.ReadAsStringAsync();
var propertiesData = JsonConvert.DeserializeObject <List <PropertyResponse> >(strResponse);
var properties = propertiesData.First().Properties;
var defaultProperty = properties.FirstOrDefault(p => p.PropertyName == defaultPropertyName);
dbEntity.UnitDescriptor = defaultProperty?.UnitDescriptor;
}
}
}
}
}
var sytemBrowserLinks =
dataItem.Links.Where(l => l.Rel.ToLowerInvariant() == "systembrowser").ToList();
var objectStored = new TaskCompletionSource <bool>();
dbEntities.Add(dbEntity, objectStored);
foreach (var sytemBrowserLink in sytemBrowserLinks)
{
await InputQueue.SendAsync(new ScanItemTask
{
ParentSystemObject = dbEntity, Url = "api/" + sytemBrowserLink.Href,
ScanRequest = scanItemTask.ScanRequest, ParentObjectStored = objectStored
});
}
}
return(dbEntities.Select(e => new StoreItemTask
{
SystemObject = e.Key,
ObjectStored = e.Value,
ParentSystemObject = scanItemTask.ParentSystemObject,
ParentObjectStored = scanItemTask.ParentObjectStored,
ScanRequest = scanItemTask.ScanRequest
}).ToList());
}
finally
{
var scansInProcess = Interlocked.Decrement(ref _scansInProcess);
var scanQueueLength = InputQueue.Count;
_logger.LogInformation($"Scans in process: {scansInProcess}, Queue length: {scanQueueLength}");
if (scansInProcess == 0 && scanQueueLength == 0)
{
_logger.LogInformation($"Queue is empty, last scan task finished, completing the pipeline");
InputQueue.Complete();
}
}
}
