public async Task UpdateStructureIfRequiredAsync(string assetPair)
{
assetPair = assetPair.ToLower();
if (string.IsNullOrWhiteSpace(assetPair) || _assetPairs.Contains(assetPair))
{
return;
}
_assetPairs.Add(assetPair);
var tablesStructure = new TablesStructure
{
Tables = _assetPairs.Select(GetTableStructureForAssetPair).ToList(),
};
await _blobSaver.CreateOrUpdateTablesStructureAsync(tablesStructure);
}
public bool IsAllBlobsReprocessingRequired(TablesStructure currentStructure)
{
if (currentStructure?.Tables == null || currentStructure.Tables.Count <= 1)
{
return(true);
}
foreach (var tableStructure in currentStructure.Tables)
{
string assetPair = ExtractAssetPairFromDirectory(tableStructure.AzureBlobFolder);
if (assetPair == null)
{
continue;
}
_assetPairs.Add(assetPair);
}
return(false);
}
public TablesStructure GetTablesStructure()
{
if (_type == null)
{
_type = _processingTypeResolver.ResolveProcessingTypeAsync().GetAwaiter().GetResult();
}
var result = new TablesStructure {
Tables = new List <TableStructure>()
};
AddStructureLevel(
result,
_type,
null,
null,
null);
return(result);
}
public async Task <bool> CreateOrUpdateTablesStructureAsync(TablesStructure tablesStructure, bool compareWithExisting = false)
{
string newStructureJson = tablesStructure.ToJson();
var blob = _blobContainer.GetBlockBlobReference(_tablesStructureFileName);
if (compareWithExisting)
{
bool exists = await blob.ExistsAsync();
if (exists)
{
string structureJson = await blob.DownloadTextAsync(null, _blobRequestOptions, null);
if (!string.IsNullOrWhiteSpace(structureJson))
{
var structure = structureJson.DeserializeJson <TablesStructure>();
if (CompareStructures(tablesStructure, structure))
{
return(false);
}
}
_log.WriteWarning(
nameof(CreateOrUpdateTablesStructureAsync),
"Table structure change",
$"Table structure is changed from {structureJson} to {newStructureJson}");
}
}
else
{
_log.WriteWarning(nameof(CreateOrUpdateTablesStructureAsync), "Table structure change", $"New table structure is {newStructureJson}");
}
await blob.UploadTextAsync(newStructureJson, null, _blobRequestOptions, null);
await SetContentTypeAsync(blob);
return(true);
}
private bool CompareStructures(TablesStructure newVersion, TablesStructure oldVersion)
{
if (oldVersion?.Tables == null)
{
return(false);
}
if (oldVersion.Tables.Count != newVersion.Tables.Count)
{
return(false);
}
foreach (var table in newVersion.Tables)
{
var oldTable = oldVersion.Tables.FirstOrDefault(t =>
t.AzureBlobFolder == table.AzureBlobFolder || t.TableName == table.TableName);
if (oldTable == null)
{
return(false);
}
var oldTableColumns = oldTable.Columns ?? oldTable.Colums;
if (table.Columns.Count != oldTableColumns.Count)
{
return(false);
}
for (int i = 0; i < table.Columns.Count; ++i)
{
if (table.Columns[i].ColumnName != oldTableColumns[i].ColumnName ||
table.Columns[i].ColumnType != oldTableColumns[i].ColumnType)
{
return(false);
}
}
}
return(true);
}
private Dictionary <string, string> GetColumnsListsFromStructure(TablesStructure tablesStructure)
{
var result = new Dictionary <string, string>();
foreach (var tableStructure in tablesStructure.Tables)
{
var strBuilder = new StringBuilder();
var columns = tableStructure.Columns ?? tableStructure.Colums;
foreach (var columnInfo in columns)
{
if (strBuilder.Length > 0)
{
strBuilder.Append(", ");
}
strBuilder.Append($"[{columnInfo.ColumnName}] ");
if (columnInfo.ColumnType == typeof(DateTime).Name)
{
strBuilder.Append("DATETIME");
}
else if (columnInfo.ColumnType == typeof(double).Name ||
columnInfo.ColumnType == typeof(decimal).Name)
{
strBuilder.Append("Decimal(30,8)");
}
else if (columnInfo.ColumnType == typeof(bool).Name)
{
strBuilder.Append("Bit");
}
else
{
strBuilder.Append("VARCHAR(256)");
}
}
result[tableStructure.TableName] = strBuilder.ToString();
}
return(result);
}
public bool IsAllBlobsReprocessingRequired(TablesStructure currentStructure)
{
return(false);
}
private void AddStructureLevel(
TablesStructure tablesStructure,
Type type,
Type parentType,
string parentIdPropertyName,
string parentIdPropertyTypeName)
{
var processingresult = ProcessProperties(
tablesStructure,
type,
parentType,
parentIdPropertyName,
parentIdPropertyTypeName);
if (processingresult.OneChildrenProperties.Count == 0 && processingresult.ManyChildrenProperties.Count == 0)
{
return;
}
Type typeForChildren = type;
string idPropertyName = processingresult.IdProperty?.Name;
string idPropertyTypeName = processingresult.IdProperty?.PropertyType.Name;
if (parentType != null && processingresult.ValueProperties.Count == 0)
{
typeForChildren = parentType;
}
if (idPropertyName == null)
{
bool idWasFoundInChildren = false;
Type childWithIdType = null;
PropertyInfo childWithIdProperty = null;
PropertyInfo idPropertyInChild = null;
foreach (var childTypePair in processingresult.OneChildrenProperties)
{
var(childType, childIdProperty) = GetIdPropertyTypeName(childTypePair.Item2);
if (childIdProperty == null)
{
continue;
}
if (idWasFoundInChildren)
{
childWithIdType = null;
idPropertyInChild = null;
childWithIdProperty = null;
}
else
{
childWithIdType = childType;
idPropertyInChild = childIdProperty;
childWithIdProperty = childTypePair.Item1;
}
idWasFoundInChildren = true;
}
if (idPropertyInChild != null)
{
idPropertyName = idPropertyInChild.Name;
idPropertyTypeName = idPropertyInChild.PropertyType.Name;
if (idPropertyInChild.Name == IdPropertyName)
{
typeForChildren = childWithIdType;
idPropertyName = $"{childWithIdType.Name}{IdPropertyName}";
}
var typeData = PropertiesMap[type];
typeData.ChildWithIdProperty = childWithIdProperty;
typeData.IdPropertyInChild = idPropertyInChild;
}
}
else
{
idPropertyName = $"{typeForChildren.Name}{IdPropertyName}";
}
if (idPropertyName == null)
{
idPropertyName = parentIdPropertyName;
idPropertyTypeName = parentIdPropertyTypeName;
}
if (idPropertyName == null && processingresult.RelationProperty != null)
{
idPropertyName = processingresult.RelationProperty.Name;
idPropertyTypeName = processingresult.RelationProperty.PropertyType.Name;
PropertiesMap[type].RelationProperty = processingresult.RelationProperty;
}
foreach (var childTypePair in processingresult.OneChildrenProperties)
{
AddStructureLevel(
tablesStructure,
childTypePair.Item2,
typeForChildren,
idPropertyName,
idPropertyTypeName);
}
foreach (var childType in processingresult.ManyChildrenProperties)
{
AddStructureLevel(
tablesStructure,
childType,
typeForChildren,
idPropertyName,
idPropertyTypeName);
}
}
private PropertiesProcessingResult ProcessProperties(
TablesStructure tablesStructure,
Type type,
Type parentType,
string parentIdPropertyName,
string parentIdPropertyTypeName)
{
string typeName = type.Name;
_excludedPropertiesMap.TryGetValue(typeName, out var excludedProperties);
_idPropertiesMap.TryGetValue(typeName, out var idPropertyName);
_relationPropertiesMap.TryGetValue(typeName, out var relationPropertyName);
var valueProperties = new List <PropertyInfo>();
var oneChildrenProperties = new List <(PropertyInfo, Type)>();
var manyChildrenProperties = new List <(PropertyInfo, Type)>();
string parentTypeName = parentType?.Name;
var collector = new TableStructure
{
TableName = string.IsNullOrWhiteSpace(parentTypeName)
? (string.IsNullOrWhiteSpace(_instanceTag) ? typeName : $"{typeName}_{_instanceTag}")
: (string.IsNullOrWhiteSpace(_instanceTag) ? $"{parentTypeName}{typeName}" : $"{parentTypeName}{typeName}_{_instanceTag}"),
AzureBlobFolder = typeName.ToLower(),
Columns = new List <ColumnInfo>(),
};
var topLevelProperties = type.GetProperties(BindingFlags.Public | BindingFlags.Instance);
PropertyInfo idProperty = null;
PropertyInfo relationProperty = null;
foreach (var property in topLevelProperties)
{
if (excludedProperties != null && excludedProperties.Contains(property.Name))
{
continue;
}
var propertyType = property.PropertyType;
if ((propertyType.IsClass || propertyType.IsInterface) && propertyType != typeof(string))
{
if (propertyType.IsArray)
{
manyChildrenProperties.Add((property, propertyType.GetElementType()));
}
else if (propertyType.IsGenericType && GenericCollectionTypes.Any(t => t == propertyType.GetGenericTypeDefinition()))
{
var genericType = propertyType.GetGenericArguments()[0];
if (genericType.IsClass && genericType != typeof(string))
{
manyChildrenProperties.Add((property, propertyType.GetGenericArguments()[0]));
}
else
{
string genericTypeName = genericType.IsEnum ? typeof(string).Name : genericType.Name;
int ind = propertyType.Name.IndexOf('`');
var propertyTypeName = $"{propertyType.Name.Substring(0, ind + 1)}{genericTypeName}";
collector.Columns.Add(
new ColumnInfo
{
ColumnName = property.Name,
ColumnType = propertyTypeName,
});
valueProperties.Add(property);
}
}
else
{
oneChildrenProperties.Add((property, propertyType));
}
}
else
{
if (idProperty == null && (property.Name == IdPropertyName || property.Name == idPropertyName))
{
idProperty = property;
}
else
{
string propertyTypeName = propertyType.Name;
if (propertyType.IsGenericType)
{
var genericType = propertyType.GetGenericArguments()[0];
string genericTypeName = genericType.IsEnum ? typeof(string).Name : genericType.Name;
int ind = propertyTypeName.IndexOf('`');
propertyTypeName = $"{propertyTypeName.Substring(0, ind + 1)}{genericTypeName}";
}
else if (propertyType.IsEnum)
{
propertyTypeName = typeof(string).Name;
}
if (property.Name == relationPropertyName)
{
relationProperty = property;
}
collector.Columns.Add(
new ColumnInfo
{
ColumnName = property.Name,
ColumnType = propertyTypeName,
});
}
valueProperties.Add(property);
}
}
if (collector.Columns.Count > 0)
{
if (parentType != type)
{
if (parentIdPropertyName != null)
{
var parentIdPropertyInChild = type.GetProperty(parentIdPropertyName);
if (parentIdPropertyInChild == null)
{
collector.Columns.Insert(
0,
new ColumnInfo
{
ColumnName = parentIdPropertyName,
ColumnType = parentIdPropertyTypeName,
});
}
}
else if (valueProperties.Count > 0 && parentType != null && PropertiesMap[parentType].ValueProperties.Count > 0)
{
throw new InvalidOperationException(
$"Type {typeName} must have any identificators that can be used to make relations to its parent type {parentType.Name}");
}
}
if (idProperty != null)
{
idPropertyName = parentType != null && parentType != type && PropertiesMap[parentType].OneChildrenProperties.Count > 1
? idProperty.Name
: IdPropertyName;
collector.Columns.Insert(
0,
new ColumnInfo
{
ColumnName = idPropertyName,
ColumnType = idProperty.PropertyType.Name,
});
}
tablesStructure.Tables.Add(collector);
}
if (!PropertiesMap.ContainsKey(type))
{
PropertiesMap.Add(
type,
new TypeData
{
ValueProperties = valueProperties,
OneChildrenProperties = oneChildrenProperties.Select(i => i.Item1).ToList(),
ManyChildrenProperties = manyChildrenProperties.Select(i => i.Item1).ToList(),
ParentIdPropertyName = parentIdPropertyName,
});
}
return(new PropertiesProcessingResult
{
IdProperty = idProperty,
RelationProperty = relationProperty,
ValueProperties = valueProperties,
OneChildrenProperties = oneChildrenProperties,
ManyChildrenProperties = manyChildrenProperties.Select(i => i.Item2).ToList(),
});
}
