protected virtual FileInstance GetFileInstance(FileInfo fileInfo, ScanResult scanResult)
{
var existing = repo.RetrieveFileInstance(fileInfo);
if (existing != null)
{
var tp = repo.RetrieveFileThumbPrint(existing.FileThumbPrintId);
if (tp.FileSize != fileInfo.Length || tp.MD5 != scanResult.MD5)
{
repo.SoftDeleteFileThumbPrint(tp.Id);
var newTP = GetFileThumbPrint(fileInfo, scanResult);
existing.FileThumbPrintId = newTP.Id;
existing = repo.UpdateFileInstance(existing);
}
return(existing);
}
else
{
var tp = GetFileThumbPrint(fileInfo, scanResult);
var newFI = new FileInstance()
{
FileThumbPrintId = tp.Id,
FileName = fileInfo.Name,
FilePath = fileInfo.DirectoryName,
Created = fileInfo.CreationTime,
Modified = fileInfo.LastWriteTime,
};
repo.CreateFileInstance(newFI);
return(newFI);
}
}
/**
* 0 - value_id,
* 1 - attribute_id,
* 2 - store_id,
* 3 - entity_id,
* 4 - value
*/
public static void Run(FileInstance file, List <string> args)
{
if (!file.Tables.ContainsKey(TABLE))
{
return;
}
var memory = new Dictionary <int, Dictionary <string, bool> >();
var lastId = 1;
foreach (var insert in file.Tables[TABLE].Inserts)
{
foreach (var row in insert.Rows)
{
if (row[1] != args[0])
{
continue;
}
var store = int.Parse(row[2]);
if (!memory.ContainsKey(store))
{
memory.Add(store, new Dictionary <string, bool>());
}
if (memory[store].ContainsKey(row[4]))
{
row[4] = "'" + Helper.RemoveTags(row[4]) + REPEAT_HASH + lastId++ + "'";
}
memory[store].Add(row[4], true);
}
}
}
protected virtual void Dispose(bool disposing)
{
if (disposing)
{
FileInstance?.Dispose();
}
}
public override Int32 Open(
String FileName,
UInt32 CreateOptions,
UInt32 GrantedAccess,
out Object FileNode,
out Object FileDesc0,
out FileInfo FileInfo,
out String NormalizedName)
{
FileNode = default;
NormalizedName = default;
if (string.IsNullOrEmpty(FileName) || FileName == "\\")
{
// No file path? Return some info about the root dir..
var fileDesc = new FileInstance(null, this);
FileDesc0 = fileDesc;
return(fileDesc.GetFileInfo(out FileInfo));
}
var foundEntry = FindFile(FileName);
if (foundEntry == null)
{
FileInfo = default;
FileDesc0 = null;
return(STATUS_OBJECT_NAME_NOT_FOUND);
}
var fileDesc2 = new FileInstance(foundEntry, this);
FileDesc0 = fileDesc2;
return(fileDesc2.GetFileInfo(out FileInfo));
}
public static void BuildFk(FileInstance file, int fileId, Config.Config config)
{
var register = Register.Registers[fileId];
foreach (var table in file.Tables)
{
if (config.Files[fileId].Tables != null &&
config.Files[fileId].Tables.ContainsKey(table.Key) &&
config.Files[fileId].Tables[table.Key].ForeignKey != null)
{
foreach (var fkConfig in config.Files[fileId].Tables[table.Key].ForeignKey)
{
register.AddFK(table.Key, fkConfig.Column, fkConfig.TargetTable);
}
}
if (config.Files[fileId].Tables != null &&
config.Files[fileId].Tables.ContainsKey(table.Key) &&
config.Files[fileId].Tables[table.Key].ForeignKeyRule != null)
{
foreach (var fkRule in config.Files[fileId].Tables[table.Key].ForeignKeyRule)
{
register.AddFkRule(table.Key, fkRule.ForeignKey.Column, fkRule);
}
}
foreach (var foreignKey in table.Value.ForeignKeys)
{
register.AddFK(table.Key, foreignKey.Key, foreignKey.Value.Table);
}
}
}
public static void Move(List <MoveDataConfig> configs, FileInstance file, Table table)
{
Console.WriteLine($"--||-- Data move in table: {table.Name}");
foreach (var config in configs)
{
var targetTable = file.Tables[config.To];
var columnId = table.GetColumnId(config.Where.Column);
var stripColumnIds = new List <int>();
foreach (var columnName in config.StripTagsFor)
{
stripColumnIds.Add(table.GetColumnId(columnName));
}
if (columnId == -1)
{
continue;
}
foreach (var insert in table.Inserts)
{
var moveInsert = new Insert
{
ID = table.ID,
Table = targetTable.Name,
Rows = new List <List <string> >()
};
for (var r = 0; r < insert.Rows.Count; r++)
{
if (insert.Rows[r][columnId] != config.Where.Value)
{
continue;
}
foreach (var stripId in stripColumnIds)
{
if (insert.Rows[r][stripId].Length >= 2 && insert.Rows[r][stripId] != "NULL")
{
insert.Rows[r][stripId] = Helper.RemoveTags(insert.Rows[r][stripId]);
}
}
moveInsert.Rows.Add(insert.Rows[r]);
insert.Rows.RemoveAt(r);
r--;
}
if (moveInsert.Rows.Count > 0)
{
targetTable.Inserts.Add(moveInsert);
targetTable.AddedCopy = true;
}
Console.WriteLine($"--||--||-- Moved {moveInsert.Rows.Count} rows");
}
}
}
public async Task <FileInstance> AddAsync(FileInstance file)
{
var savedFile = await context.Files.AddAsync(file);
await context.SaveChangesAsync();
return(file);
}
public override Boolean ReadDirectoryEntry(
Object FileNode,
Object FileDesc0,
String Pattern,
String Marker,
ref Object Context,
out String FileName,
out FileInfo FileInfo)
{
var Instance = (FileInstance)FileDesc0;
var Enumerator = (IEnumerator <IFileEntry>)Context;
if (Enumerator == null)
{
var childArr = RootFiles;
if (Instance.FileEntry != null)
{
childArr = Instance.FileEntry.Children;
}
Enumerator = childArr.GetEnumerator();
Context = Enumerator;
int Index = 0;
if (null != Marker)
{
Index = childArr.FindIndex(s => s.Name == Marker);
if (0 <= Index)
{
Index++;
}
else
{
Index = ~Index;
}
}
if (Index > 0)
{
for (int i = 0; i < Index; i++)
{
Enumerator.MoveNext();
}
}
}
if (Enumerator.MoveNext())
{
var entry = Enumerator.Current;
FileName = entry.Name;
var desc = new FileInstance(entry, this);
desc.GetFileInfo(out FileInfo);
return(true);
}
FileName = default;
FileInfo = default;
return(false);
}
public static bool Run(FileInstance outputFile, List <FileInstance> rawFiles, bool firstTime = false)
{
if (!outputFile.Tables.ContainsKey(TABLE_NAME))
{
return(false);
}
var output = false;
hashData.Clear();
// TODO: Add support for multiple files
var start = 1;
var end = rawFiles.Count;
if (firstTime)
{
start = 1;
end = 2;
}
else
{
BuildFirstHash(outputFile);
}
for (var f = start; f < end; f++)
{
foreach (var insert in rawFiles[f].Tables[TABLE_NAME].Inserts)
{
for (var r = 0; r < insert.Rows.Count; r++)
{
var key = GetHashName(insert.Rows[r]);
if (hashData.ContainsKey(key))
{
// Merge
Register.Registers[f].UpdatePk(TABLE_NAME, insert.Rows[r][0], hashData[key]);
Console.WriteLine($"--||-- Merging same user {insert.Rows[r][2]}");
var email = Helper.RemoveTags(insert.Rows[r][2]);
if (!LogData.ContainsKey(email))
{
LogData.Add(email, int.Parse(insert.Rows[r][5]));
}
insert.Rows.RemoveAt(r);
r--;
output = true;
}
else
{
hashData.Add(key, insert.Rows[r][0]);
}
}
}
}
return(output);
}
private static void BuildFirstHash(FileInstance headFile)
{
foreach (var insert in headFile.Tables[TABLE_NAME].Inserts)
{
foreach (var row in insert.Rows)
{
hashData.Add(GetHashName(row), row[0]);
}
}
}
public FileInstance UpdateFileInstance(FileInstance instance)
{
using (var ctx = new FreeSpaceContext())
{
instance.UpdatedStamp = DateTime.Now;
ctx.FileInstances.Attach(instance);
ctx.SaveChanges();
return(instance);
}
}
public void Allocate(RobloxBinaryFile file)
{
foreach (int instId in InstanceIds)
{
FileInstance inst = new FileInstance(TypeName);
file.Instances[instId] = inst;
}
file.Types[TypeIndex] = this;
}
/// <summary>
/// Saves the content of the buffer to the current file and clears it beforehand.
/// </summary>
public override void DumpBufferToCurrentFile()
{
string bufferContent;
lock (Mutex)
{
bufferContent = Storage.GetText(0, Storage.GetLength() - 1);
}
FileInstance.Clear();
FileInstance.WriteToFile(0, bufferContent);
}
private static Instance Reflect_BIN(FileInstance fileInst, Type objType)
{
if (!typeof(Instance).IsAssignableFrom(objType))
{
throw new Exception("'T' must be an Instance.");
}
Instance obj = Activator.CreateInstance(objType) as Instance;
foreach (FileProperty prop in fileInst.Properties)
{
string propertyName = prop.Name.Replace(" ", "_");
FieldInfo field = objType.GetField(propertyName, fieldInfoFlags);
if (field != null)
{
PropertyType propType = prop.Type;
object value = prop.Value;
if (propType == PropertyType.Enum)
{
Type fieldType = field.FieldType;
if (fieldType.IsEnum)
{
uint enumIndex = (uint)prop.Value;
int index = (int)enumIndex;
value = Enum.ToObject(fieldType, index);
}
}
if (value != null)
{
field.SetValue(obj, value);
}
}
}
foreach (FileInstance child in fileInst.GetChildren())
{
Type classType = GetClassType(child.ClassName);
if (classType != null)
{
Instance childObj = Reflect_BIN(child, classType);
childObj.Parent = obj;
}
}
return(obj);
}
public HttpResponseMessage AddFolder(FileInstance folder)
{
try
{
BlogService.CreateDirectory(folder.FullPath + "/" + folder.Name);
return(Request.CreateResponse(HttpStatusCode.OK));
}
catch (UnauthorizedAccessException)
{
return(Request.CreateResponse(HttpStatusCode.Unauthorized));
}
catch (Exception)
{
return(Request.CreateResponse(HttpStatusCode.InternalServerError));
}
}
public static void Merge(ref FileInstance a, FileInstance b)
{
// First RUN
if (a.Tables.Count == 0)
{
a.Tables = b.Tables;
return;
}
foreach (var table in b.Tables)
{
// Table Exists
if (a.Tables.ContainsKey(table.Key))
{
foreach (var column in table.Value.Columns)
{
// Column Exists
if (a.Tables[table.Key].Columns.ContainsKey(column.Key))
{
// If columns are equal (do nothing) -> skip
if (CompareColumns(a.Tables[table.Key].Columns[column.Key], column.Value))
{
continue;
}
Logger.LogErrorMessage($"Merging error: Columns Differ - ${column.Key}");
}
// Column Does NOT Exist
else
{
a.Tables[table.Key].Columns.Add(column.Key, column.Value);
a.Tables[table.Key].Columns[column.Key].Order = a.Tables[table.Key].Columns.Count - 1;
// TODO: Fix Inserts
}
}
// a.Tables[table.Key].Inserts.AddRange(table.Value.Inserts);
}
// Table Does NOT Exist
else
{
// TODO: Fix order
a.Tables.Add(table.Key, table.Value);
}
}
}
public void Assemble(RobloxBinaryFile file)
{
for (int i = 0; i < NumRelations; i++)
{
int childId = ChildrenIds[i];
int parentId = ParentIds[i];
FileInstance child = file.Instances[childId];
FileInstance parent = null;
if (parentId >= 0)
{
parent = file.Instances[parentId];
}
else
{
parent = file.Contents;
}
child.Parent = parent;
}
}
/// <summary>
/// Loads the content of the file to the buffer.
/// </summary>
public override void FillBufferFromFile()
{
lock (Mutex)
{
if (Storage.GetLength() > 0)
{
Storage.Delete(0, Storage.GetLength() - 1);
}
/*
* for (var i = 0; i <= _file.FileSize / blockSize; i++)
* {
* var fileContent = _file.ReadFromFile(i * blockSize, blockSize);
* _storage.Insert(fileContent, i * blockSize);
* }
*/
var fileContent = FileInstance.ReadFromFile(0, (int)FileInstance.FileSize);
Storage.Insert(fileContent, 0);
Counter.CountFileContent(this);
}
}
public static void AddLastIdReference(FileInstance baseFile,
Dictionary <string, CustomAttributeOptionConfig> config)
{
if (!baseFile.Tables.ContainsKey(TABLE) ||
!baseFile.Tables.ContainsKey(OPTION_TABLE) ||
!baseFile.Tables.ContainsKey(OPTION_VALUE_TABLE))
{
return;
}
var codeColumnId = baseFile.Tables[TABLE].GetColumnId("attribute_code");
if (codeColumnId < 0)
{
return;
}
LastId = 0;
foreach (var customOptionAttributeConfig in config)
{
var configLastId = customOptionAttributeConfig.Value.Offset +
customOptionAttributeConfig.Value.Map.Count + 1;
if (LastId < configLastId)
{
LastId = configLastId;
}
}
if (LastId != 0)
{
referenceRow = new List <string> {
LastId.ToString(), "1", "0"
};
referenceRowValue = new List <string> {
LastId.ToString(), LastId.ToString(), "0", "tmp"
};
baseFile.Tables[OPTION_TABLE].Inserts[^ 1].Rows.Add(referenceRow);
private static void ApplyChange(ConfigTable config, Table table, Column column, FileInstance file)
{
var columnConfig = config.Columns[column.Name];
if (columnConfig.Change != null)
{
ApplyValueChange(config, table, column);
}
if (columnConfig.ChangeWhere != null && columnConfig.ChangeWhere.Count > 0)
{
ApplyValueChangeWhere(config, table, column);
}
//column.IsLikable = columnConfig.IsLikable;
if (!string.IsNullOrEmpty(columnConfig.Default))
{
column.Default = columnConfig.Default;
}
if (!string.IsNullOrEmpty(columnConfig.DataType))
{
column.DataType = columnConfig.DataType;
}
if (!string.IsNullOrEmpty(columnConfig.NewName))
{
column.NewName = columnConfig.NewName;
}
}
public static bool Merge(FileInstance file, bool firstTime = false)
{
var didMerge = false;
var canDoMerge = true;
var canUpdatePk = true;
cacheSameFields.Clear();
// -- Tables
foreach (var table in file.Tables)
{
Console.WriteLine($"--- Merging INSERT in table: {table.Key}");
var pkId = 0;
if (table.Value.PrimaryKey != null && table.Value.PrimaryKey.Length > 0)
{
pkId = table.Value.GetColumnId(table.Value.PrimaryKey[0]);
}
config = MergingController.Config.Global;
lastId = GetLastId(table.Key, pkId);
if (MergingController.Config.Tables.ContainsKey(table.Key))
{
// Changes configuration from global to local
if (MergingController.Config.Tables[table.Key].Config != null)
{
config = MergingController.Config.Tables[table.Key].Config;
}
// Builds Cache of ("same" compression) field IDs
if (!cacheSameFields.ContainsKey(table.Key))
{
cacheSameFields.Add(table.Key, new Dictionary <string, int>());
var comparedFields = MergingController.Config.Tables[table.Key].SameOn;
if (comparedFields != null)
{
foreach (var field in comparedFields)
{
cacheSameFields[table.Key].Add(field, table.Value.GetColumnId(field));
}
}
}
// Gets info to whether or not do merge on first time
if (MergingController.FirstRun)
{
canDoMerge = MergingController.Config.Tables[table.Key].FirstTimeMerge;
canUpdatePk = MergingController.Config.Tables[table.Key].UpdatePk;
}
}
// -- Inserts
for (var i = 0; i < table.Value.Inserts.Count; i++)
{
var pkCount = table.Value.PrimaryKey?.Length ?? 0;
var insert = table.Value.Inserts[i];
// -- Rows
for (var r = 0; r < insert.Rows.Count; r++)
{
//Console.WriteLine($"----- table: {table} | insert: {i}/{table.Value.Inserts.Count} | row {r}/{insert.Rows.Count}");
// Removes element if in black box
if (pkCount == 1)
{
if (Register.Registers[insert.ID].InBlackBox(insert.Table, insert.Rows[r][pkId]))
{
Console.WriteLine($"!-- Skipping/Removing Black Box element: {insert.Rows[r][pkId]}");
insert.Rows.RemoveAt(r);
r--;
continue;
}
}
if (table.Value.Name == "catalog_product_super_attribute")
{
if (ConfigProductRegister.ConfigParent.Contains(insert.Rows[r][1]))
{
Register.Registers[insert.ID].AddToBlackBox(table.Value.Name, insert.Rows[r][0]);
insert.Rows.RemoveAt(r);
r--;
continue;
}
}
var sameValues = (config.WhenSame == ActionName.SkipCheck) ? (-1, null) : SameValues(table.Value, i, r, insert.Rows[r]);
if (sameValues.row != null && canDoMerge)
{
if (BindAction(
(sameValues.fileId == -999 ? ActionName.Ignore : config.WhenSame),
insert.Table,
sameValues.fileId,
insert.ID,
sameValues.row,
insert.Rows[r],
pkId
))
{
insert.Rows.RemoveAt(r);
r--;
}
// If something was merged return true
if (config.WhenSame == ActionName.BindToThis || config.WhenSame == ActionName.Add)
{
didMerge = true;
}
}
else
{
if (table.Value.PrimaryKey != null && table.Value.PrimaryKey.Length == 1)
{
// NOTE: When Same IDs always adds
//(config.WhenIdSame == ActionName.SkipCheck) ? (-1, null) :
var sameIds = SameId(table.Value, insert.Rows[r][pkId], table.Value.AddedCopy, firstTime);
if (sameIds != -1 && canUpdatePk)
{
BindAction(
//config.WhenIdSame,
ActionName.Add,
insert.Table,
sameIds,
insert.ID,
null,
insert.Rows[r],
pkId
);
didMerge = true;
}
// Registers new PK
else
{
Register.Registers[insert.ID].AddPK(insert.Table, insert.Rows[r][pkId]);
}
}
}
}
}
lastIdDictionary[table.Key] = lastId;
}
return(didMerge);
}
public static void Cleanup(FileInstance file)
{
var register = Register.Registers[file.Tables.First().Value.ID];
bool repeat;
do
{
repeat = false;
foreach (var table in file.Tables)
{
var pkId = 0;
if (table.Value.PrimaryKey != null && table.Value.PrimaryKey.Length > 0)
{
pkId = table.Value.GetColumnId(table.Value.PrimaryKey[0]);
}
// If tables has no FK references, then we can skip
// the whole table check
if (!register.ForeignKeys.ContainsKey(table.Key))
{
continue;
}
Console.WriteLine($"- Cleanup of table: {table.Key}");
// Lets check all columns with FK references and
// remove them if they are in BB
var columns = register.ForeignKeys[table.Key].Keys.ToList();
for (var c = 0; c < columns.Count; c++)
{
// We need to get column ids by knowing their names
var columnId = table.Value.GetColumnId(columns[c]);
// Table to which reference is made
var targetTable = register.ForeignKeys[table.Key][columns[c]];
var canBeZero = false;
if (MergingController.Config.Tables.ContainsKey(targetTable))
{
canBeZero = MergingController.Config.Tables[targetTable].ZeroIdInTable;
}
// -- Inserts
foreach (var insert in table.Value.Inserts)
{
// -- Rows
for (var r = 0; r < insert.Rows.Count; r++)
{
var originalId = insert.Rows[r][columnId];
// TODO: Zero check, some tables have value as 0
if (originalId == "NULL" || (originalId == "0" && canBeZero == false) || !register.InBlackBox(targetTable, originalId))
{
continue;
}
if (table.Value.PrimaryKey != null && table.Value.PrimaryKey.Length == 1)
{
register.AddToBlackBox(table.Key, insert.Rows[r][pkId]);
}
insert.Rows.RemoveAt(r);
repeat = true;
r--;
}
}
}
}
var tables = register.ForeignKeysRule.Keys.ToList();
foreach (var tableName in tables)
{
var table = file.Tables[tableName];
var pkId = 0;
if (table.PrimaryKey != null && table.PrimaryKey.Length > 0)
{
pkId = table.GetColumnId(table.PrimaryKey[0]);
}
Console.WriteLine($"- Cleanup via RULE of table: {tableName}");
// Lets check all columns with FK references and
// remove them if they are in BB
var columns = register.ForeignKeysRule[tableName].Keys.ToList();
for (var c = 0; c < columns.Count; c++)
{
// We need to get column ids by knowing their names
var columnId = table.GetColumnId(columns[c]);
// -- Inserts
foreach (var insert in table.Inserts)
{
// -- Each value rule on columns
foreach (var frValue in register.ForeignKeysRule[table.Name][columns[c]])
{
var targetTable = register.ForeignKeysRule[table.Name][columns[c]][frValue.Key].ForeignKey.TargetTable;
// Column ID which holds value that's needed to be checked
var ruleColumnId = table.GetColumnId(
register.ForeignKeysRule[table.Name][columns[c]][frValue.Key].Rule.Column);
// Value that's required to be equal to execute FK reference
var ruleValue = register.ForeignKeysRule[table.Name][columns[c]][frValue.Key].Rule.Value;
// -- Rows
for (var r = 0; r < insert.Rows.Count; r++)
{
// If values are not equal -> skip
if (insert.Rows[r][ruleColumnId] != ruleValue)
{
continue;
}
var originalId = insert.Rows[r][columnId];
if (!register.InBlackBox(targetTable, originalId))
{
continue;
}
Console.WriteLine($"--- Removed {insert.Rows[r][pkId]}");
register.AddToBlackBox(tableName, insert.Rows[r][pkId]);
insert.Rows.RemoveAt(r);
repeat = true;
r--;
}
}
}
}
}
} while (repeat);
}
public FileInterpreter()
{
fileInst = new FileInstance();
}
public FileInstance GetFileInstance()
{
return(FileInstance.GetInstance(token, "fil.1"));
}
public static void Apply(FileConfig configLocal, FileInstance file, Config.Config configGlobal)
{
var configTables = new[]
{
configGlobal.Tables, configLocal.Tables
};
foreach (var tables in configTables)
{
if (tables == null)
{
continue;
}
foreach (var configTable in tables)
{
if (!file.Tables.ContainsKey(configTable.Key))
{
Logger.LogErrorMessage(
$"Table: {configTable.Key} not found, while applying file level configuration");
continue;
}
var table = file.Tables[configTable.Key];
if (!string.IsNullOrEmpty(configTable.Value.NewName))
{
table.NewName = configTable.Value.NewName;
}
if (configTable.Value.IgnoreColumns != null)
{
ApplyIgnoreColumn(configTable.Value, table);
}
if (configTable.Value.NewColumns != null)
{
ApplyNewColumn(configTable.Value, table);
}
if (configTable.Value.Columns != null)
{
foreach (var column in configTable.Value.Columns)
{
if (!table.Columns.ContainsKey(column.Key))
{
Logger.LogErrorMessage(
$"Column: {column.Key} not found in table {configTable.Key}, while applying file level configuration");
continue;
}
ApplyChange(configTable.Value, table, table.Columns[column.Key], file);
}
}
if (configTable.Value.DropColumns != null)
{
ApplyDropColumn(configTable.Value, table);
}
if (!string.IsNullOrEmpty(configTable.Value.ForceName))
{
foreach (var insert in table.Inserts)
{
insert.Table = configTable.Value.ForceName;
}
file.Tables.Add(configTable.Value.ForceName, table);
file.Tables.Remove(table.Name);
table.Name = configTable.Value.ForceName;
}
}
}
}
public static void UpdateFkRule(FileInstance file, int executionId)
{
var register = Register.Registers[file.Tables.First().Value.ID];
var tables = register.ForeignKeysRule.Keys.ToList();
var orgRegister = Register.Registers[0];
foreach (var tableName in tables)
{
var table = file.Tables[tableName];
var exceptionOnMissingFK = true;
if (MergingController.Config.Tables.ContainsKey(tableName))
{
exceptionOnMissingFK = MergingController.Config.Tables[tableName].ExceptionOnMissingFk;
}
Console.WriteLine($"- Update of FK via RULE in table: {tableName}");
// Lets check all columns with FK references and
// remove them if they are in BB
var columns = register.ForeignKeysRule[tableName].Keys.ToList();
for (var c = 0; c < columns.Count; c++)
{
// We need to get column ids by knowing their names
var columnId = table.GetColumnId(columns[c]);
// -- Inserts
foreach (var insert in table.Inserts)
{
// -- Each value rule on columns
foreach (var frValue in register.ForeignKeysRule[table.Name][columns[c]])
{
// Column ID which holds value that's needed to be checked
var ruleColumnId = table.GetColumnId(
register.ForeignKeysRule[table.Name][columns[c]][frValue.Key].Rule.Column);
// Value that's required to be equal to execute FK reference
var ruleValue = register.ForeignKeysRule[table.Name][columns[c]][frValue.Key].Rule.Value;
// -- Rows
for (var r = 0; r < insert.Rows.Count; r++)
{
// If values are not equal -> skip
if (insert.Rows[r][ruleColumnId] != ruleValue)
{
continue;
}
var originalId = insert.Rows[r][columnId];
if (originalId != "NULL")
{
try
{
insert.Rows[r][columnId] = register.GetPkRule(tableName, columns[c], frValue.Key, originalId);
}
catch (Exception)
{
if (executionId == 0)
{
if (exceptionOnMissingFK == false)
{
insert.Rows[r][columnId] = "NULL";
}
else
{
throw new Exception("Missing FK");
}
}
}
}
}
}
}
}
}
}
public static void UpdateFk(FileInstance file, int executionId)
{
var register = Register.Registers[file.Tables.First().Value.ID];
var orgRegister = Register.Registers[0];
foreach (var table in file.Tables)
{
var exceptionOnMissingFK = true;
if (MergingController.Config.Tables.ContainsKey(table.Key))
{
exceptionOnMissingFK = MergingController.Config.Tables[table.Key].ExceptionOnMissingFk;
}
// If tables has no FK references, then we can skip
// the whole table check
if (!register.ForeignKeys.ContainsKey(table.Key))
{
continue;
}
Console.WriteLine($"- Update of FK in table: {table.Key}");
// Lets check all columns with FK references and
// remove them if they are in BB
var columns = register.ForeignKeys[table.Key].Keys.ToList();
for (var c = 0; c < columns.Count; c++)
{
// We need to get column ids by knowing their names
var columnId = table.Value.GetColumnId(columns[c]);
// Table to which reference is made
var targetTable = register.ForeignKeys[table.Key][columns[c]];
var canBeZero = false;
if (MergingController.Config.Tables.ContainsKey(targetTable))
{
canBeZero = MergingController.Config.Tables[targetTable].ZeroIdInTable;
}
var removeIfMissing = false;
if (MergingController.Config.Files.Count > table.Value.ID &&
MergingController.Config.Files[table.Value.ID].Tables != null &&
MergingController.Config.Files[table.Value.ID].Tables.ContainsKey(table.Key) &&
MergingController.Config.Files[table.Value.ID].Tables[table.Key].Columns != null &&
MergingController.Config.Files[table.Value.ID].Tables[table.Key].Columns.ContainsKey(columns[c])
)
{
removeIfMissing = MergingController.Config.Files[table.Value.ID].Tables[table.Key]
.Columns[columns[c]].DropIfMissingFk;
}
// -- Inserts
foreach (var insert in table.Value.Inserts)
{
// -- Rows
for (var r = 0; r < insert.Rows.Count; r++)
{
var originalId = insert.Rows[r][columnId];
if ((originalId != "0" || canBeZero) && originalId != "NULL")
{
try
{
insert.Rows[r][columnId] = register.GetPK(table.Key, columns[c], originalId);
}
catch (Exception)
{
if (exceptionOnMissingFK == false)
{
insert.Rows[r][columnId] = "NULL";
}
else
{
if (!removeIfMissing)
{
throw new Exception($"Missing FK for table {table.Value} on column {columnId} with value {originalId}!");
}
if (table.Value.PrimaryKey.Length == 1)
{
register.AddToBlackBox(table.Key, insert.Rows[r][table.Value.GetColumnId(table.Value.PrimaryKey[0])]);
}
insert.Rows.RemoveAt(r);
r--;
}
}
}
}
}
}
}
}
public void Merge()
{
var outputFile = new FileInstance();
//if (Config.Cache != null && Config.Cache.On)
//{
// CacheManager.CleanUp(rawFiles);
//}
// Step 1. Build Tables
for (var i = 0; i < rawFiles.Count; i++)
{
// Step 1.1 Apply each file config to each file
ApplyConfigFile.Apply(Config.Files[i], rawFiles[i], Config);
// Step 1.2 While Building Tables create register of TB/PK and TB/FK
RegisterBuilder.BuildFk(rawFiles[i], i, Config);
// Step 1.3 Merge tables
//Merger.Merge(ref outputFile, rawFiles[i]);
}
// Custom Attribute Option Registration
if (Config.Cache != null && !Config.Cache.On && Config.Cache.CustomAttributeOption != null &&
Config.Cache.CustomAttributeOption.Count != 0)
{
CustomAttributeOptionBuilder.AddLastIdReference(rawFiles[0], Config.Cache.CustomAttributeOption);
}
// Merge file inserts
for (var f = 0; f < Config.Files.Count; f++)
{
if (Config.Files[f].MergeFileInto == -1)
{
continue;
}
// Copies inserts into target file
var targetFile = Config.Files[f].MergeFileInto;
foreach (var table in rawFiles[f].Tables)
{
rawFiles[targetFile].Tables[table.Key].Inserts.AddRange(table.Value.Inserts);
}
// Removes files from config and rawFile
Config.Files.RemoveAt(f);
rawFiles.RemoveAt(f);
f--;
}
// Move Data
for (var f = 0; f < Config.Files.Count; f++)
{
if (Config.Files[f].Tables == null)
{
continue;
}
foreach (var configTable in Config.Files[f].Tables)
{
if (configTable.Value.MoveData == null || configTable.Value.MoveData.Count == 0)
{
continue;
}
if (!rawFiles[f].Tables.ContainsKey(configTable.Key))
{
continue;
}
MoveData.Move(configTable.Value.MoveData, rawFiles[f], rawFiles[f].Tables[configTable.Key]);
}
}
// Copy Data
for (var f = 0; f < Config.Files.Count; f++)
{
if (Config.Files[f].Tables == null)
{
continue;
}
foreach (var configTable in Config.Files[f].Tables)
{
if (configTable.Value.CopyData == null || configTable.Value.CopyData.Count == 0)
{
continue;
}
if (!rawFiles[f].Tables.ContainsKey(configTable.Key))
{
continue;
}
CopyData.Copy(configTable.Value.CopyData, rawFiles[f].Tables[configTable.Key]);
}
}
if (Config.Cache != null && Config.Cache.On)
{
CacheManager.Build(rawFiles, Config.Cache);
}
// Step 1.3 Merge tables
Merger.Merge(ref outputFile, rawFiles[0]);
// Config product register
for (var f = 1; f < Config.Files.Count; f++)
{
if (rawFiles[f].Tables.ContainsKey("catalog_product_super_link"))
{
ConfigProductRegister.Build(rawFiles[f].Tables["catalog_product_super_link"]);
}
}
bool didMerge;
var executionId = 0;
do
{
foreach (var table in outputFile.Tables)
{
var register = table.Value.PrimaryKey != null && table.Value.PrimaryKey.Length == 1;
if (!register)
{
continue;
}
var pkId = table.Value.GetColumnId(table.Value.PrimaryKey[0]);
for (var i = 0; i < table.Value.Inserts.Count; i++)
{
var rows = table.Value.Inserts[i].Rows;
for (var r = 0; r < rows.Count; r++)
{
Register.Registers[0].AddPK(table.Key, rows[r][pkId]);
}
}
}
HashRegister.register.Clear();
var cacheSameFields = new Dictionary <string, Dictionary <string, int> >();
foreach (var table in outputFile.Tables)
{
var hash = Config.Tables.ContainsKey(table.Key) && Config.Tables[table.Key].SameOn != null;
if (!hash)
{
continue;
}
var comparedFields = Config.Tables[table.Key].SameOn;
if (comparedFields != null)
{
cacheSameFields.Add(table.Key, new Dictionary <string, int>());
foreach (var field in comparedFields)
{
cacheSameFields[table.Key].Add(field, table.Value.GetColumnId(field));
}
}
for (var i = 0; i < table.Value.Inserts.Count; i++)
{
var rows = table.Value.Inserts[i].Rows;
for (var r = 0; r < rows.Count; r++)
{
var hashName = new StringBuilder();
hashName.Append(table.Key + "::");
foreach (var cacheSameId in cacheSameFields[table.Key])
{
hashName.Append(rows[r][cacheSameId.Value].ToLower() + "$__::__$");
}
HashRegister.AddHashValue(table.Key, hashName.ToString(), new HashValue()
{
FileId = table.Value.ID,
InsertId = i,
RowId = r
});
}
}
}
didMerge = false;
if (FirstRun)
{
for (var f = 1; f < Config.Files.Count; f++)
{
foreach (var tableConfig in Config.Files[f].Tables)
{
if (tableConfig.Value.ForceMerge == null || tableConfig.Value.ForceMerge.Count == 0)
{
continue;
}
if (!rawFiles[f].Tables.ContainsKey(tableConfig.Key))
{
continue;
}
if (didMerge)
{
ForceMerger.Merge(rawFiles[f].Tables[tableConfig.Key], tableConfig.Value.ForceMerge);
}
else
{
didMerge = ForceMerger.Merge(rawFiles[f].Tables[tableConfig.Key], tableConfig.Value.ForceMerge);
}
}
}
}
for (var f = 1; f < rawFiles.Count; f++)
{
if (didMerge)
{
FileInsertMerger.Merge(rawFiles[f], FirstRun);
}
else
{
didMerge = FileInsertMerger.Merge(rawFiles[f], FirstRun);
}
}
if (FirstRun)
{
didMerge = true;
}
if (didMerge)
{
SameUsers.Run(outputFile, rawFiles, FirstRun);
}
else
{
didMerge = SameUsers.Run(outputFile, rawFiles, FirstRun);
}
if (!didMerge)
{
break;
}
for (var f = 1; f < rawFiles.Count; f++)
{
FileBlackBoxRemover.Cleanup(rawFiles[f]);
}
for (var f = 1; f < rawFiles.Count; f++)
{
FileFkUpdater.UpdateFk(rawFiles[f], executionId);
FileFkUpdater.UpdateFkRule(rawFiles[f], executionId);
}
for (var f = 1; f < rawFiles.Count; f++)
{
if (rawFiles[f].Tables.ContainsKey("catalog_category_entity"))
{
// Rebuild catalog_category_entity.path
foreach (var insert in rawFiles[f].Tables["catalog_category_entity"].Inserts)
{
CatalogCategoryPath.Run(insert);
}
}
if (rawFiles[f].Tables.ContainsKey("url_rewrite"))
{
// Rebuild url_rewrite.target_path
foreach (var insert in rawFiles[f].Tables["url_rewrite"].Inserts)
{
UrlRewriteTargetPath.Run(insert);
}
}
}
executionId++;
Register.Registers[0].PrimaryKeys.Clear();
Register.Registers[0].BlackBox.Clear();
for (var f = 1; f < rawFiles.Count; f++)
{
Register.Registers[f].RefreshPk();
Register.Registers[f].BlackBox.Clear();
}
if (FirstRun)
{
FirstRun = false;
}
} while (didMerge);
for (var f = 1; f < rawFiles.Count; f++)
{
if (rawFiles[f].Tables.ContainsKey("inventory_source_item"))
{
SkuRegister.Update(rawFiles[f].Tables["inventory_source_item"]);
}
}
SameUsers.SaveLog();
// Custom Attribute Option Builder - removes tmp insert
if (Config.Cache != null && !Config.Cache.On && Config.Cache.CustomAttributeOption != null &&
Config.Cache.CustomAttributeOption.Count != 0)
{
CustomAttributeOptionBuilder.RemoveReferenceRows(rawFiles[0]);
}
for (var f = 1; f < rawFiles.Count; f++)
{
foreach (var table in rawFiles[f].Tables)
{
//table.Value.BuildInsertColumnText();
outputFile.Tables[table.Key].Inserts.AddRange(table.Value.Inserts);
}
}
foreach (var table in outputFile.Tables)
{
table.Value.BuildInsertColumnText();
}
if (Config.Handlers.ContainsKey("ProductSameUrl"))
{
SameUrlAddressFixer.Run(outputFile, Config.Handlers["ProductSameUrl"]);
}
// Custom Attribute Option Builder
if (Config.Cache != null && !Config.Cache.On && Config.Cache.CustomAttributeOption != null &&
Config.Cache.CustomAttributeOption.Count != 0)
{
CustomAttributeOptionBuilder.Build(outputFile, Config.Cache.CustomAttributeOption);
}
// Step 3. Write SQL
using var file = new StreamWriter(Config.Output);
if (Config.Append != null && !string.IsNullOrEmpty(Config.Append.StartBeforeSet))
{
file.WriteLine(Config.Append.StartBeforeSet);
}
file.WriteLine("SET FOREIGN_KEY_CHECKS=0;");
file.WriteLine("SET NAMES utf8;");
file.WriteLine("SET SESSION sql_mode='NO_AUTO_VALUE_ON_ZERO';");
//file.WriteLine("SET GLOBAL max_allowed_packet=5073741824;");
if (Config.Append != null && !string.IsNullOrEmpty(Config.Append.StartAfterSet))
{
file.WriteLine(Config.Append.StartAfterSet);
}
foreach (var fInstTable in outputFile.Tables)
{
fInstTable.Value.GetSql(in file);
}
if (Config.Append != null && !string.IsNullOrEmpty(Config.Append.EndBeforeSet))
{
file.WriteLine(Config.Append.EndBeforeSet);
}
file.WriteLine("SET FOREIGN_KEY_CHECKS=1;");
// file.WriteLine("SET GLOBAL max_allowed_packet=16777216;");
if (Config.Append != null && !string.IsNullOrEmpty(Config.Append.EndAfterSet))
{
if (Config.Append.DoReplace)
{
Config.Append.EndAfterSet = Config.Append.EndAfterSet.Replace("\\'", "'");
}
file.WriteLine(Config.Append.EndAfterSet);
}
}
public HttpResponseMessage AddFolder(FileInstance folder)
{
BlogService.CreateDirectory(folder.FullPath + "/" + folder.Name);
return(Request.CreateResponse(HttpStatusCode.OK));
}
public static void SaveFile()
{
FileInstance.Save(FilePath);
}
