public static Dictionary <int, Zmapping> getBody()
{
Dictionary <int, Zmapping> myMappings = new Dictionary <int, Zmapping>();
using (var db = new ZudelloContext())
{
int i = 0;
var MapID = db.Zmapping.Where(y => y.IsOutgoing == 1).ToList();
foreach (var mappings in MapID.OrderBy(y => y.ProcessOrder).ThenByDescending(l => l.connection_id))
{
Zmapping Data = new Zmapping()
{
DocType = mappings.DocType,
Body = mappings.Body,
Section = mappings.Section,
database = mappings.database,
Id = mappings.Id,
connection_id = mappings.connection_id
};
myMappings.Add(i, Data);
i++;
}
db.DisposeAsync();
return(myMappings);
}
}
public static bool InQueue(int?mappingId, string toSearch)
{
using (var db = new ZudelloContext())
{
//Maybe hash the records
var MapID = db.Zqueue.Where(i => i.MappingId == mappingId && i.Body.Contains(toSearch));
int count = MapID.Count();
db.DisposeAsync();
if (count > 0)
{
return(true);
}
else
{
return(false);
}
}
}
public static ExoMappingAndDB GetMappingAndDatabase(string Type = "", string connectionId = "")
{
ExoMappingAndDB mapDb = new ExoMappingAndDB();
using (var db = new ZudelloContext())
{
var myConnectionID = db.Zconnections.Where(c => c.ConnectionUuid == connectionId).FirstOrDefault();
var MapID = db.Zmapping.Where(i => i.DocType == Type && i.IsOutgoing == 0 && i.connection_id == myConnectionID.Id);
int count = 0;
try
{
count = MapID.Count();
}
catch
{
}
if (count > 1)
{
return(null);
}
foreach (var id in MapID)
{
mapDb.Id = myConnectionID.Id;
mapDb.InitialCatalog = myConnectionID.InitialCatalog;
mapDb.DataSource = myConnectionID.DataSource;
mapDb.IntergrationType = myConnectionID.IntergrationType;
mapDb.mappingID = id.Id;
return(mapDb);
}
db.DisposeAsync();
}
return(null);
}
public static int?SaveToDBQueue(string rawJson, int type, string status, ExoMappingAndDB map, int?linkedId = null)
{
try
{
using (var db = new ZudelloContext())
{
var res = db.Zqueue.Add(new Zqueue {
Body = rawJson,
MappingId = type,
Status = status,
ConnectionId = map.Id,
Queue_Id = linkedId
});
db.SaveChanges();
db.DisposeAsync();
return(res.Entity.Id);
}
}
catch (Exception ex)
{
return(null);
}
}
public static void RunSuccessUpdate(ConcurrentDictionary <int, ProccessResponse> successDictionary)
{
foreach (var update in successDictionary)
{
ProccessResponse pr = successDictionary.Values.FirstOrDefault();
int counter = 0;
string success = "Succuess";
string msg = pr.Information;
if (pr.Successful == false)
{
success = "Failed";
}
using (var db = new ZudelloContext())
{
var queue = db.Zqueue.Where(i => i.Id == update.Key).FirstOrDefault();
queue.Status = success;
queue.Exception = msg;
db.SaveChangesAsync();
counter++; //Maybe add in logic later if all is not updated then handel errors.
db.DisposeAsync();
}
}
}
public static async Task PushExoDataToZudello()
{
string Token = ZudelloLogin.Login();
SQLCredentials ConnectionString = new SQLCredentials();
Dictionary <int, string> Connection = ConnectionString.ConnectionStringBuilder();
/****
*
* Add threading into here for the foreach loop
* will need to chunk based on amount of connections
*/
//get the query from mappings table
Dictionary <int, Zmapping> MappingsBody = getBody();
int numberOfLogicalCores = Environment.ProcessorCount;
List <Thread> threads = new List <Thread>(numberOfLogicalCores);
int sizeOfOneChunk = (MappingsBody.Count / numberOfLogicalCores) + 1;
ConcurrentBag <string> cb = new ConcurrentBag <string>();
ConcurrentBag <int> cbMaps = new ConcurrentBag <int>();
for (int i = 0; i < numberOfLogicalCores; i++)
{
int ab = i;
var thr = new Thread(
() =>
{
try
{
int count = 0;
Dictionary <int, Zmapping> MappingsChunked = MappingsBody.Skip(ab * sizeOfOneChunk)
.Take(sizeOfOneChunk).ToDictionary(p => p.Key, p => p.Value);
foreach (var Mappings in MappingsChunked)
{
bool isXMl = true;
//Get the SQL statement
string mySql = Mappings.Value.Body;
//Get the Mappings
dynamic zudelloObject = Mappings.Value;
string SQLQuery = ExoTools.RenderToSql(mySql, zudelloObject);
#warning make this in config also for process method
string cmd = SQLQuery;
//Open SQL connection and run the SQL query
//Get connection details by Key ID
SqlConnection con = new SqlConnection(Connection[Mappings.Value.connection_id]);
SqlCommand SelectCommand = new SqlCommand(cmd, con);
// SqlDataReader myreader;
con.Open();
if (cmd.ToLower().Contains("for xml"))
{
isXMl = true;
}
if (String.IsNullOrEmpty(cmd))
{
continue;
}
var jsonResult = new StringBuilder();
var xmlConvert = new StringBuilder();
//SelectCommand.CommandText = cmd;
var myreader = SelectCommand.ExecuteReader();
if (!myreader.HasRows)
{
//if there is no data then close connection and next loop
con.Close();
con.Dispose();
continue;
}
else
{
while (myreader.Read())
{
jsonResult.Append(myreader.GetValue(0).ToString());
}
//Console.WriteLine(jsonResult);
dynamic obj = "";
string ConvertedXml = "";
if (isXMl == true)
{
ConvertedXml = ExoTools.ZudelloXMLConverter(jsonResult.ToString());
// string ConvertedXml = ExoTools.ZudelloXMLConverter(jsonResult.ToString());
obj = JsonConvert.DeserializeObject <ExpandoObject>(ConvertedXml);
}
else
{
obj = JsonConvert.DeserializeObject <ExpandoObject>(jsonResult.ToString());
}
foreach (var ObjectType in obj)
{
//not sure why .Value is causing error but not affecting build
foreach (var dataValues in ObjectType.Value)
{
string data = "";
if (isXMl == true)
{
data = JsonConvert.SerializeObject(dataValues);
}
else
{
MyQueueObject dataWrapper = new MyQueueObject();
dataWrapper.data = dataValues;
data = JsonConvert.SerializeObject(dataWrapper);
}
// add to concurrent bag
cb.Add(data);
count++;
// Console.WriteLine(data);
}
}
cbMaps.Add(Mappings.Value.Id);
}
con.Close();
con.Dispose();
}
}
catch (Exception ex)
{
}
}
);
threads.Add(thr);
}
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
threads.Clear();
int sizeOfOneChunkQueue = (cb.Count / numberOfLogicalCores) + 1;
for (int x = 0; x < numberOfLogicalCores; x++)
{
int abx = x;
var SendToQueuethr = new Thread(
() =>
{
try
{
int count = 0;
List <string> queuedBag = cb.Skip(abx * sizeOfOneChunkQueue)
.Take(sizeOfOneChunkQueue).ToList();
foreach (var Mappings in queuedBag)
{
Console.WriteLine(ZudelloLogin.SendToZudelloQueue(Token, Mappings));
}
}
catch (Exception ex)
{
}
});
threads.Add(SendToQueuethr);
}
foreach (var qthread in threads)
{
qthread.Start();
}
foreach (var qthread in threads)
{
qthread.Join();
}
/*
* List<Task> bagConsumeTasks = new List<Task>();
* int itemsInBag = 0;
* while (!cb.IsEmpty)
* {
* bagConsumeTasks.Add(Task.Run(() =>
* {
* string item;
* if (cb.TryTake(out item))
* {
*
* Console.WriteLine(ZudelloLogin.SendToZudelloQueue(Token, item));
* itemsInBag++;
* }
* }));
* } */
// Task.WaitAll(bagConsumeTasks.ToArray());
foreach (int id in cbMaps)
{
using (var db = new ZudelloContext())
{
var lastSync = db.Zlastsync.Where(s => s.MappingId == id).FirstOrDefault();
lastSync.LastSync = DateTime.Now.ToString();
db.SaveChanges();
db.DisposeAsync();
}
}
}
public static async Task AddtoQueue()
{
//Check Mappings
ZudelloLogin.GetZudelloMappings().Wait();
string Token = ZudelloLogin.Login();
List <Zconnections> teams = new List <Zconnections>();
using (var db = new ZudelloContext())
{
//Maybe hash the records
teams = db.Zconnections.ToList();
await db.DisposeAsync();
}
int numberOfLogicalCores = Environment.ProcessorCount;
List <Thread> threads = new List <Thread>(numberOfLogicalCores);
int sizeOfOneChunk = (teams.Count / numberOfLogicalCores) + 1;
ConcurrentDictionary <int, dynamic> cb = new ConcurrentDictionary <int, dynamic>();
ConcurrentDictionary <dynamic, LastSync> lastSyncBag = new ConcurrentDictionary <dynamic, LastSync>();
for (int i = 0; i < numberOfLogicalCores; i++)
{
int ab = i;
var thr = new Thread(
() =>
{
int count = 0;
List <Zconnections> teamChunked = teams.Skip(ab * sizeOfOneChunk)
.Take(sizeOfOneChunk).ToList();
foreach (var team in teamChunked)
{
try
{
dynamic ZudelloTeam = JsonConvert.DeserializeObject <ExpandoObject>(team.ZudelloCredentials);
string InvoiceList = ZudelloLogin.CallZudelloDocs(Token, ZudelloTeam.team, team.Id);
dynamic ToSync = JObject.Parse(InvoiceList);
foreach (var data in ToSync.data)
{
try
{
string myDataString = data.ToString();
// Console.WriteLine(Id.uuid + "," + Id.items);
Console.WriteLine(data);
ExoMappingAndDB mapId = GetMappingAndDatabase(data.document.docType.ToString(),
data.document.connectionUuid.ToString());
int?rowId = null;
if (mapId != null)
{
rowId = SaveToDBQueue(myDataString, mapId.mappingID, "Waiting", mapId);
}
var validate = Validations.ExoSupplierInvoice(myDataString, mapId);
if (validate.CreateSupplier == true)
{
//HAVE to hard code type for the supplier and inventory?
mapId = GetMappingAndDatabase(data.document.supplier.docType.ToString(), data.document.connectionUuid.ToString());
// Check if already in the queue
if (InQueue(mapId.mappingID, data.document.supplier.code.ToString()) == false)
{
SaveToDBQueue(data.document.supplier.ToString(), mapId.mappingID, "Waiting", mapId, rowId);
}
}
if (validate.CreateInventory == true)
{
foreach (var itemData in data.document.lines)
{
mapId = GetMappingAndDatabase(itemData.docType.ToString(), data.document.connectionUuid.ToString());
if (validate.InventoryToCreate.Contains(itemData.item.sku.ToString()))
{
// Check if already in the queue
if (InQueue(mapId.mappingID, itemData.item.sku.ToString()) == false)
{
SaveToDBQueue(itemData.ToString(), mapId.mappingID, "Waiting", mapId, rowId);
}
}
}
}
LastSync lastSync = new LastSync();
lastSync.teamId = team.Id;
lastSync.lastUpdated = data.created_at.ToString("yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'");
lastSyncBag.TryAdd(data.uuid, lastSync);
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
catch (Exception ex)
{
//Write to loggings tables
Console.WriteLine(ex.Message);
}
}
}
);
threads.Add(thr);
}
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
var dataList = lastSyncBag.Values.ToList();
foreach (var lastSync in dataList)
{
using (var db = new ZudelloContext())
{
var updateLastSync = (from a in db.Zlastsync
join c in db.Zmapping on a.MappingId equals c.Id
where c.DocType == "CallZudelloDocs" && c.connection_id == lastSync.teamId
select a).FirstOrDefault();
if (DateTime.Parse(updateLastSync.LastSync) > DateTime.Parse(lastSync.lastUpdated))
{
}
else
{
updateLastSync.LastSync = lastSync.lastUpdated;
db.SaveChanges();
}
db.Dispose();
}
}
}
public static async Task GetZudelloMappings(string team = null)
{
string Token = ZudelloLogin.Login();
//Output object should be this
List <Zconnections> teams = new List <Zconnections>();
using (var db = new ZudelloContext())
{
//Maybe hash the records
teams = db.Zconnections.ToList();
await db.DisposeAsync();
}
int numberOfLogicalCores = Environment.ProcessorCount;
List <Thread> threads = new List <Thread>(numberOfLogicalCores);
int sizeOfOneChunk = (teams.Count / numberOfLogicalCores) + 1;
ConcurrentBag <Zmapping> cb = new ConcurrentBag <Zmapping>();
ConcurrentDictionary <int, dynamic> cbd = new ConcurrentDictionary <int, dynamic>();
for (int i = 0; i < numberOfLogicalCores; i++)
{
int ab = i;
var thr = new Thread(
() =>
{
int count = 0;
List <Zconnections> teamChunked = teams.Skip(ab * sizeOfOneChunk)
.Take(sizeOfOneChunk).ToList();
foreach (var t in teamChunked)
{
dynamic ZudelloTeam = JsonConvert.DeserializeObject <ExpandoObject>(t.ZudelloCredentials);
bool newMappings = false;
string GetMapping = ZudelloLogin.CallZudelloMapping(Token, ZudelloTeam.team);
ZmappingJson MappingFiles = JsonConvert.DeserializeObject <ZmappingJson>(GetMapping);
dynamic zudelloObject = JObject.Parse(GetMapping);
foreach (Zmapping map in MappingFiles.data)
{
Console.WriteLine(map.UpdatedAt);
cb.Add(map);
}
}
}
);
threads.Add(thr);
}
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
foreach (var map in cb)
{
bool newMappings = false;
using (var db = new ZudelloContext())
{
try {
Zmapping AddtoMap = map;
Zmapping MapID = db.Zmapping.Where(i => i.DocType == map.DocType &&
i.Type == map.Type &&
i.connection_id == map.connection_id).FirstOrDefault();
if (MapID.Id > 0)
{
MapID.Body = map.Body; //= map.ShallowCopy();
MapID.IsOutgoing = map.IsOutgoing;
MapID.ProcessOrder = map.ProcessOrder;
MapID.IsMasterData = map.IsMasterData;
MapID.connection_id = map.connection_id;
MapID.Type = map.Type;
MapID.IntergrationUuid = map.IntergrationUuid;
// MapID.uuid = map.uuid;
db.Update(MapID);
db.SaveChanges();
var updateLastSync = (from a in db.Zlastsync
join c in db.Zmapping on a.MappingId equals c.Id
where c.DocType == "CallZudelloMapping"
select a).FirstOrDefault();
string pr = String.Format(DateTime.UtcNow.ToString(), "yyyy'-'MM'-'dd'T'HH':'mm':'ss'.'fff'Z'");
updateLastSync.LastSync = pr;
// db.SaveChanges();
}
else
{
//add to mapping if it does not exist.
db.Zmapping.Add(AddtoMap);
db.SaveChanges();
Console.WriteLine(String.Format("Doc Type {0} has been added: Type {1} has been added", map.DocType, map.Type));
newMappings = true;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);//some error with mappings
try
{
Zmapping AddtoMap = map;
db.Zmapping.Add(AddtoMap);
db.SaveChanges();
Console.WriteLine(String.Format("Doc Type {0} has been added: Type {1} has been added", map.DocType, map.Type));
newMappings = true;
}
catch (Exception exm)
{
Console.WriteLine(exm.Message);
}
finally
{
if (map.Type == "SQL_LITE_CMD")
{
Console.WriteLine(Tools.RunSQLLiteCmd(Token, map));
}
}
}
finally
{
if (map.Type == "SQL_LITE_CMD")
{
Console.WriteLine(Tools.RunSQLLiteCmd(Token, map));
}
}
db.DisposeAsync();
}
if (newMappings == true)
{
try //Add into sync table if there are any new records
{
SetSyncTable().Wait();
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
}
}
}
}
public static string CallZudelloMapping(string token, string myTeam = null)
{
string result = "";
string website = "";
string key = "";
string apiurl;
string ApiResult = "";
string entity = ZudelloInitalSettings.GetZudelloSettings().ZudelloMappingEndpoint;
string intergration = ZudelloInitalSettings.GetZudelloSettings().intergrationType;
string team = "";
//Multiple Database Teams
if (myTeam == null)
{
team = ZudelloInitalSettings.GetZudelloSettings().ZudelloTeam;
}
else
{
team = myTeam;
}
string createTimeQuery = "?updated_at=";
string createdate = "2010-11-20T05:55:29.000Z";
using (var db = new ZudelloContext())
{
try
{
var updateLastSync = (from a in db.Zlastsync
join c in db.Zmapping on a.MappingId equals c.Id
where c.DocType == "CallZudelloMapping"
select a).FirstOrDefault();
createdate = updateLastSync.LastSync;//DateupdateLastSync.LastSync;
// db.SaveChanges();
db.DisposeAsync();
}
catch (Exception ex)
{
db.DisposeAsync();
}
}
createTimeQuery = createTimeQuery + createdate;
// work out API call
apiurl = website + entity + intergration + createTimeQuery;
// work out API call
//apiurl = website + entity + uuid;
// do the api call
ServicePointManager.ServerCertificateValidationCallback = delegate { return(true); };
var request = (HttpWebRequest)WebRequest.Create(apiurl);
// var data = Encoding.ASCII.GetBytes(apibody);
request.Method = "GET";
//request.ContentLength = data.Length;
request.ContentType = "application/json";
request.Accept = "application/json";
string platform = ZudelloInitalSettings.GetZudelloSettings().ZudelloPlatform;
//"4rUtOPCRnGA5I1qXXLPPn16BdgjDPw6R"
request.Headers.Add("x-team", team);
request.Headers.Add("x-platform", platform);
request.Headers.Add("Authorization", "Bearer " + token);
/* using (var stream = request.GetRequestStream())
* {
* stream.Write(data, 0, data.Length);
* } */
var response = (HttpWebResponse)request.GetResponse();
var responseString = new StreamReader(response.GetResponseStream()).ReadToEnd();
if (response.StatusCode == HttpStatusCode.OK)
{
result = responseString;
}
else
{
result = responseString;
}
return(result);
}
public static async Task ProcessRecords()
{
int?MasterDataAwaiting = null;
List <QueueProcess> queueToProcess = new List <QueueProcess>();
using (var db = new ZudelloContext())
{
var listQueued = (from queue in db.Zqueue
join map in db.Zmapping on queue.MappingId equals map.Id
where map.IsMasterData == 1 && queue.Status.Trim() == "Waiting" //Will make more generic later
select new
{
queue,
map
}).ToList();
foreach (var data in listQueued)
{
QueueProcess m = new QueueProcess();
m.map = data.map;
m.queue = data.queue;
queueToProcess.Add(m);
}
db.DisposeAsync();
}
int numberOfLogicalCores = Environment.ProcessorCount;
List <Thread> threads = new List <Thread>(numberOfLogicalCores);
int sizeOfOneChunk = (queueToProcess.Count / numberOfLogicalCores) + 1;
ConcurrentBag <string> cb = new ConcurrentBag <string>();
ConcurrentDictionary <int, ProccessResponse> successDictionary = new ConcurrentDictionary <int, ProccessResponse>();
for (int i = 0; i < numberOfLogicalCores; i++)
{
int ab = i;
var thr = new Thread(
() =>
{
int count = 0;
List <QueueProcess> chunkedMaster = queueToProcess.Skip(ab * sizeOfOneChunk)
.Take(sizeOfOneChunk).ToList();
foreach (var queueData in chunkedMaster)
{
ProccessResponse success = new ProccessResponse();
success = ProcessMethod(queueData);
successDictionary.TryAdd(queueData.queue.Id, success);
}
});
threads.Add(thr);
}
foreach (var thread in threads)
{
thread.Start();
}
foreach (var thread in threads)
{
thread.Join();
}
//clear threads list
threads.Clear();
RunSuccessUpdate(successDictionary);
//clear Success Dictionary
successDictionary.Clear();
//Get transactional data to process
//Clear Queue List
// queueToProcess.Clear();
List <QueueProcess> tQueueToProcess = new List <QueueProcess>();
using (var db = new ZudelloContext())
{
var filterQueue = db.Zqueue.Where(x => x.Queue_Id != null &&
x.Status == "Failed").Select(x => x.Queue_Id).ToArray();
var listQueued = (from queue in db.Zqueue
join map in db.Zmapping on queue.MappingId equals map.Id
where map.IsMasterData != 1 &&
queue.Status.Trim() == "Waiting" &&
!filterQueue.Contains(queue.Id)
select new
{
queue,
map
}).ToList();
foreach (var data in listQueued)
{
QueueProcess m = new QueueProcess();
m.map = data.map;
m.queue = data.queue;
tQueueToProcess.Add(m);
}
}
//Clear concurrent Dictionary
//Transactional Data
/*
*
*
*
*/
//Reset sizeOf the Chunk
sizeOfOneChunk = (tQueueToProcess.Count / numberOfLogicalCores) + 1;
for (int x = 0; x < numberOfLogicalCores; x++)
{
int xb = x;
var transThr = new Thread(
() =>
{
int count = 0;
List <QueueProcess> chunkedTrans = tQueueToProcess.Skip(xb * sizeOfOneChunk)
.Take(sizeOfOneChunk).ToList();
foreach (var queueData in chunkedTrans)
{
//think this was bug
ProccessResponse success = new ProccessResponse();
success = ProcessMethod(queueData);
successDictionary.TryAdd(queueData.queue.Id, success);
}
});
threads.Add(transThr);
}
foreach (var transThread in threads)
{
transThread.Start();
}
foreach (var transThread in threads)
{
transThread.Join();
}
//clear threads list
// threads.Clear();
RunSuccessUpdate(successDictionary);
RunResponse(successDictionary);
}
