private APIReturnValue ProcessDocuments(string EntityName, DataRow Row, string CustomFields)
{
List <string> lstFailedFiles = new List <string>();
List <string> lstPanIds = new List <string>();
//string tempFilePath = Path.Combine("temp", Path.GetFileName(Row["filePath"].ToString()));
string tempFilePath = Row["filePath"].ToString();
string APIPath = string.Empty;
APIReturnValue retVal = new APIReturnValue();
string AppVenId = Preprocessor.vendorMasterInfoList[Convert.ToInt32(Row["venId"])].appVendorId;
string strResp = string.Empty;
//File.Copy(Row["filePath"].ToString(), tempFilePath, true);
FileMetadata metadata = Util.CreateObjectFromRow <FileMetadata>(Row);
metadata.appVendorId = AppVenId;
metadata.projectId = ProjectId;
switch (EntityName)
{
case "SolicitationDocument":
{
APIPath = $"projects({ProjectId})/doc-attach/objectstore/documents";
metadata.projectVendorId = Preprocessor.GetProjectVendorId(AppVenId, ProjectId);
break;
}
case "CreditDocument":
{
APIPath = $"vendorCredits({Convert.ToInt32(Row["PanopticParentId"])})/doc-attach/objectstore/documents";
break;
}
}
if (Row.Table.Columns.Contains("DocumentType") && Row["DocumentType"] != null)
{
metadata.docType = Preprocessor.documentTypeList[Row["DocumentType"].ToString().ToLower()];
}
Dictionary <string, Stream> files = null;
if (Path.GetExtension(tempFilePath).ToLower() == ".msg")
{
//files = GetFileStreams(tempFilePath);
files = ExtractMSG(tempFilePath);
Logger.log.Debug($"Get File Stream finished for {tempFilePath}");
}
else
{
files = new Dictionary <string, Stream>();
files.Add(Path.GetFileName(tempFilePath), new FileStream(tempFilePath, FileMode.Open, FileAccess.Read));
}
if (files != null)
{
foreach (KeyValuePair <string, Stream> kvp in files)
{
try
{
metadata.fileName = kvp.Key;
metadata.aliasName = kvp.Key;
Dictionary <string, string> kvfn = new Dictionary <string, string>();
if (!string.IsNullOrEmpty(CustomFields))
{
dynamic expando = Util.ConvertToExpando(metadata);
List <String> lst = CustomFields.Split(",").ToList();
Dictionary <string, object> tempProps = new Dictionary <string, object>();
foreach (string str in lst)
{
if (Row.Table.Columns.Contains(str) && Row[str] != null)
{
tempProps.Add(str, Row[str]);
}
}
Util.AddProperties(expando, tempProps);
kvfn.Add(metadata.fileName, JsonConvert.SerializeObject(expando));
}
else
{
kvfn.Add(metadata.fileName, JsonConvert.SerializeObject(metadata));
}
Dictionary <string, Dictionary <string, string> > fileMetadata = new Dictionary <string, Dictionary <string, string> >();
fileMetadata.Add("fileMetadata", kvfn);
string s = JsonConvert.SerializeObject(fileMetadata);
/*
* using (FileStream outputFileStream = new FileStream(kvp.Key, FileMode.Create))
* {
* kvp.Value.CopyTo(outputFileStream);
* }
*/
HttpResponseMessage respMsg = Util.AuditAPIURL
.AppendPathSegment(APIPath).WithOAuthBearerToken(Util.bearerToken)
.PostMultipartAsync(mp => mp.AddStringParts(new { request = s }).AddFile("file", kvp.Value, kvp.Key)).Result;
strResp = respMsg.Content.ReadAsStringAsync().Result;
Logger.log.Debug($"Upload finished for {kvp.Key}");
if (respMsg.IsSuccessStatusCode)
{
List <Dictionary <string, object> > objResp = JsonConvert.DeserializeObject <List <Dictionary <string, object> > >(strResp);
retVal.Status = StatusCode.PartiallyComplete;
lstPanIds.Add(objResp[0]["documentId"].ToString());
}
else
{
retVal.StatusDesc += strResp;
lstFailedFiles.Add(kvp.Key);
Logger.log.Error($"Error Occurred while Processing File {kvp.Key} for Record Id: {Row["RecordId"]}, Response: {strResp}");
}
}
catch (AggregateException ae)
{
lstFailedFiles.Add(kvp.Key);
ae.Handle(e =>
{
if (e is FlurlHttpException)
{
string es = ((FlurlHttpException)e).GetResponseStringAsync().Result;
retVal.StatusDesc += es;
Logger.log.Error(e, $"Error Occurred while Processing File {kvp.Key} for Record Id: {Row["RecordId"]}, Response: {es}");
}
else
{
Logger.log.Error(e, $"Error Occurred while Processing File {kvp.Key} for Record Id: {Row["RecordId"]}");
retVal.StatusDesc += strResp;
}
return(true); //continue execution... set to false to break.
});
}
catch (Exception e)
{
lstFailedFiles.Add(kvp.Key);
retVal.StatusDesc += strResp;
Logger.log.Error(e, $"Error Occurred while Processing File {kvp.Key} for Record Id: {Row["RecordId"]}");
}
}
}
else
{
throw new Exception("Error Occurred while Reading File Stream");
}
if ((lstPanIds.Count > 0) && (lstFailedFiles.Count == 0))
{
retVal.Status = StatusCode.Complete;
}
else if ((lstPanIds.Count > 0) && (lstFailedFiles.Count > 0))
{
retVal.Status = StatusCode.PartiallyComplete;
}
else
{
retVal.Status = StatusCode.Error;
}
if (lstFailedFiles.Count > 0)
{
retVal.StatusDesc = $"Failed Files: {string.Join(",", lstFailedFiles)}, Response: {retVal.StatusDesc}";
}
if (lstPanIds.Count > 0)
{
retVal.PanopticId = string.Join(",", lstPanIds);
}
//File.Delete(tempFilePath);
return(retVal);
}
private APIReturnValue ProcessComments(string EntityName, DataRow Row, string CustomFields)
{
APIReturnValue retVal = new APIReturnValue();
int ParentId;
string qry = string.Empty;
switch (EntityName)
{
case "SolicitationComment":
{
ParentId = Convert.ToInt32(Preprocessor.GetProjectVendorId(Preprocessor.vendorMasterInfoList[Convert.ToInt32(Row["VenId"])].appVendorId, ProjectId));
qry = $"vendorSolicitation({ParentId})/comment-attach/comments";
break;
}
case "CreditComment":
{
ParentId = Convert.ToInt32(Row["PanopticParentId"]);
qry = $"vendorCredits({ParentId})/comment-attach/comments";
break;
}
case "ClaimComment":
{
ParentId = Convert.ToInt32(Row["PanopticParentId"]);
qry = $"claims({ParentId})/comment-attach/comments";
break;
}
}
string strResp = string.Empty;
CommentRequest objComReq = Util.CreateObjectFromRow <CommentRequest>(Row);
HttpResponseMessage respMsg;
if (!string.IsNullOrEmpty(CustomFields))
{
dynamic expando = Util.ConvertToExpando(objComReq);
List <String> lst = CustomFields.Split(",").ToList();
Dictionary <string, object> tempProps = new Dictionary <string, object>();
foreach (string str in lst)
{
if (Row.Table.Columns.Contains(str) && Row[str] != null)
{
tempProps.Add(str, Row[str]);
}
}
Util.AddProperties(expando, tempProps);
string strPL = JsonConvert.SerializeObject(expando);
respMsg = Util.AuditAPIURL.AppendPathSegment(qry)
.WithOAuthBearerToken(Util.bearerToken).WithHeader("Content-Type", "application/json")
.PostStringAsync(strPL).Result;
}
else
{
respMsg = Util.AuditAPIURL.AppendPathSegment(qry)
.WithOAuthBearerToken(Util.bearerToken).WithHeader("Content-Type", "application/json")
.PostJsonAsync(objComReq).Result;
}
strResp = respMsg.Content.ReadAsStringAsync().Result;
retVal.StatusDesc = strResp;
if (respMsg.IsSuccessStatusCode)
{
IDictionary <string, object> objResp = JsonConvert.DeserializeObject <ExpandoObject>(strResp);
retVal.Status = StatusCode.Complete;
retVal.PanopticId = objResp["id"].ToString();
}
else
{
retVal.Status = StatusCode.Error;
}
return(retVal);
}
public void ProcessData()
{
try
{
Preprocessor.Preprocess();
DataTable dtSource = null;
Stopwatch sw = new Stopwatch();
DataSet ds = Util.GetMigrationConfigurations();
if (ds.Tables.Count > 0 && ds.Tables[0].Rows.Count > 0)
{
DataTable ProjTbl = ds.Tables[0].DefaultView.ToTable(true, "ClientCode", "ProjectId")
.AsEnumerable().CopyToDataTable();
foreach (DataRow ProjRow in ProjTbl.Rows)
{
MigrationClientCode = ProjRow["ClientCode"].ToString();
ProjectId = Convert.ToInt32(ProjRow["ProjectId"]);
CompletedCountList.Clear();
var ModuleRS = ds.Tables[0].AsEnumerable().Where(x => x.Field <string>("ClientCode") == MigrationClientCode &&
x.Field <int>("ProjectId") == ProjectId);
foreach (DataRow ModuleRow in ModuleRS)
{
string entityName = ModuleRow["Name"].ToString();
int iterationCount = Convert.ToInt32(ModuleRow["IterationCount"]);
int batchSize = Convert.ToInt32(ModuleRow["BatchSize"]);
CompletedCountList.TryAdd(entityName, 0);
sw.Restart();
Logger.log.Info($"Processing {entityName}s...");
int cnt = iterationCount;
int ttl = 0;
while (cnt > 0 && batchSize > 0)
{
dtSource = Util.GetMigrationData(entityName, MigrationClientCode, ProjectId, ModuleRow["SPName"].ToString(), batchSize).Tables[0];
int rowCount = dtSource.Rows.Count;
if (dtSource != null && rowCount > 0)
{
Logger.log.Info($"Processing Interation {iterationCount - cnt + 1}, Count: {rowCount} ... ");
ttl += dtSource.Rows.Count;
Parallel.ForEach <DataRow>(dtSource.AsEnumerable(),
new ParallelOptions()
{
//Sets the maximum number of concurrent tasks
MaxDegreeOfParallelism = Convert.ToInt32(ModuleRow["ThreadCount"])
},
(row =>
{
ProcessRow(entityName, row, ModuleRow["CustomFields"].ToString());
Console.Write($"\r{((CompletedCountList[entityName] - ((iterationCount - cnt) * batchSize)) * 100 / rowCount)}%");
}));
}
else
{
break;
}
--cnt;
Console.WriteLine();
}
if (ttl > 0)
{
Logger.log.Info($"{entityName}s {CompletedCountList[entityName]} of {ttl} Processed Successfully in {sw.Elapsed.TotalMinutes} Minutes.");
}
else
{
Logger.log.Warn($"No records found to Process {entityName}s");
}
}
}
}
else
{
Logger.log.Warn($"No Client or Project Information found to Process");
}
}
catch (AggregateException ae)
{
ae.Handle(e =>
{
if (e is FlurlHttpException)
{
string es = ((FlurlHttpException)e).GetResponseStringAsync().Result;
Logger.log.Error(e, $"HTTP Exception Occurred while Processing Data. Response: {es}");
}
else
{
Logger.log.Error(e, "Error Occurred while Processing Data.");
}
return(false); //break execution.
});
}
}
public static bool Preprocess()
{
PopulateVendorInfo();
foreach (ExpandoObject o in GetAuditLookupValues("Vendor"))
{
string nm = o.FirstOrDefault(x => x.Key == "name").Value.ToString();
List <Object> f = (List <Object>)o.FirstOrDefault(x => x.Key == "dataObjectFieldValues").Value;
Dictionary <string, string> tempD = new Dictionary <string, string>();
foreach (ExpandoObject obj in f)
{
string k = obj.FirstOrDefault(x => x.Key == "value").Value.ToString().ToLower();
string v = obj.FirstOrDefault(x => x.Key == "id").Value.ToString();
if (!tempD.ContainsKey(k))
{
tempD.Add(k, v);
}
}
if (!auditFieldLookupValues.ContainsKey(nm))
{
auditFieldLookupValues.Add(nm, tempD);
}
}
foreach (ExpandoObject o in GetClaimLookupValues("Claim"))
{
string nm = o.FirstOrDefault(x => x.Key == "name").Value.ToString();
List <Object> f = (List <Object>)o.FirstOrDefault(x => x.Key == "dataObjectFieldValues").Value;
Dictionary <string, string> tempD = new Dictionary <string, string>();
foreach (ExpandoObject obj in f)
{
string k = obj.FirstOrDefault(x => x.Key == "value").Value.ToString().ToLower();
string v = obj.FirstOrDefault(x => x.Key == "id").Value.ToString();
if (!tempD.ContainsKey(k))
{
tempD.Add(k, v);
}
}
if (!claimFieldLookupValues.ContainsKey(nm))
{
claimFieldLookupValues.Add(nm, tempD);
}
}
foreach (ExpandoObject o in Preprocessor.GetDocumentTypes())
{
int v = int.Parse((o.FirstOrDefault(x => x.Key == "documentTypeId").Value.ToString()));
string av = (o.FirstOrDefault(x => x.Key == "documentType").Value.ToString().ToLower());
if (!documentTypeList.ContainsKey(av))
{
documentTypeList.Add(av, v);
}
}
foreach (ExpandoObject o in GetVendorStatementActions())
{
int v = int.Parse((o.FirstOrDefault(x => x.Key == "vendorStatementActionId").Value.ToString()));
string av = (o.FirstOrDefault(x => x.Key == "actionTaken").Value.ToString());
if (!vendorStatementActionList.ContainsKey(av))
{
vendorStatementActionList.Add(av, v);
}
}
foreach (ExpandoObject o in GetProjectVendorStatuses())
{
int v = int.Parse((o.FirstOrDefault(x => x.Key == "projectVendorStatusId").Value.ToString()));
string av = (o.FirstOrDefault(x => x.Key == "vendorStatus").Value.ToString().ToLower());
if (!projectVendorStatusList.ContainsKey(av))
{
projectVendorStatusList.Add(av, v);
}
}
return(true);
}
