private APIReturnValue ProcessDocuments(string EntityName, DataRow Row, string CustomFields) { List <string> lstFailedFiles = new List <string>(); List <string> lstPanIds = new List <string>(); //string tempFilePath = Path.Combine("temp", Path.GetFileName(Row["filePath"].ToString())); string tempFilePath = Row["filePath"].ToString(); string APIPath = string.Empty; APIReturnValue retVal = new APIReturnValue(); string AppVenId = Preprocessor.vendorMasterInfoList[Convert.ToInt32(Row["venId"])].appVendorId; string strResp = string.Empty; //File.Copy(Row["filePath"].ToString(), tempFilePath, true); FileMetadata metadata = Util.CreateObjectFromRow <FileMetadata>(Row); metadata.appVendorId = AppVenId; metadata.projectId = ProjectId; switch (EntityName) { case "SolicitationDocument": { APIPath = $"projects({ProjectId})/doc-attach/objectstore/documents"; metadata.projectVendorId = Preprocessor.GetProjectVendorId(AppVenId, ProjectId); break; } case "CreditDocument": { APIPath = $"vendorCredits({Convert.ToInt32(Row["PanopticParentId"])})/doc-attach/objectstore/documents"; break; } } if (Row.Table.Columns.Contains("DocumentType") && Row["DocumentType"] != null) { metadata.docType = Preprocessor.documentTypeList[Row["DocumentType"].ToString().ToLower()]; } Dictionary <string, Stream> files = null; if (Path.GetExtension(tempFilePath).ToLower() == ".msg") { //files = GetFileStreams(tempFilePath); files = ExtractMSG(tempFilePath); Logger.log.Debug($"Get File Stream finished for {tempFilePath}"); } else { files = new Dictionary <string, Stream>(); files.Add(Path.GetFileName(tempFilePath), new FileStream(tempFilePath, FileMode.Open, FileAccess.Read)); } if (files != null) { foreach (KeyValuePair <string, Stream> kvp in files) { try { metadata.fileName = kvp.Key; metadata.aliasName = kvp.Key; Dictionary <string, string> kvfn = new Dictionary <string, string>(); if (!string.IsNullOrEmpty(CustomFields)) { dynamic expando = Util.ConvertToExpando(metadata); List <String> lst = CustomFields.Split(",").ToList(); Dictionary <string, object> tempProps = new Dictionary <string, object>(); foreach (string str in lst) { if (Row.Table.Columns.Contains(str) && Row[str] != null) { tempProps.Add(str, Row[str]); } } Util.AddProperties(expando, tempProps); kvfn.Add(metadata.fileName, JsonConvert.SerializeObject(expando)); } else { kvfn.Add(metadata.fileName, JsonConvert.SerializeObject(metadata)); } Dictionary <string, Dictionary <string, string> > fileMetadata = new Dictionary <string, Dictionary <string, string> >(); fileMetadata.Add("fileMetadata", kvfn); string s = JsonConvert.SerializeObject(fileMetadata); /* * using (FileStream outputFileStream = new FileStream(kvp.Key, FileMode.Create)) * { * kvp.Value.CopyTo(outputFileStream); * } */ HttpResponseMessage respMsg = Util.AuditAPIURL .AppendPathSegment(APIPath).WithOAuthBearerToken(Util.bearerToken) .PostMultipartAsync(mp => mp.AddStringParts(new { request = s }).AddFile("file", kvp.Value, kvp.Key)).Result; strResp = respMsg.Content.ReadAsStringAsync().Result; Logger.log.Debug($"Upload finished for {kvp.Key}"); if (respMsg.IsSuccessStatusCode) { List <Dictionary <string, object> > objResp = JsonConvert.DeserializeObject <List <Dictionary <string, object> > >(strResp); retVal.Status = StatusCode.PartiallyComplete; lstPanIds.Add(objResp[0]["documentId"].ToString()); } else { retVal.StatusDesc += strResp; lstFailedFiles.Add(kvp.Key); Logger.log.Error($"Error Occurred while Processing File {kvp.Key} for Record Id: {Row["RecordId"]}, Response: {strResp}"); } } catch (AggregateException ae) { lstFailedFiles.Add(kvp.Key); ae.Handle(e => { if (e is FlurlHttpException) { string es = ((FlurlHttpException)e).GetResponseStringAsync().Result; retVal.StatusDesc += es; Logger.log.Error(e, $"Error Occurred while Processing File {kvp.Key} for Record Id: {Row["RecordId"]}, Response: {es}"); } else { Logger.log.Error(e, $"Error Occurred while Processing File {kvp.Key} for Record Id: {Row["RecordId"]}"); retVal.StatusDesc += strResp; } return(true); //continue execution... set to false to break. }); } catch (Exception e) { lstFailedFiles.Add(kvp.Key); retVal.StatusDesc += strResp; Logger.log.Error(e, $"Error Occurred while Processing File {kvp.Key} for Record Id: {Row["RecordId"]}"); } } } else { throw new Exception("Error Occurred while Reading File Stream"); } if ((lstPanIds.Count > 0) && (lstFailedFiles.Count == 0)) { retVal.Status = StatusCode.Complete; } else if ((lstPanIds.Count > 0) && (lstFailedFiles.Count > 0)) { retVal.Status = StatusCode.PartiallyComplete; } else { retVal.Status = StatusCode.Error; } if (lstFailedFiles.Count > 0) { retVal.StatusDesc = $"Failed Files: {string.Join(",", lstFailedFiles)}, Response: {retVal.StatusDesc}"; } if (lstPanIds.Count > 0) { retVal.PanopticId = string.Join(",", lstPanIds); } //File.Delete(tempFilePath); return(retVal); }
private APIReturnValue ProcessComments(string EntityName, DataRow Row, string CustomFields) { APIReturnValue retVal = new APIReturnValue(); int ParentId; string qry = string.Empty; switch (EntityName) { case "SolicitationComment": { ParentId = Convert.ToInt32(Preprocessor.GetProjectVendorId(Preprocessor.vendorMasterInfoList[Convert.ToInt32(Row["VenId"])].appVendorId, ProjectId)); qry = $"vendorSolicitation({ParentId})/comment-attach/comments"; break; } case "CreditComment": { ParentId = Convert.ToInt32(Row["PanopticParentId"]); qry = $"vendorCredits({ParentId})/comment-attach/comments"; break; } case "ClaimComment": { ParentId = Convert.ToInt32(Row["PanopticParentId"]); qry = $"claims({ParentId})/comment-attach/comments"; break; } } string strResp = string.Empty; CommentRequest objComReq = Util.CreateObjectFromRow <CommentRequest>(Row); HttpResponseMessage respMsg; if (!string.IsNullOrEmpty(CustomFields)) { dynamic expando = Util.ConvertToExpando(objComReq); List <String> lst = CustomFields.Split(",").ToList(); Dictionary <string, object> tempProps = new Dictionary <string, object>(); foreach (string str in lst) { if (Row.Table.Columns.Contains(str) && Row[str] != null) { tempProps.Add(str, Row[str]); } } Util.AddProperties(expando, tempProps); string strPL = JsonConvert.SerializeObject(expando); respMsg = Util.AuditAPIURL.AppendPathSegment(qry) .WithOAuthBearerToken(Util.bearerToken).WithHeader("Content-Type", "application/json") .PostStringAsync(strPL).Result; } else { respMsg = Util.AuditAPIURL.AppendPathSegment(qry) .WithOAuthBearerToken(Util.bearerToken).WithHeader("Content-Type", "application/json") .PostJsonAsync(objComReq).Result; } strResp = respMsg.Content.ReadAsStringAsync().Result; retVal.StatusDesc = strResp; if (respMsg.IsSuccessStatusCode) { IDictionary <string, object> objResp = JsonConvert.DeserializeObject <ExpandoObject>(strResp); retVal.Status = StatusCode.Complete; retVal.PanopticId = objResp["id"].ToString(); } else { retVal.Status = StatusCode.Error; } return(retVal); }
public void ProcessData() { try { Preprocessor.Preprocess(); DataTable dtSource = null; Stopwatch sw = new Stopwatch(); DataSet ds = Util.GetMigrationConfigurations(); if (ds.Tables.Count > 0 && ds.Tables[0].Rows.Count > 0) { DataTable ProjTbl = ds.Tables[0].DefaultView.ToTable(true, "ClientCode", "ProjectId") .AsEnumerable().CopyToDataTable(); foreach (DataRow ProjRow in ProjTbl.Rows) { MigrationClientCode = ProjRow["ClientCode"].ToString(); ProjectId = Convert.ToInt32(ProjRow["ProjectId"]); CompletedCountList.Clear(); var ModuleRS = ds.Tables[0].AsEnumerable().Where(x => x.Field <string>("ClientCode") == MigrationClientCode && x.Field <int>("ProjectId") == ProjectId); foreach (DataRow ModuleRow in ModuleRS) { string entityName = ModuleRow["Name"].ToString(); int iterationCount = Convert.ToInt32(ModuleRow["IterationCount"]); int batchSize = Convert.ToInt32(ModuleRow["BatchSize"]); CompletedCountList.TryAdd(entityName, 0); sw.Restart(); Logger.log.Info($"Processing {entityName}s..."); int cnt = iterationCount; int ttl = 0; while (cnt > 0 && batchSize > 0) { dtSource = Util.GetMigrationData(entityName, MigrationClientCode, ProjectId, ModuleRow["SPName"].ToString(), batchSize).Tables[0]; int rowCount = dtSource.Rows.Count; if (dtSource != null && rowCount > 0) { Logger.log.Info($"Processing Interation {iterationCount - cnt + 1}, Count: {rowCount} ... "); ttl += dtSource.Rows.Count; Parallel.ForEach <DataRow>(dtSource.AsEnumerable(), new ParallelOptions() { //Sets the maximum number of concurrent tasks MaxDegreeOfParallelism = Convert.ToInt32(ModuleRow["ThreadCount"]) }, (row => { ProcessRow(entityName, row, ModuleRow["CustomFields"].ToString()); Console.Write($"\r{((CompletedCountList[entityName] - ((iterationCount - cnt) * batchSize)) * 100 / rowCount)}%"); })); } else { break; } --cnt; Console.WriteLine(); } if (ttl > 0) { Logger.log.Info($"{entityName}s {CompletedCountList[entityName]} of {ttl} Processed Successfully in {sw.Elapsed.TotalMinutes} Minutes."); } else { Logger.log.Warn($"No records found to Process {entityName}s"); } } } } else { Logger.log.Warn($"No Client or Project Information found to Process"); } } catch (AggregateException ae) { ae.Handle(e => { if (e is FlurlHttpException) { string es = ((FlurlHttpException)e).GetResponseStringAsync().Result; Logger.log.Error(e, $"HTTP Exception Occurred while Processing Data. Response: {es}"); } else { Logger.log.Error(e, "Error Occurred while Processing Data."); } return(false); //break execution. }); } }
public static bool Preprocess() { PopulateVendorInfo(); foreach (ExpandoObject o in GetAuditLookupValues("Vendor")) { string nm = o.FirstOrDefault(x => x.Key == "name").Value.ToString(); List <Object> f = (List <Object>)o.FirstOrDefault(x => x.Key == "dataObjectFieldValues").Value; Dictionary <string, string> tempD = new Dictionary <string, string>(); foreach (ExpandoObject obj in f) { string k = obj.FirstOrDefault(x => x.Key == "value").Value.ToString().ToLower(); string v = obj.FirstOrDefault(x => x.Key == "id").Value.ToString(); if (!tempD.ContainsKey(k)) { tempD.Add(k, v); } } if (!auditFieldLookupValues.ContainsKey(nm)) { auditFieldLookupValues.Add(nm, tempD); } } foreach (ExpandoObject o in GetClaimLookupValues("Claim")) { string nm = o.FirstOrDefault(x => x.Key == "name").Value.ToString(); List <Object> f = (List <Object>)o.FirstOrDefault(x => x.Key == "dataObjectFieldValues").Value; Dictionary <string, string> tempD = new Dictionary <string, string>(); foreach (ExpandoObject obj in f) { string k = obj.FirstOrDefault(x => x.Key == "value").Value.ToString().ToLower(); string v = obj.FirstOrDefault(x => x.Key == "id").Value.ToString(); if (!tempD.ContainsKey(k)) { tempD.Add(k, v); } } if (!claimFieldLookupValues.ContainsKey(nm)) { claimFieldLookupValues.Add(nm, tempD); } } foreach (ExpandoObject o in Preprocessor.GetDocumentTypes()) { int v = int.Parse((o.FirstOrDefault(x => x.Key == "documentTypeId").Value.ToString())); string av = (o.FirstOrDefault(x => x.Key == "documentType").Value.ToString().ToLower()); if (!documentTypeList.ContainsKey(av)) { documentTypeList.Add(av, v); } } foreach (ExpandoObject o in GetVendorStatementActions()) { int v = int.Parse((o.FirstOrDefault(x => x.Key == "vendorStatementActionId").Value.ToString())); string av = (o.FirstOrDefault(x => x.Key == "actionTaken").Value.ToString()); if (!vendorStatementActionList.ContainsKey(av)) { vendorStatementActionList.Add(av, v); } } foreach (ExpandoObject o in GetProjectVendorStatuses()) { int v = int.Parse((o.FirstOrDefault(x => x.Key == "projectVendorStatusId").Value.ToString())); string av = (o.FirstOrDefault(x => x.Key == "vendorStatus").Value.ToString().ToLower()); if (!projectVendorStatusList.ContainsKey(av)) { projectVendorStatusList.Add(av, v); } } return(true); }