private void loopWithTotalCount(List <PawnDocRegVO> totList) { int threadNameCount = 1; int allocatedCount = 0; //Start dao thread GetDocumentToArchTempAsync(); while (true) { if (totList.Count == 0) { break; } int allowedCount = getAllowedCount(ref runningList); int jobCount = 0; int endCount = 20; List <string> subList = new List <string>(); List <PawnDocRegVO> subList1 = new List <PawnDocRegVO>(); while (jobCount < allowedCount) { if (totList.Count < endCount) { endCount = totList.Count; } subList = new List <string>(); for (int i = 0; i < endCount; i++) { subList.Add(totList[i].StorageID); subList1.Add(totList[i]); } AddTestDocThread th = new AddTestDocThread(subList, vo, getDocID); Thread jobThread = null; jobThread = new Thread(new ThreadStart(th.process)); string threhName = "Thread" + threadNameCount; threadNameCount++; jobThread.Name = threhName; runningList.Add(jobThread); log.Info(string.Format("Thread {0} started with count {1}", threhName, subList.Count)); allocatedCount += subList.Count; jobThread.Start(); foreach (var VARIABLE in subList1) { totList.Remove(VARIABLE); } if (totList.Count == 0) { break; } jobCount++; } Thread.Sleep(250); } }
/* private List<string> loopAndGetDocs() * { * List<string> totList = new List<string>(); * DocumentDAO dao = DocumentDAO.getInstance(); * Dictionary<string, string> dict = new Dictionary<string, string>(); * int pendCount = 0; * int errorCode = 0; * string errorMsg = ""; * int lastitem = 0; * int initCount = 0; * List<PawnDocRegVO> vo1 = new List<PawnDocRegVO>(); * while(true) * { * vo1 = new List<PawnDocRegVO>(); * dao.GetDocumentSets(lastitem, out vo1, out pendCount, out errorCode, out errorMsg); * * foreach (var pawnDocRegVO in vo1) * { * totList.Add(pawnDocRegVO.StorageID); * log.Info("DocID :"+pawnDocRegVO.DocID+": storage :"+pawnDocRegVO.StorageID); * // totList.Add("Input :"+lastitem+": DocID :"+pawnDocRegVO.DocID+": storage :"+pawnDocRegVO.StorageID); * } * if (pendCount == 0) * { * break; * } * lastitem = vo1[vo1.Count - 1].DocID; * // log.Debug("Break in between..................."); * // break; * } * initCount = totList.Count; * log.Info("Initial count:" + initCount); * foreach (string VARIABLE in totList) * { * if (!dict.ContainsKey(VARIABLE)) * { * dict.Add(VARIABLE, VARIABLE); * } * else * { * log.Debug(VARIABLE); * } * } * totList = new List<string>(dict.Values); * log.Info("No of dupl=" + (initCount - totList.Count)); * // log.Info("Filtered Count :" + totList.Count); * return totList; * } * * * private List<string> loopAndGetDocs4() * { * List<string> totList = new List<string>(); * DocumentDAO dao = DocumentDAO.getInstance(); * Dictionary<string, string> dict = new Dictionary<string, string>(); * int pendCount = 0; * int errorCode = 0; * string errorMsg = ""; * * int initCount = 0; * List<PawnDocRegVO> vo1 = new List<PawnDocRegVO>(); * vo1 = new List<PawnDocRegVO>(); * * dao.GetDocumentSets(lastitem, out vo1, out pendCount, out errorCode, out errorMsg); * * foreach (var pawnDocRegVO in vo1) * { * totList.Add(pawnDocRegVO.StorageID); * log.Info("DocID :" + pawnDocRegVO.DocID + ": storage :" + pawnDocRegVO.StorageID); * // totList.Add("Input :"+lastitem+": DocID :"+pawnDocRegVO.DocID+": storage :"+pawnDocRegVO.StorageID); * } * lastitem = vo1[vo1.Count - 1].DocID; * log.Info("Initial count:" + totList.Count); * return totList; * } * * //For 2 new procs from sree * private List<string> loopAndGetDocs3() * { * List<string> totList = new List<string>(); * DocumentDAO dao = DocumentDAO.getInstance(); * Dictionary<string, string> dict = new Dictionary<string, string>(); * int pendCount = 0; * int errorCode = 0; * string errorMsg = ""; * int lastitem = 0; * int initCount = 0; * List<PawnDocRegVO> vo1 = new List<PawnDocRegVO>(); * * //First call to get count * dao.CountProcTest(0, out vo1, out pendCount, out errorCode, out errorMsg); * log.Info("Total Count Retruned by New Proc : " + pendCount); * while (true) * { * vo1 = new List<PawnDocRegVO>(); * dao.GetDocumentSets(lastitem, out vo1, out pendCount, out errorCode, out errorMsg); * log.Info("#################################"); * log.Info("Input Value :" + lastitem); * log.Info("#################################"); * foreach (var pawnDocRegVO in vo1) * { * totList.Add(pawnDocRegVO.StorageID); * log.Info("DocID :" + pawnDocRegVO.DocID + ": storage :" + pawnDocRegVO.StorageID); * // totList.Add("Input :"+lastitem+": DocID :"+pawnDocRegVO.DocID+": storage :"+pawnDocRegVO.StorageID); * } * if (vo1.Count == 0) * { * break; * } * lastitem = vo1[vo1.Count - 1].DocID; * } * initCount = totList.Count; * log.Info("Initial count By Proc2:" + initCount); * foreach (string VARIABLE in totList) * { * if (!dict.ContainsKey(VARIABLE)) * { * dict.Add(VARIABLE, VARIABLE); * } * else * { * log.Debug(VARIABLE); * } * } * totList = new List<string>(dict.Values); * log.Info("No of dupl=" + (initCount - totList.Count)); * // log.Info("Filtered Count :" + totList.Count); * return totList; * }*/ public void populateData(CouchVo vo, bool isDelete, string getDocID) { setCouchValues(vo); string fileWithPath = "C:\\dev\\dev_jak2\\devwork_jkingsley\\Phase2App\\CouchConnectorApp\\test\\sample.pdf"; bool added = false; List <string> docIds = null; List <string> totList = new List <string>(); //totList = loopAndGetDocs(); //log.Info("Calling million docs"); //totList = loopAndGetDocs4(); // proc list below //totList = loopAndGetDocs(); //million doc by temp query below totList = loopAndGetDocs2(); //log.Debug("Aborted total exec..."); //return; /*GetDocumentSets(ref docIds, 0,1000); * totList.AddRange(docIds); * GetDocumentSets(ref docIds, 2000, 3000); * totList.AddRange(docIds); * GetDocumentSets(ref docIds, 9000, 10000); * totList.AddRange(docIds); * GetDocumentSets(ref docIds, 15000,20000); * totList.AddRange(docIds); * GetDocumentSets(ref docIds, 15000, 20000); * totList.AddRange(docIds);*/ /*foreach (var docId in docIds) * { * addDocumentToCouch(fileWithPath, docId, out added); * }*/ List <Thread> runningList = new List <Thread>(); Stopwatch sw = new Stopwatch(); sw.Start(); //var addJob = null; int threadNameCount = 1; int allocatedCount = 0; int totCount = totList.Count; while (true) { if (totList.Count == 0) { break; } int allowedCount = getAllowedCount(ref runningList); int jobCount = 0; int endCount = 1000; List <string> subList = new List <string>(); while (jobCount < allowedCount) { if (totList.Count < endCount) { endCount = totList.Count; } subList = new List <string>(); for (int i = 0; i < endCount; i++) { subList.Add(totList[i]); } AddTestDocThread th = new AddTestDocThread(subList, vo, getDocID); Thread jobThread = null; if (isDelete) { jobThread = new Thread(new ThreadStart(th.processDelete)); } else { jobThread = new Thread(new ThreadStart(th.process)); } string threhName = "Thread" + threadNameCount; threadNameCount++; jobThread.Name = threhName; runningList.Add(jobThread); log.Info(string.Format("Thread {0} started with count {1}", threhName, subList.Count)); allocatedCount += subList.Count; jobThread.Start(); foreach (var VARIABLE in subList) { totList.Remove(VARIABLE); } if (totList.Count == 0) { break; } jobCount++; } Thread.Sleep(250); } while (true) { List <Thread> clearList = new List <Thread>(); foreach (var thread in runningList) { if (!thread.IsAlive) { clearList.Add(thread); } } foreach (var thread in clearList) { runningList.Remove(thread); } if (runningList.Count == 0) { break; } Thread.Sleep(100); } sw.Stop(); log.Info("Total jobs............................." + totList.Count); log.Info("Allocated count :" + allocatedCount); log.Info("Total Time" + sw.Elapsed); log.Info("All Done........."); }