static async Task MainAsync() { GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["DBNameURL"]; conn = await getSecretCmdlet.GetSecretAsync(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerAcctNameURL"]; SharePointClassLibrary.Configuration.UserName = await getSecretCmdlet.GetSecretAsync(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerAcctPWDURL"]; SharePointClassLibrary.Configuration.PassWord = await getSecretCmdlet.GetSecretAsync(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerSPAzAccountKey"]; Configuration.azureAccountKey = await getSecretCmdlet.GetSecretAsync(); LogEvents("Information", "YM_SharepointUpload", "SP Upload Process started"); using (YETIDBEntities yeticontext = new YETIDBEntities()) { yeticontext.Yammer_GetYearForSharepoint(Environment.MachineName); } if (string.IsNullOrEmpty(Year)) { Environment.Exit(0); } Configuration.targetDocLib = Configuration.targetDocLib + Year; LogEvents("Information", "YM_SharepointUpload", "Resubmit Process started"); ResubmitFailedBatches(); LogEvents("Information", "YM_SharepointUpload", "Resubmit Process completed"); LogEvents("Information", "YM_SharepointUpload", "Submit New Process started"); SubmitNewBatchs(); LogEvents("Information", "YM_SharepointUpload", "Submit New Process completed"); LogEvents("Information", "YM_SharepointUpload", "Check Job status Process started"); CheckRunningJobs(); LogEvents("Information", "YM_SharepointUpload", "Check Job status Process completed"); }
static async Task MainAsync() { GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet(); //LogEvents("StartDownload", "Start"); //PafHelper obj = new PafHelper(); // to get all the config from paf getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["RobocopyOneFileCommandTemplate"]; RobocopyOneFileCommandTemplate = await getSecretCmdlet.GetSecretAsync(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["DurationInHrs"]; DurationInHrs = await getSecretCmdlet.GetSecretAsync(); using (YETIDBEntities yeticontext = new YETIDBEntities()) { Year = yeticontext.Yammer_YearDownloadRequest(Environment.MachineName).ToString(); } if (!string.IsNullOrEmpty(Year)) { getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["dirPath"]; dirPath = await getSecretCmdlet.GetSecretAsync(); dirPath = dirPath + "\\" + Year; await StoppedRequests(); await NewRequests(); Year = string.Empty; } }
static async Task MainAsync() { //Declare list variables List <ThreadTobeDeleted> ListThreadsTobeDeleted = new List <ThreadTobeDeleted>(); List <ThreadTobeRetained> ListThreadsTobeRetained = new List <ThreadTobeRetained>(); GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["DBNameURL"]; //await getSecretCmdlet.GetSecretAsync(); //SharePointClassLibrary.Configuration.UserName = PafHelper.YammerConfiguration.serviceAccountName; //SharePointClassLibrary.Configuration.PassWord = PafHelper.YammerConfiguration.serviceAccountPass; getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerAcctNameURL"]; SharePointClassLibrary.Configuration.UserName = await getSecretCmdlet.GetSecretAsync(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerAcctPWDURL"]; SharePointClassLibrary.Configuration.PassWord = await getSecretCmdlet.GetSecretAsync(); List <SharePointClassLibrary.SPFileInfo> ListSPFiles = new List <SharePointClassLibrary.SPFileInfo>(); List <string> filepaths = new List <string>(); //Get list of top level folders List <string> ToplevelFolderlist = new List <string>(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["DBNameURL"]; string conn = await getSecretCmdlet.GetSecretAsync(); //Fetch list of threads to be deleted from Sharepoint from DB. // SharePointClassLibrary.AzureSharePointHelper.GetSubFolders("DevDayYammer2017", "Groups3Aug", "", 2017, out ListSPFiles); //Fetch list of Top level Directories of the doclib and insert records in to DataSet ds = new DataSet(); string DocLibName = string.Empty; string DocLibProcessStage = string.Empty; using (SqlConnection con = new SqlConnection(conn)) { con.Open(); SqlCommand cmd = new SqlCommand(); cmd.Connection = con; cmd.CommandTimeout = 0; cmd.CommandType = CommandType.StoredProcedure; cmd.CommandText = "DupThreads_FetchThreadstobeDeleted"; SqlDataAdapter da = new SqlDataAdapter(cmd); da.Fill(ds); con.Close(); } if (ds.Tables.Count > 0) { DataTable DThreads = new DataTable(); //Access threads data to be deleted. DThreads = ds.Tables[0]; var ListThreadsTobeDeleted1 = DThreads.AsEnumerable().ToList(); DThreads = ds.Tables[1]; var ListThreadsTobeRetained2 = DThreads.AsEnumerable().ToList(); string message = ""; int CountofDeletedThreads = 0; DataTable DeletedThreads = new DataTable(); DeletedThreads.Columns.Add("DocLibName", System.Type.GetType("System.String")); DeletedThreads.Columns.Add("ThreadID_FileName", System.Type.GetType("System.String")); DeletedThreads.Columns.Add("ThreadID_Size", System.Type.GetType("System.Int64")); DeletedThreads.Columns.Add("ThreadID_Filepath", System.Type.GetType("System.String")); foreach (DataRow ThreadD in ListThreadsTobeDeleted1) { try { var ThreadR = ListThreadsTobeRetained2.Where(r => r.Field <string>("ThreadID_Filename") == ThreadD["threadID_Filename"].ToString()).SingleOrDefault(); if ( (Convert.ToInt64(ThreadR["ThreadID_Size"].ToString()) > Convert.ToInt64(ThreadD["ThreadID_Size"].ToString())) && (Convert.ToDateTime(ThreadR["ThreadID_CreatedDate"].ToString()) > Convert.ToDateTime(ThreadD["ThreadID_CreatedDate"].ToString())) && SharePointClassLibrary.AzureSharePointHelper.CheckIfFileExists(ThreadR["ThreadID_Path"].ToString(), ThreadR["DocLibname"].ToString(), ThreadR["ThreadID_FileName"].ToString(), Convert.ToInt64(ThreadR["ThreadID_Size"].ToString())) && (ThreadR["ThreadID_Filename"].ToString() == ThreadD["ThreadID_Filename"].ToString()) ) { //Can delete the Thread bool ThreadDeleted = SharePointClassLibrary.AzureSharePointHelper.FileExistAndDelete(ThreadD["ThreadID_Path"].ToString(), ThreadD["DocLibname"].ToString(), ThreadD["ThreadID_Filename"].ToString()); // Update Database to indicate that the duplicate thread is deleted from Sharepoint if (ThreadDeleted) { CountofDeletedThreads = CountofDeletedThreads + 1; DeletedThreads.Rows.Add(ThreadD["DocLibName"].ToString(), ThreadD["ThreadID_Filename"].ToString(), Convert.ToInt64(ThreadD["ThreadID_Size"].ToString()), ThreadD["ThreadID_Path"].ToString()); if (CountofDeletedThreads == 1000) { using (SqlConnection con = new SqlConnection(conn)) { con.Open(); SqlCommand cmd = new SqlCommand(); cmd.Connection = con; cmd.CommandTimeout = 0; cmd.CommandType = CommandType.StoredProcedure; cmd.CommandText = "DupThreads_UpdateDeletedThreadStatus"; cmd.Parameters.AddWithValue("DeletedThreads", DeletedThreads); //cmd.Parameters.AddWithValue("ThreadID_Path", ThreadD["ThreadID_Path"].ToString()); //cmd.Parameters.AddWithValue("ThreadID_Size", Convert.ToInt64(ThreadD["ThreadID_Size"])); //cmd.Parameters.AddWithValue("DocLibName", ThreadD["DocLibName"].ToString()); cmd.ExecuteNonQuery(); con.Close(); CountofDeletedThreads = 0; DeletedThreads.Clear(); } } } } } catch (Exception e) { message = "Exception in calling linq query"; Console.WriteLine(message.ToString()); } } } // Console.ReadLine(); }
static async Task MainAsync() { //PafHelper obj = new PafHelper(); // to get all the config from paf GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerCmpPath"]; YammerCmpPath = await getSecretCmdlet.GetSecretAsync(); using (YETIDBEntities yeticontext = new YETIDBEntities()) { Year = yeticontext.Yammer_YearCompressingRequest(Environment.MachineName, RangeInDays).ToString(); } LogEvents("Information", "FileCompress", "Got year -" + Year + " for compression"); if (!string.IsNullOrEmpty(Year)) { using (YETIDBEntities yeticontext = new YETIDBEntities()) { List <Yammer_Compress_GetProcessingCount_Result> Result = yeticontext.Yammer_Compress_GetProcessingCount(Convert.ToInt32(Year), Convert.ToInt32(ConfigurationManager.AppSettings["RangeInMonths"]), Convert.ToInt32(ConfigurationManager.AppSettings["RangeInDays"])).ToList(); if (Result.Count > 0) { if (string.IsNullOrEmpty(Convert.ToString(Result[0].StartDate))) { LogEvents("Information", "FileCompress", "No data available compress"); Environment.Exit(0); } uploadStartDate = Result[0].StartDate.ToString(); uploadEndDate = Result[0].EndDate.ToString(); LogEvents("Information", "FileCompress", "All generated"); if (ZipFolders()) //Compress folders { //int folderCount = 1; string folderName = string.Empty; getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["SPdirPath"]; string SPDir = await getSecretCmdlet.GetSecretAsync(); SPDir = Path.Combine(SPDir, Year); if (!Directory.Exists(SPDir)) { Directory.CreateDirectory(SPDir); } //else //{ //folderCount = Directory.GetDirectories(SPDir, "*.*", SearchOption.TopDirectoryOnly).Length + 1; folderName = SharePointClassLibrary.CommonHelper.FrameFolderName(uploadStartDate, uploadEndDate, true); //} string SPNewDir = Path.Combine(SPDir, folderName.ToString()); if (!Directory.Exists(SPNewDir)) { Directory.CreateDirectory(SPNewDir); } using (System.Diagnostics.Process process = new System.Diagnostics.Process()) { // Fetch RobocopyMovecommandTemplate and Yammer Compression patch secrets from Keyvault getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["RobocopyMoveCommandTemplate"]; string RobocopyMoveCommandTemplate = await getSecretCmdlet.GetSecretAsync(); process.StartInfo.UseShellExecute = false; process.StartInfo.CreateNoWindow = true; process.StartInfo.RedirectStandardOutput = false; process.StartInfo.FileName = "ROBOCOPY"; process.StartInfo.Arguments = string.Format(System.Globalization.CultureInfo.InvariantCulture, RobocopyMoveCommandTemplate, Path.Combine(YammerCmpPath, Year), SPNewDir); process.Start(); process.WaitForExit(2400 * 60 * 1000); if (process.HasExited) { if (process.ExitCode > 8) { LogEvents("Error", "RoboCopyLog", " Compressed folders not moved to SharePoint Directory"); Environment.Exit(0); } } } UpdateStatus(uploadStartDate, uploadEndDate, Year); LoadSPDirectoryMapping(uploadStartDate, uploadEndDate, SPNewDir); UpdateYearStatus(Year); } } Year = string.Empty; } } LogEvents("Information", "FileCompress", "Compression method completed"); }
/// <summary> /// call Yammer export API to download date during the period we configured /// </summary> /// <param name="start"></param> /// <param name="end"></param> public static async Task ExportAPICall(string start, string end) { string startDate = start; string endDate = end; //string token = PafHelper.YammerConfiguration.exportApiToken; //string token = YammerEncryption.Decrypt(HttpUtility.UrlDecode(PafHelper.YammerConfiguration.exportApiToken.Replace("%%", "%"))); GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerExportTokenURL"]; string token = await getSecretCmdlet.GetSecretAsync(); DateTime tempEndate = Convert.ToDateTime(endDate); string URL = "https://export.yammer.com/api/v1/export?since=" + startDate + "&until=" + endDate + "&access_token=" + token; //string URL = "https://export.yammer.com/api/v1/export?since=" + startDate + "&until=" + endDate + "&model=Message&include=csv&access_token=" + token; try { if (timesTried > Convert.ToInt32(ConfigurationManager.AppSettings["MaxAllowableRetries"])) { UpdateStatus(startDate, endDate, "Failed", "DownloadFailed", timesTried); SplitDownloadPeriod(startDate, endDate); return; } else { bool deleteCorrupted = false; UpdateStatus(startDate, endDate, "In Progress", "", timesTried); if (!Directory.Exists(dirPath + "\\" + tempEndate.Month)) { Directory.CreateDirectory(dirPath + "\\" + tempEndate.Month); } Thread.Sleep(5000); if (!Directory.Exists(AppDomain.CurrentDomain.BaseDirectory + @"DownloadFile")) { Directory.CreateDirectory(AppDomain.CurrentDomain.BaseDirectory + @"DownloadFile"); } string tempPath = AppDomain.CurrentDomain.BaseDirectory + @"DownloadFile\\export-" + endDate.Replace(':', '-') + ".zip"; using (WebClient wc = new WebClient()) { wc.Proxy = null; wc.DownloadFile(URL, tempPath); wc.Dispose(); } Thread.Sleep(5000); if (File.Exists(tempPath)) { if (!Directory.Exists(ConfigurationManager.AppSettings["logpath"] + "\\RobocopyLogs")) { Directory.CreateDirectory(ConfigurationManager.AppSettings["logpath"] + "\\RobocopyLogs"); } getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["LogPath"]; string logPath = await getSecretCmdlet.GetSecretAsync(); string logName = string.Format(CultureInfo.InvariantCulture, "{0}\\{1}_{2}_{3}.log", logPath + "\\RobocopyLogs", "FileCopyProcessing", Year + "_" + tempEndate.Month, DateTime.Now.Ticks); using (System.Diagnostics.Process process = new System.Diagnostics.Process()) { process.StartInfo.UseShellExecute = false; process.StartInfo.CreateNoWindow = true; process.StartInfo.RedirectStandardOutput = false; process.StartInfo.FileName = "ROBOCOPY"; process.StartInfo.Arguments = string.Format(CultureInfo.InvariantCulture, RobocopyOneFileCommandTemplate, AppDomain.CurrentDomain.BaseDirectory + @"DownloadFile", dirPath + "\\" + tempEndate.Month, "export-" + endDate.Replace(':', '-') + ".zip", logName); process.Start(); process.WaitForExit(2400 * 60 * 1000); if (process.HasExited) { if (process.ExitCode <= 8) { File.Delete(logName); } else { LogEvents("RoboCopyLog", "check this path: " + logName); Environment.Exit(0); } } } Thread.Sleep(5000); if (File.Exists(dirPath + "\\" + tempEndate.Month + "\\export-" + endDate.Replace(':', '-') + ".zip")) { File.Delete(tempPath); } } Thread.Sleep(5000); using (ZipArchive archive = ZipFile.OpenRead(dirPath + "\\" + tempEndate.Month + "\\export-" + endDate.Replace(':', '-') + ".zip")) { foreach (ZipArchiveEntry entry in archive.Entries) { try { var stream = entry.Open(); //If files can be read, then zip file is not corrupted } catch (Exception ex) { LogEvents("export - " + endDate.Replace(':', '-') + ".zip", ex.ToString()); UpdateStatus(startDate, endDate, "Stopped", "DownloadFailed", timesTried + 1); deleteCorrupted = true; } } } if (deleteCorrupted) { File.Delete(dirPath + "\\" + tempEndate.Month + "\\export-" + endDate.Replace(':', '-') + ".zip"); deleteCorrupted = false; LogEvents("export - " + endDate.Replace(':', '-') + ".zip", " CorruptedRemoved " + dirPath + "\\" + tempEndate.Month + "\\export-" + endDate.Replace(':', '-') + ".zip"); return; } UpdateStatus(startDate, endDate, "Completed", "DownloadCompleted", timesTried + 1); } } catch (Exception ex) when(ex is WebException) { if (ex.Message == "The remote server returned an error: (429)." || ex.Message == "The operation has timed out") { UpdateStatus(startDate, endDate, "Stopped", "DownloadFailed", timesTried); } else { UpdateStatus(startDate, endDate, "Stopped", "DownloadFailed", timesTried + 1); } LogEvents("export - " + endDate.Replace(':', '-') + ".zip", ex.ToString()); if (File.Exists(dirPath + "\\" + tempEndate.Month + "\\export-" + endDate.Replace(':', '-') + ".zip")) { File.Delete(dirPath + "\\" + tempEndate.Month + "\\export-" + endDate.Replace(':', '-') + ".zip"); } } }
static async Task MainAsync() { GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet(); List <SharePointClassLibrary.SPFileInfo> ListSPFiles = new List <SharePointClassLibrary.SPFileInfo>(); List <string> filepaths = new List <string>(); //Get list of top level folders List <string> ToplevelFolderlist = new List <string>(); getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["DBNameURL"]; conn = await getSecretCmdlet.GetSecretAsync(); //Fetch list of Top level Directories of the doclib and insert records in to DataSet ds = new DataSet(); string DocLibName = string.Empty; string DocLibProcessStage = string.Empty; // Check if any deduplication has to start string Year = string.Empty; using (YETIDBEntities yeticontext = new YETIDBEntities()) { Year = yeticontext.Yammer_GetYearForDedup(Environment.MachineName).ToString(); } if (string.IsNullOrEmpty(Year)) { Environment.Exit(0); } List <DupThreads_FetchDocLibStatus_Result> List = new List <DupThreads_FetchDocLibStatus_Result>(); using (YETIDBEntities yeticontext = new YETIDBEntities()) { List = yeticontext.DupThreads_FetchDocLibStatus().ToList(); } if (List.Count > 0) { foreach (DupThreads_FetchDocLibStatus_Result Listitem in List) { DocLibName = Listitem.DocLibName.ToString(); DocLibProcessStage = Listitem.ProcessStage.ToString(); } //call method to get list of TopLevel Folders ToplevelFolderlist = SharePointClassLibrary.AzureSharePointHelper.GetFolderListFromSharePoint(DocLibName); DataTable TopLevelFolders = new DataTable(); TopLevelFolders.Columns.Add("TopLevelFolder", System.Type.GetType("System.String")); foreach (string TopLevelFolder in ToplevelFolderlist) { TopLevelFolders.Rows.Add(TopLevelFolder); } // insert the top level folder names in to a different table and update the status as "TopFoldersFetched" in DocLibStatus table using (SqlConnection con = new SqlConnection(conn)) { con.Open(); SqlCommand cmd = new SqlCommand(); cmd.Connection = con; cmd.CommandTimeout = 0; cmd.CommandType = CommandType.StoredProcedure; cmd.CommandText = "DupThreads_InsertTopLevelSubfolders"; cmd.Parameters.AddWithValue("TopLevelSubfolders", TopLevelFolders); cmd.Parameters.AddWithValue("DocLibName", DocLibName); cmd.ExecuteNonQuery(); con.Close(); } if (DocLibProcessStage == "TopLevelFoldersInserted") { List <string> TopFolders = new List <string>(); using (YETIDBEntities yeticontext = new YETIDBEntities()) { TopFolders = yeticontext.DupThreads_FetchTopLevelFolders(DocLibName).ToList(); } if (TopFolders.Count > 0) { //Read TopLevelFolders and passthrough each of them to get list of threads foreach (string SubFolderName in TopFolders) { //fetch threads info SharePointClassLibrary.AzureSharePointHelper.GetSubFolders(DocLibName, SubFolderName, "", 2017, out ListSPFiles); DataTable TableSPFiles = new DataTable(); //TableSPFiles.Columns.Add("ThreadID", System.Type.GetType("System.String")); TableSPFiles.Columns.Add("DocLibName", System.Type.GetType("System.String")); TableSPFiles.Columns.Add("TopLevelSubFolder", System.Type.GetType("System.String")); TableSPFiles.Columns.Add("ThreadID_FileName", System.Type.GetType("System.String")); TableSPFiles.Columns.Add("ThreadID_Size", System.Type.GetType("System.Int64")); TableSPFiles.Columns.Add("ThreadID_CreatedDate", System.Type.GetType("System.DateTime")); TableSPFiles.Columns.Add("ThreadID_Path", System.Type.GetType("System.String")); //Insert Threadinfo to DB foreach (SharePointClassLibrary.SPFileInfo SPfile in ListSPFiles) { TableSPFiles.Rows.Add(DocLibName, SubFolderName, SPfile.Filename, SPfile.filesize, SPfile.filecreateddate, SPfile.Filepath); } using (YETIDBEntities yeticontext = new YETIDBEntities()) { yeticontext.DupThreads_FetchTopLevelFolders(DocLibName).ToList(); } using (SqlConnection con = new SqlConnection(conn)) { con.Open(); SqlCommand cmd = new SqlCommand(); cmd.Connection = con; cmd.CommandTimeout = 0; cmd.CommandType = CommandType.StoredProcedure; cmd.CommandText = "DupThreads_InsertThreadInfo"; cmd.Parameters.AddWithValue("SPFileInfo", TableSPFiles); cmd.ExecuteNonQuery(); con.Close(); } } } } DataSet dsThreadsTobeDeleted = new DataSet(); List <DupThreads_FetchThreadstobeDeleted_Result> ListThreadstoDelete = new List <DupThreads_FetchThreadstobeDeleted_Result>(); using (YETIDBEntities yeticontext = new YETIDBEntities()) { ListThreadstoDelete = yeticontext.DupThreads_FetchThreadstobeDeleted(Environment.MachineName).ToList(); } if (ListThreadstoDelete.Count > 0) { DataTable DThreads = new DataTable(); //Access threads data to be deleted. DThreads = ds.Tables[0]; var ListThreadsTobeDeleted1 = DThreads.AsEnumerable().ToList(); string message = ""; foreach (DupThreads_FetchThreadstobeDeleted_Result ThreadToDelete in ListThreadstoDelete) { try { //Can delete the Thread bool ThreadDeleted = SharePointClassLibrary.AzureSharePointHelper.FileExistAndDelete(ThreadToDelete.ThreadID_Path.ToString(), ThreadToDelete.DocLibName.ToString(), ThreadToDelete.ThreadID_Filename.ToString()); // Update Database to indicate that the duplicate thread is deleted from Sharepoint if (ThreadDeleted) { using (YETIDBEntities yeticontext = new YETIDBEntities()) { yeticontext.DupThreads_UpdateDeletedThreadStatus(ThreadToDelete.ThreadID_Filename.ToString(), ThreadToDelete.ThreadID_Path.ToString(), ThreadToDelete.ThreadID_Size, ThreadToDelete.DocLibName); } } //} } catch (Exception e) { message = "Exception in calling linq query"; Console.WriteLine(message.ToString()); } } int DedupFinished = 0; using (YETIDBEntities yeticontext = new YETIDBEntities()) { DedupFinished = yeticontext.DupThreads_CheckDeleteThreadStatus(DocLibName.ToString()); } if (DedupFinished == 1) { UpdateYearStatus(Year); } } else { UpdateYearStatus(Year); } } }