static void UpdateStatus(string uploadStartDate, string uploadEndDate, string Year)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Common_UpdateStatus(uploadStartDate, uploadEndDate, "FilesCompressed", "CompressionCompleted", null, null);
}
}
/// <summary>
/// Log the job information and error
/// </summary>
/// <param name="fileName"></param>
/// <param name="errorDescription"></param>
public static void LogEvents(string fileName, string errorDescription)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Common_LogEvent("YM_Download", "Error", fileName, errorDescription, Environment.MachineName);
}
}
private static void LoadSPDirectoryMapping(string uploadStartDate, string uploadEndDate, string folderPath)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_LoadSPDirectoryMapping(uploadStartDate, uploadEndDate, folderPath, Directory.EnumerateFiles(folderPath, "*.zip", SearchOption.AllDirectories).Count(), Environment.MachineName);
}
}
private static void UpdateYearStatus(string Year)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Update_YearStatus(Year, 0, 0, 1, 0);
}
}
static void UpdateYearStatus(string Year)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Update_DownloadCompletion(Year);
}
}
/// <summary>
/// continue to download the data in status 'Stopped' or 'In progress' if time tried is less than configured.
/// </summary>
private static async Task StoppedRequests()
{
string startDate = string.Empty;
string endDate = string.Empty;
try
{
DataSet ds = new DataSet();
var isoDateTimeFormat = CultureInfo.InvariantCulture.DateTimeFormat;
string newStartDate = string.Empty;
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
List <Yammer_Download_GetTimesTried_Result> Result = yeticontext.Yammer_Download_GetTimesTried(Year, Environment.MachineName).ToList();
foreach (Yammer_Download_GetTimesTried_Result Listitem in Result)
{
startDate = Convert.ToDateTime(Listitem.StartDate).ToString(isoDateTimeFormat.SortableDateTimePattern);
endDate = Convert.ToDateTime(Listitem.EndDate).ToString(isoDateTimeFormat.SortableDateTimePattern);
timesTried = Convert.ToInt32(Listitem.TimesTried);
await ExportAPICall(startDate, endDate);
}
}
}
catch (Exception ex) when(ex is WebException)
{
LogEvents("export - " + endDate.Replace(':', '-') + ".zip", ex.ToString());
}
}
public static void LogEvents(string evenType, string fileName, string errorDescription)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Common_LogEvent("YM_Compressing", evenType.ToString(), fileName, errorDescription, Environment.MachineName);
}
}
private static void UpdateStatus(string startDateTime, string endDateTime, string status, string stage, int timesTried, bool toMove = false)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Common_UpdateStatus(startDateTime, endDateTime, status, stage, timesTried, toMove);
}
}
private static void UpdateSPMapping(int BatchNo)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_UpdateSPMapping(BatchNo, Convert.ToInt32(Year), Convert.ToDateTime(uploadStartDate), Convert.ToDateTime(uploadEndDate), Environment.MachineName);
}
}
static void updateStatus(string JobId, string status)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_UpdateSharepointStatus(JobId, status);
}
}
private static void UpdateBatchStatus(int batchNo, string status, int count = 0)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_UpdateBatchStatus(batchNo, status, Convert.ToInt32(ConfigurationManager.AppSettings["TimesTried"]), count);
}
}
/// <summary>
/// if one period file is downloaded failed in configured times,
/// the time duration will be split to 1 hr and the day’s data will be in 24 parts
/// </summary>
/// <param name="startDate"></param>
/// <param name="endDate"></param>
public static void SplitDownloadPeriod(string startDate, string endDate)
{
string newEndDate = string.Empty;
var isoDateTimeFormat = CultureInfo.InvariantCulture.DateTimeFormat;
DateTime newstartdate = Convert.ToDateTime(startDate);
TimeSpan diff = Convert.ToDateTime(endDate).AddSeconds(1) - Convert.ToDateTime(startDate);
if (diff.Minutes == 5)
{
return;
}
else if (diff.Hours != 1)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Download_DeleteExportDetails(startDate, endDate);
for (int i = 0; i < Convert.ToInt32(DurationInHrs); i++)
{
DateTime tempEndDate = Convert.ToDateTime(newstartdate).AddHours(1).AddMilliseconds(-1);
newEndDate = tempEndDate.ToString(isoDateTimeFormat.SortableDateTimePattern);
yeticontext.Yammer_Download_NewExportDetails(newstartdate.ToString(), newEndDate, "Stopped", Environment.MachineName);
newstartdate = Convert.ToDateTime(newstartdate).AddHours(1);
}
}
}
else if (diff.Hours == 1)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Download_DeleteExportDetails(startDate, endDate);
for (int i = 0; i < 12; i++) //splitting 1 hr data for 5 min so 12 parts
{
DateTime tempEndDate = Convert.ToDateTime(newstartdate).AddMinutes(5).AddMilliseconds(-1);
newEndDate = tempEndDate.ToString(isoDateTimeFormat.SortableDateTimePattern);
yeticontext.Yammer_Download_NewExportDetails(newstartdate.ToString(), newEndDate, "Stopped", Environment.MachineName);
newstartdate = Convert.ToDateTime(newstartdate).AddMinutes(5);
}
}
}
}
private static bool CheckAnyBatchInProgress(bool isNew)
{
try
{
int returnValue = 0;
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
returnValue = Convert.ToInt32(yeticontext.Yammer_CheckBatchInProgress(Convert.ToInt32(Year), Environment.MachineName));
}
if (isNew)
{
if (returnValue >= Configuration.totalNewBatchCount)
{
return(true);
}
else
{
Configuration.totalNewBatchCount = Configuration.totalNewBatchCount - returnValue;
return(false);
}
}
else
{
if (returnValue == 0)
{
return(true);
}
if (returnValue >= Configuration.totalResubmitBatchCount)
{
return(true);
}
else
{
Configuration.totalResubmitBatchCount = Configuration.totalResubmitBatchCount - returnValue;
return(false);
}
}
}
catch (Exception ex)
{
LogEvents("Error", "YM_SharepointUpload", ex.ToString());
Environment.Exit(0);
return(true);
}
}
static async Task MainAsync()
{
GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet();
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["DBNameURL"];
conn = await getSecretCmdlet.GetSecretAsync();
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerAcctNameURL"];
SharePointClassLibrary.Configuration.UserName = await getSecretCmdlet.GetSecretAsync();
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerAcctPWDURL"];
SharePointClassLibrary.Configuration.PassWord = await getSecretCmdlet.GetSecretAsync();
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerSPAzAccountKey"];
Configuration.azureAccountKey = await getSecretCmdlet.GetSecretAsync();
LogEvents("Information", "YM_SharepointUpload", "SP Upload Process started");
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_GetYearForSharepoint(Environment.MachineName);
}
if (string.IsNullOrEmpty(Year))
{
Environment.Exit(0);
}
Configuration.targetDocLib = Configuration.targetDocLib + Year;
LogEvents("Information", "YM_SharepointUpload", "Resubmit Process started");
ResubmitFailedBatches();
LogEvents("Information", "YM_SharepointUpload", "Resubmit Process completed");
LogEvents("Information", "YM_SharepointUpload", "Submit New Process started");
SubmitNewBatchs();
LogEvents("Information", "YM_SharepointUpload", "Submit New Process completed");
LogEvents("Information", "YM_SharepointUpload", "Check Job status Process started");
CheckRunningJobs();
LogEvents("Information", "YM_SharepointUpload", "Check Job status Process completed");
}
static async Task MainAsync()
{
GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet();
//LogEvents("StartDownload", "Start");
//PafHelper obj = new PafHelper(); // to get all the config from paf
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["RobocopyOneFileCommandTemplate"];
RobocopyOneFileCommandTemplate = await getSecretCmdlet.GetSecretAsync();
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["DurationInHrs"];
DurationInHrs = await getSecretCmdlet.GetSecretAsync();
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
Year = yeticontext.Yammer_YearDownloadRequest(Environment.MachineName).ToString();
}
if (!string.IsNullOrEmpty(Year))
{
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["dirPath"];
dirPath = await getSecretCmdlet.GetSecretAsync();
dirPath = dirPath + "\\" + Year;
await StoppedRequests();
await NewRequests();
Year = string.Empty;
}
}
private static async Task SetDuration()
{
string startDate = string.Empty;
string endDate = string.Empty;
int inProgressCount = 0;
int countDownCount = 0;
try
{
var isoDateTimeFormat = CultureInfo.InvariantCulture.DateTimeFormat;
string newStartDate = string.Empty;
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
newStartDate = yeticontext.Yammer_Download_GetLastEnddate(Year).ToString();
inProgressCount = Convert.ToInt32(yeticontext.Yammer_Download_GetInProgressCount(Year, Environment.MachineName));
countDownCount = 5 - inProgressCount;
if (inProgressCount >= 5)
{
return;
}
if (!string.IsNullOrEmpty(newStartDate))
{
DateTime startdate = Convert.ToDateTime(newStartDate);
if (startdate.Year.ToString() == Year)
{
newStartDate = startdate.ToString(isoDateTimeFormat.SortableDateTimePattern);
}
else
{
UpdateYearStatus(Year);
return;
}
}
else
{
newStartDate = Year + "-01-01T00:00:00";
}
}
var countdownEvent = new CountdownEvent(countDownCount);
for (int i = 0; i < countDownCount; i++)
{
startDate = newStartDate;
DateTime tempEndDate = Convert.ToDateTime(startDate).AddHours(Convert.ToInt32(DurationInHrs)).AddMilliseconds(-1);
endDate = tempEndDate.ToString(isoDateTimeFormat.SortableDateTimePattern);
//if (tempEndDate < DateTime.Now.AddHours(-DateTime.Now.Hour).AddMinutes(-DateTime.Now.Minute).AddSeconds(-DateTime.Now.Second))
if (tempEndDate < DateTime.Now)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_Download_NewExportDetails(startDate, endDate, "Not Started", Environment.MachineName);
}
DateTime startDateD = Convert.ToDateTime(endDate).AddSeconds(1);
if (startDateD.Year.ToString() == Year)
{
newStartDate = startDateD.ToString(isoDateTimeFormat.SortableDateTimePattern);
}
else
{
i = countDownCount;
}
//new Thread(delegate ()
{
await ExportAPICall(startDate, endDate);
countdownEvent.Signal();
}//).Start();
Thread.Sleep(10000); //Giving 10 sec gap to start the download for the assigned start & end date
}
else
{
return;
}
}
countdownEvent.Wait();
countdownEvent.Dispose();
}
catch (Exception ex) when(ex is WebException)
{
LogEvents("export - " + endDate.Replace(':', '-') + ".zip", ex.ToString());
}
}
private static void SubmitNewBatchs()
{
try
{
List <Yammer_GetSPDirectoryForUpload_Result> SPDirectories = new List <Yammer_GetSPDirectoryForUpload_Result>();
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
SPDirectories = yeticontext.Yammer_GetSPDirectoryForUpload(Convert.ToInt32(Year)).ToList();
}
if (SPDirectories.Count > 0)
{
var isoDateTimeFormat = CultureInfo.InvariantCulture.DateTimeFormat;
SPDirPath = SPDirectories[0].FolderPath;
ThreadCount = Convert.ToInt64(SPDirectories[0].ThreadCount);
uploadStartDate = Convert.ToDateTime(SPDirectories[0].UploadStartDate).ToString(isoDateTimeFormat.SortableDateTimePattern); //);
uploadEndDate = Convert.ToDateTime(SPDirectories[SPDirectories.Count - 1].UploadEndDate).ToString(isoDateTimeFormat.SortableDateTimePattern);
Configuration.SPMappingId = Convert.ToInt32(SPDirectories[SPDirectories.Count - 1].Id);
}
if (!string.IsNullOrEmpty(uploadEndDate))
{
LogEvents("Information", "YM_SharepointUpload", "Splitting folders");
SplitFolders();
LogEvents("Information", "YM_SharepointUpload", "ArrangeGroupsForBatches");
ArrangeGroupsForBatches();
LogEvents("Information", "YM_SharepointUpload", "SplitFoldersForBatches");
SplitFoldersForBatches();
LogEvents("Information", "YM_SharepointUpload", "Year " + Year + " is ready to upload to sharepoint");
DataSet JobIdDs = new DataSet();
LogEvents("Information", "YM_SharepointUpload", "Checking if any Job is in progress");
if (!CheckAnyBatchInProgress(true))
{
LogEvents("Information", "YM_SharepointUpload", "Submitting batch");
bool islibraryCreated = false;
try
{
islibraryCreated = SharePointClassLibrary.AzureSharePointHelper.CreateDocumentLibrary(Configuration.targetDocLib);
}
catch (Exception ex)
{
if (ex.Message == "A list, survey, discussion board, or document library with the specified title already exists in this Web site. Please choose another title.")
{
islibraryCreated = true;
}
else
{
islibraryCreated = false;
LogEvents("Error", "YM_SharepointUpload", "LibraryCreation Failed. " + ex.ToString());
}
}
if (islibraryCreated)
{
SubmitBatchToExchange(Configuration.targetDocLib, true);
}
}
}
}
catch (Exception ex)
{
LogEvents("Error", "SubmitNew", ex.ToString());
}
}
static bool ExecuteBatch(PSCredential cred, string sourceFilePath, string groupFolderName, string sourcePackage, string targetPackage, int BatchNo, int ItemCount)
{
try
{
LogEvents("Information", "YM_SharepointUpload", "New bacth creation");
List <Command> cmds = new List <Command>();
List <ErrorRecord> errors = new List <ErrorRecord>();
PSCommand cmd = new PSCommand();
cmd.AddCommand("New-SPOMigrationPackage");
cmd.AddParameter("-SourceFilesPath", sourceFilePath);
cmd.AddParameter("-OutputPackagePath", sourcePackage);
cmd.AddParameter("-IncludeFileSharePermissions");
cmd.AddParameter("-TargetWebUrl", Configuration.Url);
cmd.AddParameter("-TargetDocumentLibraryPath", Configuration.targetDocLib);
cmd.AddParameter("-TargetDocumentLibrarySubFolderPath", groupFolderName);
Results result = RunComand(cmd);
var directory = new DirectoryInfo(sourcePackage);
var logFile = directory.GetFiles("CreateMigrationPackage*.log")
.OrderByDescending(f => f.LastWriteTime)
.First();
StreamReader reader = System.IO.File.OpenText(logFile.FullName);
string line;
List <string> errorList = new List <string>();
while ((line = reader.ReadLine()) != null)
{
string[] items = line.Split('\t');
if (items.Count() > 1)
{
if (items[1].Contains("Error"))
{
errorList.Add(items[2]);
}
}
}
if (errorList.Count > 0)
{
LogEvents("Error", logFile.Name, "Issue with upload check the log file @: " + logFile.FullName);
return(false);
}
LogEvents("Information", "SharePoint", "Package Created");
PSCommand cmd1 = new PSCommand();
cmd1.AddCommand("ConvertTo-SPOMigrationTargetedPackage");
cmd1.AddParameter("-SourceFilesPath", sourceFilePath);
cmd1.AddParameter("-SourcePackagePath", sourcePackage);
cmd1.AddParameter("-OutputPackagePath", targetPackage);
cmd1.AddParameter("-TargetWebUrl", Configuration.Url);
cmd1.AddParameter("-TargetDocumentLibraryPath", Configuration.targetDocLib);
cmd1.AddParameter("-TargetDocumentLibrarySubFolderPath", groupFolderName);
cmd1.AddParameter("-Credentials", cred);
result = RunComand(cmd1);
directory = new DirectoryInfo(targetPackage);
logFile = directory.GetFiles("ConvertMigrationPackage*.log")
.OrderByDescending(f => f.LastWriteTime)
.First();
reader = System.IO.File.OpenText(logFile.FullName);
line = string.Empty;
errorList = new List <string>();
while ((line = reader.ReadLine()) != null)
{
string[] items = line.Split('\t');
if (items.Count() > 1)
{
if (items[1].Contains("Error"))
{
errorList.Add(items[2]);
}
}
}
if (errorList.Count > 0)
{
LogEvents("Error", logFile.Name, "Issue with upload check the log file @: " + logFile.FullName);
return(false);
}
LogEvents("Information", "SharePoint", "Package Convertion Completed");
PSCommand cmd2 = new PSCommand();
cmd2.AddCommand("Set-SPOMigrationPackageAzureSource");
cmd2.AddParameter("-SourceFilesPath", sourceFilePath);
cmd2.AddParameter("-SourcePackagePath", targetPackage);
cmd2.AddParameter("-AccountName", Configuration.azureAccountName);
cmd2.AddParameter("-AccountKey", Configuration.azureAccountKey);
result = RunComand(cmd2);
errors = result.Errors;
directory = new DirectoryInfo(targetPackage);
logFile = directory.GetFiles("CopyMigrationPackage*.log")
.OrderByDescending(f => f.LastWriteTime)
.First();
reader = System.IO.File.OpenText(logFile.FullName);
line = string.Empty;
errorList = new List <string>();
while ((line = reader.ReadLine()) != null)
{
string[] items = line.Split('\t');
if (items.Count() > 1)
{
if (items[1].Contains("Error") && !items[2].Contains("file timed out (max time"))
{
errorList.Add(items[2]);
}
}
}
if (errorList.Count > 0)
{
LogEvents("Error", logFile.Name, "Issue with upload check the log file @: " + logFile.FullName);
return(false);
}
LogEvents("Information", "SharePoint", "Azure upload completed");
Collection <PSObject> ObjReturned = result.ObjReturned;
string FileContainerUri = string.Empty;
string PackageContainerUri = string.Empty;
string ReportingQueueUri = string.Empty;
dynamic obj = ObjReturned[0].BaseObject;
try
{
FileContainerUri = ((dynamic)obj).FileContainerUri.ToString();
PackageContainerUri = ((dynamic)obj).PackageContainerUri.ToString();
ReportingQueueUri = ((dynamic)obj).ReportingQueueUri.ToString();
}
catch (Exception ex)
{
return(false);
}
LogEvents("Information", "ContainerName", FileContainerUri);
if (errors.Count == 0)
{
cmds = new List <Command>();
PSCommand cmd3 = new PSCommand();
cmd3.AddCommand("Submit-SPOMigrationJob");
cmd3.AddParameter("-FileContainerUri", FileContainerUri.Trim());
cmd3.AddParameter("-PackageContainerUri", PackageContainerUri.Trim());
cmd3.AddParameter("-TargetWebUrl", Configuration.Url.Substring(0, Configuration.Url.Length - 1));
cmd3.AddParameter("-Credentials", cred);
result = RunComand(cmd3);
errors = result.Errors;
string configGUID = result.ObjReturned[0].ToString();
LogEvents("Information", "ContainerName", "Package submitted to SharePoint.");
if (errors.Count == 0)
{
string containerName = FileContainerUri.Split('?')[0];
containerName = containerName.Substring(containerName.LastIndexOf("/") + 1);
containerName = containerName.Substring(0, containerName.LastIndexOf("-"));
UpdateBatchStatus(BatchNo, "Processing");
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.Yammer_ContainerMapping(containerName, configGUID, Convert.ToInt32(Year), Configuration.targetDocLib, "Processing", "PackageSubmitted", Environment.MachineName, sourceFilePath, ItemCount, FileContainerUri, PackageContainerUri, ReportingQueueUri, BatchNo);
}
}
else
{
return(false);
}
return(true);
}
else
{
return(false);
}
}
catch (Exception ex)
{
LogEvents("Error", "SubmitBatchToExchange", ex.ToString());
return(false);
}
}
static async Task MainAsync()
{
//PafHelper obj = new PafHelper(); // to get all the config from paf
GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet();
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["YammerCmpPath"];
YammerCmpPath = await getSecretCmdlet.GetSecretAsync();
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
Year = yeticontext.Yammer_YearCompressingRequest(Environment.MachineName, RangeInDays).ToString();
}
LogEvents("Information", "FileCompress", "Got year -" + Year + " for compression");
if (!string.IsNullOrEmpty(Year))
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
List <Yammer_Compress_GetProcessingCount_Result> Result = yeticontext.Yammer_Compress_GetProcessingCount(Convert.ToInt32(Year), Convert.ToInt32(ConfigurationManager.AppSettings["RangeInMonths"]), Convert.ToInt32(ConfigurationManager.AppSettings["RangeInDays"])).ToList();
if (Result.Count > 0)
{
if (string.IsNullOrEmpty(Convert.ToString(Result[0].StartDate)))
{
LogEvents("Information", "FileCompress", "No data available compress");
Environment.Exit(0);
}
uploadStartDate = Result[0].StartDate.ToString();
uploadEndDate = Result[0].EndDate.ToString();
LogEvents("Information", "FileCompress", "All generated");
if (ZipFolders()) //Compress folders
{
//int folderCount = 1;
string folderName = string.Empty;
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["SPdirPath"];
string SPDir = await getSecretCmdlet.GetSecretAsync();
SPDir = Path.Combine(SPDir, Year);
if (!Directory.Exists(SPDir))
{
Directory.CreateDirectory(SPDir);
}
//else
//{
//folderCount = Directory.GetDirectories(SPDir, "*.*", SearchOption.TopDirectoryOnly).Length + 1;
folderName = SharePointClassLibrary.CommonHelper.FrameFolderName(uploadStartDate, uploadEndDate, true);
//}
string SPNewDir = Path.Combine(SPDir, folderName.ToString());
if (!Directory.Exists(SPNewDir))
{
Directory.CreateDirectory(SPNewDir);
}
using (System.Diagnostics.Process process = new System.Diagnostics.Process())
{
// Fetch RobocopyMovecommandTemplate and Yammer Compression patch secrets from Keyvault
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["RobocopyMoveCommandTemplate"];
string RobocopyMoveCommandTemplate = await getSecretCmdlet.GetSecretAsync();
process.StartInfo.UseShellExecute = false;
process.StartInfo.CreateNoWindow = true;
process.StartInfo.RedirectStandardOutput = false;
process.StartInfo.FileName = "ROBOCOPY";
process.StartInfo.Arguments = string.Format(System.Globalization.CultureInfo.InvariantCulture, RobocopyMoveCommandTemplate, Path.Combine(YammerCmpPath, Year), SPNewDir);
process.Start();
process.WaitForExit(2400 * 60 * 1000);
if (process.HasExited)
{
if (process.ExitCode > 8)
{
LogEvents("Error", "RoboCopyLog", " Compressed folders not moved to SharePoint Directory");
Environment.Exit(0);
}
}
}
UpdateStatus(uploadStartDate, uploadEndDate, Year);
LoadSPDirectoryMapping(uploadStartDate, uploadEndDate, SPNewDir);
UpdateYearStatus(Year);
}
}
Year = string.Empty;
}
}
LogEvents("Information", "FileCompress", "Compression method completed");
}
static async Task MainAsync()
{
GetSecretCmdlet getSecretCmdlet = new GetSecretCmdlet();
List <SharePointClassLibrary.SPFileInfo> ListSPFiles = new List <SharePointClassLibrary.SPFileInfo>();
List <string> filepaths = new List <string>();
//Get list of top level folders
List <string> ToplevelFolderlist = new List <string>();
getSecretCmdlet.secretURI = ConfigurationManager.AppSettings["DBNameURL"];
conn = await getSecretCmdlet.GetSecretAsync();
//Fetch list of Top level Directories of the doclib and insert records in to
DataSet ds = new DataSet();
string DocLibName = string.Empty;
string DocLibProcessStage = string.Empty;
// Check if any deduplication has to start
string Year = string.Empty;
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
Year = yeticontext.Yammer_GetYearForDedup(Environment.MachineName).ToString();
}
if (string.IsNullOrEmpty(Year))
{
Environment.Exit(0);
}
List <DupThreads_FetchDocLibStatus_Result> List = new List <DupThreads_FetchDocLibStatus_Result>();
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
List = yeticontext.DupThreads_FetchDocLibStatus().ToList();
}
if (List.Count > 0)
{
foreach (DupThreads_FetchDocLibStatus_Result Listitem in List)
{
DocLibName = Listitem.DocLibName.ToString();
DocLibProcessStage = Listitem.ProcessStage.ToString();
}
//call method to get list of TopLevel Folders
ToplevelFolderlist = SharePointClassLibrary.AzureSharePointHelper.GetFolderListFromSharePoint(DocLibName);
DataTable TopLevelFolders = new DataTable();
TopLevelFolders.Columns.Add("TopLevelFolder", System.Type.GetType("System.String"));
foreach (string TopLevelFolder in ToplevelFolderlist)
{
TopLevelFolders.Rows.Add(TopLevelFolder);
}
// insert the top level folder names in to a different table and update the status as "TopFoldersFetched" in DocLibStatus table
using (SqlConnection con = new SqlConnection(conn))
{
con.Open();
SqlCommand cmd = new SqlCommand();
cmd.Connection = con;
cmd.CommandTimeout = 0;
cmd.CommandType = CommandType.StoredProcedure;
cmd.CommandText = "DupThreads_InsertTopLevelSubfolders";
cmd.Parameters.AddWithValue("TopLevelSubfolders", TopLevelFolders);
cmd.Parameters.AddWithValue("DocLibName", DocLibName);
cmd.ExecuteNonQuery();
con.Close();
}
if (DocLibProcessStage == "TopLevelFoldersInserted")
{
List <string> TopFolders = new List <string>();
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
TopFolders = yeticontext.DupThreads_FetchTopLevelFolders(DocLibName).ToList();
}
if (TopFolders.Count > 0)
{
//Read TopLevelFolders and passthrough each of them to get list of threads
foreach (string SubFolderName in TopFolders)
{
//fetch threads info
SharePointClassLibrary.AzureSharePointHelper.GetSubFolders(DocLibName, SubFolderName, "", 2017, out ListSPFiles);
DataTable TableSPFiles = new DataTable();
//TableSPFiles.Columns.Add("ThreadID", System.Type.GetType("System.String"));
TableSPFiles.Columns.Add("DocLibName", System.Type.GetType("System.String"));
TableSPFiles.Columns.Add("TopLevelSubFolder", System.Type.GetType("System.String"));
TableSPFiles.Columns.Add("ThreadID_FileName", System.Type.GetType("System.String"));
TableSPFiles.Columns.Add("ThreadID_Size", System.Type.GetType("System.Int64"));
TableSPFiles.Columns.Add("ThreadID_CreatedDate", System.Type.GetType("System.DateTime"));
TableSPFiles.Columns.Add("ThreadID_Path", System.Type.GetType("System.String"));
//Insert Threadinfo to DB
foreach (SharePointClassLibrary.SPFileInfo SPfile in ListSPFiles)
{
TableSPFiles.Rows.Add(DocLibName, SubFolderName, SPfile.Filename, SPfile.filesize, SPfile.filecreateddate, SPfile.Filepath);
}
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.DupThreads_FetchTopLevelFolders(DocLibName).ToList();
}
using (SqlConnection con = new SqlConnection(conn))
{
con.Open();
SqlCommand cmd = new SqlCommand();
cmd.Connection = con;
cmd.CommandTimeout = 0;
cmd.CommandType = CommandType.StoredProcedure;
cmd.CommandText = "DupThreads_InsertThreadInfo";
cmd.Parameters.AddWithValue("SPFileInfo", TableSPFiles);
cmd.ExecuteNonQuery();
con.Close();
}
}
}
}
DataSet dsThreadsTobeDeleted = new DataSet();
List <DupThreads_FetchThreadstobeDeleted_Result> ListThreadstoDelete = new List <DupThreads_FetchThreadstobeDeleted_Result>();
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
ListThreadstoDelete = yeticontext.DupThreads_FetchThreadstobeDeleted(Environment.MachineName).ToList();
}
if (ListThreadstoDelete.Count > 0)
{
DataTable DThreads = new DataTable();
//Access threads data to be deleted.
DThreads = ds.Tables[0];
var ListThreadsTobeDeleted1 = DThreads.AsEnumerable().ToList();
string message = "";
foreach (DupThreads_FetchThreadstobeDeleted_Result ThreadToDelete in ListThreadstoDelete)
{
try
{
//Can delete the Thread
bool ThreadDeleted = SharePointClassLibrary.AzureSharePointHelper.FileExistAndDelete(ThreadToDelete.ThreadID_Path.ToString(), ThreadToDelete.DocLibName.ToString(), ThreadToDelete.ThreadID_Filename.ToString());
// Update Database to indicate that the duplicate thread is deleted from Sharepoint
if (ThreadDeleted)
{
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
yeticontext.DupThreads_UpdateDeletedThreadStatus(ThreadToDelete.ThreadID_Filename.ToString(), ThreadToDelete.ThreadID_Path.ToString(), ThreadToDelete.ThreadID_Size, ThreadToDelete.DocLibName);
}
}
//}
}
catch
(Exception e)
{
message = "Exception in calling linq query";
Console.WriteLine(message.ToString());
}
}
int DedupFinished = 0;
using (YETIDBEntities yeticontext = new YETIDBEntities())
{
DedupFinished = yeticontext.DupThreads_CheckDeleteThreadStatus(DocLibName.ToString());
}
if (DedupFinished == 1)
{
UpdateYearStatus(Year);
}
}
else
{
UpdateYearStatus(Year);
}
}
}
