static async Task GetAndUploadFileToTeamsChannel(String aadAccessToken, String selectedTeamId, SemaphoreSlim semaphore, Combined.AttachmentsMapping combinedAttachmentsMapping, string fileAttachmentsPath, AttachmentIdFilePathMode attachmentIdMode)
{
//string fileId = "";
Tuple <string, string> fileIdAndUrl;
try
{
var filename = combinedAttachmentsMapping.attachmentFileName;
filename = filename.Replace(":", "-");
switch (attachmentIdMode)
{
case AttachmentIdFilePathMode.Directory:
filename = combinedAttachmentsMapping.attachmentId + "/" + filename;
break;
case AttachmentIdFilePathMode.Prefix:
filename = "(" + combinedAttachmentsMapping.attachmentId + ") " + filename;
break;
case AttachmentIdFilePathMode.Suffix:
var extensionIndex = filename.LastIndexOf(".");
if (extensionIndex < 0)
{
filename = filename + " (" + combinedAttachmentsMapping.attachmentId + ")";
}
else
{
filename = filename.Substring(0, extensionIndex) + " (" + combinedAttachmentsMapping.attachmentId + ")" + filename.Substring(extensionIndex);
}
break;
}
var pathToItem = "/" + combinedAttachmentsMapping.msChannelName + fileAttachmentsPath + filename;
bool deleteFile = true;
string fileToUpload = "";
if (combinedAttachmentsMapping.attachmentUrl.StartsWith("https") || combinedAttachmentsMapping.attachmentUrl.StartsWith("http"))
{
Console.WriteLine("Downloading attachment to local file system " + combinedAttachmentsMapping.attachmentId);
var request = new HttpClient();
using (HttpResponseMessage response =
await request.GetAsync(combinedAttachmentsMapping.attachmentUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false))
{
// do something with response
using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync())
{
fileToUpload = Path.GetTempFileName();
using (Stream streamToWriteTo = File.Open(fileToUpload, FileMode.Create))
{
await streamToReadFrom.CopyToAsync(streamToWriteTo);
}
}
}
}
else
{
fileToUpload = combinedAttachmentsMapping.attachmentUrl;
deleteFile = false;
}
fileIdAndUrl = await UploadFileToTeamsChannel(aadAccessToken, selectedTeamId, fileToUpload, pathToItem, false);
combinedAttachmentsMapping.msSpoId = fileIdAndUrl.Item1;
combinedAttachmentsMapping.msSpoUrl = fileIdAndUrl.Item2;
if (deleteFile)
{
File.Delete(fileToUpload);
Console.WriteLine("Deleting local copy of attachment " + combinedAttachmentsMapping.attachmentId);
}
}
catch (Exception ex)
{
// do something
Console.WriteLine("Exception " + ex);
Console.WriteLine("On this file " + combinedAttachmentsMapping.attachmentId + " " + combinedAttachmentsMapping.attachmentUrl);
}
finally
{
// don't forget to release
semaphore.Release();
}
return;
}
static List <Models.Combined.AttachmentsMapping> GetAndUploadMessages(Models.Combined.ChannelsMapping channelsMapping, string basePath,
List <ViewModels.SimpleUser> slackUserList, String aadAccessToken, String selectedTeamId, bool copyFileAttachments, bool formatTimeToLocal,
int nbMessagesPerFile, string fileAttachmentsPath, AttachmentIdFilePathMode attachmentIdMode, string jsonMessagesPath, string htmlMessagesPath)
{
var messageList = new List <ViewModels.SimpleMessage>();
messageList.Clear();
var messageListJsonSource = new JArray();
messageListJsonSource.Clear();
List <Models.Combined.AttachmentsMapping> attachmentsToUpload = new List <Models.Combined.AttachmentsMapping>();
attachmentsToUpload.Clear();
Console.WriteLine("Migrating messages in channel " + channelsMapping.slackChannelName);
foreach (var file in Directory.GetFiles(Path.Combine(basePath, channelsMapping.slackChannelName)))
{
Console.WriteLine("File " + file);
using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read))
using (StreamReader sr = new StreamReader(fs))
using (JsonTextReader reader = new JsonTextReader(sr))
{
while (reader.Read())
{
if (reader.TokenType == JsonToken.StartObject)
{
JObject obj = JObject.Load(reader);
// SelectToken returns null not an empty string if nothing is found
// I'm too lazy right now for strongly typed classes
// deal with message basics: when, body, who
var messageTs = (string)obj.SelectToken("ts");
var messageText = (string)obj.SelectToken("text");
var messageId = channelsMapping.slackChannelId + "." + messageTs;
//messageText = RegexDetector.DetectSlackParens(messageText, slackUserList);
var messageSender = Utils.Messages.FindMessageSender(obj, slackUserList);
// create a list of attachments to upload
// deal with "attachments" that are files
// specifically, files hosted by Slack
var fileAttachments = new List <ViewModels.SimpleMessage.FileAttachment>();
var files = (JArray)obj.SelectToken("files");
if (files != null)
{
foreach (var attach in files)
{
// SelectToken returns null not an empty string if nothing is found
var fileUrl = (string)attach.SelectToken("url_private");
var fileId = (string)attach.SelectToken("id");
var fileMode = (string)attach.SelectToken("mode");
var fileName = (string)attach.SelectToken("name");
var local = (string)attach.SelectToken("url_private_download");
if (local != null && local.StartsWith("_files"))
{
fileUrl = Path.Join(basePath, local);
}
if (fileMode != "external" && fileId != null && fileUrl != null)
{
Console.WriteLine("Message attachment found with ID " + fileId);
attachmentsToUpload.Add(new Models.Combined.AttachmentsMapping
{
attachmentId = fileId,
attachmentUrl = fileUrl,
attachmentChannelId = channelsMapping.slackChannelId,
attachmentFileName = fileName,
msChannelName = channelsMapping.displayName
});
// map the attachment to fileAttachment which is used in the viewmodel
fileAttachments.Add(new ViewModels.SimpleMessage.FileAttachment
{
id = fileId,
originalName = (string)attach.SelectToken("name"),
originalTitle = (string)attach.SelectToken("title"),
originalUrl = (string)attach.SelectToken("permalink")
});
}
}
}
// deal with "attachments" that aren't files
List <ViewModels.SimpleMessage.Attachments> attachmentsList = new List <ViewModels.SimpleMessage.Attachments>();
List <ViewModels.SimpleMessage.Attachments.Fields> fieldsList = new List <ViewModels.SimpleMessage.Attachments.Fields>();
var attachmentsObject = (JArray)obj.SelectToken("attachments");
if (attachmentsObject != null)
{
foreach (var attachmentItem in attachmentsObject)
{
var attachmentText = (string)attachmentItem.SelectToken("text");
var attachmentTextFallback = (string)attachmentItem.SelectToken("fallback");
var attachmentItemToAdd = new ViewModels.SimpleMessage.Attachments();
if (!String.IsNullOrEmpty(attachmentText))
{
attachmentItemToAdd.text = attachmentText;
}
else if (!String.IsNullOrEmpty(attachmentTextFallback))
{
attachmentItemToAdd.text = attachmentTextFallback;
}
var attachmentServiceName = (string)attachmentItem.SelectToken("service_name");
if (!String.IsNullOrEmpty(attachmentServiceName))
{
attachmentItemToAdd.service_name = attachmentServiceName;
}
var attachmentFromUrl = (string)attachmentItem.SelectToken("from_url");
if (!String.IsNullOrEmpty(attachmentFromUrl))
{
attachmentItemToAdd.url = attachmentFromUrl;
}
var attachmentColor = (string)attachmentItem.SelectToken("color");
if (!String.IsNullOrEmpty(attachmentColor))
{
attachmentItemToAdd.color = attachmentColor;
}
var fieldsObject = (JArray)attachmentItem.SelectToken("fields");
if (fieldsObject != null)
{
fieldsList.Clear();
foreach (var fieldItem in fieldsObject)
{
fieldsList.Add(new ViewModels.SimpleMessage.Attachments.Fields()
{
title = (string)fieldItem.SelectToken("title"),
value = (string)fieldItem.SelectToken("value"),
shortWidth = (bool)fieldItem.SelectToken("short")
});
}
attachmentItemToAdd.fields = fieldsList;
}
else
{
attachmentItemToAdd.fields = null;
}
attachmentsList.Add(attachmentItemToAdd);
}
}
else
{
attachmentsList = null;
}
// do some stuff with slack message threading at some point
messageList.Add(new ViewModels.SimpleMessage
{
id = messageId,
text = messageText,
ts = messageTs,
user = messageSender,
fileAttachments = fileAttachments,
attachments = attachmentsList,
});
}
}
}
}
if (copyFileAttachments)
{
Utils.FileAttachments.ArchiveMessageFileAttachments(aadAccessToken, selectedTeamId, attachmentsToUpload, fileAttachmentsPath, attachmentIdMode).Wait();
foreach (var messageItem in messageList)
{
if (messageItem.fileAttachments.Count > 0)
{
foreach (var attach in messageItem.fileAttachments)
{
var messageItemWithFileAttachment = attachmentsToUpload.Find(w => String.Equals(attach.id, w.attachmentId, StringComparison.CurrentCultureIgnoreCase));
if (messageItemWithFileAttachment != null)
{
attach.spoId = messageItemWithFileAttachment.msSpoId;
attach.spoUrl = messageItemWithFileAttachment.msSpoUrl;
}
}
}
}
}
Utils.Messages.CreateSlackMessageJsonArchiveFile(basePath, channelsMapping, messageList, aadAccessToken, selectedTeamId, nbMessagesPerFile, jsonMessagesPath);
Utils.Messages.CreateSlackMessageHtmlArchiveFile(basePath, channelsMapping, messageList, aadAccessToken, selectedTeamId, slackUserList, formatTimeToLocal, nbMessagesPerFile, htmlMessagesPath);
return(attachmentsToUpload);
}
public static async Task ArchiveMessageFileAttachments(String aadAccessToken, String selectedTeamId, List <Combined.AttachmentsMapping> combinedAttachmentsMapping, string fileAttachmentsPath, AttachmentIdFilePathMode attachmentIdMode, int maxDls = 10)
{
var tasks = new List <Task>();
// semaphore, allow to run maxDLs (default 10) tasks in parallel
SemaphoreSlim semaphore = new SemaphoreSlim(maxDls);
foreach (var v in combinedAttachmentsMapping)
{
// await here until there is a room for this task
await semaphore.WaitAsync();
tasks.Add(GetAndUploadFileToTeamsChannel(aadAccessToken, selectedTeamId, semaphore, v, fileAttachmentsPath, attachmentIdMode));
}
// await for the rest of tasks to complete
await Task.WhenAll(tasks);
}
public static void ScanMessagesByChannel(List <Models.Combined.ChannelsMapping> channelsMapping, string basePath,
List <ViewModels.SimpleUser> slackUserList, String aadAccessToken, String selectedTeamId, bool copyFileAttachments, bool formatTimeToLocal,
int nbMessagesPerFile, string fileAttachmentsPath, AttachmentIdFilePathMode attachmentIdMode, string jsonMessagesPath, string htmlMessagesPath)
{
foreach (var v in channelsMapping)
{
var channelAttachmentsToUpload = GetAndUploadMessages(v, basePath, slackUserList, aadAccessToken, selectedTeamId, copyFileAttachments, formatTimeToLocal, nbMessagesPerFile, fileAttachmentsPath, attachmentIdMode, jsonMessagesPath, htmlMessagesPath);
}
return;
}