static async Task GetAndUploadFileToTeamsChannel(String aadAccessToken, String selectedTeamId, SemaphoreSlim semaphore, Combined.AttachmentsMapping combinedAttachmentsMapping, string fileAttachmentsPath, AttachmentIdFilePathMode attachmentIdMode)
        {
            //string fileId = "";
            Tuple <string, string> fileIdAndUrl;

            try
            {
                var filename = combinedAttachmentsMapping.attachmentFileName;
                filename = filename.Replace(":", "-");
                switch (attachmentIdMode)
                {
                case AttachmentIdFilePathMode.Directory:
                    filename = combinedAttachmentsMapping.attachmentId + "/" + filename;
                    break;

                case AttachmentIdFilePathMode.Prefix:
                    filename = "(" + combinedAttachmentsMapping.attachmentId + ") " + filename;
                    break;

                case AttachmentIdFilePathMode.Suffix:
                    var extensionIndex = filename.LastIndexOf(".");
                    if (extensionIndex < 0)
                    {
                        filename = filename + " (" + combinedAttachmentsMapping.attachmentId + ")";
                    }
                    else
                    {
                        filename = filename.Substring(0, extensionIndex) + " (" + combinedAttachmentsMapping.attachmentId + ")" + filename.Substring(extensionIndex);
                    }
                    break;
                }
                var pathToItem = "/" + combinedAttachmentsMapping.msChannelName + fileAttachmentsPath + filename;

                bool   deleteFile   = true;
                string fileToUpload = "";
                if (combinedAttachmentsMapping.attachmentUrl.StartsWith("https") || combinedAttachmentsMapping.attachmentUrl.StartsWith("http"))
                {
                    Console.WriteLine("Downloading attachment to local file system " + combinedAttachmentsMapping.attachmentId);
                    var request = new HttpClient();
                    using (HttpResponseMessage response =
                               await request.GetAsync(combinedAttachmentsMapping.attachmentUrl, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false))
                    {
                        // do something with response
                        using (Stream streamToReadFrom = await response.Content.ReadAsStreamAsync())
                        {
                            fileToUpload = Path.GetTempFileName();
                            using (Stream streamToWriteTo = File.Open(fileToUpload, FileMode.Create))
                            {
                                await streamToReadFrom.CopyToAsync(streamToWriteTo);
                            }
                        }
                    }
                }
                else
                {
                    fileToUpload = combinedAttachmentsMapping.attachmentUrl;
                    deleteFile   = false;
                }

                fileIdAndUrl = await UploadFileToTeamsChannel(aadAccessToken, selectedTeamId, fileToUpload, pathToItem, false);

                combinedAttachmentsMapping.msSpoId  = fileIdAndUrl.Item1;
                combinedAttachmentsMapping.msSpoUrl = fileIdAndUrl.Item2;
                if (deleteFile)
                {
                    File.Delete(fileToUpload);
                    Console.WriteLine("Deleting local copy of attachment " + combinedAttachmentsMapping.attachmentId);
                }
            }
            catch (Exception ex)
            {
                // do something
                Console.WriteLine("Exception " + ex);
                Console.WriteLine("On this file " + combinedAttachmentsMapping.attachmentId + " " + combinedAttachmentsMapping.attachmentUrl);
            }
            finally
            {
                // don't forget to release
                semaphore.Release();
            }
            return;
        }
Beispiel #2
0
        static List <Models.Combined.AttachmentsMapping> GetAndUploadMessages(Models.Combined.ChannelsMapping channelsMapping, string basePath,
                                                                              List <ViewModels.SimpleUser> slackUserList, String aadAccessToken, String selectedTeamId, bool copyFileAttachments, bool formatTimeToLocal,
                                                                              int nbMessagesPerFile, string fileAttachmentsPath, AttachmentIdFilePathMode attachmentIdMode, string jsonMessagesPath, string htmlMessagesPath)
        {
            var messageList = new List <ViewModels.SimpleMessage>();

            messageList.Clear();

            var messageListJsonSource = new JArray();

            messageListJsonSource.Clear();

            List <Models.Combined.AttachmentsMapping> attachmentsToUpload = new List <Models.Combined.AttachmentsMapping>();

            attachmentsToUpload.Clear();

            Console.WriteLine("Migrating messages in channel " + channelsMapping.slackChannelName);
            foreach (var file in Directory.GetFiles(Path.Combine(basePath, channelsMapping.slackChannelName)))
            {
                Console.WriteLine("File " + file);
                using (FileStream fs = new FileStream(file, FileMode.Open, FileAccess.Read))
                    using (StreamReader sr = new StreamReader(fs))
                        using (JsonTextReader reader = new JsonTextReader(sr))
                        {
                            while (reader.Read())
                            {
                                if (reader.TokenType == JsonToken.StartObject)
                                {
                                    JObject obj = JObject.Load(reader);

                                    // SelectToken returns null not an empty string if nothing is found
                                    // I'm too lazy right now for strongly typed classes

                                    // deal with message basics: when, body, who

                                    var messageTs   = (string)obj.SelectToken("ts");
                                    var messageText = (string)obj.SelectToken("text");
                                    var messageId   = channelsMapping.slackChannelId + "." + messageTs;
                                    //messageText = RegexDetector.DetectSlackParens(messageText, slackUserList);
                                    var messageSender = Utils.Messages.FindMessageSender(obj, slackUserList);

                                    // create a list of attachments to upload
                                    // deal with "attachments" that are files
                                    // specifically, files hosted by Slack

                                    var fileAttachments = new List <ViewModels.SimpleMessage.FileAttachment>();

                                    var files = (JArray)obj.SelectToken("files");
                                    if (files != null)
                                    {
                                        foreach (var attach in files)
                                        {
                                            // SelectToken returns null not an empty string if nothing is found
                                            var fileUrl  = (string)attach.SelectToken("url_private");
                                            var fileId   = (string)attach.SelectToken("id");
                                            var fileMode = (string)attach.SelectToken("mode");
                                            var fileName = (string)attach.SelectToken("name");
                                            var local    = (string)attach.SelectToken("url_private_download");
                                            if (local != null && local.StartsWith("_files"))
                                            {
                                                fileUrl = Path.Join(basePath, local);
                                            }

                                            if (fileMode != "external" && fileId != null && fileUrl != null)
                                            {
                                                Console.WriteLine("Message attachment found with ID " + fileId);
                                                attachmentsToUpload.Add(new Models.Combined.AttachmentsMapping
                                                {
                                                    attachmentId        = fileId,
                                                    attachmentUrl       = fileUrl,
                                                    attachmentChannelId = channelsMapping.slackChannelId,
                                                    attachmentFileName  = fileName,
                                                    msChannelName       = channelsMapping.displayName
                                                });

                                                // map the attachment to fileAttachment which is used in the viewmodel
                                                fileAttachments.Add(new ViewModels.SimpleMessage.FileAttachment
                                                {
                                                    id            = fileId,
                                                    originalName  = (string)attach.SelectToken("name"),
                                                    originalTitle = (string)attach.SelectToken("title"),
                                                    originalUrl   = (string)attach.SelectToken("permalink")
                                                });
                                            }
                                        }
                                    }

                                    // deal with "attachments" that aren't files

                                    List <ViewModels.SimpleMessage.Attachments>        attachmentsList = new List <ViewModels.SimpleMessage.Attachments>();
                                    List <ViewModels.SimpleMessage.Attachments.Fields> fieldsList      = new List <ViewModels.SimpleMessage.Attachments.Fields>();
                                    var attachmentsObject = (JArray)obj.SelectToken("attachments");
                                    if (attachmentsObject != null)
                                    {
                                        foreach (var attachmentItem in attachmentsObject)
                                        {
                                            var attachmentText         = (string)attachmentItem.SelectToken("text");
                                            var attachmentTextFallback = (string)attachmentItem.SelectToken("fallback");

                                            var attachmentItemToAdd = new ViewModels.SimpleMessage.Attachments();

                                            if (!String.IsNullOrEmpty(attachmentText))
                                            {
                                                attachmentItemToAdd.text = attachmentText;
                                            }
                                            else if (!String.IsNullOrEmpty(attachmentTextFallback))
                                            {
                                                attachmentItemToAdd.text = attachmentTextFallback;
                                            }

                                            var attachmentServiceName = (string)attachmentItem.SelectToken("service_name");
                                            if (!String.IsNullOrEmpty(attachmentServiceName))
                                            {
                                                attachmentItemToAdd.service_name = attachmentServiceName;
                                            }

                                            var attachmentFromUrl = (string)attachmentItem.SelectToken("from_url");
                                            if (!String.IsNullOrEmpty(attachmentFromUrl))
                                            {
                                                attachmentItemToAdd.url = attachmentFromUrl;
                                            }

                                            var attachmentColor = (string)attachmentItem.SelectToken("color");
                                            if (!String.IsNullOrEmpty(attachmentColor))
                                            {
                                                attachmentItemToAdd.color = attachmentColor;
                                            }

                                            var fieldsObject = (JArray)attachmentItem.SelectToken("fields");
                                            if (fieldsObject != null)
                                            {
                                                fieldsList.Clear();
                                                foreach (var fieldItem in fieldsObject)
                                                {
                                                    fieldsList.Add(new ViewModels.SimpleMessage.Attachments.Fields()
                                                    {
                                                        title      = (string)fieldItem.SelectToken("title"),
                                                        value      = (string)fieldItem.SelectToken("value"),
                                                        shortWidth = (bool)fieldItem.SelectToken("short")
                                                    });
                                                }
                                                attachmentItemToAdd.fields = fieldsList;
                                            }
                                            else
                                            {
                                                attachmentItemToAdd.fields = null;
                                            }
                                            attachmentsList.Add(attachmentItemToAdd);
                                        }
                                    }
                                    else
                                    {
                                        attachmentsList = null;
                                    }

                                    // do some stuff with slack message threading at some point

                                    messageList.Add(new ViewModels.SimpleMessage
                                    {
                                        id              = messageId,
                                        text            = messageText,
                                        ts              = messageTs,
                                        user            = messageSender,
                                        fileAttachments = fileAttachments,
                                        attachments     = attachmentsList,
                                    });
                                }
                            }
                        }
            }

            if (copyFileAttachments)
            {
                Utils.FileAttachments.ArchiveMessageFileAttachments(aadAccessToken, selectedTeamId, attachmentsToUpload, fileAttachmentsPath, attachmentIdMode).Wait();

                foreach (var messageItem in messageList)
                {
                    if (messageItem.fileAttachments.Count > 0)
                    {
                        foreach (var attach in messageItem.fileAttachments)
                        {
                            var messageItemWithFileAttachment = attachmentsToUpload.Find(w => String.Equals(attach.id, w.attachmentId, StringComparison.CurrentCultureIgnoreCase));
                            if (messageItemWithFileAttachment != null)
                            {
                                attach.spoId  = messageItemWithFileAttachment.msSpoId;
                                attach.spoUrl = messageItemWithFileAttachment.msSpoUrl;
                            }
                        }
                    }
                }
            }
            Utils.Messages.CreateSlackMessageJsonArchiveFile(basePath, channelsMapping, messageList, aadAccessToken, selectedTeamId, nbMessagesPerFile, jsonMessagesPath);
            Utils.Messages.CreateSlackMessageHtmlArchiveFile(basePath, channelsMapping, messageList, aadAccessToken, selectedTeamId, slackUserList, formatTimeToLocal, nbMessagesPerFile, htmlMessagesPath);

            return(attachmentsToUpload);
        }
        public static async Task ArchiveMessageFileAttachments(String aadAccessToken, String selectedTeamId, List <Combined.AttachmentsMapping> combinedAttachmentsMapping, string fileAttachmentsPath, AttachmentIdFilePathMode attachmentIdMode, int maxDls = 10)
        {
            var tasks = new List <Task>();

            // semaphore, allow to run maxDLs (default 10) tasks in parallel
            SemaphoreSlim semaphore = new SemaphoreSlim(maxDls);

            foreach (var v in combinedAttachmentsMapping)
            {
                // await here until there is a room for this task
                await semaphore.WaitAsync();

                tasks.Add(GetAndUploadFileToTeamsChannel(aadAccessToken, selectedTeamId, semaphore, v, fileAttachmentsPath, attachmentIdMode));
            }

            // await for the rest of tasks to complete
            await Task.WhenAll(tasks);
        }
Beispiel #4
0
        public static void ScanMessagesByChannel(List <Models.Combined.ChannelsMapping> channelsMapping, string basePath,
                                                 List <ViewModels.SimpleUser> slackUserList, String aadAccessToken, String selectedTeamId, bool copyFileAttachments, bool formatTimeToLocal,
                                                 int nbMessagesPerFile, string fileAttachmentsPath, AttachmentIdFilePathMode attachmentIdMode, string jsonMessagesPath, string htmlMessagesPath)
        {
            foreach (var v in channelsMapping)
            {
                var channelAttachmentsToUpload = GetAndUploadMessages(v, basePath, slackUserList, aadAccessToken, selectedTeamId, copyFileAttachments, formatTimeToLocal, nbMessagesPerFile, fileAttachmentsPath, attachmentIdMode, jsonMessagesPath, htmlMessagesPath);
            }

            return;
        }