Пример #1
0
        public static async Task <string> UploadFile(string attachmentPath, Team team, Channel channel, ImportContext importContext, int tryCount = 0)
        {
            var fileSizeBytes = new IO.FileInfo(attachmentPath).Length;
            var fileName      = IO.Path.GetFileName(IO.Path.GetDirectoryName(attachmentPath)) + IO.Path.GetExtension(attachmentPath);
            var uploadPath    = channel.DisplayName + "/" + fileName;

            var itemRequest = importContext.GraphServiceClient.Groups[team.Id].Drive.Root.ItemWithPath(uploadPath);

            try
            {
                using (var stream = IO.File.OpenRead(attachmentPath))
                {
                    // 4MB is the maximum for a simple upload, staying safe with 3.6.
                    if (fileSizeBytes / 1024 / 1024 > 3.6)
                    {
                        var session = await itemRequest.CreateUploadSession().Request().PostAsync();

                        var maxChunkSize = 320 * 4 * 1024;
                        var provider     = new ChunkedUploadProvider(session, importContext.GraphServiceClient, stream, maxChunkSize);

                        var       chunckRequests = provider.GetUploadChunkRequests();
                        var       exceptions     = new List <Exception>();
                        var       readBuffer     = new byte[maxChunkSize];
                        DriveItem itemResult     = null;

                        foreach (var request in chunckRequests)
                        {
                            var result = await provider.GetChunkRequestResponseAsync(request, readBuffer, exceptions);

                            if (result.UploadSucceeded)
                            {
                                itemResult = result.ItemResponse;
                            }
                        }

                        // Retry if upload failed.
                        if (itemResult == null)
                        {
                            if (tryCount < 5)
                            {
                                TimestampedConsole.WriteLine($"Uploading the large attachment \"{attachmentPath}\" failed. Retrying.");
                                return(await UploadFile(attachmentPath, team, channel, importContext, ++tryCount));
                            }
                            else
                            {
                                throw new Exception($"The attachment \"{attachmentPath}\" couldn't be uploaded into the \"{channel.DisplayName}\" channel (\"{team.DisplayName}\" team) because upload failed repeatedly. You can try again as this might be just a temporary error. If the issue isn't resolved then delete the file so it's not uploaded.");
                            }
                        }
                    }
                    else
                    {
                        await itemRequest.Content.Request().PutAsync <DriveItem>(stream);
                    }

                    return((await itemRequest.Request().GetAsync()).WebUrl);
                }
            }
            catch (ServiceException ex) when(ex.Message.Contains("Unable to provision resource."))
            {
                var waitMinutes = 5;

                TimestampedConsole.WriteLine($"The team's SharePoint site is not yet set up (you can check this under the channel's Files tab) so attachments can't be uploaded. Waiting {waitMinutes} minutes.");
                await Task.Delay(waitMinutes * 60000);

                TimestampedConsole.WriteLine($"Retrying upload.");

                return(await UploadFile(attachmentPath, team, channel, importContext, 0));
            }
        }
Пример #2
0
        public static async Task ImportChannelsFromRoomsAsync(ImportContext importContext, bool checkTeams = true)
        {
            var configuration = importContext.Configuration;
            var graphApi      = importContext.GraphApi;

            if (!File.Exists(CursorPath))
            {
                await UpdateCursor(new ImportCursor());
            }

            var cursor = JsonConvert.DeserializeObject <ImportCursor>(await File.ReadAllTextAsync(CursorPath));

            if (cursor == null)
            {
                cursor = new ImportCursor();
            }

            var rooms = JsonConvert
                        .DeserializeObject <RoomContainer[]>(await File.ReadAllTextAsync(Path.Combine(configuration.ExportFolderPath, "rooms.json")))
                        .Select(roomContainer => roomContainer.Room)
                        .Skip(cursor.SkipRooms);

            var teams = (await graphApi.GetMyTeamsAsync()).Items.ToDictionary(team => team.DisplayName);

            if (checkTeams)
            {
                // Creating teams in advance so there's less waiting on the provisioning of the corresponding SharePoint
                // sites.
                Console.WriteLine("======================");
                TimestampedConsole.WriteLine("Creating teams that don't exist.");

                var teamNamesToUse = configuration.HipChatRoomsToTeams.Values;

                foreach (var teamName in teamNamesToUse.Distinct())
                {
                    if (!teams.ContainsKey(teamName))
                    {
                        TimestampedConsole.WriteLine($"The team \"{teamName}\" doesn't exist, so attempting to create it.");

                        using (var response = await graphApi.CreateTeamAsync(new Team {
                            DisplayName = teamName
                        }))
                        {
                            if (!response.Headers.TryGetValues("Location", out var locations) || locations.Count() != 1)
                            {
                                throw new Exception($"Attempted to create the \"{teamName}\" team but the operation didn't return correctly. Try to create the team manually.");
                            }

                            var location  = locations.Single();
                            var operation = await graphApi.GetAsyncOperation(location);

                            var tries = 0;
                            while (operation.Status != "succeeded")
                            {
                                if (tries > 10)
                                {
                                    throw new Exception($"Attempted to create the \"{teamName}\" team but the operation didn't succeed after plenty of time. Try to create the team manually.");
                                }

                                // https://docs.microsoft.com/en-us/graph/api/resources/teamsasyncoperation?view=graph-rest-beta
                                // says to wait >30s.
                                await Task.Delay(31000);

                                operation = await graphApi.GetAsyncOperation(location);

                                tries++;
                            }

                            teams[teamName] = new Team
                            {
                                Id          = operation.TargetResourceId,
                                DisplayName = teamName
                            };
                        }

                        TimestampedConsole.WriteLine($"Created the \"{teamName}\" team.");
                    }
                }

                TimestampedConsole.WriteLine("Created all new teams.");
                Console.WriteLine("======================");
            }

            foreach (var room in rooms)
            {
                if (!configuration.HipChatRoomsToTeams.TryGetValue(room.Name, out string teamNameToImportInto))
                {
                    if (room.IsArchived && configuration.HipChatRoomsToTeams.ContainsKey("$Archived default"))
                    {
                        teamNameToImportInto = configuration.HipChatRoomsToTeams["$Archived default"];
                    }
                    else if (configuration.HipChatRoomsToTeams.ContainsKey("$Default"))
                    {
                        teamNameToImportInto = configuration.HipChatRoomsToTeams["$Default"];
                    }
                }

                var teamToImportInto = teams[teamNameToImportInto];

                try
                {
                    if (!configuration.HipChatRoomsToChannels.TryGetValue(room.Name, out string channelName))
                    {
                        channelName = room.Name;
                    }

                    // This can be used to test a single room again and again, without having to delete anything.
                    //channelName += new Random().Next();
                    //room.Id = 4726816;

                    Console.WriteLine("======================");

                    TimestampedConsole.WriteLine($"Starting processing the \"{room.Name}\" room.");

                    if (_unsupportedChannelNameCharacters.Any(character => channelName.Contains(character)))
                    {
                        channelName = string.Join("", channelName.Split(_unsupportedChannelNameCharacters, StringSplitOptions.RemoveEmptyEntries));
                        TimestampedConsole.WriteLine(
                            $"* The \"{room.Name}\" room's name contains at least one character not allowed in channel names " +
                            $"({_unsupportedChannelNameCharactersString})). Offending characters were removed: \"{channelName}\".");
                    }

                    Channel channel = (await graphApi.GetChannelsAsync(teamToImportInto.Id))
                                      .Items
                                      .FirstOrDefault(c => c.DisplayName == channelName);

                    if (channel == null)
                    {
                        channel = await graphApi.CreateChannelAsync(
                            teamToImportInto.Id,
                            new Channel
                        {
                            DisplayName = channelName,
                            Description = room.Topic
                        });

                        TimestampedConsole.WriteLine($"Created the \"{channelName}\" channel. Starting importing messages.");
                    }
                    else
                    {
                        TimestampedConsole.WriteLine($"Starting importing messages into the existing \"{channelName}\" channel.");
                    }

                    var roomFolderPath  = Path.Combine(configuration.ExportFolderPath, "rooms", room.Id.ToString());
                    var historyFilePath = Path.Combine(roomFolderPath, "history.json");
                    var messages        = JsonConvert
                                          .DeserializeObject <Message[]>(await File.ReadAllTextAsync(historyFilePath), new MessageJsonConverter())
                                          .Where(message => message != null && !(message is ArchiveRoomMessage))
                                          // To preserve the original order at least we need to reverse the data set. This wouldn't be
                                          // needed if the created message's timestamp would actually take effect, see:
                                          // https://github.com/Lombiq/HipChat-to-Microsoft-Teams-Migration-Utility/issues/1
                                          .Reverse()
                                          .Skip(cursor.SkipMessages);

                    var batchSize = configuration.NumberOfHipChatMessagesToImportIntoTeamsMessage;

                    if (importContext.MessageBatchSizeOverride > 0)
                    {
                        batchSize = importContext.MessageBatchSizeOverride;
                    }

                    while (messages.Any())
                    {
                        var messageBatch = messages.Take(batchSize);

                        var chatMessage = new ChatMessage
                        {
                            Body = new ItemBody
                            {
                                ContentType = "1"
                            },
                            CreatedDateTime = messageBatch.First().Timestamp
                        };

                        var attachments = new List <ChatMessageAttachment>();

                        var batchedMessageBody = string.Empty;

                        foreach (var message in messageBatch)
                        {
                            var messageBody = message.Body;

                            if (importContext.ShortenNextMessage)
                            {
                                messageBody =
                                    messageBody.Substring(0, configuration.ShortenLongMessagesToCharacterCount) + "...";
                            }

                            if (message is UserMessage userMessage)
                            {
                                // Users are not fetched yet and this doesn't work, so using a hack to show
                                // the user's name in the message body for now.
                                //chatMessage.From = new IdentitySet
                                //{
                                //    User = new User
                                //    {
                                //        DisplayName = userMessage.Sender.Name
                                //    }
                                //};

                                if (messageBody.StartsWith("/code "))
                                {
                                    messageBody = $"<pre><code>{messageBody.Substring(6)}</code></pre>";
                                }

                                if (messageBody.StartsWith("/quote "))
                                {
                                    messageBody = $"<blockquote>{messageBody.Substring(7)}</blockquote>";
                                }

                                messageBody = $"{userMessage.Sender.Name}:<br>{messageBody}";

                                if (configuration.UploadAttachments && userMessage.Attachment != null)
                                {
                                    var attachmentPathSegments = userMessage.Attachment.Path.Split(new[] { '/' });
                                    var attachmentPath         = Path.Combine(roomFolderPath, "files", attachmentPathSegments[0], attachmentPathSegments[1]);

                                    // Attachments can't be embedded into messages (because they won't get attached to
                                    // them for some reason) but the files can be linked to.
                                    if (File.Exists(attachmentPath))
                                    {
                                        var fileUrl = await AttachmentUploader.UploadFile(attachmentPath, teamToImportInto, channel, importContext);

                                        var extension = Path.GetExtension(attachmentPath);

                                        if (!string.IsNullOrEmpty(extension))
                                        {
                                            var contentType = MimeTypeMap.List.MimeTypeMap
                                                              .GetMimeType(extension)
                                                              .FirstOrDefault();

                                            if (contentType.StartsWith("image/"))
                                            {
                                                messageBody += $"<br><img src=\"{fileUrl}\">";
                                            }
                                            else
                                            {
                                                messageBody += $"<br><a href=\"{fileUrl}\">Attachment</a>";
                                            }
                                        }
                                        else
                                        {
                                            messageBody += $"<br><a href=\"{fileUrl}\">Attachment</a>";
                                        }

                                        attachments.Add(new ChatMessageAttachment
                                        {
                                            ContentUrl  = fileUrl,
                                            ContentType = "reference",
                                            Name        = Path.GetFileName(fileUrl)
                                        });
                                    }
                                }
                            }
                            else
                            {
                                // This doesn't work.
                                //chatMessage.From = new From
                                //{
                                //    Guest = new User
                                //    {
                                //        DisplayName = ((NotificationMessage)message).Sender
                                //    }
                                //};

                                messageBody = $"{((NotificationMessage)message).Sender}:<br>{messageBody}";
                            }

                            messageBody = message.Timestamp.ToString() + " UTC " + messageBody;
                            if (batchSize != 1)
                            {
                                messageBody += "<hr>";
                            }

                            batchedMessageBody += messageBody;
                            importContext.ShortenNextMessage = false;
                        }

                        chatMessage.Body.Content = batchedMessageBody;
                        chatMessage.Attachments  = attachments;

                        await graphApi.CreateThreadAsync(
                            teamToImportInto.Id,
                            channel.Id,
                            new ChatThread
                        {
                            RootMessage = chatMessage
                        });

                        cursor.SkipMessages += batchSize;
                        await UpdateCursor(cursor);

                        _throttlingCooldownMinutes = DefaultThrottlingCooldownMinutes;

                        // Waiting a bit to avoid API throttling.
                        await Task.Delay(500);

                        if (cursor.SkipMessages % 50 == 0)
                        {
                            TimestampedConsole.WriteLine($"{cursor.SkipMessages} messages imported into the channel.");
                        }

                        messages = messages.Skip(batchSize);

                        batchSize = configuration.NumberOfHipChatMessagesToImportIntoTeamsMessage;
                        importContext.MessageBatchSizeOverride = batchSize;
                    }


                    cursor.SkipRooms++;
                    cursor.SkipMessages = 0;
                    await UpdateCursor(cursor);

                    TimestampedConsole.WriteLine($"Messages imported into the \"{channel.DisplayName}\" channel.");

                    Console.WriteLine("======================");
                }
                catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.TooManyRequests)
                {
                    TimestampedConsole.WriteLine($"API requests are being throttled. Waiting for {_throttlingCooldownMinutes} minutes, then retrying. If this happens again and again then close the app and wait some time (more than an hour, or sometimes even a day) before starting it again.");

                    // While some APIs return a Retry-After header to indicate when you should retry a throttled
                    // request (see: https://docs.microsoft.com/en-us/graph/throttling) the Teams endpoints don't.
                    // Also, all endpoints seem to have their own limits, because after the message creation is
                    // throttled e.g. the user APIs still work. So we need to use such hacks.
                    await Task.Delay(_throttlingCooldownMinutes * 60000);

                    _throttlingCooldownMinutes *= 2;

                    await ImportChannelsFromRoomsAsync(importContext);

                    return;
                }
                catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.ServiceUnavailable || ex.StatusCode == HttpStatusCode.InternalServerError)
                {
                    var waitSeconds = 10;

                    TimestampedConsole.WriteLine($"A request failed with the error Service Unavailable. Waiting {waitSeconds}s, then retrying.");
                    await Task.Delay(waitSeconds * 1000);

                    await ImportChannelsFromRoomsAsync(importContext);

                    return;
                }
                catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.RequestEntityTooLarge)
                {
                    if (importContext.MessageBatchSizeOverride == configuration.NumberOfHipChatMessagesToImportIntoTeamsMessage &&
                        importContext.MessageBatchSizeOverride > 5)
                    {
                        importContext.MessageBatchSizeOverride = 5;
                    }
                    else if (importContext.MessageBatchSizeOverride != 1)
                    {
                        importContext.MessageBatchSizeOverride = 1;
                    }
                    else
                    {
                        if (configuration.ShortenLongMessagesToCharacterCount > 0)
                        {
                            importContext.ShortenNextMessage = true;

                            await ImportChannelsFromRoomsAsync(importContext, false);

                            return;
                        }
                        else
                        {
                            throw new Exception($"The next message to import from the \"{room.Name}\" room is too large. You need to manually shorten it in the corresponding JSON file in the HipChat export package.");
                        }
                    }

                    TimestampedConsole.WriteLine($"Importing {configuration.NumberOfHipChatMessagesToImportIntoTeamsMessage} HipChat messages into a Teams message resulted in a message too large. Retrying with just {importContext.MessageBatchSizeOverride} messages.");

                    await ImportChannelsFromRoomsAsync(importContext, false);

                    return;
                }
                catch (ApiException ex) when(ex.StatusCode == HttpStatusCode.Unauthorized)
                {
                    throw;
                }
                catch (Exception ex)
                {
                    throw new Exception($"Importing the room \"{room.Name}\" with the description \"{room.Topic}\" failed. This error can't be retried.", ex);
                }
            }
        }