public static void AddBubbles(BubbleGroup group, VisualBubble[] bubbles) { lock (OperationLock) { var file = GetLocation(@group); using (var stream = File.Open(file, FileMode.Append, FileAccess.Write)) { BubbleGroupIndex.UpdateLastBubbleOrAndLastModifiedIndex(group.ID, bubbles.Last(), stream.Position); foreach (var b in bubbles) { using (var ms = new MemoryStream()) { Serializer.Serialize(ms, b); var bubbleData = ms.ToArray(); var bubbleHeader = b.GetType().Name + ":" + b.ID + ":" + b.Time; BubbleGroupDatabasePrimitives.WriteBubbleData(stream, bubbleData); BubbleGroupDatabasePrimitives.WriteBubbleHeader(stream, bubbleHeader); } } } } }
public static UnifiedBubbleGroup CreateUnified(List <BubbleGroup> groups, BubbleGroup primaryGroup) { lock (BubbleGroupDatabase.OperationLock) { var unifiedGroupsToKill = new HashSet <UnifiedBubbleGroup>(); foreach (var group in groups) { if (group.IsUnified) { unifiedGroupsToKill.Add(group.Unified); @group.DeregisterUnified(); } } foreach (var unifiedGroup in unifiedGroupsToKill) { BubbleGroupManager.BubbleGroupsRemove(unifiedGroup); } BubbleGroupIndex.RemoveUnified(unifiedGroupsToKill.Select(x => x.ID).ToArray()); var unified = CreateUnifiedInternal(groups, primaryGroup); BubbleGroupIndex.AddUnified(unified); BubbleGroupManager.BubbleGroupsAdd(unified); BubbleGroupSettingsManager.SetUnreadIndicatorGuid(unified, unified.LastBubbleSafe().ID, false); return(unified); } }
public static void BubbleGroupsRemove(BubbleGroup group) { lock (BubbleGroupsLock) { BubbleGroups.Remove(group); if (!(group is UnifiedBubbleGroup)) { BubbleGroupIndex.Remove(group.ID); } } }
public static void BubbleGroupsAdd(BubbleGroup group, bool initialLoad = false) { lock (BubbleGroupsLock) { BubbleGroups.Add(group); if (!initialLoad && !(group is UnifiedBubbleGroup)) { BubbleGroupIndex.Add(group); } } }
public static List <BubbleGroup> DeleteUnified(UnifiedBubbleGroup unifiedGroup, bool deleteInnerGroups = true) { lock (BubbleGroupDatabase.OperationLock) { foreach (var group in unifiedGroup.Groups) { @group.DeregisterUnified(); if (deleteInnerGroups) { Delete(@group); } } BubbleGroupManager.BubbleGroupsRemove(unifiedGroup); BubbleGroupIndex.RemoveUnified(unifiedGroup.ID); return(deleteInnerGroups ? null : unifiedGroup.Groups); } }
internal static void LoadAllPartiallyIfPossible() { BubbleGroupIndex.Load(); BubbleGroupSettingsManager.Load(); BubbleGroupCacheManager.LoadAll(); }
public static bool LoadFullyIfNeeded(BubbleGroup group, bool sync = false) { if (@group == null) { return(false); } var loadedSomething = false; lock (BubbleGroupDatabase.OperationLock) { var unifiedGroup = @group as UnifiedBubbleGroup; var adjustUnifiedGroupUnreadIndicatorIfExists = false; var associatedGroups = unifiedGroup != null ? unifiedGroup.Groups.ToList() : new[] { @group }.ToList(); var associatedPartiallyLoadedGroups = associatedGroups.Where(x => x.PartiallyLoaded).ToList(); foreach (var partiallyLoadedGroup in associatedPartiallyLoadedGroups) { var partiallyLoadedBubblesToRemove = partiallyLoadedGroup.Bubbles.ToList(); var rollingBack = false; TryAgain: try { loadedSomething = true; partiallyLoadedGroup.PartiallyLoaded = false; foreach (var bubble in BubbleGroupDatabase.FetchBubbles(partiallyLoadedGroup).Reverse()) { partiallyLoadedGroup.Bubbles.Add(bubble); } foreach (var partiallyLoadedBubbleToRemove in partiallyLoadedBubblesToRemove) { partiallyLoadedGroup.Bubbles.Remove(partiallyLoadedBubbleToRemove); } if (partiallyLoadedGroup.Bubbles.Count < 1) { foreach (var partiallyLoadedBubbleToRemove in partiallyLoadedBubblesToRemove) { partiallyLoadedGroup.Bubbles.Add(partiallyLoadedBubbleToRemove); } } } catch (Exception ex) { Utils.DebugPrint("Failed to fully load partially loaded group " + partiallyLoadedGroup.ID + ": " + ex); if (!rollingBack) { Utils.DebugPrint("Attempting to roll back the last transaction...."); BubbleGroupSettingsManager.SetUnreadIndicatorGuid(partiallyLoadedGroup, null, false); adjustUnifiedGroupUnreadIndicatorIfExists = true; rollingBack = true; var lastModifiedIndex = BubbleGroupIndex.GetLastModifiedIndex(partiallyLoadedGroup.ID); if (lastModifiedIndex.HasValue) { try { BubbleGroupDatabase.RollBackTo(partiallyLoadedGroup, lastModifiedIndex.Value); goto TryAgain; } catch (Exception ex2) { Utils.DebugPrint("Failed to rollback: " + ex2); // fall-through. It's unrecoverable! } } else { // fall-through. It's unrecoverable! } } Utils.DebugPrint("Partially loaded group is dead. Killing and restarting (lost data occurred)."); BubbleGroupDatabase.Kill(partiallyLoadedGroup); BubbleGroupSync.RemoveFromSync(partiallyLoadedGroup); BubbleGroupDatabase.AddBubbles(partiallyLoadedGroup, partiallyLoadedBubblesToRemove.ToArray()); } } if (unifiedGroup != null && !unifiedGroup.UnifiedGroupLoaded) { Populate(unifiedGroup); } if (unifiedGroup != null && adjustUnifiedGroupUnreadIndicatorIfExists) { BubbleGroupSettingsManager.SetUnreadIndicatorGuid(unifiedGroup, null, false); } } if (!sync && loadedSomething) { Task.Factory.StartNew(() => { var unifiedGroup = @group as UnifiedBubbleGroup; if (unifiedGroup != null) { BubbleQueueManager.Send(unifiedGroup.Groups.Where(x => ServiceManager.IsRunning(x.Service)) .Select(x => x.Service.Information.ServiceName).ToArray()); } else if (ServiceManager.IsRunning(@group.Service)) { BubbleQueueManager.Send(new[] { @group.Service.Information.ServiceName }); } BubbleQueueManager.SetNotQueuedToFailures(@group); }); } return(loadedSomething); }
public static bool UpdateBubble(BubbleGroup group, VisualBubble[] bubbles, int searchDepth = 100) { //we can't operate if a thread/worker is concurrently processing a new bubble lock (OperationLock) { var groupDatabaseLocation = GetLocation(@group); var bubbleTuples = bubbles.Select(x => new Tuple <string, VisualBubble>(x.ID, x)).ToList(); VisualBubble lastBubble = null; long insertPosition = -1; using (var stream = File.Open(groupDatabaseLocation, FileMode.Open, FileAccess.ReadWrite)) { stream.Seek(stream.Length, SeekOrigin.Begin); for (var i = 0; i < (searchDepth != -1 ? searchDepth : Int32.MaxValue); i++) { if (stream.Position == 0) { break; } byte[] headerBytes; int headerBytesLength; int endPosition; BubbleGroupDatabasePrimitives.ReadBubbleHeader(stream, out headerBytes, out headerBytesLength); BubbleGroupDatabasePrimitives.FindBubbleHeaderDelimiter(headerBytes, headerBytesLength, 0, out endPosition); var bubbleDataLength = BubbleGroupDatabasePrimitives.JumpBubbleData(stream); //we need to seek over the data. var guid = BubbleGroupDatabasePrimitives.ReadBubbleHeaderStruct(headerBytes, headerBytesLength, endPosition + 1, out endPosition); Tuple <string, VisualBubble> bubbleTuple; if ((bubbleTuple = bubbleTuples.FirstOrDefault(x => x.Item1 == guid)) == null) { continue; } var b = bubbleTuple.Item2; var bubbleSize = headerBytesLength + bubbleDataLength + 8; var cutStart = stream.Position + bubbleSize; var cutLength = stream.Length - cutStart; insertPosition = stream.Position; using (var ms = new MemoryStream()) { Serializer.Serialize(ms, b); var updatedBubbleData = ms.ToArray(); var updatedBubbleHeader = b.GetType().Name + ":" + b.ID + ":" + b.Time; var updatedBubbleSize = updatedBubbleHeader.Length + updatedBubbleData.Length + 8; var bubbleInjectDelta = bubbleSize - updatedBubbleSize; //enough room if (bubbleInjectDelta != 0) { var injectPosition = stream.Position; stream.Position = cutStart; //higher var cut = new byte[cutLength]; stream.Read(cut, 0, (int)cutLength); //should always work as long as the count ain't crazy high //var bw = new BinaryWriter(stream, Encoding.ASCII); stream.Position = injectPosition; BubbleGroupDatabasePrimitives.WriteBubbleData(stream, updatedBubbleData); BubbleGroupDatabasePrimitives.WriteBubbleHeader(stream, updatedBubbleHeader); stream.Write(cut, 0, cut.Length); stream.SetLength(stream.Position); } //they're the same! else if (bubbleInjectDelta == 0) { //var bw = new BinaryWriter(stream, Encoding.ASCII); BubbleGroupDatabasePrimitives.WriteBubbleData(stream, updatedBubbleData); BubbleGroupDatabasePrimitives.WriteBubbleHeader(stream, updatedBubbleHeader); } } if (cutLength == 0) { lastBubble = b; } bubbleTuples.Remove(bubbleTuple); if (!bubbleTuples.Any()) { break; } } BubbleGroupIndex.UpdateLastBubbleOrAndLastModifiedIndex(group.ID, lastBubble, insertPosition); } if (!bubbleTuples.Any()) { return(true); } return(false); } }
public static bool InsertBubblesByTime(BubbleGroup group, VisualBubble[] bubbles, int maxDepth = 1000, bool guidCheck = false, bool insertAtTop = false) { //we can't operate if a thread/worker is concurrently processing a new bubble lock (OperationLock) { var groupDatabaseLocation = GetLocation(@group); var bubbleTuples = bubbles.Select(x => new Tuple <long, VisualBubble>(x.Time, x)).ToList(); VisualBubble lastBubble = null; long insertPosition = -1; using (var stream = File.Open(groupDatabaseLocation, FileMode.Open, FileAccess.ReadWrite)) { stream.Seek(stream.Length, SeekOrigin.Begin); for (var i = 0; i < (maxDepth != -1 ? maxDepth : Int32.MaxValue); i++) { if (stream.Position == 0) { if (insertAtTop) { insertPosition = 0; var cut = new byte[(int)stream.Length]; stream.Read(cut, 0, (int)stream.Length); stream.Position = 0; var bubblesToInsert = bubbleTuples.Select(x => x.Item2).ToList(); bubblesToInsert.TimSort((x, y) => { // IMPORTANT: Our Time field is specified in seconds, however scenarios are appearing // (e.g., bots) where messages are sent in on the same second but still require // proper ordering. In this case, Services may set a flag specifying a fallback // to the ID assigned by the Service (e.g. Telegram). if ((x.Time == y.Time) && (x.IsServiceIdSequence && y.IsServiceIdSequence)) { return(string.Compare( strA: x.IdService, strB: y.IdService, ignoreCase: false, culture: CultureInfo.InvariantCulture)); } // OK, simpler scenario, just compare the Time field for the bubbles else { return(x.Time.CompareTo(y.Time)); } }); foreach (var bubbleToInsert in bubblesToInsert) { using (var ms = new MemoryStream()) { Serializer.Serialize(ms, bubbleToInsert); var bubbleToInsertData = ms.ToArray(); var bubbleToInsertHeader = bubbleToInsert.GetType().Name + ":" + bubbleToInsert.ID + ":" + bubbleToInsert.Time; BubbleGroupDatabasePrimitives.WriteBubbleData(stream, bubbleToInsertData); BubbleGroupDatabasePrimitives.WriteBubbleHeader(stream, bubbleToInsertHeader); } } stream.Write(cut, 0, cut.Length); stream.SetLength(stream.Position); break; } else { break; } } byte[] headerBytes; int headerBytesLength; int endPosition; int bubbleDataLength; BubbleGroupDatabasePrimitives.ReadBubbleHeader(stream, out headerBytes, out headerBytesLength); BubbleGroupDatabasePrimitives.FindBubbleHeaderDelimiter(headerBytes, headerBytesLength, 0, out endPosition); //we need to seek over the data. var bubbleData = BubbleGroupDatabasePrimitives.ReadBubbleData(stream, out bubbleDataLength); var nBubble = Deserialize(bubbleData); var guid = BubbleGroupDatabasePrimitives.ReadBubbleHeaderStruct(headerBytes, headerBytesLength, endPosition + 1, out endPosition); var time = BubbleGroupDatabasePrimitives.AsciiBytesToString(headerBytes, endPosition + 1, headerBytesLength); long longTime; Int64.TryParse(time, out longTime); { var bubblesToInsert = bubbleTuples.Where(x => { if (guidCheck && guid == x.Item2.ID) { return(false); } // IMPORTANT: Our Time field is specified in seconds, however scenarios are appearing // (e.g., bots) where messages are sent in on the same second but still require // proper ordering. In this case, Services may set a flag specifying a fallback // to the ID assigned by the Service (e.g. Telegram). if ((longTime == x.Item1) && (nBubble.IsServiceIdSequence && x.Item2.IsServiceIdSequence)) { if (string.Compare( strA: nBubble.IdService, strB: x.Item2.IdService, ignoreCase: false, culture: CultureInfo.InvariantCulture) < 0) { // // Incoming bubble qualifies to be placed AFTER current bubble we are evaluating // return(true); } else { // // Incoming bubble DOES NOT qualify to be placed AFTER current bubble we are evaluating // return(false); } } // OK, simpler scenario, DOES incoming bubble qualify to be placed AFTER current bubble we are evaluating if (longTime <= x.Item1) { return(true); } else { return(false); } }).ToList(); if (!bubblesToInsert.Any()) { continue; } var bubbleSize = headerBytesLength + bubbleDataLength + 8; var insertLocation = stream.Position + bubbleSize; stream.Seek(insertLocation, SeekOrigin.Begin); var cutLength = stream.Length - insertLocation; var cut = new byte[cutLength]; stream.Read(cut, 0, (int)cutLength); //should always work as long as the count ain't crazy high stream.Seek(insertLocation, SeekOrigin.Begin); insertPosition = stream.Position; foreach (var bubbleToInsert in bubblesToInsert.Select(x => x.Item2)) { using (var ms = new MemoryStream()) { Serializer.Serialize(ms, bubbleToInsert); var bubbleToInsertData = ms.ToArray(); var bubbleToInsertHeader = bubbleToInsert.GetType().Name + ":" + bubbleToInsert.ID + ":" + bubbleToInsert.Time; BubbleGroupDatabasePrimitives.WriteBubbleData(stream, bubbleToInsertData); BubbleGroupDatabasePrimitives.WriteBubbleHeader(stream, bubbleToInsertHeader); } } stream.Write(cut, 0, cut.Length); stream.SetLength(stream.Position); // adding to end if (cut.Length == 0) { lastBubble = bubblesToInsert.Last().Item2; } foreach (var bubbleToInsert in bubblesToInsert) { bubbleTuples.Remove(bubbleToInsert); } if (!bubbleTuples.Any()) { break; } } } BubbleGroupIndex.UpdateLastBubbleOrAndLastModifiedIndex(group.ID, lastBubble, insertPosition); } if (!bubbleTuples.Any()) { return(true); } return(false); } }
public static bool InsertBubblesByTime(BubbleGroup group, VisualBubble[] bubbles, int maxDepth = 1000, bool guidCheck = false, bool insertAtTop = false) { //we can't operate if a thread/worker is concurrently processing a new bubble lock (OperationLock) { var groupDatabaseLocation = GetLocation(@group); var bubbleTuples = bubbles.Select(x => new Tuple <long, VisualBubble>(x.Time, x)).ToList(); VisualBubble lastBubble = null; long insertPosition = -1; using (var stream = File.Open(groupDatabaseLocation, FileMode.Open, FileAccess.ReadWrite)) { stream.Seek(stream.Length, SeekOrigin.Begin); for (var i = 0; i < (maxDepth != -1 ? maxDepth : Int32.MaxValue); i++) { if (stream.Position == 0) { if (insertAtTop) { insertPosition = 0; var cut = new byte[(int)stream.Length]; stream.Read(cut, 0, (int)stream.Length); stream.Position = 0; var bubblesToInsert = bubbleTuples.Select(x => x.Item2).ToList(); bubblesToInsert.TimSort((x, y) => x.Time.CompareTo(y.Time)); foreach (var bubbleToInsert in bubblesToInsert) { using (var ms = new MemoryStream()) { Serializer.Serialize(ms, bubbleToInsert); var bubbleToInsertData = ms.ToArray(); var bubbleToInsertHeader = bubbleToInsert.GetType().Name + ":" + bubbleToInsert.ID + ":" + bubbleToInsert.Time; BubbleGroupDatabasePrimitives.WriteBubbleData(stream, bubbleToInsertData); BubbleGroupDatabasePrimitives.WriteBubbleHeader(stream, bubbleToInsertHeader); } } stream.Write(cut, 0, cut.Length); stream.SetLength(stream.Position); } else { break; } } byte[] headerBytes; int headerBytesLength; int endPosition; BubbleGroupDatabasePrimitives.ReadBubbleHeader(stream, out headerBytes, out headerBytesLength); BubbleGroupDatabasePrimitives.FindBubbleHeaderDelimiter(headerBytes, headerBytesLength, 0, out endPosition); var bubbleDataLength = BubbleGroupDatabasePrimitives.JumpBubbleData(stream); //we need to seek over the data. var guid = BubbleGroupDatabasePrimitives.ReadBubbleHeaderStruct(headerBytes, headerBytesLength, endPosition + 1, out endPosition); var time = BubbleGroupDatabasePrimitives.AsciiBytesToString(headerBytes, endPosition + 1, headerBytesLength); long longTime; Int64.TryParse(time, out longTime); { var bubblesToInsert = bubbleTuples.Where(x => { if (guidCheck && guid == x.Item2.ID) { return(false); } if (longTime <= x.Item1) { return(true); } return(false); }).ToList(); if (!bubblesToInsert.Any()) { continue; } var bubbleSize = headerBytesLength + bubbleDataLength + 8; var insertLocation = stream.Position + bubbleSize; stream.Seek(insertLocation, SeekOrigin.Begin); var cutLength = stream.Length - insertLocation; var cut = new byte[cutLength]; stream.Read(cut, 0, (int)cutLength); //should always work as long as the count ain't crazy high stream.Seek(insertLocation, SeekOrigin.Begin); insertPosition = stream.Position; foreach (var bubbleToInsert in bubblesToInsert.Select(x => x.Item2)) { using (var ms = new MemoryStream()) { Serializer.Serialize(ms, bubbleToInsert); var bubbleToInsertData = ms.ToArray(); var bubbleToInsertHeader = bubbleToInsert.GetType().Name + ":" + bubbleToInsert.ID + ":" + bubbleToInsert.Time; BubbleGroupDatabasePrimitives.WriteBubbleData(stream, bubbleToInsertData); BubbleGroupDatabasePrimitives.WriteBubbleHeader(stream, bubbleToInsertHeader); } } stream.Write(cut, 0, cut.Length); stream.SetLength(stream.Position); // adding to end if (cut.Length == 0) { lastBubble = bubblesToInsert.Last().Item2; } foreach (var bubbleToInsert in bubblesToInsert) { bubbleTuples.Remove(bubbleToInsert); } if (!bubbleTuples.Any()) { break; } } } BubbleGroupIndex.UpdateLastBubbleOrAndLastModifiedIndex(group.ID, lastBubble, insertPosition); } if (!bubbleTuples.Any()) { return(true); } return(false); } }