Example #1
0
        public static async Task SerializePostprocessorOutput(
            Task <ILogPartToken> logPartToken,
            ILogPartTokenFactories logPartTokenFactories,
            IEnumerableAsync <M.Event[]> events,
            Task <ISameNodeDetectionToken> sameNodeDetectionTokenTask,
            ISameNodeDetectionTokenFactories nodeDetectionTokenFactories,
            Func <object, TextLogEventTrigger> triggersConverter,
            string contentsEtagAttr,
            Func <Task <Stream> > openOutputStream,
            ITempFilesManager tempFiles,
            CancellationToken cancellation
            )
        {
            events       = events ?? new List <M.Event[]>().ToAsync();
            logPartToken = logPartToken ?? Task.FromResult <ILogPartToken>(null);
            sameNodeDetectionTokenTask = sameNodeDetectionTokenTask ?? Task.FromResult <ISameNodeDetectionToken>(null);

            var eventsTmpFile = tempFiles.GenerateNewName();

            Func <Task <Stream> > openTempFile(string fileName) => () => Task.FromResult <Stream>(new FileStream(fileName, FileMode.OpenOrCreate));

            var serializeMessagingEvents = events.SerializePostprocessorOutput <M.Event, M.EventsSerializer, M.IEventsVisitor>(
                triggerSerializer => new M.EventsSerializer(triggerSerializer),
                null, logPartTokenFactories, triggersConverter, null, messagingEventsElementName, openTempFile(eventsTmpFile), tempFiles, cancellation
                );

            await Task.WhenAll(serializeMessagingEvents, logPartToken, sameNodeDetectionTokenTask);

            using (var outputWriter = XmlWriter.Create(await openOutputStream(), new XmlWriterSettings()
            {
                Indent = true, Async = true, CloseOutput = true
            }))
                using (var messagingEventsReader = XmlReader.Create(eventsTmpFile))
                {
                    outputWriter.WriteStartElement("root");

                    new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
                    logPartTokenFactories.SafeWriteTo(await logPartToken, outputWriter);
                    nodeDetectionTokenFactories.SafeWriteTo(await sameNodeDetectionTokenTask, outputWriter);

                    messagingEventsReader.ReadToFollowing(messagingEventsElementName);
                    await outputWriter.WriteNodeAsync(messagingEventsReader, false);

                    outputWriter.WriteEndElement();             // root
                }

            File.Delete(eventsTmpFile);
        }
        public static async Task SerializePostprocessorOutput <Evt, Serializer, EvtVisitor>(
            this IEnumerableAsync <Evt[]> events,
            Func <Action <object, XElement>, Serializer> serializerFactory,
            Task <ILogPartToken> rotatedLogPartToken,
            ILogPartTokenFactories rotatedLogPartFactories,
            Func <object, TextLogEventTrigger> triggersConverter,
            string contentsEtagAttr,
            string rootElementName,
            string outputFileName,
            ITempFilesManager tempFiles,
            CancellationToken cancellation
            ) where Evt : IVisitable <EvtVisitor> where Serializer : class, IEventsSerializer, EvtVisitor
        {
            rotatedLogPartToken = rotatedLogPartToken ?? Task.FromResult <ILogPartToken>(null);
            var        sortKeyAttr     = XName.Get("__key");
            var        chunks          = new List <string>();
            Serializer serializer      = null;
            Action     resetSerializer = () =>
            {
                if (serializer?.Output?.Count > 0)
                {
                    string chunkFileName = tempFiles.GenerateNewName();
                    chunks.Add(chunkFileName);
                    using (var writer = XmlWriter.Create(chunkFileName, new XmlWriterSettings()
                    {
                        OmitXmlDeclaration = true,
                        ConformanceLevel = ConformanceLevel.Fragment
                    }))
                    {
                        foreach (var e in serializer.Output.OrderBy(e => e.Attribute(sortKeyAttr).Value))
                        {
                            e.WriteTo(writer);
                        }
                    }
                }
                serializer = serializerFactory((trigger, elt) =>
                {
                    triggersConverter(trigger).Save(elt);
                    elt.SetAttributeValue(sortKeyAttr, ((IOrderedTrigger)trigger).Index.ToString("x8"));
                });
            };

            resetSerializer();
            await events.ForEach(batch =>
            {
                foreach (var e in batch)
                {
                    e.Visit(serializer);
                    if (serializer.Output.Count >= 8 * 1024)
                    {
                        resetSerializer();
                    }
                }
                return(Task.FromResult(!cancellation.IsCancellationRequested));
            });

            resetSerializer();

            if (cancellation.IsCancellationRequested)
            {
                return;
            }

            using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
            {
                Indent = true
            }))
            {
                outputWriter.WriteStartElement(rootElementName);
                new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
                rotatedLogPartFactories.SafeWriteTo(await rotatedLogPartToken, outputWriter);
                var readersSettings = new XmlReaderSettings()
                {
                    ConformanceLevel = ConformanceLevel.Fragment
                };
                var readers = chunks.Select(chunkFileName => XmlReader.Create(chunkFileName, readersSettings)).ToList();
                try
                {
                    var q = new VCSKicksCollection.PriorityQueue <KeyValuePair <XmlReader, XElement> >(Comparer <KeyValuePair <XmlReader, XElement> > .Create((item1, item2) =>
                    {
                        return(string.CompareOrdinal(item1.Value.Attribute(sortKeyAttr).Value, item2.Value.Attribute(sortKeyAttr).Value));
                    }));
                    Action <XmlReader> enqueueReader = reader =>
                    {
                        if (!reader.EOF)
                        {
                            if (reader.MoveToContent() != XmlNodeType.Element)
                            {
                                throw new InvalidOperationException("bad chunk");
                            }
                            q.Enqueue(new KeyValuePair <XmlReader, XElement>(reader, (XElement)XNode.ReadFrom(reader)));
                        }
                    };
                    readers.ForEach(enqueueReader);
                    while (q.Count > 0)
                    {
                        var item = q.Dequeue();
                        item.Value.Attribute(sortKeyAttr).Remove();
                        item.Value.WriteTo(outputWriter);
                        enqueueReader(item.Key);
                    }
                }
                finally
                {
                    readers.ForEach(r => r.Dispose());
                    chunks.ForEach(chunkFileName => tempFiles.DeleteIfTemporary(chunkFileName));
                }
                outputWriter.WriteEndElement();                 // end of root node
            }
        }
Example #3
0
        public static async Task SerializePostprocessorOutput(
            IEnumerableAsync <M.Event[]> events,
            IEnumerableAsync <TLBlock.Event[]> timelineComments,
            IEnumerableAsync <SIBlock.Event[]> stateInspectorComments,
            Task <ILogPartToken> logPartToken,
            ILogPartTokenFactories logPartTokenFactories,
            Func <object, TextLogEventTrigger> triggersConverter,
            string contentsEtagAttr,
            string outputFileName,
            ITempFilesManager tempFiles,
            CancellationToken cancellation
            )
        {
            events                 = events ?? new List <M.Event[]>().ToAsync();
            timelineComments       = timelineComments ?? new List <TLBlock.Event[]>().ToAsync();
            stateInspectorComments = stateInspectorComments ?? new List <SIBlock.Event[]>().ToAsync();
            logPartToken           = logPartToken ?? Task.FromResult <ILogPartToken>(null);

            var eventsTmpFile                = tempFiles.GenerateNewName();
            var timelineCommentsTmpFile      = tempFiles.GenerateNewName();
            var stateInsectorCommentsTmpFile = tempFiles.GenerateNewName();

            var serializeMessagingEvents = events.SerializePostprocessorOutput <M.Event, M.EventsSerializer, M.IEventsVisitor>(
                triggerSerializer => new M.EventsSerializer(triggerSerializer),
                null, logPartTokenFactories, triggersConverter, null, messagingEventsElementName, eventsTmpFile, tempFiles, cancellation
                );

            var serializeTimelineComments = timelineComments.SerializePostprocessorOutput <TLBlock.Event, TLBlock.EventsSerializer, TLBlock.IEventsVisitor>(
                triggerSerializer => new TLBlock.EventsSerializer(triggerSerializer),
                null, logPartTokenFactories, triggersConverter, null, timelineCommentsElementName, timelineCommentsTmpFile, tempFiles, cancellation
                );

            var serializeStateInspectorComments = stateInspectorComments.SerializePostprocessorOutput <SIBlock.Event, SIBlock.EventsSerializer, SIBlock.IEventsVisitor>(
                triggerSerializer => new SIBlock.EventsSerializer(triggerSerializer),
                null, logPartTokenFactories, triggersConverter, null, stateCommentsElementName, stateInsectorCommentsTmpFile, tempFiles, cancellation
                );

            await Task.WhenAll(serializeMessagingEvents, serializeTimelineComments, serializeStateInspectorComments, logPartToken);

            using (var outputWriter = XmlWriter.Create(outputFileName, new XmlWriterSettings()
            {
                Indent = true, Async = true
            }))
                using (var messagingEventsReader = XmlReader.Create(eventsTmpFile))
                    using (var timelineCommentsReader = XmlReader.Create(timelineCommentsTmpFile))
                        using (var stateInspectorCommentsReader = XmlReader.Create(stateInsectorCommentsTmpFile))
                        {
                            outputWriter.WriteStartElement("root");

                            new PostprocessorOutputETag(contentsEtagAttr).Write(outputWriter);
                            logPartTokenFactories.SafeWriteTo(await logPartToken, outputWriter);

                            messagingEventsReader.ReadToFollowing(messagingEventsElementName);
                            await outputWriter.WriteNodeAsync(messagingEventsReader, false);

                            timelineCommentsReader.ReadToFollowing(timelineCommentsElementName);
                            await outputWriter.WriteNodeAsync(timelineCommentsReader, false);

                            stateInspectorCommentsReader.ReadToFollowing(stateCommentsElementName);
                            await outputWriter.WriteNodeAsync(stateInspectorCommentsReader, false);

                            outputWriter.WriteEndElement();     // root
                        }

            File.Delete(eventsTmpFile);
            File.Delete(timelineCommentsTmpFile);
            File.Delete(stateInsectorCommentsTmpFile);
        }