public MultiThreadedStrategy(ILogMedia media, Encoding encoding, IRegex headerRe,
MessagesSplitterFlags splitterFlags, TextStreamPositioningParams textStreamPositioningParams,
string parentLoggingPrefix, ITraceSourceFactory traceSourceFactory)
: this(media, encoding, headerRe, splitterFlags, false, textStreamPositioningParams, parentLoggingPrefix, traceSourceFactory)
{
BytesToParsePerThread = GetBytesToParsePerThread(textStreamPositioningParams);
}
public static async Task Test(IMediaBasedReaderFactory factory, ILogMedia media, ExpectedLog expectation)
{
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (IPositionedMessagesReader reader = factory.CreateMessagesReader(new MediaBasedReaderParams(threads, media)))
{
await reader.UpdateAvailableBounds(false);
List <IMessage> msgs = new List <IMessage>();
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition)), async parser =>
{
for (; ;)
{
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
msgs.Add(msg);
}
});
expectation.StartVerification();
for (int i = 0; i < msgs.Count; ++i)
{
expectation.Verify(i, msgs[i]);
}
expectation.FinishVerification();
}
}
public static void Test(IMediaBasedReaderFactory factory, ILogMedia media, ExpectedLog expectation)
{
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (IPositionedMessagesReader reader = factory.CreateMessagesReader(new MediaBasedReaderParams(threads, media, tempFilesManager, new TraceSourceFactory())))
{
reader.UpdateAvailableBounds(false);
List <IMessage> msgs = new List <IMessage>();
using (var parser = reader.CreateParser(new CreateParserParams(reader.BeginPosition)))
{
for (; ;)
{
var msg = parser.ReadNext();
if (msg == null)
{
break;
}
msgs.Add(msg);
}
}
expectation.StartVerification();
for (int i = 0; i < msgs.Count; ++i)
{
expectation.Verify(i, msgs[i]);
}
expectation.FinishVerification();
}
}
public BaseStrategy(ILogMedia media, Encoding encoding, IRegex headerRe, TextStreamPositioningParams textStreamPositioningParams)
{
this.media = media;
this.encoding = encoding;
this.headerRe = headerRe;
this.textStreamPositioningParams = textStreamPositioningParams;
}
/// <summary>
/// 사용할 LogMedia를 추가합니다.
/// 이미 추가된 인스턴스인 경우 무시됩니다.
/// </summary>
/// <param name="media">추가할 LogMedia 인스턴스</param>
public static void AddLogger(ILogMedia media)
{
lock (_listMedia)
{
if (_listMedia.Contains(media) == false)
_listMedia.Add(media);
}
}
public MediaBasedReaderParams(ILogSourceThreads threads, ILogMedia media, ITempFilesManager tempFilesManager, MessagesReaderFlags flags = MessagesReaderFlags.None,
Settings.IGlobalSettingsAccessor settingsAccessor = null)
{
Threads = threads;
Media = media;
Flags = flags;
TempFilesManager = tempFilesManager;
SettingsAccessor = settingsAccessor ?? Settings.DefaultSettingsAccessor.Instance;
}
public MediaBasedReaderParams(ILogSourceThreadsInternal threads, ILogMedia media,
MessagesReaderFlags flags = MessagesReaderFlags.None,
Settings.IGlobalSettingsAccessor settingsAccessor = null, string parentLoggingPrefix = null)
{
Threads = threads;
Media = media;
Flags = flags;
SettingsAccessor = settingsAccessor ?? Settings.DefaultSettingsAccessor.Instance;
ParentLoggingPrefix = parentLoggingPrefix;
}
public MediaBasedReaderParams(ILogSourceThreadsInternal threads, ILogMedia media, ITempFilesManager tempFilesManager, ITraceSourceFactory traceSourceFactory, MessagesReaderFlags flags = MessagesReaderFlags.None,
Settings.IGlobalSettingsAccessor settingsAccessor = null, string parentLoggingPrefix = null)
{
Threads = threads;
Media = media;
Flags = flags;
TempFilesManager = tempFilesManager;
TraceSourceFactory = traceSourceFactory;
SettingsAccessor = settingsAccessor ?? Settings.DefaultSettingsAccessor.Instance;
ParentLoggingPrefix = parentLoggingPrefix;
}
public StreamLogProvider(
ILogProviderHost host,
ILogProviderFactory factory,
IConnectionParams connectParams,
StreamBasedFormatInfo formatInfo,
Type readerType
) :
base(host, factory, connectParams)
{
using (tracer.NewFrame)
{
tracer.Info("readerType={0}", readerType);
if (connectionParams[ConnectionParamsKeys.RotatedLogFolderPathConnectionParam] != null)
{
media = new RollingFilesMedia(
LogMedia.FileSystemImpl.Instance,
readerType,
formatInfo,
tracer,
new GenericRollingMediaStrategy(connectionParams[ConnectionParamsKeys.RotatedLogFolderPathConnectionParam]),
host.TempFilesManager,
host.TraceSourceFactory
);
}
else
{
media = new SimpleFileMedia(connectParams);
}
reader = (IPositionedMessagesReader)Activator.CreateInstance(
readerType, new MediaBasedReaderParams(this.threads, media, host.TempFilesManager, host.TraceSourceFactory,
settingsAccessor: host.GlobalSettings, parentLoggingPrefix: tracer.Prefix), formatInfo);
ITimeOffsets initialTimeOffset;
if (LogJoint.TimeOffsets.TryParse(
connectionParams[ConnectionParamsKeys.TimeOffsetConnectionParam] ?? "", out initialTimeOffset))
{
reader.TimeOffsets = initialTimeOffset;
}
StartAsyncReader("Reader thread: " + connectParams.ToString(), reader);
InitPathDependentMembers(connectParams);
}
}
internal MultiThreadedStrategy(ILogMedia media, Encoding encoding, IRegex headerRe, MessagesSplitterFlags splitterFlags, bool useMockThreading, TextStreamPositioningParams textStreamPositioningParams)
: base(media, encoding, headerRe, textStreamPositioningParams)
{
this.streamDataPool = new ThreadSafeObjectPool <Byte[]>(pool =>
{
var ret = new Byte[BytesToParsePerThread];
tracer.Info("Allocating new piece of stream data: {0}", ret.GetHashCode());
return(ret);
});
this.outputBuffersPool = new ThreadSafeObjectPool <List <PostprocessedMessage> >(pool =>
{
var ret = new List <PostprocessedMessage>(1024 * 8);
tracer.Info("Allocating new output buffer: {0}", ret.GetHashCode());
return(ret);
});
this.useMockThreading = useMockThreading;
this.splitterFlags = splitterFlags;
}
public StreamLogProvider(
ILogProviderHost host,
ILogProviderFactory factory,
IConnectionParams connectParams,
Func <MediaBasedReaderParams, IPositionedMessagesReader> readerCreator
) :
base(host, factory, connectParams)
{
using (tracer.NewFrame)
{
StartAsyncReader(async() => {
if (connectionParams[ConnectionParamsKeys.RotatedLogFolderPathConnectionParam] != null)
{
media = new RollingFilesMedia(
host.FileSystem,
readerCreator,
tracer,
new GenericRollingMediaStrategy(
connectionParams[ConnectionParamsKeys.RotatedLogFolderPathConnectionParam],
ConnectionParamsUtils.GetRotatedLogPatterns(connectParams)
)
);
}
else
{
media = await SimpleFileMedia.Create(host.FileSystem, connectParams);
}
reader = readerCreator(new MediaBasedReaderParams(this.threads, media,
settingsAccessor: host.GlobalSettings, parentLoggingPrefix: tracer.Prefix));
ITimeOffsets initialTimeOffset;
if (LogJoint.TimeOffsets.TryParse(
connectionParams[ConnectionParamsKeys.TimeOffsetConnectionParam] ?? "", out initialTimeOffset))
{
reader.TimeOffsets = initialTimeOffset;
}
return(reader);
});
InitPathDependentMembers(connectParams);
}
}
public Parser(ILogMedia media)
{
media.DataStream.Position = 0;
TextReader r = new StreamReader(media.DataStream);
int val = int.Parse(r.ReadToEnd().Trim());
if (val == EmptyFileContent)
{
messageRead = true;
}
else if (val == InvalidFileContent)
{
throw new InvalidFormatException();
}
else
{
time = startOfTime.Add(TimeSpan.FromHours(val));
}
}
public static void Test(IMediaBasedReaderFactory factory, ILogMedia media, ExpectedLog expectation)
{
using (ILogSourceThreads threads = new LogSourceThreads())
using (IPositionedMessagesReader reader = factory.CreateMessagesReader(new MediaBasedReaderParams(threads, media, tempFilesManager)))
{
reader.UpdateAvailableBounds(false);
List <IMessage> msgs = new List <IMessage>();
using (var parser = reader.CreateParser(new CreateParserParams(reader.BeginPosition)))
{
for (; ;)
{
var msg = parser.ReadNext();
if (msg == null)
{
break;
}
msgs.Add(msg);
}
}
expectation.StartVerification();
int frameLevel = 0;
for (int i = 0; i < msgs.Count; ++i)
{
switch (msgs[i].Flags & MessageFlag.TypeMask)
{
case MessageFlag.StartFrame:
++frameLevel;
break;
case MessageFlag.EndFrame:
--frameLevel;
break;
}
expectation.Verify(i, msgs[i], frameLevel);
}
expectation.FinishVerification();
}
}
internal MediaBasedPositionedMessagesReader(
ILogMedia media,
BoundFinder beginFinder,
BoundFinder endFinder,
MessagesReaderExtensions.XmlInitializationParams extensionsInitData,
TextStreamPositioningParams textStreamPositioningParams,
MessagesReaderFlags flags,
Settings.IGlobalSettingsAccessor settingsAccessor
)
{
this.beginFinder = beginFinder;
this.endFinder = endFinder;
this.media = media;
this.textStreamPositioningParams = textStreamPositioningParams;
this.singleThreadedStrategy = new Lazy <BaseStrategy>(CreateSingleThreadedStrategy);
this.multiThreadedStrategy = new Lazy <BaseStrategy>(CreateMultiThreadedStrategy);
this.extensions = new MessagesReaderExtensions(this, extensionsInitData);
this.flags = flags;
this.settingsAccessor = settingsAccessor;
}
internal MultiThreadedStrategy(ILogMedia media, Encoding encoding, IRegex headerRe, MessagesSplitterFlags splitterFlags,
bool useMockThreading, TextStreamPositioningParams textStreamPositioningParams, string parentLoggingPrefix, ITraceSourceFactory traceSourceFactory)
: base(media, encoding, headerRe, textStreamPositioningParams)
{
if (parentLoggingPrefix != null)
{
this.tracer = traceSourceFactory.CreateTraceSource("LogSource", string.Format("{0}.mts_{1:x4}", parentLoggingPrefix, Hashing.GetShortHashCode(this.GetHashCode())));
}
this.streamDataPool = new ThreadSafeObjectPool <Byte[]>(pool =>
{
var ret = new Byte[BytesToParsePerThread];
tracer.Info("Allocating new piece of stream data: {0:x8}", ret.GetHashCode());
return(ret);
});
this.outputBuffersPool = new ThreadSafeObjectPool <List <PostprocessedMessage> >(pool =>
{
var ret = new List <PostprocessedMessage>(1024 * 8);
tracer.Info("Allocating new output buffer: {0:x8}", ret.GetHashCode());
return(ret);
});
this.useMockThreading = useMockThreading;
this.splitterFlags = splitterFlags;
}
async Task <PreprocessingStepParams> ExecuteInternal(IPreprocessingStepCallback callback)
{
await callback.BecomeLongRunning();
string factoryName = @params.Argument;
callback.TempFilesCleanupList.Add(@params.Location);
Action <double?> setStepDescription = prctComplete =>
{
var str = new StringBuilder();
str.Append(@params.FullPath);
str.Append(": fixing timestamp anomalies...");
if (prctComplete != null)
{
str.AppendFormat(" {0}%", (int)(prctComplete.Value * 100));
}
callback.SetStepDescription(str.ToString());
};
setStepDescription(null);
string tmpFileName = callback.TempFilesManager.GenerateNewName();
var factoryNameSplit = factoryName.Split('\\');
if (factoryNameSplit.Length != 2)
{
throw new InvalidFormatException();
}
var factory = logProviderFactoryRegistry.Find(factoryNameSplit[0], factoryNameSplit[1]);
if (factory == null)
{
throw new InvalidDataException("factory not found: " + factoryName);
}
var readerFactory = factory as IMediaBasedReaderFactory;
if (readerFactory == null)
{
throw new InvalidDataException("bad factory: " + factoryName);
}
using (ILogMedia fileMedia = await SimpleFileMedia.Create(fileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(@params.Location)))
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (var reader = readerFactory.CreateMessagesReader(
new MediaBasedReaderParams(threads, fileMedia)))
{
var readerImpl = reader as MediaBasedPositionedMessagesReader; // todo: do not use real classes; have stream encoding in an interface.
if (readerImpl == null)
{
throw new InvalidDataException("bad reader was made by factory " + factoryName);
}
await reader.UpdateAvailableBounds(false);
var range = new FileRange.Range(reader.BeginPosition, reader.EndPosition);
double rangeLen = range.Length;
using (var progress = progressAggregator.CreateProgressSink())
using (var writer = new StreamWriter(tmpFileName, false, readerImpl.StreamEncoding))
await DisposableAsync.Using(await reader.CreateParser(new CreateParserParams(reader.BeginPosition,
flags: MessagesParserFlag.DisableDejitter | MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading)), async parser =>
{
var queue = new VCSKicksCollection.PriorityQueue <IMessage>(
new MessagesComparer(ignoreConnectionIds: true));
Action dequeue = () => writer.WriteLine(queue.Dequeue().RawText.ToString());
double lastPrctComplete = 0;
var cancellation = callback.Cancellation;
for (long msgIdx = 0;; ++msgIdx)
{
if (cancellation.IsCancellationRequested)
{
break;
}
var msg = await parser.ReadNext();
if (msg == null)
{
break;
}
if ((msgIdx % progressUpdateThreshold) == 0 && rangeLen > 0)
{
var prctComplete = (double)(msg.Position - range.Begin) / rangeLen;
progress.SetValue(prctComplete);
if (prctComplete - lastPrctComplete > 0.05)
{
setStepDescription(prctComplete);
lastPrctComplete = prctComplete;
}
}
queue.Enqueue(msg);
if (queue.Count > queueSize)
{
dequeue();
}
}
while (queue.Count > 0)
{
dequeue();
}
});
}
return(new PreprocessingStepParams(
tmpFileName,
@params.FullPath + " (reordered)",
@params.PreprocessingHistory.Add(new PreprocessingHistoryItem(name, factoryName))
));
}
public SingleThreadedStrategy(ILogMedia media, Encoding encoding, IRegex headerRe, MessagesSplitterFlags splitterFlags, TextStreamPositioningParams textStreamPositioningParams)
: base(media, encoding, headerRe, textStreamPositioningParams)
{
this.textSplitter = new ReadMessageFromTheMiddleProblem(new MessagesSplitter(new StreamTextAccess(media.DataStream, encoding, textStreamPositioningParams), headerRe, splitterFlags));
}
protected override async Task LiveLogListen(CancellationToken stopEvt, LiveLogXMLWriter output)
{
using (ILogMedia media = await SimpleFileMedia.Create(
host.FileSystem,
SimpleFileMedia.CreateConnectionParamsFromFileName(fileName)))
using (FileSystemWatcher watcher = new FileSystemWatcher(Path.GetDirectoryName(fileName),
Path.GetFileName(fileName)))
using (AutoResetEvent fileChangedEvt = new AutoResetEvent(true))
{
IMessagesSplitter splitter = new MessagesSplitter(
new StreamTextAccess(media.DataStream, Encoding.ASCII, TextStreamPositioningParams.Default),
host.RegexFactory.Create(@"^(?<body>.+)$", ReOptions.Multiline)
);
watcher.NotifyFilter = NotifyFilters.LastWrite | NotifyFilters.Size;
watcher.Changed += delegate(object sender, FileSystemEventArgs e)
{
fileChangedEvt.Set();
};
//watcher.EnableRaisingEvents = true;
long lastLinePosition = 0;
long lastStreamLength = 0;
WaitHandle[] events = new WaitHandle[] { stopEvt.WaitHandle, fileChangedEvt };
var capture = new TextMessageCapture();
for (; ;)
{
if (WaitHandle.WaitAny(events, 250, false) == 0)
{
break;
}
await media.Update();
if (media.Size == lastStreamLength)
{
continue;
}
lastStreamLength = media.Size;
sizeInBytesStat = lastStreamLength;
DateTime lastModified = media.LastModified;
await splitter.BeginSplittingSession(new FileRange.Range(0, lastStreamLength), lastLinePosition, MessagesParserDirection.Forward);
try
{
for (; ;)
{
if (!await splitter.GetCurrentMessageAndMoveToNextOne(capture))
{
break;
}
lastLinePosition = capture.BeginPosition;
XmlWriter writer = output.BeginWriteMessage(false);
writer.WriteStartElement("m");
writer.WriteAttributeString("d", Listener.FormatDate(lastModified));
writer.WriteString(XmlUtils.RemoveInvalidXMLChars(capture.MessageHeader));
writer.WriteEndElement();
output.EndWriteMessage();
}
}
finally
{
splitter.EndSplittingSession();
}
}
}
}
public MessagesReader(MediaBasedReaderParams readerParams, object fmtInfo)
{
this.media = readerParams.Media;
}
public MultiThreadedStrategy(ILogMedia media, Encoding encoding, IRegex headerRe, MessagesSplitterFlags splitterFlags, TextStreamPositioningParams textStreamPositioningParams)
: this(media, encoding, headerRe, splitterFlags, false, textStreamPositioningParams)
{
BytesToParsePerThread = GetBytesToParsePerThread(textStreamPositioningParams);
}
/// <summary>
/// LogMedia 목록에서 지정한 LogMedia를 제거합니다.
/// 정상적으로 제거된 경우 해당 LogMedia의 Release가 호출됩니다.
/// </summary>
/// <param name="media">제거할 LogMedia 인스턴스</param>
public static void RemoveLogger(ILogMedia media)
{
lock (_listMedia)
{
if (_listMedia.Remove(media) == true)
media.Release();
}
}
