Type CompileUserCodeToTypeInternal(CompilationTargetFx targetFx, Func <string, string> assemblyLocationResolver)
{
using (CSharpCodeProvider prov = new CSharpCodeProvider())
using (var perfop = new Profiling.Operation(trace, "compile user code"))
{
List <string> refs = new List <string>();
string fullCode = MakeMessagesBuilderCode(inputFieldNames, extensions, outputFields, refs);
CompilerParameters cp = new CompilerParameters();
if (targetFx == CompilationTargetFx.RunningFx)
{
cp.ReferencedAssemblies.Add("System.dll");
cp.CompilerOptions = "/optimize";
}
else if (targetFx == CompilationTargetFx.Silverlight)
{
var silverLightDir = GetSilverlightDir();
foreach (var silverlightAsm in new string[] { "mscorlib.dll", "System.Core.dll", "System.dll" })
{
cp.ReferencedAssemblies.Add(silverLightDir + silverlightAsm);
}
cp.CompilerOptions = "/optimize /nostdlib+";
}
List <string> resolvedRefs = new List <string>();
resolvedRefs.Add(assemblyLocationResolver(Assembly.GetExecutingAssembly().FullName));
resolvedRefs.Add(assemblyLocationResolver(typeof(StringSlice).Assembly.FullName));
foreach (string refAsm in refs)
{
resolvedRefs.Add(assemblyLocationResolver(refAsm));
}
foreach (string resoledAsm in resolvedRefs)
{
cp.ReferencedAssemblies.Add(resoledAsm);
}
string tempDir = tempFilesManager.GenerateNewName();
Directory.CreateDirectory(tempDir);
cp.OutputAssembly = string.Format("{0}{2}UserCode{1}.dll", tempDir, Guid.NewGuid().ToString("N"), Path.DirectorySeparatorChar);
// Temp folder will be cleaned when LogJoint starts next time.
cp.TempFiles = new TempFileCollection(tempDir, false);
cp.TreatWarningsAsErrors = false;
CompilerResults cr;
using (new CodedomEnvironmentConfigurator())
{
perfop.Milestone("started compile");
cr = prov.CompileAssemblyFromSource(cp, fullCode);
}
if (cr.Errors.HasErrors)
{
ThrowBadUserCodeException(fullCode, cr);
}
perfop.Milestone("getting type");
return(cr.CompiledAssembly.GetType("GeneratedMessageBuilder"));
}
}
public Command(
CommandType t,
LogProviderCommandPriority priority,
LJTraceSource trace,
CancellationToken cancellation,
IAsyncLogProviderCommandHandler handler)
{
Type = t;
Id = Interlocked.Increment(ref lastCommandId);
Priority = priority;
Cancellation = cancellation;
Handler = handler;
Perfop = new Profiling.Operation(trace, this.ToString());
}
void FillCacheRanges(CancellationToken cancellationToken)
{
using (tracer.NewFrame)
using (var perfop = new Profiling.Operation(tracer, "FillRanges"))
{
bool updateStarted = false;
// Iterate through the ranges
for (; ;)
{
cancellationToken.ThrowIfCancellationRequested();
currentRange = buffer.GetNextRangeToFill();
if (currentRange == null) // Nothing to fill
{
break;
}
currentMessagesContainer = buffer;
tracer.Info("currentRange={0}", currentRange);
bool failed = false;
try
{
if (!updateStarted)
{
tracer.Info("Starting to update the messages.");
updateStarted = true;
}
long messagesRead = 0;
ResetFlags();
// Start reading elements
using (IPositionedMessagesParser parser = reader.CreateParser(new CreateParserParams(
currentRange.GetPositionToStartReadingFrom(), currentRange.DesirableRange,
MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading,
MessagesParserDirection.Forward,
postprocessor: null,
cancellation: cancellationToken)))
{
tracer.Info("parser created");
for (; ;)
{
cancellationToken.ThrowIfCancellationRequested();
ResetFlags();
if (!ReadNextMessage(parser))
{
cancellationToken.ThrowIfCancellationRequested();
break;
}
ProcessLastReadMessageAndFlush();
++messagesRead;
}
}
tracer.Info("reading finished");
FlushBuffer();
}
catch
{
failed = true;
throw;
}
finally
{
if (!failed)
{
tracer.Info("Loading of the range finished successfully. Completing the range.");
currentRange.Complete();
tracer.Info("Disposing the range.");
}
else
{
tracer.Info("Loading failed. Disposing the range without completion.");
}
currentRange.Dispose();
currentRange = null;
currentMessagesContainer = null;
perfop.Milestone("range completed");
}
}
UpdateLoadedTimeStats(reader);
if (updateStarted)
{
perfop.Milestone("great success");
owner.SetMessagesCache(new AsyncLogProviderDataCache()
{
Messages = new MessagesContainers.ListBasedCollection(
buffer.Forward(0, int.MaxValue).Select(m => m.Message)),
MessagesRange = buffer.ActiveRange,
});
}
}
}
public void Complete()
{
Perfop.Dispose();
Perfop = Profiling.Operation.Null;
}
void FillCacheRanges(CancellationToken preemptionToken)
{
using (tracer.NewFrame)
using (var perfop = new Profiling.Operation(tracer, "FillRanges"))
{
bool updateStarted = false;
try
{
// Iterate through the ranges
for (; ;)
{
currentRange = buffer.GetNextRangeToFill();
if (currentRange == null) // Nothing to fill
{
break;
}
currentMessagesContainer = buffer;
tracer.Info("currentRange={0}", currentRange);
try
{
if (!updateStarted)
{
tracer.Info("Starting to update the messages.");
updateStarted = true;
}
long messagesRead = 0;
ResetFlags();
// Start reading elements
using (IPositionedMessagesParser parser = reader.CreateParser(new CreateParserParams(
currentRange.GetPositionToStartReadingFrom(), currentRange.DesirableRange,
MessagesParserFlag.HintParserWillBeUsedForMassiveSequentialReading,
MessagesParserDirection.Forward)))
{
tracer.Info("parser created");
for (; ;)
{
if (preemptionToken.IsCancellationRequested)
{
loadingInterrupted = true;
preemptionToken.ThrowIfCancellationRequested();
}
ResetFlags();
ReadNextMessage(parser);
ProcessLastReadMessageAndFlush();
ReportLoadErrorIfAny();
if (breakAlgorithm)
{
break;
}
++messagesRead;
}
}
tracer.Info("reading finished");
FlushBuffer();
}
finally
{
if (!loadingInterrupted)
{
tracer.Info("Loading of the range finished completly. Completing the range.");
currentRange.Complete();
tracer.Info("Disposing the range.");
}
else
{
tracer.Info("Loading was interrupted. Disposing the range without completion.");
}
currentRange.Dispose();
currentRange = null;
currentMessagesContainer = null;
perfop.Milestone("range completed");
}
if (loadingInterrupted)
{
tracer.Info("Loading interrupted. Stopping to read ranges.");
break;
}
}
UpdateLoadedTimeStats(reader);
}
catch (Exception e)
{
tracer.Error(e, "failed to update cache");
loadingInterrupted = true;
}
finally
{
if (updateStarted)
{
tracer.Info("Finishing to update the messages.");
if (!loadingInterrupted)
{
owner.SetMessagesCache(new AsyncLogProviderDataCache()
{
Messages = new MessagesContainers.ListBasedCollection(
buffer.Forward(0, int.MaxValue).Select(m => m.Message)),
MessagesRange = buffer.ActiveRange,
AvailableRange = currentStats.PositionsRange,
AvailableTime = currentStats.AvailableTime,
});
}
}
else
{
tracer.Info("There were no ranges to read");
}
}
}
}
static async Task <DetectedFormat> DetectFormat(
string fileName,
string loggableName,
Func <ILogProviderFactory, int> mruIndexGetter,
ILogProviderFactoryRegistry factoriesRegistry,
CancellationToken cancellation,
IFormatAutodetectionProgress progress,
ITraceSourceFactory traceSourceFactory,
LogMedia.IFileSystem fileSystem)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentException("fileName");
}
if (mruIndexGetter == null)
{
throw new ArgumentNullException("mru");
}
Func <Task <SimpleFileMedia> > createFileMedia = () => SimpleFileMedia.Create(fileSystem, SimpleFileMedia.CreateConnectionParamsFromFileName(fileName));
var log = traceSourceFactory.CreateTraceSource("App", string.Format("fdtc.{0}", Interlocked.Increment(ref lastPerfOp)));
using (new Profiling.Operation(log, string.Format("format detection of {0}", loggableName)))
using (ILogSourceThreadsInternal threads = new LogSourceThreads())
using (var localCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellation))
{
var candidateFactories = GetOrderedListOfRelevantFactories(fileName, mruIndexGetter, factoriesRegistry).ToArray();
var ret = (await Task.WhenAll(candidateFactories.Select((factory, index) => (factory, index)).Select(async candidate =>
{
var(factory, idx) = candidate;
try
{
using (var perfOp = new Profiling.Operation(log, factory.ToString()))
using (var fileMedia = await createFileMedia())
using (var reader = ((IMediaBasedReaderFactory)factory).CreateMessagesReader(
new MediaBasedReaderParams(threads, fileMedia,
MessagesReaderFlags.QuickFormatDetectionMode, parentLoggingPrefix: log.Prefix)))
{
if (progress != null)
{
progress.Trying(factory);
}
if (localCancellation.IsCancellationRequested)
{
perfOp.Milestone("cancelled");
return(fmt: (DetectedFormat)null, idx);
}
await reader.UpdateAvailableBounds(false);
perfOp.Milestone("bounds detected");
var parser = await reader.CreateParser(new CreateParserParams(0, null,
MessagesParserFlag.DisableMultithreading | MessagesParserFlag.DisableDejitter, MessagesParserDirection.Forward));
try
{
if (await parser.ReadNext() != null)
{
log.Info("Autodetected format of {0}: {1}", fileName, factory);
localCancellation.Cancel();
return(fmt: new DetectedFormat(factory, ((IFileBasedLogProviderFactory)factory).CreateParams(fileName)), idx);
}
}
finally
{
await parser.Dispose();
}
}
}
catch (Exception e)
{
log.Error(e, "Failed to load '{0}' as {1}", fileName, factory);
}
return(fmt: (DetectedFormat)null, idx);
}))).Where(x => x.fmt != null).OrderBy(x => x.idx).Select(x => x.fmt).FirstOrDefault();
if (ret != null)
{
return(ret);
}
using (var fileMedia = await createFileMedia())
{
if (!await IOUtils.IsBinaryFile(fileMedia.DataStream))
{
log.Info("File does not look binary");
var factory = factoriesRegistry.Find(
PlainText.Factory.CompanyName, PlainText.Factory.FormatName) as IFileBasedLogProviderFactory;
if (factory != null)
{
log.Info("Fall back to plaintext format");
return(new DetectedFormat(factory, factory.CreateParams(fileName)));
}
}
}
}
return(null);
}
static DetectedFormat DetectFormat(
string fileName,
string loggableName,
Func <ILogProviderFactory, int> mruIndexGetter,
ILogProviderFactoryRegistry factoriesRegistry,
CancellationToken cancellation,
IFormatAutodetectionProgress progress,
ITempFilesManager tempFilesManager)
{
if (string.IsNullOrEmpty(fileName))
{
throw new ArgumentException("fileName");
}
if (mruIndexGetter == null)
{
throw new ArgumentNullException("mru");
}
Func <SimpleFileMedia> createFileMedia = () => new SimpleFileMedia(SimpleFileMedia.CreateConnectionParamsFromFileName(fileName));
var log = new LJTraceSource("App", string.Format("fdtc.{0}", Interlocked.Increment(ref lastPerfOp)));
using (new Profiling.Operation(log, string.Format("format detection of {0}", loggableName)))
using (ILogSourceThreads threads = new LogSourceThreads())
using (var localCancellation = CancellationTokenSource.CreateLinkedTokenSource(cancellation))
{
var ret = GetOrderedListOfRelevantFactories(fileName, mruIndexGetter, factoriesRegistry).AsParallel().Select(factory =>
{
try
{
using (var perfOp = new Profiling.Operation(log, factory.ToString()))
using (var fileMedia = createFileMedia())
using (var reader = ((IMediaBasedReaderFactory)factory).CreateMessagesReader(
new MediaBasedReaderParams(threads, fileMedia, tempFilesManager, MessagesReaderFlags.QuickFormatDetectionMode, parentLoggingPrefix: log.Prefix)))
{
if (progress != null)
{
progress.Trying(factory);
}
if (localCancellation.IsCancellationRequested)
{
perfOp.Milestone("cancelled");
return(null);
}
reader.UpdateAvailableBounds(false);
perfOp.Milestone("bounds detected");
using (var parser = reader.CreateParser(new CreateParserParams(0, null,
MessagesParserFlag.DisableMultithreading | MessagesParserFlag.DisableDejitter, MessagesParserDirection.Forward)))
{
if (parser.ReadNext() != null)
{
log.Info("Autodetected format of {0}: {1}", fileName, factory);
localCancellation.Cancel();
return(new DetectedFormat(factory, ((IFileBasedLogProviderFactory)factory).CreateParams(fileName)));
}
}
}
}
catch (Exception e)
{
log.Error(e, "Failed to load '{0}' as {1}", fileName, factory);
}
return(null);
}).FirstOrDefault(x => x != null);
if (ret != null)
{
return(ret);
}
using (var fileMedia = createFileMedia())
{
if (!IOUtils.IsBinaryFile(fileMedia.DataStream))
{
log.Info("File does not look binary");
var factory = factoriesRegistry.Find(
PlainText.Factory.CompanyName, PlainText.Factory.FormatName) as IFileBasedLogProviderFactory;
if (factory != null)
{
log.Info("Fall back to plaintext format");
return(new DetectedFormat(factory, factory.CreateParams(fileName)));
}
}
}
}
return(null);
}