public FilteredLogReader(LogStreamerSession logStreamer, Dictionary<string, ulong> tablePos, HashSet<string> tableFilter)
{
_logStreamer = logStreamer;
// make a copy of the table position dictionary, stripping out prefix
if (tablePos != null && tablePos.Count > 0)
{
_tablePos = new Dictionary<string, ulong>(tablePos.Count);
foreach (var kv in tablePos)
{
_tablePos.Add(StripDatabasePrefix(kv.Key), kv.Value);
}
}
// make a copy of the table filter set, stripping out prefix
if (tableFilter != null)
{
_tableFilter = new HashSet<string>();
foreach (var tableNameOrId in tableFilter)
{
var tableName = StripDatabasePrefix(tableNameOrId);
if (tableName != string.Empty)
{
_tableFilter.Add(tableName);
}
}
}
_reader = _logStreamer.LogManager.OpenLog(Starcounter.Internal.StarcounterEnvironment.DatabaseNameLower, LogStreamerParent.TransactionLogDirectory, new LogPosition { commit_id = FindStartCommitId() });
}
internal FunctionsController(
CloudStorageAccount account,
CloudBlobClient blobClient,
IFunctionInstanceLookup functionInstanceLookup,
IFunctionLookup functionLookup,
IFunctionIndexReader functionIndexReader,
IHeartbeatValidityMonitor heartbeatMonitor,
IAborter aborter,
IRecentInvocationIndexReader recentInvocationsReader,
IRecentInvocationIndexByFunctionReader recentInvocationsByFunctionReader,
IRecentInvocationIndexByJobRunReader recentInvocationsByJobRunReader,
IRecentInvocationIndexByParentReader recentInvocationsByParentReader,
IFunctionStatisticsReader statisticsReader,
ILogReader reader)
{
_account = account;
_blobClient = blobClient;
_functionInstanceLookup = functionInstanceLookup;
_functionLookup = functionLookup;
_functionIndexReader = functionIndexReader;
_heartbeatMonitor = heartbeatMonitor;
_aborter = aborter;
_recentInvocationsReader = recentInvocationsReader;
_recentInvocationsByFunctionReader = recentInvocationsByFunctionReader;
_recentInvocationsByJobRunReader = recentInvocationsByJobRunReader;
_recentInvocationsByParentReader = recentInvocationsByParentReader;
_statisticsReader = statisticsReader;
_reader = reader;
}
public BlockService(IDataStore dataStore, ILogReader logFileDataSource, IIPLocator ipLocator, IFirewallIpBlocker ipBlocker, IIPBlockPolicyFactory policyFactory)
{
this.dataStore = dataStore;
this.logFileDataSource = logFileDataSource;
this.ipLocator = ipLocator;
this.ipBlocker = ipBlocker;
this.policyFactory = policyFactory;
Run();
}
/// <summary>
/// Transfrmation to stream
/// </summary>
/// <param name="reader">Reader</param>
/// <param name="begin">Begin</param>
/// <param name="end">End</param>
/// <returns>Transformation result</returns>
public static void ToStream(this ILogReader reader, Stream stream, uint begin, uint end)
{
IEnumerable <object> en = reader.Load(begin, end);
foreach (object o in en)
{
formatter.Serialize(stream, o);
}
}
/// <summary>
/// Realtime analysis
/// </summary>
/// <param name="dataConsumer">Data consumer</param>
/// <param name="input">Input</param>
/// <param name="stop">Stop function</param>
/// <param name="reason">Reason</param>
/// <param name="timeType">Time type</param>
/// <param name="isAbsoluteTime">The absolute time "sign"</param>
public static bool RealtimeAnalysis(this IDataConsumer dataConsumer,
ILogReader input, Func <object, bool> stop, string reason,
TimeType timeType, bool isAbsoluteTime)
{
ILogReader lr = input as ILogReader;
IEnumerable <object> enu = input.Load(0, 0);
return(dataConsumer.RealtimeAnalysis(enu, stop, reason, timeType, isAbsoluteTime));
}
// Verify that only the expected log items occur in the given window.
// logs should be sorted in reverse chronological order.
private async Task Verify(ILogReader reader, DateTime start, DateTime end, params FunctionInstanceLogItem[] expected)
{
var recent = await GetRecentAsync(reader, CommonFuncName1, start, end);
Assert.Equal(expected.Length, recent.Length);
for (int i = 0; i < expected.Length; i++)
{
Assert.Equal(expected[i].FunctionInstanceId, recent[i].FunctionInstanceId);
}
}
static async Task Reader(string accountConnectionString, DateTime startDate, DateTime endDate)
{
CloudStorageAccount account = CloudStorageAccount.Parse(accountConnectionString);
CloudTableClient client = account.CreateCloudTableClient();
ILogTableProvider tableProvider = LogFactory.NewLogTableProvider(client);
ILogReader reader = LogFactory.NewReader(tableProvider);
await Reader(reader, startDate, endDate);
}
static Logger()
{
var loggerFactory = new LoggerFactory();
LoggerInterface = loggerFactory.GetLoggerRepository();
var logReaderFactory = new LogReaderFactory();
LogReader = logReaderFactory.GetLogReaderRepository();
}
public GetMtaLogsQuery(ILogReader logReader, IRolesService rolesService,
IReadOnlyCharacterService characterService, IReadOnlyUserService userService,
IHttpContextReader httpContextReader)
{
this.logReader = logReader;
this.rolesService = rolesService;
this.characterService = characterService;
this.userService = userService;
this.httpContextReader = httpContextReader;
}
public async Task TimeRange()
{
// Make some very precise writes and verify we read exactly what we'd expect.
var table = GetNewLoggingTable();
try
{
ILogWriter writer = LogFactory.NewWriter("c1", table);
ILogReader reader = LogFactory.NewReader(table);
// Time that functios are called.
DateTime[] times = new DateTime[] {
new DateTime(2010, 3, 6, 10, 11, 20),
new DateTime(2010, 3, 7, 10, 11, 20),
};
DateTime tBefore0 = times[0].AddMinutes(-1);
DateTime tAfter0 = times[0].AddMinutes(1);
DateTime tBefore1 = times[1].AddMinutes(-1);
DateTime tAfter1 = times[1].AddMinutes(1);
var logs = Array.ConvertAll(times, time => new FunctionInstanceLogItem
{
FunctionInstanceId = Guid.NewGuid(),
FunctionName = CommonFuncName1,
StartTime = time
});
var tasks = Array.ConvertAll(logs, log => WriteAsync(writer, log));
await Task.WhenAll(tasks);
await writer.FlushAsync();
// Try various combinations.
await Verify(reader, DateTime.MinValue, DateTime.MaxValue, logs[1], logs[0]); // Infinite range, includes all.
await Verify(reader, tBefore0, tAfter1, logs[1], logs[0]); // barely hugs both instances
await Verify(reader, DateTime.MinValue, tBefore0);
await Verify(reader, DateTime.MinValue, tAfter0, logs[0]);
await Verify(reader, DateTime.MinValue, tBefore1, logs[0]);
await Verify(reader, DateTime.MinValue, tAfter1, logs[1], logs[0]);
await Verify(reader, tAfter0, tBefore1); // inbetween, 0
await Verify(reader, tBefore1, tAfter1, logs[1]);
await Verify(reader, tBefore1, DateTime.MaxValue, logs[1]);
}
finally
{
table.DeleteIfExists();
}
}
private List <MessageTrackingLogEntry> GetMessageLog(RpcReason rpcReason, ILogReader logReader, TrackingLogPrefix logPrefix, ProxyAddressCollection senderAddresses)
{
Fqdn key = Fqdn.Parse(logReader.Server);
Dictionary <ProxyAddressCollection, List <MessageTrackingLogEntry>[]> dictionary = null;
List <MessageTrackingLogEntry>[] array = null;
if (!this.cacheBySender.TryGetValue(key, out dictionary))
{
dictionary = new Dictionary <ProxyAddressCollection, List <MessageTrackingLogEntry>[]>(1, LogCache.proxyAddressCollectionComparer);
this.cacheBySender.Add(key, dictionary);
}
if (!dictionary.TryGetValue(senderAddresses, out array))
{
array = new List <MessageTrackingLogEntry> [LogCache.prefixValues.Length];
for (int i = 0; i < LogCache.prefixValues.Length; i++)
{
List <MessageTrackingLogEntry> list = logReader.ReadLogs(rpcReason, LogCache.prefixValues[i], senderAddresses, this.startTime, this.endTime, this.eventBudget);
array[i] = list;
Dictionary <string, List <MessageTrackingLogEntry>[]> dictionary2 = null;
bool flag = !this.cache.TryGetValue(key, out dictionary2);
HashSet <string> hashSet = new HashSet <string>();
foreach (MessageTrackingLogEntry messageTrackingLogEntry in list)
{
if (!flag && !dictionary2.ContainsKey(messageTrackingLogEntry.MessageId) && !hashSet.Contains(messageTrackingLogEntry.MessageId))
{
hashSet.Add(messageTrackingLogEntry.MessageId);
}
}
if (flag)
{
dictionary2 = new Dictionary <string, List <MessageTrackingLogEntry>[]>(hashSet.Count);
this.cache.Add(key, dictionary2);
}
foreach (MessageTrackingLogEntry messageTrackingLogEntry2 in list)
{
if (hashSet.Contains(messageTrackingLogEntry2.MessageId))
{
List <MessageTrackingLogEntry>[] array2;
if (!dictionary2.TryGetValue(messageTrackingLogEntry2.MessageId, out array2))
{
array2 = new List <MessageTrackingLogEntry> [LogCache.prefixValues.Length];
dictionary2.Add(messageTrackingLogEntry2.MessageId, array2);
}
if (array2[i] == null)
{
array2[i] = new List <MessageTrackingLogEntry>();
}
array2[i].Add(messageTrackingLogEntry2);
}
}
}
dictionary.Add(senderAddresses, array);
}
return(array[(int)logPrefix]);
}
public async Task LogStart()
{
// Make some very precise writes and verify we read exactly what we'd expect.
var table = GetNewLoggingTable();
try
{
ILogWriter writer = LogFactory.NewWriter("c1", table);
ILogReader reader = LogFactory.NewReader(table);
string Func1 = "alpha";
var t1a = new DateTime(2010, 3, 6, 10, 11, 20, DateTimeKind.Utc);
FunctionInstanceLogItem l1 = new FunctionInstanceLogItem
{
FunctionInstanceId = Guid.NewGuid(),
FunctionName = Func1,
StartTime = t1a,
LogOutput = "one"
// inferred as Running since no end time.
};
await writer.AddAsync(l1);
await writer.FlushAsync();
// Start event should exist.
var entries = await GetRecentAsync(reader, Func1);
Assert.Equal(1, entries.Length);
Assert.Equal(entries[0].Status, FunctionInstanceStatus.Running);
Assert.Equal(entries[0].EndTime, null);
l1.EndTime = l1.StartTime.Add(TimeSpan.FromSeconds(1));
l1.Status = FunctionInstanceStatus.CompletedSuccess;
await writer.AddAsync(l1);
await writer.FlushAsync();
// Should overwrite the previous row.
entries = await GetRecentAsync(reader, Func1);
Assert.Equal(1, entries.Length);
Assert.Equal(entries[0].Status, FunctionInstanceStatus.CompletedSuccess);
Assert.Equal(entries[0].EndTime.Value.DateTime, l1.EndTime);
}
finally
{
// Cleanup
table.DeleteIfExists();
}
}
/// <summary>
/// 读取数据到数据库
/// </summary>
private void PreProcess(string fpath)
{
ILogReader ilr = GetLogReader(fpath);
if (ilr == null)
{
return;
}
ilr.ReadBlock();
}
private ParamContainer CreateParamContainer(ILogger loggerController, IActionExecuter actionExecuter)
{
ParamContainer paramContainer = ActionExecuterHelper.CreateParamContainer(_logger, actionExecuter);
IRoutings routings = Routings.Routings.GetRoutings();
ILogReader logReader = Logger.GetLogReader();
ResponseBuilder responseBuilder = new ResponseBuilder(loggerController, logReader, new CronJobInitData(loggerController.TransactionId));
paramContainer.AddKey(CommonConst.CommonValue.PARAM_RESPONBUILDER, () => { return(responseBuilder); });
paramContainer.AddKey(CommonConst.CommonValue.PARAM_ROUTING_OBJECT, () => { return(routings); });
return(paramContainer);
}
public async Task Casing()
{
// Make some very precise writes and verify we read exactly what we'd expect.
var table = GetNewLoggingTable();
try
{
ILogWriter writer = LogFactory.NewWriter("c1", table);
ILogReader reader = LogFactory.NewReader(table);
string FuncOriginal = "UPPER-lower";
string Func2 = FuncOriginal.ToLower(); // casing permutations
string Func3 = Func2.ToLower();
var t1a = new DateTime(2010, 3, 6, 10, 11, 20, DateTimeKind.Utc);
FunctionInstanceLogItem l1 = new FunctionInstanceLogItem
{
FunctionInstanceId = Guid.NewGuid(),
FunctionName = FuncOriginal,
StartTime = t1a,
LogOutput = "one"
// inferred as Running since no end time.
};
await writer.AddAsync(l1);
await writer.FlushAsync();
// Start event should exist.
var definitionSegment = await reader.GetFunctionDefinitionsAsync(null);
Assert.Equal(1, definitionSegment.Results.Length);
Assert.Equal(FuncOriginal, definitionSegment.Results[0].Name);
// Lookup various casings
foreach (var name in new string[] { FuncOriginal, Func2, Func3 })
{
var entries = await GetRecentAsync(reader, name);
Assert.Equal(1, entries.Length);
Assert.Equal(entries[0].Status, FunctionInstanceStatus.Running);
Assert.Equal(entries[0].EndTime, null);
Assert.Equal(entries[0].FunctionName, FuncOriginal); // preserving.
}
}
finally
{
// Cleanup
table.DeleteIfExists();
}
}
// Verify that only the expected log items occur in the given window.
// logs should be sorted in reverse chronological order.
private async Task Verify(ILogReader reader, DateTime start, DateTime end, params FunctionInstanceLogItem[] expected)
{
var recent = await GetRecentAsync(reader, commonFuncId1, start, end);
Assert.Equal(expected.Length, recent.Length);
for (int i = 0; i < expected.Length; i++)
{
Assert.Equal(expected[i].FunctionInstanceId, recent[i].FunctionInstanceId);
}
}
public LogViewModel(ILogReader logReader)
{
this.logReader = logReader;
this.WhenActivated((disposables) =>
{
this.logReader.ReadNextEntriesAsync()
.ToObservable()
.Subscribe(AddEntry)
.DisposeWith(disposables);
});
}
public void ActivateLog()
{
byte[] buffer = new byte[1024 * 1024 * 10];
LogTunnel tunnelLog;
m_log = tunnelLog = CreateLog();
m_cfl = new ContinuesBinaryFileLogger(new InMemoryStreamProvider(new MemoryStream(buffer)), new SyncSubmitLogEntryFactory(), BufferPoolFactory.Instance);
m_cfl.AttachToTunnelLog(tunnelLog);
var readerFactory = new WhitelogBinaryReaderFactory();
readerFactory.RegisterReaderFactory(new ExpandableLogReaderFactory());
m_testConsumer = new TestConsumer();
m_logReader = readerFactory.GetLogReader(new MemoryStream(buffer), m_testConsumer);
}
/// <summary>
/// Performs operation
/// </summary>
/// <param name="reader">Reader</param>
/// <param name="stop">Stop</param>
public void Perform(ILogReader reader, Func <object, bool> stop)
{
if (item is IBufferDirectory)
{
directory = item as IBufferDirectory;
WriteTypes();
IEnumerable <object> en = consumer.RealtimeAnalysisEnumerable(reader, stop,
StaticExtensionEventInterfaces.RealtimeLogAnalysis, TimeType.Second, true);
IEnumerable <byte[]> data = Transform(en);
IBufferData d = directory.CreateData(data, reader.Name, reader.FileName, "");
d.Types = typeBytes;
StaticExtensionDataPerformerInterfaces.Data.SubmitChanges();
}
}
public async Task LargeWritesWithParametersAreTruncated()
{
ILogWriter writer = LogFactory.NewWriter(defaultHost, "c1", this);
ILogReader reader = LogFactory.NewReader(this);
// Max table request size is 4mb. That gives roughly 40kb per row.
string largeValue = new string('x', 100 * 1000);
string truncatedPrefix = largeValue.Substring(0, 100);
List <Guid> functionIds = new List <Guid>();
for (int i = 0; i < 90; i++)
{
var functionId = Guid.NewGuid();
functionIds.Add(functionId);
var now = DateTime.UtcNow;
var item = new FunctionInstanceLogItem
{
FunctionInstanceId = functionId,
Arguments = new Dictionary <string, string>(),
StartTime = now,
EndTime = now.AddSeconds(3),
FunctionName = "tst2",
LogOutput = largeValue,
ErrorDetails = largeValue,
TriggerReason = largeValue
};
for (int j = 0; j < 1000; j++)
{
string paramName = "p" + j.ToString();
item.Arguments[paramName] = largeValue;
}
await writer.AddAsync(item);
}
// If we didn't truncate, then this would throw with a 413 "too large" exception.
await writer.FlushAsync();
// If we got here without an exception, then we successfully truncated the rows.
// Lookup and verify
var instance = await reader.LookupFunctionInstanceAsync(functionIds[0]);
Assert.True(instance.LogOutput.StartsWith(truncatedPrefix));
Assert.True(instance.ErrorDetails.StartsWith(truncatedPrefix));
Assert.True(instance.TriggerReason.StartsWith(truncatedPrefix));
Assert.Equal(0, instance.Arguments.Count); // totally truncated.
}
static async Task <IRecentFunctionEntry[]> GetRecentAsync(ILogReader reader, FunctionId functionId,
DateTime start, DateTime end)
{
var query = await reader.GetRecentFunctionInstancesAsync(new RecentFunctionQuery
{
FunctionId = functionId,
Start = start,
End = end,
MaximumResults = 1000
}, null);
var results = query.Results;
return(results);
}
static Logger()
{
var loggerFactory = new LoggerFactory();
LoggerInterface = loggerFactory.GetLoggerRepository();
var logReaderFactory = new LogReaderFactory();
LogReader = logReaderFactory.GetLogReaderRepository();
LogReaderV2 = logReaderFactory.GetLogReaderV2Repository();
var transactionFactory = new TransactionFactory();
TransactionInterface = transactionFactory.GetTransactionRepository();
}
/// <summary>
/// Will copy all logs, which returns <b>true</b> from the <paramref name="logEntryFilterPredicate"/> function, from the <paramref name="log"/> and write them to a text file with the <paramref name="filename"/> in the given <paramref name="fileStore"/>.
/// </summary>
/// <remarks>
/// <code>
/// // The following characters are encoded into, and decoded from, the following:
/// // Percent (%) %<25>
/// // Semicolon (;) %<3B>
/// // Double-Quote (") %<22>
/// // Newline ( ) %<0D>
/// </code>
/// </remarks>
/// <param name="log">The logger to read log from.</param>
/// <param name="fileStore">The file store to write the log file to.</param>
/// <param name="filename">The name of the file in the file store to write the logs to.</param>
/// <param name="writeLogEntriesUsingLocalTime">Sets whether the date is written to the log relative to UTC or the local computer's timezone.</param>
/// <param name="logEntryFilterPredicate">The <see cref="Func{T, TResult}"/> should return <b>true</b> to write the log to the archive file; return <b>false</b> to skip writing the log.</param>
/// <returns>The number of logs written to the log file.</returns>
public static int ArchiveLogs(ILogReader log,
FileStorage.IFileStore fileStore,
string filename,
bool writeLogEntriesUsingLocalTime,
Func <ILogEntry, bool> logEntryFilterPredicate)
{
return(ArchiveLogs(log,
fileStore,
filename,
writeLogEntriesUsingLocalTime,
false,
0,
0,
logEntryFilterPredicate));
}
/// <summary>
/// Transfrmation to bytes
/// </summary>
/// <param name="reader">Reader</param>
/// <param name="begin">Begin</param>
/// <param name="end">End</param>
/// <returns>Transformation result</returns>
public static IEnumerable <byte[]> ToBytes(this ILogReader reader, uint begin, uint end)
{
IEnumerable <object> en = reader.Load(begin, end);
foreach (object o in en)
{
using (MemoryStream stream = new MemoryStream())
{
formatter.Serialize(stream, o);
byte[] b = stream.GetBuffer();
byte[] outp = new byte[stream.Length];
Array.Copy(b, outp, outp.Length);
yield return(outp);
}
}
}
private void AddReaderWithFileNameCheck(string path)
{
if (!FileMask.IsMatch(Path.GetFileName(path)))
{
return;
}
lock (_readers)
{
if (!_readers.ContainsKey(path))
{
ILogReader reader = _logReaderFactory.CreateReader(path, new FileInfo(path).Length, Encoding);
reader.LineReaded += EvaluateEvents;
reader.BeginRead();
_readers.Add(path, reader);
}
}
}
/// <summary>
/// Will copy all logs from the <paramref name="log"/> and write them to a text file with the <paramref name="filename"/> in the given <paramref name="fileStore"/>.
/// </summary>
/// <remarks>
/// <code>
/// // The following characters are encoded into, and decoded from, the following:
/// // Percent (%) %<25>
/// // Semicolon (;) %<3B>
/// // Double-Quote (") %<22>
/// // Newline ( ) %<0D>
/// </code>
/// </remarks>
/// <param name="log">The logger to read log from.</param>
/// <param name="fileStore">The file store to write the log file to.</param>
/// <param name="filename">The name of the file in the file store to write the logs to.</param>
/// <param name="writeLogEntriesUsingLocalTime">Sets whether the date is written to the log relative to UTC or the local computer's timezone.</param>
/// <param name="truncateLogArchive">Sets whether the <paramref name="fileStore"/> will be automatically truncated.</param>
/// <param name="truncateLogArchiveMaximumFiles">The total number of files allowed in the <paramref name="fileStore"/> before auto truncation is performed.</param>
/// <param name="truncateLogArchivePercentageToRemove">The percentage of log archive files to remove once <paramref name="truncateLogArchiveMaximumFiles"/> has been exceeded.</param>
/// <returns>The number of logs written to the log file.</returns>
public static int ArchiveLogs(ILogReader log,
FileStorage.IFileStore fileStore,
string filename,
bool writeLogEntriesUsingLocalTime,
bool truncateLogArchive,
int truncateLogArchiveMaximumFiles,
double truncateLogArchivePercentageToRemove)
{
return(ArchiveLogs(log,
fileStore,
filename,
writeLogEntriesUsingLocalTime,
truncateLogArchive,
truncateLogArchiveMaximumFiles,
truncateLogArchivePercentageToRemove,
new Func <ILogEntry, bool>((e) => { return true; })));
}
/// <summary>
/// Realtime analysis
/// </summary>
/// <param name="dataConsumer">Data consumer</param>
/// <param name="input">Input</param>
/// <param name="stop">Stop function</param>
/// <param name="reason">Reason</param>
/// <param name="timeType">Time type</param>
/// <param name="isAbsoluteTime">The absolute time "sign"</param>
public static void RealtimeAnalysis(this IDataConsumer dataConsumer,
object input, Func <object, bool> stop, string reason,
TimeType timeType, bool isAbsoluteTime)
{
cadr = 0;
IComponentCollection cc = dataConsumer.CreateCollection(reason);
if (input is ILogReader)
{
ILogReader lr = input as ILogReader;
cc.ForEach((ICalculationReason re) => { re.CalculationReason = reason; });
cc.ForEach((IRealTimeStartStop ss) => { ss.Start(); });
dataConsumer.RealtimeAnalysis(lr, stop, reason, timeType, isAbsoluteTime);
cc.ForEach((IRealTimeStartStop ss) => { ss.Stop(); });
return;
}
if (input is ILogReaderCollection)
{
// IComponentCollection cc = dataConsumer.CreateCollection(reason);
cc.ForEach((ICalculationReason re) => { re.CalculationReason = reason; });
cc.ForEach((IRealTimeStartStop ss) => { ss.Start(); });
IEnumerable <ILogReader> lr = (input as ILogReaderCollection).Readers;
foreach (ILogReader r in lr)
{
cc.ForEach((IRealTimeStartStop ss) => { ss.Start(); });
bool b = dataConsumer.RealtimeAnalysis(r, stop, reason, timeType, isAbsoluteTime);
cc.ForEach((IRealTimeStartStop ss) => { ss.Stop(); });
if (b)
{
return;
}
}
cc.ForEach((IRealTimeStartStop ss) => { ss.Stop(); });
}
if (input is IIterator)
{
IIterator it = input as IIterator;
cc.ForEach((ICalculationReason re) => { re.CalculationReason = reason; });
cc.ForEach((IRealTimeStartStop ss) => { ss.Start(); });
// bool b = dataConsumer.RealtimeAnalysis(it, stop, reason, timeType, isAbsoluteTime);
cc.ForEach((IRealTimeStartStop ss) => { ss.Stop(); });
return;
}
}
public List <MessageTrackingLogEntry> GetMessageLog(RpcReason rpcReason, ILogReader reader, TrackingLogPrefix logPrefix, string messageId, MessageTrackingSource?sourceFilter, HashSet <MessageTrackingEvent> eventIdFilterSet)
{
List <MessageTrackingLogEntry> messageLog = this.GetMessageLog(rpcReason, reader, logPrefix, messageId);
List <MessageTrackingLogEntry> list = new List <MessageTrackingLogEntry>();
if (messageLog == null)
{
return(list);
}
foreach (MessageTrackingLogEntry messageTrackingLogEntry in messageLog)
{
if ((sourceFilter == null || !(messageTrackingLogEntry.Source != sourceFilter)) && eventIdFilterSet.Contains(messageTrackingLogEntry.EventId))
{
list.Add(messageTrackingLogEntry);
}
}
return(list);
}
public List <MessageTrackingLogEntry> GetMessageLog(RpcReason rpcReason, ILogReader reader, TrackingLogPrefix logPrefix, string messageId, long mailItemId)
{
List <MessageTrackingLogEntry> messageLog = this.GetMessageLog(rpcReason, reader, logPrefix, messageId);
List <MessageTrackingLogEntry> list = new List <MessageTrackingLogEntry>();
if (messageLog == null)
{
return(list);
}
foreach (MessageTrackingLogEntry messageTrackingLogEntry in messageLog)
{
if (mailItemId == messageTrackingLogEntry.ServerLogKeyMailItemId)
{
list.Add(messageTrackingLogEntry);
}
}
return(list);
}
public static List <TWLogMessage> GetAllLogMessages()
{
var list = new List <TWLogMessage>();
ILogReader reader = LogReader.Instance;
while (true)
{
LogMessage message = reader.GetNextLogMessage();
if (message == null)
{
break;
}
list.Add(new TWLogMessage(ToTWSeverity(message.Severity), message.Template,
message.Objects, message.Exception, message.Time));
}
return(list);
}
/// <summary>
/// Realtime analysis enumerable
/// </summary>
/// <param name="dataConsumer">Data consumer</param>
/// <param name="input">Input</param>
/// <param name="stop">Stop function</param>
/// <param name="reason">Reason</param>
/// <param name="timeType">Time type</param>
/// <param name="isAbsoluteTime">The absolute time "sign"</param>
/// <returns>The enumerable</returns>
public static IEnumerable <object> RealtimeAnalysisEnumerable(this IDataConsumer dataConsumer,
object input, Func <object, bool> stop, string reason,
TimeType timeType, bool isAbsoluteTime)
{
if (input is ILogReader)
{
ILogReader lr = input as ILogReader;
return(dataConsumer.RealtimeAnalysisEnumerable(lr, stop, reason, timeType, isAbsoluteTime));
}
if (input is ILogReaderCollection)
{
ILogReaderCollection lr = input as ILogReaderCollection;
return(dataConsumer.RealtimeAnalysisEnumerable(lr, stop, reason, timeType, isAbsoluteTime));
}
if (input is IIterator)
{
}
return(null);
}
public async Task LogStart()
{
// Make some very precise writes and verify we read exactly what we'd expect.
ILogWriter writer = LogFactory.NewWriter(defaultHost, "c1", this);
ILogReader reader = LogFactory.NewReader(this);
string Func1 = "alpha";
var t1a = new DateTime(2010, 3, 6, 10, 11, 20, DateTimeKind.Utc);
FunctionInstanceLogItem l1 = new FunctionInstanceLogItem
{
FunctionInstanceId = Guid.NewGuid(),
FunctionName = Func1,
StartTime = t1a,
LogOutput = "one"
// inferred as Running since no end time.
};
await writer.AddAsync(l1);
await writer.FlushAsync();
// Start event should exist.
var entries = await GetRecentAsync(reader, l1.FunctionId);
Assert.Single(entries);
Assert.Equal(entries[0].GetStatus(), FunctionInstanceStatus.Running);
Assert.Equal(entries[0].EndTime, null);
l1.EndTime = l1.StartTime.Add(TimeSpan.FromSeconds(1));
await writer.AddAsync(l1);
await writer.FlushAsync();
// Should overwrite the previous row.
entries = await GetRecentAsync(reader, l1.FunctionId);
Assert.Single(entries);
Assert.Equal(entries[0].GetStatus(), FunctionInstanceStatus.CompletedSuccess);
Assert.Equal(entries[0].EndTime.Value, l1.EndTime);
}
public async Task FunctionInstance()
{
var table = GetNewLoggingTable();
try
{
ILogReader reader = LogFactory.NewReader(table);
TimeSpan poll = TimeSpan.FromMilliseconds(50);
TimeSpan poll5 = TimeSpan.FromMilliseconds(poll.TotalMilliseconds * 5);
var logger1 = new CloudTableInstanceCountLogger("c1", table, 100)
{
PollingInterval = poll
};
Guid g1 = Guid.NewGuid();
DateTime startTime = DateTime.UtcNow;
logger1.Increment(g1);
await Task.Delay(poll5); // should get at least 1 poll entry in
logger1.Decrement(g1);
await Task.WhenAll(logger1.StopAsync());
DateTime endTime = DateTime.UtcNow;
// Now read.
// We may get an arbitrary number of raw poll entries since the
// low poll latency combined with network delay can be unpredictable.
var values = await reader.GetVolumeAsync(startTime, endTime, 1);
double totalVolume = (from value in values select value.Volume).Sum();
Assert.True(totalVolume > 0);
double totalInstance = (from value in values select value.InstanceCounts).Sum();
Assert.Equal(1, totalInstance);
}
finally
{
table.DeleteIfExists();
}
}
static public FlexReadBuffer Deserialize(ILogReader S,
FlexReadBuffer flexBuf)
{
var messageSize = S.ReadInt(flexBuf._sizeBuf);
flexBuf._curSize = messageSize.Item1 + messageSize.Item2;
flexBuf._curSizeSize = messageSize.Item2;
if (flexBuf.Buffer != null)
{
throw new Exception("Flexbuffer should have been null in Deserialize");
}
flexBuf.GetBuffer();
var bytesRead = S.ReadAllRequiredBytes(flexBuf._buffer, messageSize.Item2, messageSize.Item1);
if (bytesRead < messageSize.Item1)
{
throw new Exception("Error deserializing buffer in stream");
}
return(flexBuf);
}
public LogsIndexModelBuilder(IFileManager fileManager, ILogReader logReader)
{
_fileManager = fileManager;
_logReader = logReader;
}
/// <summary>
/// Loads the files.
/// </summary>
/// <param name="reader">The log reader.</param>
/// <param name="logs">The list to add open logs to.</param>
/// <param name="files">The files too open.</param>
/// <param name="initialVersion">The initial log version.</param>
/// <param name="ignoreVersionChanges">If set to <c>true</c> version information in the log is ignored.</param>
/// <param name="progress">The progress.</param>
private void LoadFiles(ILogReader reader,
ICollection<PacketLog> logs,
string[] files,
float initialVersion,
bool ignoreVersionChanges,
ProgressCallback progress)
{
foreach (string fileName in files)
{
try
{
// Create new log for each file
PacketLog log = new PacketLog();
log.Version = initialVersion;
log.IgnoreVersionChanges = ignoreVersionChanges;
m_progress.SetDescription("Loading file: " + fileName + "...");
// Check if file exists
FileInfo fileInfo = new FileInfo(fileName);
if (!fileInfo.Exists)
{
Log.Info("File \"" + fileInfo.FullName + "\" doesn't exist, ignored.");
continue;
}
// Add all packets
using(FileStream stream = File.Open(fileName, FileMode.Open, FileAccess.Read, FileShare.Read))
{
log.AddRange(reader.ReadLog(new BufferedStream(stream, 64*1024), progress));
}
// Initialize log
m_progress.SetDescription("Initializing log and packets...");
log.Init(LogManager, 3, progress);
// Set stream name
log.StreamName = fileInfo.FullName;
AddRecentFile(fileInfo.FullName);
logs.Add(log);
}
catch (Exception e)
{
Log.Error("loading files", e);
}
}
}
public LogSystem(ILogReader aLogReader, ILogController aLogController)
{
LogReader = aLogReader;
LogController = aLogController;
}
public XmlLogService(ILogIt logIt, ILogReader logReader)
: base(logIt, logReader)
{
}
public void OpenLog() {
if (_reader != null)
throw new InvalidOperationException("OpenLog() already called");
if (IsSendNothingFilter()) {
Status = string.Format("Not opening log as table filter is single empty string");
return;
}
ulong commit_id = FindStartCommitId();
Status = string.Format("Opening log to commit ID {0}", commit_id);
var started = DateTime.Now; // Using DateTime instead of StopWatch as OpenLog() can take a very long time
_reader = _logStreamer.LogManager.OpenLog(_databaseName, _transactionLogDirectory, new LogPosition { CommitID = commit_id });
Status = string.Format("Opening log took {0}", DateTime.Now.Subtract(started));
}
protected void WhenLogReaderIsCreated()
{
Target = LogReader.Create(LogFileName, InitialPosition);
}
public LogService(ILogIt logIt, ILogReader logReader)
{
this.logIt = logIt;
this.logReader = logReader;
}
static Task<IRecentFunctionEntry[]> GetRecentAsync(ILogReader reader, string functionName)
{
return GetRecentAsync(reader, functionName, DateTime.MinValue, DateTime.MaxValue);
}
private FunctionSnapshot[] _snapshots = null; // Cache of function definitions
public FastTableReader(CloudTable table)
{
_reader = LogFactory.NewReader(table);
}
public FastTableReader(ILogReader reader)
{
_reader = reader;
}
public HomeController(IConfigProvider configProvider, ILogReader logReader, IFileSystem fileSystem)
{
this.configProvider = configProvider;
this.logReader = logReader;
this.fileSystem = fileSystem;
}
static async Task<IRecentFunctionEntry[]> GetRecentAsync(ILogReader reader, string functionName,
DateTime start, DateTime end)
{
var query = await reader.GetRecentFunctionInstancesAsync(new RecentFunctionQuery
{
FunctionName = functionName,
Start = start,
End = end,
MaximumResults = 1000
}, null);
var results = query.Results;
return results;
}
private void ShowReport(ILogReader reader)
{
if (reader != this)
return;
UpdateReport();
report.ShowDialog();
}
public LogControlProvider(ILogReader aLogReader, ILogController aLogController)
{
iLogReader = aLogReader;
iLogController = aLogController;
}
