static void Main(string[] args)
{
//Write100MillionLinesOfLogItemsToFile();
var currentPage = new PagedLogItems();
currentPage.LoadType = LoadType.NextPage;
currentPage.LogType = LogType.All;
//Load next page log items of all type
currentPage = LoadLogItemByPage(currentPage);
Console.WriteLine(currentPage);
//Load next page log items of all type
currentPage = LoadLogItemByPage(currentPage);
Console.WriteLine(currentPage);
//Load previous page log items of all type
currentPage.LoadType = LoadType.PreviousPage;
currentPage = LoadLogItemByPage(currentPage);
Console.WriteLine(currentPage);
//Load next page log items of all type 100 times
currentPage.LoadType = LoadType.NextPage;
for (var i = 0; i < 10; ++i)
{
currentPage = LoadLogItemByPage(currentPage);
Console.WriteLine(currentPage);
}
//Load next page log items of E type 100 times
//Note: When change the log type to load, you must renew a new loading from the begin of the large file.
currentPage = new PagedLogItems();
currentPage.LogType = LogType.E;
currentPage.LoadType = LoadType.NextPage;
for (var i = 0; i < 10; ++i)
{
currentPage = LoadLogItemByPage(currentPage);
Console.WriteLine(currentPage);
}
//Load next page log items of E type 100 times
//Note: When change the log type to load, you must renew a new loading from the begin of the large file.
currentPage = new PagedLogItems();
currentPage.LogType = LogType.L;
currentPage.LoadType = LoadType.NextPage;
for (var i = 0; i < 10; ++i)
{
currentPage = LoadLogItemByPage(currentPage);
Console.WriteLine(currentPage);
}
Console.WriteLine("Press any key to exit...");
Console.ReadKey();
}
private static ScannedResult ScanLines(string[] lines, PagedLogItems currentPage)
{
var result = new ScannedResult();
if (lines == null || lines.Length <= 0)
{
return(result);
}
if (currentPage == null || currentPage.LogItems == null)
{
return(result);
}
if (currentPage.LogItems.Count >= PAGED_LINES_COUNT)
{
currentPage.LogItems.Clear();
}
foreach (var line in lines)
{
if (currentPage.LogItems.Count >= PAGED_LINES_COUNT)
{
break;
}
var logItem = line.ToLogItem(currentPage.LogType);
if (logItem == null)
{
result.ScannedByteLength += System.Text.Encoding.UTF8.GetBytes(line + "\n").LongLength;
continue;
}
else
{
result.ScannedItemsCount++;
currentPage.LogItems.Add(logItem);
result.ScannedByteLength += System.Text.Encoding.UTF8.GetBytes(line + "\n").LongLength;
}
}
return(result);
}
private static void SetPositionOfFileStream(PagedLogItems currentPage, FileStream fs)
{
if (currentPage.LoadType == LoadType.PreviousPage)
{
if (currentPage.CurrentEndPosition.Count > 0)
{
//if current page is not loaded completely, go on to next block and continue loading
if (currentPage.CurrentEndPosition.Any(p => !p.LoadCompleted))
{
currentPage.CurrentStartPosition = currentPage.CurrentStartPosition.Where(p => p.LoadCompleted).ToList();
currentPage.CurrentEndPosition = currentPage.CurrentEndPosition.Where(p => p.LoadCompleted).ToList();
fs.Position = currentPage.PreviousStartPosition - BUFFER_SIZE_OF_96_KB < 0 ? 0 : currentPage.PreviousStartPosition - BUFFER_SIZE_OF_96_KB;
}
else
{
//else, go to previous block
if (currentPage.CurrentEndPosition.Count >= 2)
{
currentPage.CurrentStartPosition.RemoveAt(currentPage.CurrentStartPosition.Count - 1);
currentPage.CurrentEndPosition.RemoveAt(currentPage.CurrentEndPosition.Count - 1);
currentPage.CurrentStartPosition.RemoveAt(currentPage.CurrentStartPosition.Count - 1);
currentPage.CurrentEndPosition.RemoveAt(currentPage.CurrentEndPosition.Count - 1);
fs.Position = currentPage.PreviousStartPosition;
}
else
{
fs.Position = 0;
}
}
}
else
{
fs.Position = 0;
}
}
else
{
if (currentPage.CurrentEndPosition.Count > 0)
{
//if current page is not loaded completely, go on to next block and continue loading
if (currentPage.CurrentEndPosition.Any(p => !p.LoadCompleted))
{
currentPage.CurrentStartPosition = currentPage.CurrentStartPosition.Where(p => p.LoadCompleted).ToList();
currentPage.CurrentEndPosition = currentPage.CurrentEndPosition.Where(p => p.LoadCompleted).ToList();
fs.Position = currentPage.PreviousStartPosition + BUFFER_SIZE_OF_96_KB >= fs.Length ? fs.Length : currentPage.PreviousStartPosition + BUFFER_SIZE_OF_96_KB;
}
else
{
fs.Position = currentPage.CurrentEndPosition.Last().Position;
}
}
else
{
fs.Position = 0;
}
}
}
private static PagedLogItems LoadLogItemByPage(PagedLogItems currentPage)
{
var stopWatch = new Stopwatch();
Console.WriteLine("Start load log items from file \"test.log\"....");
stopWatch.Start();
var buffer = new byte[BUFFER_SIZE_OF_96_KB];
using (var fs = new FileStream("test.log", FileMode.Open, FileAccess.Read, FileShare.Read))
{
currentPage.TotalSize = fs.Length;
var errorCount = 0;
SetPositionOfFileStream(currentPage, fs);
while (fs.Read(buffer, 0, BUFFER_SIZE_OF_96_KB) > 0)
{
var content = System.Text.Encoding.UTF8.GetString(buffer);
content = content.TrimEnd('\0');
if (string.IsNullOrEmpty(content))
{
break;
}
var lines = content.Split('\n');
if (lines == null || lines.Length <= 0)
{
break;
}
ScannedResult scannedResult = null;
try
{
scannedResult = ScanLines(lines, currentPage);
}
catch (Exception ex)
{
Console.WriteLine("ScanLines Failed! error message:{0}", ex.Message);
errorCount++;
currentPage.LogItems = currentPage.LogItems.Take(currentPage.LogItems.Count - scannedResult.ScannedItemsCount).ToList();
buffer = new byte[BUFFER_SIZE_OF_96_KB];
var newStartPosition = (fs.Position - BUFFER_SIZE_OF_96_KB - errorCount) > 0 ? (fs.Position - BUFFER_SIZE_OF_96_KB - errorCount) : 0;
fs.Position = newStartPosition;
continue;
}
errorCount = 0;
var currentStartPosition = (fs.Position - BUFFER_SIZE_OF_96_KB);
var currentEndPosition = (currentStartPosition + scannedResult.ScannedByteLength) >= fs.Length ? fs.Length : currentStartPosition + scannedResult.ScannedByteLength;
if (currentPage.CurrentStartPosition == null || currentPage.CurrentStartPosition.Count <= 0)
{
currentPage.PreviousStartPosition = 0;
}
else
{
currentPage.PreviousStartPosition = currentPage.CurrentStartPosition.Last().Position;
}
//Load give count of log items in the first loading
if (currentPage.LogItems.Count >= PAGED_LINES_COUNT)
{
currentPage.CurrentStartPosition.Add(new PositionWithStatus()
{
Position = currentStartPosition, LoadCompleted = true
});
currentPage.CurrentEndPosition.Add(new PositionWithStatus()
{
Position = currentEndPosition, LoadCompleted = true
});
break;
}
else
{
currentPage.CurrentStartPosition.Add(new PositionWithStatus()
{
Position = currentStartPosition, LoadCompleted = false
});
currentPage.CurrentEndPosition.Add(new PositionWithStatus()
{
Position = currentEndPosition, LoadCompleted = false
});
buffer = new byte[BUFFER_SIZE_OF_96_KB];
SetPositionOfFileStream(currentPage, fs);
}
}
}
stopWatch.Stop();
Console.WriteLine("Loading log items from file \"test.log\" ended.");
Console.WriteLine("Time consumed:{0} milliseconds", stopWatch.ElapsedMilliseconds);
return(currentPage);
}