private void captureLoop()
{
try
{
while (!stopUpdate)
{
try
{
frame = desktopDuplicator.GetLatestFrame();
}
catch { }
if (frame == null)
{
continue;
}
// uncomment the line bellow to enable drawing on screen
// if(updateBgEnabled) BackgroundImage = frame.DesktopImage;
PerformanceMonitorEvent?.BeginInvoke(sw.Elapsed.TotalMilliseconds, null, null);
if (PerformanceMonitoringIsActive)
{
sw.Restart();
}
}
}
catch (ThreadAbortException) { }
}
private void InitConnection()
{
DesktopFrame frame = null;
while (true)
{
try
{
frame = desktopDuplicator.GetLatestFrame();
break;
}
catch { desktopDuplicator = new DesktopDuplicator(0); };
}
resX = frame.DesktopImage.Width;
resY = frame.DesktopImage.Height;
if (proc != null)
{
try { proc.Kill(); }
catch { };
}
if (audioProc != null)
{
try { audioProc.Kill(); }
catch { };
}
audioProc = new Process()
{
StartInfo = new ProcessStartInfo()
{
FileName = "ffmpeg.exe",
Arguments = string.Format(ffmpeg_audio_args, ip),
UseShellExecute = false,
RedirectStandardInput = true,
RedirectStandardOutput = true,
CreateNoWindow = true
}
};
audioProc.Start();
proc = new Process()
{
StartInfo = new ProcessStartInfo()
{
FileName = "ffmpeg.exe",
Arguments = string.Format(ffmpeg_args, resX, resY, quality, ip),
UseShellExecute = false,
RedirectStandardInput = true,
RedirectStandardOutput = true,
CreateNoWindow = true
}
};
proc.Start();
}
public void CanGetDesktopThumbnail()
{
var foo = DesktopFrame.FromFile("./mocks/datapattern.bmp");
var thumbby = foo.GetThumbnailImage(640, 480);
Assert.True(thumbby.Length > 0);
}
public void GlobalCleanup()
{
AlignmentMap = null;
AlignmentFrame = null;
AlphaFrame = null;
DataFrame = null;
TestFrame = null;
}
public void Draw()
{
UpdateTexture();
ApplyTexture();
if (_latestFrame != null)
{
_latestFrame = null;
}
}
public static IEnumerable <BitmapBufferedData> Create(DesktopFrame frame)
{
foreach (var croppedImage in BitmapCropper.GetCroppedBitmaps(frame))
{
var bitmapBufferData = new BitmapBufferedData(croppedImage.Rectangle);
bitmapBufferData.Buffer = EncodePicture(croppedImage.Bitmap);
croppedImage.Dispose();
yield return(bitmapBufferData);
}
}
public void GlobalSetup()
{
var mapJson = File.ReadAllText("./mocks/alignmentmap.json");
AlignmentMap = JsonSerializer.Deserialize <IList <int> >(mapJson);
AlignmentFrame = DesktopFrame.FromFile("./mocks/alignpattern.bmp");
AlphaFrame = DesktopFrame.FromFile("./mocks/alphapattern.bmp");
DataFrame = DesktopFrame.FromFile("./mocks/datapattern.bmp");
TestFrame = DesktopFrame.FromFile("./mocks/testpattern.bmp");
}
private unsafe void StreamerLoop()
{
DesktopFrame frame = null;
while (true)
{
if (!started)
{
running = false;
Thread.Sleep(100);
continue;
}
try { frame = desktopDuplicator.GetLatestFrame(); }
catch
{
desktopDuplicator = new DesktopDuplicator(0);
continue;
}
if (frame == null)
{
continue;
}
if (resX != frame.DesktopImage.Width || resY != frame.DesktopImage.Height)
{
InitConnection();
continue;
}
var bData = frame.DesktopImage.LockBits(
new Rectangle(0, 0, frame.DesktopImage.Width, frame.DesktopImage.Height),
ImageLockMode.ReadWrite, frame.DesktopImage.PixelFormat);
using (var ms = new UnmanagedMemoryStream(
(byte *)bData.Scan0.ToPointer(),
bData.Width * bData.Height * 4))
{
try
{
proc.StandardInput.BaseStream.Flush();
ms.CopyTo(proc.StandardInput.BaseStream);
running = true;
}
catch { Stop(); }
frame.DesktopImage.UnlockBits(bData);
}
}
}
private static DesktopFrame GetScreenPicDxgi()
{
DesktopFrame frame = null;
try
{
frame = _desktopDuplicator.GetLatestFrame();
}
catch (Exception)
{
_desktopDuplicator = new DesktopDuplicator(0);
}
return(frame);
}
/// <summary>
/// 通过DXGI获取桌面图形，只能用于win8以上系统,效果媲美Mirror driver
/// 返回值有可能是null（超时会返回0），注意判断使用
/// </summary>
/// <returns></returns>
public static DesktopFrame getScreenPicDXGI()
{
DesktopFrame frame = null;
try
{
frame = desktopDuplicator.GetLatestFrame();
}
catch (Exception ex)
{
Console.WriteLine("-------------->>>" + ex.Message);
desktopDuplicator = new DesktopDuplicator(0);
}
return(frame);
}
public void CanDecodeAlphaPattern()
{
var mapJson = File.ReadAllText("./mocks/alignmentmap.json");
var map = JsonSerializer.Deserialize <IList <int> >(mapJson);
var foo = DesktopFrame.FromFile("./mocks/alphapattern.bmp");
var settings = new MatrixSettings()
{
Map = map,
DataFormat = DataMatrixFormat.Text,
};
var dataMatrix = foo.DecodeMatrixFrame(settings);
var matrixJson = JsonSerializer.Serialize(dataMatrix);
dataMatrix = JsonSerializer.Deserialize <MatrixFrame>(matrixJson);
var testData = dataMatrix.Data.ToString();
Assert.Equal((map.Count / 2) * 3 - 6, testData.Length);
Assert.Equal(120, dataMatrix.FrameId);
int width = 156;
int height = 46;
Assert.Equal(width, dataMatrix.Metadata[0]);
Assert.Equal(height, dataMatrix.Metadata[1]);
var frameType = dataMatrix.Metadata[2] >> 4;
var pixelSize = dataMatrix.Metadata[2] % 16;
Assert.Equal(2, frameType);
Assert.Equal(2, pixelSize);
var errors = 0;
for (int i = 0; i < testData.Length; i++)
{
if (testData[i] != TestPattern[i % TestPattern.Length])
{
errors++;
}
}
Assert.Equal(0, errors);
}
private void TraceFrameDetails(DesktopFrame frame)
{
//there are many frames per second and we need to extract useful information and only log those!
if (frame == null)
{
//if the frame is null, this can mean two things. the timeout from the desktop duplication api was reached
//before the monitor content changed or there was some other error.
if (!_lastNotNullFrameDateTime.HasValue)
{
_lastNotNullFrameDateTime = DateTime.UtcNow;
}
if (!_lastNotNullFrameLoggedDateTime.HasValue ||
DateTime.UtcNow - _lastNotNullFrameLoggedDateTime.Value > TimeSpan.FromSeconds(30))
{
_lastNotNullFrameLoggedDateTime = DateTime.UtcNow;
_log.Debug("The frames are null since {0}", _lastNotNullFrameDateTime);
_framesAreNullMessagePrinted = true;
}
}
else
{
if (_lastNotNullFrameDateTime.HasValue)
{
if (_framesAreNullMessagePrinted)
{
_log.Debug("There is again a frame which is not null!");
_framesAreNullMessagePrinted = false;
}
_lastNotNullFrameDateTime = null;
}
if (_lastObservedHeight == null || _lastObservedWidth == null ||
_lastObservedHeight != frame.DesktopImage.Height ||
_lastObservedWidth != frame.DesktopImage.Width)
{
_log.Debug("The frame size changed from {0}x{1} to {2}x{3}"
, _lastObservedWidth, _lastObservedHeight
, frame.DesktopImage.Width, frame.DesktopImage.Height);
_lastObservedWidth = frame.DesktopImage.Width;
_lastObservedHeight = frame.DesktopImage.Height;
}
}
}
public void CanGenerateAlignmentMap()
{
var foo = DesktopFrame.FromFile("./mocks/alignpattern.bmp");
var alignmentMap = foo.GenerateAlignmentMap(2);
Assert.Equal(156 * 46, alignmentMap.Count);
var flatPoints = new List <int>(alignmentMap.Count * 2);
foreach (var point in alignmentMap)
{
flatPoints.Add(point.X);
flatPoints.Add(point.Y);
}
//var map = JsonSerializer.Serialize(flatPoints);
//File.WriteAllText("map.json", map);
}
private void FormDemo_Shown(object sender, EventArgs e)
{
while (true)
{
Application.DoEvents();
DesktopFrame frame = desktopDuplicator.GetLatestFrame();
if (frame != null)
{
DataBox mapSource = desktopDuplicator.getCurrentDataBoxBytes();
int width = desktopDuplicator.getFrameWidth();
int height = desktopDuplicator.getFrameHeight();
//Image processing
Stopwatch performanceWatch = new Stopwatch();
performanceWatch.Start();
unsafe
{
processDetection((byte *)mapSource.DataPointer, width, height);
}
performanceWatch.Stop();
Console.WriteLine("Elapsed milliseconds: {0}", performanceWatch.Elapsed.Ticks / 10000.0);
Console.WriteLine("Elapsed fps: {0}", 1000.0 / (performanceWatch.Elapsed.Ticks / 10000.0));
count++;
total += performanceWatch.Elapsed.Ticks;
Console.WriteLine("Average milliseconds: {0}", total / 10000.0 / count);
Console.WriteLine("Average fps: {0}", 1000.0 / (total / 10000.0 / count));
//End image processing
LabelCursor.Location = frame.CursorLocation;
LabelCursor.Visible = frame.CursorVisible;
this.BackgroundImage = desktopDuplicator.getImageFromDataBox(mapSource);
desktopDuplicator.releaseCurrentDataBoxBytes();
desktopDuplicator.ReleaseFrame();
System.GC.Collect();
}
}
}
private void Form1_Shown(object sender, EventArgs e)
{
using (var engine = new TesseractEngine(@"./tessdata", "eng", EngineMode.Default))
{
engine.SetVariable("tessedit_char_backlist", "!?@#$%&*()<>_-+=:;'\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
engine.SetVariable("tessedit_char_whitelist", "/.,0123456789");
engine.SetVariable("classify_bln_numeric_mode", "1");
var rect1 = new Rectangle(55, 785, 130, 50);
while (true)
{
Application.DoEvents();
var elapsed = DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond;
DesktopFrame frame = null;
try
{
frame = desktopDuplicator.GetLatestFrame();
}
catch
{
desktopDuplicator = new DesktopDuplicator(0);
continue;
}
if (frame != null)
{
Bitmap gauges = this.CropImage(frame.DesktopImage, rect1);
using (Image <Gray, Byte> image = new Image <Gray, Byte>(gauges))
using (var tresh = image.ThresholdBinary(new Gray(180), new Gray(255)))
{
this.pictureBox1.Image = tresh.ToBitmap();
using (var page = engine.Process(tresh.ToBitmap(), new Rect(0, 0, 130, 50)))
{
this.label1.Text = page.GetText();
this.label2.Text = String.Format("{0}ms", (DateTime.Now.Ticks / TimeSpan.TicksPerMillisecond) - elapsed);
}
}
}
}
}
}
private void TimerDXGI_Tick(object sender, EventArgs e)
{
var s = Stopwatch.StartNew();
DesktopFrame frame = null;
try
{
frame = desktopDuplicator.GetLatestFrame();
}
catch
{
desktopDuplicator = new DesktopDuplicator(0);
}
if (frame != null)
{
this.BackgroundImage = frame.DesktopImage;
}
fps++;
tim += s.ElapsedMilliseconds;
}
public override void RegisterMessageHandlers()
{
Network.RegisterMessageHandler<RequestScreenshotMessage>(OnRequestScreenshotMessageReceived);
while (true)
{
Application.DoEvents();
try
{
frame = MirrorDriver.GetLatestFrame();
}
catch
{
MirrorDriver = new DesktopDuplicator(0);
continue;
}
if (frame != null)
{
//LabelCursor.Location = frame.CursorLocation;
//LabelCursor.Visible = frame.CursorVisible;
//Debug.WriteLine("--------------------------------------------------------");
//foreach (var moved in frame.MovedRegions)
//{
// Debug.WriteLine(String.Format("Moved: {0} -> {1}", moved.Source, moved.Destination));
// MovedRegion.Location = moved.Destination.Location;
// MovedRegion.Size = moved.Destination.Size;
//}
//foreach (var updated in frame.UpdatedRegions)
//{
// Debug.WriteLine(String.Format("Updated: {0}", updated.ToString()));
// UpdatedRegion.Location = updated.Location;
// UpdatedRegion.Size = updated.Size;
//}
//this.BackgroundImage = frame.DesktopImage;
}
}
}
public static CroppedBitmap[] GetCroppedBitmaps(DesktopFrame frame)
{
var bmps = new List <CroppedBitmap>();
try
{
foreach (var rect in frame.UpdatedRegions)
{
var splittedRects = SplitRectangle(rect);
foreach (var splittedRect in splittedRects)
{
bmps.Add(new CroppedBitmap(frame.DesktopImage.Clone(splittedRect, frame.DesktopImage.PixelFormat), splittedRect));
}
}
}
catch (Exception ex)
{
}
return(bmps.ToArray());
}
private void FormDemo_Shown(object sender, EventArgs e)
{
while (true)
{
Application.DoEvents();
DesktopFrame frame = null;
try
{
frame = desktopDuplicator.GetLatestFrame();
}
catch
{
desktopDuplicator = new DesktopDuplicator(0);
continue;
}
if (frame != null)
{
LabelCursor.Location = frame.CursorLocation;
LabelCursor.Visible = frame.CursorVisible;
//Debug.WriteLine("--------------------------------------------------------");
//foreach (var moved in frame.MovedRegions)
//{
// Debug.WriteLine(String.Format("Moved: {0} -> {1}", moved.Source, moved.Destination));
// MovedRegion.Location = moved.Destination.Location;
// MovedRegion.Size = moved.Destination.Size;
//}
//foreach (var updated in frame.UpdatedRegions)
//{
// Debug.WriteLine(String.Format("Updated: {0}", updated.ToString()));
// UpdatedRegion.Location = updated.Location;
// UpdatedRegion.Size = updated.Size;
//}
this.BackgroundImage = frame.DesktopImage;
fps++;
}
}
}
/// <summary>
/// Gets the next screen frame. DISPOSE of the bitmap when you're done using it!
/// </summary>
/// <returns></returns>
public static Bitmap GetNextFrame()
{
if (!isInitialized)
{
Initialize();
}
try
{
DesktopFrame frame = desktopDuplicator.GetLatestFrame();
if (frame != null)
{
Bitmap frameBitmap = frame.DesktopImage;
return(frameBitmap);
}
}
catch (Exception)
{
desktopDuplicator.Dispose();
desktopDuplicator = new DesktopDuplicator(0);
Debug.WriteLine("Exception in DesktopDuplication API occurred. User probably switched windows");
}
return(null);
}
public void CanDecodeMatrixEventData()
{
var mapJson = File.ReadAllText("./mocks/alignmentmap.json");
var map = JsonSerializer.Deserialize <IList <int> >(mapJson);
var foo = DesktopFrame.FromFile("./mocks/datapattern.bmp");
var settings = new MatrixSettings()
{
Map = map,
DataFormat = DataMatrixFormat.MatrixEvents,
};
var dataMatrix = foo.DecodeMatrixFrame(settings);
var matrixJson = JsonSerializer.Serialize(dataMatrix);
dataMatrix = JsonSerializer.Deserialize <MatrixFrame>(matrixJson);
var testData = JsonSerializer.Deserialize <IList <MatrixEvent> >(((JsonElement)dataMatrix.Data).GetRawText());
Assert.Equal(45, testData.Count);
Assert.Equal(115, dataMatrix.FrameId);
int width = 156;
int height = 46;
Assert.Equal(width, dataMatrix.Metadata[0]);
Assert.Equal(height, dataMatrix.Metadata[1]);
var frameType = dataMatrix.Metadata[2] >> 4;
var pixelSize = dataMatrix.Metadata[2] % 16;
Assert.Equal(0, frameType);
Assert.Equal(2, pixelSize);
var playerData = (JsonElement)testData.FirstOrDefault(td => td.Topic == "player").Data;
Assert.Equal("Sleepyhead", playerData.GetProperty("n").GetString());
}
void GetFrame()
{
while (true)
{
lock (_frameLock)
{
if (!_requestNew)
{
continue;
}
try
{
_latestFrame = _desktopDuplicator.GetLatestFrame();
_requestNew = false;
}
catch
{
return; // Do nothing, it'll probably just spam the console.
}
}
Thread.Sleep(5);
}
}
//on screen shared initiated old
private void OnRequestScreenshotMessageReceived(MessageEventArgs <RequestScreenshotMessage> e)
{
while (true)
{
Application.DoEvents();
DesktopFrame frame = null;
try
{
frame = MirrorDriver.GetLatestFrame();
}
catch (Exception ex)
{
MirrorDriver = new DesktopDuplicator(0);
continue;
}
if (frame != null)
{
var screenshot = frame.DesktopImage;
var stream = new MemoryStream();
screenshot.Save(stream, ImageFormat.Bmp);
SendFragmentedBitmap(stream.ToArray(), Screen.PrimaryScreen.Bounds);
}
}
}
static void Main(string[] args)
{
ActiveProcessObserver observer = new ActiveProcessObserver();
DesktopDuplicator duplicator = null;
observer.ProcessChanged += Observer_ProcessChanged;
observer.Observe();
try
{
duplicator = new DesktopDuplicator(0);
using (var engine = new TesseractEngine(@"./tessdata", "eng", EngineMode.Default))
{
engine.SetVariable("tessedit_char_backlist", "!?@#$%&*()<>_-+=:;'\"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz");
engine.SetVariable("tessedit_char_whitelist", "/.,0123456789");
engine.SetVariable("classify_bln_numeric_mode", "1");
do
{
if (pathOfExileHasFocus)
{
DesktopFrame frame = null;
try
{
frame = duplicator.GetLatestFrame();
}
catch
{
duplicator = new DesktopDuplicator(0);
continue;
}
if (frame != null)
{
// TODO(Olivier): Probably a worthless optimization since the screen will be a game
// but we could verify that our target rectangle is whitin bounds of the frame's
// changed areas
Bitmap area = CropImage(frame.DesktopImage, PATH_OF_EXILE_HEALTH_REGION);
using (Image <Gray, Byte> image = new Image <Gray, Byte>(area))
using (var tresh = image.ThresholdBinary(new Gray(BINARY_TRESHOLD), new Gray(255)))
using (var page = engine.Process(tresh.ToBitmap()))
{
//TODO(Olivier): Use a regex to validate that the page's text is actually what we want
}
}
}
else
{
Thread.Sleep(1000);
}
}while (true);
}
}
catch (Exception e)
{
Console.WriteLine(e);
Console.Read();
}
}
public void CanDecodeTestPattern()
{
var mapJson = File.ReadAllText("./mocks/alignmentmap.json");
var map = JsonSerializer.Deserialize <IList <int> >(mapJson);
var foo = DesktopFrame.FromFile("./mocks/testpattern.bmp");
var settings = new MatrixSettings()
{
Map = map,
DataFormat = DataMatrixFormat.Raw,
};
var dataMatrix = foo.DecodeMatrixFrame(settings);
var matrixJson = JsonSerializer.Serialize(dataMatrix);
dataMatrix = JsonSerializer.Deserialize <MatrixFrame>(matrixJson);
var testData = JsonSerializer.Deserialize <byte[]>(((JsonElement)dataMatrix.Data).GetRawText());
Assert.Equal((map.Count / 2) * 3, testData.Length);
Assert.Equal(119, dataMatrix.FrameId);
int width = 156;
int height = 46;
Assert.Equal(width, dataMatrix.Metadata[0]);
Assert.Equal(height, dataMatrix.Metadata[1]);
var frameType = dataMatrix.Metadata[2] >> 4;
var pixelSize = dataMatrix.Metadata[2] % 16;
Assert.Equal(1, frameType);
Assert.Equal(2, pixelSize);
var errors = 0;
// Test the full range
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
// Skip the metadata
if (y == 0 && (x == 0 || x == 1))
{
continue;
}
var ix = y * width * 3 + x * 3;
var x1 = testData[ix];
var y1 = testData[ix + 1];
var z = testData[ix + 2];
if (x + 1 != x1 || y + 1 != y1 || z != 255)
{
errors++;
}
}
}
Assert.Equal(0, errors);
}
private void copyScreenToBlockingQueue()
{
Stopwatch sw = new Stopwatch();
sw.Start();
int fps = 0;//统计FPS
bool nullFrame = true;
while (isSendPic)
{
while (!picFlag)
{
try
{
manulResetEvent.WaitOne();
}
catch (ThreadInterruptedException ex)
{
manulResetEvent.Reset();
Console.WriteLine(ex.Message);
ErrorInfo.getErrorWriter().writeErrorMassageToFile(ex.Message);
return;
}
}
while (picFlag)
{
try
{
Thread.Sleep(dynamicTime);
// Console.WriteLine("screenCopyThread***********");
if (isWin8Above)//above win8 version
{
/**采用DXGI形式获取桌面，只能使用在win8以上系统，效率比较高，用来代替Mirror Driver*/
DesktopFrame frame = CopyScreen.getScreenPicDXGI();
if (frame != null)
{
if (nullFrame)
{
//第一针总是黑屏，所以直接舍弃
nullFrame = false;
continue;
}
fps++;
BitmapWithCursor bitmapWithCursor = new BitmapWithCursor();
bitmapWithCursor.setCursorPoint(new ShortPoint(System.Windows.Forms.Cursor.Position.X, System.Windows.Forms.Cursor.Position.Y));
bitmapWithCursor.setScreenBitmap(frame.DesktopImage);
bitmapWithCursor.dirtyRecs = frame.UpdatedRegions;
screenCopyQueue.Enqueue(bitmapWithCursor);
textBoxCSQ.Text = "" + screenCopyQueue.getQueueSize();
}
}
else//below win8
{
int cursorX, cursorY;
/* 采用的GDI形式获取桌面图形，效率比较低*/
Bitmap btm = CopyScreen.getScreenPic(out cursorX, out cursorY);
if (btm != null)
{
fps++;
BitmapWithCursor bitmapWithCursor = new BitmapWithCursor();
bitmapWithCursor.setCursorPoint(new ShortPoint(cursorX, cursorY));
bitmapWithCursor.setScreenBitmap(btm);
screenCopyQueue.Enqueue(bitmapWithCursor);
textBoxCSQ.Text = "" + screenCopyQueue.getQueueSize();
}
}
/**fps count*/
if (sw.ElapsedMilliseconds > 1000)
{
sw.Restart();
textBoxFPS.Text = "" + fps;
fps = 0;
}
}
catch (Exception ex)
{
Console.WriteLine(ex.Message);
ErrorInfo.getErrorWriter().writeErrorMassageToFile(ex.Message);
return;
}
}
}
}
