/// <summary>
/// Extracts the current piece from the board.
/// The piece is only searched in the original orientation and x coordinate, must therefore be untouched.
/// </summary>
/// <param name="screenshot">The screenshot to extract the piece from.</param>
/// <param name="maxFallingDistance">Expected maximal falling distance.</param>
/// <returns>The piece and it's probability.</returns>
public ProbabilisticResult <Piece> ExtractSpawnedPieceFuzzy(IScreenshot screenshot, int maxFallingDistance)
{
if (screenshot == null)
{
throw new ArgumentNullException(nameof(screenshot));
}
if (maxFallingDistance < 0)
{
throw new ArgumentException("maxFallingDistance must be positive");
}
double bestProbability = 0;
Piece expectedPiece = null;
for (int yDelta = 0; yDelta <= maxFallingDistance; yDelta++)
{
foreach (var tetromino in Tetriminos.All)
{
var piece = new Piece(tetromino, 0, 0, -yDelta);
var probability = _matcher.GetProbabilityCurrentPiece(screenshot, piece);
if (probability > bestProbability)
{
bestProbability = probability;
expectedPiece = piece;
}
}
}
return(new ProbabilisticResult <Piece>(expectedPiece, bestProbability));
}
/// <summary>
/// Extracts the next piece visible on the screenshot.
/// </summary>
/// <param name="screenshot">The screenshot to extract the piece from.</param>
/// <returns>The next Tetrimino.</returns>
public ProbabilisticResult <Tetrimino> ExtractNextPieceFuzzy(IScreenshot screenshot)
{
// relevant tiles on the screen: x : 14 - 17, y : 13 - 16
double bestProbability = double.NegativeInfinity;
Tetrimino?bestTetrimino = null;
foreach (var tetrimino in Tetriminos.All)
{
//var piece = new Piece(tetromino, 0, TetrisConstants.NextPieceTemplateTileCoordinates.X, TetrisConstants.NextPieceTemplateTileCoordinates.Y);
//var probability = _matcher.GetProbabilityCurrentPiece(screenshot, piece);
var probability = _matcher.GetProbabilityNextPiece(screenshot, tetrimino);
if (probability > bestProbability)
{
bestProbability = probability;
bestTetrimino = tetrimino;
}
}
if (!bestTetrimino.HasValue)
{
throw new Exception("No possible value for tetrimino found!");
}
return(new ProbabilisticResult <Tetrimino>(bestTetrimino.Value, bestProbability));
}
public double GetProbability(IScreenshot screenshot, Piece expected)
{
int errors = 0;
for (int x = -1; x < 3; x++)
{
for (int y = -1; y < 3; y++)
{
var coordinates = Coordinates.PieceToTile(expected.X + x, expected.Y + y);
var isBlockReal = expected.Shape.IsSquareOccupied(x, y);
var isBlockExpected = IsTileBlock(screenshot, coordinates.X, coordinates.Y);
if (isBlockReal != isBlockExpected)
{
errors++;
}
}
}
if (errors > 0 && expected.Tetrimino == Tetrimino.I && expected.Y == 0 && expected.Orientation == 1)
{
// TODO: remove this ugly hack
// subtract one because the I piece is not completly visible in upright position
errors--;
}
double normalizedError = errors / 16.0;
return(1 - normalizedError);
}
/// <summary>
/// Extracts the current piece from the board.
/// </summary>
/// <param name="screenshot">The screenshot to extract the piece from.</param>
/// <param name="piece">The piece to search.</param>
/// <param name="maxFallingDistance">Expected maximal falling distance.</param>
/// <returns>The piece and it's probability.</returns>
public ProbabilisticResult <Piece> ExtractKnownPieceFuzzy(IScreenshot screenshot, Piece piece, int maxFallingDistance)
{
if (screenshot == null)
{
throw new ArgumentNullException(nameof(screenshot));
}
if (maxFallingDistance < 0)
{
throw new ArgumentException("maxFallingDistance must be positive");
}
double bestProbability = 0;
Piece expectedPiece = null;
Piece testPiece = new Piece(piece);
for (int yDelta = 0; yDelta <= maxFallingDistance; yDelta++)
{
var probability = _matcher.GetProbabilityCurrentPiece(screenshot, testPiece);
if (probability > bestProbability)
{
bestProbability = probability;
expectedPiece = new Piece(testPiece);
}
testPiece.Fall();
}
return(new ProbabilisticResult <Piece>(expectedPiece, bestProbability));
}
public bool IsGameOverSingleplayer(IScreenshot screenshot)
{
return(IsMatch(screenshot,
_gameoverSingleplayerReferenceImage,
_roiBoard,
_thresholdGameoverSingleplayer));
}
/// <summary>
/// Extracts the next piece visible on the screenshot.
/// Throws an exception, if the next piece is not visible on the screenshot.
/// </summary>
/// <param name="screenshot">The screenshot to extract the piece from.</param>
/// <returns>The next Tetrimino.</returns>
public Tetrimino ExtractNextPiece(IScreenshot screenshot)
{
// relevant tiles on the screen: x : 14 - 17, y : 13 - 16
ushort mask = 0;
for (int x = 0; x < 4; x++)
{
for (int y = 0; y < 4; y++)
{
byte mean = screenshot.GetTileMean(TetrisConstants.NextPieceTileOrigin.X + x, TetrisConstants.NextPieceTileOrigin.Y + y);
if (IsBlock(mean))
{
int index = 4 * (2 - y) + (x);
mask |= (ushort)(1 << index);
}
}
}
var nextPiece = Piece.FromMask(mask);
if (nextPiece?.Tetrimino == null)
{
throw new ApplicationException("Next piece not visible on the screenshot");
}
return(nextPiece.Tetrimino);
}
public Board Update(IScreenshot screenshot, Board board, Piece piece)
{
var newBoard = new Board();
var headInBoardCoordinates = piece.Shape.Head
.Select(h => Coordinates.PieceToBoard(piece.X + h.X, piece.Y + h.Y))
.ToList();
for (int x = 0; x < board.Width; x++)
{
for (int y = 0; y < GameBoyConstants.ScreenHeight / GameBoyConstants.TileSize; y++)
{
if (headInBoardCoordinates.Any(h => h.X == x && h.Y <= y))
{
break;
}
if (_matcher.GetProbabilityBoardBlock(screenshot, x, y) >= _thresholdBlock)
{
newBoard.Occupy(x, y);
}
}
}
return(newBoard);
}
public bool IsStart(IScreenshot screenshot)
{
return(IsMatch(screenshot,
_startMultiplayerReferenceImage,
_roiBoard,
_thresholdStartMultiplayer));
}
public bool IsGameOverMultiplayer(IScreenshot screenshot)
{
return(IsMatch(screenshot,
_gameoverMultiplayerReferenceImage,
_roiMultiplayerGameover,
_thresholdGameoverMultiplayer));
}
// Searches the piece only in the spawning area on top of the board and a specifiec number of tiles below
// Searches only pieces with the orientation 0
// returns null, if the piece was not found in the spawning area or the specifiec number of tiles below
public Piece ExtractSpawnedPiece(IScreenshot screenshot, int searchHeight)
{
if (searchHeight < 0)
{
throw new ArgumentException("searchHeight must be positive.");
}
for (int yDelta = 0; yDelta <= searchHeight; yDelta++)
{
ushort mask = 0;
for (int x = 0; x < 4; x++)
{
for (int y = 0; y < 3; y++)
{
if (IsTileBlock(screenshot, TetrisConstants.CurrentTileOrigin.X + x, TetrisConstants.CurrentTileOrigin.Y + y + yDelta))
{
int index = 4 * (2 - y) + (x);
mask |= (ushort)(1 << index);
}
}
}
var piece = Piece.FromMask(mask);
if (piece != null)
{
piece.Fall(yDelta);
return(piece);
}
}
// not found
return(null);
}
public Piece ExtractCurrentPiece(IScreenshot screenshot, Tetrimino?tetrimino, int maxFallDistance)
{
if (tetrimino.HasValue)
{
return(ExtractCurrentPieceKnown(screenshot, tetrimino.Value, maxFallDistance));
}
return(ExtractCurrentPieceUnknown(screenshot, maxFallDistance));
}
public int Update(IScreenshot screenshot)
{
string sql = @"UPDATE Screenshots
SET Url = @Url, CreatedAt = @CreatedAt, UpdatedAt = @UpdatedAt
WHERE Url = @Url";
return db.Execute(sql, screenshot);
}
public ScreenshotCreatedEventArgs(IScreenshot screenshot)
{
if (screenshot == null)
{
throw new ArgumentNullException("screenshot");
}
Screenshot = screenshot;
}
public void Store(IScreenshot screenshot)
{
var name = Guid.NewGuid() + ".jpg";
File.WriteAllBytes(Path.Combine(_path, name), screenshot.Content);
RaiseScreenshotStored(new ScreenshotStoredEventArgs(name));
}
public void NotRecognizeNextPiece(string imageKey, IScreenshot screenshot)
{
var result = _pieceExtractor.ExtractNextPieceFuzzy(screenshot);
Assert.True(result.IsRejected(_probabilityThreshold));
Assert.GreaterOrEqual(result.Probability, 0.0);
Assert.Less(result.Probability, _probabilityThreshold);
}
static IPlatformScreenshot AsPlatform(this IScreenshot screenshot)
{
if (screenshot is not IPlatformScreenshot platform)
{
throw new PlatformNotSupportedException("This implementation of IScreenshot does not implement IPlatformScreenshot.");
}
return(platform);
}
// This is needed from rendering transparent background as black
private Bitmap DrawImageOnWhiteBackground(IScreenshot screenshot, int width, int height)
{
Bitmap blank = new Bitmap(width, height);
Graphics g = Graphics.FromImage(blank);
g.Clear(Color.White);
g.DrawImage(screenshot.Image, 0, 0, width, height);
return(new Bitmap(blank));
}
private Piece ExtractCurrentPieceKnown(IScreenshot screenshot, Tetrimino tetrimino, int maxFallDistance)
{
var foundPiece = _tetrisExtractor.ExtractSpawnedPiece(screenshot, maxFallDistance);
if (foundPiece != null && foundPiece.Tetrimino == tetrimino && foundPiece.IsUntouched)
{
return(foundPiece);
}
return(null);
}
public string UploadScreenshotToAmazonS3RecordTransaction()
{
//Upload screenshot to Amazon S3
screenshot = upload.AmazonS3();
//Insert screenshot into the Screenshots table
repository.Add(screenshot);
return screenshot.Url;
}
/*
* private bool IsMatch(IScreenshot screenshot, Mat reference, double threshold)
* {
* var result = new Mat();
*
* CvInvoke.AbsDiff(screenshot.Image, reference, result);
* var mean = CvInvoke.Mean(result);
*
* return mean.V0 <= threshold * 255;
* }
*/
private bool IsMatch(IScreenshot screenshot, Mat reference, Rectangle roi, double threshold)
{
var result = new Mat();
var screenshotRoi = new Mat(screenshot.Image, roi);
var referenceRoi = new Mat(reference, roi);
CvInvoke.AbsDiff(screenshotRoi, referenceRoi, result);
var mean = CvInvoke.Mean(result);
return(mean.V0 <= threshold * 255);
}
public GameState Extract(IScreenshot screenshot, GameState currentGameState)
{
//var board = ExtractBoard();
var board = new Board();
var currentPiece = ExtractSpawnedPieceOrigin(screenshot);
var nextPiece = ExtractNextPiece(screenshot);
var gameState = new GameState(board, currentPiece, nextPiece);
return(gameState);
}
public void RecognizeNextPiece(string imageKey, IScreenshot screenshot, Tetrimino nextPieceExpected)
{
_nextPiece++;
var tetromino = _extractor.ExtractNextPiece(screenshot);
Assert.NotNull(tetromino);
Assert.AreEqual(nextPieceExpected, tetromino);
_nextPieceRecognized++;
}
public Tetrimino?ExtractNextPiece(IScreenshot screenshot)
{
var result = _pieceExtractor.ExtractNextPieceFuzzy(screenshot);
if (result.IsAccepted(_thresholdNextPiece))
{
return(result.Result);
}
return(null);
}
public ScreenCapture(int width, int height)
{
//win8/win10
if (Environment.OSVersion.Version >= new Version(6, 2))
{
_screenshot = new DXGIScreenshot();
}
else
{
_screenshot = new GDIScreenshot(width, height);
}
}
public void RecognizeUnknownSpawnedPiece(string imageKey, IScreenshot screenshot, Piece currentPieceExpected)
{
_unknownSpawnedPiece++;
var searchHeight = currentPieceExpected.FallHeight;
var piece = _extractor.ExtractSpawnedPiece(screenshot, searchHeight);
Assert.NotNull(piece);
Assert.AreEqual(currentPieceExpected, piece);
_unknownSpawnedPieceRecognized++;
}
public void NotRecognizeUnknownSpawnedPiece(string imageKey, IScreenshot screenshot)
{
_unknownSpawnedPiece++;
// TODO: make tests with higher search distance!
var searchHeight = 3;
var piece = _extractor.ExtractSpawnedPiece(screenshot, searchHeight);
Assert.Null(piece);
_unknownSpawnedPieceRecognized++;
}
public void RecognizeNextPiece(string imageKey, IScreenshot screenshot, Tetrimino nextPieceExpected)
{
_nextPiece++;
var result = _pieceExtractor.ExtractNextPieceFuzzy(screenshot);
Assert.True(result.IsAccepted(_probabilityThreshold));
Assert.AreEqual(nextPieceExpected, result.Result);
Assert.GreaterOrEqual(result.Probability, _probabilityThreshold);
Assert.LessOrEqual(result.Probability, 1.0);
_nextPieceRecognized++;
}
public void NotRecognizeUnknownSpawnedPiece(string imageKey, IScreenshot screenshot)
{
_unknownSpawnedPiece++;
// TODO: make tests with higher search distance!
var maxFallingDistance = 3;
var result = _pieceExtractor.ExtractSpawnedPieceFuzzy(screenshot, maxFallingDistance);
Assert.True(result.IsRejected(_probabilityThreshold));
Assert.GreaterOrEqual(result.Probability, 0.0);
Assert.Less(result.Probability, _probabilityThreshold);
_unknownSpawnedPieceRecognized++;
}
private int GetAddedLines(IScreenshot screenshot, Board board, double threshold)
{
int numLines = 0;
for (; numLines < 4; numLines++)
{
var probability = GetHorizonRaisedProbability(screenshot, board, numLines);
if (probability < threshold)
{
break;
}
}
return(numLines);
}
private double GetHorizonRaisedProbability(IScreenshot screenshot, Board board, int height = 0)
{
double probabilitySum = 0.0;
foreach (var block in board.GetHorizon(height))
{
if (block.Y < board.Height)
{
probabilitySum += _matcher.GetProbabilityBoardBlock(screenshot, block.X, block.Y);
}
}
return(probabilitySum / board.Width);
}
// x and y are in board coordinates
private byte GetTileBlockMean(IScreenshot screenshot, int x, int y)
{
// TODO: make relative to board size?
// ignore walls
if (x < 2 || x > 11)
{
return(255);
}
if (y < 0 || y > 17)
{
return(255);
}
return(screenshot.GetTileMean(x, y));
}
public Upload()
{
config = ConfigurationManager.OpenMappedExeConfiguration(new ExeConfigurationFileMap() { ExeConfigFilename = "App.config" }, ConfigurationUserLevel.None);
string AWSAccessKey = config.AppSettings.Settings["AWSAccessKey"].Value;
string AWSSecretKey = config.AppSettings.Settings["AWSSecretKey"].Value;
s3Client = new AmazonS3Client(AWSAccessKey, AWSSecretKey, RegionEndpoint.USEast1);
s3RegionEndpoint = RegionEndpoint.USEast1;
FilePath = Path.GetDirectoryName(Assembly.GetEntryAssembly().Location);
FileName = config.AppSettings.Settings["FileName"].Value;
BucketName = config.AppSettings.Settings["BucketName"].Value;
screenshot = new Screenshot();
}
public void PieceMatchingNextPiecePositives(string imageKey, IScreenshot screenshot, Tetrimino nextPieceExpected)
{
_nextPiecesTotal++;
var probabilityNextPiece = _matcher.GetProbabilityNextPiece(screenshot, nextPieceExpected);
_logger.Info($"PieceMatchingNextPiecePositives: {probabilityNextPiece * 100.0:F}");
var nextPieceFound = probabilityNextPiece >= _probabilityThreshold;
if (nextPieceFound)
{
_nextPiecesRecognized++;
}
Assert.True(nextPieceFound);
}
public int Add(IScreenshot screenshot)
{
string sql = "INSERT INTO Screenshots (Url, CreatedAt, UpdatedAt) VALUES (@Url, @CreatedAt, @UpdatedAt);";
return db.Execute(sql, screenshot);
}