void camera_NewFrame(object sender, AForge.Video.NewFrameEventArgs eventArgs)
{
globFrameCount++;
DateTime captureTime = DateTime.Now;
Bitmap frame = eventArgs.Frame;
BitmapData bmpData = frame.LockBits(new Rectangle(0, 0, frame.Width, frame.Height), ImageLockMode.ReadWrite,
frame.PixelFormat);
Image <Bgr, byte> image = new Image <Bgr, byte>(frame.Width, frame.Height, bmpData.Stride, bmpData.Scan0);
//to prevent any corrupted memory errors that crop up for some reason
Image <Bgr, byte> image2 = image.Clone();
//dispose all unused image data to prevent memory leaks
image.Dispose();
frame.UnlockBits(bmpData);
frame.Dispose();
eventArgs.Frame.Dispose();
CapturedImage capturedImage = new CapturedImage(image2, captureTime);
MostRecentImage = capturedImage;
_logic.UpdateAndPredictNotes(capturedImage);
TimeSpan timeElapsed = DateTime.Now - captureTime;
if (timeElapsed.TotalMilliseconds > 45)
{
Debug.WriteLine("WARNING: Capture took {0} ms.", timeElapsed.TotalMilliseconds);
}
ImageQueue.Enqueue(capturedImage);
}
private void button2_Click_1(object sender, EventArgs e)
{
secondImage = new CapturedImage(new Image<Bgr, byte>(@"D:\drumbot\panic attack partial\" + imageId.ToString().PadLeft(6, '0') + ".bmp"), DateTime.Now);
MostRecentImage = secondImage;
logic.UpdateAndPredictNotes(MostRecentImage);
DrawBigPicture();
RefreshPictureBoxes();
//just to get the debug console output
//foreach(Note n in logic.CurrentNotes)
//{
// double d = n.FramesUntilHit;
//}
imageId++;
}
private void button2_Click_1(object sender, EventArgs e)
{
secondImage = new CapturedImage(new Image <Bgr, byte>(@"D:\drumbot\panic attack partial\" + imageId.ToString().PadLeft(6, '0') + ".bmp"), DateTime.Now);
MostRecentImage = secondImage;
logic.UpdateAndPredictNotes(MostRecentImage);
DrawBigPicture();
RefreshPictureBoxes();
//just to get the debug console output
//foreach(Note n in logic.CurrentNotes)
//{
// double d = n.FramesUntilHit;
//}
imageId++;
}
private void ReloadTracks()
{
logic.CurrentNotes.Clear();
if (firstImage != null)
{
firstImage.Dispose();
}
if (secondImage != null)
{
secondImage.Dispose();
}
firstImage = new CapturedImage(new Image <Bgr, byte>(@"D:\drumbot\panic attack partial\" + "000475.bmp"), DateTime.Now);
secondImage = new CapturedImage(new Image <Bgr, byte>(@"D:\drumbot\panic attack partial\" + "000476.bmp"),
firstImage.CaptureTime.AddMilliseconds(40));
MostRecentImage = firstImage;
logic.UpdateAndPredictNotes(MostRecentImage);
MostRecentImage = secondImage;
logic.UpdateAndPredictNotes(MostRecentImage);
DrawBigPicture();
}
private void button2_Click_1(object sender, EventArgs e)
{
_fakeOutput.ResetHitIndicators();
previousImage = currentImage;
currentImage = new CapturedImage(new Image<Bgr, byte>(_imageFileList[nextImageIndex]),
previousImage.CaptureTime.AddMilliseconds(40));
nextImageIndex++;
MostRecentImage = currentImage;
logic.UpdateAndPredictNotes(MostRecentImage);
DrawBigPicture();
RefreshPictureBoxes();
if (_fakeOutput.RedHit)
redButton.BackColor = Color.Red;
else
redButton.BackColor = Color.Gray;
if (_fakeOutput.YellowHit)
yellowButton.BackColor = Color.Yellow;
else
yellowButton.BackColor = Color.Gray;
if (_fakeOutput.BlueHit)
blueButton.BackColor = Color.Blue;
else
blueButton.BackColor = Color.Gray;
if (_fakeOutput.GreenHit)
greenButton.BackColor = Color.LawnGreen;
else
greenButton.BackColor = Color.Gray;
if (_fakeOutput.OrangeHit)
orangeButton.BackColor = Color.Orange;
else
orangeButton.BackColor = Color.Gray;
//just to get the debug console output
//foreach(Note n in logic.CurrentNotes)
//{
// double d = n.FramesUntilHit;
//}
}
public void ImageWriter()
{
int frameCount = 0;
while (true)
{
if (Flagged)
{
_flagId++;
System.IO.Directory.CreateDirectory(@"D:\flagged\" + _flagId);
CapturedImage[] imageArray = ImageQueue.ToArray();
for (int i = 0; i < imageArray.Length; i++)
{
CapturedImage image = imageArray[i];
string filePath = string.Format(@"D:\flagged\{0}\{1}.png", _flagId, i.ToString().PadLeft(5, '0'));
image.Save(filePath);
}
Flagged = false;
}
//keep the last 100 images in the buffer
if (ImageQueue.Count > 100)
{
if (globCaptureStart == DateTime.MinValue)
{
globCaptureStart = DateTime.Now;
}
CapturedImage image = ImageQueue.Peek();
if (Recording)
{
image.Save("D:\\captures\\" + frameCount.ToString().PadLeft(5, '0') + ".png");
frameCount++;
}
ImageQueue.Dequeue();
image.Dispose();
}
Thread.Sleep(1);
}
}
public void UpdateAndPredictNotes(CapturedImage newFrame)
{
List<Note> updatedNotes = new List<Note>();
foreach (Note newNote in newFrame.Notes)
newNote.MatchedToOldNote = false;
//if we keep the enum with orange at the bottom (highest int conversion) then normal notes will get priority matching
foreach (Note oldNote in CurrentNotes.OrderByDescending(n => n.DetectedInFrames).ThenBy(n => n.Color))
{
oldNote.MatchedToNewNote = false;
//find a new note that best matches the old one
//the note has to be in the same track and has to be below the previous position
//it does NOT have to be detect as the same color because our detection is not perfect
//we have to account for that and then give it the most likely color
Note note = oldNote;
var possibleMatches =
newFrame.Notes.Where(
nn =>
nn.TrackColor == note.TrackColor && nn.Rectangle.Y >= note.Rectangle.Y
&& nn.MatchedToOldNote == false);
//find the note closest to the previous position by euclidean distance
//if we don't have any velocity data yet then this will simply return the previous position
Rectangle predictedPosition = oldNote.PredictedPosition;
//what to do if this gives us 0 results ?
int matchCount = possibleMatches.Count();
if (matchCount == 0)
{
//went offscreen ?
continue;
}
//if we have a lot of notes with the right color to pick from we might as well assume that all notes
//have been classified correctly
var colorMatches = possibleMatches.Where(pm => pm.Color == oldNote.Color);
if(colorMatches.Count() > 0)
possibleMatches = colorMatches;
//find the note closest to the predicted positions CENTER by euclidean distance
var bestMatch =
possibleMatches.OrderBy(
nn =>
Math.Sqrt(Math.Pow(nn.Rectangle.Center().X - predictedPosition.Center().X, 2)
+ Math.Pow(nn.Rectangle.Center().Y - predictedPosition.Center().Y, 2))).First();
bestMatch.PerFrameVelocityX = bestMatch.Rectangle.Center().X - oldNote.Rectangle.Center().X;
bestMatch.PerFrameVelocityY = bestMatch.Rectangle.Center().Y - oldNote.Rectangle.Center().Y;
double xMismatch = bestMatch.Rectangle.Center().X - predictedPosition.Center().X;
double yMismatch = bestMatch.Rectangle.Center().Y - predictedPosition.Center().Y;
//made up numbers
if(bestMatch.Color != oldNote.Color && xMismatch < 2 && yMismatch < 3)
{
bestMatch.Color = oldNote.Color;
}
//Debug.WriteLine("XDif:{0} \nYDif:{1}", xMismatch, yMismatch);
//just to make sure we dont accidentially miscolor bass notes
double ratio = bestMatch.Rectangle.Width/(double) bestMatch.Rectangle.Height;
if (ratio >= 4 && bestMatch.Color != NoteType.Orange)
bestMatch.Color = NoteType.Orange;
else if (ratio < 3.0)
bestMatch.Color = bestMatch.TrackColor;
//if the ratio is between 3.0 and 4.0 we cannot be certain if it is a normal or a bass note so we keep the original guess
oldNote.MatchedToNewNote = true;
bestMatch.MatchedToOldNote = true;
//keep track of how many times we have seen this note
bestMatch.DetectedInFrames += oldNote.DetectedInFrames;
bestMatch.FramesSinceLastDetection = 0;
updatedNotes.Add(bestMatch);
}
//add the new notes
updatedNotes.AddRange(newFrame.Notes.Where(n => n.MatchedToOldNote == false));
bool orangeNoteTriggered = false;
foreach(Note note in CurrentNotes.Where(n=>n.MatchedToNewNote == false && n.FramesSinceLastDetection < 10))
{
//get the predicted position in the CURRENT frame
note.UpdateUsingPrediction();
//check if the predicted position in the next frame is at the target
if (note.AtTargetPoint(note.PredictedPosition) || note.FramesUntilHit <= 2)
{
if (note.Color == NoteType.Orange)
{
if (orangeNoteTriggered || _framesSinceLastOrangeNote < 2)
continue;
orangeNoteTriggered = true;
}
_drumHardware.HitNote(note.Color);
}
else
{
updatedNotes.Add(note);
}
}
if (orangeNoteTriggered)
_framesSinceLastOrangeNote = 0;
else
_framesSinceLastOrangeNote++;
CurrentNotes = updatedNotes;
}
public void UpdateAndPredictNotes(CapturedImage newFrame)
{
List <Note> updatedNotes = new List <Note>();
foreach (Note newNote in newFrame.Notes)
{
newNote.MatchedToOldNote = false;
}
//if we keep the enum with orange at the bottom (highest int conversion) then normal notes will get priority matching
foreach (Note oldNote in CurrentNotes.OrderByDescending(n => n.DetectedInFrames).ThenBy(n => n.Color))
{
oldNote.MatchedToNewNote = false;
//find a new note that best matches the old one
//the note has to be in the same track and has to be below the previous position
//it does NOT have to be detect as the same color because our detection is not perfect
//we have to account for that and then give it the most likely color
Note note = oldNote;
var possibleMatches =
newFrame.Notes.Where(
nn =>
nn.TrackColor == note.TrackColor && nn.Rectangle.Y >= note.Rectangle.Y &&
nn.MatchedToOldNote == false);
//find the note closest to the previous position by euclidean distance
//if we don't have any velocity data yet then this will simply return the previous position
Rectangle predictedPosition = oldNote.PredictedPosition;
//what to do if this gives us 0 results ?
int matchCount = possibleMatches.Count();
if (matchCount == 0)
{
//went offscreen ?
continue;
}
//if we have a lot of notes with the right color to pick from we might as well assume that all notes
//have been classified correctly
var colorMatches = possibleMatches.Where(pm => pm.Color == oldNote.Color);
if (colorMatches.Count() > 0)
{
possibleMatches = colorMatches;
}
//find the note closest to the predicted positions CENTER by euclidean distance
var bestMatch =
possibleMatches.OrderBy(
nn =>
Math.Sqrt(Math.Pow(nn.Rectangle.Center().X - predictedPosition.Center().X, 2)
+ Math.Pow(nn.Rectangle.Center().Y - predictedPosition.Center().Y, 2))).First();
bestMatch.PerFrameVelocityX = bestMatch.Rectangle.Center().X - oldNote.Rectangle.Center().X;
bestMatch.PerFrameVelocityY = bestMatch.Rectangle.Center().Y - oldNote.Rectangle.Center().Y;
double xMismatch = bestMatch.Rectangle.Center().X - predictedPosition.Center().X;
double yMismatch = bestMatch.Rectangle.Center().Y - predictedPosition.Center().Y;
//made up numbers
if (bestMatch.Color != oldNote.Color && xMismatch < 2 && yMismatch < 3)
{
bestMatch.Color = oldNote.Color;
}
//Debug.WriteLine("XDif:{0} \nYDif:{1}", xMismatch, yMismatch);
//just to make sure we dont accidentially miscolor bass notes
double ratio = bestMatch.Rectangle.Width / (double)bestMatch.Rectangle.Height;
if (ratio >= 4 && bestMatch.Color != NoteType.Orange)
{
bestMatch.Color = NoteType.Orange;
}
else if (ratio < 3.0)
{
bestMatch.Color = bestMatch.TrackColor;
}
//if the ratio is between 3.0 and 4.0 we cannot be certain if it is a normal or a bass note so we keep the original guess
oldNote.MatchedToNewNote = true;
bestMatch.MatchedToOldNote = true;
//keep track of how many times we have seen this note
bestMatch.DetectedInFrames += oldNote.DetectedInFrames;
bestMatch.FramesSinceLastDetection = 0;
updatedNotes.Add(bestMatch);
}
//add the new notes
updatedNotes.AddRange(newFrame.Notes.Where(n => n.MatchedToOldNote == false));
bool orangeNoteTriggered = false;
foreach (Note note in CurrentNotes.Where(n => n.MatchedToNewNote == false && n.FramesSinceLastDetection < 10))
{
//get the predicted position in the CURRENT frame
note.UpdateUsingPrediction();
//check if the predicted position in the next frame is at the target
if (note.AtTargetPoint(note.PredictedPosition) || note.FramesUntilHit <= 2)
{
if (note.Color == NoteType.Orange)
{
if (orangeNoteTriggered || _framesSinceLastOrangeNote < 2)
{
continue;
}
orangeNoteTriggered = true;
}
_drumHardware.HitNote(note.Color);
}
else
{
updatedNotes.Add(note);
}
}
if (orangeNoteTriggered)
{
_framesSinceLastOrangeNote = 0;
}
else
{
_framesSinceLastOrangeNote++;
}
CurrentNotes = updatedNotes;
}
private void ReloadTracks()
{
logic.CurrentNotes.Clear();
if (firstImage != null)
firstImage.Dispose();
if (secondImage != null)
secondImage.Dispose();
firstImage = new CapturedImage(new Image<Bgr, byte>(@"D:\drumbot\panic attack partial\" + "000475.bmp"), DateTime.Now);
secondImage = new CapturedImage(new Image<Bgr, byte>(@"D:\drumbot\panic attack partial\" + "000476.bmp"),
firstImage.CaptureTime.AddMilliseconds(40));
MostRecentImage = firstImage;
logic.UpdateAndPredictNotes(MostRecentImage);
MostRecentImage = secondImage;
logic.UpdateAndPredictNotes(MostRecentImage);
DrawBigPicture();
}
private void ReloadTracks()
{
_fakeOutput.ResetHitIndicators();
logic.CurrentNotes.Clear();
if (previousImage != null)
{
previousImage.Dispose();
}
if (currentImage != null)
{
currentImage.Dispose();
}
previousImage = new CapturedImage(new Image <Bgr, byte>(_imageFileList[0]), DateTime.Now);
currentImage = new CapturedImage(new Image <Bgr, byte>(_imageFileList[1]), previousImage.CaptureTime.AddMilliseconds(40));
nextImageIndex = 2;
MostRecentImage = previousImage;
logic.UpdateAndPredictNotes(MostRecentImage);
MostRecentImage = currentImage;
logic.UpdateAndPredictNotes(MostRecentImage);
DrawBigPicture();
if (_fakeOutput.RedHit)
{
redButton.BackColor = Color.Red;
}
else
{
redButton.BackColor = Color.Gray;
}
if (_fakeOutput.YellowHit)
{
yellowButton.BackColor = Color.Yellow;
}
else
{
yellowButton.BackColor = Color.Gray;
}
if (_fakeOutput.BlueHit)
{
blueButton.BackColor = Color.Blue;
}
else
{
blueButton.BackColor = Color.Gray;
}
if (_fakeOutput.GreenHit)
{
greenButton.BackColor = Color.LawnGreen;
}
else
{
greenButton.BackColor = Color.Gray;
}
if (_fakeOutput.OrangeHit)
{
orangeButton.BackColor = Color.Orange;
}
else
{
orangeButton.BackColor = Color.Gray;
}
}
private void button2_Click_1(object sender, EventArgs e)
{
_fakeOutput.ResetHitIndicators();
previousImage = currentImage;
currentImage = new CapturedImage(new Image <Bgr, byte>(_imageFileList[nextImageIndex]),
previousImage.CaptureTime.AddMilliseconds(40));
nextImageIndex++;
MostRecentImage = currentImage;
logic.UpdateAndPredictNotes(MostRecentImage);
DrawBigPicture();
RefreshPictureBoxes();
if (_fakeOutput.RedHit)
{
redButton.BackColor = Color.Red;
}
else
{
redButton.BackColor = Color.Gray;
}
if (_fakeOutput.YellowHit)
{
yellowButton.BackColor = Color.Yellow;
}
else
{
yellowButton.BackColor = Color.Gray;
}
if (_fakeOutput.BlueHit)
{
blueButton.BackColor = Color.Blue;
}
else
{
blueButton.BackColor = Color.Gray;
}
if (_fakeOutput.GreenHit)
{
greenButton.BackColor = Color.LawnGreen;
}
else
{
greenButton.BackColor = Color.Gray;
}
if (_fakeOutput.OrangeHit)
{
orangeButton.BackColor = Color.Orange;
}
else
{
orangeButton.BackColor = Color.Gray;
}
//just to get the debug console output
//foreach(Note n in logic.CurrentNotes)
//{
// double d = n.FramesUntilHit;
//}
}
private void ReloadTracks()
{
_fakeOutput.ResetHitIndicators();
logic.CurrentNotes.Clear();
if (previousImage != null)
previousImage.Dispose();
if (currentImage != null)
currentImage.Dispose();
previousImage = new CapturedImage(new Image<Bgr, byte>(_imageFileList[0]), DateTime.Now);
currentImage = new CapturedImage(new Image<Bgr, byte>(_imageFileList[1]),previousImage.CaptureTime.AddMilliseconds(40));
nextImageIndex = 2;
MostRecentImage = previousImage;
logic.UpdateAndPredictNotes(MostRecentImage);
MostRecentImage = currentImage;
logic.UpdateAndPredictNotes(MostRecentImage);
DrawBigPicture();
if(_fakeOutput.RedHit)
redButton.BackColor = Color.Red;
else
redButton.BackColor = Color.Gray;
if(_fakeOutput.YellowHit)
yellowButton.BackColor = Color.Yellow;
else
yellowButton.BackColor = Color.Gray;
if(_fakeOutput.BlueHit)
blueButton.BackColor = Color.Blue;
else
blueButton.BackColor = Color.Gray;
if (_fakeOutput.GreenHit)
greenButton.BackColor = Color.LawnGreen;
else
greenButton.BackColor = Color.Gray;
if(_fakeOutput.OrangeHit)
orangeButton.BackColor = Color.Orange;
else
orangeButton.BackColor = Color.Gray;
}
void camera_NewFrame(object sender, AForge.Video.NewFrameEventArgs eventArgs)
{
globFrameCount++;
DateTime captureTime = DateTime.Now;
Bitmap frame = eventArgs.Frame;
BitmapData bmpData = frame.LockBits(new Rectangle(0, 0, frame.Width, frame.Height), ImageLockMode.ReadWrite,
frame.PixelFormat);
Image<Bgr, byte> image = new Image<Bgr, byte>(frame.Width, frame.Height, bmpData.Stride, bmpData.Scan0);
//to prevent any corrupted memory errors that crop up for some reason
Image<Bgr, byte> image2 = image.Clone();
//dispose all unused image data to prevent memory leaks
image.Dispose();
frame.UnlockBits(bmpData);
frame.Dispose();
eventArgs.Frame.Dispose();
CapturedImage capturedImage = new CapturedImage(image2, captureTime);
MostRecentImage = capturedImage;
_logic.UpdateAndPredictNotes(capturedImage);
TimeSpan timeElapsed = DateTime.Now - captureTime;
if (timeElapsed.TotalMilliseconds > 45)
{
Debug.WriteLine("WARNING: Capture took {0} ms.", timeElapsed.TotalMilliseconds);
}
ImageQueue.Enqueue(capturedImage);
}
