Author: szh

Can you beat Mario Bros 1-1 with ONLY your face? 

(Turns out yes, but with many trials, slow game play, and very little chance to become a top ranked gamer.)

I downloaded FaceOSC, investigating how the different components worked, and read over the examples + their code to see what I could do with FaceOSC, and how the program is ran.

I really love the "Mario Bros 1-1 but with a twist" trend so I thought it would be funny to make a "Can you beat Mario Bros 1-1 with ONLY your face?"

As I tested my own game, the exact threshold for where the "center" of the game is was rather unclear, because I was estimated of where the face would be (assuming it was relatively center). Therefore, I later then added a "centering" section in the beginning, where the program would wait a couple seconds for the player to calibrate themselves before setting the thresholds for the left and right side.

I also later switch the left and right because the camera is flips the user so that left is their left eye, but can be confusing for when the player is actually playing cause it looks to be the opposite of what the intended result actually is.

FaceOSC and Processing needs to be installed before usage:

//
// Sabrina Zhai
// 9.23.19
//
// This program is intended to be used with SuperMarioBros, 
// allowing players to play 1-1 with a twist.
//
// This code is adapted from FaceOSCReceiver.
//
// A template for receiving face tracking osc messages from
// Kyle McDonald's FaceOSC https://github.com/kylemcdonald/ofxFaceTracker
//
// 2012 Dan Wilcox danomatika.com
// for the IACD Spring 2012 class at the CMU School of Art
//
// adapted from from Greg Borenstein's 2011 example
// http://www.gregborenstein.com/
// https://gist.github.com/1603230
// 
//

import oscP5.*;
OscP5 oscP5;

import java.awt.*;
import java.awt.event.*;
import java.awt.event.KeyEvent.*;

// num faces found
int found;

// pose
float poseScale;
PVector posePosition = new PVector();
PVector poseOrientation = new PVector();

// gesture
float mouthHeight;
float mouthWidth;
float eyeLeft;
float eyeRight;
float eyebrowLeft;
float eyebrowRight;
float jaw;
float nostrils;

Robot robot;

float openThreshold = 4;
float leftThreshold; // for the left SIDE and not the left EYE
float rightThreshold;

float previousMouthHeight;
float previousPosition;

int begin; 
int duration = 3;
int time = 3;
boolean faceSet = false;

void setup() {
  size(640, 480);
  frameRate(30);

  begin = millis();  

  oscP5 = new OscP5(this, 8338);
  oscP5.plug(this, "found", "/found");
  oscP5.plug(this, "poseScale", "/pose/scale");
  oscP5.plug(this, "posePosition", "/pose/position");
  oscP5.plug(this, "", "/pose/orientation");
  oscP5.plug(this, "mouthWidthReceived", "/gesture/mouth/width");
  oscP5.plug(this, "mouthHeightReceived", "/gesture/mouth/height");
  oscP5.plug(this, "eyeLeftReceived", "/gesture/eye/left");
  oscP5.plug(this, "eyeRightReceived", "/gesture/eye/right");
  oscP5.plug(this, "eyebrowLeftReceived", "/gesture/eyebrow/left");
  oscP5.plug(this, "eyebrowRightReceived", "/gesture/eyebrow/right");
  oscP5.plug(this, "jawReceived", "/gesture/jaw");
  oscP5.plug(this, "nostrilsReceived", "/gesture/nostrils");

  //Sets up the Robot to type into the computer
  try {
    robot = new Robot();
    robot.setAutoDelay(0);
  } 
  catch (AWTException e) { // (Exception e) {
    e.printStackTrace();
  }
}

void draw() {
  background(255);
  stroke(0);
  textSize(18);

  if (time > 0) { 
    time = duration - (millis() - begin)/1000;
    text("Setting current face position as center in..." + time, 10, 20);
  } else if (!faceSet) {  
    text("Setting current face position as center in...Face set!", 10, 20);

    //Set the face's threshold positions
    leftThreshold = posePosition.x - 75; 
    rightThreshold = posePosition.x + 75; 
    faceSet = true;
  }

  //Helps user see where the threshold to move their head is
  line(leftThreshold, 0, leftThreshold, height);
  line(rightThreshold, 0, rightThreshold, height);

   // Actions after a face is found
  if (found > 0) { 

    // Draw the face
    translate(posePosition.x, posePosition.y);
    scale(poseScale/2);
    noFill();
    ellipse(-20, eyeLeft * -9, 20, 7);
    ellipse(20, eyeRight * -9, 20, 7);
    ellipse(0, 20, mouthWidth* 3, mouthHeight * 3);
    ellipse(-5, nostrils * -1, 7, 3);
    ellipse(5, nostrils * -1, 7, 3);
    rectMode(CENTER);
    fill(0);
    rect(-20, eyebrowLeft * -5, 25, 5);
    rect(20, eyebrowRight * -5, 25, 5);

    // Makes Mario jump
    if (mouthHeight > openThreshold) { // Mouth open (continuously)
      robot.keyPress(java.awt.event.KeyEvent.VK_UP);
      if (previousMouthHeight < openThreshold) { // If the mouth is only opened ONCE (closed)
        robot.keyRelease(java.awt.event.KeyEvent.VK_UP);
      }
    }
    previousMouthHeight = mouthHeight;

    // Moves Mario to the left (user moves to the right)
    if (posePosition.x < leftThreshold && previousPosition < leftThreshold) {
      robot.keyPress(java.awt.event.KeyEvent.VK_LEFT);
    } else {
      robot.keyRelease(java.awt.event.KeyEvent.VK_LEFT);
    }

    // Moves Mario to the right (user moves to the left)
    if (posePosition.x > rightThreshold && previousPosition > rightThreshold) {
      robot.keyPress(java.awt.event.KeyEvent.VK_RIGHT);
    } else {
      robot.keyRelease(java.awt.event.KeyEvent.VK_RIGHT);
    }
    previousPosition = posePosition.x;
  }
}


// OSC CALLBACK FUNCTIONS
public void found(int i) {
  //println("found: " + i);
  found = i;
}

public void poseScale(float s) {
  //println("scale: " + s);
  poseScale = s;
}

public void posePosition(float x, float y) {
  println("pose position\tX: " + x + " Y: " + y );
  posePosition.set(x, y, 0);
}

public void poseOrientation(float x, float y, float z) {
  println("pose orientation\tX: " + x + " Y: " + y + " Z: " + z);
  poseOrientation.set(x, y, z);
}

public void mouthWidthReceived(float w) {
  //println("mouth Width: " + w);
  mouthWidth = w;
}

public void mouthHeightReceived(float h) {
  //println("mouth height: " + h);
  mouthHeight = h;
}

public void eyeLeftReceived(float f) {
  //println("eye left: " + f);
  eyeLeft = f;
}

public void eyeRightReceived(float f) {
  //println("eye right: " + f);
  eyeRight = f;
}

public void eyebrowLeftReceived(float f) {
  //println("eyebrow left: " + f);
  eyebrowLeft = f;
}

public void eyebrowRightReceived(float f) {
  //println("eyebrow right: " + f);
  eyebrowRight = f;
}

public void jawReceived(float f) {
  //println("jaw: " + f);
  jaw = f;
}

public void nostrilsReceived(float f) {
  //println("nostrils: " + f);
  nostrils = f;
}

// all other OSC messages end up here
void oscEvent(OscMessage m) {
  if (m.isPlugged() == false) {
    //println("UNPLUGGED: " + m);
  }
}