MediaPipe: A Complete Guide to Computer Vision

MediaPipe is an open-source library developed by Google that makes it easier to build computer vision and multimedia processing applications.

1. Installation and Setup

# Install dependencies
pip install mediapipe opencv-python numpy

# Import the required libraries
import mediapipe as mp
import cv2
import numpy as np

2. Pose Detection

2.1 Basic Setup

# Initialize MediaPipe Pose
mp_pose = mp.solutions.pose
pose = mp_pose.Pose(
    min_detection_confidence=0.5,
    min_tracking_confidence=0.5
)
mp_draw = mp.solutions.drawing_utils

def detect_pose(image):
    # Convert to RGB
    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    
    # Detect the pose
    results = pose.process(image_rgb)
    
    # Draw the landmarks
    if results.pose_landmarks:
        mp_draw.draw_landmarks(
            image,
            results.pose_landmarks,
            mp_pose.POSE_CONNECTIONS
        )
    
    return image

2.2 Movement Analysis

def analyze_movement(landmarks):
    # Extract key points
    left_shoulder = np.array([
        landmarks.landmark[mp_pose.PoseLandmark.LEFT_SHOULDER.value].x,
        landmarks.landmark[mp_pose.PoseLandmark.LEFT_SHOULDER.value].y
    ])
    
    right_shoulder = np.array([
        landmarks.landmark[mp_pose.PoseLandmark.RIGHT_SHOULDER.value].x,
        landmarks.landmark[mp_pose.PoseLandmark.RIGHT_SHOULDER.value].y
    ])
    
    # Calculate the shoulder angle
    shoulder_angle = np.arctan2(
        right_shoulder[1] - left_shoulder[1],
        right_shoulder[0] - left_shoulder[0]
    )
    
    return shoulder_angle

3. Face Detection

3.1 Setup

# Initialize MediaPipe Face Detection
mp_face_detection = mp.solutions.face_detection
face_detection = mp_face_detection.FaceDetection(
    min_detection_confidence=0.5
)

def detect_faces(image):
    # Convert to RGB
    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    
    # Detect faces
    results = face_detection.process(image_rgb)
    
    # Draw the bounding boxes
    if results.detections:
        for detection in results.detections:
            bbox = detection.location_data.relative_bounding_box
            h, w, _ = image.shape
            x = int(bbox.xmin * w)
            y = int(bbox.ymin * h)
            width = int(bbox.width * w)
            height = int(bbox.height * h)
            
            cv2.rectangle(image, (x, y), (x + width, y + height), (0, 255, 0), 2)
    
    return image

3.2 Expression Analysis

def analyze_expression(landmarks):
    # Extract face points
    left_eye = np.array([
        landmarks.landmark[mp_face_detection.FaceLandmark.LEFT_EYE.value].x,
        landmarks.landmark[mp_face_detection.FaceLandmark.LEFT_EYE.value].y
    ])
    
    right_eye = np.array([
        landmarks.landmark[mp_face_detection.FaceLandmark.RIGHT_EYE.value].x,
        landmarks.landmark[mp_face_detection.FaceLandmark.RIGHT_EYE.value].y
    ])
    
    # Calculate the distance between the eyes
    eye_distance = np.linalg.norm(right_eye - left_eye)
    
    return eye_distance

4. Hand Detection

4.1 Setup

# Initialize MediaPipe Hands
mp_hands = mp.solutions.hands
hands = mp_hands.Hands(
    min_detection_confidence=0.7,
    min_tracking_confidence=0.5
)

def detect_hands(image):
    # Convert to RGB
    image_rgb = cv2.cvtColor(image, cv2.COLOR_BGR2RGB)
    
    # Detect hands
    results = hands.process(image_rgb)
    
    # Draw the landmarks
    if results.multi_hand_landmarks:
        for hand_landmarks in results.multi_hand_landmarks:
            mp_draw.draw_landmarks(
                image,
                hand_landmarks,
                mp_hands.HAND_CONNECTIONS
            )
    
    return image

4.2 Gesture Recognition

def recognize_gesture(hand_landmarks):
    # Extract finger points
    thumb_tip = np.array([
        hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP.value].x,
        hand_landmarks.landmark[mp_hands.HandLandmark.THUMB_TIP.value].y
    ])
    
    index_tip = np.array([
        hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP.value].x,
        hand_landmarks.landmark[mp_hands.HandLandmark.INDEX_FINGER_TIP.value].y
    ])
    
    # Calculate the distance between the thumb and index finger
    pinch_distance = np.linalg.norm(thumb_tip - index_tip)
    
    return pinch_distance

5. Complete Application

def main():
    # Initialize the camera
    cap = cv2.VideoCapture(0)
    
    while cap.isOpened():
        success, image = cap.read()
        if not success:
            continue
            
        # Pose detection
        image = detect_pose(image)
        
        # Face detection
        image = detect_faces(image)
        
        # Hand detection
        image = detect_hands(image)
        
        # Display
        cv2.imshow('MediaPipe Demo', image)
        
        if cv2.waitKey(1) & 0xFF == ord('q'):
            break
    
    cap.release()
    cv2.destroyAllWindows()

if __name__ == '__main__':
    main()

6. Performance Optimization

6.1 Reducing Latency

def optimize_performance():
    # Configuration for better performance
    pose = mp_pose.Pose(
        min_detection_confidence=0.5,
        min_tracking_confidence=0.5,
        model_complexity=1,  # 0=Lite, 1=Full, 2=Heavy
        smooth_landmarks=True,
        enable_segmentation=False
    )

6.2 Memory Management

def manage_memory():
    # Clean up resources
    pose.close()
    face_detection.close()
    hands.close()

7. Best Practices

1. Image Preprocessing

   • Resize images for better performance
   • Normalize pixel values
   • Handle different image formats

2. Error Handling

   • Check that the camera is available
   • Handle cases where no detection is possible
   • Implement appropriate timeouts

3. Optimization

   • Use lite mode on mobile devices
   • Reduce the detection frequency if needed
   • Implement a caching system for results

Conclusion

Key takeaways:

• Accurate pose, face, and hand detection
• Simple and intuitive API
• Excellent performance
• Cross-platform support

Recommendations:

• Adjust model complexity to your needs
• Optimize performance for mobile devices
• Implement robust error handling
• Use lite mode for real-time applications