Google Mediapipe 手勢偵測結合SVM模型進行多手勢識別

import cv2
import mediapipe as mp
import numpy as np
import time
import os

# Initialize mediapipe Hands
mp_hands = mp.solutions.hands
hands = mp_hands.Hands()

def compute_distances(landmarks):
    distances = []
    
    # Define pairs for distance calculation
    pairs = [(0, 1), (0, 2), (0, 3), (0, 4),
             (0, 5), (0, 6), (0, 7), (0, 8),
             (0, 9), (0, 10), (0, 11), (0, 12),
             (0, 13), (0, 14), (0, 15), (0, 16),
             (0, 17), (0, 18), (0, 19), (0, 20),
             (4, 8), (8, 12), (12, 16), (16, 20)]

    reference_pair = (0, 9)
    p_ref1 = np.array([landmarks.landmark[reference_pair[0]].x, landmarks.landmark[reference_pair[0]].y])
    p_ref2 = np.array([landmarks.landmark[reference_pair[1]].x, landmarks.landmark[reference_pair[1]].y])
    reference_distance = np.linalg.norm(p_ref1 - p_ref2)
    
    for pair in pairs:
        p1 = np.array([landmarks.landmark[pair[0]].x, landmarks.landmark[pair[0]].y])
        p2 = np.array([landmarks.landmark[pair[1]].x, landmarks.landmark[pair[1]].y])
        distance = np.linalg.norm(p1 - p2) / reference_distance
        distances.append(distance)

    return distances

# Ask user for filename
filename = input("Please enter the filename for data: ")
save_path = "dataset_V3/" + filename

# Check if the 'dataset' directory exists, if not, create it
if not os.path.exists("dataset_V3"):
    os.makedirs("dataset_V3")

cap = cv2.VideoCapture(0)
data_collection = []

collecting = False
start_time = None

while True:
    ret, frame = cap.read()
    if not ret:
        continue

    if not collecting:
        cv2.putText(frame, "Press SPACE to start data collection", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)
    else:
        elapsed_time = int(time.time() - start_time)
        remaining_time = 10 - elapsed_time
        cv2.putText(frame, f"Time left: {remaining_time} seconds", (50, 50), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv2.LINE_AA)
        if elapsed_time >= 10:
            break

    rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(rgb_frame)

    if results.multi_hand_landmarks and collecting:
        for landmarks in results.multi_hand_landmarks:
            distances = compute_distances(landmarks)
            data_collection.append(distances)

    cv2.imshow("Data Collection", frame)
    key = cv2.waitKey(1)
    
    if key == 32 and not collecting:
        collecting = True
        start_time = time.time()
    elif key == ord('q'):
        break

cap.release()
cv2.destroyAllWindows()

# Convert the data_collection list to numpy array and save
np.save(save_path, np.array(data_collection))
print(f"Data saved to {save_path}")

import numpy as np
import os
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
import joblib

dataset_path = "dataset_V3"
label_file = "labels_V3.txt"

# Fetch all .npy files from the dataset directory
files = [f for f in os.listdir(dataset_path) if f.endswith('.npy')]

X = []
y = []

labels = []

# Load data and labels
for idx, file in enumerate(files):
    data = np.load(os.path.join(dataset_path, file))
    X.extend(data)  # Note: using extend instead of append
    label = file.split('.')[0]
    labels.append(label)
    y.extend([idx] * data.shape[0])

X = np.array(X)
y = np.array(y)

# Save labels to labels.txt
with open(label_file, 'w') as f:
    for label in labels:
        f.write(label + "\n")

# Preprocess data
scaler = StandardScaler()
X = scaler.fit_transform(X)

# Train SVM model
clf = SVC(kernel='linear', probability=True)
clf.fit(X, y)

# Save the model and the scaler
model_filename = "svm_model_V3.pkl"
joblib.dump(clf, model_filename)
scaler_filename = "scaler_V3.pkl"
joblib.dump(scaler, scaler_filename)

print(f"Model saved to {model_filename}")
print(f"Scaler saved to {scaler_filename}")
print(f"Labels saved to {label_file}")

import cv2
import mediapipe as mp
import numpy as np
import joblib

# Initialize mediapipe Hands Detection
mp_hands = mp.solutions.hands
hands = mp_hands.Hands()

# Load the SVM model and scaler
model_filename = "svm_model_V3.pkl"
clf = joblib.load(model_filename)
scaler_filename = "scaler_V3.pkl"
scaler = joblib.load(scaler_filename)

# Load labels
label_file = "labels_V3.txt"
with open(label_file, 'r') as f:
    labels = f.readlines()
labels = [label.strip() for label in labels]

def compute_distances(landmarks):
    # Define pairs for distance calculation
    pairs = [(0, 1), (0, 2), (0, 3), (0, 4),
             (0, 5), (0, 6), (0, 7), (0, 8),
             (0, 9), (0, 10), (0, 11), (0, 12),
             (0, 13), (0, 14), (0, 15), (0, 16),
             (0, 17), (0, 18), (0, 19), (0, 20),
             (4, 8), (8, 12), (12, 16), (16, 20)]

    distances = []
    reference_distance = np.linalg.norm(
        np.array([landmarks.landmark[0].x, landmarks.landmark[0].y]) -
        np.array([landmarks.landmark[9].x, landmarks.landmark[9].y])
    )

    for pair in pairs:
        p1 = np.array([landmarks.landmark[pair[0]].x, landmarks.landmark[pair[0]].y])
        p2 = np.array([landmarks.landmark[pair[1]].x, landmarks.landmark[pair[1]].y])
        distance = np.linalg.norm(p1 - p2)
        distances.append(distance/reference_distance)  # Normalize the distance using the reference distance

    return distances

cap = cv2.VideoCapture(0)

while True:
    ret, frame = cap.read()
    if not ret:
        break

    rgb_frame = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
    results = hands.process(rgb_frame)

    if results.multi_hand_landmarks:
        for index, landmarks in enumerate(results.multi_hand_landmarks):
            # Distinguish between left and right hand
            hand_label = "Right" if results.multi_handedness[index].classification[0].label == "Left" else "Left"

            distances = compute_distances(landmarks)
            distances = scaler.transform([distances])

            prediction = clf.predict(distances)
            confidence = np.max(clf.predict_proba(distances))

            label = labels[prediction[0]]
            display_text = f"{hand_label} Hand: {label} ({confidence*100:.2f}%)"

            cv2.putText(frame, display_text, (10, 30 + (index * 40)), cv2.FONT_HERSHEY_SIMPLEX, 1, (0, 255, 0), 2, cv2.LINE_AA)

            # To visualize the landmarks of the hand
            mp.solutions.drawing_utils.draw_landmarks(frame, landmarks, mp_hands.HAND_CONNECTIONS)

    cv2.imshow('Demo', frame)

    if cv2.waitKey(1) & 0xFF == 27:  # ESC key
        break

cap.release()
cv2.destroyAllWindows()