File size: 9,049 Bytes
5aad5c7 46007cf 812ac03 46007cf dd5e693 cdd0122 64040d2 dd5e693 812ac03 46007cf 812ac03 64040d2 812ac03 4e5c0fd 812ac03 46007cf cdd0122 2b3857b cdd0122 46007cf 4b1fa52 46007cf 64040d2 46007cf 4e5c0fd 46007cf cdd0122 46007cf cdd0122 46007cf df94744 46007cf df94744 46007cf 812ac03 46007cf 812ac03 cdd0122 46007cf cdd0122 46007cf cdd0122 64040d2 cdd0122 64040d2 2b3857b 64040d2 df94744 812ac03 ecc3c55 812ac03 4e5c0fd 812ac03 df94744 46007cf cdd0122 |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 |
import cv2
import numpy as np
from PIL import Image
from PIL import ImageColor
import mediapipe as mp
import time
import gradio as gr
import glob
import os
width_, height_ = 144, 96
drawing_flag = False
sleepy_time = time.time()
output_frames = []
def is_hex(hexq):
valid = ['0','1','2','3','4','5','6',
'7','8','9','A','a','B','b',
'C','c','D','d','E','e','F',
'f']
hexq = str(hexq)
if len(hexq) == 7:
if hexq[0] == '#':
for h in hexq[1:]:
if h in valid:
return True
else:
return False
else:
return False
else:
return False
def hex2rgb(hex):
if is_hex(hex):
return ImageColor.getcolor(hex, "RGB")
else:
return (0, 0, 0)
def find_hands(brain, img):
if img is not None:
img_rgb = cv2.cvtColor(img, cv2.COLOR_BGR2RGB) # opencv image is in BGR form but mp is trained with RGB
results = brain.process(img_rgb) # process finds the hands and outputs classification and 21 landmarks for each hand
all_hands = [] # initializing array to hold the dictionary for the hands
h, w, _ = img.shape # get height and width of image for scaling
if results.multi_hand_landmarks:
for hand_type, hand_lms in zip(results.multi_handedness,
results.multi_hand_landmarks): # elegant solution for mp list object traversal
hand = {} # initializing dict for each hand
lm_list = [] # landmarks array for all 21 point of the hand
for lm in hand_lms.landmark:
px, py, pz = int(lm.x * w), int(lm.y * h), int(
lm.z * w) # scaling landmark points to image size for frame coordinates
lm_list.append([px, py, pz])
hand["lm_list"] = lm_list # add "lm_list" key for all landmark points of the hand
hand["type"] = hand_type.classification[0].label # adds the label (left/right) for the hand
all_hands.append(hand) # appends the dict
return all_hands
else:
return 0
def is_drawing(index, thumb): # proximity function with arbitrary threshold
npindex = np.array((index[0], index[1]))
npthumb = np.array((thumb[0], thumb[1]))
if np.linalg.norm(npindex - npthumb) < 20:
return True
else:
return False
def save(landmarks): # brute force finger orientation checking
if landmarks[8][1] < landmarks[6][1]:
if landmarks[12][1] < landmarks[10][1]:
if landmarks[16][1] < landmarks[14][1]:
if landmarks[20][1] < landmarks[18][1]:
return True
else:
return False
def clear(landmarks): # brute force finger orientation checking
if landmarks[4][1] < landmarks[3][1] < landmarks[2][1] < landmarks[8][1]:
return True
else:
return False
def show(video, dominant_hand, hex_color='#FFFFFF'): # main
cam = cv2.VideoCapture(video) # get the video file from path
width = cam.get(cv2.CAP_PROP_FRAME_WIDTH)
height = cam.get(cv2.CAP_PROP_FRAME_HEIGHT)
detector = mp.solutions.hands.Hands(min_detection_confidence=0.8) # initialize detector
# paper = np.zeros((width, height, 4), np.uint8)
paper = np.zeros((int(height), int(width), 3), dtype=np.uint8) # create blank page
paper.fill(255)
color = hex2rgb(hex_color)
past_holder = () # hold previous index coordinates
palette = cv2.imread('palette_small.jpg')
page_num = 0 # iterating for saving (not a viable function for gradio)
global sleepy_time # get sleep time for multiple gestures
while cam.isOpened():
x, rgb_image = cam.read()
rgb_image_f = cv2.flip(rgb_image, 1) # mirrored video
hands = find_hands(detector, rgb_image_f)
if x: # return flag for cv2
try: # for error handling
if hands:
hand1 = hands[0] if hands[0]["type"] == dominant_hand else hands[1]
lm_list1 = hand1["lm_list"] # List of 21 Landmarks
handedness = hand1["type"]
if handedness == dominant_hand:
idx_coords = lm_list1[8][0], lm_list1[8][1] # 0 is width (bigger)
# print(idx_coords)
cv2.circle(rgb_image_f, idx_coords, 5, color, cv2.FILLED)
### Discontinued function due to gradio limitations ###
# if idx_coords[1] < 72: # brute force but should be extremely marginally faster lol
# if idx_coords[0] < 71: # red
# color = (0, 0, 255)
# if 71 < idx_coords[0] < 142: # orange
# color = (0, 115, 255)
# if 142 < idx_coords[0] < 213: # yellow
# color = (0, 229, 255)
# if 213 < idx_coords[0] < 284: # green
# color = (0, 195, 88)
# if 284 < idx_coords[0] < 356: # blue
# color = (195, 85, 0)
# if 356 < idx_coords[0] < 427: # indigo
# color = (195, 0, 68)
# if 427 < idx_coords[0] < 498: # violet
# color = (195, 0, 143)
# if 498 < idx_coords[0] < 569: # black
# color = (0, 0, 0)
# if 569 < idx_coords[0]: # white / eraser
# color = (255, 255, 255)
if len(past_holder) and drawing_flag: # start drawing
cv2.line(paper, past_holder, idx_coords, color, 5)
cv2.line(rgb_image_f, past_holder, idx_coords, color, 5)
# paper[idx_coords[0]][idx_coords[1]][0] = 255
# paper[idx_coords[0]][idx_coords[1]][3] = 255
cv2.circle(rgb_image_f, idx_coords, 5, color, cv2.FILLED)
### Discontinued function due to gradio limitations ###
# if save(lm_list1) and time.time() - sleepy_time > 3: # save / output
# paper[0:height_, w - width_: w] = 255 # presenter eraser
# paper = cv2.cvtColor(paper, cv2.COLOR_BGR2RGB)
# im = Image.fromarray(paper)
# im.save("paper%s.png" % page_num)
# print("saved")
# sleepy_time = time.time()
# paper = cv2.cvtColor(paper, cv2.COLOR_RGB2BGR)
# page_num += 1
if clear(lm_list1) and time.time() - sleepy_time > 3: # reset paper
paper = np.zeros((height, width, 3), dtype=np.uint8)
paper.fill(255)
print("page cleared")
sleepy_time = time.time()
past_holder = idx_coords
if is_drawing(idx_coords, lm_list1[4]): # 4 is thumb
drawing_flag = True
else:
drawing_flag = False
except:
pass
finally:
if True:
presenter = cv2.resize(rgb_image_f, (width_, height_))
h, w, _ = rgb_image_f.shape
paper[0:height_, w - width_: w] = presenter
else:
break
paper = cv2.cvtColor(paper, cv2.COLOR_RGB2BGR)
im = Image.fromarray(paper)
output_frames.append(paper)
im.save("paper%s.png" % page_num)
page_num += 1
img_array = []
for filename in glob.glob('*.png'):
imggg = cv2.imread(filename)
img_array.append(imggg)
os.remove(filename)
video_output = cv2.VideoWriter('any.webm', cv2.VideoWriter_fourcc(*'VP80'), 30, (640, 480))
for i in range(len(img_array)):
video_output.write(img_array[i])
video_output.release()
return 'any.webm'
title = 'Air Draw'
desc = 'A mediapipe hands wrapper for drawing in the air. Draw holding an invisible pen, ' \
'and open up your hand to lift the "pen" from the "paper". Use a "thumbs up" gesture to clear the drawing ' \
'paper. '
iface = gr.Interface(
fn=show,
inputs=[
gr.inputs.Video(source="webcam", label="Record yourself drawing in the air!"),
gr.inputs.Radio(['Right', 'Left'], label="Dominant Hand"),
gr.inputs.Textbox(placeholder="#355C7D", label="Hex Color")
],
outputs='video',
title=title,
description=desc)
iface.launch(share=True, enable_queue=True)
|