Custom Model Tutorial#
Run on Google Colab | View source on GitHub | Download notebook |
Start EVA server#
We are reusing the start server notebook for launching the EVA server.
!wget -nc "https://raw.githubusercontent.com/georgia-tech-db/eva/master/tutorials/00-start-eva-server.ipynb"
%run 00-start-eva-server.ipynb
cursor = connect_to_server()
File ‘00-start-eva-server.ipynb’ already there; not retrieving.
nohup eva_server > eva.log 2>&1 &
Note: you may need to restart the kernel to use updated packages.
Download custom user-defined function (UDF), model, and video#
# Download UDF
!wget -nc https://www.dropbox.com/s/nht5lwjemmclqx3/gender.py?raw=1 -O gender.py
# Download built-in Face Detector
!wget -nc https://raw.githubusercontent.com/georgia-tech-db/eva/master/eva/udfs/face_detector.py
# Download models
!wget -nc https://www.dropbox.com/s/yyp7awyczv7esf4/gender.pth?raw=1 -O gender.pth
# Download videos
!wget -nc https://www.dropbox.com/s/f5rorxf0840ajjd/short.mp4?raw=1 -O short.mp4
File ‘gender.py’ already there; not retrieving.
File ‘face_detector.py’ already there; not retrieving.
File ‘gender.pth’ already there; not retrieving.
File ‘short.mp4’ already there; not retrieving.
Load video for analysis#
cursor.execute("DROP TABLE TIKTOK;")
response = cursor.fetch_all()
print(response)
cursor.execute("LOAD VIDEO 'short.mp4' INTO TIKTOK;")
response = cursor.fetch_all()
print(response)
cursor.execute("""SELECT id FROM TIKTOK WHERE id < 5""")
response = cursor.fetch_all()
print(response)
@status: ResponseStatus.FAIL
@batch:
None
@error: Table: TIKTOK does not exist
@status: ResponseStatus.SUCCESS
@batch:
0
0 Number of loaded VIDEO: 1
@query_time: 2.2460366699997394
@status: ResponseStatus.SUCCESS
@batch:
tiktok.id
0 0
1 1
2 2
3 3
4 4
@query_time: 0.0720941989993662
Visualize Video#
from IPython.display import Video
Video("short.mp4", embed=True)
Create GenderCNN and FaceDetector UDFs#
cursor.execute("""DROP UDF GenderCNN;""")
response = cursor.fetch_all()
print(response)
cursor.execute("""CREATE UDF IF NOT EXISTS
GenderCNN
INPUT (data NDARRAY UINT8(3, 224, 224))
OUTPUT (label TEXT(10))
TYPE Classification
IMPL 'gender.py';
""")
response = cursor.fetch_all()
print(response)
cursor.execute("""CREATE UDF IF NOT EXISTS
FaceDetector
INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4),
scores NDARRAY FLOAT32(ANYDIM))
TYPE FaceDetection
IMPL 'face_detector.py';
""")
response = cursor.fetch_all()
print(response)
@status: ResponseStatus.SUCCESS
@batch:
0
0 UDF GenderCNN successfully dropped
@query_time: 0.13388806600050884
@status: ResponseStatus.SUCCESS
@batch:
0
0 UDF GenderCNN successfully added to the database.
@query_time: 0.44734299300034763
@status: ResponseStatus.SUCCESS
@batch:
0
0 UDF FaceDetector already exists, nothing added.
@query_time: 0.006585866000023088
Run Face Detector on video#
cursor.execute("""SELECT id, FaceDetector(data).bboxes
FROM TIKTOK WHERE id < 10""")
response = cursor.fetch_all()
response.as_df()
tiktok.id | facedetector.bboxes | |
---|---|---|
0 | 0 | [[81, 194, 282, 470]] |
1 | 1 | [[82, 194, 283, 469]] |
2 | 2 | [[82, 195, 283, 470]] |
3 | 3 | [[82, 194, 284, 472]] |
4 | 4 | [[84, 197, 283, 472]] |
5 | 5 | [[85, 199, 283, 471]] |
6 | 6 | [[84, 199, 284, 472]] |
7 | 7 | [[84, 198, 284, 473]] |
8 | 8 | [[85, 197, 283, 472]] |
9 | 9 | [[86, 198, 282, 476]] |
Composing UDFs in a query#
Detect gender of the faces detected in the video by composing a set of UDFs (GenderCNN, FaceDetector, and Crop)
cursor.execute("""SELECT id, bbox, GenderCNN(Crop(data, bbox))
FROM TIKTOK JOIN LATERAL UNNEST(FaceDetector(data)) AS Face(bbox, conf)
WHERE id < 50;""")
response = cursor.fetch_all()
response.as_df()
tiktok.id | Face.bbox | gendercnn.label | |
---|---|---|---|
0 | 0 | [81, 194, 282, 470] | female |
1 | 1 | [82, 194, 283, 469] | female |
2 | 2 | [82, 195, 283, 470] | female |
3 | 3 | [82, 194, 284, 472] | female |
4 | 4 | [84, 197, 283, 472] | female |
5 | 5 | [85, 199, 283, 471] | female |
6 | 6 | [84, 199, 284, 472] | female |
7 | 7 | [84, 198, 284, 473] | female |
8 | 8 | [85, 197, 283, 472] | female |
9 | 9 | [86, 198, 282, 476] | female |
10 | 10 | [85, 199, 283, 477] | female |
11 | 11 | [85, 200, 282, 478] | female |
12 | 12 | [94, 199, 288, 481] | female |
13 | 13 | [93, 199, 285, 480] | female |
14 | 14 | [95, 203, 288, 481] | female |
15 | 15 | [95, 201, 287, 479] | female |
16 | 16 | [95, 205, 287, 476] | female |
17 | 17 | [94, 204, 287, 475] | female |
18 | 18 | [84, 203, 283, 484] | female |
19 | 19 | [86, 196, 283, 474] | female |
20 | 20 | [84, 198, 280, 476] | female |
21 | 21 | [82, 206, 278, 483] | female |
22 | 22 | [81, 207, 277, 481] | female |
23 | 23 | [80, 206, 277, 482] | female |
24 | 24 | [80, 207, 276, 482] | female |
25 | 25 | [82, 209, 276, 481] | female |
26 | 26 | [83, 200, 278, 477] | female |
27 | 27 | [82, 206, 279, 485] | female |
28 | 28 | [82, 204, 280, 485] | female |
29 | 29 | [86, 202, 279, 476] | female |
30 | 30 | [88, 197, 282, 473] | female |
31 | 31 | [87, 205, 284, 482] | female |
32 | 32 | [89, 207, 284, 482] | female |
33 | 33 | [94, 202, 288, 475] | female |
34 | 34 | [94, 205, 287, 481] | female |
35 | 35 | [95, 203, 288, 479] | female |
36 | 36 | [96, 203, 290, 480] | female |
37 | 37 | [96, 202, 288, 477] | female |
38 | 38 | [96, 201, 289, 475] | female |
39 | 39 | [96, 203, 290, 478] | female |
40 | 40 | [94, 199, 289, 475] | female |
41 | 41 | [93, 199, 288, 474] | female |
42 | 42 | [92, 197, 289, 472] | female |
43 | 43 | [95, 199, 291, 471] | female |
44 | 44 | [93, 199, 292, 470] | female |
45 | 45 | [93, 200, 291, 468] | female |
46 | 46 | [94, 209, 285, 480] | female |
47 | 47 | [90, 207, 285, 482] | female |
48 | 48 | [88, 206, 284, 480] | female |
49 | 49 | [88, 207, 283, 480] | female |
Visualize Output#
import cv2
from matplotlib import pyplot as plt
def annotate_video(detections, input_video_path, output_video_path):
color=(207, 248, 64)
thickness=4
vcap = cv2.VideoCapture(input_video_path)
width = int(vcap.get(3))
height = int(vcap.get(4))
fps = vcap.get(5)
fourcc = cv2.VideoWriter_fourcc(*'MP4V') #codec
video=cv2.VideoWriter(output_video_path, fourcc, fps, (width,height))
frame_id = 0
# Capture frame-by-frame
ret, frame = vcap.read() # ret = 1 if the video is captured; frame is the image
while ret:
df = detections
df = df[['Face.bbox', 'gendercnn.label']][df['tiktok.id'] == frame_id]
if df.size:
for bbox, label in df.values:
x1, y1, x2, y2 = bbox
x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
frame=cv2.rectangle(frame, (x1, y1), (x2, y2), color, thickness) # object bbox
cv2.putText(frame, str(label), (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, color, thickness-1) # object label
video.write(frame)
# Show every fifth frame
if frame_id % 5 == 0:
plt.imshow(frame)
plt.show()
if frame_id == 50:
return
frame_id+=1
ret, frame = vcap.read()
video.release()
vcap.release()
#!pip install ipywidgets
from ipywidgets import Video
input_path = 'short.mp4'
output_path = 'annotated_short.mp4'
dataframe = response.as_df()
annotate_video(dataframe, input_path, output_path)