Custom Model Tutorial

Run on Google Colab

View source on GitHub

Download notebook

Start EVA server

We are reusing the start server notebook for launching the EVA server.

!wget -nc "https://raw.githubusercontent.com/georgia-tech-db/eva/master/tutorials/00-start-eva-server.ipynb"
%run 00-start-eva-server.ipynb
cursor = connect_to_server()

File ‘00-start-eva-server.ipynb’ already there; not retrieving.

nohup eva_server > eva.log 2>&1 &
Note: you may need to restart the kernel to use updated packages.

Download custom user-defined function (UDF), model, and video

# Download UDF
!wget -nc https://www.dropbox.com/s/nht5lwjemmclqx3/gender.py?raw=1 -O gender.py

# Download built-in Face Detector
!wget -nc https://raw.githubusercontent.com/georgia-tech-db/eva/master/eva/udfs/face_detector.py

# Download models
!wget -nc https://www.dropbox.com/s/yyp7awyczv7esf4/gender.pth?raw=1 -O gender.pth

# Download videos
!wget -nc https://www.dropbox.com/s/f5rorxf0840ajjd/short.mp4?raw=1 -O short.mp4

File ‘gender.py’ already there; not retrieving.
File ‘face_detector.py’ already there; not retrieving.

File ‘gender.pth’ already there; not retrieving.
File ‘short.mp4’ already there; not retrieving.

Load video for analysis

cursor.execute("DROP TABLE TIKTOK;")
response = cursor.fetch_all()
print(response)
cursor.execute("LOAD VIDEO 'short.mp4' INTO TIKTOK;")
response = cursor.fetch_all()
print(response)
cursor.execute("""SELECT id FROM TIKTOK WHERE id < 5""")
response = cursor.fetch_all()
print(response)

@status: ResponseStatus.FAIL
@batch: 
 None
@error: Table: TIKTOK does not exist
@status: ResponseStatus.SUCCESS
@batch: 
                            0
0  Number of loaded VIDEO: 1
@query_time: 2.2460366699997394
@status: ResponseStatus.SUCCESS
@batch: 
    tiktok.id
0          0
1          1
2          2
3          3
4          4
@query_time: 0.0720941989993662

Visualize Video

from IPython.display import Video
Video("short.mp4", embed=True)

Create GenderCNN and FaceDetector UDFs

cursor.execute("""DROP UDF GenderCNN;""")
response = cursor.fetch_all()
print(response)

cursor.execute("""CREATE UDF IF NOT EXISTS 
                  GenderCNN
                  INPUT (data NDARRAY UINT8(3, 224, 224)) 
                  OUTPUT (label TEXT(10)) 
                  TYPE  Classification 
                  IMPL 'gender.py';
        """)
response = cursor.fetch_all()
print(response)

cursor.execute("""CREATE UDF IF NOT EXISTS
                  FaceDetector
                  INPUT  (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
                  OUTPUT (bboxes NDARRAY FLOAT32(ANYDIM, 4),
                          scores NDARRAY FLOAT32(ANYDIM))
                  TYPE  FaceDetection
                  IMPL  'face_detector.py';
        """)
response = cursor.fetch_all()
print(response)

@status: ResponseStatus.SUCCESS
@batch: 
                                     0
0  UDF GenderCNN successfully dropped
@query_time: 0.13388806600050884
@status: ResponseStatus.SUCCESS
@batch: 
                                                    0
0  UDF GenderCNN successfully added to the database.
@query_time: 0.44734299300034763
@status: ResponseStatus.SUCCESS
@batch: 
                                                  0
0  UDF FaceDetector already exists, nothing added.
@query_time: 0.006585866000023088

Run Face Detector on video

cursor.execute("""SELECT id, FaceDetector(data).bboxes 
                  FROM TIKTOK WHERE id < 10""")
response = cursor.fetch_all()
response.as_df()

	tiktok.id	facedetector.bboxes
0	0	[[81, 194, 282, 470]]
1	1	[[82, 194, 283, 469]]
2	2	[[82, 195, 283, 470]]
3	3	[[82, 194, 284, 472]]
4	4	[[84, 197, 283, 472]]
5	5	[[85, 199, 283, 471]]
6	6	[[84, 199, 284, 472]]
7	7	[[84, 198, 284, 473]]
8	8	[[85, 197, 283, 472]]
9	9	[[86, 198, 282, 476]]

Composing UDFs in a query

Detect gender of the faces detected in the video by composing a set of UDFs (GenderCNN, FaceDetector, and Crop)

cursor.execute("""SELECT id, bbox, GenderCNN(Crop(data, bbox)) 
                  FROM TIKTOK JOIN LATERAL  UNNEST(FaceDetector(data)) AS Face(bbox, conf)  
                  WHERE id < 50;""")
response = cursor.fetch_all()
response.as_df()

	tiktok.id	Face.bbox	gendercnn.label
0	0	[81, 194, 282, 470]	female
1	1	[82, 194, 283, 469]	female
2	2	[82, 195, 283, 470]	female
3	3	[82, 194, 284, 472]	female
4	4	[84, 197, 283, 472]	female
5	5	[85, 199, 283, 471]	female
6	6	[84, 199, 284, 472]	female
7	7	[84, 198, 284, 473]	female
8	8	[85, 197, 283, 472]	female
9	9	[86, 198, 282, 476]	female
10	10	[85, 199, 283, 477]	female
11	11	[85, 200, 282, 478]	female
12	12	[94, 199, 288, 481]	female
13	13	[93, 199, 285, 480]	female
14	14	[95, 203, 288, 481]	female
15	15	[95, 201, 287, 479]	female
16	16	[95, 205, 287, 476]	female
17	17	[94, 204, 287, 475]	female
18	18	[84, 203, 283, 484]	female
19	19	[86, 196, 283, 474]	female
20	20	[84, 198, 280, 476]	female
21	21	[82, 206, 278, 483]	female
22	22	[81, 207, 277, 481]	female
23	23	[80, 206, 277, 482]	female
24	24	[80, 207, 276, 482]	female
25	25	[82, 209, 276, 481]	female
26	26	[83, 200, 278, 477]	female
27	27	[82, 206, 279, 485]	female
28	28	[82, 204, 280, 485]	female
29	29	[86, 202, 279, 476]	female
30	30	[88, 197, 282, 473]	female
31	31	[87, 205, 284, 482]	female
32	32	[89, 207, 284, 482]	female
33	33	[94, 202, 288, 475]	female
34	34	[94, 205, 287, 481]	female
35	35	[95, 203, 288, 479]	female
36	36	[96, 203, 290, 480]	female
37	37	[96, 202, 288, 477]	female
38	38	[96, 201, 289, 475]	female
39	39	[96, 203, 290, 478]	female
40	40	[94, 199, 289, 475]	female
41	41	[93, 199, 288, 474]	female
42	42	[92, 197, 289, 472]	female
43	43	[95, 199, 291, 471]	female
44	44	[93, 199, 292, 470]	female
45	45	[93, 200, 291, 468]	female
46	46	[94, 209, 285, 480]	female
47	47	[90, 207, 285, 482]	female
48	48	[88, 206, 284, 480]	female
49	49	[88, 207, 283, 480]	female

Visualize Output

import cv2
from matplotlib import pyplot as plt

def annotate_video(detections, input_video_path, output_video_path):
    color=(207, 248, 64)
    thickness=4

    vcap = cv2.VideoCapture(input_video_path)
    width = int(vcap.get(3))
    height = int(vcap.get(4))
    fps = vcap.get(5)
    fourcc = cv2.VideoWriter_fourcc(*'MP4V') #codec
    video=cv2.VideoWriter(output_video_path, fourcc, fps, (width,height))

    frame_id = 0
    # Capture frame-by-frame
    ret, frame = vcap.read()  # ret = 1 if the video is captured; frame is the image

    while ret:
        df = detections
        df = df[['Face.bbox', 'gendercnn.label']][df['tiktok.id'] == frame_id]
        
        if df.size:
            for bbox, label in df.values:
                x1, y1, x2, y2 = bbox
                x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
                frame=cv2.rectangle(frame, (x1, y1), (x2, y2), color, thickness) # object bbox
                cv2.putText(frame, str(label), (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, color, thickness-1) # object label
            
            video.write(frame)
            # Show every fifth frame
            if frame_id % 5 == 0:
                plt.imshow(frame)
                plt.show()

        if frame_id == 50:
            return

        frame_id+=1
        ret, frame = vcap.read()

    video.release()
    vcap.release()

#!pip install ipywidgets
from ipywidgets import Video
input_path = 'short.mp4'
output_path = 'annotated_short.mp4'

dataframe = response.as_df()
annotate_video(dataframe, input_path, output_path)