Object Detection Tutorial#
 Run on Google Colab
|
 View source on GitHub
|
 Download notebook
|
Start EVA server#
We are reusing the start server notebook for launching the EVA server.
!wget -nc "https://raw.githubusercontent.com/georgia-tech-db/eva/master/tutorials/00-start-eva-server.ipynb"
%run 00-start-eva-server.ipynb
cursor = connect_to_server()
File ‘00-start-eva-server.ipynb’ already there; not retrieving.
[ -z "$(lsof -ti:5432)" ] || kill -9 "$(lsof -ti:5432)"
nohup eva_server > eva.log 2>&1 &
Note: you may need to restart the kernel to use updated packages.
Download the Videos#
# Getting the video files
!wget -nc https://www.dropbox.com/s/k00wge9exwkfxz6/ua_detrac.mp4
# Getting the FastRCNN object detector
!wget -nc https://raw.githubusercontent.com/georgia-tech-db/eva/master/eva/udfs/fastrcnn_object_detector.py
File ‘ua_detrac.mp4’ already there; not retrieving.
File ‘fastrcnn_object_detector.py’ already there; not retrieving.
Load the surveillance videos for analysis#
cursor.execute('DROP TABLE ObjectDetectionVideos')
response = cursor.fetch_all()
print(response)
cursor.execute('LOAD FILE "ua_detrac.mp4" INTO ObjectDetectionVideos;')
response = cursor.fetch_all()
print(response)
@status: ResponseStatus.SUCCESS
@batch:
0
0 Table Successfully dropped: ObjectDetectionVideos
@query_time: 0.03250785917043686
@status: ResponseStatus.SUCCESS
@batch:
0
0 Video successfully added at location: ua_detrac.mp4
@query_time: 0.04492253065109253
Visualize Video#
from IPython.display import Video
Video("ua_detrac.mp4", embed=True)
Register FasterRCNN Object Detection model in an User-Defined Function (UDF) in EVA#
cursor.execute("""CREATE UDF IF NOT EXISTS FastRCNNObjectDetector
INPUT (frame NDARRAY UINT8(3, ANYDIM, ANYDIM))
OUTPUT (labels NDARRAY STR(ANYDIM), bboxes NDARRAY FLOAT32(ANYDIM, 4),
scores NDARRAY FLOAT32(ANYDIM))
TYPE Classification
IMPL 'fastrcnn_object_detector.py';
""")
response = cursor.fetch_all()
print(response)
@status: ResponseStatus.SUCCESS
@batch:
0
0 UDF FastRCNNObjectDetector already exists, nothing added.
@query_time: 0.02033083327114582
Run Object Detector on the video#
cursor.execute("""SELECT id, FastRCNNObjectDetector(data)
FROM ObjectDetectionVideos
WHERE id < 20""")
response = cursor.fetch_all()
print(response)
@status: ResponseStatus.SUCCESS
@batch:
objectdetectionvideos.id \
0 0
1 1
2 2
3 3
4 4
5 5
6 6
7 7
8 8
9 9
10 10
11 11
12 12
13 13
14 14
15 15
16 16
17 17
18 18
19 19
fastrcnnobjectdetector.labels \
0 [person, car, car, car, car, car, car, car, car, car, car, person, car, car, car, car, car, car,...
1 [person, car, car, car, car, car, car, car, car, car, person, car, car, car, car, car, car, car,...
2 [person, car, car, car, car, car, car, car, car, car, car, person, car, car, car, car, car, car,...
3 [person, car, car, car, car, car, car, car, car, car, car, car, car, person, car, car, car, car,...
4 [person, car, car, car, car, car, car, car, car, car, car, car, car, car, person, motorcycle, ca...
5 [person, car, car, car, car, car, car, car, car, car, motorcycle, car, car, person, car, car, ca...
6 [person, car, car, car, car, car, car, car, car, car, car, car, motorcycle, person, car, car, ca...
7 [person, car, car, car, car, car, car, car, car, car, car, car, motorcycle, person, car, car, ca...
8 [person, car, car, car, car, car, car, car, car, car, car, car, car, motorcycle, person, car, ca...
9 [person, car, car, car, car, car, car, car, car, car, motorcycle, car, car, car, car, car, car, ...
10 [person, car, car, car, car, car, car, car, car, car, motorcycle, car, car, car, car, person, ca...
11 [person, car, car, car, car, car, car, car, car, car, car, car, motorcycle, motorcycle, car, car...
12 [person, car, car, car, car, car, car, car, car, motorcycle, car, car, car, car, car, car, car, ...
13 [person, car, car, car, car, car, car, motorcycle, car, car, car, car, car, car, car, car, car, ...
14 [person, car, car, car, car, car, car, car, car, motorcycle, car, car, car, car, car, car, car, ...
15 [person, car, car, car, car, car, car, car, car, car, motorcycle, car, car, car, car, car, car, ...
16 [person, car, car, car, car, car, car, car, car, car, car, car, car, car, car, car, motorcycle, ...
17 [person, car, car, car, car, car, car, car, car, car, car, car, car, car, car, car, motorcycle, ...
18 [person, car, car, car, car, car, car, car, car, car, car, car, car, car, bicycle, car, car, car...
19 [person, car, car, car, car, car, car, car, car, car, car, car, car, car, car, car, car, car, ca...
fastrcnnobjectdetector.bboxes \
0 [[636.2461 363.62677 670.9286 421.68674], [795.974 198.13597 905.6497 239.72404], [926.5628...
1 [[636.3488 364.41104 671.457 425.17358], [797.5898 198.4674 909.43506 239.6631 ], [926.5780...
2 [[636.69507 367.78067 671.4994 425.38022], [609.8011 215.57028 726.95276 277.29547], [799.7043...
3 [[637.3386 370.06775 671.1379 430.22446], [926.5658 108.94128 960. 130.84044], [841.2449...
4 [[637.39386 371.9589 671.4067 429.7561 ], [844.38947 235.53317 960. 292.27747], [926.5494...
5 [[637.3752 374.44894 671.94543 431.65906], [846.1043 235.29305 959.76 292.48502], [926.5932...
6 [[637.50885 376.1439 672.65 435.27576], [851.6319 283.0303 960. 353.44882], [848.5116...
7 [[636.91095 379.39893 672.60986 441.0763 ], [855.6536 284.40753 960. 353.15482], [926.5692...
8 [[637.67236 381.3916 673.5471 443.23373], [622.0833 218.62851 740.9644 284.6785 ], [859.5268...
9 [[637.6342 384.85025 673.79156 443.4543 ], [851.62354 235.61482 960. 289.6144 ], [926.5669...
10 [[638.32996 386.24515 675.5105 449.4082 ], [853.97705 235.86105 960. 290.53348], [926.6726...
11 [[638.0419 387.4773 675.44586 452.0051 ], [926.71765 109.181984 960. 130.68192 ], [856....
12 [[638.6666 389.93994 675.5014 455.6136 ], [858.50885 238.62434 960. 290.6553 ], [926.8178...
13 [[638.99316 393.1623 675.5218 459.98587], [862.4889 237.9856 959.6673 289.5376], [926.9093 10...
14 [[639.02594 395.4505 675.83563 458.40024], [864.2598 237.59782 959.9527 290.08386], [926.9506...
15 [[639.6993 397.50488 676.6463 458.95874], [866.5562 239.41269 959.6509 290.25293], [890.41 ...
16 [[639.6608 398.67078 677.3171 461.35507], [867.6135 239.08667 959.7032 292.32498], [643.4708...
17 [[639.988 401.79584 676.39813 464.388 ], [927.07587 109.3211 959.9775 130.52295], [642.4912...
18 [[639.6279 404.75647 677.6399 468.1819 ], [171.9567 113.68296 223.0494 137.04474], [872.4612...
19 [[639.65326 407.688 677.55304 471.05777], [170.65564 114.0763 221.7475 137.03217], [646.9618...
fastrcnnobjectdetector.scores
0 [0.9973132, 0.9954666, 0.99453676, 0.9928416, 0.9928219, 0.99168164, 0.9894707, 0.98463666, 0.97...
1 [0.9986808, 0.99661547, 0.9946049, 0.9925746, 0.9925718, 0.9904755, 0.9891768, 0.9887982, 0.9882...
2 [0.99889356, 0.9963032, 0.99464333, 0.9944518, 0.99270344, 0.9906961, 0.99067444, 0.98912454, 0....
3 [0.99847347, 0.99472106, 0.9937597, 0.9915018, 0.99106944, 0.9899698, 0.98859274, 0.98738176, 0....
4 [0.99799746, 0.99536246, 0.9944917, 0.9930253, 0.9925647, 0.9924131, 0.9900523, 0.98391956, 0.98...
5 [0.99744403, 0.9945592, 0.9944694, 0.9940614, 0.9938199, 0.99164, 0.9875629, 0.9839294, 0.982379...
6 [0.99849856, 0.99723524, 0.9956939, 0.99471045, 0.99313545, 0.9920499, 0.99203074, 0.988704, 0.9...
7 [0.9991289, 0.99689096, 0.9950283, 0.99449337, 0.99406683, 0.99135935, 0.98904794, 0.98687, 0.98...
8 [0.99855894, 0.99789375, 0.99742484, 0.9951934, 0.99357045, 0.99098235, 0.98661834, 0.9811693, 0...
9 [0.99793184, 0.9957092, 0.99510276, 0.99408025, 0.9926242, 0.992222, 0.98884636, 0.98143584, 0.9...
10 [0.9983885, 0.9960192, 0.99516207, 0.9947366, 0.99309456, 0.98936135, 0.9875754, 0.9850686, 0.98...
11 [0.99706155, 0.9951617, 0.99343115, 0.9920042, 0.9912471, 0.99058783, 0.98849005, 0.9863391, 0.9...
12 [0.99758244, 0.9971852, 0.99477625, 0.99245197, 0.99241346, 0.99238765, 0.9909242, 0.9853243, 0....
13 [0.998033, 0.9956489, 0.99477553, 0.9936738, 0.9918057, 0.99007344, 0.9884399, 0.9858212, 0.9848...
14 [0.99647397, 0.99619615, 0.9947871, 0.9924697, 0.9908516, 0.98961943, 0.9889376, 0.986597, 0.984...
15 [0.99818736, 0.99694735, 0.99624455, 0.99479103, 0.99371606, 0.99213445, 0.9918663, 0.9894098, 0...
16 [0.99894565, 0.99597675, 0.9959545, 0.99492985, 0.9949143, 0.9911158, 0.9899811, 0.98385, 0.9678...
17 [0.998359, 0.9952448, 0.9951807, 0.994565, 0.99443567, 0.99266374, 0.9913652, 0.9903177, 0.98383...
18 [0.99799997, 0.9964407, 0.99579465, 0.9955603, 0.99490297, 0.99179196, 0.9901037, 0.9847661, 0.9...
19 [0.9981029, 0.9973521, 0.99511445, 0.9949681, 0.9949438, 0.99476826, 0.99410397, 0.9827383, 0.97...
@query_time: 6.682086415588856
Visualizing output of the Object Detector on the video#
import cv2
from pprint import pprint
from matplotlib import pyplot as plt
def annotate_video(detections, input_video_path, output_video_path):
color1=(207, 248, 64)
color2=(255, 49, 49)
thickness=4
vcap = cv2.VideoCapture(input_video_path)
width = int(vcap.get(3))
height = int(vcap.get(4))
fps = vcap.get(5)
fourcc = cv2.VideoWriter_fourcc('m', 'p', '4', 'v') #codec
video=cv2.VideoWriter(output_video_path, fourcc, fps, (width,height))
frame_id = 0
# Capture frame-by-frame
# ret = 1 if the video is captured; frame is the image
ret, frame = vcap.read()
while ret:
df = detections
df = df[['fastrcnnobjectdetector.bboxes', 'fastrcnnobjectdetector.labels']][df.index == frame_id]
if df.size:
dfLst = df.values.tolist()
for bbox, label in zip(dfLst[0][0], dfLst[0][1]):
x1, y1, x2, y2 = bbox
x1, y1, x2, y2 = int(x1), int(y1), int(x2), int(y2)
# object bbox
img=cv2.rectangle(frame, (x1, y1), (x2, y2), color1, thickness)
# object label
cv2.putText(img, label, (x1, y1-10), cv2.FONT_HERSHEY_SIMPLEX, 0.9, color1, thickness)
# frame label
cv2.putText(img, 'Frame ID: ' + str(frame_id), (700, 500), cv2.FONT_HERSHEY_SIMPLEX, 1.2, color2, thickness)
video.write(img)
# Show every fifth frame
if frame_id % 5 == 0:
plt.imshow(img)
plt.show()
frame_id+=1
ret, frame = vcap.read()
video.release()
vcap.release()
from ipywidgets import Video, Image
input_path = 'ua_detrac.mp4'
output_path = 'video.mp4'
dataframe = response.batch.frames
annotate_video(dataframe, input_path, output_path)
Video.from_file(output_path)
Dropping an User-Defined Function (UDF)#
cursor.execute("DROP UDF FastRCNNObjectDetector;")
response = cursor.fetch_all()
print(response)
@status: ResponseStatus.SUCCESS
@batch:
0
0 UDF FastRCNNObjectDetector successfully dropped
@query_time: 0.030028007924556732