Skip to content

Commit 82a26ba

Browse files
LukasHedegaardiliiliiliilinegarhdrad-daniel
authored
Benchmark au (#231)
* Add Human Activity Recognition benchmark scripts * Remove pandas dependency in activity_recognition * Remove pyav dependency * Add decoder option to kinetics loader * Add script for har benchmark install * Update har-benchmark install script * Add variable batch_sizes to har benchmark script * Add har benchmark RTX2080Ti results * Update har benchmark scripts * Add xavier benchmark har results * Update har benchmark scripts with CPU batch_sizes * Add har benchmark CPU results * Fix black lint issue * Remove obsolete dependency * Redo cox3d RTX2080Ti benchmark * Update har benchmark batch sizes * Fix install script env vars * Fix exception handling in kinetics * Update Xavier har benchmark * Add har results on TX2 * Add batch size for tx2 in har benchmark scripts * Add 3D object detection benchmark * Fix bounding box to str * Add detection_3d and tracking_2d benchmarks * Fix imports not working in python 3.6 * Update file names * Update far mot names * Add AB3DMOT benchmark * Update media tracking 3d * benchmark_stgcn added * benchmark_stgcn added * benchmark_stgcn added * benchmark_stgcn added * benchmark_stgcn added * benchmark_stgcn added * benchmark_stgcn added * benchmark_stgcn added * benchmark_stgcn modified * benchmark_stgcn modified * benchmark_stgcn modified * benchmark_stgcn modified * benchmark_stbln modified * benchmark_stbln modified * benchmark_stbln modified * benchmark_stbln modified * benchmark_stbln modified * mcdo is modified * mcdo is modified * mcdo is modified * mcdo is modified * mcdo is modified * mcdo is modified * Fix style in kinetics * Fix W503 * code style fixed * style fixed * Fix style errors * Fix kinetics unused torchvision * Remove activity recognition results * Add av dependency back Co-authored-by: Illia Oleksiienko <[email protected]> Co-authored-by: Negar <[email protected]> Co-authored-by: ad-daniel <[email protected]>
1 parent 5490b71 commit 82a26ba

File tree

35 files changed

+3509
-33
lines changed

35 files changed

+3509
-33
lines changed

Diff for: ‎projects/perception/activity_recognition/benchmark/README.md

+28
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
# Human Activity Recognition Benchmark
2+
This folder contains a script for benchmarking the inference of Human Activity Recognition learners found at
3+
```python
4+
from opendr.perception.activity_recognition import X3DLearner
5+
from opendr.perception.activity_recognition import CoX3DLearner
6+
```
7+
8+
The script include logging of FLOPS and params for `learner.model`, inference timing, and energy-consumption (NVIDIA Jetson only).
9+
10+
The benchmarking runs twice; Once using `learner.infer` and once using `learner.model.forward`. The results of each are printed accordingly.
11+
12+
13+
## Setup
14+
Please install [`pytorch-benchmark`](https://github.com/LukasHedegaard/pytorch-benchmark):
15+
```bash
16+
pip install pytorch-benchmark
17+
```
18+
19+
## Running the benchmark
20+
X3D
21+
```bash
22+
./benchmark_x3d.py
23+
```
24+
25+
CoX3D
26+
```bash
27+
./benchmark_cox3d.py
28+
```

Diff for: ‎projects/perception/activity_recognition/benchmark/benchmark_cox3d.py

+127
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,127 @@
1+
# Copyright 2020-2022 OpenDR European Project
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
import torch
17+
import yaml
18+
from opendr.perception.activity_recognition import CoX3DLearner
19+
20+
from pytorch_benchmark import benchmark
21+
import logging
22+
from typing import List, Union
23+
from opendr.engine.target import Category
24+
from opendr.engine.data import Image
25+
26+
logger = logging.getLogger("benchmark")
27+
logging.basicConfig()
28+
logger.setLevel("DEBUG")
29+
30+
31+
def benchmark_cox3d():
32+
temp_dir = "./projects/perception/activity_recognition/benchmark/tmp"
33+
34+
num_runs = 100
35+
36+
# As found in src/opendr/perception/activity_recognition/x3d/hparams
37+
input_shape = {
38+
"xs": (3, 160, 160),
39+
"s": (3, 160, 160),
40+
"m": (3, 224, 224),
41+
"l": (3, 312, 312),
42+
}
43+
44+
# Max power of 2
45+
# batch_size = { # RTX2080Ti
46+
# "xs": 128,
47+
# "s": 64,
48+
# "m": 64,
49+
# "l": 8,
50+
# }
51+
# batch_size = { # TX2
52+
# "xs": 32,
53+
# "s": 16,
54+
# "m": 8,
55+
# "l": 4,
56+
# }
57+
# batch_size = { # Xavier
58+
# "xs": 64,
59+
# "s": 32,
60+
# "m": 16,
61+
# "l": 8,
62+
# }
63+
batch_size = { # CPU - larger batch sizes don't increase throughput
64+
"xs": 1,
65+
"s": 1,
66+
"m": 1,
67+
"l": 1,
68+
}
69+
70+
for backbone in ["s", "m", "l"]:
71+
print(f"==== Benchmarking CoX3DLearner ({backbone}) ====")
72+
73+
learner = CoX3DLearner(
74+
device="cuda" if torch.cuda.is_available() else "cpu",
75+
temp_path=temp_dir,
76+
backbone=backbone,
77+
)
78+
79+
sample = torch.randn(
80+
batch_size[backbone], *input_shape[backbone]
81+
) # (B, C, T, H, W)
82+
image_samples = [Image(v) for v in sample]
83+
image_sample = [Image(sample[0])]
84+
85+
def get_device_fn(*args):
86+
nonlocal learner
87+
return next(learner.model.parameters()).device
88+
89+
def transfer_to_device_fn(
90+
sample: Union[torch.Tensor, List[Category], List[Image]],
91+
device: torch.device,
92+
):
93+
if isinstance(sample, torch.Tensor):
94+
return sample.to(device=device)
95+
96+
assert isinstance(sample, list)
97+
98+
if isinstance(sample[0], Image):
99+
# Image.data i a numpy array, which is always on CPU
100+
return sample
101+
102+
assert isinstance(sample[0], Category)
103+
return [
104+
Category(prediction=s.data, confidence=s.confidence.to(device=device),)
105+
for s in sample
106+
]
107+
108+
print("== Benchmarking learner.infer ==")
109+
results1 = benchmark(
110+
model=learner.infer,
111+
sample=image_samples,
112+
sample_with_batch_size1=image_sample,
113+
num_runs=num_runs,
114+
get_device_fn=get_device_fn,
115+
transfer_to_device_fn=transfer_to_device_fn,
116+
batch_size=batch_size[backbone],
117+
print_fn=print,
118+
)
119+
print(yaml.dump({"learner.infer": results1}))
120+
121+
print("== Benchmarking model directly ==")
122+
results2 = benchmark(learner.model, sample, num_runs=num_runs, print_fn=print)
123+
print(yaml.dump({"learner.model.forward": results2}))
124+
125+
126+
if __name__ == "__main__":
127+
benchmark_cox3d()

Diff for: ‎projects/perception/activity_recognition/benchmark/benchmark_x3d.py

+128
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,128 @@
1+
# Copyright 2020-2022 OpenDR European Project
2+
#
3+
# Licensed under the Apache License, Version 2.0 (the "License");
4+
# you may not use this file except in compliance with the License.
5+
# You may obtain a copy of the License at
6+
#
7+
# http://www.apache.org/licenses/LICENSE-2.0
8+
#
9+
# Unless required by applicable law or agreed to in writing, software
10+
# distributed under the License is distributed on an "AS IS" BASIS,
11+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
# See the License for the specific language governing permissions and
13+
# limitations under the License.
14+
15+
16+
import torch
17+
import yaml
18+
from opendr.perception.activity_recognition import X3DLearner
19+
20+
from pytorch_benchmark import benchmark
21+
import logging
22+
from typing import List, Union
23+
from opendr.engine.target import Category
24+
from opendr.engine.data import Video
25+
26+
logger = logging.getLogger("benchmark")
27+
logging.basicConfig()
28+
logger.setLevel("DEBUG")
29+
30+
31+
def benchmark_x3d():
32+
temp_dir = "./projects/perception/activity_recognition/benchmark/tmp"
33+
34+
num_runs = 100
35+
36+
# As found in src/opendr/perception/activity_recognition/x3d/hparams
37+
input_shape = {
38+
"xs": (3, 4, 160, 160),
39+
"s": (3, 13, 160, 160),
40+
"m": (3, 16, 224, 224),
41+
"l": (3, 16, 312, 312),
42+
}
43+
44+
# Max power of 2
45+
# batch_size = { # RTX2080Ti
46+
# "xs": 32,
47+
# "s": 16,
48+
# "m": 8,
49+
# "l": 2,
50+
# }
51+
# batch_size = { # TX2
52+
# "xs": 16,
53+
# "s": 8,
54+
# "m": 4,
55+
# "l": 2,
56+
# }
57+
# batch_size = { # Xavier
58+
# "xs": 32,
59+
# "s": 16,
60+
# "m": 8,
61+
# "l": 2,
62+
# }
63+
batch_size = { # CPU - larger batch sizes don't increase throughput
64+
"xs": 1,
65+
"s": 1,
66+
"m": 1,
67+
"l": 1,
68+
}
69+
70+
for backbone in ["xs", "s", "m", "l"]:
71+
print(f"==== Benchmarking X3DLearner ({backbone}) ====")
72+
73+
learner = X3DLearner(
74+
device="cuda" if torch.cuda.is_available() else "cpu",
75+
temp_path=temp_dir,
76+
backbone=backbone,
77+
)
78+
learner.model.eval()
79+
80+
sample = torch.randn(
81+
batch_size[backbone], *input_shape[backbone]
82+
) # (B, C, T, H, W)
83+
video_samples = [Video(v) for v in sample]
84+
video_sample = [Video(sample[0])]
85+
86+
def get_device_fn(*args):
87+
nonlocal learner
88+
return next(learner.model.parameters()).device
89+
90+
def transfer_to_device_fn(
91+
sample: Union[torch.Tensor, List[Category], List[Video]],
92+
device: torch.device,
93+
):
94+
if isinstance(sample, torch.Tensor):
95+
return sample.to(device=device)
96+
97+
assert isinstance(sample, list)
98+
99+
if isinstance(sample[0], Video):
100+
# Video.data i a numpy array, which is always on CPU
101+
return sample
102+
103+
assert isinstance(sample[0], Category)
104+
return [
105+
Category(prediction=s.data, confidence=s.confidence.to(device=device),)
106+
for s in sample
107+
]
108+
109+
print("== Benchmarking learner.infer ==")
110+
results1 = benchmark(
111+
model=learner.infer,
112+
sample=video_samples,
113+
sample_with_batch_size1=video_sample,
114+
num_runs=num_runs,
115+
get_device_fn=get_device_fn,
116+
transfer_to_device_fn=transfer_to_device_fn,
117+
batch_size=batch_size[backbone],
118+
print_fn=print,
119+
)
120+
print(yaml.dump({"learner.infer": results1}))
121+
122+
print("== Benchmarking model directly ==")
123+
results2 = benchmark(learner.model, sample, num_runs=num_runs, print_fn=print)
124+
print(yaml.dump({"learner.model.forward": results2}))
125+
126+
127+
if __name__ == "__main__":
128+
benchmark_x3d()

Diff for: ‎projects/perception/activity_recognition/benchmark/install_on_server.sh

+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/bin/bash
2+
3+
conda create --name opendr python=3.8 -y
4+
conda activate opendr
5+
conda env config vars set OPENDR_HOME=$PWD
6+
conda env config vars set PYTHONPATH=$PWD/src:$PYTHONPATH
7+
8+
# Reactivate env to let env vars take effect
9+
conda deactivate
10+
conda activate opendr
11+
12+
pip install torch==1.7.1+cu110 torchvision==0.8.2+cu110 -f https://download.pytorch.org/whl/torch_stable.html
13+
pip install pytorch_lightning==1.2.3 onnxruntime==1.3.0 joblib>=1.0.1 pytorch_benchmark

Diff for: ‎projects/perception/activity_recognition/benchmark/requirements.txt

+1
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
pytorch-benchmark >= 0.2

0 commit comments

Comments
 (0)