Skip to content

Commit f745e7d

Browse files
amyerobertsamyeroberts
authored
Remove repeated prepare_images in processor tests (#33163)
* Remove repeated prepare_images * Address comments - update docstring; explanatory comment
1 parent 0574fa6 commit f745e7d

21 files changed

+140
-304
lines changed

‎tests/models/align/test_processor_align.py

-12
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import tempfile
1919
import unittest
2020

21-
import numpy as np
2221
import pytest
2322

2423
from transformers import BertTokenizer, BertTokenizerFast
@@ -30,8 +29,6 @@
3029

3130

3231
if is_vision_available():
33-
from PIL import Image
34-
3532
from transformers import AlignProcessor, EfficientNetImageProcessor
3633

3734

@@ -86,15 +83,6 @@ def get_image_processor(self, **kwargs):
8683
def tearDown(self):
8784
shutil.rmtree(self.tmpdirname)
8885

89-
def prepare_image_inputs(self):
90-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
91-
or a list of PyTorch tensors if one specifies torchify=True.
92-
"""
93-
94-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
95-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
96-
return image_inputs
97-
9886
def test_save_load_pretrained_default(self):
9987
tokenizer_slow = self.get_tokenizer()
10088
tokenizer_fast = self.get_rust_tokenizer()

‎tests/models/blip/test_processor_blip.py

+5-15
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,22 @@
1515
import tempfile
1616
import unittest
1717

18-
import numpy as np
1918
import pytest
2019

2120
from transformers.testing_utils import require_vision
2221
from transformers.utils import is_vision_available
2322

23+
from ...test_processing_common import ProcessorTesterMixin
2424

25-
if is_vision_available():
26-
from PIL import Image
2725

26+
if is_vision_available():
2827
from transformers import AutoProcessor, BertTokenizer, BlipImageProcessor, BlipProcessor, PreTrainedTokenizerFast
2928

3029

3130
@require_vision
32-
class BlipProcessorTest(unittest.TestCase):
31+
class BlipProcessorTest(ProcessorTesterMixin, unittest.TestCase):
32+
processor_class = BlipProcessor
33+
3334
def setUp(self):
3435
self.tmpdirname = tempfile.mkdtemp()
3536

@@ -49,17 +50,6 @@ def get_image_processor(self, **kwargs):
4950
def tearDown(self):
5051
shutil.rmtree(self.tmpdirname)
5152

52-
def prepare_image_inputs(self):
53-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
54-
or a list of PyTorch tensors if one specifies torchify=True.
55-
"""
56-
57-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
58-
59-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
60-
61-
return image_inputs
62-
6353
def test_save_load_pretrained_additional_features(self):
6454
processor = BlipProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
6555
processor.save_pretrained(self.tmpdirname)

‎tests/models/blip_2/test_processor_blip_2.py

+5-15
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,22 @@
1515
import tempfile
1616
import unittest
1717

18-
import numpy as np
1918
import pytest
2019

2120
from transformers.testing_utils import require_vision
2221
from transformers.utils import is_vision_available
2322

23+
from ...test_processing_common import ProcessorTesterMixin
2424

25-
if is_vision_available():
26-
from PIL import Image
2725

26+
if is_vision_available():
2827
from transformers import AutoProcessor, Blip2Processor, BlipImageProcessor, GPT2Tokenizer, PreTrainedTokenizerFast
2928

3029

3130
@require_vision
32-
class Blip2ProcessorTest(unittest.TestCase):
31+
class Blip2ProcessorTest(ProcessorTesterMixin, unittest.TestCase):
32+
processor_class = Blip2Processor
33+
3334
def setUp(self):
3435
self.tmpdirname = tempfile.mkdtemp()
3536

@@ -49,17 +50,6 @@ def get_image_processor(self, **kwargs):
4950
def tearDown(self):
5051
shutil.rmtree(self.tmpdirname)
5152

52-
def prepare_image_inputs(self):
53-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
54-
or a list of PyTorch tensors if one specifies torchify=True.
55-
"""
56-
57-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
58-
59-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
60-
61-
return image_inputs
62-
6353
def test_save_load_pretrained_additional_features(self):
6454
processor = Blip2Processor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
6555
processor.save_pretrained(self.tmpdirname)

‎tests/models/chinese_clip/test_processor_chinese_clip.py

+10-15
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,24 @@
1818
import tempfile
1919
import unittest
2020

21-
import numpy as np
2221
import pytest
2322

2423
from transformers import BertTokenizer, BertTokenizerFast
2524
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
2625
from transformers.testing_utils import require_vision
2726
from transformers.utils import FEATURE_EXTRACTOR_NAME, is_vision_available
2827

28+
from ...test_processing_common import ProcessorTesterMixin
2929

30-
if is_vision_available():
31-
from PIL import Image
3230

31+
if is_vision_available():
3332
from transformers import ChineseCLIPImageProcessor, ChineseCLIPProcessor
3433

3534

3635
@require_vision
37-
class ChineseCLIPProcessorTest(unittest.TestCase):
36+
class ChineseCLIPProcessorTest(ProcessorTesterMixin, unittest.TestCase):
37+
processor_class = ChineseCLIPProcessor
38+
3839
def setUp(self):
3940
self.tmpdirname = tempfile.mkdtemp()
4041

@@ -76,6 +77,11 @@ def setUp(self):
7677
with open(self.image_processor_file, "w", encoding="utf-8") as fp:
7778
json.dump(image_processor_map, fp)
7879

80+
tokenizer = self.get_tokenizer()
81+
image_processor = self.get_image_processor()
82+
processor = ChineseCLIPProcessor(tokenizer=tokenizer, image_processor=image_processor)
83+
processor.save_pretrained(self.tmpdirname)
84+
7985
def get_tokenizer(self, **kwargs):
8086
return BertTokenizer.from_pretrained(self.tmpdirname, **kwargs)
8187

@@ -88,17 +94,6 @@ def get_image_processor(self, **kwargs):
8894
def tearDown(self):
8995
shutil.rmtree(self.tmpdirname)
9096

91-
def prepare_image_inputs(self):
92-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
93-
or a list of PyTorch tensors if one specifies torchify=True.
94-
"""
95-
96-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
97-
98-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
99-
100-
return image_inputs
101-
10297
def test_save_load_pretrained_default(self):
10398
tokenizer_slow = self.get_tokenizer()
10499
tokenizer_fast = self.get_rust_tokenizer()

‎tests/models/clip/test_processor_clip.py

-14
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import tempfile
1919
import unittest
2020

21-
import numpy as np
2221
import pytest
2322

2423
from transformers import CLIPTokenizer, CLIPTokenizerFast
@@ -30,8 +29,6 @@
3029

3130

3231
if is_vision_available():
33-
from PIL import Image
34-
3532
from transformers import CLIPImageProcessor, CLIPProcessor
3633

3734

@@ -79,17 +76,6 @@ def get_image_processor(self, **kwargs):
7976
def tearDown(self):
8077
shutil.rmtree(self.tmpdirname)
8178

82-
def prepare_image_inputs(self):
83-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
84-
or a list of PyTorch tensors if one specifies torchify=True.
85-
"""
86-
87-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
88-
89-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
90-
91-
return image_inputs
92-
9379
def test_save_load_pretrained_default(self):
9480
tokenizer_slow = self.get_tokenizer()
9581
tokenizer_fast = self.get_rust_tokenizer()

‎tests/models/clipseg/test_processor_clipseg.py

+5-14
Original file line numberDiff line numberDiff line change
@@ -18,23 +18,24 @@
1818
import tempfile
1919
import unittest
2020

21-
import numpy as np
2221
import pytest
2322

2423
from transformers import CLIPTokenizer, CLIPTokenizerFast
2524
from transformers.models.clip.tokenization_clip import VOCAB_FILES_NAMES
2625
from transformers.testing_utils import require_vision
2726
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available
2827

28+
from ...test_processing_common import ProcessorTesterMixin
2929

30-
if is_vision_available():
31-
from PIL import Image
3230

31+
if is_vision_available():
3332
from transformers import CLIPSegProcessor, ViTImageProcessor
3433

3534

3635
@require_vision
37-
class CLIPSegProcessorTest(unittest.TestCase):
36+
class CLIPSegProcessorTest(ProcessorTesterMixin, unittest.TestCase):
37+
processor_class = CLIPSegProcessor
38+
3839
def setUp(self):
3940
self.tmpdirname = tempfile.mkdtemp()
4041

@@ -75,16 +76,6 @@ def get_image_processor(self, **kwargs):
7576
def tearDown(self):
7677
shutil.rmtree(self.tmpdirname)
7778

78-
def prepare_image_inputs(self):
79-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
80-
or a list of PyTorch tensors if one specifies torchify=True."""
81-
82-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
83-
84-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
85-
86-
return image_inputs
87-
8879
def test_save_load_pretrained_default(self):
8980
tokenizer_slow = self.get_tokenizer()
9081
tokenizer_fast = self.get_rust_tokenizer()

‎tests/models/flava/test_processor_flava.py

+5-15
Original file line numberDiff line numberDiff line change
@@ -19,18 +19,17 @@
1919
import tempfile
2020
import unittest
2121

22-
import numpy as np
2322
import pytest
2423

2524
from transformers import BertTokenizer, BertTokenizerFast
2625
from transformers.models.bert.tokenization_bert import VOCAB_FILES_NAMES
2726
from transformers.testing_utils import require_vision
2827
from transformers.utils import IMAGE_PROCESSOR_NAME, is_vision_available
2928

29+
from ...test_processing_common import ProcessorTesterMixin
3030

31-
if is_vision_available():
32-
from PIL import Image
3331

32+
if is_vision_available():
3433
from transformers import FlavaImageProcessor, FlavaProcessor
3534
from transformers.models.flava.image_processing_flava import (
3635
FLAVA_CODEBOOK_MEAN,
@@ -41,7 +40,9 @@
4140

4241

4342
@require_vision
44-
class FlavaProcessorTest(unittest.TestCase):
43+
class FlavaProcessorTest(ProcessorTesterMixin, unittest.TestCase):
44+
processor_class = FlavaProcessor
45+
4546
def setUp(self):
4647
self.tmpdirname = tempfile.mkdtemp()
4748

@@ -91,17 +92,6 @@ def get_image_processor(self, **kwargs):
9192
def tearDown(self):
9293
shutil.rmtree(self.tmpdirname)
9394

94-
def prepare_image_inputs(self):
95-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
96-
or a list of PyTorch tensors if one specifies torchify=True.
97-
"""
98-
99-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
100-
101-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
102-
103-
return image_inputs
104-
10595
def test_save_load_pretrained_default(self):
10696
tokenizer_slow = self.get_tokenizer()
10797
tokenizer_fast = self.get_rust_tokenizer()

‎tests/models/git/test_processor_git.py

+5-15
Original file line numberDiff line numberDiff line change
@@ -15,21 +15,22 @@
1515
import tempfile
1616
import unittest
1717

18-
import numpy as np
1918
import pytest
2019

2120
from transformers.testing_utils import require_vision
2221
from transformers.utils import is_vision_available
2322

23+
from ...test_processing_common import ProcessorTesterMixin
2424

25-
if is_vision_available():
26-
from PIL import Image
2725

26+
if is_vision_available():
2827
from transformers import AutoProcessor, BertTokenizer, CLIPImageProcessor, GitProcessor, PreTrainedTokenizerFast
2928

3029

3130
@require_vision
32-
class GitProcessorTest(unittest.TestCase):
31+
class GitProcessorTest(ProcessorTesterMixin, unittest.TestCase):
32+
processor_class = GitProcessor
33+
3334
def setUp(self):
3435
self.tmpdirname = tempfile.mkdtemp()
3536

@@ -51,17 +52,6 @@ def get_image_processor(self, **kwargs):
5152
def tearDown(self):
5253
shutil.rmtree(self.tmpdirname)
5354

54-
def prepare_image_inputs(self):
55-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
56-
or a list of PyTorch tensors if one specifies torchify=True.
57-
"""
58-
59-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
60-
61-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
62-
63-
return image_inputs
64-
6555
def test_save_load_pretrained_additional_features(self):
6656
processor = GitProcessor(tokenizer=self.get_tokenizer(), image_processor=self.get_image_processor())
6757
processor.save_pretrained(self.tmpdirname)

‎tests/models/grounding_dino/test_processor_grounding_dino.py

-15
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,6 @@
1818
import tempfile
1919
import unittest
2020

21-
import numpy as np
2221
import pytest
2322

2423
from transformers import BertTokenizer, BertTokenizerFast, GroundingDinoProcessor
@@ -35,8 +34,6 @@
3534
from transformers.models.grounding_dino.modeling_grounding_dino import GroundingDinoObjectDetectionOutput
3635

3736
if is_vision_available():
38-
from PIL import Image
39-
4037
from transformers import GroundingDinoImageProcessor
4138

4239

@@ -96,18 +93,6 @@ def get_image_processor(self, **kwargs):
9693
def tearDown(self):
9794
shutil.rmtree(self.tmpdirname)
9895

99-
# Copied from tests.models.clip.test_processor_clip.CLIPProcessorTest.prepare_image_inputs
100-
def prepare_image_inputs(self):
101-
"""This function prepares a list of PIL images, or a list of numpy arrays if one specifies numpify=True,
102-
or a list of PyTorch tensors if one specifies torchify=True.
103-
"""
104-
105-
image_inputs = [np.random.randint(255, size=(3, 30, 400), dtype=np.uint8)]
106-
107-
image_inputs = [Image.fromarray(np.moveaxis(x, 0, -1)) for x in image_inputs]
108-
109-
return image_inputs
110-
11196
def get_fake_grounding_dino_output(self):
11297
torch.manual_seed(42)
11398
return GroundingDinoObjectDetectionOutput(

0 commit comments

Comments
 (0)