test fixes and lints

Summary:
- followup recent pyre change D63415925
- make tests remove temporary files
- weights_only=True in torch.load
- lint fixes

3 test fixes from VRehnberg in #1914
- imageio channels fix
- frozen decorator in test_config
- load_blobs positional

Reviewed By: MichaelRamamonjisoa

Differential Revision: D66162167

fbshipit-source-id: 7737e174691b62f1708443a4fae07343cec5bfeb

Author: bottler

dev/linter.sh

@@ -36,5 +36,5 @@ then
 
   echo "Running pyre..."
   echo "To restart/kill pyre server, run 'pyre restart' or 'pyre kill' in fbcode/"
-  ( cd ~/fbsource/fbcode; pyre -l vision/fair/pytorch3d/ )
+  ( cd ~/fbsource/fbcode; arc pyre check //vision/fair/pytorch3d/... )
 fi

packaging/pytorch3d/meta.yaml

@@ -32,7 +32,6 @@ requirements:
 
 build:
   string: py{{py}}_{{ environ['CU_VERSION'] }}_pyt{{ environ['PYTORCH_VERSION_NODOT']}}
-  # script: LD_LIBRARY_PATH=$PREFIX/lib:$BUILD_PREFIX/lib:$LD_LIBRARY_PATH python setup.py install --single-version-externally-managed --record=record.txt # [not win]
   script: python setup.py install --single-version-externally-managed --record=record.txt # [not win]
   script_env:
     - CUDA_HOME
@@ -57,7 +56,6 @@ test:
     - pandas
     - sqlalchemy
   commands:
-    #pytest .
     python -m unittest discover -v -s tests -t .
 
 

projects/implicitron_trainer/impl/model_factory.py

@@ -116,7 +116,9 @@ def __call__(
                         "cuda:%d" % 0: "cuda:%d" % accelerator.local_process_index
                     }
                 model_state_dict = torch.load(
-                    model_io.get_model_path(model_path), map_location=map_location
+                    model_io.get_model_path(model_path),
+                    map_location=map_location,
+                    weights_only=True,
                 )
 
                 try:

projects/implicitron_trainer/impl/optimizer_factory.py

@@ -241,7 +241,7 @@ def _get_optimizer_state(
                     map_location = {
                         "cuda:%d" % 0: "cuda:%d" % accelerator.local_process_index
                     }
-                optimizer_state = torch.load(opt_path, map_location)
+                optimizer_state = torch.load(opt_path, map_location, weights_only=True)
             else:
                 raise FileNotFoundError(f"Optimizer state {opt_path} does not exist.")
         return optimizer_state

projects/nerf/nerf/dataset.py

@@ -84,9 +84,9 @@ def get_nerf_datasets(
 
     if autodownload and any(not os.path.isfile(p) for p in (cameras_path, image_path)):
         # Automatically download the data files if missing.
-        download_data((dataset_name,), data_root=data_root)
+        download_data([dataset_name], data_root=data_root)
 
-    train_data = torch.load(cameras_path)
+    train_data = torch.load(cameras_path, weights_only=True)
     n_cameras = train_data["cameras"]["R"].shape[0]
 
     _image_max_image_pixels = Image.MAX_IMAGE_PIXELS

projects/nerf/test_nerf.py

@@ -63,7 +63,7 @@ def main(cfg: DictConfig):
         raise ValueError(f"Model checkpoint {checkpoint_path} does not exist!")
 
     print(f"Loading checkpoint {checkpoint_path}.")
-    loaded_data = torch.load(checkpoint_path)
+    loaded_data = torch.load(checkpoint_path, weights_only=True)
     # Do not load the cached xy grid.
     # - this allows setting an arbitrary evaluation image size.
     state_dict = {

projects/nerf/train_nerf.py

@@ -77,7 +77,7 @@ def main(cfg: DictConfig):
         # Resume training if requested.
         if cfg.resume and os.path.isfile(checkpoint_path):
             print(f"Resuming from checkpoint {checkpoint_path}.")
-            loaded_data = torch.load(checkpoint_path)
+            loaded_data = torch.load(checkpoint_path, weights_only=True)
             model.load_state_dict(loaded_data["model"])
             stats = pickle.loads(loaded_data["stats"])
             print(f"   => resuming from epoch {stats.epoch}.")

pytorch3d/implicitron/models/feature_extractor/resnet_feature_extractor.py

@@ -106,7 +106,7 @@ def __post_init__(self):
             self.layers = torch.nn.ModuleList()
             self.proj_layers = torch.nn.ModuleList()
             for stage in range(self.max_stage):
-                stage_name = f"layer{stage+1}"
+                stage_name = f"layer{stage + 1}"
                 feature_name = self._get_resnet_stage_feature_name(stage)
                 if (stage + 1) in self.stages:
                     if (
@@ -139,7 +139,7 @@ def __post_init__(self):
         self.stages = set(self.stages)  # convert to set for faster "in"
 
     def _get_resnet_stage_feature_name(self, stage) -> str:
-        return f"res_layer_{stage+1}"
+        return f"res_layer_{stage + 1}"
 
     def _resnet_normalize_image(self, img: torch.Tensor) -> torch.Tensor:
         return (img - self._resnet_mean) / self._resnet_std

pytorch3d/implicitron/tools/model_io.py

@@ -111,10 +111,10 @@ def load_model(fl, map_location: Optional[dict]):
     flstats = get_stats_path(fl)
     flmodel = get_model_path(fl)
     flopt = get_optimizer_path(fl)
-    model_state_dict = torch.load(flmodel, map_location=map_location)
+    model_state_dict = torch.load(flmodel, map_location=map_location, weights_only=True)
     stats = load_stats(flstats)
     if os.path.isfile(flopt):
-        optimizer = torch.load(flopt, map_location=map_location)
+        optimizer = torch.load(flopt, map_location=map_location, weights_only=True)
     else:
         optimizer = None
 

pytorch3d/io/experimental_gltf_io.py

@@ -163,9 +163,8 @@ def _read_chunks(
     if binary_data is not None:
         binary_data = np.frombuffer(binary_data, dtype=np.uint8)
 
-    # pyre-fixme[7]: Expected `Optional[Tuple[Dict[str, typing.Any],
-    #  ndarray[typing.Any, typing.Any]]]` but got `Tuple[typing.Any,
-    #  Optional[ndarray[typing.Any, dtype[typing.Any]]]]`.
+    assert binary_data is not None
+
     return json_data, binary_data
 
 

pytorch3d/io/ply_io.py

@@ -1246,13 +1246,10 @@ def _save_ply(
         return
 
     color_np_type = np.ubyte if colors_as_uint8 else np.float32
-    verts_dtype = [("verts", np.float32, 3)]
+    verts_dtype: list = [("verts", np.float32, 3)]
     if verts_normals is not None:
         verts_dtype.append(("normals", np.float32, 3))
     if verts_colors is not None:
-        # pyre-fixme[6]: For 1st argument expected `Tuple[str,
-        #  Type[floating[_32Bit]], int]` but got `Tuple[str,
-        #  Type[Union[floating[_32Bit], unsignedinteger[typing.Any]]], int]`.
         verts_dtype.append(("colors", color_np_type, 3))
 
     vert_data = np.zeros(verts.shape[0], dtype=verts_dtype)

pytorch3d/renderer/mesh/clip.py

@@ -168,7 +168,7 @@ def _get_culled_faces(face_verts: torch.Tensor, frustum: ClipFrustum) -> torch.T
             position of the clipping planes.
 
     Returns:
-        faces_culled: An boolean tensor of size F specifying whether or not each face should be
+        faces_culled: boolean tensor of size F specifying whether or not each face should be
             culled.
     """
     clipping_planes = (

pytorch3d/renderer/mesh/textures.py

@@ -726,15 +726,17 @@ def __init__(
                     for each face
             verts_uvs: (N, V, 2) tensor giving the uv coordinates per vertex
                     (a FloatTensor with values between 0 and 1).
-            maps_ids: Used if there are to be multiple maps per face. This can be either a list of map_ids [(F,)]
+            maps_ids: Used if there are to be multiple maps per face.
+                    This can be either a list of map_ids [(F,)]
                     or a long tensor of shape (N, F) giving the id of the texture map
                     for each face. If maps_ids is present, the maps has an extra dimension M
                     (so maps_padded is (N, M, H, W, C) and maps_list has elements of
                     shape (M, H, W, C)).
                     Specifically, the color
-                    of a vertex V is given by an average of maps_padded[i, maps_ids[i, f], u, v, :]
+                    of a vertex V is given by an average of
+                       maps_padded[i, maps_ids[i, f], u, v, :]
                     over u and v integers adjacent to
-                    _verts_uvs_padded[i, _faces_uvs_padded[i, f, 0], :] .
+                       _verts_uvs_padded[i, _faces_uvs_padded[i, f, 0], :] .
             align_corners: If true, the extreme values 0 and 1 for verts_uvs
                     indicate the centers of the edge pixels in the maps.
             padding_mode: padding mode for outside grid values
@@ -1237,7 +1239,8 @@ def sample_textures(self, fragments, **kwargs) -> torch.Tensor:
             texels = texels.reshape(N, K, C, H_out, W_out).permute(0, 3, 4, 1, 2)
             return texels
         else:
-            # We have maps_ids_padded: (N, F), textures_map: (N, M, Hi, Wi, C),fragmenmts.pix_to_face: (N, Ho, Wo, K)
+            # We have maps_ids_padded: (N, F), textures_map: (N, M, Hi, Wi, C),
+            # fragments.pix_to_face: (N, Ho, Wo, K)
             # Get pixel_to_map_ids: (N, K, Ho, Wo) by indexing pix_to_face into maps_ids
             N, M, H_in, W_in, C = texture_maps.shape  # 3 for RGB
 
@@ -1827,7 +1830,7 @@ def sample_textures(self, fragments, faces_packed=None) -> torch.Tensor:
                 representation) which overlap the pixel.
 
         Returns:
-            texels: An texture per pixel of shape (N, H, W, K, C).
+            texels: A texture per pixel of shape (N, H, W, K, C).
             There will be one C dimensional value for each element in
             fragments.pix_to_face.
         """

tests/implicitron/test_config.py

@@ -246,7 +246,7 @@ def __init__(self, a: Any = 1, b: Any = 2):
 
         enable_get_default_args(Foo)
 
-        @dataclass()
+        @dataclass(frozen=True)
         class Bar:
             aa: int = 9
             bb: int = 9

tests/implicitron/test_extending_orm_types.py

@@ -87,7 +87,9 @@ def build(
         sequence_annotation: types.SequenceAnnotation,
         load_blobs: bool = True,
     ) -> CanineFrameData:
-        frame_data = super().build(frame_annotation, sequence_annotation, load_blobs)
+        frame_data = super().build(
+            frame_annotation, sequence_annotation, load_blobs=load_blobs
+        )
         frame_data.num_dogs = frame_annotation.num_dogs or 101
         frame_data.magnetic_field_average_flux_density = (
             frame_annotation.magnetic_field.average_flux_density

tests/pulsar/test_forward.py

@@ -76,7 +76,7 @@ def test_bg_weight(self):
                         "test_out",
                         "test_forward_TestForward_test_bg_weight_hits.png",
                     ),
-                    (hits * 255.0).cpu().to(torch.uint8).numpy(),
+                    (hits * 255.0).cpu().to(torch.uint8).squeeze(2).numpy(),
                 )
             self.assertEqual(hits[500, 500, 0].item(), 1.0)
             self.assertTrue(
@@ -139,7 +139,7 @@ def test_basic_3chan(self):
                         "test_out",
                         "test_forward_TestForward_test_basic_3chan_hits.png",
                     ),
-                    (hits * 255.0).cpu().to(torch.uint8).numpy(),
+                    (hits * 255.0).cpu().to(torch.uint8).squeeze(2).numpy(),
                 )
             self.assertEqual(hits[500, 500, 0].item(), 1.0)
             self.assertTrue(
@@ -194,15 +194,15 @@ def test_basic_1chan(self):
                         "test_out",
                         "test_forward_TestForward_test_basic_1chan.png",
                     ),
-                    (result * 255.0).cpu().to(torch.uint8).numpy(),
+                    (result * 255.0).cpu().to(torch.uint8).squeeze(2).numpy(),
                 )
                 imageio.imsave(
                     path.join(
                         path.dirname(__file__),
                         "test_out",
                         "test_forward_TestForward_test_basic_1chan_hits.png",
                     ),
-                    (hits * 255.0).cpu().to(torch.uint8).numpy(),
+                    (hits * 255.0).cpu().to(torch.uint8).squeeze(2).numpy(),
                 )
             self.assertEqual(hits[500, 500, 0].item(), 1.0)
             self.assertTrue(
@@ -264,7 +264,7 @@ def test_basic_8chan(self):
                         "test_out",
                         "test_forward_TestForward_test_basic_8chan_hits.png",
                     ),
-                    (hits * 255.0).cpu().to(torch.uint8).numpy(),
+                    (hits * 255.0).cpu().to(torch.uint8).squeeze(2).numpy(),
                 )
             self.assertEqual(hits[500, 500, 0].item(), 1.0)
             self.assertTrue(

tests/test_build.py

@@ -43,7 +43,7 @@ def test_valid_ipynbs(self):
         tutorials = sorted(tutorials_dir.glob("*.ipynb"))
 
         for tutorial in tutorials:
-            with open(tutorial) as f:
+            with open(tutorial, encoding="utf8") as f:
                 json.load(f)
 
     @unittest.skipIf(in_conda_build or in_re_worker, "In conda build, or RE worker")

tests/test_io_gltf.py

@@ -53,7 +53,7 @@ def _write(mesh, path, **kwargs) -> None:
     io.save_mesh(mesh, path, **kwargs)
 
     with open(path, "rb") as f:
-        _, stored_length = _read_header(f)
+        _, stored_length = _read_header(f)  # pyre-ignore
     assert stored_length == os.path.getsize(path)
 
 
@@ -191,14 +191,14 @@ def test_save_cow(self):
         mesh = _load(glb, device=device)
 
         # save the mesh to a glb file
-        glb = DATA_DIR / "cow_write.glb"
-        _write(mesh, glb)
+        glb_reload = DATA_DIR / "cow_write.glb"
+        _write(mesh, glb_reload)
 
         # load again
-        glb_reload = DATA_DIR / "cow_write.glb"
         self.assertTrue(glb_reload.is_file())
         device = torch.device("cuda:0")
         mesh_reload = _load(glb_reload, device=device)
+        glb_reload.unlink()
 
         # assertions
         self.assertEqual(mesh_reload.faces_packed().shape, (5856, 3))
@@ -232,6 +232,7 @@ def test_save_ico_sphere(self):
         # reload the ico_sphere
         device = torch.device("cuda:0")
         mesh_reload = _load(glb, device=device, include_textures=False)
+        glb.unlink()
 
         self.assertClose(
             ico_sphere_mesh.verts_padded().cpu(),
@@ -299,9 +300,9 @@ def test_load_save_load_cow_texturesvertex(self):
         _write(mesh, glb)
 
         # reload the mesh glb file saved in TexturesVertex format
-        glb = DATA_DIR / "cow_write_texturesvertex.glb"
         self.assertTrue(glb.is_file())
         mesh_dash = _load(glb, device=device)
+        glb.unlink()
         self.assertEqual(len(mesh_dash), 1)
 
         self.assertEqual(mesh_dash.faces_packed().shape, (5856, 3))
@@ -381,3 +382,4 @@ def test_save_toy(self):
 
         glb = DATA_DIR / "example_write_texturesvertex.glb"
         _write(mesh, glb)
+        glb.unlink()

tests/test_points_to_volumes.py

@@ -196,7 +196,7 @@ def init_cube_mesh(batch_size: int, device: str):
         Generate a batch of `batch_size` cube meshes.
         """
 
-        device = torch.device(device)
+        device_ = torch.device(device)
 
         verts, faces = [], []
 
@@ -213,7 +213,7 @@ def init_cube_mesh(batch_size: int, device: str):
                     [0.0, 0.0, 1.0],
                 ],
                 dtype=torch.float32,
-                device=device,
+                device=device_,
             )
             verts.append(v)
             faces.append(
@@ -233,7 +233,7 @@ def init_cube_mesh(batch_size: int, device: str):
                         [0, 1, 6],
                     ],
                     dtype=torch.int64,
-                    device=device,
+                    device=device_,
                 )
             )
 
@@ -316,7 +316,7 @@ def test_from_point_cloud(self, interp_mode="trilinear"):
                             outfile = (
                                 outdir
                                 + f"/rgb_{interp_mode}"
-                                + f"_{str(volume_size).replace(' ','')}"
+                                + f"_{str(volume_size).replace(' ', '')}"
                                 + f"_{vidx:003d}_sldim{slice_dim}.png"
                             )
                             im.save(outfile)

tests/test_raysampling.py

@@ -639,4 +639,4 @@ def test_heterogeneous_sampling(self, batch_size=8):
                                 origin1, origin2, rtol=1e-4, atol=1e-4
                             ) == (id1 == id2), (origin1, origin2, id1, id2)
                             assert not torch.allclose(dir1, dir2), (dir1, dir2)
-                            self.assertClose(len1, len2), (len1, len2)
+                            self.assertClose(len1, len2)

tests/test_render_volumes.py

@@ -689,7 +689,7 @@ def test_rotating_cube_volume_render(self):
                             outfile = (
                                 outdir
                                 + f"/rgb_{sample_mode}"
-                                + f"_{str(volume_size).replace(' ','')}"
+                                + f"_{str(volume_size).replace(' ', '')}"
                                 + f"_{imidx:003d}"
                             )
                             if image_ is image: