183 lines
6.8 KiB
Python
183 lines
6.8 KiB
Python
# Copyright (c) Meta Platforms, Inc. and affiliates.
|
|
# All rights reserved.
|
|
|
|
# This source code is licensed under the license found in the
|
|
# LICENSE file in the root directory of this source tree.
|
|
|
|
from typing import Optional, Tuple, Type
|
|
|
|
import torch
|
|
from torch import nn
|
|
|
|
from ..position_encoding import PositionEmbeddingRandom
|
|
|
|
from ..sam2_utils import LayerNorm2d
|
|
|
|
|
|
class PromptEncoder(nn.Module):
|
|
def __init__(
|
|
self,
|
|
embed_dim: int,
|
|
image_embedding_size: Tuple[int, int],
|
|
input_image_size: Tuple[int, int],
|
|
mask_in_chans: int,
|
|
activation: Type[nn.Module] = nn.GELU,
|
|
) -> None:
|
|
"""
|
|
Encodes prompts for input to SAM's mask decoder.
|
|
|
|
Arguments:
|
|
embed_dim (int): The prompts' embedding dimension
|
|
image_embedding_size (tuple(int, int)): The spatial size of the
|
|
image embedding, as (H, W).
|
|
input_image_size (int): The padded size of the image as input
|
|
to the image encoder, as (H, W).
|
|
mask_in_chans (int): The number of hidden channels used for
|
|
encoding input masks.
|
|
activation (nn.Module): The activation to use when encoding
|
|
input masks.
|
|
"""
|
|
super().__init__()
|
|
self.embed_dim = embed_dim
|
|
self.input_image_size = input_image_size
|
|
self.image_embedding_size = image_embedding_size
|
|
self.pe_layer = PositionEmbeddingRandom(embed_dim // 2)
|
|
|
|
self.num_point_embeddings: int = 4 # pos/neg point + 2 box corners
|
|
point_embeddings = [
|
|
nn.Embedding(1, embed_dim) for i in range(self.num_point_embeddings)
|
|
]
|
|
self.point_embeddings = nn.ModuleList(point_embeddings)
|
|
self.not_a_point_embed = nn.Embedding(1, embed_dim)
|
|
|
|
self.mask_input_size = (
|
|
4 * image_embedding_size[0],
|
|
4 * image_embedding_size[1],
|
|
)
|
|
self.mask_downscaling = nn.Sequential(
|
|
nn.Conv2d(1, mask_in_chans // 4, kernel_size=2, stride=2),
|
|
LayerNorm2d(mask_in_chans // 4),
|
|
activation(),
|
|
nn.Conv2d(mask_in_chans // 4, mask_in_chans, kernel_size=2, stride=2),
|
|
LayerNorm2d(mask_in_chans),
|
|
activation(),
|
|
nn.Conv2d(mask_in_chans, embed_dim, kernel_size=1),
|
|
)
|
|
self.no_mask_embed = nn.Embedding(1, embed_dim)
|
|
|
|
def get_dense_pe(self) -> torch.Tensor:
|
|
"""
|
|
Returns the positional encoding used to encode point prompts,
|
|
applied to a dense set of points the shape of the image encoding.
|
|
|
|
Returns:
|
|
torch.Tensor: Positional encoding with shape
|
|
1x(embed_dim)x(embedding_h)x(embedding_w)
|
|
"""
|
|
return self.pe_layer(self.image_embedding_size).unsqueeze(0)
|
|
|
|
def _embed_points(
|
|
self,
|
|
points: torch.Tensor,
|
|
labels: torch.Tensor,
|
|
pad: bool,
|
|
) -> torch.Tensor:
|
|
"""Embeds point prompts."""
|
|
points = points + 0.5 # Shift to center of pixel
|
|
if pad:
|
|
padding_point = torch.zeros((points.shape[0], 1, 2), device=points.device)
|
|
padding_label = -torch.ones((labels.shape[0], 1), device=labels.device)
|
|
points = torch.cat([points, padding_point], dim=1)
|
|
labels = torch.cat([labels, padding_label], dim=1)
|
|
point_embedding = self.pe_layer.forward_with_coords(
|
|
points, self.input_image_size
|
|
)
|
|
point_embedding[labels == -1] = 0.0
|
|
point_embedding[labels == -1] += self.not_a_point_embed.weight
|
|
point_embedding[labels == 0] += self.point_embeddings[0].weight
|
|
point_embedding[labels == 1] += self.point_embeddings[1].weight
|
|
point_embedding[labels == 2] += self.point_embeddings[2].weight
|
|
point_embedding[labels == 3] += self.point_embeddings[3].weight
|
|
return point_embedding
|
|
|
|
def _embed_boxes(self, boxes: torch.Tensor) -> torch.Tensor:
|
|
"""Embeds box prompts."""
|
|
boxes = boxes + 0.5 # Shift to center of pixel
|
|
coords = boxes.reshape(-1, 2, 2)
|
|
corner_embedding = self.pe_layer.forward_with_coords(
|
|
coords, self.input_image_size
|
|
)
|
|
corner_embedding[:, 0, :] += self.point_embeddings[2].weight
|
|
corner_embedding[:, 1, :] += self.point_embeddings[3].weight
|
|
return corner_embedding
|
|
|
|
def _embed_masks(self, masks: torch.Tensor) -> torch.Tensor:
|
|
"""Embeds mask inputs."""
|
|
mask_embedding = self.mask_downscaling(masks)
|
|
return mask_embedding
|
|
|
|
def _get_batch_size(
|
|
self,
|
|
points: Optional[Tuple[torch.Tensor, torch.Tensor]],
|
|
boxes: Optional[torch.Tensor],
|
|
masks: Optional[torch.Tensor],
|
|
) -> int:
|
|
"""
|
|
Gets the batch size of the output given the batch size of the input prompts.
|
|
"""
|
|
if points is not None:
|
|
return points[0].shape[0]
|
|
elif boxes is not None:
|
|
return boxes.shape[0]
|
|
elif masks is not None:
|
|
return masks.shape[0]
|
|
else:
|
|
return 1
|
|
|
|
def _get_device(self) -> torch.device:
|
|
return self.point_embeddings[0].weight.device
|
|
|
|
def forward(
|
|
self,
|
|
points: Optional[Tuple[torch.Tensor, torch.Tensor]],
|
|
boxes: Optional[torch.Tensor],
|
|
masks: Optional[torch.Tensor],
|
|
) -> Tuple[torch.Tensor, torch.Tensor]:
|
|
"""
|
|
Embeds different types of prompts, returning both sparse and dense
|
|
embeddings.
|
|
|
|
Arguments:
|
|
points (tuple(torch.Tensor, torch.Tensor) or none): point coordinates
|
|
and labels to embed.
|
|
boxes (torch.Tensor or none): boxes to embed
|
|
masks (torch.Tensor or none): masks to embed
|
|
|
|
Returns:
|
|
torch.Tensor: sparse embeddings for the points and boxes, with shape
|
|
BxNx(embed_dim), where N is determined by the number of input points
|
|
and boxes.
|
|
torch.Tensor: dense embeddings for the masks, in the shape
|
|
Bx(embed_dim)x(embed_H)x(embed_W)
|
|
"""
|
|
bs = self._get_batch_size(points, boxes, masks)
|
|
sparse_embeddings = torch.empty(
|
|
(bs, 0, self.embed_dim), device=self._get_device()
|
|
)
|
|
if points is not None:
|
|
coords, labels = points
|
|
point_embeddings = self._embed_points(coords, labels, pad=(boxes is None))
|
|
sparse_embeddings = torch.cat([sparse_embeddings, point_embeddings], dim=1)
|
|
if boxes is not None:
|
|
box_embeddings = self._embed_boxes(boxes)
|
|
sparse_embeddings = torch.cat([sparse_embeddings, box_embeddings], dim=1)
|
|
|
|
if masks is not None:
|
|
dense_embeddings = self._embed_masks(masks)
|
|
else:
|
|
dense_embeddings = self.no_mask_embed.weight.reshape(1, -1, 1, 1).expand(
|
|
bs, -1, self.image_embedding_size[0], self.image_embedding_size[1]
|
|
)
|
|
|
|
return sparse_embeddings, dense_embeddings
|