Merge branch 'master' into bugfix/multiclass_jaccard

CHANGELOG.md

@@ -16,7 +16,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
--
+- Changed `update_count` and `update_called` from private to public methods ([#1370](https://github.com/Lightning-AI/metrics/pull/1370))
 
 
 ### Deprecated

src/torchmetrics/detection/mean_ap.py

@@ -40,11 +40,11 @@
 def compute_area(input: List[Any], iou_type: str = "bbox") -> Tensor:
     """Compute area of input depending on the specified iou_type.
 
-    Default output for empty input is torch.Tensor([])
+    Default output for empty input is :class:`~torch.Tensor`
     """
     if len(input) == 0:
 
-        return torch.Tensor([])
+        return Tensor([])
 
     if iou_type == "bbox":
         return box_area(torch.stack(input))
@@ -124,7 +124,7 @@ class COCOMetricResults(BaseMetricResults):
     )
 
 
-def _segm_iou(det: List[Tuple[np.ndarray, np.ndarray]], gt: List[Tuple[np.ndarray, np.ndarray]]) -> torch.Tensor:
+def _segm_iou(det: List[Tuple[np.ndarray, np.ndarray]], gt: List[Tuple[np.ndarray, np.ndarray]]) -> Tensor:
     """
     Compute IOU between detections and ground-truths using mask-IOU. Based on pycocotools toolkit for mask_utils
     Args:
@@ -431,7 +431,7 @@ def _move_list_states_to_cpu(self) -> None:
             current_to_cpu = []
             if isinstance(current_val, Sequence):
                 for cur_v in current_val:
-                    # Cannot handle RLE as torch.Tensor
+                    # Cannot handle RLE as Tensor
                     if not isinstance(cur_v, tuple):
                         cur_v = cur_v.to("cpu")
                     current_to_cpu.append(cur_v)

src/torchmetrics/functional/image/gradients.py

@@ -56,7 +56,7 @@ def image_gradients(img: Tensor) -> Tuple[Tensor, Tensor]:
 
     Raises:
         TypeError:
-            If ``img`` is not of the type ``torch.Tensor``.
+            If ``img`` is not of the type :class:`~torch.Tensor`.
         RuntimeError:
             If ``img`` is not a 4D tensor.
 

src/torchmetrics/functional/text/bert.py

@@ -66,11 +66,11 @@ def _get_embeddings_and_idf_scale(
         user_forward_fn:
             A user's own forward function used in a combination with ``user_model``. This function must
             take ``user_model`` and a python dictionary of containing ``"input_ids"`` and ``"attention_mask"``
-            represented by ``torch.Tensor`` as an input and return the model's output represented by the single
-            ``torch.Tensor``.
+            represented by :class:`~torch.Tensor` as an input and return the model's output represented by the single
+            :class:`~torch.Tensor`.
 
     Return:
-        A tuple of ``torch.Tensor``s containing the model's embeddings and the normalized tokens IDF.
+        A tuple of :class:`~torch.Tensor`s containing the model's embeddings and the normalized tokens IDF.
         When ``idf = False``, tokens IDF is not calculated, and a matrix of mean weights is returned instead.
         For a single sentence, ``mean_weight = 1/seq_len``, where ``seq_len`` is a sum over the corresponding
         ``attention_mask``.
@@ -273,15 +273,15 @@ def bert_score(
         user_tokenizer:
             A user's own tokenizer used with the own model. This must be an instance with the ``__call__`` method.
             This method must take an iterable of sentences (``List[str]``) and must return a python dictionary
-            containing ``"input_ids"`` and ``"attention_mask"`` represented by ``torch.Tensor``.
-            It is up to the user's model of whether ``"input_ids"`` is a ``torch.Tensor`` of input ids or embedding
-            vectors. his tokenizer must prepend an equivalent of ``[CLS]`` token and append an equivalent of ``[SEP]``
-            token as `transformers` tokenizer does.
+            containing ``"input_ids"`` and ``"attention_mask"`` represented by :class:`~torch.Tensor`.
+            It is up to the user's model of whether ``"input_ids"`` is a :class:`~torch.Tensor` of input ids
+            or embedding vectors. his tokenizer must prepend an equivalent of ``[CLS]`` token and append an equivalent
+            of ``[SEP]`` token as `transformers` tokenizer does.
         user_forward_fn:
             A user's own forward function used in a combination with ``user_model``.
             This function must take ``user_model`` and a python dictionary of containing ``"input_ids"``
-            and ``"attention_mask"`` represented by ``torch.Tensor`` as an input and return the model's output
-            represented by the single ``torch.Tensor``.
+            and ``"attention_mask"`` represented by :class:`~torch.Tensor` as an input and return the model's output
+            represented by the single :class:`~torch.Tensor`.
         verbose: An indication of whether a progress bar to be displayed during the embeddings' calculation.
         idf: An indication of whether normalization using inverse document frequencies should be used.
         device: A device to be used for calculation.

src/torchmetrics/functional/text/helper_embedding_metric.py

@@ -147,7 +147,7 @@ def _check_shape_of_model_output(output: Tensor, input_ids: Tensor) -> None:
     invalid_out_shape = len(output.shape) != 3 or output.shape[0] != bs or output.shape[1] != seq_len
     if invalid_out_shape:
         raise ValueError(
-            "The model output must be `torch.Tensor` of a shape `[batch_size, seq_len, model_dim]` "
+            "The model output must be `Tensor` of a shape `[batch_size, seq_len, model_dim]` "
             f"i.e. [{bs}, {seq_len}. , `model_dim`], but got {output.shape}."
         )
 
@@ -249,7 +249,7 @@ def _get_tokens_idf_default_value(self) -> float:
 
     @staticmethod
     def _set_of_tokens(input_ids: Tensor) -> Set:
-        """Return set of tokens from the `input_ids` `torch.Tensor`."""
+        """Return set of tokens from the `input_ids` :class:`~torch.Tensor`."""
         return set(input_ids.tolist())
 
 
@@ -265,10 +265,8 @@ def __init__(
     ) -> None:
         """
         Args:
-            input_ids:
-                Input ids (`torch.Tensor`).
-            attention_mask:
-                Attention mask (`torch.Tensor`).
+            input_ids: Input indexes
+            attention_mask: Attention mask
             idf:
                 An indication of whether calculate token inverse document frequencies to weight the model embeddings.
             tokens_idf:

src/torchmetrics/metric.py

@@ -141,9 +141,20 @@ def __init__(
 
     @property
     def _update_called(self) -> bool:
-        # Needed for lightning integration
+        # TODO: this is needed for internal lightning, remove after v0.12 and update on lightning side
         return self._update_count > 0
 
+    @property
+    def update_called(self) -> bool:
+        """Returns `True` if `update` or `forward` has been called initialization or last `reset`."""
+        return self._update_count > 0
+
+    @property
+    def update_count(self) -> int:
+        """Get the number of times `update` and/or `forward` has been called since initialization or last
+        `reset`."""
+        return self._update_count
+
     def add_state(
         self,
         name: str,
@@ -155,8 +166,8 @@ def add_state(
 
         Args:
             name: The name of the state variable. The variable will then be accessible at ``self.name``.
-            default: Default value of the state; can either be a ``torch.Tensor`` or an empty list. The state will be
-                reset to this value when ``self.reset()`` is called.
+            default: Default value of the state; can either be a :class:`~torch.Tensor` or an empty list.
+                The state will be reset to this value when ``self.reset()`` is called.
             dist_reduce_fx (Optional): Function to reduce state across multiple processes in distributed mode.
                 If value is ``"sum"``, ``"mean"``, ``"cat"``, ``"min"`` or ``"max"`` we will use ``torch.sum``,
                 ``torch.mean``, ``torch.cat``, ``torch.min`` and ``torch.max``` respectively, each with argument
@@ -171,9 +182,10 @@ def add_state(
 
             The metric states would be synced as follows
 
-            - If the metric state is ``torch.Tensor``, the synced value will be a stacked ``torch.Tensor`` across
-              the process dimension if the metric state was a ``torch.Tensor``. The original ``torch.Tensor`` metric
-              state retains dimension and hence the synchronized output will be of shape ``(num_process, ...)``.
+            - If the metric state is :class:`~torch.Tensor`, the synced value will be a stacked :class:`~torch.Tensor`
+              across the process dimension if the metric state was a :class:`~torch.Tensor`. The original
+              :class:`~torch.Tensor` metric state retains dimension and hence the synchronized output will be of shape
+              ``(num_process, ...)``.
 
             - If the metric state is a ``list``, the synced value will be a ``list`` containing the
               combined elements from all processes.

src/torchmetrics/text/bert.py

@@ -59,14 +59,15 @@ class BERTScore(Metric):
         user_tokenizer:
             A user's own tokenizer used with the own model. This must be an instance with the `__call__` method.
             This method must take an iterable of sentences (`List[str]`) and must return a python dictionary
-            containing `"input_ids"` and `"attention_mask"` represented by `torch.Tensor`. It is up to the user's model
-            of whether `"input_ids"` is a `torch.Tensor` of input ids or embedding vectors.
-            This tokenizer must prepend an equivalent of `[CLS]` token and append an equivalent of `[SEP]` token
-            as `transformers` tokenizer does.
+            containing `"input_ids"` and `"attention_mask"` represented by :class:`~torch.Tensor`.
+            It is up to the user's model of whether `"input_ids"` is a :class:`~torch.Tensor` of input ids or embedding
+            vectors. This tokenizer must prepend an equivalent of `[CLS]` token and append an equivalent of `[SEP]`
+            token as `transformers` tokenizer does.
         user_forward_fn:
             A user's own forward function used in a combination with `user_model`. This function must take `user_model`
-            and a python dictionary of containing `"input_ids"` and `"attention_mask"` represented by `torch.Tensor`
-            as an input and return the model's output represented by the single `torch.Tensor`.
+            and a python dictionary of containing `"input_ids"` and `"attention_mask"` represented
+            by :class:`~torch.Tensor` as an input and return the model's output represented by the single
+            :class:`~torch.Tensor`.
         verbose: An indication of whether a progress bar to be displayed during the embeddings' calculation.
         idf: An indication whether normalization using inverse document frequencies should be used.
         device: A device to be used for calculation.

tests/unittests/bases/test_collections.py

@@ -423,7 +423,7 @@ def test_check_compute_groups_correctness(self, metrics, expected, preds, target
             m2.update(preds, target)
 
             for _, member in m.items():
-                assert member._update_called
+                assert member.update_called
 
             assert m.compute_groups == expected
             assert m2.compute_groups == {}
@@ -433,7 +433,7 @@ def test_check_compute_groups_correctness(self, metrics, expected, preds, target
             m2.update(preds, target)
 
             for _, member in m.items():
-                assert member._update_called
+                assert member.update_called
 
             # compare results for correctness
             res_cg = m.compute()

tests/unittests/bases/test_metric.py

@@ -476,3 +476,23 @@ def test_compute_on_cpu_arg_forward(method):
     val = metric.compute()
     assert all(str(v.device) == "cpu" for v in val)
     assert all(torch.allclose(v, x.cpu()) for v in val)
+
+
+@pytest.mark.parametrize("method", ["forward", "update"])
+@pytest.mark.parametrize("metric", [DummyMetricSum, DummyListMetric])
+def test_update_properties(metric, method):
+    m = metric()
+    x = torch.randn(
+        1,
+    ).squeeze()
+    for i in range(10):
+        if method == "update":
+            m.update(x)
+        if method == "forward":
+            _ = m(x)
+        assert m.update_called
+        assert m.update_count == i + 1
+
+    m.reset()
+    assert not m.update_called
+    assert m.update_count == 0

tests/unittests/retrieval/helpers.py

@@ -41,14 +41,14 @@
 
 
 def get_group_indexes(indexes: Union[Tensor, np.ndarray]) -> List[Union[Tensor, np.ndarray]]:
-    """Given an integer `torch.Tensor` or `np.ndarray` `indexes`, return a `torch.Tensor` or `np.ndarray` of
-    indexes for each different value in `indexes`.
+    """Given an integer :class:`~torch.Tensor` or `np.ndarray` `indexes`, return a :class:`~torch.Tensor` or
+    `np.ndarray` of indexes for each different value in `indexes`.
 
     Args:
-        indexes: a `torch.Tensor` or `np.ndarray` of integers
+        indexes: a :class:`~torch.Tensor` or `np.ndarray` of integers
 
     Return:
-        A list of integer `torch.Tensor`s or `np.ndarray`s
+        A list of integer :class:`~torch.Tensor`s or `np.ndarray`s
 
     Example:
         >>> indexes = torch.tensor([0, 0, 0, 1, 1, 1, 1])