From 76f551c9f410a69f856834f08ab8c690a740c944 Mon Sep 17 00:00:00 2001 From: ooooo <3164076421@qq.com> Date: Fri, 5 Jul 2024 20:44:50 +0800 Subject: [PATCH 1/4] [Typing][A-52] Add type annotations for python/paddle/hapi/model.py --- python/paddle/hapi/model.py | 135 +++++++++++++++++++++++++----------- 1 file changed, 93 insertions(+), 42 deletions(-) diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 63a88939584cfd..9071bccb08185a 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -11,6 +11,7 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations import contextlib import inspect @@ -19,11 +20,14 @@ import socket import time import warnings +from typing import TYPE_CHECKING, Any, Sequence, Union import numpy as np +from typing_extensions import TypeAlias import paddle import paddle.distributed as dist +import paddle.optimizer from paddle import base from paddle.autograd import no_grad from paddle.base import core @@ -45,6 +49,22 @@ from .callbacks import EarlyStopping, config_callbacks from .model_summary import summary +if TYPE_CHECKING: + import numpy.typing as npt + + from paddle import Tensor + from paddle._typing import DTypeLike + + from .callbacks import Callback + + _InputBatch: TypeAlias = Union[ + Tensor, + npt.NDArray[Any], + list[Tensor], + list[npt.NDArray[Any]], + ] + + __all__ = [] _parallel_context_initialized = False @@ -1161,7 +1181,16 @@ class Model: ... """ - def __init__(self, network, inputs=None, labels=None): + mode: str + network: paddle.nn.Layer + stop_training: bool + + def __init__( + self, + network: paddle.nn.Layer, + inputs: Input | Sequence[Input] | dict[str, Input] | None = None, + labels: Input | Sequence[Input] | None = None, + ) -> None: self.mode = 'train' self.network = network self._inputs = None @@ -1191,7 +1220,12 @@ def __init__(self, network, inputs=None, labels=None): else: self._adapter = StaticGraphAdapter(self) - def train_batch(self, inputs, labels=None, update=True): + def train_batch( + self, + inputs: _InputBatch, + labels: _InputBatch | None = None, + update: bool = True, + ) -> list[float] | tuple[list[npt.NDattay[Any]], list[float]]: """ Run one training step on one batch of data. And using `update` indicates @@ -1248,7 +1282,9 @@ def train_batch(self, inputs, labels=None, update=True): return loss @no_grad() - def eval_batch(self, inputs, labels=None): + def eval_batch( + self, inputs: _InputBatch, labels: _InputBatch | None = None + ) -> list[float] | tuple[list[npt.NDarray[Any]], list[float]]: """ Run one evaluating step on a batch of data. @@ -1304,7 +1340,7 @@ def eval_batch(self, inputs, labels=None): return loss @no_grad() - def predict_batch(self, inputs): + def predict_batch(self, inputs: _InputBatch) -> list[npt.NDArray[Any]]: """ Run one predicting step on a batch of data. @@ -1353,7 +1389,7 @@ def predict_batch(self, inputs): self._update_inputs() return loss - def save(self, path, training=True): + def save(self, path: str, training: bool = True) -> None: """ This function saves parameters, optimizer information or model and @@ -1420,7 +1456,12 @@ def save(self, path, training=True): else: self._adapter.save(path) - def load(self, path, skip_mismatch=False, reset_optimizer=False): + def load( + self, + path: str, + skip_mismatch: bool = False, + reset_optimizer: bool = False, + ) -> None: """ Load from files storing the model states and optimizer states. The file @@ -1533,7 +1574,7 @@ def _strip_postfix(path): else: return self._adapter.load(matched_param_state, optim_state) - def parameters(self, *args, **kwargs): + def parameters(self, *args, **kwargs) -> list[Tensor]: """ Returns a list of parameters of the model. @@ -1668,8 +1709,12 @@ def _check_amp_configs(amp_config_key_set): self._adapter._amp_configs[key] = amp_configs[key] def prepare( - self, optimizer=None, loss=None, metrics=None, amp_configs=None - ): + self, + optimizer: paddle.optimizer.Optimizer | None = None, + loss: paddle.nn.Layer | Callback[...] | None = None, + metrics: Metric | list[Metric] | None = None, + amp_configs: str | dict[str, Any] | None = None, + ) -> None: """ Configures the model before running. @@ -1749,22 +1794,22 @@ def prepare( def fit( self, - train_data=None, - eval_data=None, - batch_size=1, - epochs=1, - eval_freq=1, - log_freq=10, - save_dir=None, - save_freq=1, - verbose=2, - drop_last=False, - shuffle=True, - num_workers=0, - callbacks=None, - accumulate_grad_batches=1, - num_iters=None, - ): + train_data: Dataset | DataLoader = None, + eval_data: Dataset | DataLoader = None, + batch_size: int | list[int] = 1, + epochs: int = 1, + eval_freq: int = 1, + log_freq: int = 10, + save_dir: str | None = None, + save_freq: int = 1, + verbose: int = 2, + drop_last: bool = False, + shuffle: bool = True, + num_workers: int = 0, + callbacks: Callback | None = None, + accumulate_grad_batches: int = 1, + num_iters: int | None = None, + ) -> None: """ Trains the model for a fixed number of epochs. If `eval_data` is set, @@ -1998,14 +2043,14 @@ def fit( def evaluate( self, - eval_data, - batch_size=1, - log_freq=10, - verbose=2, - num_workers=0, - callbacks=None, - num_iters=None, - ): + eval_data: Dataset | DataLoader, + batch_size: int = 1, + log_freq: int = 10, + verbose: int = 2, + num_workers: int = 0, + callbacks: Callback | None = None, + num_iters: int | None = None, + ) -> dict[str, float | npt.NDArray[Any]]: """ Evaluate the loss and metrics of the model on input dataset. @@ -2109,13 +2154,13 @@ def evaluate( def predict( self, - test_data, - batch_size=1, - num_workers=0, - stack_outputs=False, - verbose=1, - callbacks=None, - ): + test_data: Dataset | DataLoader, + batch_size: int = 1, + num_workers: int = 0, + stack_outputs: bool = False, + verbose: int = 1, + callbacks: Callback | None = None, + ) -> list[npt.npt.NDArray[Any] | tuple[npt.NDArray[Any], ...]]: """ Compute the output predictions on testing data. @@ -2222,7 +2267,7 @@ def predict( cbks.on_end('predict', logs) return outputs - def _save_inference_model(self, path): + def _save_inference_model(self, path: str) -> None: """ Save inference model can be used in static or dynamic mode. @@ -2372,7 +2417,13 @@ def _run_one_epoch( return logs, outputs return logs - def summary(self, input_size=None, dtype=None): + def summary( + self, + input_size: ( + tuple[int, ...] | Input | list[tuple[int, ...] | Input] | None + ) = None, + dtype: DTypeLike | None = None, + ) -> dict[str, int]: """Prints a string summary of the network. Args: From 595161fb083b04829b6952986a98671c9bcb7b47 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Sun, 7 Jul 2024 16:18:07 +0800 Subject: [PATCH 2/4] fix many typing --- python/paddle/hapi/model.py | 89 ++++++++++++++++++++++++++++--------- 1 file changed, 68 insertions(+), 21 deletions(-) diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 9071bccb08185a..86b9851bb06013 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -20,7 +20,15 @@ import socket import time import warnings -from typing import TYPE_CHECKING, Any, Sequence, Union +from typing import ( + TYPE_CHECKING, + Any, + Callable, + Literal, + Sequence, + Union, + overload, +) import numpy as np from typing_extensions import TypeAlias @@ -53,9 +61,10 @@ import numpy.typing as npt from paddle import Tensor - from paddle._typing import DTypeLike + from paddle._typing.dtype_like import _DTypeLiteral from .callbacks import Callback + from .model_summary import ModelSummary _InputBatch: TypeAlias = Union[ Tensor, @@ -1181,7 +1190,7 @@ class Model: ... """ - mode: str + mode: Literal["train", "eval", "test"] network: paddle.nn.Layer stop_training: bool @@ -1225,7 +1234,7 @@ def train_batch( inputs: _InputBatch, labels: _InputBatch | None = None, update: bool = True, - ) -> list[float] | tuple[list[npt.NDattay[Any]], list[float]]: + ) -> list[float] | tuple[list[npt.NDArray[Any]], list[float]]: """ Run one training step on one batch of data. And using `update` indicates @@ -1284,7 +1293,7 @@ def train_batch( @no_grad() def eval_batch( self, inputs: _InputBatch, labels: _InputBatch | None = None - ) -> list[float] | tuple[list[npt.NDarray[Any]], list[float]]: + ) -> list[float] | tuple[list[npt.NDArray[Any]], list[float]]: """ Run one evaluating step on a batch of data. @@ -1574,7 +1583,7 @@ def _strip_postfix(path): else: return self._adapter.load(matched_param_state, optim_state) - def parameters(self, *args, **kwargs) -> list[Tensor]: + def parameters(self, *args: Any, **kwargs: Any) -> list[Tensor]: """ Returns a list of parameters of the model. @@ -1711,7 +1720,9 @@ def _check_amp_configs(amp_config_key_set): def prepare( self, optimizer: paddle.optimizer.Optimizer | None = None, - loss: paddle.nn.Layer | Callback[...] | None = None, + loss: ( + paddle.nn.Layer | Callable[[Tensor, Tensor], Tensor] | None + ) = None, metrics: Metric | list[Metric] | None = None, amp_configs: str | dict[str, Any] | None = None, ) -> None: @@ -1794,8 +1805,8 @@ def prepare( def fit( self, - train_data: Dataset | DataLoader = None, - eval_data: Dataset | DataLoader = None, + train_data: Dataset | DataLoader | None = None, + eval_data: Dataset | DataLoader | None = None, batch_size: int | list[int] = 1, epochs: int = 1, eval_freq: int = 1, @@ -1806,7 +1817,7 @@ def fit( drop_last: bool = False, shuffle: bool = True, num_workers: int = 0, - callbacks: Callback | None = None, + callbacks: list[Callback] | None = None, accumulate_grad_batches: int = 1, num_iters: int | None = None, ) -> None: @@ -1848,7 +1859,7 @@ def fit( subprocess used and loading data in main process. When train_data and eval_data are both the instance of Dataloader, this parameter will be ignored. Default: 0. - callbacks (Callback|None, optional): A list of `Callback` instances to apply + callbacks (list[Callback]|None, optional): A list of `Callback` instances to apply during training. If None, :ref:`api_paddle_callbacks_ProgBarLogger` and :ref:`api_paddle_callbacks_ModelCheckpoint` are automatically inserted. Default: None. accumulate_grad_batches (int, optional): The number of batches to accumulate gradient @@ -2048,7 +2059,7 @@ def evaluate( log_freq: int = 10, verbose: int = 2, num_workers: int = 0, - callbacks: Callback | None = None, + callbacks: list[Callback] | None = None, num_iters: int | None = None, ) -> dict[str, float | npt.NDArray[Any]]: """ @@ -2069,7 +2080,7 @@ def evaluate( 0 for no subprocess used and loading data in main process. When train_data and eval_data are both the instance of Dataloader, this parameter will be ignored. Default: 0. - callbacks (Callback|None, optional): A list of `Callback` instances to apply + callbacks (list[Callback]|None, optional): A list of `Callback` instances to apply during training. If None, `ProgBarLogger` and `ModelCheckpoint` are automatically inserted. Default: None. num_iters (int|None, optional): The number of iterations to evaluate the model. @@ -2152,15 +2163,51 @@ def evaluate( return eval_result + @overload def predict( self, test_data: Dataset | DataLoader, - batch_size: int = 1, - num_workers: int = 0, - stack_outputs: bool = False, - verbose: int = 1, - callbacks: Callback | None = None, - ) -> list[npt.npt.NDArray[Any] | tuple[npt.NDArray[Any], ...]]: + batch_size: int = ..., + num_workers: int = ..., + stack_outputs: Literal[True] = ..., + verbose: int = ..., + callbacks: Callback | None = ..., + ) -> list[npt.NDArray[Any]]: + ... + + @overload + def predict( + self, + test_data: Dataset | DataLoader, + batch_size: int = ..., + num_workers: int = ..., + stack_outputs: Literal[False] = ..., + verbose: int = ..., + callbacks: Callback | None = ..., + ) -> list[tuple[npt.NDArray[Any], ...]]: + ... + + @overload + def predict( + self, + test_data: Dataset | DataLoader, + batch_size: int = ..., + num_workers: int = ..., + stack_outputs: bool = ..., + verbose: int = ..., + callbacks: Callback | None = ..., + ) -> list[npt.NDArray[Any] | tuple[npt.NDArray[Any], ...]]: + ... + + def predict( + self, + test_data, + batch_size=1, + num_workers=0, + stack_outputs=False, + verbose=1, + callbacks=None, + ): """ Compute the output predictions on testing data. @@ -2422,8 +2469,8 @@ def summary( input_size: ( tuple[int, ...] | Input | list[tuple[int, ...] | Input] | None ) = None, - dtype: DTypeLike | None = None, - ) -> dict[str, int]: + dtype: _DTypeLiteral | None = None, + ) -> ModelSummary: """Prints a string summary of the network. Args: From 01f8d95deb5a362314f2eae368bae5fd2298f62f Mon Sep 17 00:00:00 2001 From: SigureMo Date: Sun, 7 Jul 2024 17:34:54 +0800 Subject: [PATCH 3/4] add timeout for save --- python/paddle/hapi/model.py | 1 + 1 file changed, 1 insertion(+) diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index 86b9851bb06013..ce001432efa1f9 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -1429,6 +1429,7 @@ def save(self, path: str, training: bool = True) -> None: .. code-block:: python + >>> # doctest: +TIMEOUT(30) >>> import paddle >>> import paddle.nn as nn >>> import paddle.vision.transforms as T From 9bc53e3806f5354b69353a840febf3624441ffe2 Mon Sep 17 00:00:00 2001 From: SigureMo Date: Sun, 7 Jul 2024 18:12:43 +0800 Subject: [PATCH 4/4] inc timeout for save (80s) --- python/paddle/hapi/model.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/paddle/hapi/model.py b/python/paddle/hapi/model.py index ce001432efa1f9..04716144afad4e 100644 --- a/python/paddle/hapi/model.py +++ b/python/paddle/hapi/model.py @@ -1429,7 +1429,7 @@ def save(self, path: str, training: bool = True) -> None: .. code-block:: python - >>> # doctest: +TIMEOUT(30) + >>> # doctest: +TIMEOUT(80) >>> import paddle >>> import paddle.nn as nn >>> import paddle.vision.transforms as T