[AutoParallel] Fix ViT embed #1
Conversation
| @@ -622,7 +623,8 @@ def __init__(self, *args, **kwargs): | |||
| # self.is_pretraining = True | |||
|
|
|||
| def _wrap_for_dist_loader(self, train_dataloader): | |||
| dist_loader = super()._wrap_for_dist_loader(train_dataloader) | |||
| dtensor_idx = [2, 3] | |||
There was a problem hiding this comment.
dense_tensor_idx 是否好一些，dtensor_idx 容易有歧义
| num_attention_heads=config.num_attention_heads, | ||
| ) | ||
|
|
||
| def get_tensor_parallel_split_mappings(num_layers): |
There was a problem hiding this comment.
这个函数是给动手用的，其中的内容可以去掉，换成pass。因为整个去掉此函数可能导致 from_pretrain 报错
| super().__init__(config) | ||
| self.visual = Qwen2VisionTransformerPretrainedModel._from_config(config.vision_config) | ||
| # mesh = fleet.auto.get_mesh() | ||
| mesh = dist.ProcessMesh([[0], [1], [2], [3], [4], [5], [6], [7]], dim_names=["dp", "mp"]) |
There was a problem hiding this comment.
是否这样会通用一些
mesh = dist.get_mesh()
if "pp" in mesh.dim_names:
mesh = mesh.get_mesh_with_dim("pp")[0]
|
是否也要修改 |
是的，这里需要修改，忽略了 |
| ColumnParallelLinear = linear_utils.ColumnParallelLinear | ||
| RowParallelLinear = linear_utils.RowParallelLinear | ||
|
|
||
| if config.tensor_parallel_degree > 1: |
There was a problem hiding this comment.
类似判断 config.tensor_parallel_degree > 1 的分支都可以去掉，这些分支只有动手需要
jeff41404
merged commit 5f32352
into
jeff41404:verify_auto_parallel_intermediate_api_in_paddlemix
Fix ViT embed