groupuser/phi-1_5
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Improves type hinting on configuration arguments.

Browse Source
This commit is contained in:
Gustavo de Rosa 2023-11-01 23:40:19 +00:00 committed by huggingface-web
parent 45f4b21525
commit 92557d03bb
1 changed files with 12 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
configuration_mixformer_sequential.py
View File

@ -21,24 +21,24 @@ class MixFormerSequentialConfig(PretrainedConfig):
def __init__( def __init__(
self, self,
vocab_size: Optional[int] = 50304, vocab_size: int = 50304,
n_positions: Optional[int] = 2048, n_positions: int = 2048,
n_embd: Optional[int] = 1024, n_embd: int = 1024,
n_layer: Optional[int] = 20, n_layer: int = 20,
n_inner: Optional[int] = None, n_inner: Optional[int] = None,
n_head: Optional[int] = 16, n_head: int = 16,
n_head_kv: Optional[int] = None, n_head_kv: Optional[int] = None,
rotary_dim: Optional[int] = 32, rotary_dim: Optional[int] = 32,
activation_function: Optional[str] = "gelu_new", activation_function: Optional[str] = "gelu_new",
flash_rotary: bool = False, flash_rotary: bool = False,
fused_dense: bool = False, fused_dense: bool = False,
attn_pdrop: Optional[float] = 0.0, attn_pdrop: float = 0.0,
embd_pdrop: Optional[float] = 0.0, embd_pdrop: float = 0.0,
resid_pdrop: Optional[float] = 0.0, resid_pdrop: float = 0.0,
layer_norm_epsilon: Optional[float] = 1e-5, layer_norm_epsilon: float = 1e-5,
initializer_range: Optional[float] = 0.02, initializer_range: float = 0.02,
tie_word_embeddings: Optional[bool] = False, tie_word_embeddings: bool = False,
pad_vocab_size_multiple: Optional[int] = 64, pad_vocab_size_multiple: int = 64,
**kwargs **kwargs
) -> None: ) -> None:
self.vocab_size = int(math.ceil(vocab_size / pad_vocab_size_multiple) * pad_vocab_size_multiple) self.vocab_size = int(math.ceil(vocab_size / pad_vocab_size_multiple) * pad_vocab_size_multiple)