Skip to content

CosineDecay

minnt.schedulers.CosineDecay

Bases: GenericDecay

A cosine decay learning rate scheduler with optional linear warmup.

This scheduler is a convenience wrapper around minnt.schedulers.GenericDecay with the decay parameter set to "cosine".

Source code in minnt/schedulers/cosine_decay.py 
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
class CosineDecay(GenericDecay):
    """A cosine decay learning rate scheduler with optional linear warmup.

    This scheduler is a convenience wrapper around [minnt.schedulers.GenericDecay][]
    with the `decay` parameter set to `"cosine"`.
    """
    def __init__(
        self,
        optimizer: torch.optim.Optimizer,
        total_steps: int,
        *,
        final_decay: float = 0.0,
        warmup: int | float = 0,
        last_epoch: int = -1,
        warn_about_exceeding_steps: bool = True,
    ) -> None:
        """Creates a new CosineDecay scheduler instance.

        Please refer to the documentation of [minnt.schedulers.GenericDecay][] for details.
        """
        super().__init__(
            optimizer,
            total_steps,
            decay="cosine",
            final_decay=final_decay,
            warmup=warmup,
            last_epoch=last_epoch,
            warn_about_exceeding_steps=warn_about_exceeding_steps,
        )

__init__ 

__init__(
    optimizer: Optimizer,
    total_steps: int,
    *,
    final_decay: float = 0.0,
    warmup: int | float = 0,
    last_epoch: int = -1,
    warn_about_exceeding_steps: bool = True
) -> None

Creates a new CosineDecay scheduler instance.

Please refer to the documentation of minnt.schedulers.GenericDecay for details.

Source code in minnt/schedulers/cosine_decay.py 
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
def __init__(
    self,
    optimizer: torch.optim.Optimizer,
    total_steps: int,
    *,
    final_decay: float = 0.0,
    warmup: int | float = 0,
    last_epoch: int = -1,
    warn_about_exceeding_steps: bool = True,
) -> None:
    """Creates a new CosineDecay scheduler instance.

    Please refer to the documentation of [minnt.schedulers.GenericDecay][] for details.
    """
    super().__init__(
        optimizer,
        total_steps,
        decay="cosine",
        final_decay=final_decay,
        warmup=warmup,
        last_epoch=last_epoch,
        warn_about_exceeding_steps=warn_about_exceeding_steps,
    )