
    ǄgsI                        d Z ddlZddlZddlZddlmZ ddlmZmZm	Z	m
Z
mZmZmZmZ ddlZddlmZ ddlmZ ddlmZmZ ddlmZ d	d
lmZ g dZddlmZ eeedf   e
e   f   ZddZd ZddZd Z  G d de      Z! ejD                         	 dde	e   dedeee#ejH                  f      fd       Z% G d de      Z&y)z>Implementation for Stochastic Weight Averaging implementation.    N)deepcopy)AnyCallableIterableListLiteralOptionalTupleUnion)Tensor)Module)_format_paramLRScheduler)&_get_foreach_kernels_supported_devices   )	Optimizer)AveragedModel	update_bnSWALRget_ema_multi_avg_fnget_swa_multi_avg_fnget_ema_avg_fnget_swa_avg_fn)"_group_tensors_by_device_and_dtype.c                 Z     t        j                         dt        dt        f fd       }|S )zRGet the function applying exponential moving average (EMA) across multiple params.ema_param_listcurrent_param_listc                     t        j                  | d         st        j                  | d         rt        j                  | |dz
         y t	        | |      D ]"  \  }}|j                  |z  |dz
  z  z          $ y )Nr   r   )torchis_floating_point
is_complex_foreach_lerp_zipcopy_)r   r   _p_emap_modeldecays        ]/home/mcse/projects/flask_80/flask-venv/lib/python3.12/site-packages/torch/optim/swa_utils.py
ema_updatez(get_ema_multi_avg_fn.<locals>.ema_update%   s     "">!#459I9I1:
   1CQYO"%n6H"I CwEEMGq5y,AABC    )r   no_grad
PARAM_LISTr(   r*   s   ` r)   r   r   "   s7     ]]_C: C: C C r+   c            	      z    t        j                         dt        dt        dt        t        t
        f   fd       } | S )zQGet the function applying stochastic weight average (SWA) across multiple params.averaged_param_listr   num_averagedc                 x   t        j                  | d         st        j                  | d         rt        j                  | |d|dz   z         y t        j                  ||       }t        |t              r(t        j                  | ||dz   gt        |       z         y t        j                  | |d|dz   z         y )Nr   r   g      ?)alpha)
r   r    r!   r"   _foreach_sub
isinstancer   _foreach_addcdiv_len_foreach_add_)r0   r   r1   diffss       r)   
swa_updatez(get_swa_multi_avg_fn.<locals>.swa_update6   s     ""#6q#9:e>N>N"?
   #%7lQ>N9O &&'9;NOE,/'''!A%&-@)AA ##'c\A=M6Nr+   )r   r,   r-   r   r   intr:   s    r)   r   r   3   sG     ]]_'& FCK( 2 r+   c                 Z     t        j                         dt        dt        f fd       }|S )zQGet the function applying exponential moving average (EMA) across a single param.	ema_paramcurrent_paramc                      | z  dz
  |z  z   S Nr    )r>   r?   r1   r(   s      r)   r*   z"get_ema_avg_fn.<locals>.ema_updateV   s    y AI#>>>r+   )r   r,   r   r.   s   ` r)   r   r   S   s3     ]]_?f ?V ? ? r+   c            	      z    t        j                         dt        dt        dt        t        t        f   fd       } | S )zPGet the function applying stochastic weight average (SWA) across a single param.averaged_paramr?   r1   c                     | || z
  |dz   z  z   S rA   rB   )rD   r?   r1   s      r)   r:   z"get_swa_avg_fn.<locals>.swa_update`   s     !?LSTDT UUUr+   )r   r,   r   r   r;   r<   s    r)   r   r   ]   sK     ]]_VV/5VEJ6SV;EWV V
 r+   c                        e Zd ZU dZeed<   	 	 	 	 ddedeee	e
j                  f      deeeeeee	f   gef      deeeeeee	f   gdf      f fdZd	 Zdefd
Z xZS )r   a  Implements averaged model for Stochastic Weight Averaging (SWA) and Exponential Moving Average (EMA).

    Stochastic Weight Averaging was proposed in `Averaging Weights Leads to
    Wider Optima and Better Generalization`_ by Pavel Izmailov, Dmitrii
    Podoprikhin, Timur Garipov, Dmitry Vetrov and Andrew Gordon Wilson
    (UAI 2018).

    Exponential Moving Average is a variation of `Polyak averaging`_,
    but using exponential weights instead of equal weights across iterations.

    AveragedModel class creates a copy of the provided module :attr:`model`
    on the device :attr:`device` and allows to compute running averages of the
    parameters of the :attr:`model`.

    Args:
        model (torch.nn.Module): model to use with SWA/EMA
        device (torch.device, optional): if provided, the averaged model will be
            stored on the :attr:`device`
        avg_fn (function, optional): the averaging function used to update
            parameters; the function must take in the current value of the
            :class:`AveragedModel` parameter, the current value of :attr:`model`
            parameter, and the number of models already averaged; if None,
            an equally weighted average is used (default: None)
        multi_avg_fn (function, optional): the averaging function used to update
            parameters inplace; the function must take in the current values of the
            :class:`AveragedModel` parameters as a list, the current values of :attr:`model`
            parameters as a list, and the number of models already averaged; if None,
            an equally weighted average is used (default: None)
        use_buffers (bool): if ``True``, it will compute running averages for
            both the parameters and the buffers of the model. (default: ``False``)

    Example:
        >>> # xdoctest: +SKIP("undefined variables")
        >>> loader, optimizer, model, loss_fn = ...
        >>> swa_model = torch.optim.swa_utils.AveragedModel(model)
        >>> scheduler = torch.optim.lr_scheduler.CosineAnnealingLR(optimizer,
        >>>                                     T_max=300)
        >>> swa_start = 160
        >>> swa_scheduler = SWALR(optimizer, swa_lr=0.05)
        >>> for i in range(300):
        >>>      for input, target in loader:
        >>>          optimizer.zero_grad()
        >>>          loss_fn(model(input), target).backward()
        >>>          optimizer.step()
        >>>      if i > swa_start:
        >>>          swa_model.update_parameters(model)
        >>>          swa_scheduler.step()
        >>>      else:
        >>>          scheduler.step()
        >>>
        >>> # Update bn statistics for the swa_model at the end
        >>> torch.optim.swa_utils.update_bn(loader, swa_model)

    You can also use custom averaging functions with the `avg_fn` or `multi_avg_fn` parameters.
    If no averaging function is provided, the default is to compute
    equally-weighted average of the weights (SWA).

    Example:
        >>> # xdoctest: +SKIP("undefined variables")
        >>> # Compute exponential moving averages of the weights and buffers
        >>> ema_model = torch.optim.swa_utils.AveragedModel(model,
        >>>             torch.optim.swa_utils.get_ema_multi_avg_fn(0.9), use_buffers=True)

    .. note::
        When using SWA/EMA with models containing Batch Normalization you may
        need to update the activation statistics for Batch Normalization.
        This can be done either by using the :meth:`torch.optim.swa_utils.update_bn`
        or by setting :attr:`use_buffers` to `True`. The first approach updates the
        statistics in a post-training step by passing data through the model. The
        second does it during the parameter update phase by averaging all buffers.
        Empirical evidence has shown that updating the statistics in normalization
        layers increases accuracy, but you may wish to empirically test which
        approach yields the best results in your problem.

    .. note::
        :attr:`avg_fn` and `multi_avg_fn` are not saved in the :meth:`state_dict` of the model.

    .. note::
        When :meth:`update_parameters` is called for the first time (i.e.
        :attr:`n_averaged` is `0`) the parameters of `model` are copied
        to the parameters of :class:`AveragedModel`. For every subsequent
        call of :meth:`update_parameters` the function `avg_fn` is used
        to update the parameters.

    .. _Averaging Weights Leads to Wider Optima and Better Generalization:
        https://arxiv.org/abs/1803.05407
    .. _There Are Many Consistent Explanations of Unlabeled Data: Why You Should
        Average:
        https://arxiv.org/abs/1806.05594
    .. _SWALP: Stochastic Weight Averaging in Low-Precision Training:
        https://arxiv.org/abs/1904.11943
    .. _Stochastic Weight Averaging in Parallel: Large-Batch Training That
        Generalizes Well:
        https://arxiv.org/abs/2001.02312
    .. _Polyak averaging:
        https://paperswithcode.com/method/polyak-averaging
    
n_averagedNmodeldeviceavg_fnmulti_avg_fnc                 2   t         |           |	|J d       t        |      | _        | | j                  j	                  |      | _        | j                  dt        j                  dt        j                  |             || _	        || _
        || _        y )Nz6Only one of avg_fn and multi_avg_fn should be providedrG   r   )dtyperI   )super__init__r   moduletoregister_bufferr   tensorlongrJ   rK   use_buffers)selfrH   rI   rJ   rK   rU   	__class__s         r)   rO   zAveragedModel.__init__   s     	Nl2	DC	D2uo++..0DK%,,q

6J	
 (&r+   c                 &     | j                   |i |S )zForward pass.)rP   )rV   argskwargss      r)   forwardzAveragedModel.forward   s    t{{D+F++r+   c           	      &   | j                   rFt        j                  | j                  j	                         | j                  j                               n| j	                         }| j                   r2t        j                  |j	                         |j                               n|j	                         }g }g }t        ||      D ]  \  }}|j                         j                  |j                        }|j                  |j                                |j                  |       | j                  dk(  so|j                         j                  |        | j                  dkD  r| j                  | j                   t        ||g      }	|	j!                         D ]  \  \  }
}\  \  }}}| j                  r-| j                  ||| j                  j                  |
             H|
D|
j"                  t%               v r.t'               } |||| j                  j                  |
             t)               }| j                  j                  |
      }t        ||      D ]  \  }}|j                   ||||                nwt        ||      D ]h  \  }}| j                  j                  |j                        }|j                         j                  | j                  |j                         ||             j | j                   st        | j                  j                         |j                               D ]K  \  }}|j                         j                  |j                         j                  |j                               M | xj                  dz  c_        y)zUpdate model parameters.r   Nr   )rU   	itertoolschainrP   
parametersbuffersr#   detachrQ   rI   appendrG   r$   rK   rJ   r   itemstyper   r   r   )rV   rH   
self_parammodel_paramself_param_detachedmodel_param_detached
p_averagedr'   p_model_grouped_tensorsrI   r%   self_paramsmodel_paramsrK   rJ   rG   b_swab_models                      r)   update_parameterszAveragedModel.update_parameters   s     OODKK224dkk6I6I6KL" 	  OOE,,.@!!# 	
 7979#&z;#? 	4J~~'**:+<+<=H&&z'8'8':; ''1!#!!#))(3	4 ??Q  ,0C"D(*>?# %**,V KVQ "/[,(())'t7I7I&7Q *"KK+Q+SS';'=$'t7I7I&7Q "0!1%)__%7%7%?
36{L3Q V/J&,,VJ-TUV'V, ,/')=, 'J "&!3!3J4E4E!FJ%%'--J$5$5$7*M	  #&dkk&9&9&;U]]_"M Hw$$W^^%5%8%8%FGH1r+   )NNNF)__name__
__module____qualname____doc__r   __annotations__r   r	   r   r;   r   rI   r   r-   rO   r[   rp   __classcell__rW   s   @r)   r   r   i   s    `D 
 6:SW '' sELL012' 6653E"F"NOP	'
 j*eFCK.@A4GH
'0,=v =r+   r   loaderrH   rI   c                 2   i }|j                         D ]Z  }t        |t        j                  j                   j                  j
                        s<|j                          |j                  ||<   \ |sy|j                  }|j                          |j                         D ]	  }d|_         | D ]8  }t        |t        t        f      r|d   }||j                  |      } ||       : |j                         D ]  }||   |_         |j                  |       y)a  Update BatchNorm running_mean, running_var buffers in the model.

    It performs one pass over data in `loader` to estimate the activation
    statistics for BatchNorm layers in the model.

    Args:
        loader (torch.utils.data.DataLoader): dataset loader to compute the
            activation statistics on. Each data batch should be either a
            tensor, or a list/tuple whose first element is a tensor
            containing data.
        model (torch.nn.Module): model for which we seek to update BatchNorm
            statistics.
        device (torch.device, optional): If set, data will be transferred to
            :attr:`device` before being passed into :attr:`model`.

    Example:
        >>> # xdoctest: +SKIP("Undefined variables")
        >>> loader, model = ...
        >>> torch.optim.swa_utils.update_bn(loader, model)

    .. note::
        The `update_bn` utility assumes that each data batch in :attr:`loader`
        is either a tensor or a list or tuple of tensors; in the latter case it
        is assumed that :meth:`model.forward()` should be called on the first
        element of the list or tuple corresponding to the data batch.
    Nr   )modulesr5   r   nn	batchnorm
_BatchNormreset_running_statsmomentumtrainingtrainkeyslisttuplerQ   )rx   rH   rI   momentarP   was_traininginput	bn_modules           r)   r   r   *  s   @ G--/ .fehh..88CCD&&($ooGFO.
 >>L	KKM,,.   edE]+!HEHHV$Ee \\^ 0	$Y/	0	KKr+   c                   t     e Zd ZdZ	 	 	 ddededed   f fdZed        Z	ed        Z
ed	        Zd
 Z xZS )r   aD  Anneals the learning rate in each parameter group to a fixed value.

    This learning rate scheduler is meant to be used with Stochastic Weight
    Averaging (SWA) method (see `torch.optim.swa_utils.AveragedModel`).

    Args:
        optimizer (torch.optim.Optimizer): wrapped optimizer
        swa_lrs (float or list): the learning rate value for all param groups
            together or separately for each group.
        annealing_epochs (int): number of epochs in the annealing phase
            (default: 10)
        annealing_strategy (str): "cos" or "linear"; specifies the annealing
            strategy: "cos" for cosine annealing, "linear" for linear annealing
            (default: "cos")
        last_epoch (int): the index of the last epoch (default: -1)

    The :class:`SWALR` scheduler can be used together with other
    schedulers to switch to a constant learning rate late in the training
    as in the example below.

    Example:
        >>> # xdoctest: +SKIP("Undefined variables")
        >>> loader, optimizer, model = ...
        >>> lr_lambda = lambda epoch: 0.9
        >>> scheduler = torch.optim.lr_scheduler.MultiplicativeLR(optimizer,
        >>>        lr_lambda=lr_lambda)
        >>> swa_scheduler = torch.optim.swa_utils.SWALR(optimizer,
        >>>        anneal_strategy="linear", anneal_epochs=20, swa_lr=0.05)
        >>> swa_start = 160
        >>> for i in range(300):
        >>>      for input, target in loader:
        >>>          optimizer.zero_grad()
        >>>          loss_fn(model(input), target).backward()
        >>>          optimizer.step()
        >>>      if i > swa_start:
        >>>          swa_scheduler.step()
        >>>      else:
        >>>          scheduler.step()

    .. _Averaging Weights Leads to Wider Optima and Better Generalization:
        https://arxiv.org/abs/1803.05407
    	optimizerswa_lranneal_strategycoslinearc                 X   t        d||      }t        ||j                        D ]
  \  }}||d<    |dvrt        d|       |dk(  r| j                  | _        n|dk(  r| j                  | _        t        |t              r|dk  rt        d|       || _	        t        | -  ||       y )Nr   r   z>anneal_strategy must by one of 'cos' or 'linear', instead got r   r   r   z3anneal_epochs must be equal or greater than 0, got )r   r#   param_groups
ValueError_cosine_annealanneal_func_linear_annealr5   r;   anneal_epochsrN   rO   )	rV   r   r   r   r   
last_epochswa_lrsgrouprW   s	           r)   rO   zSWALR.__init__  s      )V< )*@*@A 	%MFE$E(O	%"33./1  %#22D(#22D--1BEm_U  +J/r+   c                     | S NrB   ts    r)   r   zSWALR._linear_anneal  s    r+   c                 Z    dt        j                  t         j                  | z        z
  dz  S )Nr      )mathr   pir   s    r)   r   zSWALR._cosine_anneal  s#    DHHTWWq[))Q..r+   c                 ,    |dk(  r|S | ||z  z
  d|z
  z  S rA   rB   )lrr   r3   s      r)   _get_initial_lrzSWALR._get_initial_lr  s&    A:MUV^#E	22r+   c                    | j                   st        j                  dt               | j                  dz
  }| j
                  dk(  rt        d|      }t        dt        d|dz
  t        d| j
                        z              }| j                  |      }| j                  j                  D cg c]  }| j                  |d   |d   |       }}t        dt        d|t        d| j
                        z              }| j                  |      }t        | j                  j                  |      D cg c]  \  }}|d   |z  |d|z
  z  z    c}}S c c}w c c}}w )zGet learning rate.zTTo get the last learning rate computed by the scheduler, please use `get_last_lr()`.r   r   r   r   )_get_lr_called_within_stepwarningswarnUserWarning_step_countr   maxminr   r   r   r   r#   )	rV   stepprev_t
prev_alphar   prev_lrsr   r3   r   s	            r)   get_lrzSWALR.get_lr  sO    ..MM. !#"q$<DQAqC43E3E,FFGH%%f-
 44
   teHozJ
 
 3q$Q(:(:!;;<=  # !!<!<hG
r (Oe#bAI&66
 	


s   * E-E)
   r   )rq   rr   rs   rt   r   floatr   rO   staticmethodr   r   r   r   rv   rw   s   @r)   r   r   e  s{    )^ 4900 0
 !106   / / 3 3

r+   r   )g+?r   )'rt   r]   r   r   copyr   typingr   r   r   r   r   r	   r
   r   r   r   torch.nnr   torch.optim.lr_schedulerr   r   torch.utils._foreach_utilsr   r   r   __all__r   r-   r   r   r   r   r   r,   r;   rI   r   r   rB   r+   r)   <module>r      s    E     Q Q Q    ? M   J 5%tF|34
"@	~F ~B  267SM77 U3,-.7 7tn
K n
r+   