
    ij                         d dl mZ d dlmZ d dlmZ  edg       G d dej                               Zej                  ,ej                  j                  dej                        e_        yy)	    )ops)keras_export)	optimizerz"keras.optimizers.ScheduleFreeAdamWc                   \     e Zd ZdZ	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 d fd	Z fdZd Z fdZ xZS )ScheduleFreeAdamWaq  Optimizer that implements the Schedule-Free AdamW algorithm.

    Schedule-Free learning is a method that avoids the need for a learning rate
    schedule by maintaining a combination of interpolation and averaging.
    This approach eliminates the requirement to specify stopping time in advance
    and typically matches or outperforms cosine and linear decay schedules.

    The optimizer maintains two sets of variables internally:
    - `momentum`: The sequence where gradient updates are applied
    - `x`: The averaged sequence used for evaluation

    During training, the model parameters are set to an interpolation between
    `momentum` and `x`.

    Args:
        learning_rate: A float, a
            `keras.optimizers.schedules.LearningRateSchedule` instance, or
            a callable that takes no arguments and returns the actual value to
            use. The learning rate. Defaults to `0.0025`.
        beta_1: A float value or a constant float tensor, or a callable
            that takes no arguments and returns the actual value to use. The
            exponential decay rate for the 1st moment estimates and controls
            the interpolation between `momentum` and `x`. Defaults to `0.9`.
        beta_2: A float value or a constant float tensor, or a callable
            that takes no arguments and returns the actual value to use. The
            exponential decay rate for the 2nd moment estimates.
            Defaults to `0.999`.
        epsilon: A small constant for numerical stability.
            Defaults to `1e-8`.
        warmup_steps: Number of warmup steps for learning rate warmup.
            During warmup, the learning rate linearly increases from 0 to the
            specified learning rate. Defaults to `0`.
        {{base_optimizer_keyword_args}}

    References:

    - [Defazio et al., 2024](https://arxiv.org/abs/2405.15682)
    - [Schedule-Free repository](
        https://github.com/facebookresearch/schedule_free)

    Example:

    >>> optimizer = keras.optimizers.ScheduleFreeAdamW(learning_rate=0.0025)
    >>> model.compile(optimizer=optimizer, loss="mse")
    >>> model.fit(x_train, y_train)

    c                 t    t        |   d||||||	|
||||d| || _        || _        || _        || _        y )N)learning_ratenameweight_decayclipnorm	clipvalueglobal_clipnormuse_emaema_momentumema_overwrite_frequencyloss_scale_factorgradient_accumulation_steps )super__init__beta_1beta_2epsilonwarmup_steps)selfr	   r   r   r   r   r   r   r   r   r   r   r   r   r   r
   kwargs	__class__s                    }/var/www/html/emotional.easysim.app/public_html/venv/lib/python3.12/site-packages/keras/src/optimizers/schedule_free_adamw.pyr   zScheduleFreeAdamW.__init__8   sa    & 	 	
'%+%$;/(C	
 	
 (    c                    | j                   ryt        | 	  |       | j                  |ddg      \  | _        | _        t        | j                  |      D ]-  \  }}|	| j                  |t        j                  |             / y)aa  Initialize optimizer variables.

        ScheduleFreeAdamW optimizer has the following variables:
        - `momentum`: Auxiliary variable where gradient updates are applied
        - `velocity`: Exponential moving average of squared gradients (Adam)

        Args:
            var_list: list of model variables to build optimizer variables on.
        Nmomentumvelocity)
builtr   buildadd_optimizer_variables
_momentums_velocitieszipassignr   copy)r   var_listr!   varr   s       r   r$   zScheduleFreeAdamW.build^   s{     ::h,0,H,Hz:.-
))
 !(; 	5MHc#Hchhsm4	5r   c           
      X   t        j                  ||j                        }t        j                  ||j                        }t        j                  | j                  dz   |j                        }t        j                  | j                  |j                        }t        j                  | j
                  |j                        }t        j                  | j                  |j                        }| j                  dkD  rHt        j                  | j                  |j                        }	t        j                  ||	z  d      }
||
z  }| j                  |      }| j                  |   }| j                  |   }|j                  }dt        j                  ||      z
  }| j                  |t        j                  t        j                   t        j"                  |      |      d|z
               t        j$                  t        j&                  ||z        |      }t        j(                  t        j                  ||      |      }| j+                  ||       d|z  }|}t        j(                  t        j                   |t        j                  d|z
  |            |      }t        j$                  t        j                  d|z
  |      t        j                  ||            }t        j$                  t        j                  d|z
  |      t        j                  ||            }| j-                  ||       y)z=Update step given gradient and the associated model variable.   r   g      ?N)r   castdtype
iterationsr   r   r   r   minimum_get_variable_indexr&   r'   valuepower
assign_addmultiplysubtractsquareaddsqrtdivide
assign_subr)   )r   gradientvariabler	   lr
local_stepr   r   r   r   warmup_factor	var_indexr!   r"   momentum_oldbias_correction_2denomgrad_scaledweighty_oldx_oldx_newy_news                          r   update_stepzScheduleFreeAdamW.update_stept   s^   XXmX^^488Hhnn5XXdoo18>>B
$++x~~6$++x~~6((4<<8 q 88D$5$5x~~FLKK
\(A3GMm#B,,X6	??9-##I.  ~~ 		&* == 	LLSZZ18<a&j	
 ,=!=>H jjb(!;UC+. z! 

LLQZ FG
 LLVU+S\\&(-K
 LLVX.VU0K
 	He$r   c                     t         |          }|j                  | j                  | j                  | j
                  | j                  d       |S )N)r   r   r   r   )r   
get_configupdater   r   r   r   )r   configr   s     r   rO   zScheduleFreeAdamW.get_config   sF    #%++++<< $ 1 1		
 r   )g{Gzd?g?g+?g:0yE>r   NNNNFgGz?NNNN)	__name__
__module____qualname____doc__r   r$   rM   rO   __classcell__)r   s   @r   r   r      sT    .d  $$(!$)L5,A%F
 
r   r   Nz{{base_optimizer_keyword_args}})	keras.srcr   keras.src.api_exportr   keras.src.optimizersr   	Optimizerr   rU   replacebase_optimizer_keyword_argsr   r   r   <module>r]      st     - * 345z	++ z 6zz ( 1 9 9 A A)9+P+P! )r   