
    ij                     l   d dl Z d dlZd dlmZ d dlZd dlZd dlmZ d dlm	Z
 d dlmZ d dlmZ d dlmZ d dlmZ d d	lmZ d dlmZ d d
lmZ d dlmZ d dlmZ d dlmZ d dlmZ d dlm Z   e       rd dl!m"Z" e"jF                  Z#nejF                  Z# G d dejH                        Z%ddZ& G d de      Z'y)    N)partial)backend)	callbacks)
optimizers)tree)config)distribution_lib)is_nnx_enabled)trainer)array_slicing)data_adapter_utils)EpochIterator)traceback_utils)pythonify_logs)nnxc                   ^    e Zd Z fdZ	 	 ddZd Zd Zd Zd ZddZ	ddZ
dd	Zdd
Zej                  	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 	 dd       Zej                  	 	 	 	 	 	 	 	 dd       Zej                  	 dd       Z	 	 	 	 ddZ	 	 	 ddZd Zd Zd Zd Z	 	 	 	 ddZ	 	 	 	 	 ddZ xZS )
JAXTrainerc                 Z    t         |           d | _        d | _        d | _        d| _        y )NT)super__init__train_functiontest_functionpredict_function_jax_state_synced)self	__class__s    r/var/www/html/emotional.easysim.app/public_html/venv/lib/python3.12/site-packages/keras/src/backend/jax/trainer.pyr   zJAXTrainer.__init__    s.    "! $!%    c	           
      .   i }	| j                   r||	d<    | j                  |||fddi|	\  }
}}|r!| j                  j                          || _        | j	                  ||||||
||      \  }}|r| j                  j                          |\  }}}|}|ro| j
                  ct        t        | j
                  j                  |            }t        j                  |      5  | j
                  j                  |      }ddd       |||
||ffS # 1 sw Y   xY w)z?This method is stateless and is intended for use with jax.grad.trainingreturn_lossesT)xyy_predsample_weightr    Nstate_mapping)_call_has_training_argstateless_call_losses_overrideclearstateless_compute_loss	optimizerlistzip	variablesr   StatelessScope
scale_loss)r   trainable_variablesnon_trainable_variablesmetrics_variablesr"   r#   r%   r    optimizer_variableskwargsr$   losseslossr0   unscaled_lossmappings                   r   compute_loss_and_updatesz#JAXTrainer.compute_loss_and_updates'   sS    &&!)F: 3F$2E2E#3
 	3

 3
/' !!'')$*D!55#' 6 	
i !!'') 	J	57H
 23t~~779LMNG''g> 7~~0067#	
 
 	
7 7s   DDc           
         t        j                  t        | j                  |      D cg c]	  \  }}||f c}}      5 }	| j                  j                  |t        d t        j                  |      D              j                  d          | j                  ||||      }
d d d        g }| j                  D ]2  }	j                  |      }||j                  }|j                  |       4 
|fS c c}}w # 1 sw Y   VxY w)Nr&   c              3   &   K   | ]	  }||  y wN ).0is     r   	<genexpr>z7JAXTrainer._update_metrics_variables.<locals>.<genexpr>m   s      #!-A#s   r   )r%   )r   r1   r/   r5   _loss_trackerupdate_statenextr   flattenshapecompute_metricsget_current_valuevalueappend)r   r5   r:   r"   r#   r$   r%   ref_vvscopelogsnew_metrics_variablesnew_vs                r   _update_metrics_variablesz$JAXTrainer._update_metrics_variablesb   s    ## !$D$:$:<M NE1 

 	E
 ++" ##||A# % ,  ''1fmDD	E !#++ 	0E++E2E}!((/		0
 ***'	E 	Es   C1
 A"C77D c           
      D   |\  }}}}t        j                  |      \  }}}	t        j                  | j                  d      }
 |
||||||	d|      \  \  }}}|\  }}}}| j
                  j                  |||      \  }}| j                  ||||||	      \  }}||||f}||fS )NT)has_aux)r    r6   )r   unpack_x_y_sample_weightjaxvalue_and_gradr<   r-   stateless_applyrS   )r   statedatar3   r4   r6   r5   r"   r#   r%   grad_fnr9   auxgradsr:   r$   rP   s                    r   
train_stepzJAXTrainer.train_step{   s     	
#0II$O1m$$))4
 %# 3	
sU  	L 79J NN**(;
	

 #'"@"@}aFM#

  #	
 U{r   c           	          |\  }}}t        j                  |      \  }}}| j                  ||||||d      \  }	}
|
\  }}}}| j                  ||||||      \  }}|||f}||fS )NF)r    )r   rV   r<   rS   )r   rZ   r[   r3   r4   r5   r"   r#   r%   r9   r]   r:   r$   rP   s                 r   	test_stepzJAXTrainer.test_step   s    
 		
#0II$O1m11# 2 
	c  	L 79J #'"@"@}aFM#

  #

 U{r   c                     |\  }}i }| j                   rd|d<   t        j                  |      \  }}} | j                  |||fi |\  }}||fS )NFr    )r(   r   rV   r)   )	r   rZ   r[   r3   r4   r7   r"   _outputss	            r   predict_stepzJAXTrainer.predict_step   sr    7<44&&!&F:$==dC1a+>4+>+>!8!,
?E,
(( ///r   c                       j                   dkD  r9|r/d  j                  s j                  rt               fd}|S  fd}|S fd}|S )N   c                 T    | d   }| dd  D ]  }t        j                  d ||      } |S )Nr   rg   c                 D    t         j                  j                  | |g      S r?   )rW   numpyconcatenate)t1t2s     r   <lambda>z@JAXTrainer._make_function.<locals>.concatenate.<locals>.<lambda>   s    399+@+@"b+J r   )r   map_structure)rd   outputnext_outputs      r   rk   z.JAXTrainer._make_function.<locals>.concatenate   s@    $QZF'.qr{ !%!3!3J"'" "Mr   c                     t        |      } | |      \  }} |g}	 t        j                  dz
        D ]*  }t        |      } | |      \  }} |j                  |       , 	  |      }|| fS # t        $ r Y w xY wNrg   )rF   rangesteps_per_executionrL   StopIteration)	rZ   iteratorr[   rd   rc   _outputsrk   r   step_functions	         r   iterator_stepz0JAXTrainer._make_function.<locals>.iterator_step   s    >D%25$%?NGU&iG!&t'?'?!'C!D 5A#'>D.;E4.HOHe#NN845 *'2G"E>) ) s   AA/ /	A;:A;c                     t        |      } | |      \  }} 	 t        j                  dz
        D ]  }t        |      } | |      \  }}  	 || fS # t        $ r Y || fS w xY wrs   )rF   rt   ru   rv   )rZ   rw   r[   rd   rc   r   ry   s        r   rz   z0JAXTrainer._make_function.<locals>.iterator_step   s    >D%25$%?NGU!&t'?'?!'C!D HA#'>D-:5$-GNGUH
 #E>) ) "E>)s   4A 	A"!A"c                 (     | t        |            S r?   )rF   )rZ   rw   ry   s     r   rz   z0JAXTrainer._make_function.<locals>.iterator_step   s    $UDN;;r   )ru   run_eagerlyjit_compilejit)r   ry   concatenate_outputsrz   rk   s   ``  @r   _make_functionzJAXTrainer._make_function   s^    ##a'"" ''D,<,<"%k"2K*@ !	*  < r   c                 F     j                   |sy  j                  sa j                  rUd }t        j                          j                         }d |f}t               r fd}n j                  }t        |d|      }n j                  } j                  |      }| _         y )Nc                 <    t              j                  | |      S r?   )typer_   rZ   r[   r   s     r   rn   z0JAXTrainer.make_train_function.<locals>.<lambda>  s    d4j.C.C%/ r   r   donate_argnumsout_shardings)
r   r}   r~   r	   distribution_get_state_sharding_specr
   r_   r   r   )r   forcer   state_shardingsstep_fnr_   ry   s   `      r   make_train_functionzJAXTrainer.make_train_function  s    *5D$4$4 M,,.:"&"?"?"A!% 7 // +J J++J7+r   c                 Z     j                   |sy  j                  sk j                  r_d }t        j                          j                         \  }}}}|||f}d |f}t               r fd}n j                  }t        |d|      }	n j                  }	 j                  |	      }
|
 _         y )Nc                 <    t              j                  | |      S r?   )r   ra   r   s     r   rn   z/JAXTrainer.make_test_function.<locals>.<lambda>2  s    d4j.B.B%/ r   r   r   )
r   r}   r~   r	   r   r   r
   ra   r   r   )r   r   r   trainable_shardingsnon_trainable_shardingsrc   metrics_shardingsr   r   ra   ry   s   `          r   make_test_functionzJAXTrainer.make_test_function  s    )%D$4$4 M,,.: 113'+% (+%#
 "& 7 .. +I I++I6*r   c                 6   	  j                   |s j                   S  fd} j                  sM j                  rAd }t        j                          j                         \  }}}}||f}d |f}t        |d|      } j                  |d      		fd}| _         y )Nc                 @    j                  | |      \  }}|| d   |ffS )Nr   )re   )rZ   r[   rd   r4   r   s       r   re   z6JAXTrainer.make_predict_function.<locals>.predict_stepG  s0    /3/@/@/M,G,U1X'>???r   r   r   T)r   c                 $     | |      \  }} || fS r?   r@   )rZ   rw   rd   _step_functions      r   ry   z7JAXTrainer.make_predict_function.<locals>.step_functionc  s    +E8<NGUE>!r   )r   r}   r~   r	   r   r   r   r   )
r   r   re   r   r   r   rc   r   ry   r   s
   `        @r   make_predict_functionz JAXTrainer.make_predict_functionC  s      ,U(((	@ D$4$4 M,,.: 113'+ (+# "& 7 +L ,,d - 
	" !.r   c                    | j                  d       t        j                         }|r||k  rt        j                  d|z         |}d | _        |r#|!t        j                  |||f|      \  \  }}}}|t        j                  |      \  }}}t        ||||||	|
| j                        }| j                  |       |j                          t        |t        j                         s)t        j                   |d|dk7  |||j"                  |       }| j%                          d	| _        i }d	}|j)                          | j*                  xs |}	 t-        ||      D ]  }| j/                          |j1                  |       d| _        |j5                         5  |D ]  \  }}}|j7                  |       | j2                  r| j9                  ddddd
      }d	| _        | j;                  |      \  }}|\  }}} }!||| |!d| _        |j?                  ||       | j&                  s n d d d        | jA                          tC        | jE                              }"|| jG                  ||      rtI        | dd       %t        |xs || j                  |d	      | _        | jK                  |xs |||dd      }#|#jM                         D $%ci c]  \  }$}%d|$ |% }#}$}%|"jO                  |#       |jQ                  ||"       |"}| j&                  s n d}| jA                          t        | jR                  tT        jV                        r*|dkD  r%| jR                  jY                  | jZ                         tI        | dd       | `|r|j]                  |       d | _        | j^                  S # 1 sw Y   xY wc c}%}$w # | jA                          t        | jR                  tT        jV                        r*|dkD  r%| jR                  jY                  | jZ                         tI        | dd       | `|r|j]                  |       d | _        w xY w)NfitzLimiting epochs to %d)validation_split)r"   r#   r%   
batch_sizesteps_per_epochshuffleclass_weightru   rw   Tr   )add_historyadd_progbarverboseepochsstepsmodelFr3   r4   r6   r5   purge_model_variablesr3   r4   r6   r5   _eval_epoch_iterator)r"   r#   r%   r   ru   r   r   )r"   r#   r%   r   r   r   return_dict_use_cached_eval_datasetval_)rP   )0_assert_compile_calledr   
max_epochswarningswarnr   r   train_validation_splitr   rV   JAXEpochIteratorru   _symbolic_buildreset
isinstancecallbacks_moduleCallbackListnum_batchesr   stop_trainingon_train_begin_initial_epochrt   reset_metricson_epoch_beginr   catch_stop_iterationon_train_batch_begin_get_jax_stater   
_jax_stateon_train_batch_endjax_state_syncdict_get_metrics_result_or_logs_should_evalgetattrevaluateitemsupdateon_epoch_endr-   optimizers_module	Optimizerfinalize_variable_valuestrainable_weightson_train_endhistory)&r   r"   r#   r   r   r   r   r   validation_datar   r   r%   initial_epochr   validation_stepsvalidation_batch_sizevalidation_freqr   val_xval_yval_sample_weightepoch_iteratortraining_logstraining_finishedepoch
begin_stepend_steprw   rZ   rP   r3   r4   r6   r5   
epoch_logsval_logsnamevals&                                         r   r   zJAXTrainer.fiti  s   ( 	##E*&&(
*v-MM1J>?F$(! 7 44A}%8H%A}
 &
 #;;OL	! *'!+% $ 8 8	
 	n5 )%5%B%BC(55 #qL$00I 	  ""!  "++<}k	#}f5 Y""$((/)-&#88: '":H &"6
Hh!66zB  11$($7$7488<48266: %8 %E 6;D2&*&9&9%&Je "/3/- 4G7N3F1B	+ "44XtD-- "M&"'"Z ##% "$"B"B4"HI
 #.43D3D?4 t%;TBJ4D##*;'<'J
040H0H,<$)51  $}}&7#8#FJ."+$(15  - 	 H =ENN<L /8tS$tfs* H   %%h/&&uj9 *%%sYt !% !4>>+<+F+FGQJ778N8NO t3T:F- &&M&:"DO||M'" '"X  !4>>+<+F+FGQJ778N8NO t3T:F- &&M&:"DOs@   AN< B
N)N)"B(N< 
N64N< N< )N3	.N< <B
Qc	           	         | j                  d       |	j                  dd      }
|	rt        d|	       |
r| j                  }nt	        |||||d| j
                        }| j                  |       |j                          t        |t        j                        s(t        j                  ||dk7  |d|j                  | 	      }| j                          d| _        |j                          i }| j                          d
| _        |j#                         5  |D ]  \  }}}|j%                  |       | j                   r| j'                  d
d
d
d
      }d| _        | j)                  |      \  }}|\  }}}|||d| _        |j-                  ||       | j                  s n d d d        | j/                          t1        | j3                  |            }|j5                  |       d | _        |r|S | j7                  |      S # 1 sw Y   `xY w)Nr   r   FzArguments not recognized: )r"   r#   r%   r   r   r   ru   r   r   rg   r   r   r   r   r   Tr3   r4   r5   r   r3   r4   r5   )r   pop
ValueErrorr   r   ru   r   r   r   r   r   r   r   stop_evaluatingon_test_beginr   r   r   on_test_batch_beginr   r   r   on_test_batch_endr   r   r   on_test_end_flatten_metrics_in_order)r   r"   r#   r   r   r%   r   r   r   r7   use_cached_eval_datasetr   rP   r   r   rw   rZ   r3   r4   r5   s                       r   r   zJAXTrainer.evaluate%  s0    	##J/"(**-G"O9&BCC"!66N .+% %$($<$<N 	n5 )%5%B%BC(55#qL$00I 	!$!!%002 #	2@ ".
Hh--j9)) //,004*..2	 0 E .3D*"00Ae
 	'+% ,?/F):# ++Hd;''E"#	L 	d>>tDEd#K--d33]#	 #	s   BG=G==Hc                 p   t        |||d| j                        }t        d | j                         D              sv|D ]a  \  }}}t	        j
                  t        |            \  }}}t               r	 | |       n%t        j                         5   | |       d d d         n |j                          t        |t        j                        s(t        j                  ||dk7  |d|j                  |       }| j                          d| _        |j#                          d }	d| _        d }
d }|j'                         5  |D ]  \  }}}|j)                  |       | j$                  r| j+                  ddd	      }d| _        | j-                  |      \  }}|\  }}||d
| _         |	||
      }
|j1                  |d|i       | j                   s n d d d        | j3                          |j5                          d | _        t7        j8                  t:        j<                  |
      S # 1 sw Y   xY w# 1 sw Y   bxY w)NF)r"   r   r   r   ru   c              3   4   K   | ]  }|j                     y wr?   builtrA   layers     r   rC   z%JAXTrainer.predict.<locals>.<genexpr>       C55;;C   r   rg   r   c                 n    |t        j                  d |       }|S t        j                  | d ||        |S )Nc                     | gS r?   r@   )batch_outputs    r   rn   z?JAXTrainer.predict.<locals>.append_to_outputs.<locals>.<lambda>  s    , r   c                 $    | j                  |      S r?   )rL   )rp   r   s     r   rn   z?JAXTrainer.predict.<locals>.append_to_outputs.<locals>.<lambda>  s    |1L r   )r   ro   map_structure_up_to)batch_outputsrd   s     r   append_to_outputsz-JAXTrainer.predict.<locals>.append_to_outputs  sG    ,,7! N ((!L!	 Nr   T)r3   r4   r   r3   r4   rd   )r   ru   all_flatten_layersr   rV   rF   r
   r   r1   r   r   r   r   r   r   stop_predictingon_predict_beginr   r   on_predict_batch_beginr   r   r   on_predict_batch_endr   on_predict_endr   r   nprk   )r   r"   r   r   r   r   r   rc   rw   r  rd   r4   r   r   rZ   r   r3   s                    r   predictzJAXTrainer.predict  sS   
 *!! $ 8 8
 CD,@,@,BCC"0 
1h,EEN1a "#G //1  Q 
   ")%5%B%BC(55#qL$00I 	""$$""$	 "&"&002 	2@ .
Hh00<)) //,004.2 0 E
 .3D*'+'<'<UH'M$u '+ ,? 0G	# ,M7C ..y-8 '';	@ 	  "''r~~wOOQ   J	 	s   	H8BH,H,H)	,H5c                    | j                  d       |)t        d d|       t        j                  |      fd}| j	                  t         |                    | j                          | j                          | j                  ddddd      }d| _	        | j                  | |             \  }}|\  }	}
}}|	|
||d	| _        | j                          t        |      }|r|S | j                  |      S )
Ntrain_on_batchzkArguments `sample_weight` and `class_weight` cannot be specified at the same time. Received: sample_weight=z, class_weight=c               3   .   K   t         f       y wr?   _distribute_datar%   r"   r#   s   r   r[   z'JAXTrainer.train_on_batch.<locals>.data       "Aq-#899   
data_batchTFr   r   )r   r   r   class_weight_to_sample_weightsr   rF   r   r   r   r   r   r   r   r   r   )r   r"   r#   r%   r   r   r[   rZ   rP   r3   r4   r6   r5   s    ```         r   r  zJAXTrainer.train_on_batch  s;    	##$45#( //<o >$$0>3  /MM<M	: 	TV5  " ## $$( $""' $ 
 "'))%8e 	
# $7'>#6!2	
 	 d#K--d33r   c                    | j                  d       fd}| j                  t         |                    | j                          | j	                          | j                  dddd      }d| _        | j                  | |             \  }}|\  }}	}
||	|
d| _        | j                          t        |      }|r|S | j                  |      S )Ntest_on_batchc               3   .   K   t         f       y wr?   r  r  s   r   r[   z&JAXTrainer.test_on_batch.<locals>.data3  r  r  r  TFr   r   )r   r   rF   r   r   r   r   r   r   r   r   r   )r   r"   r#   r%   r   r[   rZ   rP   r3   r4   r5   s    ```       r   r  zJAXTrainer.test_on_batch*  s     	##O4	: 	TV5! ## $$(""'	 $ 
 "'((7e KPG46G#6'>!2

 	 d#K--d33r   c                    t        d | j                         D              s%t        j                         5   |        d d d        | j	                          | j                  dddd      }d| _        fd}| j                  | |             \  }}|\  }}||d| _        | j                          t        j                  d |      }|S # 1 sw Y   xY w)Nc              3   4   K   | ]  }|j                     y wr?   r   r   s     r   rC   z.JAXTrainer.predict_on_batch.<locals>.<genexpr>U  r   r   TFr   c               3      K    f y wr?   r@   r"   s   r   r[   z)JAXTrainer.predict_on_batch.<locals>.datac  s     $Js   
r  c                 ,    t        j                  |       S r?   )r
  arrayr  s    r   rn   z-JAXTrainer.predict_on_batch.<locals>.<lambda>m  s    RXXa[ r   )r  r  r   r1   r   r   r   r   r   r   r   ro   )r   r"   rZ   r[   r   r3   r4   s    `     r   predict_on_batchzJAXTrainer.predict_on_batchT  s    CD,@,@,BCC'') Q""$## $$(#"'	 $ 
 "'	  $44UDFCu7<44#6'>
 	**+@-P/ s   	C

Cc                    t        | dd       r| j                  ry | j                  j                  dd       }| j                  j                  dd       }| j                  j                  dd       }| j                  j                  dd       }|r/t	        | j
                  |      D ]  \  }}|j                  |        |r/t	        | j                  |      D ]  \  }}|j                  |        |r9t	        | j                  j                  |      D ]  \  }}|j                  |        |r/t	        | j                  |      D ]  \  }}|j                  |        d| _        y )Nr   r3   r4   r6   r5   T)r   r   r   getr/   r3   assignr4   r-   r0   r5   )r   r3   r4   r6   r5   rM   rN   s          r   r   zJAXTrainer.jax_state_syncp  sL   t\40D4J4J"oo112GN"&//"5"5%t#
 #oo112GN OO//0CTJ 8 8:MN  qQ ",,.E  q Q   8 8:MN  qQ  6 68IJ  qQ !%r   c                    | j                   D cg c]  }|j                  j                   }}| j                  D cg c]  }|j                  j                   }}t	        | d      rD| j
                  8| j
                  j                  D cg c]  }|j                  j                   }}ng }| j                  D cg c]  }|j                  j                   }}| j                  ||||       ||||fS c c}w c c}w c c}w c c}w )Nr-   )	r3   rK   shardingr4   hasattrr-   r0   r5   _check_sharding_consistency)r   rN   r   r   optimizer_shardingsr   s         r   r   z#JAXTrainer._get_state_sharding_spec  s   &*&>&>
!"AGG
 
 '+&B&B#
!"AGG#
 #
 4%$..*D*...*B*B#%&  # # #%7;7M7MN!QWW--NN((#		
  #	
 	
+
#
#
 Os   C4C9C>;Dc           
         t        j                         yt        j                  t	        | j
                  |      t	        | j                  |      t	        t        | d      r"| j                  r| j                  j                  ng |      t	        | j                  |            }d}d}|D ]A  \  }}	t        |	t        j                  j                        r||j                  }nd}|s>|sA n |r|syt!        j"                  d|dd       y)	a  Warn if there is a mix of local and distributed variable shardings.

        When some variables have SingleDeviceSharding (created outside the
        distribution scope) and others have mesh-aware shardings (created
        inside), passing them together as `out_shardings` to `jax.jit`
        raises ``ValueError: Received incompatible devices for jitted
        computation``. This helper detects the mismatch early and emits
        an actionable warning.
        Nr-   FTzDetected a mix of local (SingleDeviceSharding) and distributed (mesh-aware) variables. This will cause a 'ValueError: Received incompatible devices for jitted computation' when JAX tries to compile the training step.

First local variable found: a  

This typically happens when the model is built or weights are loaded before the distribution is set. To fix this, call set_distribution() before creating any Keras objects:

    import keras
    keras.distribution.set_distribution(distribution)
    model = create_model()
    model.compile(...)
    model.fit(...)

Alternatively, use the distribution scope context manager:

    with distribution.scope():
        model = create_model()
        model.compile(...)
        model.fit(...)
   )
stacklevel)r	   r   	itertoolschainr/   r3   r4   r&  r-   r0   r5   r   rW   r%  SingleDeviceShardingpathr   r   )
r   r   r   r(  r   var_shard_pairsfirst_local_var_pathhas_meshrN   ss
             r   r'  z&JAXTrainer._check_sharding_consistency  s     ((*2#//((*=>,,.EF t[1dnn NN,,# &&(9:
  $# 		DAq!S\\>>?'/+,66( $		 %+
 ,@*B C''* -	
r   c                     |r| j                   D ]	  }d|_         |r| j                  D ]	  }d|_         |r"| j                  j                  D ]	  }d|_         |r| j
                  D ]	  }d|_         yy)a  Remove all the model variable for memory saving.

        During JAX training, since the training function is stateless, we have
        to pass in and get the model weights over and over, during which the
        copy of the weights that attached to the Variable are still and
        occupying extra memory. We remove those variable to save memory (for
        better memory utilization) at the beginning of the epoch, and reattach
        the value back to variables at the end of the epoch, via
        `jax_state_sync()`.
        N)r3   _valuer4   r-   r0   r5   )r   r3   r4   r6   r5   rN   s         r   _purge_model_variablesz!JAXTrainer._purge_model_variables  s    " --   "11   ^^--   ++    r   c                 .   g }|r3|j                  | j                  D cg c]  }|j                   c}       |r3|j                  | j                  D cg c]  }|j                   c}       |r=|j                  | j                  j
                  D cg c]  }|j                   c}       |r3|j                  | j                  D cg c]  }|j                   c}       |r| j                  ||||       t        |      S c c}w c c}w c c}w c c}w )Nr   )	rL   r3   rK   r4   r-   r0   r5   r6  tuple)r   r3   r4   r6   r5   r   rZ   rN   s           r   r   zJAXTrainer._get_jax_state  s     LL4+C+CDa!''DE"LL4+G+GHa!''HILL4>>+C+CDa!''DELL4+A+ABa!''BC ''$7(?$7"3	 (  U| EHDBs   DDDD)FN)F)NNNrg   autoNg        NTNNr   NNNrg   )NNNr9  NNNF)Nr9  NN)NNNF)NNF)FFFF)FFFFF)__name__
__module____qualname__r   r<   rS   r_   ra   re   r   r   r   r   r   filter_tracebackr   r   r  r  r  r   r   r   r'  r6  r   __classcell__)r   s   @r   r   r      sO   &  9
v+2*X>
01f,4"+H$.L %% 
"#y &yv %% 
d4 &d4L %%HL^P &^PF ;4@ (4T8&4
:H
X " %! @ " %!#r   r   c                 *   t        j                         Y|fd}t        j                  ||       }t	        t
        j                  j                        }t        j                  || |      S t        j                  t        j                  |       S )Nc                 @    | y j                  | j                        S r?   )get_data_layoutrH   dr   s    r   
get_layoutz$_distribute_data.<locals>.get_layout0  s     9#33AGG<<r   )batch_dim_name)
r	   r   r   ro   r   jax_distribution_libdistribute_data_inputrE  rW   
device_put)r[   layoutsrD  jax_dist_data_inputr   s       @r   r  r  *  s    #002L?=
 ((T:G% 66'66
 !!"5tWEEcnnd33r   c                   $    e Zd Zd Zd Zd Zd Zy)r   c                 ,    t        | j                        S r?   )rF   _epoch_iterator)r   s    r   __next__zJAXEpochIterator.__next__@  s    D(())r   c                     t        j                         }|| j                  |      S | j                  j	                         }t        j                         dk(  r|S | j                  |      S )Ncpu)r	   r   _get_distributed_iteratordata_adapterget_jax_iteratorrW   default_backend_one_batch_ahead_iterator)r   r   rw   s      r   _get_iteratorzJAXEpochIterator._get_iteratorC  sb    '446#11,??((99;H""$-11(;;r   c              #      K   d}| j                   j                         D ]-  }|fd}t        j                  ||      }t	        ||       / yw)zALazily compute layouts to reduce host to device transfer latency.Nc                 T    | y j                  | j                        j                  S r?   )rA  rH   backend_layoutrB  s    r   rD  z>JAXEpochIterator._get_distributed_iterator.<locals>.get_layoutT  s&    y#'77@OOOr   )rR  rS  r   ro   r  )r   r   rI  r[   rD  s    `   r   rQ  z*JAXEpochIterator._get_distributed_iteratorN  sV     %%668 		2DP
 ,,Z>"411		2s   AAc              #   \   K   d}|D ]  }t        |      }||}|}|}|  || yyw)a7  Initiate transfers to the device one batch ahead.

        This utility takes an iterator and returns a new iterator which
        initiates the transfer to device one step ahead. This can improve the
        performance of training loops significantly by overlapping compute and
        data transfer.
        Nr  )r   numpy_iterator
next_batchbatchcurrent_batchs        r   rU  z*JAXEpochIterator._one_batch_ahead_iterator\  sT      
# 	$E$U+E!"
 *"
##	$ ! "s   *,N)r:  r;  r<  rN  rV  rQ  rU  r@   r   r   r   r   ?  s    *	<2r   r   r?   )(r,  r   	functoolsr   rW   rj   r
  	keras.srcr   r   r   r   r   r   keras.src.backendr   r	   rF  keras.src.backend.configr
   keras.src.distributionkeras.src.trainersr   base_trainer keras.src.trainers.data_adaptersr   r   !keras.src.trainers.epoch_iteratorr   keras.src.utilsr   keras.src.utils.python_utilsr   flaxr   r   Trainerr   r  r   r@   r   r   <module>rl     s       
   3 5  $ F 3 3 6 : ? ; + 7
''C
''CH%% HV 4*0} 0r   