
    ijY                     t    d dl Z d dlmZ d dlmZ d dlmZ d dlmZ d dl	m
Z
  ed       G d d	e             Zy)
    N)tree)keras_export)_convert_loss_to_function)Model)serialization_libzkeras.distillation.Distillerc                        e Zd ZdZ	 	 	 d fd	Zd Zd Zd Zd Zd Z	d Z
d	 Zd
 Zd Zd Zd Zd Zd fd	ZddZ	 ddZ fdZ fdZed        Z xZS )	Distillera  Distillation model for transferring knowledge from teacher to student.

    Knowledge distillation transfers knowledge from a large, complex model
    (teacher) to a smaller, simpler model (student). The student learns
    from both ground truth labels and the teacher's predictions, often
    achieving better performance than training on labels alone.

    Arguments:
        teacher: A trained `keras.Model` that serves as the knowledge source.
            The teacher model is frozen during distillation.
        student: A `keras.Model` to be trained through distillation.
        distillation_losses: List of distillation losses to apply. Can be a
            single distillation loss or a list of distillation losses like
            `keras.distillation.LogitsDistillation`,
            `keras.distillation.FeatureDistillation`, or custom distillation
            losses.
        distillation_loss_weights: List of weights for each distillation loss.
            Must have the same length as `distillation_losses`. If `None`,
            equal weights are used.
        student_loss_weight: Weight for the student's supervised loss component.
            Must be between 0 and 1. Defaults to 0.5.
        name: Name for the distiller model. Defaults to `"distiller"`.
        **kwargs: Additional keyword arguments passed to the parent `Model`
            class.

    Attributes:
        student: The student model being trained. Access this to get the trained
            student model for independent use after distillation training.
        teacher: The teacher model providing knowledge. This model is frozen
            during training.

    Examples:

    ```python
    # Basic distillation with KerasHub models
    import keras_hub as hub

    teacher = hub.models.CausalLM.from_preset("gemma_2b_en")
    student = hub.models.CausalLM.from_preset(
        "gemma_1.1_2b_en", load_weights=False
    )

    # Single distillation loss
    distiller = Distiller(
        teacher=teacher,
        student=student,
        distillation_losses=LogitsDistillation(temperature=3.0),
    )

    # Compile the distiller (like any Keras model)
    distiller.compile(
        optimizer='adam',
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy']
    )

    # Train the distiller
    distiller.fit(x_train, y_train, epochs=10)

    # Access the trained student model
    trained_student = distiller.student

    # Multiple distillation losses
    distiller = Distiller(
        teacher=teacher,
        student=student,
        distillation_losses=[
            LogitsDistillation(temperature=3.0),
            FeatureDistillation(
                teacher_layer_name="dense_1",
                student_layer_name="dense_1"
            )
        ],
        distillation_loss_weights=[1.0, 0.5],
    )

    # Compile with custom settings
    distiller.compile(
        optimizer='adam',
        loss='sparse_categorical_crossentropy',
        metrics=['accuracy']
    )
    ```
    c                    t        	|   dd|i| | j                  ||       || _        || _        t        |t        t        f      st        dt        |             |dk  s|dkD  rt        d|       || _
        |t        d      t        |t        t        f      s|g| _        dg| _        n`|| _        |dgt        |      z  | _        nBt        |      t        |      k7  r$t        dt        |       dt        |       d	      || _        | j                  D ]  }| j!                  |||        | j#                          d
| j                  _        t&        j(                  j+                  d      | _        t&        j(                  j+                  d      | _        t&        j(                  j+                  d      | _        y )Nnamez*student_loss_weight must be a number, got               ?z5student_loss_weight must be between 0.0 and 1.0, got z'distillation_losses' cannot be `None`. Provide a distillation loss (e.g., LogitsDistillation or FeatureDistillation) or a list of distillation losses.z%Number of distillation_loss_weights (z,) must match number of distillation_losses ()Fstudent_lossr   distillation_loss
total_loss )super__init___validate_modelsteacherstudent
isinstanceintfloat
ValueErrortypestudent_loss_weightlisttupledistillation_lossesdistillation_loss_weightslen)_validate_distillation_loss_compatibility _create_multi_feature_extractors	trainablekerasmetricsMeanstudent_loss_trackerdistillation_loss_trackertotal_loss_tracker)
selfr   r   r!   r"   r   r   kwargsr   	__class__s
            u/var/www/html/emotional.easysim.app/public_html/venv/lib/python3.12/site-packages/keras/src/distillation/distiller.pyr   zDistiller.__init__`   s    	-d-f- 	gw/  -U|<<+,-/  $(;c(A*+-  $7  &I  -e}=(;'<D$.1UD*':D$(025': 2. 01S9L5MM$ 9:; < 345Q8  2K. "&!9!9 	::"3	
 	--/ "' %*MM$6$6N$6$K!).););$ *< *
& #(--"4"4,"4"G    c                 4   t        |t        j                        st        dt	        |             t        |t        j                        st        dt	        |             | j                  ||       | j                  ||       | j                  ||       y)z8Validate that teacher and student models are compatible.z#Teacher must be a keras.Model, got z#Student must be a keras.Model, got N)r   r'   r   r   r   _validate_input_compatibility_validate_output_compatibility_validate_dtype_compatibilityr-   r   r   s      r0   r   zDistiller._validate_models   s    '5;;/5d7m_E  '5;;/5d7m_E  	**7G<++GW=**7G<r1   c           
          t        |      t        |      k7  rt        d| d| d| d      t        ||      D ]%  \  }}|	|||k7  st        d| d| d| d       y)z&Assert that two shapes are compatible.Teacher and student z, shapes have different dimensions. Teacher: , Student: .Nz# shapes are incompatible. Teacher: z. All dimensions must match.)r#   r   zip)r-   shape1shape2contextdim1dim2s         r0   _assert_shapes_are_compatiblez'Distiller._assert_shapes_are_compatible   s    v;#f+%&wi 0((.x{6(!E 
 ff- 	JD$D$4 *7) 4  &x{6( ;12 	r1   c           	      8    ||k7  rt        d| d| d| d      y)z4Assert that teacher and student dtypes are the same.r8   z dtypes must match. Teacher: r9   r:   N)r   )r-   teacher_dtypestudent_dtyper>   s       r0   _assert_same_dtypezDistiller._assert_same_dtype   s;    M)&wi 0)?+m_AG  *r1   c                      t        |d      rt        |d      syt        |d      }t        |d      }||yt        j                   fd||       y)z?Validate that teacher and student have compatible input shapes.inputsNc                 R    j                  | j                  |j                  d      S NinputrA   shapetisir-   s     r0   <lambda>z9Distiller._validate_input_compatibility.<locals>.<lambda>   s!    4=="((G r1   hasattrgetattrr   map_structure)r-   r   r   teacher_inputsstudent_inputss   `    r0   r3   z'Distiller._validate_input_compatibility   sZ    w)(1K (3 (3!^%; 	
r1   c                      t        |d      rt        |d      syt        |d      }t        |d      }||yt        j                   fd||       y)z@Validate that teacher and student have compatible output shapes.outputsNc                 R    j                  | j                  |j                  d      S NoutputrK   tosor-   s     r0   rP   z:Distiller._validate_output_compatibility.<locals>.<lambda>   s!    4=="((H r1   rQ   )r-   r   r   teacher_outputsstudent_outputss   `    r0   r4   z(Distiller._validate_output_compatibility   sZ    w	*''92M!'95!'95"o&= 	
r1   c                     t        |d      rt        |d      sy|j                  |j                  yt        j                   fd|j                  |j                         t        |d      rt        |d      sy|j                  |j                  yt        j                   fd|j                  |j                         y)z=Validate that teacher and student have compatible data types.rG   Nc                 R    j                  | j                  |j                  d      S rI   rE   dtyperM   s     r0   rP   z9Distiller._validate_dtype_compatibility.<locals>.<lambda>   s    422288RXXwO r1   rX   c                 R    j                  | j                  |j                  d      S rZ   rc   r\   s     r0   rP   z9Distiller._validate_dtype_compatibility.<locals>.<lambda>  s!    422"((H r1   )rR   rG   r   rT   rX   r6   s   `  r0   r5   z'Distiller._validate_dtype_compatibility   s    w)(1K>>!W^^%;ONNNN	
 w	*''92M??"goo&= OOOO	
r1   c                 (    |j                  ||       y)zZValidate that the distillation loss is compatible with teacher
        and student models.N)validate_model_compatibility)r-   r   r   r   s       r0   r$   z3Distiller._validate_distillation_loss_compatibility  s    
 	66wHr1   c                    g }g }| j                   D ]  }t        |d      r5|j                  r)|j                  |vr|j                  |j                         t        |d      sQ|j                  s^|j                  |vsm|j                  |j                          | j                  | j                  |      | _        | j                  | j                  |      | _	        y)z?Create feature extractors for efficient multi-layer extraction.teacher_layer_namestudent_layer_nameN)
r!   rR   ri   appendrj   _create_feature_extractorr   _teacher_feature_extractorr   _student_feature_extractor)r-   teacher_layer_namesstudent_layer_namesr   s       r0   r%   z*Distiller._create_multi_feature_extractors  s      !%!9!9 	)+?@%88 &88./ (..)<< )+?@%88 &88./ (..)<<)	0 +/*H*HLL-+
' +/*H*HLL-+
'r1   c                    |syt        |d      r|j                  t        d|j                   d      t	        |t
        j                        r|j                  d   j                  }n|j                  }d|i}|D ]#  }|j                  |      }|j                  ||<   % t        j                  |j                  ||j                   d	      S )
ar  Create a feature extractor for a model.

        Arguments:
            model: The model to create an extractor for.
            layer_names: List of layer names to extract features from.

        Returns:
            Feature extractor model or `None` if no layer names provided.

        Raises:
            ValueError: If model has no symbolic inputs/outputs.
        NrG   z$Cannot create feature extractor for z-. The model has no symbolic inputs attribute.final_outputr   _multi_feature_extractor)rG   rX   r   )rR   rG   r   r   r   r'   
Sequentiallayersr[   	get_layerr   )r-   modellayer_namesrs   rX   
layer_namelayers          r0   rl   z#Distiller._create_feature_extractor=  s     uh'5<<+?6uzzl C> ? 
 eU--. <<+22L <<L!<0% 	/JOOO4E"',,GJ	/ {{<<JJ<78
 	
r1   c                 j    | j                   | j                  |d      S d| j                  |d      iS )z6Extract all teacher features in a single forward pass.Ftrainingrs   )rm   r   )r-   xs     r0   _extract_all_teacher_featuresz'Distiller._extract_all_teacher_featuresc  s<    **6221u2EE"DLLUL$CDDr1   c                 H    | j                   | j                  |d      S d|iS )z6Extract all student features in a single forward pass.Tr}   rs   )rn   )r-   r   y_preds      r0   _extract_all_student_featuresz'Distiller._extract_all_student_featuresj  s/    **6221t2DD"F++r1   c           	          |r|j                   xs d}n|j                  xs d}||vr(t        d| dt        |j	                                      ||   S )z8Get the specific features needed by a distillation loss.rs   zLayer 'z.' not found in extracted features. Available: )ri   rj   r   r   keys)r-   r   all_features
is_teacherrz   s        r0   _get_distillation_loss_featuresz)Distiller._get_distillation_loss_featuresq  sn     *==OJ*==OJ\)* &"<#4#4#678: 
 J''r1   c                     |t        d      t        j                  t        |      | _        || _        |-t        |t        t        f      st        dt        |             t        | ,  d|d|d| y)a  Compile the distiller with proper integration.

        Arguments:
            optimizer: Optimizer for training the student model.
            loss: Student loss function for the student's supervised learning.
                Can be a string identifier or a loss function instance.
            metrics: Additional metrics to track during training.
            **kwargs: Additional arguments passed to parent compile.
        Nz'loss' cannot be `None`.z%metrics must be a list or tuple, got )	optimizerlossr(   r   )r   r   rT   r   _student_loss_student_loss_for_serializationr   r   r    r   r   compile)r-   r   r   r(   r.   r/   s        r0   r   zDistiller.compile  s     <788!//0I4P/3,z'D%='I7WG  	 	
	
 		
r1   c                 ,     | j                   |fd|i|S )z)Forward pass returns student predictions.r~   )r   )r-   rG   r~   r.   s       r0   callzDistiller.call  s    t||F@X@@@r1   c                    |
 | ||      }d}| j                   dkD  r|t        j                  d | j                  ||      }t        j                  |      }t        |      dkD  r<t        j                  j                  t        j                  j                  |            n|d   }t        |d      r7t        |j                        dkD  rt        j                  j                  |      }d}	| j                   dk  rL| j                  |      }
| j                  ||      }t        | j                   | j"                        D ]  \  }}t        |d	      r6|j$                  *	 | j'                  ||
d
      }| j'                  ||d      }n|
d   }|}|j3                  ||       |j5                  ||      }t        |d      rHt        |j                        dkD  r0t)        d|j6                  j.                   d|j                   d      t        j                  j9                  |	t        j                  j;                  ||            }	 t        j                  j9                  t        j                  j;                  | j                   |      t        j                  j;                  t        j                  j=                  d| j                         |	            }| j>                  jA                  |       | jB                  jA                  |	       | jD                  jA                  |       |S # t(        $ rD}t+        dt-        |      j.                   d|j$                   d|j0                   d|       |d}~ww xY w)aM  Compute combined distillation loss.

        Arguments:
            x: Input data.
            y: Target data.
            y_pred: Model predictions.
            sample_weight: Sample weights (currently unused).
            training: Whether the model is in training mode.

        Returns:
            Combined loss tensor.
        Nr}   r   c                      | ||      S Nr   )loo_preds      r0   rP   z(Distiller.compute_loss.<locals>.<lambda>  s    Qq&\ r1      r   rL   r   ri   T)r   FzFailed to extract features for z targeting teacher layer 'z' and student layer 'z'. Original error: rs   zDistillation loss z' returned a non-scalar loss with shape z6. The compute_loss method must return a scalar tensor.)#r   r   rT   r   flattenr#   r'   opssumstackrR   rL   meanr   r   r;   r!   r"   ri   r   r   RuntimeErrorr   __name__rj   validate_outputscompute_lossr/   addmultiplysubtractr*   update_stater+   r,   )r-   r   yr   sample_weightr~   r   loss_valuesflat_lossesr   teacher_featuresstudent_featuresdistillation_loss_fnweight distillation_loss_teacher_output distillation_loss_student_outputecurrent_distillation_lossr   s                      r0   r   zDistiller.compute_loss  s~     >!h/F##c)am,,1""	K ,,{3K {#a' 		eiiook:; ^  |W-#l6H6H2IA2M$yy~~l;  ##c)#AA!D#AA!VL 14(($*H*H1 H,$f
 02FG,??K! @@ 4 0+/ A  9 !@@ 4 0+0 A  9( 8H&84 8>4 %5544 -A,M,M44-) 5w?5;;<q@$,/99BBC DA4::; <"#  %*IIMM%II&&v/HI%!KHV YY]]IIt77FII		""3(@(@A!

 	!!..|<&&334EF,,Z8A & 
!*=#$89BBC D  4 G GH I  4 G GH I//0c3  !!
!s   (L))	M62?M11M6c                     t         |           | j                  j                          | j                  j                          | j
                  j                          y)zReset all metrics.N)r   reset_metricsr*   reset_stater+   r,   )r-   r/   s    r0   r   zDistiller.reset_metrics)  sD    !!--/&&224++-r1   c           
      N   t         |          }|j                  t        j                  | j
                        t        j                  | j                        | j                  D cg c]  }t        j                  |       c}| j                  | j                  d       |S c c}w )z$Get configuration for serialization.)r   r   r!   r"   r   )
r   
get_configupdater   serialize_keras_objectr   r   r!   r"   r   )r-   configr   r/   s      r0   r   zDistiller.get_config0  s    #%,CCLL -CCLL
 .2-E-E() &<<=NO( .2-K-K'+'?'?	
  (s   %B"c                     |j                         }t        j                  |d         |d<   t        j                  |d         |d<   |d   D cg c]  }t        j                  |       c}|d<    | di |S c c}w )z#Create instance from configuration.r   r   r!   r   )copyr   deserialize_keras_object)clsr   r   s      r0   from_configzDistiller.from_configE  s      .FF9
y .FF9
y
 &,,A%B)
! 667HI)
$%
 }V})
s   A8)Ng      ?	distiller)adamNNr   )NNNNT)r   
__module____qualname____doc__r   r   rA   rE   r3   r4   r5   r$   r%   rl   r   r   r   r   r   r   r   r   classmethodr   __classcell__)r/   s   @r0   r	   r	   	   s    St #'MH^= 
"
"
4I"
H$
LE,("
8A
 IMEN.*  r1   r	   )r'   	keras.srcr   keras.src.api_exportr   (keras.src.distillation.distillation_lossr   keras.src.models.modelr   keras.src.savingr   r	   r   r1   r0   <module>r      s;      - N ( . ,-L	 L	 .L	r1   