iddlZddlmZmZmZmZmZddlZddl Z ddl Z ddl m ZddlmZmZmZmZddlmZmZddlmZmZddlmZdd lmZdd lmZdd l m!Z!dd l"m#Z#m$Z$m%Z%m&Z&dd l'm(Z(ddl)m*Z*ddl+m,Z,e$r ddl-m.cm/Z0dZ1ndZ1e%jde3Z4e#rddl5Z5e%jde3Z4dZ6dZ7dZ8dZ9Gdde*eZ:y)N)CallableDictListOptionalUnion) functional) CLIPTextModel CLIPTokenizer"Qwen2_5_VLForConditionalGenerationQwen2VLProcessor)MultiPipelineCallbacksPipelineCallback)PipelineImageInputVaeImageProcessor)KandinskyLoraLoaderMixin) AutoencoderKL)Kandinsky5Transformer3DModel)FlowMatchEulerDiscreteScheduler)is_ftfy_availableis_torch_xla_availableloggingreplace_example_docstring) randn_tensor)DiffusionPipeline)KandinskyImagePipelineOutputTFa< Examples: ```python >>> import torch >>> from diffusers import Kandinsky5I2IPipeline >>> # Available models: >>> # kandinskylab/Kandinsky-5.0-I2I-Lite-sft-Diffusers >>> # kandinskylab/Kandinsky-5.0-I2I-Lite-pretrain-Diffusers >>> model_id = "kandinskylab/Kandinsky-5.0-I2I-Lite-sft-Diffusers" >>> pipe = Kandinsky5I2IPipeline.from_pretrained(model_id, torch_dtype=torch.bfloat16) >>> pipe = pipe.to("cuda") >>> prompt = "A cat and a dog baking a cake together in a kitchen." >>> output = pipe( ... prompt=prompt, ... negative_prompt="", ... height=1024, ... width=1024, ... num_inference_steps=50, ... guidance_scale=3.5, ... ).frames[0] ``` ctrtj|}tjtj|}|j S)z Copied from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/wan/pipeline_wan.py Clean text using ftfy if available and unescape HTML entities. )rftfyfix_texthtmlunescapestriptexts /home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/diffusers/pipelines/kandinsky5/pipeline_kandinsky_i2i.py basic_cleanr(Ws; }}T" ==t, -D ::<cTtjdd|}|j}|S)z Copied from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/wan/pipeline_wan.py Normalize whitespace in text by replacing multiple spaces with single space. z\s+ )resubr$r%s r'whitespace_cleanr.cs& 66&#t $D ::text_encoder_2->transformer->vae)latentsprompt_embeds_qwenprompt_embeds_clipnegative_prompt_embeds_qwennegative_prompt_embeds_clip transformervae text_encoder tokenizertext_encoder_2 tokenizer_2 schedulerc t||j|||||||d|_d|_d|_t |j |_gd|_y)N)r8r9r:r;r<r=r>a=<|im_start|>system You are a promt engineer. Based on the provided source image (first image) and target image (second image), create an interesting text prompt that can be used together with the source image to create the target image:<|im_end|><|im_start|>user{}<|vision_start|><|image_pad|><|vision_end|><|im_end|>7)vae_scale_factor))rC))rErD))rGrF))rIrH) super__init__register_modulesprompt_template prompt_template_encode_start_idxvae_scale_factor_spatialrimage_processor resolutions) selfr8r9r:r;r<r=r> __class__s r'rKzKandinsky5I2IPipeline.__init__sn  #%)#   `02-()%0$B_B_`wr)NrCpromptimagedevicemax_sequence_lengthdtypec  |xs |j}|xs|jj}t|ts|g}|Dcgc]5}|j |j ddz|j ddzf7}}|Dcgc]}|jj|}}|j|z} |j||dddd} | jd | kDrt|D]\}} | |} | |jjk(j} | | |jjk7|jd } |jj| || z d z d}t!|dkDs| dt!| ||<t"j%d |d ||j||d| dddj'|}|jdi|ddddd dd|jdf}|ddd|jdf}t)j*|jdd}t-j.|ddj't(j0}|j'||fScc}wcc}w)a Encode prompt using Qwen2.5-VL text encoder. This method processes the input prompt through the Qwen2.5-VL model to generate text embeddings suitable for image generation. Args: prompt List[str]: Input list of prompts image (PipelineImageInput): Input list of images to condition the generation on device (torch.device): Device to run encoding on max_sequence_length (int): Maximum sequence length for tokenization dtype (torch.dtype): Data type for embeddings Returns: Tuple[torch.Tensor, torch.Tensor]: Text embeddings and cumulative sequence lengths rrrNptlongest)r&imagesvideosreturn_tensorspadding input_idsr zXThe following part of your input was truncated because `max_sequence_length` is set to z tokens: T)r&r\r] max_length truncationr^r_) return_dictoutput_hidden_states hidden_statesattention_mask)dim)rr)valuerX)_execution_devicer:rX isinstancelistresizesizerMformatrNr;shape enumerateimage_token_idsumdecodelenloggerwarningtotorchcumsumFpadint32)rRrTrUrVrWrXip full_textsmax_allowed_lenuntruncated_idsr&tokensnum_image_tokens removed_textinputsembedsrh cu_seqlenss r'_encode_prompt_qwenz)Kandinsky5I2IPipeline._encode_prompt_qwens014110**00%&GEEJK166!9>166!9>:;KK>DEd**11!4E E??BUU.. )     $ 6$Z0 4(+$*dnn.K.K$K#P#P#R $..*G*G GHInInqst#~~44VJqMNN/0 ,I &   "V* #""  !%     !$"G"G"II K  01!T5Z5Z5\2\]\\."4"4Q"7Q? UU:vQ7:::M yy++[LEs :J"J c|xs |j}|xs|jj}|j|dddddj |}|jdi|d}|j |S)a Encode prompt using CLIP text encoder. This method processes the input prompt through the CLIP model to generate pooled embeddings that capture semantic information. Args: prompt (Union[str, List[str]]): Input prompt or list of prompts device (torch.device): Device to run encoding on dtype (torch.dtype): Data type for embeddings Returns: torch.Tensor: Pooled text embeddings from CLIP MTrcrZ)rcrdadd_special_tokensr_r^ pooler_outputrl)rmr<rXr=r{)rRrTrVrXr pooled_embeds r'_encode_prompt_clipz)Kandinsky5I2IPipeline._encode_prompt_clips(14112,,22!! #  "  "V* +t**4V4_E u%%r)rnum_images_per_promptc|xs |j}|xs|jj}t|ts|g}t |}|Dcgc] }t |}}|j|||||\} } |j|||} | jd|d} | j||zd| jd} | jd|d} | j||zd} | j} | j|} tjtj dg|tj"| j%dg}| | |fScc}w)a Encodes a single prompt (positive or negative) into text encoder hidden states. This method combines embeddings from both Qwen2.5-VL and CLIP text encoders to create comprehensive text representations for image generation. Args: prompt (`str` or `List[str]`): Prompt to be encoded. num_images_per_prompt (`int`, *optional*, defaults to 1): Number of images to generate per prompt. max_sequence_length (`int`, *optional*, defaults to 1024): Maximum sequence length for text encoding. Must be less than 1024 device (`torch.device`, *optional*): Torch device. dtype (`torch.dtype`, *optional*): Torch dtype. Returns: Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - Qwen text embeddings of shape (batch_size * num_images_per_prompt, sequence_length, embedding_dim) - CLIP pooled embeddings of shape (batch_size * num_images_per_prompt, clip_embedding_dim) - Cumulative sequence lengths (`cu_seqlens`) for Qwen embeddings of shape (batch_size * num_images_per_prompt + 1,) )rTrUrVrWrX)rTrVrXrrarrVrX)rmr:rXrnrorxr0rrrepeatviewrsdiffrepeat_interleaver|cattensorrr})rRrTrUrrWrVrX batch_sizerr4prompt_cu_seqlensr5original_lengthsrepeated_lengthsrepeated_cu_seqlenss r' encode_promptz#Kandinsky5I2IPipeline.encode_prompt'sD14110**00&$'XF[ +12a,q/22150H0H 3 1I1 --"556 066 $a 044 . .4F4L4LR4P  066 $a 044ZBW5WY[\-113+== ! $ii \\1#fEKK @BRBYBYZ[B\ ] "#57JJJc3s Ec  | | dkDr td| td||fjvrTdjjDcgc]\}}d|d|dc}}}tj d|d |d |d | Lt fd | Ds8td j d| Dcgc]}|j vs|c}||| ||| td|| | || | td| | td|7t|ts't|tstdt||9t|ts(t|tstdt|yyycc}}wcc}w)a Validate input parameters for the pipeline. Args: prompt: Input prompt negative_prompt: Negative prompt for guidance image: Input image for conditioning height: Image height width: Image width prompt_embeds_qwen: Pre-computed Qwen prompt embeddings prompt_embeds_clip: Pre-computed CLIP prompt embeddings negative_prompt_embeds_qwen: Pre-computed Qwen negative prompt embeddings negative_prompt_embeds_clip: Pre-computed CLIP negative prompt embeddings prompt_cu_seqlens: Pre-computed cumulative sequence lengths for Qwen positive prompt negative_prompt_cu_seqlens: Pre-computed cumulative sequence lengths for Qwen negative prompt callback_on_step_end_tensor_inputs: Callback tensor inputs Raises: ValueError: If inputs are invalid NrCz*max_sequence_length must be less than 1024z6`image` must be provided for image-to-image generation,()z'`height` and `width` have to be one of z , but are z and z(. Dimensions will be resized accordinglyc3:K|]}|jvyw)N)_callback_tensor_inputs).0krRs r' z5Kandinsky5I2IPipeline.check_inputs..s#F 23A-- -F sz2`callback_on_step_end_tensor_inputs` has to be in z , but found zuIf any of `prompt_embeds_qwen`, `prompt_embeds_clip`, or `prompt_cu_seqlens` is provided, all three must be provided.zIf any of `negative_prompt_embeds_qwen`, `negative_prompt_embeds_clip`, or `negative_prompt_cu_seqlens` is provided, all three must be provided.zProvide either `prompt` or `prompt_embeds_qwen` (and corresponding `prompt_embeds_clip` and `prompt_cu_seqlens`). Cannot leave all undefined.z2`prompt` has to be of type `str` or `list` but is z;`negative_prompt` has to be of type `str` or `list` but is ) ValueErrorrQjoinryrzallrrnstrrotype)rRrTnegative_promptrUheightwidthr4r5r6r7rnegative_prompt_cu_seqlens"callback_on_step_end_tensor_inputsrWwhresolutions_strrs` r' check_inputsz"Kandinsky5I2IPipeline.check_inputssKJ  */BT/IIJ J =UV V 6?$"2"2 2!hhAQAQ'RA!A3as! 'RSO NN9/9J*U[T\\abgahiQR  . 9#F 7YF C DTEaEaDbbn|^pHvwbckokGkGbGpqpHoIJ   )-?-KO`Ol!)-?-GK\Kd 2 ( 3*6)5,3.6-5 2 >08`   z&#'>zRXZ^G_QRVW]R^Q_`a a  &?C0OUY9ZZ[_`o[pZqrs s:[0 '[(SpHs F 2F F rnum_channels_latentsrr generatorr3returnc z| | j||S|dt||jzt||jz|f} t|tr)t ||k7rt dt |d|dt| |||} |jj|||j|| } tj5|jj| jj| } | j!d } t#|jj$d r#| |jj$j&z} | j)d d ddd} tj*| | tj,| dddfgd} ddd| S#1swY| SxYw)a2 Prepare initial latent variables for image-to-image generation. This method creates random noise latents with encoded image, Args: image (PipelineImageInput): Input image to condition the generation on batch_size (int): Number of images to generate num_channels_latents (int): Number of channels in latent space height (int): Height of generated image width (int): Width of generated image dtype (torch.dtype): Data type for latents device (torch.device): Device to create latents on generator (torch.Generator): Random number generator latents (torch.Tensor): Pre-existing latents to use Returns: torch.Tensor: Prepared latent tensor with encoded image Nrrz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.)rrVrX)rrrk)rrscaling_factorrr .ra)r{intrOrnrorxrrrP preprocessr|no_gradr9encode latent_distsample unsqueezehasattrconfigrpermuter ones_like) rRrUrrrrrXrVrr3rs image_tensor image_latentss r'prepare_latentsz%Kandinsky5I2IPipeline.prepare_latentss>  ::V5:9 9  K488 8 J$77 7   i &3y>Z+GA#i.AQR&<'gi  u &PUV++66uVSX6Y\\]ckp\q ]]_ a HHOOL9EELLW`LaM)33A6Mtxx(89 -0N0N N *11!Q1a@Mii-QTVXWXVXQXIY9Z []_`G a as CF00F:c|jS)z%Get the current guidance scale value.)_guidance_scalerRs r'guidance_scalez$Kandinsky5I2IPipeline.guidance_scale's###r)c|jS)z&Get the number of denoising timesteps.)_num_timestepsrs r' num_timestepsz#Kandinsky5I2IPipeline.num_timesteps,s"""r)c|jS)z)Check if generation has been interrupted.) _interruptrs r' interruptzKandinsky5I2IPipeline.interrupt1sr)2g @pilTrnum_inference_stepsrr4r5r6r7rr output_typerecallback_on_step_endrct|ttfr |j}|0|.t|tr|dj n |j \}}|j |||||| | | ||||| ||f|jvrU|jtj|jDcgc]}t|d|dz ||z z c}\}}||_ d|_ |j}|jj}|t|t rd}|g}n-|t|tr t#|}n| j$d}| |j'||||||\} } }|j(dkDr|d}t|t r||gt#|zn|g}n;t#|t#|k7r$t+d t#|d t#|d | |j'||||||\} }}|j,j/|| |j,j0}|jj2j4}|j7|||z|||||| | } t9j:d| t9j:||j<zdz| t9j:||j<zdz| g}t9j:|j?jAjC| }|At9j:|j?jAjC| nd}gd}d} t#|||j,jDzz }!t#||_#|jI|5}"tK|D]E\}}#|jLr|#jOdjQ||z}$|j| jS|| jS|| jS||$jS||||| d jT}%|j(dkDrm| k|j| jS|| jS||jS||$jS||||| d jT}&|&||%|&z zz}%|j,jW|%ddddf|#| ddddddddd|fdd| ddddddddd|f<|~i}'|D]}(tY|(|'|(<||||#|'})|)j[d| } |)j[d| } |)j[d| } |)j[d| } |)j[d|}|t#|dz k(s'|dz|!kDr/|dz|j,jDzdk(r|"j]t^s2tajbH ddd| ddddddddd|f} |dk7r| jS|jdj} | jg||d||j<z||j<z|} | jidddddd} | jg||z|||j<z||j<z} | |jdj2jjz } |jdjm| jT}|jnjq||}n| }|js|s|fStu|Scc}w#1swYNxYw)a The call function to the pipeline for image-to-image generation. Args: image (`PipelineImageInput`): The input image to condition the generation on. Must be an image, a list of images or a `torch.Tensor`. prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, pass `prompt_embeds` instead. negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts to avoid during image generation. If not defined, pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale` < `1`). height (`int`): The height in pixels of the generated image. width (`int`): The width in pixels of the generated image. num_inference_steps (`int`, defaults to `50`): The number of denoising steps. guidance_scale (`float`, defaults to `5.0`): Guidance scale as defined in classifier-free guidance. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): A torch generator to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents. prompt_embeds_qwen (`torch.Tensor`, *optional*): Pre-generated Qwen text embeddings. prompt_embeds_clip (`torch.Tensor`, *optional*): Pre-generated CLIP text embeddings. negative_prompt_embeds_qwen (`torch.Tensor`, *optional*): Pre-generated Qwen negative text embeddings. negative_prompt_embeds_clip (`torch.Tensor`, *optional*): Pre-generated CLIP negative text embeddings. prompt_cu_seqlens (`torch.Tensor`, *optional*): Pre-generated cumulative sequence lengths for Qwen positive prompt. negative_prompt_cu_seqlens (`torch.Tensor`, *optional*): Pre-generated cumulative sequence lengths for Qwen negative prompt. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated image. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`KandinskyImagePipelineOutput`]. callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): A function that is called at the end of each denoising step. callback_on_step_end_tensor_inputs (`List`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. max_sequence_length (`int`, defaults to `1024`): The maximum sequence length for text and image qwen encoding. Must be less than 1024 Examples: Returns: [`~KandinskyImagePipelineOutput`] or `tuple`: If `return_dict` is `True`, [`KandinskyImagePipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated images. Nr) rTrrUrrr4r5r6r7rrrrWrF)rTrUrrWrVrX?z9`negative_prompt` must have same length as `prompt`. Got z vs .)rV) rUrrrrrXrVrr3r)rrr)totalT) rgencoder_hidden_statespooled_projectionstimestepvisual_rope_pos text_rope_pos scale_factor sparse_paramsre)rer3r4r5r6r7latentr r)r)rU);rnrr tensor_inputsrorqrrQnpargminabsrrrmr8rXrrxrsrrrr> set_timesteps timestepsr in_visual_dimrr|arangerOrmaxitemorderr progress_barrtrrrr{rsteplocalspopupdate XLA_AVAILABLExm mark_stepr9reshaperrrwrP postprocessmaybe_free_model_hooksr)*rRrUrTrrrrrrrr3r4r5r6r7rrrrerrrWrrVrXrrrrrnegative_text_rope_posrrnum_warmup_stepsrtr pred_velocityuncond_pred_velocitycallback_kwargsrcallback_outputss* r'__call__zKandinsky5I2IPipeline.__call__6sf *-=?U,V W1E1S1S . >em-7t-DE!HMM%**ME6 +11(C(C/'A/Q 3   6?$"2"2 2 ,, $JZJZ[Q3!qt ?@[\ME6 .''  &&  *VS"9JXF  Jvt$<VJ+11!4J  %HLHZHZ&;$7 I[I E  24E    $&"$/3/EKEW?"3c&k"A^m]n_%V4 OPSTcPdOeeijmntjuivvwx+2&&.#.C,?%# 'e+-HJd $$%8$HNN,,  $//66DD&&!$99!5'   LL6 * LL4#@#@@AEf U LL$"?"??1DV T   %6%;%;%=%A%A%C%H%H%JSYZ *5 LL388:>>@EEGPV W '  y>,?$..BVBV,VV!)n   %8  9; #\!),: #1>>;;q>00>S1ST!% 0 0")**U"3*<*?*?*F'9'<']]qMq;r$rM=A^^=P=P!!Q$'GAq!Q@UAU@U4U,Vdi>Q>>1a$9%9$99:(3&(O?9-3Xa[*9';D!Q'X$.229gFG)9)=)=>RTf)g&)9)=)=>RTf)g&2B2F2F57R3/3C2F2F57R3/I**A9I/IqSTuX\XfXfXlXlNlpqNq '') LLNu: #; #|!Q1&;';&;;< ( "jj0Goo%$777666$ GooaAq!Q7Goo22$$777666 G > >>GHHOOG,33E((44U 4TEE ##%8O+%88A\~; #; #s+!\> !H<]]] )NNrCN)NN)rrCNN)NNNNNNNN)rCrCNNNN)-__name__ __module__ __qualname____doc__model_cpu_offload_seqrrrr r r r rrKrrrrr|rVrrXrrrTensorrr GeneratorrpropertyrrrrrEXAMPLE_DOC_STRINGfloatboolrrrrr __classcell__)rSs@r'r2r2xs4Mx1xx9 x $ x & x#x3x>/3)-#''+ I,S I,*+I,& I, ! I,  $ I,\*.'+ "&c49n%"&&"& $ "&P&'#')-'+[Kc49n%[K||[K # [K ! [K & [K $[KH $($(#'+/ \tD%''+)-MQ*.C!CC" C  C  C $C&CE%//43H"HIJC%,,'C CJ$$##U]]_12)-;? $##% #/0MQ*.5959>B>B48=A%*  9B #'1g9!g9c49n%g9"%T#Y"78 g9  g9 } g9!g9g9 (}g9E%//43H"HIJg9%,,'g9%U\\2g9%U\\2g9&.ell%;g9&.ell%;g9 $ELL1!g9"%-U\\$:#g9$c]%g9&'g9(' (Cd+T124DF\\ ] )g9.-1I/g90!1g93g9r)r2);r"typingrrrrrnumpyrregexr,r|torch.nnrr~ transformersr r r r callbacksrrrPrrloadersrmodelsrmodels.transformersr schedulersrutilsrrrrutils.torch_utilsrpipeline_utilsrpipeline_outputrtorch_xla.core.xla_modelcore xla_modelrr get_loggerr ryr rr(r.r0r2rlr)r'r's 88 $kkAD/#?9 ..9))MM   H %   H %: g 9-/Gg 9r)