iddlZddlmZmZmZmZmZddlZddl Z ddl Z ddl m ZddlmZmZmZmZddlmZmZddlmZddlmZdd lmZdd lmZdd lm Z dd l!m"Z"m#Z#m$Z$m%Z%dd l&m'Z'ddl(m)Z)ddl*m+Z+e#r ddl,m-cm.Z/dZ0ndZ0e$jbe2Z3e"rddl4Z4e$jbe2Z3dZ5dZ6dZ7dZ8Gdde)eZ9y)N)CallableDictListOptionalUnion) functional) CLIPTextModel CLIPTokenizer"Qwen2_5_VLForConditionalGenerationQwen2VLProcessor)MultiPipelineCallbacksPipelineCallback)VaeImageProcessor)KandinskyLoraLoaderMixin) AutoencoderKL)Kandinsky5Transformer3DModel)FlowMatchEulerDiscreteScheduler)is_ftfy_availableis_torch_xla_availableloggingreplace_example_docstring) randn_tensor)DiffusionPipeline)KandinskyImagePipelineOutputTFa< Examples: ```python >>> import torch >>> from diffusers import Kandinsky5T2IPipeline >>> # Available models: >>> # kandinskylab/Kandinsky-5.0-T2I-Lite-sft-Diffusers >>> # kandinskylab/Kandinsky-5.0-T2I-Lite-pretrain-Diffusers >>> model_id = "kandinskylab/Kandinsky-5.0-T2I-Lite-sft-Diffusers" >>> pipe = Kandinsky5T2IPipeline.from_pretrained(model_id, torch_dtype=torch.bfloat16) >>> pipe = pipe.to("cuda") >>> prompt = "A cat and a dog baking a cake together in a kitchen." >>> output = pipe( ... prompt=prompt, ... negative_prompt="", ... height=1024, ... width=1024, ... num_inference_steps=50, ... guidance_scale=3.5, ... ).frames[0] ``` ctrtj|}tjtj|}|j S)z Copied from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/wan/pipeline_wan.py Clean text using ftfy if available and unescape HTML entities. )rftfyfix_texthtmlunescapestriptexts /home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/diffusers/pipelines/kandinsky5/pipeline_kandinsky_t2i.py basic_cleanr'Ws; }}T" ==t, -D ::<cTtjdd|}|j}|S)z Copied from https://github.com/huggingface/diffusers/blob/main/src/diffusers/pipelines/wan/pipeline_wan.py Normalize whitespace in text by replacing multiple spaces with single space. z\s+ )resubr#r$s r&whitespace_cleanr-cs& 66&#t $D ::text_encoder_2->transformer->vae)latentsprompt_embeds_qwenprompt_embeds_clipnegative_prompt_embeds_qwennegative_prompt_embeds_clip transformervae text_encoder tokenizertext_encoder_2 tokenizer_2 schedulerc t||j|||||||d|_d|_d|_t |j |_gd|_y)N)r7r8r9r:r;r<r=z<|im_start|>system You are a promt engineer. Describe the image by detailing the color, shape, size, texture, quantity, text, spatial relationships of the objects and background:<|im_end|> <|im_start|>user {}<|im_end|>))vae_scale_factor))rB))rDrC))rFrE))rHrG) super__init__register_modulesprompt_template prompt_template_encode_start_idxvae_scale_factor_spatialrimage_processor resolutions) selfr7r8r9r:r;r<r= __class__s r&rJzKandinsky5T2IPipeline.__init__sn  #%)#   02-()%0$B_B_`wr(Npromptdevicemax_sequence_lengthdtypec |xs |j}|xs|jj}|Dcgc]}|jj |}}|j |z}|j |ddddd}|jd|kDrt|D]t\} } || |j d} |j j| |dz d} t| d kDsH| dt|  || <tjd |d | v|j |dd|d dd j|} |j| dd d dddd|j df}| ddd|j df}tj|j!dd }t#j$|dd jtj&}|j||fScc}w)aI Encode prompt using Qwen2.5-VL text encoder. This method processes the input prompt through the Qwen2.5-VL model to generate text embeddings suitable for image generation. Args: prompt List[str]: Input list of prompts device (torch.device): Device to run encoding on max_sequence_length (int): Maximum sequence length for tokenization dtype (torch.dtype): Data type for embeddings Returns: Tuple[torch.Tensor, torch.Tensor]: Text embeddings and cumulative sequence lengths Nptlongest)r%imagesvideosreturn_tensorspadding input_idsrrzXThe following part of your input was truncated because `max_sequence_length` is set to z tokens: T)r%r[r\ max_length truncationr]r^)r_ return_dictoutput_hidden_states hidden_statesattention_maskr)dim)rr)value)rW)_execution_devicer9rWrLformatrMr:shape enumeratedecodelenloggerwarningtotorchcumsumsumFpadint32)rQrTrUrVrWp full_textsmax_allowed_lenuntruncated_idsir%tokens removed_textinputsembedsrg cu_seqlenss r&_encode_prompt_qwenz)Kandinsky5T2IPipeline._encode_prompt_qwens@,14110**00>DEd**11!4E E??BUU.. )     $ 6$Z0 4(+D,Q,QTVW#~~44VJqMNN/0 ,I  &   "V* ""[)!%#     !$"G"G"II K  01!T5Z5Z5\2\]\\."4"4Q"7Q? UU:vQ7:::M yy++SFs"G-c|xs |j}|xs|jj}|j|dddddj |}|jdi|d}|j |S)a Encode prompt using CLIP text encoder. This method processes the input prompt through the CLIP model to generate pooled embeddings that capture semantic information. Args: prompt (Union[str, List[str]]): Input prompt or list of prompts device (torch.device): Device to run encoding on dtype (torch.dtype): Data type for embeddings Returns: torch.Tensor: Pooled text embeddings from CLIP MTrbrY)rbrcadd_special_tokensr^r] pooler_output)rjr;rWr<rr)rQrTrUrWr pooled_embeds r&_encode_prompt_clipz)Kandinsky5T2IPipeline._encode_prompt_clips(14112,,22!! #  "  "V* +t**4V4_E u%%r(rnum_images_per_promptc|xs |j}|xs|jj}t|ts|g}t |}|Dcgc] }t |}}|j||||\}} |j|||} |jd|d}|j||zd|jd}| jd|d} | j||zd} | j} | j|} tjtj dg|tj"| j%dg} || | fScc}w)a Encodes a single prompt (positive or negative) into text encoder hidden states. This method combines embeddings from both Qwen2.5-VL and CLIP text encoders to create comprehensive text representations for image generation. Args: prompt (`str` or `List[str]`): Prompt to be encoded. num_images_per_prompt (`int`, *optional*, defaults to 1): Number of images to generate per prompt. max_sequence_length (`int`, *optional*, defaults to 512): Maximum sequence length for text encoding. Must be less than 1024 device (`torch.device`, *optional*): Torch device. dtype (`torch.dtype`, *optional*): Torch dtype. Returns: Tuple[torch.Tensor, torch.Tensor, torch.Tensor]: - Qwen text embeddings of shape (batch_size * num_images_per_prompt, sequence_length, embedding_dim) - CLIP pooled embeddings of shape (batch_size * num_images_per_prompt, clip_embedding_dim) - Cumulative sequence lengths (`cu_seqlens`) for Qwen embeddings of shape (batch_size * num_images_per_prompt + 1,) )rTrUrVrW)rTrUrWrr`rrUrW)rjr9rW isinstancelistror/rrrepeatviewrldiffrepeat_interleaverscattensorrxrt)rQrTrrVrUrW batch_sizeryr3prompt_cu_seqlensr4original_lengthsrepeated_lengthsrepeated_cu_seqlenss r& encode_promptz#Kandinsky5T2IPipeline.encode_prompt!sB14110**00&$'XF[ +12a,q/22150H0H 3 1I1 --"556 066 $a 044 . .4F4L4LR4P  066 $a 044ZBW5WY[\-113+== ! $ii \\1#fEKK @BRBYBYZ[B\ ] "#57JJJa3s Ec | | dkDr td||fjvrTdjjD cgc]\} }d| d|dc}} }tj d|d|d |d | Lt fd | Ds8td j d | Dcgc]}|j vs|c}||| ||| td||| ||| td| | td|7t|ts't|tstdt||9t|ts(t|tstdt|yyycc}} wcc}w)al Validate input parameters for the pipeline. Args: prompt: Input prompt negative_prompt: Negative prompt for guidance height: Image height width: Image width prompt_embeds_qwen: Pre-computed Qwen prompt embeddings prompt_embeds_clip: Pre-computed CLIP prompt embeddings negative_prompt_embeds_qwen: Pre-computed Qwen negative prompt embeddings negative_prompt_embeds_clip: Pre-computed CLIP negative prompt embeddings prompt_cu_seqlens: Pre-computed cumulative sequence lengths for Qwen positive prompt negative_prompt_cu_seqlens: Pre-computed cumulative sequence lengths for Qwen negative prompt callback_on_step_end_tensor_inputs: Callback tensor inputs Raises: ValueError: If inputs are invalid NrBz*max_sequence_length must be less than 1024,()z'`height` and `width` have to be one of z , but are z and z(. Dimensions will be resized accordinglyc3:K|]}|jvyw)N)_callback_tensor_inputs).0krQs r& z5Kandinsky5T2IPipeline.check_inputs..s#F 23A-- -F sz2`callback_on_step_end_tensor_inputs` has to be in z , but found zuIf any of `prompt_embeds_qwen`, `prompt_embeds_clip`, or `prompt_cu_seqlens` is provided, all three must be provided.zIf any of `negative_prompt_embeds_qwen`, `negative_prompt_embeds_clip`, or `negative_prompt_cu_seqlens` is provided, all three must be provided.zProvide either `prompt` or `prompt_embeds_qwen` (and corresponding `prompt_embeds_clip` and `prompt_cu_seqlens`). Cannot leave all undefined.z2`prompt` has to be of type `str` or `list` but is z;`negative_prompt` has to be of type `str` or `list` but is ) ValueErrorrPjoinrprqallrrstrrtype)rQrTnegative_promptheightwidthr3r4r5r6rnegative_prompt_cu_seqlens"callback_on_step_end_tensor_inputsrVwhresolutions_strrs` r& check_inputsz"Kandinsky5T2IPipeline.check_inputs|s:F  */BT/IIJ J 6?$"2"2 2!hhAQAQ'RA!A3as! 'RSO NN9/9J*U[T\\abgahiQR  . 9#F 7YF C DTEaEaDbbn|^pHvwbckokGkGbGpqpHoIJ   )-?-KO`Ol!)-?-GK\Kd 2 ( 3*6)5,3.6-5 2 >08`   z&#'>zRXZ^G_QRVW]R^Q_`a a  &?C0OUY9ZZ[_`o[pZqrs s:[0 '[(SpHsE8 %E>9E>rBrnum_channels_latentsrr generatorr2returnc &||j||S|dt||jzt||jz|f} t|tr)t ||k7rt dt |d|dt| |||}|S)a Prepare initial latent variables for text-to-image generation. This method creates random noise latents Args: batch_size (int): Number of images to generate num_channels_latents (int): Number of channels in latent space height (int): Height of generated image width (int): Width of generated image dtype (torch.dtype): Data type for latents device (torch.device): Device to create latents on generator (torch.Generator): Random number generator latents (torch.Tensor): Pre-existing latents to use Returns: torch.Tensor: Prepared latent tensor rrz/You have passed a list of generators of length z+, but requested an effective batch size of z@. Make sure the batch size matches the length of the generators.)rrUrW)rrintrNrrrorr) rQrrrrrWrUrr2rls r&prepare_latentsz%Kandinsky5T2IPipeline.prepare_latentss:  ::V5:9 9  K488 8 J$77 7   i &3y>Z+GA#i.AQR&<'gi  u &PUVr(c|jS)z%Get the current guidance scale value.)_guidance_scalerQs r&guidance_scalez$Kandinsky5T2IPipeline.guidance_scales###r(c|jS)z&Get the number of denoising timesteps.)_num_timestepsrs r& num_timestepsz#Kandinsky5T2IPipeline.num_timesteps s"""r(c|jS)z)Check if generation has been interrupted.) _interruptrs r& interruptzKandinsky5T2IPipeline.interruptsr(2g @pilTrnum_inference_stepsrr3r4r5r6rr output_typerdcallback_on_step_endrc t|ttfr |j}|j ||||| | | | |||| ||f|j vrU|j t j|j Dcgc]}t|d|dz ||z z c}\}}||_ d|_ |j}|jj}|t|trd}|g}n-|t|tr t!|}n| j"d}| |j%|||||\} } }|j&dkDr|d}t|tr||gt!|zn|g}n;t!|t!|k7r$t)d t!|d t!|d | |j%|||||\} } }|j*j-|| |j*j.}|jj0j2}|j5||z||||||| } t7j8d| t7j8||j:zdz| t7j8||j:zdz| g}t7j8|j=j?jA| }|At7j8|j=j?jA| nd}gd}d}t!|||j*jBzz } t!||_"|jG|5}!tI|D]!\}}"|jJr|"jMdjO||z}#|j| jQ|| jQ|| jQ||#jQ|||||d jR}$|j&dkDrm| k|j| jQ|| jQ|| jQ||#jQ|||||d jR}%|%||$|%z zz}$|j*jU|$ddddf|"| dd} |~i}&|D]}'tW|'|&|'<||||"|&}(|(jYd| } |(jYd| } |(jYd| } |(jYd| } |(jYd| } |t!|dz k(s'|dz| kDr/|dz|j*jBzdk(r|!j[t\st_j`$ ddd| ddddddddd|f} |dk7r| jQ|jbj} | je||d||j:z||j:z|} | jgdddddd} | je||z|||j:z||j:z} | |jbj0jhz } |jbjk| jR})|jljo|)|})n| })|jq|s|)fSts|)Scc}w#1swYNxYw)aX The call function to the pipeline for text-to-image generation. Args: prompt (`str` or `List[str]`, *optional*): The prompt or prompts to guide the image generation. If not defined, pass `prompt_embeds` instead. negative_prompt (`str` or `List[str]`, *optional*): The prompt or prompts to avoid during image generation. If not defined, pass `negative_prompt_embeds` instead. Ignored when not using guidance (`guidance_scale` < `1`). height (`int`, defaults to `1024`): The height in pixels of the generated image. width (`int`, defaults to `1024`): The width in pixels of the generated image. num_inference_steps (`int`, defaults to `50`): The number of denoising steps. guidance_scale (`float`, defaults to `5.0`): Guidance scale as defined in classifier-free guidance. num_images_per_prompt (`int`, *optional*, defaults to 1): The number of images to generate per prompt. generator (`torch.Generator` or `List[torch.Generator]`, *optional*): A torch generator to make generation deterministic. latents (`torch.Tensor`, *optional*): Pre-generated noisy latents. prompt_embeds_qwen (`torch.Tensor`, *optional*): Pre-generated Qwen text embeddings. prompt_embeds_clip (`torch.Tensor`, *optional*): Pre-generated CLIP text embeddings. negative_prompt_embeds_qwen (`torch.Tensor`, *optional*): Pre-generated Qwen negative text embeddings. negative_prompt_embeds_clip (`torch.Tensor`, *optional*): Pre-generated CLIP negative text embeddings. prompt_cu_seqlens (`torch.Tensor`, *optional*): Pre-generated cumulative sequence lengths for Qwen positive prompt. negative_prompt_cu_seqlens (`torch.Tensor`, *optional*): Pre-generated cumulative sequence lengths for Qwen negative prompt. output_type (`str`, *optional*, defaults to `"pil"`): The output format of the generated image. return_dict (`bool`, *optional*, defaults to `True`): Whether or not to return a [`KandinskyImagePipelineOutput`]. callback_on_step_end (`Callable`, `PipelineCallback`, `MultiPipelineCallbacks`, *optional*): A function that is called at the end of each denoising step. callback_on_step_end_tensor_inputs (`List`, *optional*): The list of tensor inputs for the `callback_on_step_end` function. max_sequence_length (`int`, defaults to `512`): The maximum sequence length for text encoding. Examples: Returns: [`~KandinskyImagePipelineOutput`] or `tuple`: If `return_dict` is `True`, [`KandinskyImagePipelineOutput`] is returned, otherwise a `tuple` is returned where the first element is a list with the generated images. ) rTrrrr3r4r5r6rrrrVrrFN)rTrrVrUrW?z9`negative_prompt` must have same length as `prompt`. Got z vs .)rU)rrrrrWrUrr2r)rrr)totalT) rfencoder_hidden_statespooled_projectionstimestepvisual_rope_pos text_rope_pos scale_factor sparse_paramsrd)rdr2r3r4r5r6latentr )r)image):rrr tensor_inputsrrPnpargminabsrrrjr7rWrrrorlrrrr= set_timesteps timestepsconfig in_visual_dimrrsarangerNrmaxitemorderr progress_barrmr unsqueezerrrsamplesteplocalspopupdate XLA_AVAILABLExm mark_stepr8reshapepermutescaling_factorrnrO postprocessmaybe_free_model_hooksr)*rQrTrrrrrrrr2r3r4r5r6rrrrdrrrVr}rUrWrrrrrnegative_text_rope_posrrnum_warmup_stepsrtr pred_velocityuncond_pred_velocitycallback_kwargsrcallback_outputsrs* r&__call__zKandinsky5T2IPipeline.__call__sZ` *-=?U,V W1E1S1S . +11(C(C/'A/Q 3   6?$"2"2 2 ,, $JZJZ[Q3!qt ?@[\ME6 .''  &&  *VS"9JXF  Jvt$<VJ+11!4J  %HLHZHZ&;$7 I[I E  24E    $&"$/3/EKEW?"3c&k"A^m]n_%V4 OPSTcPdOeeijmntjuivvwx+2&&..C,?%# 'e+-HJd $$%8$HNN,,  $//66DD&&!$99!5'   LL6 * LL4#@#@@AEf U LL$"?"??1DV T   %6%;%;%=%A%A%C%H%H%JSYZ *5 LL388:>>@EEGPV W '  y>,?$..BVBV,VV!)n   %8  99 #\!),8 #1>>;;q>00>S1ST!% 0 0")**U"3*<*?*?*F'9'<']]qMq;r$rM..--mAqD.A1g[`-abcd'3&(O?9-3Xa[*9';D!Q'X$.229gFG)9)=)=>RTf)g&)9)=)=>RTf)g&2B2F2F57R3/3C2F2F57R3/I**A9I/IqSTuX\XfXfXlXlNlpqNq '') LLNq8 #9 #x!Q1&;';&;;< ( "jj0Goo%$777666$ GooaAq!Q7Goo22$$777666 G > >>GHHOOG,33E((44U 4TEE ##%8O+%88u\v9 #9 #s8![$ +H[)[))[3)NrSN)NN)rrSNN)NNNNNNNN)rBrBNNNN),__name__ __module__ __qualname____doc__model_cpu_offload_seqrrrr r r r rrJrrrrsrUrrWrrrrr GeneratorTensorrpropertyrrrno_gradrEXAMPLE_DOC_STRINGfloatboolrrrrr __classcell__)rRs@r&r1r1xsx4Mx1xx9 x $ x & x#x3x@*.#&'+ B,S B,&B,! B,  $ B,N*.'+ "&c49n%"&&"& $ "&N&'#&)-'+ YKc49n%YK #YK! YK & YK  $ YKB $($(#'+/ Wtx%''+)-MQ*.00"0 0  0  $ 0&0E%//43H"HIJ0%,,'0 0d$$##U]]_12)-;?#% #/0MQ*.5959>B>B48=A%*  9B #&/Z9c49n%Z9"%T#Y"78Z9 Z9  Z9 ! Z9Z9 (}Z9E%//43H"HIJZ9%,,'Z9%U\\2Z9%U\\2Z9&.ell%;Z9&.ell%;Z9$ELL1Z9 %-U\\$:!Z9"c]#Z9$%Z9&' (Cd+T124DF\\ ] 'Z9,-1I-Z9.!/Z93Z9r(r1):r!typingrrrrrnumpyrregexr+rstorch.nnrrv transformersr r r r callbacksrrrOrloadersrmodelsrmodels.transformersr schedulersrutilsrrrrutils.torch_utilsrpipeline_utilsrpipeline_outputrtorch_xla.core.xla_modelcore xla_modelrr get_loggerrrprrr'r-r/r1rr(r&rs 88 $kkA0/#?9 ..9))MM   H %   H %: z 9-/Gz 9r(