
    *igF                        d dl Z d dlmZ d dlmZ d dlZd dlmZ d dlZ	d dl
Z
dZ	 d dlmZ d dlmZ 	 d dlmZ d dlmc mZ e
j0                  j                   ZdZdZd	Zd
ZeeeeiZdZdZ g dZ!eddgiZ"d Z#ee"fdZ$d Z%efdZ&d Z'ed        Z(d(dZ)e
j0                  jU                  ed      d        Z+e
j0                  jU                  ed      e
j0                  jY                  dddg      d               Z-e
j0                  jU                  ed      e
j0                  jY                  dg d      d               Z.e
j0                  jU                  ed      e
j0                  jY                  dg d       d!               Z/e
j0                  jU                  ed      e
j0                  jY                  dg d"      d#               Z0e
j0                  jU                  e d$      d%        Z1e
j0                  jU                  ed      d&        Z2e
j0                  jU                  ed      d'        Z3y# e$ r dZdZY w xY w# e$ r dZY 
w xY w))    N)contextmanager)	timedeltaFInMemoryKmsClientTs   0123456789112345
footer_keys   1234567890123450col_keys   2345678901234501col2_keyyearn_legsanimalr   r   c                  @    t        j                  g dg dg dd      S )N)i      r     r   )   r      r      d   )FlamingoParrotDogHorsezBrittle stars	Centipeder
   )patable     o/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/pyarrow/tests/test_dataset_encryption.pycreate_sample_tabler    :   s#    888*	
 r   c           	      L    t        j                  | d|dt        d      d      S )NF
AES_GCM_V1g      @)minutes   )r   plaintext_footercolumn_keysencryption_algorithmcache_lifetimedata_key_length_bits)peEncryptionConfigurationr   )r   r&   s     r   create_encryption_configr,   K   s,    %%) -  r   c                  .    t        j                  d      S )Ni,  )r(   )r*   DecryptionConfigurationr   r   r   create_decryption_configr/   W   s    %%S99r   c           
          t        j                  | j                         D ci c]  \  }}||j                  d       c}}      S c c}}w )NzUTF-8)custom_kms_conf)r*   KmsConnectionConfigitemsdecode)keyskey_namekeys      r   create_kms_connection_configr8   [   sJ    !! "&
# cjj))
 
s   A
c                     t        |       S Nr   )kms_connection_configurations    r   kms_factoryr<   d   s    9::r   c              #   z   K   | rd  y t        j                  ||      5  d  d d d        y # 1 sw Y   y xY ww)Nmatch)pytestraises)success
error_typer?   s      r   cond_raisesrD   h   s1     ]]:U3 		 	 	s   ;/	;8;c           	          |rHt        di t        t        t        it        di t        t        |gi|j                  d      d   }nt        t        d }	 d fd	} j                  }j                         D cg c]   \  }}|D ]  }|j                  d      d    " }}}}|D cg c]  }||vr||j                  | d      s|  }	}t        |      dkD  sJ t        |	      dkD  sJ t        t        i}
j                         D ci c]  \  }}|t        k7  r|| }}} ||
|	d        ||
|d        ||
|d        |||	dd        |||dd        |||dd       t              dkD  rj                         D ]  \  }}|D ]v  }|j                  d      d   }j                         D ci c]  \  }}|t        |fv r|| }}}||k7  }|	|gz   } |||	d        ||||        |||d        |||d       x   ||	d        ||d        ||d       y c c}}}w c c}w c c}}w c c}}w )	N.r   Tc                     }t        t              }t               }t        |      }t        |       }t	        j
                  t              }	t        j                  |	||      }
t        j                  |	||      }t        j                  j                         }|j                  |
      }t        j                         }|j!                  d       t        j"                  d|||       t        j                  j                         }t%        j&                  t(        d      5  t        j                  d||       d d d        t        j*                  |      }t        j                  j                  |	      }t-        |t.        d
      5  t        j                  d||      }t-        |t.        d
      5  j1                  |      j3                  |j5                  |            sJ 	 d d d        d d d        |	j7                  ||      }t        j*                  |      }t        j                  j                  |	      }t-        |t.        d
      5  t        j                  d||      }t-        |t.        d
      5  j1                  |      j3                  |j5                  |            sJ 	 d d d        d d d        y # 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   xY w# 1 sw Y   7xY w# 1 sw Y   y xY w)Nencryption_config/sample_datasetdatabase_dirformatfile_options
filesystemzno decryptionr>   rO   rQ   decryption_configdefault_fragment_scan_optionszUnknown master key)decryption_properties)r,   FOOTER_KEY_NAMEr/   r8   r*   CryptoFactoryr<   dsParquetEncryptionConfigParquetDecryptionConfigr   datasetParquetFileFormatmake_write_optionsfs_MockFileSystem
create_dirwrite_datasetr@   rA   IOErrorParquetFragmentScanOptionsrD   
ValueErrorselectequalsto_tablefile_decryption_properties)	read_keysread_columnsto_table_successdataset_success
write_keysrI   rT   encrypt_kms_connection_configdecrypt_kms_connection_configcrypto_factoryparquet_encryption_cfgparquet_decryption_cfgpformatwrite_optionsmockfspq_scan_optsr]   rW   r&   r5   r   s                     r   assert_decryptsz>do_test_dataset_encryption_decryption.<locals>.assert_decrypts}   s    
4_kR46(DZ(P%(DY(O%))+6!#!;!;9;L"
 "$!;!;9;L"

 **..0224 3 
 ##%#
%&	
 **..0]]7*:; 	LJJ'FK	L 444
 **..*6 / 
 *<PQ 	Yjj!1'fUG-zAUV Y||L1889I9I,9WXXXY	Y !/ I I)+<!>44"7
 **..*6 / 
 *<PQ 	Yjj!1'fUG-zAUV Y||L1889I9I,9WXXXY	Y 	Y3	L 	LY Y	Y 	Y"Y Y	Y 	YsT   #K+K72K*K+K4?2K(2K4KK	KK%(K1	-K44K=F   r   )T)dictKEYSEXTRA_COL_KEY_NAMEEXTRA_COL_KEYCOLUMN_KEYSsplitcolumn_namesr3   
startswithlenrX   
FOOTER_KEY)r   extra_column_pathextra_column_namery   all_column_namesr6   r   column_nameencrypted_column_namesplaintext_column_namesfooter_key_onlyr7   column_keys_onlycolumn_key_namecolumn_key_column_namesencrypted_column_namerk   plaintext_and_one_successplaintext_and_oner&   r5   s   `                  @@r   %do_test_dataset_encryption_decryptionr   q   s   BdB1=ABV[V-?BSAT,UV-33C8;!  	DYN ))<G<M<M<O? ?"8(L1=?"- *//4Q7 ?7 ? ? 2BU"-!,4J!J08#4#?#?;-q@Q#R	 * U U
 %&***%&***&
3O-1ZZ\8)Hc#6 !# 8 8 O%;TBO%;UCO%5u= $&<eUK$&<eUK$&6uE ;!8C8I8I8K 	D4O4)@ D% )>(C(CC(H(K% 37**,P!.3 (_o,N N &s] P	 P -BEV,V)$:>S=T$T!	+A4H	+<>WX	+A5I	+;UC%D	D, D0$7D0$7D*D1m?U84Ps   %H.2#H5H:>I 
z+Parquet Encryption is not currently enabled)reasonc                  *    t        t                      y r:   )r   r    r   r   r   "test_dataset_encryption_decryptionr      s     **=*?@r   r   listzlist.list.elementc                     t        j                  g dg dg ddgdgdggt        j                  t        j                                     }t	               j                  d|      }t        ||        y )	N)rz   r      )r   r      )      	   typer   )r   arraylist_int32r    append_columnr   )r   	list_datar   s      r   test_list_encryption_decryptionr     s[    
 	Iy2$rd;XXbhhj!I  !//	BE)%=r   )mapzmap.key_value.keyzmap.key_value.valuec                    t        j                  t        j                         t        j                               }t        j                  ddgddgddgdgg g g|      }t               j                  d	|      }t        ||        y )
N)k1rz   )k2r   )r   r   )k3r   )r   r   )r   r   )k4r   r   r   )r   map_stringr   r   r    r   r   )r   map_typemap_datar   s       r   test_map_encryption_decryptionr     sx     wwryy{BHHJ/Hxx	"Y	$:Y	<RKR	
 H  !//x@E)%=r   )structz	struct.f1z	struct.f2c                    dt        j                         fdt        j                         fg}t        j                  |      }t        j                  g d|      }t               j                  d|      }t        ||        y )Nf1f2)rz   oner   twor   threer   fourr   fiver   sixr   r   )r   r   r   r   r   r    r   r   )r   struct_fieldsstruct_typestruct_datar   s        r   !test_struct_encryption_decryptionr   (  sg    
 BHHJ'$		)<=M))M*K((TK  !//+FE)%=r   )colzcol.list.element.key_value.keyz#col.list.element.key_value.value.f1z#col.list.element.key_value.value.f2c                    dt        j                         fdt        j                         fg}t        j                  |      }d}d}d}d}d}d}t        j                  t        j                         |      }	||d	}
d
|i}d|i}||d}t        j
                  |	      }|
|g}|g}|g}t        j                  ||d |d d g|      g}t               j                  d|      }t        ||        y )Nr   r   r   r   r   r   r   r   )rz   r   r   r   )r   r   r   r   )
r   r   r   r   r   r   r   r    r   r   )r   r   r   struct1struct2struct3struct4struct5struct6r   map1map2map3map4	list_typelist1list2list3r   r   s                       r   &test_deep_nested_encryption_decryptionr   8  s     BHHJ'$		)<=M))M*KGGGGGGwwrxxz;/H7#Dw<Dw<D7#D"I4LEFEFE5%udDA	RSI!//yAE)%=r   z'Parquet Encryption is currently enabledc                      t         j                  j                         } t        j                  t
              5  | j                  d      }ddd       y# 1 sw Y   yxY w)zTest write_dataset with ParquetFileFormat and test if an exception is thrown
    if you try to set encryption_config using make_write_optionsz
some valuerH   N)r   r]   r^   r@   rA   NotImplementedErrorr_   )ru   _s     r   -test_write_dataset_parquet_without_encryptionr   _  sO     jj**,G	*	+ G&&&FG G Gs   AAc            	           G d dt         j                        d} t        j                  j	                  t        j
                  t        |       D cg c]  }t        j                          c}t        j                               gdg      }t        j                         }t        j                  fd      }t        j                  dddgid	d
d      }t        j                  |||      }t        j                  ||t        j                               }t        j                   |      }t        j"                  |      }	|	j%                  |      }
|j'                  |      }t)        j*                         }|j-                  d       d}t        j.                  |||	|
|       |dz   }t1        j2                  |||      j5                         }||k(  sJ t        j6                  ||	|      }|j9                         }||k(  sJ yc c}w )z9Test encryption and decryption of a large number of rows.c                   4    e Zd ZdededefdZdededefdZy);test_large_row_encryption_decryption.<locals>.NoOpKmsClient	key_bytesr   returnc                 0    t        j                  |      }|S r:   )base64	b64encode)selfr   r   bs       r   wrap_keyzDtest_large_row_encryption_decryption.<locals>.NoOpKmsClient.wrap_keyu  s      +AHr   wrapped_keyc                 0    t        j                  |      }|S r:   )r   	b64decode)r   r   r   r   s       r   
unwrap_keyzFtest_large_row_encryption_decryption.<locals>.NoOpKmsClient.unwrap_keyy  s      -AHr   N)__name__
__module____qualname__bytesstrr   r   r   r   r   NoOpKmsClientr   t  s6    	e 	 	 		% 	C 	E 	r   r   i  r   foo)namesc                             S r:   r   )r   r   s    r   <lambda>z6test_large_row_encryption_decryption.<locals>.<lambda>  s	     r   UNIMPORTANT_KEYTF   )r   r&   double_wrappingr%   r)   rS   rU   rH   rJ   zlarge-row-test-dataset)rO   rP   rQ   z/part-0.parquetrW   rQ   rR   N)r*   	KmsClientr   Tablefrom_arraysr   rangerandomfloat32r2   rY   r+   rZ   r[   r\   r.   re   r^   r_   rj   r`   ra   rb   rc   pqParquetFilereadr]   ri   )	row_countr   r   
kms_configrr   rI   
pqe_config
pqd_configscan_optionsfile_formatrv   rj   rw   path	file_path	new_tabler]   r   s                    @r   $test_large_row_encryption_decryptionr  n  s     IHH  	&+I&67V]]_7

 	 '	 ! E '')J%%&?@N22$&0  ++
$5J ++
B$>$>$@J 00:NL&&\RK22Z2PM!/!J!J:!V!F
c#DUD"/FD ((I)C df  IjjkfEG  "IIS 8s   Hc                     t               } t               }t               }t               }t	        j
                  t              }t        j                  |||      }t        j                  |||      }t        j                  j                         }|j                  |ddg      }t        j                         }	|	j!                  d       t        j"                  | d|||	       t        j$                  |      }
t        j                  j                  |
      }t        j                  d||		      }|j'                         D ]  }|j)                  |||j*                  |	      }t-        j.                  |j*                  ||	
      5 }t1        |j2                  j4                        D ]  }|j2                  j7                  |      }|j9                  d      j:                  J |j9                  d      j:                  j<                  dk(  sJ |j9                  d      j:                  j>                  dk(  sJ |j9                  d      j:                  J |j9                  d      j:                  j<                  dk(  sJ |j9                  d      j:                  j>                  dk(  sJ |j9                  d      j:                  J  	 d d d         y # 1 sw Y   xY w)Nr   r   )rI   write_statisticsrJ   rK   rL   rS   rU   rR   r   r   r   r   rz   r   r   ) r    r,   r/   r8   r*   rY   r<   rZ   r[   r\   r   r]   r^   r_   r`   ra   rb   rc   re   get_fragmentsrj   r  r   r   r   metadatanum_row_groups	row_groupcolumn
statisticsminmax)r   rI   rT   kms_connection_configrr   rs   rt   ru   rv   rw   rx   r]   fragmentrW   parquet_filerg_idxr  s                    r   7test_dataset_encryption_with_selected_column_statisticsr    s     !E02028:%%k2N77-/@  77-/@ jj**,G..0 (+ / M
 !F
c!" 000L jj***VGjj)'fMG))+ > . I I!#4hmmV!M^^MM"7
 	>  5 5 D DE >(11;;FC	 ''*55AAA ''*5599TAAA ''*5599TAAA ''*55AAA ''*5599Q>>> ''*5599S@@@ ''*55===>	> 	>>	> 	>s   D<KKK	r:   )4r   
contextlibr   datetimer   r   
pyarrow.fsr`   pyarrowr   r@   encryption_unavailablepyarrow.parquetparquetr   pyarrow.datasetr]   rZ   ImportError pyarrow.tests.parquet.encryptionr   pyarrow.parquet.encryption
encryptionr*   mark
pytestmarkr   rX   COL_KEYCOL_KEY_NAMEr|   r~   r}   COLUMNSr   r    r,   r/   r8   r<   rD   r   skipifr   parametrizer   r   r   r   r   r  r  r   r   r   <module>r.     s  $  %        
"B++ [[  
 !

\7;# 
&h12" )8[ 	: '+ ;  J2Z #P  AA #P   1D(EF> G> 8   F>	> #P   (LM	> N	> 8   >	>4 'P  	G	G #P  88v #P  =>=>_  	B	B  "!"s"   H2 I 2	H?>H?II