bi,;dZddlmZddlZddlZddlmZmZmZm Z m Z m Z m Z ddl mZddl mZddl mZddl mZer dd lmZdd lmZGd d ZeZGd dZGddZeZdgZy)a<Make approximate assertions as "expectations" on test results. This module is designed to be used within test cases decorated with the `@pytest.mark.decorator` decorator It allows you to log scores about a test case and optionally make assertions that log as "expectation" feedback to LangSmith. Example: ```python import pytest from langsmith import expect @pytest.mark.langsmith def test_output_semantically_close(): response = oai_client.chat.completions.create( model="gpt-3.5-turbo", messages=[ {"role": "system", "content": "You are a helpful assistant."}, {"role": "user", "content": "Say hello!"}, ], ) response_txt = response.choices[0].message.content # Intended usage expect.embedding_distance( prediction=response_txt, reference="Hello!", ).to_be_less_than(0.9) # Score the test case matcher = expect.edit_distance( prediction=response_txt, reference="Hello!", ) # Apply an assertion and log 'expectation' feedback to LangSmith matcher.to_be_less_than(1) # You can also directly make assertions on values directly expect.value(response_txt).to_contain("Hello!") # Or using a custom check expect.value(response_txt).against(lambda x: "Hello" in x) # You can even use this for basic metric logging within tests expect.score(0.8) expect.score(0.7, key="similarity").to_be_greater_than(0.7) ``` ) annotationsN) TYPE_CHECKINGAnyCallableLiteralOptionalUnionoverloadclient) run_helpers) run_trees)utils)EditDistanceConfig)EmbeddingConfigc eZdZdZddZddZy) _NULL_SENTRYzA sentinel singleton class used to distinguish omitted keyword arguments from those passed in with the value None (which may have different behavior). cy)NFselfs [/home/obispo/Crisostomo_bridge/mision_env/lib/python3.12/site-packages/langsmith/_expect.py__bool__z_NULL_SENTRY.__bool__Qscy)N NOT_GIVENrrs r__repr__z_NULL_SENTRY.__repr__TsrN)returnzLiteral[False])rstr)__name__ __module__ __qualname____doc__rrrrrrrLsrrceZdZdZ d ddZdddZddZddZddZddZ ddd Z dd Z dd Z dd Z dd Zy)_Matcherz4A class for making assertions on expectation values.Nc||_||_||_|xstjd|_t j|_|jr|jj|_ y||_ y)N max_workers) _clientkeyvaluels_utilsContextThreadPoolExecutor _executorrhget_current_run_tree_rttrace_id_run_id)rr r+r,r/run_ids r__init__z_Matcher.__init__^s^  "Wh&H&HUV&W**,,0HHtxx(( & rc tjs|jstj|_|j j |jj|jd|||jr|jjnd|jr|jjndyy)N expectation)r5r+scorecomment session_id start_time) r-test_tracking_is_disabledr*rtget_cached_clientr/submitcreate_feedbackr4r2r;r<)rr9messages r_submit_feedbackz_Matcher._submit_feedbackms113<<!335 NN ! ! ,,||!26((488..26((488.. " 4rc |sJ||jdd|jd|y#t$r#}|jdt||dd}~wwxYw)Nz Success: .)rBr)rCr+AssertionErrorrepr)r conditionrB method_namees r_assertz_Matcher._assert{sd  %g %9  ! !!y !K=-Q ! R   ! !!T!W -  s,/ AAAc ~|j|j|kd|jd|d|jdy)zAssert that the expectation value is less than the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value is not less than the given value. Expected z to be less than , but got to_be_less_thanNrLr,r+rr,s rrPz_Matcher.to_be_less_thans> JJ z!25'DJJ< P  rc ~|j|j|kDd|jd|d|jdy)aAssert that the expectation value is greater than the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value is not greater than the given value. rNz to be greater than rOto_be_greater_thanNrQrRs rrTz_Matcher.to_be_greater_thans> JJ z!5eWJtzzl S  rc |j||jcxkxr|kncd|jd|d|d|jdy)a4Assert that the expectation value is between the given min and max values. Args: min_value: The minimum value (exclusive). max_value: The maximum value (exclusive). Raises: AssertionError: If the expectation value is not between the min and max. rNz to be between z and rO to_be_betweenNrQ)r min_value max_values rrVz_Matcher.to_be_betweensO  .Y .z 5 L | %   rc |jt|j|t||k(d|jd|d|jdy)akAssert that the expectation value is approximately equal to the given value. Args: value: The value to compare against. precision: The number of decimal places to round to for comparison. Raises: AssertionError: If the rounded expectation value does not equal the rounded given value. rNz to be approximately rOto_be_approximatelyN)rLroundr,r+)rr, precisions rrZz_Matcher.to_be_approximatelysL $**i (E%,C Cz!6ugZ | T ! rc ~|j|j|k(d|jd|d|jdy)aAssert that the expectation value equals the given value. Args: value: The value to compare against. Raises: AssertionError: If the expectation value does not exactly equal the given value. rNz to be equal to rOto_equalNrQrRs rr^z_Matcher.to_equals> JJ% z!1% 4::, O  rcv|j|jdud|jd|jdy)zAssert that the expectation value is `None`. Raises: AssertionError: If the expectation value is not `None`. NrNz to be None, but got to_be_nonerQrs rr`z_Matcher.to_be_nones8 JJ$ z!6tzzl C  rcd|j||jvd|jd|ddy)zAssert that the expectation value contains the given value. Args: value: The value to check for containment. Raises: AssertionError: If the expectation value does not contain the given value. rNz to contain z, but it does not to_containNrQrRs rrbz_Matcher.to_contains6 TZZ zeW4E F  rctj|}|j||jd|d|jdy)zAssert the expectation value against a custom function. Args: func: A custom function that takes the expectation value as input. Raises: AssertionError: If the custom function returns False. z Assertion z failed for againstN)inspect signaturerLr,r+)rfuncfunc_signatures rrdz_Matcher.againstsD!**40  ( TXXJ ?  r)NN) r Optional[ls_client.Client]r+rr,rr/z,Optional[ls_utils.ContextThreadPoolExecutor]r5 Optional[str]N)r9intrBrjrNone)rIboolrBrrJrrrm)r,floatrrm)rWrorXrorrm))r,ror\rlrrm)rrm)r,rrrm)rgrrrm)r r!r"r#r6rCrLrPrTrVrZr^r`rbrdrrrr%r%[s}>CG $ A* A A A @ A  A     " "     rr%ceZdZdZddddZdd ddZdd ddZddZd ddd  dd Ze dd Z e dd Z e df ddZ ddZ y)_Expectz1A class for setting expectations on test results.Nr c||_tjd|_t j |jj dy)Nr'r(T)wait)r*r-r.executoratexitregistershutdownrr s rr6z_Expect.__init__s3  ::qI  ..T:rconfigc ,ddlm}|xsi}|jdrdnd}||}|j||}||jd}|j d ||d |d |jd t |jd ||j S)aCompute the embedding distance between the prediction and reference. This logs the embedding distance to LangSmith and returns a `_Matcher` instance for making assertions on the distance value. By default, this uses the OpenAI API for computing embeddings. Args: prediction: The predicted string to compare. reference: The reference string to compare against. config: Optional configuration for the embedding distance evaluator. Supported options: - `encoder`: A custom encoder function to encode the list of input strings to embeddings. Defaults to the OpenAI API. - `metric`: The distance metric to use for comparison. Supported values: `'cosine'`, `'euclidean'`, `'manhattan'`, `'chebyshev'`, `'hamming'`. Returns: A `_Matcher` instance for the embedding distance value. Example: ```python expect.embedding_distance( prediction="hello", reference="hi", ).to_be_less_than(1.0) ``` r)EmbeddingDistanceencodercustomopenairz prediction reference)r~metricembedding_distanceUsing z , Metric: r9 source_infor:r/) 'langsmith._internal._embedding_distancer}getevaluatedistancerCr%r*ru) rrrr{r} encoder_func evaluatorr9src_infos rrz_Expect.embedding_distance sT N2#)::i#8xh %V4 ""jI"N+y7I7IJ  '#L>I)>JJ0$7  / ""jI"N$9=  '#F8= D   LL  mm   rcHt|jd||jS)aDCreate a `_Matcher` instance for making assertions on the given value. Args: value: The value to make assertions on. Returns: A `_Matcher` instance for the given value. Example: ```python expect.value(10).to_be_less_than(20) ``` r,r)r%r*rurRs rr,z _Expect.values gu NNrr9)r+ source_run_idr:cz|j||ddi||dt|j|||jS)aLog a numeric score to LangSmith. Args: score: The score value to log. key: The key to use for logging the score. Defaults to `'score'`. Example: ```python expect.score(0.8) # doctest: +ELLIPSIS expect.score(0.8, key="similarity").to_be_greater_than(0.7) ``` methodz expect.score)r9rrr:r)rCr%r*ru)rr9r+rr:s rr9z _Expect.scoresF,   (.9!."    c5DMMJJrcyrkrrRs r__call__z_Expect.__call__s36rcyrkrrys rrz_Expect.__call__sCFrcPt|}|tur|j|S|S)Nr )rrrr,)rr,r expecteds rrz_Expect.__call__s* &)  !>>%( (rc(tj}|r |jnd}tjsZ|j st j|_|jj|j jf||d|yy)N)r5r+) r0r1r3r-r=r*r>r?rur@rA)rr+results current_runr5s rrCz_Expect._submit_feedbacksz--/ )4%%$113<<!335 DMM  ,, 5; HO 4r)r ri)rrrrr{zOptional[EmbeddingConfig]rr%)rrrrr{zOptional[EditDistanceConfig]rr%)r,rrr%) r9zUnion[float, int, bool]r+rrzOptional[ls_client.ID_TYPE]r:rjrr%)r zls_client.Clientrrr)r rir,z Optional[Any]rzUnion[_Expect, _Matcher])r+rrdict) r r!r"r#r6rrr,r9r rrrCrrrrrrrs;?C;-1 ; ; ; * ;  ; D04 8 8 8 - 8  8 tO(59!% K&K K 3 K  K KF66 FF )-1 +   " rrrexpect) r# __future__rrvretypingrrrrrr r langsmithr ls_clientr r0rr>rr-rrrrrrr%rrr__all__rrrrsy0d# *'%'EG   N c c LKK\  *r