rhXddlmZddlZddlZddlmZmZddlZddl Z ddl m Z eje Z er ddlmZ ddlmZ ddlmZ ddlmZ d dd Z d dd Z d dd Z d dd Zy#e$rYywxYw#e$rY}wxYw#e$rYwxYw#e$rYwxYw)) annotationsN) TYPE_CHECKINGAny)tqdm) Elasticsearch QdrantClient) SeismicIndex) OpenSearchc  ddlm}ddlm}|j r|j tjk7r td|E| td|j r|j tjk7r td|ddd i|}d ttj} |j| id |j|jd i|j} | j!j#j%} | j'j#j%} |j)d} d}g}d}| d}t+j,|t+j.| d}t+j,|t+j.| d}t1t3| dD]}||}||}| d||j5}| ||j5}d |j7||i}|j9|t;||k\s || dz k(ss|j=| |t3||t;|z|t;|z }g}|| f}|\}} g}tj}t3|j)dD]}|j?dk(r|jj!dj#j%j5}|jj'j#j%j5}n|jj!dj#j%|k(}|jj!d|j#j%j5}|jj'|j#j%j5}|jA| |j7|||d jB} | D!cgc]}!|!jD|!jFd}"}!|j9|"tj|z }#|r||#|fS||#fS#t$r t dwxYwcc}!w)a Performs semantic search using sparse embeddings with Qdrant. Args: query_embeddings: PyTorch COO sparse tensor containing query embeddings corpus_embeddings: PyTorch COO sparse tensor containing corpus embeddings Only used if corpus_index is None corpus_index: Tuple of (QdrantClient, collection_name) If provided, uses this existing index for search top_k: Number of top results to retrieve output_index: Whether to return the Qdrant client and collection name Returns: A tuple containing: - List of search results in format [[{"corpus_id": int, "score": float}, ...], ...] - Time taken for search - (Optional) Tuple of (QdrantClient, collection_name) if output_index is True rr)modelszWPlease install the Qdrant client with `pip install qdrant-client` to use this function.z,Query embeddings must be a sparse COO tensorz9Either corpus_embeddings or corpus_index must be providedz-Corpus embeddings must be a sparse COO tensorurlzhttp://localhost:6333sparse_collection_textF)on_diskindex)collection_namevectors_configsparse_vectors_configi'left)siderightz#Processing and Upserting embeddingsdesc)indicesvalues)rvectorsids)rquerylimitusing corpus_idscore)$ qdrant_clientr qdrant_client.httpr ImportError is_sparselayouttorch sparse_coo ValueErrorinttimecreate_collectionSparseVectorParamsSparseIndexParamscoalescercpunumpyrsizenp searchsortedarangerrangetolist SparseVectorappendlenupload_collection sparse_dim query_pointspointsidr&)$query_embeddingscorpus_embeddings corpus_indextop_k output_indexkwargsr r clientrcorpus indices_arr values_arr num_vectors batch_size vectors_batch insert_idxrow_idsstartsendsistartend vec_indices vec_values vector_data all_resultssearch_start_timeq_idx q_indicesq_valuesmasksearch_resultshitformatted_results search_times$ /var/www/html/ai-insurance-compliance-backend/venv/lib/python3.12/site-packages/sentence_transformers/sparse_encoder/search_engines.pysemantic_search_qdrantrh s:u.-  % %)9)@)@EDTDT)TGHH  $XY Y **.?.F.F%JZJZ.ZLM MD"9DVD.s499;/?.@A  ./#)6+D+D6KcKclqKcKr+D+s"t ! #++-nn&**,224 ]]_((*002 ',,Q/    a."))K*@vNw +(>WMeK(/TU #A1IEq'C%a.s3::z:semantic_search_elasticsearch...(&ZPQz!U';'KA! 'K&Z*,Nrplistallrsitems rgruz0semantic_search_elasticsearch..5A  4Z3&ZUY&Z#ZZA.0UQuery embeddings must be a list of lists in the format [[('token', value), ...], ...]AEither corpus_embeddings_decoded or corpus_index must be providedc3dK|](}t|txrtd|D*yw)c3\K|]$}t|txrt|dk(&ywrmrorrs rgruz:semantic_search_elasticsearch...(*^TU:a+?+OCFaK+O*^rwNrxr{s rgruz0semantic_search_elasticsearch..5F  tT " ^s*^Y]*^'^ ^F r~VCorpus embeddings must be a list of lists in the format [[('token', value), ...], ...] sparse_index_rmappings propertiestype rank_featureskeywordtokensrErbodyUpserting embeddingsr._rEr_index_id_source rank_featureztokens.)field saturationboostboolr)shouldminimum_should_matchr8r!hitsr_scorer$zhttp://localhost:9200) elasticsearchrrjr*rpryrzr/r0r1rexistsdeletecreater@rr<mindictitemsstrreplacer?bulkrefreshsearch)query_embeddings_decodedcorpus_embeddings_decodedrHrIrJrKrrjes index_namenum_docsrQ start_idxend_idxactionsrWrkvr]r^r_ query_tokensshould_clausestokenweightr!resultrd formattedrfs rgsemantic_search_elasticsearchrsH 8 . 5SA,A>pqq $ ,`a a3T:#F 1F C uv v  =f =$S%5$67 ::  :  . JJ  J  /  "O# &y1 #   01 eAxwug=N^`kq3r"s t u&^ef2g)hi%8X^^dWeflWmnPS3s5z?S]Kn n9%&"))+ 11KK55K''E   l   f6 osL%&L/ &L5L;L,c ddlm}m}m} |i}|i}| } t |t rtd|Ds td| | tdt |t rtd|Ds td|} t|} tt| d D]} t|| }| jt| tj t |j#| tj t |j%tj& |j(| fi|}t+j*}t|}g}g}t|D]}t||}|j-tj t |j#| |j-tj t |j%tj& d |vrd |d <d|vrd|d<|j.dtj t|| |||d|}t1|d}|Dcgc]&}|Dcgc]\}}}t3||dc}}}(}}}}}t+j*|z }|r|||fS||fS#t$r t dwxYwcc}}}wcc}}}}w)a Performs semantic search using sparse embeddings with Seismic. Args: query_embeddings_decoded: List of query embeddings in format [[("token": value), ...], ...] Example: To get this format from a SparseEncoder model:: model = SparseEncoder('my-sparse-model') query_texts = ["your query text"] query_embeddings = model.encode(query_texts) query_embeddings_decoded = model.decode(query_embeddings) corpus_embeddings_decoded: List of corpus embeddings in format [[("token": value), ...], ...] Only used if corpus_index is None Can be obtained using the same decode method as query embeddings corpus_index: Tuple of (SeismicIndex, collection_name) If provided, uses this existing index for search top_k: Number of top results to retrieve output_index: Whether to return the SeismicIndex client and collection name index_kwargs: Additional arguments for SeismicIndex passed to build_from_dataset, such as centroid_fraction, min_cluster_size, summary_energy, nknn, knn_path, batched_indexing, or num_threads. search_kwargs: Additional arguments for SeismicIndex passed to batch_search, such as query_cut, heap_factor, n_knn, sorted, or num_threads. Note: query_cut and heap_factor are set to default values if not provided. Returns: A tuple containing: - List of search results in format [[{"corpus_id": int, "score": float}, ...], ...] - Time taken for search - (Optional) Tuple of (SeismicIndex, collection_name) if output_index is True r)SeismicDatasetr get_seismic_stringzMPlease install Seismic with `pip install pyseismic-lsr` to use this function.c3dK|](}t|txrtd|D*yw)c3\K|]$}t|txrt|dk(&ywrmrorrs rgruz4semantic_search_seismic...crvrwNrxr{s rgruz*semantic_search_seismic..br}r~rrc3dK|](}t|txrtd|D*yw)c3\K|]$}t|txrt|dk(&ywrmrorrs rgruz4semantic_search_seismic...mrrwNrxr{s rgruz*semantic_search_seismic..lrr~rzAdding documents to Seismicr)dtype query_cut heap_factorgffffff?) queries_idsquery_components query_valuesrc$t|ddS)Nr)r0)xs rgz)semantic_search_seismic..sC!QL)keyr$r')seismicrr rr*rpryrzr/r@rr<r add_documentrr9arraykeysrfloat32build_from_datasetr1r? batch_searchsortedr0)rrrHrIrJ index_kwargs search_kwargsrr r string_typedatasetrPidxrr^ num_queriesrrr_rresults query_result query_idxr&r%r]rfs rgsemantic_search_seismicr+sTkLL  $&K . 5SA,A>pqq $ ,`a a3T:#F 1F C uv v!"34 k*1NO C3C89F  Cfkkm,K@fmmo.bjjA  7|66wO,O  ./KL{#U4U;< l.?.?.A)B+ VWBHHT,*=*=*?%@ STU -'%' k"M)'* m$'l''HHU;/{C)!    GW"89G $ [ggg;V9eYs9~ 6gK ))+ 11KK55K''e kijjkR hs# K : K+ K$K+ K!$K+ c  ddlm}m}t |t rt d|Ds td|x| tdt |t rt d|Ds td|di|}d ttj} |jj| r|jj| |jj| d d d did didiit|} d} tt!d| | dD]n} t#| | z| } g}t!| | D]:}t%||}|j'| t)|t)||dd<|j+||p|jj-| || f}|\}} g}tj}t!t|D]f}t%||}|ddd|iiid}|j/| |}|ddDcgc]}t|d|dd}}|j'|htj|z }|r|||fS||fS#t$r tdwxYwcc}w) a Performs semantic search using sparse embeddings with OpenSearch. Args: query_embeddings_decoded: List of query embeddings in format [[("token": value), ...], ...] Example: To get this format from a SparseEncoder model:: model = SparseEncoder('my-sparse-model') query_texts = ["your query text"] query_embeddings = model.encode(query_texts) query_embeddings_decoded = model.decode(query_embeddings) corpus_embeddings_decoded: List of corpus embeddings in format [[("token": value), ...], ...] Only used if corpus_index is None Can be obtained using the same decode method as query embeddings corpus_index: Tuple of (OpenSearch, collection_name) If provided, uses this existing index for search top_k: Number of top results to retrieve output_index: Whether to return the OpenSearch client and collection name vocab: The dict to transform tokens into token ids Returns: A tuple containing: - List of search results in format [[{"corpus_id": int, "score": float}, ...], ...] - Time taken for search - (Optional) Tuple of (OpenSearch, collection_name) if output_index is True r)r rjz[Please install the OpenSearch client with `pip install opensearch-py` to use this function.c3dK|](}t|txrtd|D*yw)c3\K|]$}t|txrt|dk(&ywrmrorrs rgruz7semantic_search_opensearch...rvrwNrxr{s rgruz-semantic_search_opensearch..r}r~rrc3dK|](}t|txrtd|D*yw)c3\K|]$}t|txrt|dk(&ywrmrorrs rgruz7semantic_search_opensearch...rrwNrxr{s rgruz-semantic_search_opensearch..rr~rrrrrrrrrrrrrrr neural_sparserrrrrrr$r) opensearchpyr rjr*rpryrzr/r0r1rrrrr@rr<rrr?rrrr)rrrHrIrJrKr rj os_clientrrrQrrrrWrr]r^r_rr!rrdrrfs rgsemantic_search_opensearchrsJ 4 . 5SA,A>pqq $ ,`a a3T:#F 1F C uv vA&A $S%5$67    # #* # 5    $ $: $ 6   "O# &y1 #  ! 01 eAxrs/" %    8 $ / . ( +.248 }("}(*}(2}(  }(  }(  }(Q}(DGK59 H(;H(CH(3H(  H(  H(  H(RH(ZGK48*.+/~(;~(C~(2~(  ~(  ~( ( ~()~(Q~(FGK26 (;(C(0(  (  (  (O(y                sGB2B=CC2B:9B:=CCCCCC