
    nh              
          d dl Z d dlZd dlZd dlZd dlmZ d dlmZ d dlm	Z	 e j                  j                  e j                  j                  e      d      Ze j                  j                  e j                  j                  e      d      Ze j                  j                  e j                  j                  e      d      ZdZd	Zeefd
edededee   fdZdee   deee      fdZd Zd Zy)    N)tqdm)List)SentenceTransformerz../data/database.txtzfaiss.indexzfaiss_meta.pkliX  d   text
chunk_sizeoverlapreturnc                     g }d}|t        |       k  rCt        ||z   t        |             }|j                  | ||        |||z
  z  }|t        |       k  rC|S )Nr   )lenminappend)r   r   r	   chunksstartends         //var/www/html/axiom-chatbot/chatbot/embedder.py
chunk_textr      sc    FE
#d)
%*$c$i0d5o&g%% #d)
 M    textsc                 B    t        d      }|j                  | d      }|S )Nzall-MiniLM-L6-v2T)show_progress_bar)r   encode)r   model
embeddingss      r   embed_textsr      s%     23Eet<Jr   c                     t        t        dd      5 } | j                         }d d d        t              }t	        |      }t        |d         }t        j                  |      }|j                  t        j                  |      j                  d             t        j                  |t               t        t        d      5 } t        j                   ||        d d d        y # 1 sw Y   xY w# 1 sw Y   y xY w)Nrzutf-8)encodingr   float32wb)open	DATA_PATHreadr   r   r   faissIndexFlatL2addnparrayastypewrite_index
INDEX_PATH	META_PATHpickledump)fr   r   r   dimindexs         r   build_and_save_indexr2      s    	iw	/ 1vvxFV$J
jm
Cc"E	IIbhhz")))45	eZ(	i	 !FA   s   C"C."C+.C7c                      t        j                  t              } t        t        d      5 }t        j                  |      }d d d        | |fS # 1 sw Y   | fS xY w)Nrb)r$   
read_indexr+   r!   r,   r-   load)r1   r/   r   s      r   load_index_and_metar7   *   sO    Z(E	i	  !Q &= &=s   AA)osr-   numpyr'   r$   r   typingr   sentence_transformersr   pathjoindirname__file__r"   r+   r,   
CHUNK_SIZECHUNK_OVERLAPstrintr   floatr   r2   r7    r   r   <module>rF      s    	      5GGLL24JK	WW\\"''//(3]C
GGLL24DE	
,6} S c  Y]^aYb tCy T$u+%6 

r   