The bm25 and gpt-index scripts ? about gorilla HOT 9 CLOSED

import json
from rank_bm25 import BM25Okapi


def load_data(file_name):
    with open(file_name, 'r') as f:
        data = [json.loads(line) for line in f]
    return data

def process_json_data(data):
    texts = []
    for item in data:
        text = json.dumps(item)
        texts.append(text)
    return texts

def init_bm25_model(texts):
    tokenized_corpus = [doc.split(" ") for doc in texts]
    bm25 = BM25Okapi(tokenized_corpus)
    return bm25

def search(query, bm25):
    tokenized_query = query.split(" ")
    doc_scores = bm25.get_scores(tokenized_query)
    best_doc = bm25.get_top_n(tokenized_query, texts, n=1)[0]
    return best_doc


data = load_data('data/api/huggingface_api.jsonl')  
texts = process_json_data(data)

bm25 = init_bm25_model(texts)

query_data = load_data('eval/eval-data/questions/huggingface/questions_huggingface_bm25.jsonl')  
domains=[]
for item in query_data:
    query = item['text']
    best_doc = search(query, bm25)
    print(best_doc)

import json
import openai
openai.api_key = '****' 

def load_data(file_name):
    with open(file_name, 'r') as f:
        data = [json.loads(line) for line in f]
    return data


query_data = load_data('eval/eval-data/questions/huggingface/questions_huggingface_gpt_index.jsonl')  
querys=[]
for item in query_data:
    query = item['text']
    querys.append(query)

def text_to_embedding(text):
    response = openai.Embedding.create(
        input=text,
        model="text-embedding-ada-002"
    )
    embeddings = response['data'][0]['embedding']
    return embeddings

embeddings=[]
for query in querys:
    print(query)
    embedding = text_to_embedding(query)
    embeddings.append(embedding)  import pickle

def store_text_and_embeddings(texts, embeddings, filename):
    assert len(texts) == len(embeddings), "The length of texts and embeddings must be the same

    data = {
        'texts': texts,
        'embeddings': embeddings
    }

    with open(filename, 'wb') as f:
        pickle.dump(data, f)
        
store_text_and_embeddings(querys, embeddings, 'ada_query_texts_and_embeddings.pkl')


def process_json_data(data):
    texts = []
    for item in data:
        text = json.dumps(item)
        texts.append(text)
    return texts


API_data = load_data('data/api/huggingface_api.jsonl')  
API_texts = process_json_data(API_data)
store_text_and_embeddings(texts, embeddings, 'ada_huggingface_api_texts_and_embeddings.pkl')

from numpy import dot
from numpy.linalg import norm

def cosine_similarity(query_embedding, text_embeddings):
    query_norm = norm(query_embedding)
    similarities = []
    for text_embedding in text_embeddings:
        text_norm = norm(text_embedding)
        cosine_sim = dot(query_embedding, text_embedding) / (query_norm * text_norm)
        similarities.append(cosine_sim)
    
    return similarities

def find_most_similar_texts(query_embedding, texts, text_embeddings):
    similarities = cosine_similarity(query_embedding, text_embeddings)
    max_similarity_index = similarities.index(max(similarities))

    return texts[max_similarity_index]

import pickle
def load_texts_and_embeddings(filename):
    with open(filename, 'rb') as f:
        data = pickle.load(f)
        
    return data['texts'], data['embeddings']
texts, text_embeddings = load_texts_and_embeddings('ada_huggingface_api_texts_and_embeddings.pkl')
querys, query_embeddings= load_texts_and_embeddings('ada_query_texts_and_embeddings.pkl')  



for i in range(len(querys)):
    query=querys[i]
    query_embedding=query_embeddings[i]
    similar_text=find_most_similar_texts(query_embedding, texts, text_embeddings)
    print(similar_text)