Move-Pollination for Creativity Leveraging LLMs

numpy==1.25.2
torch==2.0.1
annoy==1.17.3
langchain==0.0.237
sentence-transformers==2.2.2

import json 
from sentence_transformers import SentenceTransformer, util
import numpy as np
import torch
import time
from annoy import AnnoyIndex

file_name = "arxiv-metadata-oai-snapshot.json"

'''
Preprocess the knowledge and construct a listing
'''
def preprocess(trail):
    records = []
    beginning = time.time()
    with open(trail,'r') as f:
        for line in f:
            records.append(json.a lot(line))
    beginning = time.time()
    sents = []
    for i in vary(len(records)):
        sents.append(records[i]['title']+'[SEP]'+records[i]['abstract'])
    finish = time.time()
    print("Time taken: ",end-start)  
    go back sents


'''
Use Sentence BERT Specter embedding to construct a numpy array
'''
def generate_embeddings(sents,type):
    embeddings = type.encode(sents,batch_size=400,show_progress_bar=True,instrument="cuda",convert_to_numpy=True)
    np.save("embeddings.npy",embeddings)
    go back embeddings


'''
Generate an annoy index. 
As an alternative of opting for angular as distance metric, different choices comparable to euclidean also are to be had
'''
def generate_annoy(embeddings):
    n_trees = 256           
    embedding_size = 768   
    top_k_hits = 5        
    annoy_index = AnnoyIndex(embedding_size, 'angular')
    for i in vary(len(embeddings)):
        annoy_index.add_item(i, embeddings[i])
    annoy_index.construct(n_trees)
    annoy_index.save("vector_index.ann")
    go back annoy_index

sents = preprocess(file_name)
type = SentenceTransformer('sentence-transformers/allenai-specter', instrument="cuda")
embeddings = generate_embeddings(sents,type)
annoy_index = generate_annoy(embeddings)

def seek(question,annoy_index,type, topK):
    query_embedding = type.encode(question,convert_to_numpy=True)
    tokens = question.strip().break up()
    hit_dict = {}
    hits = annoy_index.get_nns_by_vector(query_embedding, topK, include_distances=True)
    go back [hits, hits]

    
def collect_results(hits,sents, trail, prefix=""):
    reaction = ""
    for i in vary(len(hits[0])):
        reaction += "outcome " + prefix + str(i) + "n"
        reaction += "Identify:" + sents[hits[0][i]].break up('[SEP]')[0] + "n"
        reaction += "Summary: " + sents[hits[0][i]].break up('[SEP]')[1] + "n"
    go back reaction

fieldA = enter("Describe the sphere (or subfield) as a question:")
question = fieldA
reaction = collect_results(seek(question,an,type,30),sents_from_path1, path1)

template = """ Take a deep breath. 
    You're a researcher within the FIELD of {question}. You're tasked with figuring out analysis gaps and alternatives. Person entered the FIELD. The TOP ARTICLES from the quest engine are {reaction}. Analyze the effects and practice the directions under to spot analysis gaps within the box of {question}.
    TASK DESCRIPTION 
    On this process, your purpose is to acknowledge spaces inside of a specified FIELD the place additional analysis is wanted. 
    By means of inspecting present literature and TOP ARTICLES , records, and possibly the trajectories of ongoing analysis, the AI can assist indicate subjects or questions that have not been adequately explored or resolved. This is very important for advancing wisdom and discovering new instructions that might result in vital discoveries or trends
    As an example, imagine the area of renewable power. one may analyze an unlimited collection of analysis papers, patents, and ongoing tasks on this area. It will then determine that whilst there may be ample analysis on sun and wind power, there may well be a relative loss of analysis on harnessing tidal power or integrating various kinds of renewable power methods for extra constant energy technology.
    By means of figuring out those analysis gaps, you're going to assisint in pinpointing spaces the place researchers can center of attention their efforts to probably make novel contributions. Additionally, it could actually assist investment companies and establishments to direct assets and improve in opposition to those recognized spaces of alternative, fostering developments that may differently be overpassed.
   
    INSTRUCTIONS:
    1. Analyze each and every article returned by means of the quest engine.
    2. Establish the analysis gaps and alternatives for each and every article
    3. Summarize the analysis gaps and alternatives within the box of {question} throughout ARTICLES. The analysis hole will have to BE GROUNDED at the articles returned by means of the quest engine.
    3. Establish an orthogonal box and generate key phrase to get articles from that box that may clear up the analysis gaps.
    4. Establish a comparable box and generate key phrase to get articles from that box that may clear up the analysis gaps
    5. Generate a JSON Reaction that STRICTLY practice the TYPESCRIPT SCHEMA of sophistication Reaction under
    6. Output MUST BE a JSON  with the next fields within the SCHEMA.
    SCHEMA:
    magnificence Reaction:
        orthogonal_field: String
        keyword_orthogonal_field: String
        related_field: String
        keyword_related_field: String
        research_gaps: String
    
"""
instructed = PromptTemplate(template=template, input_variables=["query", "response"])
llm_chain = LLMChain(instructed=instructed, llm=llm)

output = ""

check out:
	output = llm_chain.run({'question': question, 'reaction':reaction})
aside from Exception as e:
  	print (e)