当前位置：首页 > news >正文

Langchain 加载网络信息实现RAG以及UnstructuredURLLoader的使用

news 来源：原创 2024/5/10 19:08:54

以下实现了从 wikipedia 加载 Android 的网页然后保存在本地的向量数据库，然后通过上下文发给大模型，让他来总结什么是android 。

from langchain_community.vectorstores import Chroma
from langchain_core.prompts import ChatPromptTemplate
from langchain_core.output_parsers import StrOutputParser
from langchain_community.embeddings import JinaEmbeddings
from langchain_community.llms import Tongyi
from langchain_core.runnables import RunnableParallel, RunnablePassthrough
import os
os.environ["DASHSCOPE_API_KEY"] = "sk-cc1c8314fdbd43ceaf26ec1824d5dd3b"
llm = Tongyi()from langchain_community.document_loaders import UnstructuredURLLoaderurls = ["https://en.wikipedia.org/wiki/Android_(operating_system)",# "https://answer.baidu.com/answer/land?params=MSZrNRfsun5P549PG8zUdV6PXibobtP6242M%2FwAkrm0sWWOs5IAFrz6XAVTZu6sZDpxyjT4AEJir6bCqxEWwPoy%2F7dCKyABf%2BFgxpeKWkX0isoUgqs7ViRSvL3B%2BBsajzbX1Ai05uEVz4Owgwf361B4xj1CwAAbsAD3PBqPnJT4%3D&from=dqa&lid=ebb4fc0600ddb6f0&word=%E4%BB%80%E4%B9%88%E6%98%AF%E8%8B%B9%E6%9E%9C",# "https://www.understandingwar.org/backgrounder/russian-offensive-campaign-assessment-february-8-2023",# "https://blog.csdn.net/oHeHui1/article/details/136261119?spm=1001.2014.3001.5502"
]loader = UnstructuredURLLoader(urls=urls)
documents = loader.load_and_split()
print(documents)
embeddings = JinaEmbeddings(jina_api_key="jina_c5d02a61c97d4d79b88234362726e94aVLMTvF38wvrElYqpGYSxFtC5Ifhj", model_name="jina-embeddings-v2-base-en"
)# # 第一次存入本地
vectorstore = Chroma.from_documents(documents, embeddings,persist_directory="./wikipedia")# # 从本地加载
# vectorstore = Chroma(persist_directory="./wikipedia", embedding_function=embeddings)retriever = vectorstore.as_retriever()
template = """Answer the question based on the context below. If the
question cannot be answered using the information provided answer
with "I don't know"     Context: {context}Question: {question}
"""
prompt = ChatPromptTemplate.from_template(template)
llm = ChatGLM.ChatGLM_LLM(verbose=False)
output_parser = StrOutputParser()
setup_and_retrieval = RunnableParallel({"context": retriever, "question": RunnablePassthrough()}
)
chain = setup_and_retrieval | prompt | llm | output_parser
# print(chain.invoke("苹果是什么"))
# print(chain.invoke("苹果有哪些功效"))
# print(chain.invoke("如何创建虚拟环境"))
print(chain.invoke("what is android"))