当前位置：首页 > news >正文

LORA模型与基座大模型合并并由transformer的AutoModel推理

news 来源：原创 2024/10/1 0:21:20

网上看了看很多资料都是关于Lora后的模型直接由Peft 去读取的，具体可以参考：LoRA 模型合并与保存这里就不再赘述了，大概原理就是

def merge_lora_to_LLM():model_name_or_path = "your_LLM_model_path"adapter_name_or_path = "your_lora_model_path"save_path = "save_model_path"tokenizer = AutoTokenizer.from_pretrained(model_name_or_path,trust_remote_code=True)model = AutoModelForCausalLM.from_pretrained(model_name_or_path,trust_remote_code=True,low_cpu_mem_usage=True,torch_dtype=torch.float16,device_map="auto")model = PeftModel.from_pretrained(model, adapter_name_or_path)model = model.merge_and_unload()tokenizer.save_pretrained(save_path)

这里主要阐述的是Lora与LLM合并后，可以直接由Transformer的AutoModel去加载与推理。具体代码如下：

from peft import PeftModel
from transformers import AutoModel, AutoTokenizer
import os
import shutilmodel_type = "/root/ld/ld_model_pretrained/Minicpmv2_6"  # Local model path or huggingface id
path_to_adapter = "/root/ld/ld_project/minicpmv2_6/MiniCPM-V/finetune/output/output_minicpmv2_lora/checkpoint-30"  # Path to the saved LoRA adapter
merge_path = "/root/ld/ld_project/minicpmv2_6/MiniCPM-V/finetune/output/merge_minicpmv"  # Path to save the merged model# 保证原始模型的各个文件不遗漏保存到merge_path中
def copy_files_not_in_B(A_path, B_path):"""Copies files from directory A to directory B if they exist in A but not in B.:param A_path: Path to the source directory (A).:param B_path: Path to the destination directory (B)."""# 保证路径存在if not os.path.exists(A_path):raise FileNotFoundError(f"The directory {A_path} does not exist.")if not os.path.exists(B_path):os.makedirs(B_path)# 获取路径A中所有非权重文件files_in_A = os.listdir(A_path)files_in_A = set([file for file in files_in_A if not (".bin" in file or "safetensors" in file)])# List all files in directory Bfiles_in_B = set(os.listdir(B_path))# 找到所有A中存在但B中不存在的文件files_to_copy = files_in_A - files_in_B# 将这些文件复制到B路径下for file in files_to_copy:src_file = os.path.join(A_path, file)dst_file = os.path.join(B_path, file)shutil.copy2(src_file, dst_file)# 加载原始模型
model = AutoModel.from_pretrained(model_type,trust_remote_code=True
)# 加载lora模块到原始模型中
lora_model = PeftModel.from_pretrained(model,path_to_adapter,device_map="auto",trust_remote_code=True
).eval()# 将加载的lora模块合并到原始模型中
merge_model = lora_model.merge_and_unload()# 将新合并的模型进行保存
merge_model.save_pretrained(merge_path, safe_serialization=False)# 加载分词器
tokenizer = AutoTokenizer.from_pretrained(model_type, trust_remote_code=True)
tokenizer.save_pretrained(merge_path)copy_files_not_in_B(model_type,merge_path)

之后的保存在merge_path的地址就可以直接用transformer的AutoModel加载，大致如下：

model = AutoModel.from_pretrained(merge_path, trust_remote_code=True, torch_dtype=torch.bfloat16)
model = model.to(device=device)
tokenizer = AutoTokenizer.from_pretrained(merge_path, trust_remote_code=True)
model.eval()