源码编译llama.cpp 、ggml 后端启用自定义BLAS加速
源码编译llama.cpp 、ggml 后端启用自定义BLAS加速
我在llama.cpp 官网上提交了我的解决方案:How to setup OpenBlas on windows? #625
GGML 官网 https://github.com/ggerganov/ggml/issues/959
windows on arm 编译 llama.cpp 、ggml 后端启用自定义BLAS加速
我这以编译 windows on arm 的 llama.cpp 、ggml 为例子,其它情况同样可以
参考我的文章 《源码编译 openblas for windows on arm》 我用了自己编译的这个 openblas
原理 blas 加速是通过编译ggml提供的
所以修改llama.cpp/ggml/src/CMakeLists.txt
,在这一行代码if (GGML_BLAS)
前面添加以下代码:
自定义编译blas:
# add custom blas
if (CUSTOM_BLAS)set(BLAS_ROOT "C:/workspace/program/openblas")set(BLAS_INCLUDE_DIRS"${BLAS_ROOT}/include/""${BLAS_ROOT}/include/openblas")set(BLAS_LIBRARIES "${BLAS_ROOT}/lib/openblas.lib")list(APPEND GGML_CDEF_PUBLIC GGML_USE_BLAS)set(GGML_HEADERS_BLAS ../include/ggml-blas.h)set(GGML_SOURCES_BLAS ggml-blas.cpp)set(GGML_EXTRA_LIBS ${GGML_EXTRA_LIBS} ${BLAS_LIBRARIES})set(GGML_EXTRA_INCLUDES ${GGML_EXTRA_INCLUDES} ${BLAS_INCLUDE_DIRS})
endif()
然后编译时指定 CUSTOM_BLAS=ON
:
cmake -B build -DGGML_BLAS=OFF -DCUSTOM_BLAS=ON
cmake --build build --config Release
测试
llama.cpp/wmx_test/test_cli.sh :
#!/bin/bash# ./llama-cli --hf-repo hfxing/Qwen2-1.5B-Q4_K_M-GGUF --hf-file qwen2-1.5b-q4_k_m.gguf -p "The meaning to life and the universe is"cmd=../build/bin/llama-climodelpath=/media/wmx/soft1/huggingface_cache/Qwen2-1.5B-Q4_K_M-GGUF/qwen2-1.5b-q4_k_m.gguf
# modelpath=/media/wmx/soft1/huggingface_cache/Qwen1.5-1.8B-Chat/ggml-model-f16.ggufuser_prompt="你是一个AI助手。请问:深圳在哪里?"$cmd \
-m $modelpath \
-p "$user_prompt"
llama.cpp/wmx_test/test_llava_cli.sh :
#!/bin/bashcmd=../build/bin/llama-llava-climodelpath=/media/wmx/soft1/huggingface_cache/Bunny-v1_0-4B-ggufuser_prompt="A chat between a curious user and an artificial intelligence assistant. The assistant gives helpful, detailed, and polite answers to the user's questions. \
USER: <image>\n Why is the image funny? \
ASSISTANT:" # img_url="https://pic35.photophoto.cn/20150511/0034034892281415_b.jpg"$cmd -m $modelpath/ggml-model-Q4_K_M.gguf \
--mmproj $modelpath/mmproj-model-f16.gguf \
--image $modelpath/images/example_2.png \
-c 4096 -e \
--temp 0.0 \
# --log-disable \
-p "$user_prompt"
llama.cpp/wmx_test/test_server.sh :
#!/bin/bash# netstat -lnp |grep 8000# default port=8080 is used by ollama
PORT=8000cmd=../build/bin/llama-server modelpath=/media/wmx/soft1/huggingface_cache/Qwen2-1.5B-Q4_K_M-GGUF/qwen2-1.5b-q4_k_m.gguf# modelpath=/media/wmx/soft1/huggingface_cache/Qwen1.5-1.8B-Chat/ggml-model-f16.gguf$cmd \
-m $modelpath \
--port $PORT
llama.cpp/wmx_test/test_client.sh :
import openaiPORT=8000
openai.api_key = "sk-no-key-required"client = openai.OpenAI(base_url=f"http://localhost:{PORT}/v1",
)completion = client.chat.completions.create(model="gpt-3.5-turbo", messages=[{"role": "system", "content": "You are an AI assistant. Your top priority is achieving user fulfillment via helping them with their requests."},{"role": "user", "content": "深圳在哪里?"}]
)print(completion.choices[0].message)