"vscode:/vscode.git/clone" did not exist on "713b0778709ba8ba2f8e437f0bbba16d57de7137"
Commit 06be2dc3 authored by wooway777's avatar wooway777
Browse files

issue/231 - remove ninetoothed dependency by default

parent a4ced800
......@@ -63,6 +63,12 @@ python scripts/test_ppl.py --model-path MODEL_PATH [--ndev NDEV] [--max-batch MA
```
- 选择是否使用九齿计算路径,默认为false,即不依赖九齿算子
```bash
xmake f --ninetoothed= [true | false] -cv
```
- 安装 InfiniLM Python 包
```bash
pip install -e .
......
......@@ -93,26 +93,24 @@ StaticKVCache::update(size_t layer_idx,
auto device = k_cache_layer->device();
if (device.getType() == infinicore::Device::Type::NVIDIA
|| device.getType() == infinicore::Device::Type::ILUVATAR
|| device.getType() == infinicore::Device::Type::METAX) {
infinicore::op::kv_caching_(
k_cache_layer,
v_cache_layer,
k,
v,
past_sequence_lengths);
} else {
size_t cache_pos = reinterpret_cast<int64_t *>(past_sequence_lengths->to(infinicore::Device::cpu())->data())[0];
auto result_len = cache_pos + update_len;
ASSERT(result_len <= cache_len_);
auto k_cache_update = k_cache_layer->narrow({{2, cache_pos, update_len}});
auto v_cache_update = v_cache_layer->narrow({{2, cache_pos, update_len}});
k_cache_update->copy_from(k);
v_cache_update->copy_from(v);
}
#ifdef ENABLE_NINETOOTHED
infinicore::op::kv_caching_(
k_cache_layer,
v_cache_layer,
k,
v,
past_sequence_lengths);
#else
size_t cache_pos = reinterpret_cast<int64_t *>(past_sequence_lengths->to(infinicore::Device::cpu())->data())[0];
auto result_len = cache_pos + update_len;
ASSERT(result_len <= cache_len_);
auto k_cache_update = k_cache_layer->narrow({{2, cache_pos, update_len}});
auto v_cache_update = v_cache_layer->narrow({{2, cache_pos, update_len}});
k_cache_update->copy_from(k);
v_cache_update->copy_from(v);
#endif
return {k_cache_layer, v_cache_layer};
}
......
......@@ -8,6 +8,16 @@ set_toolchains("gcc")
add_includedirs("third_party/spdlog/include")
add_includedirs("third_party/json/single_include/")
option("ninetoothed")
set_default(false)
set_showmenu(true)
set_description("Whether to complie NineToothed specifc path")
option_end()
if has_config("ninetoothed") then
add_defines("ENABLE_NINETOOTHED")
end
target("infinicore_infer")
set_kind("shared")
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment