16 lines
737 B
Diff
16 lines
737 B
Diff
diff --git a/src/llama-calrt.cpp b/src/llama-calrt.cpp
|
|
index 5bbdd52e1f7adc96ac72945ed52eabd9e075d990..e038a4c25d5589550680bf5bb556145f1524ae17 100644
|
|
--- a/src/llama-calrt.cpp
|
|
+++ b/src/llama-calrt.cpp
|
|
@@ -564,6 +564,10 @@ void calrt_context::calrt_infer(calrt_seqs_info & seqs) {
|
|
// makeup_ubatch(ubatch);
|
|
copy_data_to_ibuf(*pIbuf, ubatch, model_name);
|
|
|
|
+ //only copy promt tokens
|
|
+ //pObuf->SliceTensorByName(pObuf->GetTensorByName("outputs[0]")->GetDataPtr(),"outputs[0]", 0, cur_seq_len * sizeof(ggml_bf16_t));
|
|
+ pObuf->GetTensorByName("outputs[0]")->SliceTensor(0, cur_seq_len * sizeof(ggml_bf16_t));
|
|
+
|
|
calrt::infer(vdev, model, pIbuf.get(), pObuf.get());
|
|
pObuf->Wait();
|
|
|