[LLM] Support bloom prefix (#2248)

* support bloom prefix * support_bloom_prefix * support bloom prefix * Update code for bloom prefix * update code * support bloom prefix
PaddlePaddle · Oct 19, 2023 · 66a4897 · 66a4897
1 parent 2f2c824
commit 66a4897
Showing 1 changed file with 2 additions and 2 deletions.
diff --git a/llm/fastdeploy_llm/engine.py b/llm/fastdeploy_llm/engine.py
@@ -368,11 +368,11 @@ def get_alibi_slopes(num_heads):
     inputs["attention_mask"] = (
         alibi_encoder + (1 - inputs["attention_mask"]
                          ) * paddle.finfo(inputs["attention_mask"].dtype).min)
-    attention_mask = inputs["attention_mask"]
-    tgt_generation_mask  = inputs["tgt_generation_mask"]
     inputs["tgt_generation_mask"] = (
         alibi_decoder + (1 - inputs["tgt_generation_mask"]) *
         paddle.finfo(inputs["tgt_generation_mask"].dtype).min)
+    attention_mask = inputs["attention_mask"]
+    tgt_generation_mask  = inputs["tgt_generation_mask"]
 
 
 def dy_input_preprocess(inputs):