Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Support chatglm-6b #2221

Closed
wants to merge 1 commit into from
Closed

Support chatglm-6b #2221

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 43 additions & 10 deletions llm/fastdeploy_llm/engine.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -154,18 +154,24 @@ def init_dist_env(world_size, seed=20):
tgt_generation_mask = paddle.zeros(
shape=[args.batch_size, 1, 1, args.max_seq_len + args.max_dec_len],
dtype=args.dtype)
position_ids = paddle.full(
shape=[args.batch_size, args.max_seq_len], fill_value=0, dtype='int64')
if "chatglm" in args.architecture:
attention_mask = paddle.ones(
shape=(args.batch_size, 1, args.max_seq_len, args.max_seq_len),
#TODO JiangJiajun
attention_mask = paddle.zeros(
shape=(args.batch_size, 1, args.max_seq_len + args.max_dec_len,
args.max_seq_len + args.max_dec_len),
dtype=args.dtype)
tgt_pos = paddle.ones(shape=(args.batch_size, 2, 1), dtype=args.dtype)
tgt_pos = paddle.ones(shape=(args.batch_size, 2, 1), dtype="int64")
position_ids = paddle.full(
shape=[args.batch_size, 2, args.max_seq_len],
fill_value=0,
dtype='int64')
else:
attention_mask = paddle.zeros(
shape=(args.batch_size, 1, args.max_seq_len + args.max_dec_len,
args.max_seq_len + args.max_dec_len),
dtype=args.dtype)
position_ids = paddle.full(
shape=[args.batch_size, args.max_seq_len], fill_value=0, dtype='int64')


#count = 0
Expand Down Expand Up @@ -259,20 +265,32 @@ def predict(self, batch_data_dict):
"""
predict
"""
# dump_data = dict()

for k, v in batch_data_dict.items():
input_tensor = self.predictor.get_input_handle(k)
if isinstance(v, paddle.Tensor):
input_tensor.share_external_data(v)
# dump_data[k] = v.numpy()
else:
input_tensor.copy_from_cpu(v)
# dump_data[k] = v

# cache_dumps = list()
for i in range(args.num_layers):
input_tensor = self.predictor.get_input_handle('cache_kvs_' + str(
i))
input_tensor.share_external_data(cache_kvs[i])

# cache_dumps.append(cache_kvs[i].numpy())

input_tensor = self.predictor.get_input_handle('pre_ids')
input_tensor.share_external_data(pre_ids)
# dump_data["pre_ids"] = pre_ids.numpy()
# import pickle
# with open("dump_data.pkl", "wb") as f:
# pickle.dump([dump_data, cache_dumps], f)

self.predictor.run()
# NOTE: The order of return values is refered from:
# PaddleNLP/paddlenlp/experimental/transformers/generation_utils.py
Expand All @@ -294,6 +312,9 @@ def dy_input_preprocess(inputs):
stop_flags = inputs["dyinput_flags"]
dec_length = inputs["seq_len_decoder"]
bsz = len(stop_flags)

tmp = np.zeros(shape=[args.batch_size, 2, args.max_seq_len], dtype="int64")

for i in range(bsz):
if stop_flags[i] == 1:
length = int(dec_length[i, 0])
Expand Down Expand Up @@ -323,16 +344,25 @@ def dy_input_preprocess(inputs):
shape=[1, max_prefix_len + length],
dtype=args.dtype)
else:
position_ids[i, :length] = paddle.arange(length)
attention_mask[i, 0, :length, :length] = paddle.tril(
paddle.ones(
shape=[length, length], dtype=args.dtype))
if "chatglm" in args.architecture:
attention_mask[i, 0, :length, :length] = 0
attention_mask[i, 0, :length - 1, length - 1] = 1
tgt_pos[i, 0, 0] = paddle.to_tensor(
[length], dtype="int64")
else:
position_ids[i, :length] = paddle.arange(length)
attention_mask[i, 0, :length, :length] = paddle.tril(
paddle.ones(
shape=[length, length], dtype=args.dtype))
tgt_generation_mask[i, 0, 0, :length] = paddle.ones(
shape=[1, length], dtype=args.dtype)
pre_ids[i:i + 1] = -1
del inputs["dyinput_flags"]
inputs["position_ids"] = position_ids
#TODO jiangjiajun
if "chatglm" not in args.architecture:
inputs["position_ids"] = position_ids
inputs["tgt_generation_mask"] = tgt_generation_mask
inputs["tgt_pos"] = tgt_pos
if args.is_ptuning:
prefix_caches = []
for model_id in inputs['model_id']:
Expand Down Expand Up @@ -412,6 +442,9 @@ def run(infer_engine):
flag_end_array[rank] = 1
flag_begin_array[rank] = 0

# TODO ========
break


def main():
model_dir = os.path.join(args.model_dir, f"rank_{rank}")
Expand Down
56 changes: 45 additions & 11 deletions llm/fastdeploy_llm/model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,6 +99,9 @@ def _init_share_memory(self):
for i in range(5):
output_datas[keys[i]] = np.zeros(
[self.config.max_batch_size + 1, 1], dtype=np.int64)
if "chatglm" in self.config.architecture:
output_datas["tgt_pos"] = np.zeros(
[self.config.max_batch_size + 1, 2, 1], dtype=np.int64)
output_datas_size = len(pickle.dumps(output_datas)) + 6
self.shm_output_data = shared_memory.SharedMemory(
create=True, size=output_datas_size, name="shm_infer_output_data")
Expand Down Expand Up @@ -286,7 +289,12 @@ def _update_task_results(self, tasks):
for i in range(len(tasks)):
tasks[i].decode_status["finished_id"] = int(info["finished_ids"][
i])
tasks[i].decode_status["tgt_pos"] = int(info["tgt_pos"][i])
# TODO JiangJiajun
if "chatglm" in self.config.architecture:
tasks[i].decode_status["tgt_pos"] = int(info["tgt_pos"][i][0])
else:
tasks[i].decode_status["tgt_pos"] = int(info["tgt_pos"][i])

tasks[i].decode_status["step_idx"] = int(info["step_idx"][i])
tasks[i].decode_status["seq_lens_decoder"] = int(info[
"seq_lens_decoder"][i])
Expand Down Expand Up @@ -324,8 +332,26 @@ def _prepare_inputs(self, tasks, stop_num):
texts.append(task.text)
else:
texts.append("me")

input_ids, lens = self.data_processor.batch_encode_tasks(tasks)
if hasattr(task, "token_ids"):
del task.token_ids
del task.position_ids

padding = True if "chatglm" in self.config.architecture else False
max_length = self.config.max_seq_len if "chatglm" in self.config.architecture else None
input_ids, lens, position_ids = self.data_processor.batch_encode_tasks(
tasks, padding=padding, max_length=max_length)

# TODO JiangJiajun
if "chatglm" in self.config.architecture:
inst_data_pos = list()
max_len = max(map(len, input_ids))
for i in range(len(position_ids)):
inst_data_pos.append(
np.array([
list(inst) + [0] * (max_len - len(inst))
for inst in position_ids[i]
]))
position_ids = np.array(inst_data_pos).astype("int64")

for i in range(len(tasks)):
tasks[i].prompt_token_nums = lens[i]
Expand All @@ -348,6 +374,7 @@ def _prepare_inputs(self, tasks, stop_num):
inputs["min_length"] = np.array(
[task.min_dec_len for task in tasks]).astype('int64').reshape(-1,
1)
inputs["position_ids"] = position_ids
# TODO Lite model exists different method
# TODO Doesn't support eos_token id now
inputs["eos_token_id"] = np.array(
Expand Down Expand Up @@ -391,18 +418,22 @@ def _add_dynamic_batching_inputs(self, inputs, tasks, stop_nums):
sequence_lengths_decoder[i] = self.config.max_prefix_len
else:
sequence_lengths_decoder[i] = 0
# if not tasks[i].is_pad:
# sequence_lengths_encoder[i] = 0
# else:
# sequence_lengths_encoder[i] = length
tgt_ids[i] = inputs["input_ids"][i][length - 1]
stop_flags[i] = 1
elif tasks[i].status == TaskStatus.NEW:
tgt_pos.append(length - 1)
if "chatglm" in self.config.architecture:
tgt_pos += [length, 1]
else:
tgt_pos.append(length - 1)
sequence_lengths_encoder[i] = length

if self.config.is_ptuning:
sequence_lengths_decoder[
i] = length + self.config.max_prefix_len
if not getattr(tasks[i], "is_pad", False):
sequence_lengths_decoder[
i] = length + self.config.max_prefix_len
else:
sequence_lengths_decoder[i] = 0
stop_flags[i] = 1
else:
if not getattr(tasks[i], "is_pad", False):
sequence_lengths_decoder[i] = length
Expand All @@ -412,7 +443,10 @@ def _add_dynamic_batching_inputs(self, inputs, tasks, stop_nums):
tgt_ids[i] = inputs["input_ids"][i][length - 1]
dyinput_flags[i] = 1
del inputs["num_input_tokens"]
tgt_pos = np.array(tgt_pos).astype("int64").reshape(-1, 1)
if "chatglm" in self.config.architecture:
tgt_pos = np.array(tgt_pos).astype("int64").reshape(-1, 2, 1)
else:
tgt_pos = np.array(tgt_pos).astype("int64").reshape(-1, 1)
sequence_lengths_encoder = np.array(sequence_lengths_encoder).astype(
"int32").reshape(-1, 1)
sequence_lengths_decoder = np.array(sequence_lengths_decoder).astype(
Expand Down
30 changes: 15 additions & 15 deletions llm/fastdeploy_llm/processor.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,8 @@ class DataProcessor:
def __init__(self, model_dir):
self.tokenizer = paddlenlp.transformers.AutoTokenizer.from_pretrained(
model_dir)
self.pad_token_id = self.tokenizer(
[self.tokenizer.pad_token], return_tensors="np")["input_ids"][0][0]

def decode(self, tokens):
if not isinstance(tokens, list):
Expand All @@ -58,42 +60,40 @@ def decode_token(self, all_input_ids, prefix_offset, read_offset):
return self.tokenizer.decode_token(all_input_ids, prefix_offset,
read_offset)

def encode(self, text):
def encode(self, text, padding=False, max_length=None):
tokens = self.tokenizer(
text,
return_tensors="np",
padding=False,
return_attention_mask=False,
return_token_type_ids=False, )
return tokens["input_ids"][0]
text, return_tensors="np", padding=padding, max_length=max_length)
return tokens["input_ids"][0], tokens["position_ids"][0]

def batch_encode_tasks(self, tasks):
def batch_encode_tasks(self, tasks, padding=False, max_length=None):
"""
预处理，数据都在tasks中
"""
input_ids = list()
real_tokens_len = list()
position_ids = list()
for task in tasks:
if hasattr(task, "token_ids"):
token_ids = task.token_ids
if task.status != TaskStatus.NEW:
token_ids = token_ids[:2]
input_ids.append(token_ids)
real_tokens_len.append(len(token_ids))
position_ids.append(task.position_ids)
else:
text = task.text
if task.status != TaskStatus.NEW:
text = "me"
tokens = self.tokenizer(
text,
return_tensors="np",
padding=False,
return_attention_mask=False,
return_token_type_ids=False, )
return_token_type_ids=False,
padding=padding,
max_length=max_length)
input_ids.append(tokens["input_ids"][0])
real_tokens_len.append(len(tokens["input_ids"][0]))
input_ids = pad_batch_data(input_ids)
return input_ids, real_tokens_len
position_ids.append(tokens["position_ids"][0])
input_ids, real_tokens_len = pad_batch_data(
input_ids, pad_id=self.pad_token_id, return_seq_len=True)
return input_ids, real_tokens_len.tolist(), position_ids

def batch_encode(self, texts):
"""
Expand Down
3 changes: 2 additions & 1 deletion llm/fastdeploy_llm/serving/serving_model.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,8 @@ def add_request(self, task):
assert task.text.strip() != "", "The request's text cannot be empty."
try:
if not hasattr(task, "token_ids"):
task.token_ids = self.model.data_processor.encode(task.text)
task.token_ids, task.position_ids = self.model.data_processor.encode(
task.text, padding=True)
if self.config.is_ptuning:
assert len(
task.token_ids
Expand Down