附录：自定义HF导入模型高级参数详细说明

# concat_prompt
# 变量名含义
## latest_question_template：当前问题拼接规范
## history_qa_template：历史问答拼接规范
## prompt_template: 外层Prompt模板

final_prompt = ""

round = 1
need_history_round = "{round}" in latest_question_template

# 历史回话问答内容的拼接
for i in range(0, len(messages) - 1, 2):
    history_question = messages[i]["content"]
    history_answer = messages[i + 1]["content"]
    
    if need_history_round:
        final_prompt += history_qa_template.format(round=round, question=history_question, answer=history_answer)
    else:
        final_prompt += history_qa_template.format(question=history_question, answer=history_answer)
    round += 1

# 当前问题内的拼接
latest_question = messages[-1]["content"]
if need_history_round:
    final_prompt += latest_question_template.format(round=round, question=latest_question)
else:
    final_prompt += latest_question_template.format(question=latest_question)

# 外层Prompt模板拼接
final_prompt = prompt_template.format(input=final_prompt)

print("将要喂给模型的完整prompt如下：\n%s" % final_prompt)

# 进行模型推理
# inputs = tokenizer(final_prompt)
# result = model.generate(**inputs)

当前问题拼接规范：[Round {round}]\n\n问：{question}\n\n答：
历史问答拼接规范：[Round {round}]\n\n问：{question}\n\n答：{answer}\n\n
外层Prompt模板：{input}

# 请求：
{
    "messages": [
        {"role": "user", "content": "问题1"},
        {"role": "assistant", "content": "答案1"},
        {"role": "user", "content": "问题2"}
    ]
}

# 将要喂给模型的完整prompt如下：
[Round 1]

问：问题1

答：答案1

[Round 2]

问：问题2

答：

def build_model_input(model_tokenizer, messages=None, prompt=None, **kwargs):
    """
    build model generate input with model tokenizer, input messages(对话模式)
    :param model_tokenizer: tokenizer loading from uploaded model
    :param messages: input messages whose structure is same with the `messages` param in qianfan chat api
    :param kwargs: other params for further future
    :return: 1-dim token id list
    """
    token_ids = []
    # convert input messages or prompt token ids
    return token_ids

# !/usr/bin/env python3
# -*- coding: utf-8 -*-


def build_model_input(model_tokenizer, messages=None, **kwargs):
    return model_tokenizer.build_chat_input(messages[-1]["content"], history=messages[:-1]).input_ids[0].tolist()

# !/usr/bin/env python3
# -*- coding: utf-8 -*-
from transformers.pipelines.conversational import Conversation


def build_model_input(model_tokenizer, messages=None, **kwargs):
    past_user_inputs, generated_responses = [], []
    for i in range(0, len(messages) - 1, 2):
        past_user_inputs.append(messages[i]["content"])
        generated_responses.append(messages[i + 1]["content"])
    conv = Conversation(messages[-1]["content"], past_user_inputs=past_user_inputs, generated_responses=generated_responses)
    return model_tokenizer.apply_chat_template(conv)

try:
    try:
        model = load_model_class.from_pretrained(model_path, trust_remote_code=True,
                                               torch_dtype=infer_dtype, device_map="auto")
    except RuntimeError as e:
        model = load_model_class.from_pretrained(model_path, trust_remote_code=True,
                                               torch_dtype=infer_dtype, device="cuda")
except Exception as e:
    model = load_model_class.from_pretrained(model_path, trust_remote_code=True, device="cuda")

def load_model(model_path, **kwargs):
    """
    load model with model path
    :param model_path: model path
    :param kwargs: other params for further future
    :return: instance of the model class
    """
    model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,
                                       torch_dtype=torch.float16, device_map="auto")
    return model

# !/usr/bin/env python3
# -*- coding: utf-8 -*-
from transformers import AutoModelForCausalLM
import torch

def load_model(model_path, **kwargs):
    model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True,
                                       torch_dtype=torch.float16, device_map="auto")
    return model

# !/usr/bin/env python3
# -*- coding: utf-8 -*-
from transformers import AutoModelForCausalLM, GenerationConfig
import torch

def load_model(model_path, **kwargs):
    max_memory = {i: "75GB" for i in range(8)}
    model = AutoModelForCausalLM.from_pretrained(model_path, trust_remote_code=True, device_map="sequential", torch_dtype=torch.bfloat16, max_memory=max_memory, attn_implementation="eager")
    model.generation_config = GenerationConfig.from_pretrained(model_path)
    model.generation_config.pad_token_id = model.generation_config.eos_token_id
    return model