話題のMistral 7B Instruct を試す

!pip install git+https://github.com/huggingface/transformers -Uqq
!pip install accelerate sentencepiece bitsandbytes -Uqq

# テキストが見やすいようにwrapしておく
from IPython.display import HTML, display

def set_css():
  display(HTML('''
  <style>
    pre {
        white-space: pre-wrap;
    }
  </style>
  '''))
get_ipython().events.register('pre_run_cell', set_css)

import torch
from transformers import AutoTokenizer, AutoModelForCausalLM, BitsAndBytesConfig

quantization_config = BitsAndBytesConfig(
    load_in_4bit=True,
    bnb_4bit_use_double_quant=True,
    bnb_4bit_quant_type="nf4",
    bnb_4bit_compute_dtype=torch.bfloat16,
)

model_id = "mistralai/Mistral-7B-Instruct-v0.1"
tokenizer = AutoTokenizer.from_pretrained(model_id, use_fast=True)
model = AutoModelForCausalLM.from_pretrained(
    model_id,
    trust_remote_code=True,
    quantization_config=quantization_config,
    # torch_dtype=torch.float16,
    device_map='auto',
).eval()

# トークナイザーのサイズを確認。
tokenizer.vocab_size

# 本モデルのインプットのフォーマット
text = "<s>[INST] What is your favourite condiment? [/INST]"
"Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!</s> "
"[INST] Do you have mayonnaise recipes? [/INST]"

messages = [
    {"role": "user", "content": "What is your favourite condiment?"},
    {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"},
    {"role": "user", "content": "Do you have mayonnaise recipes?"}
]

encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt")
encodeds

Using sep_token, but it is not set yet.
Using pad_token, but it is not set yet.
Using cls_token, but it is not set yet.
Using mask_token, but it is not set yet.


tensor([[    1, 28792, 16289, 28793,  1824,   349,   574, 16020,  2076,  2487,
         28804,   733, 28748, 16289, 28793,  6824, 28725,   315, 28742, 28719,
          3448, 10473,   298,   264,  1179, 11322, 19961,   302,  6138, 23598,
         18342, 28723,   661, 13633,   776,   272,  1103,  3558,   302,   686,
         16944, 15637,   423,   298,  5681,   315, 28742, 28719, 13198,   582,
           297,   272,  6132, 28808,   700, 28713, 28767,   733, 16289, 28793,
          2378,   368,   506,   993,  7136,   864, 21116, 28804,   733, 28748,
         16289, 28793]])

messages = [
    {"role": "user", "content": "What is your favourite condiment?"},
    {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"},
    {"role": "user", "content": "Do you have mayonnaise recipes?"}
]

encodeds = tokenizer.apply_chat_template(messages, return_tensors="pt").to(model.device)

generated_ids = model.generate(encodeds, max_new_tokens=1000, do_sample=True)
decoded = tokenizer.batch_decode(generated_ids)
print(decoded[0])

Using sep_token, but it is not set yet.
Using pad_token, but it is not set yet.
Using cls_token, but it is not set yet.
Using mask_token, but it is not set yet.
The attention mask and the pad token id were not set. As a consequence, you may observe unexpected behavior. Please pass your input's `attention_mask` to obtain reliable results.
Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.


<s>[INST] What is your favourite condiment? [/INST]Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!</s> [INST] Do you have mayonnaise recipes? [/INST] Certainly! Here's a simple recipe for homemade mayonnaise that you can easily whip up in no time:

Ingredients:

* 2 large egg yolks
* 1 tablespoon Dijon mustard
* 1 tablespoon white wine vinegar
* 1/4 teaspoon salt
* 1/4 cup olive oil
* 1 teaspoon paprika (optional)

Instructions:

1. In a small bowl, whisk together the egg yolks, mustard, vinegar, and salt until well combined.
2. Slowly pour in the olive oil, whisking constantly as you do so.
3. Continue whisking until the mayonnaise has thickened and the colour has turned a pale yellow.
4. If you like, add a little paprika to give the mayonnaise a nice red hue.
5. Serve immediately or store in an airtight container in the refrigerator for up to a week.

I hope you enjoy this recipe! Let me know if you have any other questions.</s>

def format_input(prompt: str = "", system_prompt: str = "下記の質問に日本語で答えてください。") -> str:
    return f"<s>[INST]{system_prompt}[/INST]{prompt}</s>"

# test
format_input("日本の特徴を教えてください。")

'<s>[INST]下記の質問に日本語で答えてください。[/INST]日本の特徴を教えてください。</s>'

text = """
日本という国は、
""".strip()
text = format_input(text, "下記の文章を続けてください。")
inputs = tokenizer(text, add_special_tokens=False, return_tensors='pt').to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=100,
        do_sample=True,
        temperature=0.1,
        top_p=0.95,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

text = """
USER: りんごが5つあります。そこから2つのりんごを取り除きました。残りのりんごの数は何個でしょう？
ASSISTANT:
""".strip()
text = format_input(text)
inputs = tokenizer(text, add_special_tokens=False, return_tensors='pt')

with torch.no_grad():
    output_ids = model.generate(
        inputs['input_ids'].to(model.device),
        max_new_tokens=100,
        do_sample=True,
        temperature=0.1,
        top_p=0.95,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

text = """
質問: バットとボールの両方を買うと1100円です。バットはボールよりも1000円高いです。ボールはいくらでしょう？
回答: 一歩ずつ考えましょう。
""".strip()
text = format_input(text)
inputs = tokenizer(text, add_special_tokens=False, return_tensors='pt').to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=512,
        do_sample=True,
        temperature=0.01,
        top_p=0.95,
        pad_token_id=tokenizer.eos_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

text = """
USER: 引数kを取り、返り値としてフィボナッチ数列におけるk個目の値を返すPython関数を書いてください。
ASSISTANT:
""".strip()
text = format_input(text)
inputs = tokenizer(text, add_special_tokens=False, return_tensors='pt').to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=512,
        do_sample=True,
        temperature=0.1,
        top_p=0.95,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

Setting `pad_token_id` to `eos_token_id`:2 for open-end generation.


<s>[INST]下記の質問に日本語で答えてください。[/INST]USER: 引数kを取り、返り値としてフィボナッチ数列におけるk個目の値を返すPython関数を書いてください。
ASSISTANT:</s> 以下は、引数kを取り、返り値としてフィボナッチ数列におけるk個目の値を返すPython関数を書いています。
```python
def fibonacci(k):
    if k <= 0:
        return "Invalid input. Please enter a positive integer."
    elif k == 1:
        return 0
    elif k == 2:
        return 1
    else:
        a, b = 0, 1
        for i in range(2, k+1):
            c = a + b
            a, b = b, c
        return b
```</s>

text = """
USER: 次の内容を日本語に訳してください。"There were 3 apples and 2 oranges. How many fruits were there in total?"
ASSISTANT:
""".strip()

inputs = tokenizer(text, add_special_tokens=False, return_tensors='pt').to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=100,
        do_sample=True,
        temperature=0.1,
        top_p=0.9,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

text = """
USER: 大規模言語モデルについて説明してください。
ASSISTANT:
""".strip()
text = format_input(text)
inputs = tokenizer(text, add_special_tokens=False, return_tensors="pt").to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=200,
        do_sample=True,
        temperature=0.2,
        top_p=0.95,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

text = """
``
ゲスト: こちらの部屋はプロジェクターの備え付けはありますか？
ホスト: ご質問ありがとうございます。ご質問内容を確認しますので少々お待ち下さい。
ゲスト: 本件、その後いかがでしょうか？
``
""".strip()
text = format_input(text, "下記のシナリオにおいて、ゲストはホストからの返答を待っていますか？TrueかFalseで答えなさい。")
inputs = tokenizer(text, add_special_tokens=False, return_tensors="pt").to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=200,
        do_sample=True,
        temperature=0.01,
        top_p=0.95,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

text = """
What are the differences in training methods between Pre-training,  SFT training, Instruction training, PPO training, RHLF training in the context of LLMs?
""".strip()
text = format_input(text, "Answer the following question.")
inputs = tokenizer(text, add_special_tokens=False, return_tensors='pt').to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=2048,
        do_sample=True,
        temperature=0.1,
        top_p=0.9,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

text = """
What is the best strategy to advance your career as a software developer?
""".strip()
text = format_input(text, "Answer the following question.")
inputs = tokenizer(text, add_special_tokens=False, return_tensors='pt').to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=2048,
        do_sample=True,
        temperature=0.1,
        top_p=0.9,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

text = """
``
guest: Does the room have a projector?
host: Thank you for your interest. I will check if it does and get back to you shortly.
guest: Any updates?
``
""".strip()
text = format_input(text, "In the following scenario, is the guest waiting for a reply from the host? Answer in True or False.")
inputs = tokenizer(text, add_special_tokens=False, return_tensors="pt").to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        **inputs,
        max_new_tokens=200,
        do_sample=True,
        temperature=0.01,
        top_p=0.95,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

messages = [
    {"role": "user", "content": "JavaScriptについて勉強したいのですが何から始めればいいですか？"},
    # {"role": "assistant", "content": "Well, I'm quite partial to a good squeeze of fresh lemon juice. It adds just the right amount of zesty flavour to whatever I'm cooking up in the kitchen!"},
    # {"role": "user", "content": "Do you have mayonnaise recipes?"}
]
inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        inputs,
        max_new_tokens=2048,
        do_sample=True,
        temperature=0.1,
        top_p=0.9,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)

messages = [
    {"role": "user", "content": "JavaScriptについて勉強したいのですが何から始めればいいですか？"},
    {"role": "assistant", "content": "現状のプログラミング経験はどのくらいありますか？"},
    {"role": "user", "content": "これまでPythonでの機械学習関連のプログラミングであれば5年ほど経験があります。ただ、web関連のプログラミングはほとんど経験がありません。"}
]
inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to(model.device)

with torch.no_grad():
    output_ids = model.generate(
        inputs,
        max_new_tokens=2048,
        do_sample=True,
        temperature=0.1,
        top_p=0.9,
        pad_token_id=tokenizer.pad_token_id,
        bos_token_id=tokenizer.bos_token_id,
        eos_token_id=tokenizer.eos_token_id,
        repetition_penalty=1.1,
    )

output = tokenizer.decode(output_ids.tolist()[0])
print(output)