We have remove the pad_token for llama2

[hijkzzz]

See MR
You should convert the previous SFT/RM checkpoint with the following script:

import argparse

import torch
from openrlhf.utils import get_strategy, get_tokenizer

from openrlhf.models import Actor, RewardModel

def convert(args):
    # configure strategy
    strategy = get_strategy(args)
    strategy.setup_distributed()

    # configure model
    from_config = bool(args.load_model)
    model = RewardModel(args.pretrain, from_config)
    # model = Actor(args.pretrain, from_config)

    # load Pytorch model
    if args.load_model:
        strategy.print("Load model: ", args.load_model)
        model.model.resize_token_embeddings(32001)
        strategy.load_model(model, args.load_model)

    # configure tokenizer
    model.model.resize_token_embeddings(32000)

    # prepare models
    model = strategy.prepare(model)
    strategy.save_model(model, args.load_model + '_nopad', only_rank0=True)


if __name__ == '__main__':
    parser = argparse.ArgumentParser()
    parser.add_argument('--pretrain', type=str, default="meta-llama/Llama-2-13b-chat-hf")
    parser.add_argument('--load_model', type=str, default=None)
    parser.add_argument('--zero_stage', type=int, default=0)
    parser.add_argument('--local_rank', type=int, default=-1, help="local_rank for deepspeed")
    parser.add_argument('--bf16', action='store_false', default=True)
    parser.add_argument('--inference_tp_size', type=int, default=1)
    args = parser.parse_args()
    convert(args)

CUDA_VISIBLE_DEVICES=0 ~/.local/bin/deepspeed xxx.py --load_model ./test_scripts/ckpt/7b_llama/rm_model_anthropic_oasst_lmsys_webgpt.pt