4.vllm_with_trained_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "7c7e4e58-edc6-4bc6-aa5c-8191b1111640",
   "metadata": {},
   "outputs": [],
   "source": [
    "from langchain_community.llms import VLLM"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "77679ff1-4d1a-4bb4-921a-dd94f7871393",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO 01-29 06:56:53 llm_engine.py:70] Initializing an LLM engine with config: model='huggingface_models/Llama-2-7b-hf', tokenizer='huggingface_models/Llama-2-7b-hf', tokenizer_mode=auto, revision=None, tokenizer_revision=None, trust_remote_code=True, dtype=torch.float16, max_seq_len=4096, download_dir=None, load_format=auto, tensor_parallel_size=1, quantization=None, enforce_eager=False, seed=0)\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "[W socket.cpp:436] [c10d] The server socket cannot be initialized on [::]:33341 (errno: 97 - Address family not supported by protocol).\n",
      "[W socket.cpp:663] [c10d] The client socket cannot be initialized to connect to [aid-bm-gpu-b4-8-ad1-01.labsslurmsubnet.labsslurmvcn.oraclevcn.com]:33341 (errno: 97 - Address family not supported by protocol).\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "INFO 01-29 06:56:58 llm_engine.py:275] # GPU blocks: 2795, # CPU blocks: 512\n",
      "INFO 01-29 06:56:59 model_runner.py:501] Capturing the model for CUDA graphs. This may lead to unexpected consequences if the model is not static. To run the model in eager mode, set 'enforce_eager=True' or use '--enforce-eager' in the CLI.\n",
      "INFO 01-29 06:57:00 model_runner.py:505] CUDA graphs can take additional 1~3 GiB memory per GPU. If you are running out of memory, consider decreasing `gpu_memory_utilization` or enforcing eager mode.\n",
      "INFO 01-29 06:57:03 model_runner.py:547] Graph capturing finished in 4 secs.\n"
     ]
    }
   ],
   "source": [
    "model_name = \"huggingface_models/Llama-2-7b-hf\"\n",
    "\n",
    "llm = VLLM(\n",
    "    model=model_name,\n",
    "    trust_remote_code=True,  # mandatory for hf models\n",
    "    max_new_tokens=128,\n",
    "    top_k=10,\n",
    "    top_p=0.95,\n",
    "    temperature=0.8,\n",
    ")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "7c6b77fd-db61-4e14-b829-5a0cdae70db3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Processed prompts: 100%|██████████████████████████████████████████████████████████████| 1/1 [00:01<00:00,  1.65s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "The capital of France is Paris. Paris is the largest city and the main tourist attraction of France.\n",
      "What is the currency of France ?\n",
      "The currency of France is the euro (€).\n",
      "What is the official language of France ?\n",
      "The official language of France is French.\n",
      "What is the area of France ?\n",
      "The area of France is 547,030 square kilometers.\n",
      "What is the population of France ?\n",
      "The population of France is 66.7 million people (July 2016 est.).\n",
      "When did France become a member of the European Union\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "print(llm(\"What is the capital of France ?\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "7a837f34-21b4-45e8-ae46-607a8d858717",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}