7.fine-tune_on_structured_json-Mixtral.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "cd58e8d9-762a-483d-a152-a0602b40b04a",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "os.environ[\"CUDA_VISIBLE_DEVICES\"] = \"0\"\n",
    "os.environ[\"TOKENIZERS_PARALLELISM\"] = \"false\"\n",
    "os.environ[\"WANDB_DISABLED\"] = \"true\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "5775aae9-aa10-49ab-8f1b-d95f429e186e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[2024-02-27 13:46:15,215] [INFO] [real_accelerator.py:191:get_accelerator] Setting ds_accelerator to cuda (auto detect)\n"
     ]
    }
   ],
   "source": [
    "from trl import SFTTrainer\n",
    "from transformers import (AutoModelForCausalLM, \n",
    "                          AutoTokenizer, \n",
    "                          BitsAndBytesConfig, \n",
    "                          TrainingArguments, \n",
    "                          pipeline, \n",
    "                          logging)\n",
    "import pandas as pd\n",
    "import json\n",
    "import torch\n",
    "from tqdm import tqdm\n",
    "from datasets import load_dataset\n",
    "from peft import LoraConfig, PeftConfig\n",
    "from datasets import Dataset\n",
    "import copy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "edbeb4ed-ab61-4c29-9a67-38badb3981dc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "device = 0 if torch.cuda.is_available() else -1  # 0 for GPU, -1 for CPU\n",
    "device"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "2b2ad7e7-9649-4084-a7a5-237bca0b72bd",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_filename = \"dataset/ace/train.txt\"\n",
    "val_filename = \"dataset/ace/dev.txt\"\n",
    "test_filename = \"dataset/ace/test.txt\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c8fd1c98-5014-435a-8c8a-77cd710a7312",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_data(file_path):\n",
    "    with open(file_path, 'r') as file:\n",
    "        content = file.read()\n",
    "\n",
    "    # Split content into samples based on two newlines\n",
    "    samples = content.strip().split('\\n\\n')\n",
    "\n",
    "    # Create a list to store token-label pairs and sample IDs\n",
    "    token_label_pairs = []\n",
    "    sample_ids = []\n",
    "\n",
    "    # Iterate through samples\n",
    "    for sample in samples:\n",
    "        lines = sample.split('\\n')\n",
    "        for line in lines:\n",
    "            # Split each line into token and label\n",
    "            token, label = line.split(' ')\n",
    "            token_label_pairs.append((token, label))\n",
    "\n",
    "        # Add the sample ID for each token in the current sample\n",
    "        sample_ids.extend([len(sample_ids)] * len(lines))\n",
    "\n",
    "    # Create a DataFrame\n",
    "    df = pd.DataFrame(token_label_pairs, columns=['Token', 'Label'])\n",
    "\n",
    "    # Add a column for sample ID\n",
    "    df['Sample'] = sample_ids\n",
    "\n",
    "    # Reset the index\n",
    "    df.reset_index(drop=True, inplace=True)\n",
    "\n",
    "    return df\n",
    "\n",
    "def get_unique_labels(dataframe):\n",
    "    # Create a set to store unique labels\n",
    "    unique_labels = set()\n",
    "\n",
    "    # Iterate through the \"Label\" column in the DataFrame\n",
    "    for label in dataframe['Label']:\n",
    "        # Extract the base label (ignoring the prefix B- or I-)\n",
    "        base_label = label[2:] if label.startswith('B-') or label.startswith('I-') else label\n",
    "\n",
    "        # Add the base label to the set\n",
    "        unique_labels.add(base_label)\n",
    "\n",
    "    return list(unique_labels)\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "c8df01fc-c140-4028-b89c-689358f2b665",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_df = load_data(train_filename)\n",
    "val_df = load_data(val_filename)\n",
    "test_df = load_data(test_filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "12724919-65da-466a-86e7-9832447f5b8f",
   "metadata": {},
   "outputs": [],
   "source": [
    "unique_labels = get_unique_labels(train_df)\n",
    "unique_labels.remove(\"O\")\n",
    "entity_dict = {label: [] for label in unique_labels}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "fda833ed-7329-4fda-bd29-e92b424f9182",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_instruction(input_sentence,output):\n",
    "    instruction = f\"\"\"\n",
    "Below is an instruction that describes a task. Write a response that appropriately completes the request.\n",
    "\n",
    "### Instruction:\n",
    "Your task is a Named Entity Recognition (NER) task. Given a text , predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON payload. If no entity present then output will be empty json schema. \n",
    "Here are the entity labels with empty list. \n",
    "{json.dumps(entity_dict)}\n",
    "\n",
    "Below is an example of text and the corresponding entities in json format.\n",
    "\n",
    "### Input: \n",
    "{input_sentence}\\n\n",
    "### Output: \n",
    "{output}\\n\n",
    "### End\"\"\"\n",
    "    return instruction"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "ed35059b-246d-495b-8339-275de716ef98",
   "metadata": {},
   "outputs": [],
   "source": [
    "test_ins = \"Below is the text enclosed in square brackets and find corresponding entities from the above labels in json output.\"\n",
    "\n",
    "INSTRUCTION = f\"\"\"\n",
    "Your task is a Named Entity Recognition (NER) task. Given a text, predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON payload. If no entity present then output will be empty json schema. \n",
    "Here are the entity labels with empty list. \n",
    "{json.dumps(entity_dict)}\n",
    "\n",
    "Below is the text enclosed in square brackets and the corresponding entities in json format.\n",
    "\"\"\"\n",
    "\n",
    "TEST_INSTRUCTION = f\"\"\"\n",
    "Your task is a Named Entity Recognition (NER) task. Given a text, predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON format. If no entity present then output will be empty json schema. \n",
    "Here are the entity labels with empty list. \n",
    "{json.dumps(entity_dict)}\n",
    "\n",
    "Below is the text enclosed in square brackets and the corresponding entities in json format.\n",
    "\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "ed723c8a-d528-444a-b780-7fe8d9abccbc",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'Subject.Race': [],\n",
       " 'Subject.Age': [],\n",
       " 'Potential_therapeutic_event.Trigger': [],\n",
       " 'Time_elapsed': [],\n",
       " 'Duration': [],\n",
       " 'Treatment.Freq': [],\n",
       " 'Route': [],\n",
       " 'Gender': [],\n",
       " 'Drug': [],\n",
       " 'Subject.Sub_Disorder': [],\n",
       " 'Effect': [],\n",
       " 'Treatment.Drug': [],\n",
       " 'Adverse_event.Trigger': [],\n",
       " 'Treat_Disorder': [],\n",
       " 'Treatment.Route': [],\n",
       " 'Dosage': [],\n",
       " 'Freq': [],\n",
       " 'Treatment.Combination.Drug': [],\n",
       " 'Sub_Disorder': [],\n",
       " 'Subject': [],\n",
       " 'Race': [],\n",
       " 'Subject.Gender': [],\n",
       " 'Population': [],\n",
       " 'Subject.Population': [],\n",
       " 'Treatment.Time_elapsed': [],\n",
       " 'Treatment.Treat_Disorder': [],\n",
       " 'Combination.Drug': [],\n",
       " 'Treatment.Dosage': [],\n",
       " 'Treatment.Duration': [],\n",
       " 'Treatment': []}"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "entity_dict"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "e45ef06b-4d6f-40c4-a0c6-6b5e85c4e683",
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_instruction_samples(dataframe, entity_dict ,data_type=\"train\"):\n",
    "    instruction_samples = []\n",
    "    output_dicts = []\n",
    "\n",
    "    for sample_id, group in dataframe.groupby(\"Sample\"):\n",
    "        input_sentence = \" \".join(group[\"Token\"])\n",
    "        \n",
    "        output_dict = {label: [] for label in entity_dict}\n",
    "\n",
    "        for _, row in group.iterrows():\n",
    "            label = row[\"Label\"]\n",
    "            token = row[\"Token\"]\n",
    "\n",
    "            # Extract the base label (ignoring the prefix B- or I-)\n",
    "            base_label = label[2:] if label.startswith('B-') or label.startswith('I-') else label\n",
    "\n",
    "            # Categorize the token based on the base label\n",
    "            if base_label in output_dict:\n",
    "                output_dict[base_label].append(token)\n",
    "        \n",
    "        if data_type ==\"train\":\n",
    "            instruction = f\"\"\"{INSTRUCTION}\\n[{input_sentence}] = {json.dumps(output_dict)}\"\"\"\n",
    "        else :\n",
    "            output = \"{\"\n",
    "            instruction = f\"\"\"{TEST_INSTRUCTION}\\n[{input_sentence}] = {output}\"\"\"\n",
    "        # if data_type ==\"train\":\n",
    "        #     instruction = get_instruction(input_sentence,output_dict)\n",
    "        # else:\n",
    "        #     instruction = get_instruction(input_sentence,\"{\")\n",
    "        instruction_samples.append(instruction)\n",
    "        output_dicts.append(output_dict)\n",
    "\n",
    "    # Create a DataFrame with a single column \"text\"\n",
    "    df_instruction = pd.DataFrame({\"text\": instruction_samples})\n",
    "\n",
    "    return df_instruction,output_dicts"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "88b959a6-f96c-405a-8455-c20278d19d61",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "e3f4a673-0432-4054-8912-b6038062dc96",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_instruction_df,train_outputs = create_instruction_samples(train_df,entity_dict)\n",
    "val_instruction_df,val_outputs = create_instruction_samples(val_df,entity_dict)\n",
    "test_instruction_df,test_outputs = create_instruction_samples(test_df,entity_dict,data_type=\"test\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "975c5526-d4ab-4bff-9213-62cbc14b345e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3006, 1)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_instruction_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "334f2ca3-fa3f-4d63-9cb1-21f3026abaf3",
   "metadata": {},
   "outputs": [],
   "source": [
    "# train_instruction_df = train_instruction_df.sample(n=500)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "26a5aab0-2a75-4d61-b5ae-e00e6b5b0dca",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3006, 1)"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_instruction_df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "60c77c03-5d0a-4938-8d9d-42dcbfffc0b9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nYour task is a Named Entity Recognition (NER) task. Given a text, predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON format. If no entity present then output will be empty json schema. \\nHere are the entity labels with empty list. \\n{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [], \"Treatment.Drug\": [], \"Adverse_event.Trigger\": [], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\\n\\nBelow is the text enclosed in square brackets and the corresponding entities in json format.\\n\\n[After therapy with parenteral amiodarone ( 2300 mg in 3 days ) and other measures , signs of congestive heart failure disappeared ; subsequently the patient developed jaundice , marked increase in serum transaminase levels and fall in prothrombin time , and histologic changes of severe centrilobular necrosis were observed in hepatic biopsy .] = {'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_instruction_df.iloc[0]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "a2369c38-b9e9-446c-9f02-4ec717251df9",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nYour task is a Named Entity Recognition (NER) task. Given a text, predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON payload. If no entity present then output will be empty json schema. \\nHere are the entity labels with empty list. \\n{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [], \"Treatment.Drug\": [], \"Adverse_event.Trigger\": [], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\\n\\nBelow is the text enclosed in square brackets and the corresponding entities in json format.\\n\\n[OBJECTIVE : To test the hypothesis that tumor necrosis factor ( TNF) - alpha may mediate the loss and the dedifferentiation of subcutaneous fat tissue in the insulin - induced lipoatrophies of a diabetic patient who presented extensive lesions .] = {\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\"lipoatrophies\"], \"Treatment.Drug\": [\"insulin\"], \"Adverse_event.Trigger\": [\"induced\"], \"Treat_Disorder\": [\"diabetic\"], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_instruction_df.iloc[0]['text']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "3d44619c-b385-435e-a6e9-a0f5b6a5d2f6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Your task is a Named Entity Recognition (NER) task. Given a text, predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON payload. If no entity present then output will be empty json schema. \n",
      "Here are the entity labels with empty list. \n",
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [], \"Treatment.Drug\": [], \"Adverse_event.Trigger\": [], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\n",
      "\n",
      "Below is the text enclosed in square brackets and the corresponding entities in json format.\n",
      "\n",
      "[OBJECTIVE : To test the hypothesis that tumor necrosis factor ( TNF) - alpha may mediate the loss and the dedifferentiation of subcutaneous fat tissue in the insulin - induced lipoatrophies of a diabetic patient who presented extensive lesions .] = {\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\"lipoatrophies\"], \"Treatment.Drug\": [\"insulin\"], \"Adverse_event.Trigger\": [\"induced\"], \"Treat_Disorder\": [\"diabetic\"], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\n"
     ]
    }
   ],
   "source": [
    "print(train_instruction_df.iloc[0]['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "5d6bc33a-dd64-488f-bad6-60531b552140",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "Your task is a Named Entity Recognition (NER) task. Given a text, predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON format. If no entity present then output will be empty json schema. \n",
      "Here are the entity labels with empty list. \n",
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [], \"Treatment.Drug\": [], \"Adverse_event.Trigger\": [], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\n",
      "\n",
      "Below is the text enclosed in square brackets and the corresponding entities in json format.\n",
      "\n",
      "[After therapy with parenteral amiodarone ( 2300 mg in 3 days ) and other measures , signs of congestive heart failure disappeared ; subsequently the patient developed jaundice , marked increase in serum transaminase levels and fall in prothrombin time , and histologic changes of severe centrilobular necrosis were observed in hepatic biopsy .] = {\n"
     ]
    }
   ],
   "source": [
    "print(test_instruction_df.iloc[0]['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "a2b916a0-0e2a-409b-b898-fb45582b175a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nYour task is a Named Entity Recognition (NER) task. Given a text, predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON format. If no entity present then output will be empty json schema. \\nHere are the entity labels with empty list. \\n{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [], \"Treatment.Drug\": [], \"Adverse_event.Trigger\": [], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\\n\\nBelow is the text enclosed in square brackets and the corresponding entities in json format.\\n\\n[After therapy with parenteral amiodarone ( 2300 mg in 3 days ) and other measures , signs of congestive heart failure disappeared ; subsequently the patient developed jaundice , marked increase in serum transaminase levels and fall in prothrombin time , and histologic changes of severe centrilobular necrosis were observed in hepatic biopsy .] = {'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test_instruction_df.head()[\"text\"][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "c8bdc3f6-7e46-4ec4-9974-e9a2488272b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "train_data = Dataset.from_pandas(train_instruction_df)\n",
    "eval_data = Dataset.from_pandas(val_instruction_df)\n",
    "test_data = Dataset.from_pandas(test_instruction_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "2e733dc8-8af0-4a8a-a4de-a69653594b35",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "02c1e9eb-70bb-4800-9251-334b53e422cd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nYour task is a Named Entity Recognition (NER) task. Given a text, predict the Label of each entity, then place the entity into the list associated with the Label in an output JSON payload. If no entity present then output will be empty json schema. \\nHere are the entity labels with empty list. \\n{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [], \"Treatment.Drug\": [], \"Adverse_event.Trigger\": [], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\\n\\nBelow is the text enclosed in square brackets and the corresponding entities in json format.\\n\\n[OBJECTIVE : To test the hypothesis that tumor necrosis factor ( TNF) - alpha may mediate the loss and the dedifferentiation of subcutaneous fat tissue in the insulin - induced lipoatrophies of a diabetic patient who presented extensive lesions .] = {\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\"lipoatrophies\"], \"Treatment.Drug\": [\"insulin\"], \"Adverse_event.Trigger\": [\"induced\"], \"Treat_Disorder\": [\"diabetic\"], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}'"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_data['text'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "d3efa952-22d5-442f-a377-3f3f65c28f87",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93fac566d8e64bacb1b14554593efb07",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Loading checkpoint shards:   0%|          | 0/19 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "# Load the tokenizer and model\n",
    "model_name = \"/home/jomondal/experiments/mywork/pretrained_models/Mixtral-8x7B-Instruct-v0.1\"\n",
    "trained_model_name = f\"/home/jomondal/experiments/mywork/llama_exp/trained_models/json_{model_name.split('/')[-1]}\"\n",
    "\n",
    "compute_dtype = getattr(torch, \"float16\")\n",
    "\n",
    "bnb_config = BitsAndBytesConfig(\n",
    "    load_in_4bit=True,\n",
    "    bnb_4bit_quant_type=\"nf4\",\n",
    "    bnb_4bit_compute_dtype=compute_dtype,\n",
    "    bnb_4bit_use_double_quant=False,\n",
    ")\n",
    "\n",
    "model = AutoModelForCausalLM.from_pretrained(\n",
    "    model_name,\n",
    "    device_map=\"auto\",\n",
    "    quantization_config=bnb_config, \n",
    ")\n",
    "\n",
    "model.config.use_cache = False\n",
    "model.config.pretraining_tp = 1\n",
    "\n",
    "tokenizer = AutoTokenizer.from_pretrained(model_name, \n",
    "                                          trust_remote_code=True,\n",
    "                                         )\n",
    "tokenizer.pad_token = tokenizer.eos_token\n",
    "tokenizer.padding_side = \"right\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "23a5a807-9765-4d27-8ba5-c47f9e874c83",
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_text_after_equals_with_braces(input_string):\n",
    "    equal_index = input_string.find('=')\n",
    "    # print(input_string[equal_index:])\n",
    "    brace_index = input_string.find('{', equal_index)\n",
    "    end_brace_index = input_string.find('}', brace_index)\n",
    "    return input_string[brace_index:end_brace_index + 1]\n",
    "\n",
    "def predict(data, model, tokenizer):\n",
    "    y_pred = []\n",
    "    pipe = pipeline(task=\"text-generation\", \n",
    "                        model=model, \n",
    "                        tokenizer=tokenizer, \n",
    "                        max_new_tokens = 500, \n",
    "                        temperature = 0.001,\n",
    "                       )\n",
    "    texts = data['text']\n",
    "    \n",
    "    for i in tqdm(range(len(texts))):\n",
    "        \n",
    "        prompt = texts[i]\n",
    "        # print(prompt)\n",
    "        result = pipe(prompt,pad_token_id=tokenizer.eos_token_id)\n",
    "        # print(\"Result:\",result)\n",
    "        answer = result[0]['generated_text']\n",
    "        # print(\"Answer:\",answer)\n",
    "        answer = answer.replace('\"\"','\"')\n",
    "        # print(\"Removed double quotes:\",answer)\n",
    "        answer = extract_text_after_equals_with_braces(answer)\n",
    "        # answer = answer.replace('\\n','')\n",
    "        # print(answer)\n",
    "        try:\n",
    "            answer = json.loads(answer)\n",
    "        except Exception as e:\n",
    "            print(\"Result:\",result)\n",
    "            print(\"Answer:\",answer)\n",
    "            answer = entity_dict\n",
    "        y_pred.append(answer)\n",
    "    return y_pred"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "25695667-bd60-45c0-a93e-36259a9bc51b",
   "metadata": {},
   "outputs": [],
   "source": [
    "# def evaluate(gold,pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "209c16c1-0abb-41f5-b340-a1490fa55721",
   "metadata": {},
   "outputs": [],
   "source": [
    "# y_pred = predict(test_data, model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "bb9edf04-86af-4ec5-97c3-34b2db0a31d1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "164d0ebb152f420f88578c0fdc2979bd",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/3006 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "5f778ff5dd394eb297d34db2d528fc01",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Map:   0%|          | 0/1003 [00:00<?, ? examples/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Detected kernel version 4.18.0, which is below the recommended minimum of 5.5.0; this can cause the process to hang. It is recommended to upgrade the kernel to the minimum version or higher.\n"
     ]
    }
   ],
   "source": [
    "peft_config = LoraConfig(\n",
    "    lora_alpha=16,\n",
    "    lora_dropout=0.1,\n",
    "    r=64,\n",
    "    bias=\"none\",\n",
    "    target_modules=[\"q_proj\", \"o_proj\", \"k_proj\", \"v_proj\", \"gate_proj\", \"up_proj\", \"down_proj\"],\n",
    "    task_type=\"CAUSAL_LM\",\n",
    ")\n",
    "\n",
    "training_arguments = TrainingArguments(\n",
    "    output_dir=\"logs\",\n",
    "    num_train_epochs=3,\n",
    "    per_device_train_batch_size=4,\n",
    "    gradient_accumulation_steps=4, # 4\n",
    "    gradient_checkpointing=True,\n",
    "    optim=\"paged_adamw_32bit\",\n",
    "    save_steps=0,\n",
    "    logging_steps=25,\n",
    "    learning_rate=2e-4,\n",
    "    weight_decay=0.001,\n",
    "    fp16=True,\n",
    "    bf16=False,\n",
    "    max_grad_norm=0.3,\n",
    "    max_steps=-1,\n",
    "    warmup_ratio=0.03,\n",
    "    group_by_length=True,\n",
    "    lr_scheduler_type=\"cosine\",\n",
    "    # lr_scheduler_type=\"constant\",\n",
    "    report_to=\"tensorboard\",\n",
    "    evaluation_strategy=\"epoch\"\n",
    ")\n",
    "\n",
    "trainer = SFTTrainer(\n",
    "    model=model,\n",
    "    train_dataset=train_data,\n",
    "    eval_dataset=eval_data,\n",
    "    peft_config=peft_config,\n",
    "    dataset_text_field=\"text\",\n",
    "    tokenizer=tokenizer,\n",
    "    args=training_arguments,\n",
    "    packing=False,\n",
    "    max_seq_length=1024,\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "86e58e4a-1ef6-4af4-8393-91922b76b144",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jomondal/experiments/mywork/venv/lib/python3.8/site-packages/torch/utils/checkpoint.py:429: UserWarning: torch.utils.checkpoint: please pass in use_reentrant=True or use_reentrant=False explicitly. The default value of use_reentrant will be updated to be False in the future. To maintain current behavior, pass use_reentrant=True. It is recommended that you use use_reentrant=False. Refer to docs for more details on the differences between the two variants.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "\n",
       "    <div>\n",
       "      \n",
       "      <progress value='564' max='564' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
       "      [564/564 2:24:07, Epoch 3/3]\n",
       "    </div>\n",
       "    <table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       " <tr style=\"text-align: left;\">\n",
       "      <th>Epoch</th>\n",
       "      <th>Training Loss</th>\n",
       "      <th>Validation Loss</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <td>1</td>\n",
       "      <td>0.107100</td>\n",
       "      <td>0.117440</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>2</td>\n",
       "      <td>0.087000</td>\n",
       "      <td>0.113697</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <td>3</td>\n",
       "      <td>0.102600</td>\n",
       "      <td>0.112729</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table><p>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "TrainOutput(global_step=564, training_loss=0.14197586741007812, metrics={'train_runtime': 8664.0757, 'train_samples_per_second': 1.041, 'train_steps_per_second': 0.065, 'total_flos': 1.5713731461422776e+18, 'train_loss': 0.14197586741007812, 'epoch': 3.0})"
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Train model\n",
    "trainer.train()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "e51f5886-47d2-403d-8959-88b764c00db4",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/jomondal/experiments/mywork/venv/lib/python3.8/site-packages/peft/utils/save_and_load.py:148: UserWarning: Could not find a config file in /home/jomondal/experiments/mywork/pretrained_models/Mixtral-8x7B-Instruct-v0.1 - will assume that the vocabulary was not modified.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "('/home/jomondal/experiments/mywork/llama_exp/trained_models/json_Mixtral-8x7B-Instruct-v0.1/tokenizer_config.json',\n",
       " '/home/jomondal/experiments/mywork/llama_exp/trained_models/json_Mixtral-8x7B-Instruct-v0.1/special_tokens_map.json',\n",
       " '/home/jomondal/experiments/mywork/llama_exp/trained_models/json_Mixtral-8x7B-Instruct-v0.1/tokenizer.model',\n",
       " '/home/jomondal/experiments/mywork/llama_exp/trained_models/json_Mixtral-8x7B-Instruct-v0.1/added_tokens.json',\n",
       " '/home/jomondal/experiments/mywork/llama_exp/trained_models/json_Mixtral-8x7B-Instruct-v0.1/tokenizer.json')"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Save trained model\n",
    "trainer.model.save_pretrained(trained_model_name)\n",
    "trainer.tokenizer.save_pretrained(trained_model_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "d7e70da1-52cd-4076-93a1-1134f057c3ce",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "968"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(test_data['text'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "d07efc57-81c6-49ae-b186-6b69d0a75d85",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|████████████████████████████████████████████████████████████████████████████████████████████████████▎                                   | 714/968 [10:14:31<4:17:15, 60.77s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\"adverse\", \"effects\"], \"Treatment.Drug\": [\"tolterodine\"], \"Adverse_event.Trigger\": [\"developed\"], \"Treat_Disorder\": [\"urinary\", \"disorders\"], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [\"patients\"], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [\"Two\"], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 74%|█████████████████████████████████████████████████████████████████████████████████████████████████████▏                                  | 720/968 [10:19:12<3:06:01, 45.01s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\"Severe\", \"adenovirus\", \"pneumonia\", \"(\", \"AVP\", \" )\"], \"Treatment.Drug\": [\"infliximab\"], \"Adverse_event.Trigger\": [\"following\"], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\"infusion\"\", \"for\"\", \"the\"\", \"treatment\"\", \"of\"\", \"Crohn\", \"'s\", \"disease\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|█████████████████████████████████████████████████████████████████████████████████████████████████████▌                                  | 723/968 [10:21:55<3:30:34, 51.57s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████                                 | 734/968 [10:30:58<3:08:58, 48.45s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████▍                                | 736/968 [10:32:37<3:14:03, 50.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████▋                                | 738/968 [10:34:45<3:40:00, 57.39s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['autonomic', ',', 'neurological', 'and', 'neuromuscular', 'instability'], Treatment.Drug: ['methylene', 'blue'], Treatment.Route: ['infusion'], Treatment.Dosage: [], Treatment.Freq: [], Treatment.Duration: [], Treatment: ['for', 'parathyroidectomy']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|███████████████████████████████████████████████████████████████████████████████████████████████████████▊                                | 739/968 [10:35:50<3:46:56, 59.46s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: ['children'], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['PVT', 'HVOD'], Treatment.Drug: ['busulfan'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['chemotherapy']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|█████████████████████████████████████████████████████████████████████████████████████████████████████████▉                              | 754/968 [10:49:04<3:19:42, 55.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: ['Infant'], Subject.Age.Range: [], Subject.Age.Unit: [], Subject.Gender: [], Subject.Sub_Disorder: [], Subject: ['twins'], Subject.Population: [], Subject.Race: [], Population: [], Subject.Gender: [], Subject: [], Subject.Age: [], Subject.Age.Range: [], Subject.Age.Unit: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▍                             | 758/968 [10:52:23<3:12:39, 55.05s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['rhabdomyolysis'], Treatment.Drug: ['colchicine'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['discontinuation']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|██████████████████████████████████████████████████████████████████████████████████████████████████████████▉                             | 761/968 [10:55:13<3:19:41, 57.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [Supravenous, hyperpigmentation], Treatment.Drug: [CHOP], Treatment.Trigger: [], Treat_Disorder: [CD30, (, Ki, -, 1), anaplastic, large, -, cell, lymphoma], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 79%|███████████████████████████████████████████████████████████████████████████████████████████████████████████▉                            | 768/968 [11:02:16<3:18:58, 59.69s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Oculomotor: [], disturbances: [], associated: [], with: [], 5: [], -: [], fluorouracil: [], chemotherapy: [],.: [],.: [],.: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                           | 772/968 [11:06:01<3:08:34, 57.73s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['cortical','venous', 'thrombosis', 'dural', 'puncture', 'headache'], Treatment.Drug: ['L', '-', 'asparaginase','steroid'], Treatment.Freq: [], Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['and']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████                           | 776/968 [11:09:43<2:54:05, 54.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [Prominent, positive, U, waves], Treatment.Drug: [phenylephrine], Treatment.Route: [intravenous], Treatment.Dosage: [high, -, dose], Treatment.Freq: [], Treatment.Duration: [], Treatment: [with]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                          | 780/968 [11:12:37<2:35:27, 49.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                         | 785/968 [11:16:28<2:36:10, 51.21s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\", \"grade\", \"3\", \"proteinuria\", \"(\", \"8.5\", \"g/24\", \"h\", \")\", \"without\", \"microscopic\", \"hematuria\", \"or\", \"renal\", \"insufficiency\", \"five\", \"days\", \"after\", \"temsirolimus\", \"infusion\", \".\"], \"Treatment.Drug\": [\", \"temsirolimus\", \"infusion\", \".\"], \"Adverse_event.Trigger\": [\", \"developed\", \".\"], \"Treat_Disorder\": [\", \"advanced\", \"renal\", \"cell\", \"carcinoma\"], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [\", \"A\", \"58\", \"-\", \"year\", \"old\", \"man\", \"with\", \"advanced\", \"renal\", \"cell\", \"carcinoma\", \".\"], \"Race\": [], \"Subject.Gender\": [\", \"man\", \".\"], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\", \".\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                         | 786/968 [11:17:32<2:47:00, 55.06s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                        | 792/968 [11:22:32<2:26:45, 50.03s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Linear: [], immunoglobulin: [], A: [], bullous: [], dermatosis: [], induced: [], by: [], gemcitabine: [],.: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 82%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                        | 796/968 [11:26:17<2:40:46, 56.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['methemoglobinemia'], Treatment.Drug: [], Adverse_event.Trigger: ['development'], Treat_Disorder: [], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: ['lidocaine', 'bupivacaine'], Sub_Disorder: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['displacement', 'of', 'from', 'protein', 'binding', 'by']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                       | 800/968 [11:28:54<2:06:41, 45.25s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\", \"an\", \"acute\", \"megaloblastic\", \"anaemia\", \"and\", \"a\", \"pneumonitis\", \".\"], \"Treatment.Drug\": [\", \"methotrexate\", \".\"], \"Adverse_event.Trigger\": [\", \"occurrence\", \"of\", \"two\", \"side\", \"effects\", \"of\", \"low\", \"-\", \"dose\", \"of\", \".\"], \"Treat_Disorder\": [\", \"rheumatoid\", \"arthritis\", \".\"], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [\", \"a\", \"patient\", \"suffering\", \"from\", \".\"], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [\", \"low\", \"-\", \"dose\", \".\"], \"Treatment.Duration\": [], \"Treatment\": [\", \"a\", \".\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                       | 801/968 [11:29:38<2:05:00, 44.91s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{{\"Subject.Race\": [], \"Subject.Age\": [\"57\", \"-\", \"year\", \"-\", \"old\"], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\"acral\", \"erythrocyanosis\", \"progressing\", \"to\", \"acute\", \"digital\", \"ischemia\", \"and\", \"gangrene\"], \"Treatment.Drug\": [], \"Adverse_event.Trigger\": [\"developed\"], \"Treat_Disorder\": [\"metastatic\", \"squamous\", \"cell\", \"carcinoma\", \"of\", \"the\", \"hypopharynx\"], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [\"bleomycin\", \"methotrexate\"], \"Sub_Disorder\": [], \"Subject\": [\"a\"], \"Race\": [], \"Subject.Gender\": [\"man\"], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\"combined\", \"chemotherapy\", \"(\", \"and\", \")\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                       | 803/968 [11:31:46<2:30:25, 54.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['leucocytopenia', 'and', 'thrombocytopenia'], Treatment.Drug: ['quetiapine'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                      | 807/968 [11:35:33<2:37:46, 58.80s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['bleeding', 'risk'], Treatment.Drug: ['thromboprophylactic', 'dabigatran', 'etexilate'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                      | 810/968 [11:37:48<2:17:47, 52.33s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['suicidal', 'overdose', 'deaths'], Treatment.Drug: ['quetiapine'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                      | 811/968 [11:38:52<2:26:01, 55.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\", \"hypoadrenalism\"], \"Treatment.Drug\": [\", \"o\", \"p'\"\", \"DDD\", \"dexamethasone\"], \"Adverse_event.Trigger\": [\", \"developed\"], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [\", \"Both\"], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\", \"and\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████                      | 812/968 [11:39:56<2:31:16, 58.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: ['78', '-', 'year', '-', 'old'], Subject.Age.Unit: [], Subject.Sub_Disorder: [], Subject: ['a'], Subject.Race: [], Subject.Gender: ['woman'], Subject.Population: [], Population: [], Subject.Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['Neisseria','mucosa', 'knee', 'arthritis'], Treatment.Drug: ['sodium', 'hyaluronate'], Treatment.Route: ['injection'], Treatment.Dosage: [], Treatment.Freq: [], Treatment.Duration: [], Treatment.Treat_Disorder: [], Treatment.Combination.Drug: [], Treatment.Treatment: ['a','single'], Treatment.Treatment.Time_elapsed: [], Treatment.Treatment.Treat_Disorder: [], Treatment.Treatment.Dosage: [], Treatment.Treatment.Freq: [], Treatment.Treatment.Duration: [], Treatment.Treatment.Treatment: ['after']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                     | 813/968 [11:40:36<2:16:27, 52.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\", \"ectopic\", \"pregnancies\", \".\"], \"Treatment.Drug\": [\", \"Implanon\"], \"Adverse_event.Trigger\": [\", \"resulting\", \"in\"], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [\", \"an\", \"HIV\", \"-\", \"positive\", \"woman\"], \"Race\": [], \"Subject.Gender\": [\", \"on\", \"antiretroviral\", \"therapy\"], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                     | 816/968 [11:43:49<2:32:25, 60.17s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\" = [], \"Subject.Race\" = [], \"Subject.Age\" = [], \"Potential_therapeutic_event.Trigger\" = [], \"Time_elapsed\" = [], \"Duration\" = [], \"Treatment.Freq\" = [], \"Route\" = [], \"Gender\" = [], \"Drug\" = [], \"Subject.Sub_Disorder\" = [], \"Effect\" = [\"bone\", \"marrow\", \"suppression\", \"and\", \"oral\", \"and\", \"gastrointestinal\", \"ulceration\"], \"Treatment.Drug\" = [\"methotrexate\"], \"Adverse_event.Trigger\" = [\"toxicity\"], \"Treat_Disorder\" = [], \"Treatment.Route\" = [], \"Dosage\" = [], \"Freq\" = [], \"Treatment.Combination.Drug\" = [], \"Sub_Disorder\" = [], \"Subject\" = [], \"Race\" = [], \"Subject.Gender\" = [], \"Population\" = [], \"Subject.Population\" = [], \"Treatment.Time_elapsed\" = [], \"Treatment.Treat_Disorder\" = [], \"Combination.Drug\" = [], \"Treatment.Dosage\" = [], \"Treatment.Duration\" = [], \"Treatment\" = []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                     | 818/968 [11:45:13<2:08:22, 51.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\",\"Severe\",\"respiratory\",\"syncytial\",\"virus\",\"pulmonary\",\"infection\",\".\"], \"Treatment.Drug\": [\",\"fludarabine\",\".\"], \"Adverse_event.Trigger\": [\",\"in\",\"a\",\"patient\",\"treated\",\"with\",\"for\",\"chronic\",\"lymphocytic\",\"leukemia\",\".\"], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\",\"with\",\".\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                    | 821/968 [11:47:33<2:03:31, 50.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['Behavioral','side', 'effects', 'associated', 'with', 'clonazepam','may', 'include', 'agitation', ',', 'aggression', ',', 'hyperactivity', ',', 'irritability', ',', 'property', 'destruction', ',', 'and', 'temper', 'tantrums', '.'], Treatment.Drug: ['clonazepam'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                    | 824/968 [11:50:20<2:13:24, 55.59s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['cardiac', 'toxicity'], Treatment.Drug: ['amphotericin', 'B', 'deoxycholate'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['therapy']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 85%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                   | 827/968 [11:53:04<2:13:55, 56.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\", \"improved\", \",\", \"and\", \"her\", \"eosinophilia\", \"and\", \"radiographic\", \"abnormalities\", \"both\", \"resolved\", \".\"], \"Treatment.Drug\": [\", \"steroids\", \"captopril\"], \"Adverse_event.Trigger\": [], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\", \"systemic\", \"discontinuing\", \"and\", \"starting\", \".\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                   | 829/968 [11:54:44<2:06:33, 54.63s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['pulmonary', 'disease'], Treatment.Drug: ['calcium','stearate'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                  | 835/968 [11:59:08<1:36:03, 43.34s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍                  | 836/968 [12:00:13<1:49:07, 49.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['pulmonary', 'toxicity'], Subject: ['cases'], Population: [], Subject.Population: ['two'], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['nitrofurantoin']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌                  | 837/968 [12:00:50<1:40:36, 46.08s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Colchicine: [], -: [], induced: [], rhabdomyolysis: [],.: [], Colchicine: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉                  | 839/968 [12:02:31<1:46:10, 49.38s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['a','significant', 'increase', 'in', 'the', 'patient', \"'s\", 'INR'], Treatment.Drug: ['tolterodine'], Treatment.Freq: [], Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                  | 840/968 [12:03:35<1:54:50, 53.83s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['unusually','severe', 'adverse','reactions', '(', 'e.g.', ',', 'diarrhea', ',','stomatitis', ',','mucositis', ',','myelosuppression', ',', 'neurotoxicity', ')'], Treatment.Drug: ['5', '-', 'FU'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                 | 841/968 [12:04:03<1:37:50, 46.23s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['supratherapeutic', 'INRs'], Treatment.Drug: [], Adverse_event.Trigger: ['associated'], Treat_Disorder: [], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: ['acetaminophen','moxifloxacin'], Sub_Disorder: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['concomitant', 'and']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎                 | 842/968 [12:05:07<1:48:24, 51.62s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\" = [], \"Subject.Race\" = [], \"Subject.Age\" = [], \"Potential_therapeutic_event.Trigger\" = [], \"Time_elapsed\" = [], \"Duration\" = [], \"Treatment.Freq\" = [], \"Route\" = [], \"Gender\" = [], \"Drug\" = [], \"Subject.Sub_Disorder\" = [], \"Effect\" = [\"myopathy\"], \"Treatment.Drug\" = [], \"Adverse_event.Trigger\" = [\"associated\"], \"Treat_Disorder\" = [], \"Treatment.Route\" = [], \"Dosage\" = [], \"Freq\" = [], \"Treatment.Combination.Drug\" = [\"colchicine\", \"statin\"], \"Sub_Disorder\" = [], \"Subject\" = [], \"Race\" = [], \"Subject.Gender\" = [], \"Population\" = [], \"Subject.Population\" = [], \"Treatment.Time_elapsed\" = [], \"Treatment.Treat_Disorder\" = [], \"Combination.Drug\" = [], \"Treatment.Dosage\" = [], \"Treatment.Duration\" = [], \"Treatment\" = [\"and\", \"therapy\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊                 | 846/968 [12:07:55<1:36:59, 47.70s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏                | 848/968 [12:09:30<1:38:22, 49.19s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['Multiple','syncopal', 'episodes', 'abnormalities', 'with', 'an', 'episode', 'of','sustained','ventricular', 'tachycardia'], Treatment.Drug: ['thalidomide'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋                | 852/968 [12:12:10<1:16:11, 39.41s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['exacerbate', 'the', 'loss', 'of', 'intracellular', 'potassium', ',', 'thereby', 'increasing', 'the', 'risk', 'of', 'cardiac', 'toxicity'], Treatment.Drug: ['lithium'], Adverse_event.Trigger: ['with'], Treat_Disorder: [], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Subject: ['any', 'patient'], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████                | 855/968 [12:14:53<1:33:05, 49.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: ['induced'], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: ['rheumatoid', 'arthritis'], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['fenclofenac', 'aurothiomalate','sulphasalazine']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 88%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎               | 856/968 [12:15:58<1:40:50, 54.02s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: ['effect'], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: ['painful', 'diabetic', 'neuropathy'], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['co', '-', 'administration', 'of', 'prolonged', '-','release', 'oxycodone', 'and', 'existing', 'gabapentin', 'therapy']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍              | 864/968 [12:22:56<1:30:51, 52.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['acute','syphilitic', 'posterior', 'placoid', 'chorioretinitis', '(', 'ASPPC', ')'], Treatment.Drug: ['triamcinolone', 'acetonide'], Treatment.Route: ['intravitreal'], Treatment.Dosage: [], Treatment.Freq: [], Treatment.Duration: [], Treatment: ['injection']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████             | 876/968 [12:32:59<1:14:30, 48.60s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\", \"benzodiazepine withdrawal symptoms\", \"on discontinuation\"], \"Treatment.Drug\": [\", \"nefazodone\"], \"Adverse_event.Trigger\": [\", \"experienced\", \"symptoms\"], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [\", \"a\", \"patient\"], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\", \"discontinuation\"\", \"of\", \"an\", \"antidepressant\", \"that\", \"inhibits\", \"the\", \"cytochrome\", \"P450\", \"3A4\", \"isoenzyme\", \".\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎            | 878/968 [12:34:36<1:14:49, 49.89s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['priapism'], Treatment.Drug: ['zuclopenthixol'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋            | 880/968 [12:36:20<1:16:31, 52.18s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 91%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊            | 881/968 [12:37:05<1:12:21, 49.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\",\"hemorrhage\",\"from\",\"a\",\"meningioma\",\"may\",\"have\",\"been\",\"induced\",\"by\",\"aspirin\",\"prophylaxis\",\".\"], \"Treatment.Drug\": [\",\"aspirin\",\"prophylaxis\",\".\"], \"Adverse_event.Trigger\": [\",\"induced\",\"by\",\".\"], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌           | 887/968 [12:42:25<1:07:49, 50.24s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['symptoms', 'of', 'propafenone', 'adverse', 'effects'], Treatment.Drug: [], Adverse_event.Trigger: ['caused'], Treat_Disorder: [], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: ['propafenone', 'citalopram'], Sub_Disorder: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['and']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊           | 888/968 [12:43:28<1:12:05, 54.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['Temporary', 'neurologic', 'abnormalities'], Treatment.Drug: ['methotrexate'], Treatment.Trigger: [], Treat_Disorder: ['osteogenic','sarcoma'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: ['high', '-', 'dose'], Treatment.Duration: [], Treatment: ['chemotherapy', 'with', '(', 'HD', '-', 'MTX', ')']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉           | 889/968 [12:44:13<1:07:29, 51.26s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\", \"adverse\", \"effects\", \"(\", \"eg\", \", \"dizziness\", \", \", \"falls\", \"and\", \"mimicked\", \"coronary\", \"artery\", \"disease\", \".\"], \"Treatment.Drug\": [], \"Adverse_event.Trigger\": [\", \"caused\", \"by\"], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [\", \"propafenone\", \"citalopram\"], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\", \"and\", \".\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋          | 895/968 [12:49:09<1:05:23, 53.74s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▉          | 896/968 [12:50:13<1:08:13, 56.86s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\"Subject.Race\": [], \"Subject.Age\": [], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [\", \"a\", \"doubling\", \"of\", \"the\", \"lamotrigine\", \"blood\", \"level\", \"with\", \"symptoms\", \"of\", \"toxicity\", \".\"], \"Treatment.Drug\": [\", \"sertraline\", \"], \"Adverse_event.Trigger\": [\", \"resulted\", \"in\"], \"Treat_Disorder\": [], \"Treatment.Route\": [], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [], \"Race\": [], \"Subject.Gender\": [], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [\", \"25\", \"mg\"], \"Treatment.Duration\": [], \"Treatment\": [\", \"a\", \"total\", \"daily\", \"dose\", \"of\", \",\", \"with\", \"nondetectable\", \"sertraline\", \"and\", \"desmethylsertraline\", \"blood\", \"levels\", \",\", \".\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋         | 903/968 [12:55:34<53:11, 49.10s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: ['receiving'], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [], Subject: ['the', 'patient'], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['emtricitabine','monotherapy']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎        | 907/968 [12:58:50<48:03, 47.28s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [associated], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [less, escape, medication, use, fewer, nights, of, disturbed, sleep], Treatment.Drug: [], Adverse_event.Trigger: [], Treat_Disorder: [], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [Oxycodone, gabapentin], Sub_Disorder: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: [-, -, co, administration]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████        | 912/968 [13:03:43<54:32, 58.43s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{{\"Subject.Race\": [\"African\"], \"Subject.Age\": [\"55\", \"-\", \"year\", \"-\", \"old\"], \"Potential_therapeutic_event.Trigger\": [], \"Time_elapsed\": [], \"Duration\": [], \"Treatment.Freq\": [], \"Route\": [], \"Gender\": [], \"Drug\": [], \"Subject.Sub_Disorder\": [], \"Effect\": [], \"Treatment.Drug\": [\"triamcinolone\"], \"Adverse_event.Trigger\": [], \"Treat_Disorder\": [\"postherpetic\", \"neuralgia\"], \"Treatment.Route\": [\"epidural\"], \"Dosage\": [], \"Freq\": [], \"Treatment.Combination.Drug\": [], \"Sub_Disorder\": [], \"Subject\": [\"HIV\", \"-\", \"positive\"], \"Race\": [], \"Subject.Gender\": [\"woman\"], \"Population\": [], \"Subject.Population\": [], \"Treatment.Time_elapsed\": [], \"Treatment.Treat_Disorder\": [], \"Combination.Drug\": [], \"Treatment.Dosage\": [], \"Treatment.Duration\": [], \"Treatment\": [\"a\", \"single\", \"injection\", \"for\", \"pain\", \"relief\", \"of\"]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏       | 913/968 [13:04:36<52:09, 56.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Anaphylactoid: [], shock: [],,: [], disseminated: [], intravascular: [], coagulation: [],,: [], and: [], anuric: [], renal: [], failure: [], requiring: [], dialysis: [], occurred: [], in: [], a: [], patient: [], receiving: [], zomepirac: [], sodium: [], for: [], toothache: [],.: [],.: [],.: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 94%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎       | 914/968 [13:05:41<53:10, 59.09s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: ['young'], Subject.Age.units: [], Subject.Age.sub_disorder: [], Subject.Gender: ['female'], Subject.Sub_Disorder: [], Subject: ['volunteers'], Population: [], Subject.Population: ['six', 'two'], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['peripheral', 'edema'], Treatment.Drug: [], Treatment.Trigger: [], Treat_Disorder: [], Treatment.Route: ['intravenous'], Treatment.Dosage: [], Treatment.Freq: [], Treatment.Duration: [], Treatment: ['proton', 'pump', 'inhibitors', 'omeprazole', 'and', 'pantoprazole', ',', 'high', 'doses', 'of', 'the', 'proton', 'pump', 'inhibitors', 'were', 'applied', 'by', 'continuous', 'infusion', 'together', 'with', 'large', 'volumes', 'of', 'fluid']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏      | 920/968 [13:10:33<44:09, 55.20s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [paradoxical, raised, IOP], Treatment.Drug: [brimonidine], Treatment.Route: [topical], Treatment.Dosage: [], Treatment.Freq: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎      | 921/968 [13:11:37<45:18, 57.84s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['fatal', 'overdose'], Treatment.Drug: ['chloroquine'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: ['902','micro', 'g', '/', 'L'], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍      | 922/968 [13:12:42<45:49, 59.78s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['cardiovascular', 'events'], Treatment.Drug: ['PPI'], Drug: [], Subject.Sub_Disorder: [], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌      | 923/968 [13:13:46<45:46, 61.04s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋      | 924/968 [13:14:50<45:24, 61.93s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [], Treatment.Drug: [], Adverse_event.Trigger: [], Tret_Disorder: [], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['1', '%', 'alum','solution']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊      | 925/968 [13:15:53<44:48, 62.52s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['spinal', 'cord', 'infarction'], Treatment.Drug: ['zolmitriptan'], Treatment.Freq: [], Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▏     | 927/968 [13:17:29<38:38, 56.55s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['large', 'extent', 'of', 'hemorrhage'], Treatment.Drug: ['tissue', 'plasminogen', 'activator'], Treatment.Freq: [], Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['administration']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊     | 932/968 [13:21:22<29:36, 49.35s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [ocular, hypertension], Treatment.Drug: [ranibizumab], Treatment.Route: [intravitreally], Treatment.Dosage: [], Treatment.Duration: [], Treatment: [injection, of]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 97%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▎    | 935/968 [13:23:47<25:47, 46.90s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['a', 'condition','resembling', 'RPLS'], Treatment.Drug: [], Adverse_event.Trigger: ['induce'], Treat_Disorder: [], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: ['cyclosporine', 'cisplatin', 'tacrolimus', 'interferon', '-', 'alpha'], Sub_Disorder: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['and']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▌   | 944/968 [13:30:25<18:07, 45.30s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: [low, '-', grade, fever,, localized, pain,, myelo, '-', suppression, and, liver, dysfunction,, but, these, were, transient, and, eventually, disappeared], Treatment.Drug: [], Adverse_event.Trigger: [of], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [Lp, TAE, HT], Subject.Sub_Disorder: [], Subject: [], Race: [], Subject.Gender: [], Population: [], Subject.Population: [], Treatment.Time_elapsed: [], Treatment.Treat_Disorder: [], Combination.Drug: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: [combined, with]}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▍  | 950/968 [13:35:21<14:57, 49.88s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Anaphylaxis: [], after: [], the: [], injection: [], of: [], chymopapain: [], occurs: [], in: [], about: [], 1: [], %: [], of: [], such: [], cases: [],.: [], of: [], such: [], cases: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████ | 961/968 [13:45:12<07:07, 61.07s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['Adriamycin', 'toxicity'], Treatment.Drug: ['Adriamycin', 'doxorubicin'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['chemotherapy']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▋| 966/968 [13:48:51<01:31, 45.99s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['pulmonary', 'toxicity'], Treatment.Drug: ['CCNU'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: []}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████▊| 967/968 [13:49:55<00:51, 51.48s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{Subject.Race: [], Subject.Age: [], Potential_therapeutic_event.Trigger: [], Time_elapsed: [], Duration: [], Treatment.Freq: [], Route: [], Gender: [], Drug: [], Subject.Sub_Disorder: [], Effect: ['hypoglycemia'], Treatment.Drug: ['clarithromycin'], Treatment.Route: [], Dosage: [], Freq: [], Treatment.Combination.Drug: [], Sub_Disorder: [], Treatment.Dosage: [], Treatment.Duration: [], Treatment: ['Concomitant', 'use', 'of', 'or', 'other', 'potent', 'inhibitors', 'of', 'CYP3A4', 'with','repaglinide']}\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 968/968 [13:50:26<00:00, 51.47s/it]\n"
     ]
    }
   ],
   "source": [
    "y_pred = predict(test_data, model, tokenizer)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a809b7a0-ab81-415f-9ef7-8d14094ef2b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "len(y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "4461711f-2179-46d1-b9bb-e38ad68ec96f",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to save dictionaries to JSONL file\n",
    "def save_to_jsonl(file_path, list_of_dicts):\n",
    "    with open(file_path, 'w') as f:\n",
    "        for dictionary in list_of_dicts:\n",
    "            f.write(json.dumps(dictionary) + '\\n')\n",
    "\n",
    "# File path to save JSONL file\n",
    "pred_file_path = \"outputs/test_output_predict.jsonl\"\n",
    "\n",
    "# Save list of dictionaries to JSONL file\n",
    "save_to_jsonl(pred_file_path, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "d0ec6fef-5968-4a81-8bee-6e1024e99825",
   "metadata": {},
   "outputs": [],
   "source": [
    "gold_file_path = \"outputs/test_output_gold.jsonl\"\n",
    "save_to_jsonl(gold_file_path, test_outputs)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "64109635-a348-4837-bf31-ffab4349d3da",
   "metadata": {},
   "source": [
    "### Evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "a45ee210-bc37-4943-98b1-8be79301f989",
   "metadata": {},
   "outputs": [],
   "source": [
    "import json\n",
    "\n",
    "def read_jsonl(file_path):\n",
    "    data = []\n",
    "    with open(file_path, 'r', encoding='utf-8') as file:\n",
    "        for line in file:\n",
    "            data.append(json.loads(line))\n",
    "    return data\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "fde35f90-8ff1-416c-a952-f883536298d5",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Label-specific Metrics:\n",
      "Label\tPrecision\tRecall\tF1\n",
      "Subject.Race\t1.0\t0.36\t0.53\n",
      "Subject.Age\t0.99\t0.59\t0.74\n",
      "Potential_therapeutic_event.Trigger\t0.44\t0.11\t0.18\n",
      "Time_elapsed\t0\t0.0\t0\n",
      "Duration\t0\t0.0\t0\n",
      "Treatment.Freq\t0\t0.0\t0\n",
      "Route\t0\t0.0\t0\n",
      "Gender\t0\t0\t0\n",
      "Drug\t0\t0.0\t0\n",
      "Subject.Sub_Disorder\t0.52\t0.11\t0.18\n",
      "Effect\t0.84\t0.53\t0.65\n",
      "Treatment.Drug\t0.92\t0.59\t0.72\n",
      "Adverse_event.Trigger\t0.69\t0.46\t0.55\n",
      "Treat_Disorder\t0.7\t0.47\t0.56\n",
      "Treatment.Route\t0.59\t0.43\t0.5\n",
      "Dosage\t0\t0.0\t0\n",
      "Freq\t0\t0.0\t0\n",
      "Treatment.Combination.Drug\t0.67\t0.42\t0.52\n",
      "Sub_Disorder\t0\t0.0\t0\n",
      "Subject\t0.66\t0.43\t0.52\n",
      "Race\t0\t0\t0\n",
      "Subject.Gender\t0.94\t0.54\t0.69\n",
      "Population\t0\t0\t0\n",
      "Subject.Population\t0.68\t0.42\t0.52\n",
      "Treatment.Time_elapsed\t0.41\t0.22\t0.29\n",
      "Treatment.Treat_Disorder\t0.0\t0.0\t0\n",
      "Combination.Drug\t0\t0.0\t0\n",
      "Treatment.Dosage\t0.75\t0.4\t0.52\n",
      "Treatment.Duration\t0.57\t0.42\t0.48\n",
      "Treatment\t0.56\t0.37\t0.45\n",
      "\n",
      "Overall Micro Metrics:\n",
      "Micro Precision: 0.75\n",
      "Micro Recall: 0.46\n",
      "Micro F1-score: 0.57\n",
      "\n",
      "Overall Macro Metrics:\n",
      "Macro Precision: 0.4\n",
      "Macro Recall: 0.23\n",
      "Macro F1-score: 0.29\n"
     ]
    }
   ],
   "source": [
    "from collections import defaultdict\n",
    "from sklearn.metrics import precision_score, recall_score, f1_score\n",
    "\n",
    "def evaluate(gold, predict):\n",
    "    # Initialize dictionaries to store true positives, false positives, and false negatives for each label\n",
    "    tp = defaultdict(int)\n",
    "    fp = defaultdict(int)\n",
    "    fn = defaultdict(int)\n",
    "    \n",
    "    # Iterate over each document in gold and predicted data\n",
    "    for gold_doc, predict_doc in zip(gold, predict):\n",
    "        # Iterate over each NER label in the document\n",
    "        for label in gold_doc.keys():\n",
    "            # Calculate true positives\n",
    "            tp[label] += len(set(predict_doc.get(label, [])) & set(gold_doc[label]))\n",
    "            # Calculate false positives\n",
    "            fp[label] += len(set(predict_doc.get(label, [])) - set(gold_doc[label]))\n",
    "            # Calculate false negatives\n",
    "            fn[label] += len(set(gold_doc[label]) - set(predict_doc.get(label, [])))\n",
    "    \n",
    "    # Initialize dictionaries to store precision, recall, and f1-score for each label\n",
    "    precision = {}\n",
    "    recall = {}\n",
    "    f1 = {}\n",
    "    \n",
    "    # Calculate precision, recall, and f1-score for each label\n",
    "    for label in tp.keys():\n",
    "        precision[label] = tp[label] / (tp[label] + fp[label]) if (tp[label] + fp[label]) > 0 else 0\n",
    "        recall[label] = tp[label] / (tp[label] + fn[label]) if (tp[label] + fn[label]) > 0 else 0\n",
    "        f1[label] = 2 * precision[label] * recall[label] / (precision[label] + recall[label]) if (precision[label] + recall[label]) > 0 else 0\n",
    "    \n",
    "    # Calculate macro-averaged precision, recall, and f1-score for labels\n",
    "    macro_precision = sum(precision.values()) / len(precision)\n",
    "    macro_recall = sum(recall.values()) / len(recall)\n",
    "    macro_f1 = sum(f1.values()) / len(f1)\n",
    "    \n",
    "    # Calculate overall micro-averaged precision, recall, and f1-score\n",
    "    overall_tp = sum(tp.values())\n",
    "    overall_fp = sum(fp.values())\n",
    "    overall_fn = sum(fn.values())\n",
    "    overall_precision = overall_tp / (overall_tp + overall_fp) if (overall_tp + overall_fp) > 0 else 0\n",
    "    overall_recall = overall_tp / (overall_tp + overall_fn) if (overall_tp + overall_fn) > 0 else 0\n",
    "    overall_f1 = 2 * overall_precision * overall_recall / (overall_precision + overall_recall) if (overall_precision + overall_recall) > 0 else 0\n",
    "    \n",
    "    # Return results\n",
    "    return {\n",
    "        'Precision': precision,\n",
    "        'Recall': recall,\n",
    "        'F1': f1,\n",
    "        'Macro Precision': macro_precision,\n",
    "        'Macro Recall': macro_recall,\n",
    "        'Macro F1': macro_f1,\n",
    "        'Overall Micro Precision': overall_precision,\n",
    "        'Overall Micro Recall': overall_recall,\n",
    "        'Overall Micro F1': overall_f1\n",
    "    }\n",
    "gold_data = read_jsonl(gold_file_path)\n",
    "predict_data = read_jsonl(pred_file_path)\n",
    "\n",
    "evaluation_results = evaluate(gold_data, predict_data)\n",
    "\n",
    "print(\"Label-specific Metrics:\")\n",
    "print(\"Label\\tPrecision\\tRecall\\tF1\")\n",
    "for label in evaluation_results['Precision'].keys():\n",
    "    precision = round(evaluation_results['Precision'][label], 2)\n",
    "    recall = round(evaluation_results['Recall'][label], 2)\n",
    "    f1 = round(evaluation_results['F1'][label], 2)\n",
    "    print(f\"{label}\\t{precision}\\t{recall}\\t{f1}\")\n",
    "\n",
    "print()\n",
    "micro_precision = round(evaluation_results['Overall Micro Precision'], 2)\n",
    "micro_recall = round(evaluation_results['Overall Micro Recall'], 2)\n",
    "micro_f1 = round(evaluation_results['Overall Micro F1'], 2)\n",
    "print(\"Overall Micro Metrics:\")\n",
    "print(f\"Micro Precision: {micro_precision}\")\n",
    "print(f\"Micro Recall: {micro_recall}\")\n",
    "print(f\"Micro F1-score: {micro_f1}\")\n",
    "\n",
    "print()\n",
    "macro_precision = round(evaluation_results['Macro Precision'], 2)\n",
    "macro_recall = round(evaluation_results['Macro Recall'], 2)\n",
    "macro_f1 = round(evaluation_results['Macro F1'], 2)\n",
    "print(\"Overall Macro Metrics:\")\n",
    "print(f\"Macro Precision: {macro_precision}\")\n",
    "print(f\"Macro Recall: {macro_recall}\")\n",
    "print(f\"Macro F1-score: {macro_f1}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbf4bbc5-8fc4-454e-8f06-e83d3a0d9758",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "venv",
   "language": "python",
   "name": "venv"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}