7-1-NLTK-with-the-Greek-Script.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# NLTK with non-Latin scripts (Greek)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 1. Cleaning text"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'αυτος είναι ο χορός της βροχής της φυλής, ό,τι περίεργο.'"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sentence = \"ΑΥΤΟΣ είναι ο χορός της βροχής της φυλής, ό,τι περίεργο.\"\n",
    "sentence = sentence.lower()\n",
    "sentence"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "A package called [`unidecode`](https://pypi.org/project/Unidecode) can be used to transliterate any Unicode string into the “closest possible representation” in ASCII text:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'autos einai o khoros tes brokhes tes phules, o,ti periergo.'"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from unidecode import unidecode\n",
    "\n",
    "sentence_latin = unidecode(sentence)\n",
    "sentence_latin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'αυτος ειναι ο χορος της βροχης της φυλης, ο,τι περιεργο.'"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import unicodedata\n",
    "\n",
    "def strip_accents(s):\n",
    "    return ''.join(c for c in unicodedata.normalize('NFD', s) # NFD = Normalization Form Canonical Decomposition, one of four Unicode normalization forms.\n",
    "                   if unicodedata.category(c) != 'Mn') # The character category \"Mn\" stands for Nonspacing_Mark\n",
    "sentence_no_accents = strip_accents(sentence)\n",
    "sentence_no_accents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['αυτος',\n",
       " 'ειναι',\n",
       " 'ο',\n",
       " 'χορος',\n",
       " 'της',\n",
       " 'βροχης',\n",
       " 'της',\n",
       " 'φυλης,',\n",
       " 'ο,τι',\n",
       " 'περιεργο.']"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.tokenize import WhitespaceTokenizer\n",
    "\n",
    "tokens = WhitespaceTokenizer().tokenize(sentence_no_accents)\n",
    "tokens"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['αυτος',\n",
       " 'ειναι',\n",
       " 'ο',\n",
       " 'χορος',\n",
       " 'της',\n",
       " 'βροχης',\n",
       " 'της',\n",
       " 'φυλης',\n",
       " 'ο,τι',\n",
       " 'περιεργο']"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from string import punctuation\n",
    "\n",
    "new_tokens = []\n",
    "\n",
    "for token in tokens:\n",
    "    if token == 'ο,τι':\n",
    "        new_tokens.append('ο,τι')\n",
    "    else:\n",
    "        new_tokens.append(token.translate(str.maketrans({key: None for key in punctuation})))\n",
    "\n",
    "new_tokens_with_stopwords = new_tokens\n",
    "new_tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 2. Removing stopwords"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "83"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Greek stopwords adapted from https://github.com/6/stopwords-json however better lists with more stopwords are available: https://www.translatum.gr/forum/index.php?topic=3550.0?topic=3550.0\n",
    "greek_stopwords = [\"αλλα\",\"αν\",\"αντι\",\"απο\",\"αυτα\",\"αυτες\",\"αυτη\",\"αυτο\",\"αυτοι\",\"αυτος\",\"αυτους\",\"αυτων\",\"για\",\"δε\",\"δεν\",\"εαν\",\"ειμαι\",\"ειμαστε\",\"ειναι\",\"εισαι\",\"ειστε\",\"εκεινα\",\"εκεινες\",\"εκεινη\",\"εκεινο\",\"εκεινοι\",\"εκεινος\",\"εκεινους\",\"εκεινων\",\"ενω\",\"επι\",\"η\",\"θα\",\"ισως\",\"κ\",\"και\",\"κατα\",\"κι\",\"μα\",\"με\",\"μετα\",\"μη\",\"μην\",\"να\",\"ο\",\"οι\",\"ομως\",\"οπως\",\"οσο\",\"οτι\",\"ο,τι\",\"παρα\",\"ποια\",\"ποιες\",\"ποιο\",\"ποιοι\",\"ποιος\",\"ποιους\",\"ποιων\",\"που\",\"προς\",\"πως\",\"σε\",\"στη\",\"στην\",\"στο\",\"στον\",\"στης\",\"στου\",\"στους\",\"στις\",\"στα\",\"τα\",\"την\",\"της\",\"το\",\"τον\",\"τοτε\",\"του\",\"των\",\"τις\",\"τους\",\"ως\"]\n",
    "len(greek_stopwords)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['χορος', 'βροχης', 'φυλης', 'περιεργο']"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "new_tokens_set = set(new_tokens)\n",
    "greek_stopwords_set = set(greek_stopwords)\n",
    "intersection_set = new_tokens_set.intersection(greek_stopwords_set)\n",
    "intersection_set\n",
    "\n",
    "for element in intersection_set:\n",
    "    new_tokens = list(filter((element).__ne__, new_tokens)) # __ne__ is the != operator.\n",
    "new_tokens"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 3. Other packages"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "There are more interesting packages like [`polyglot`](https://pypi.org/project/polyglot/) and [`greek-stemmer`](https://pypi.org/project/greek-stemmer/). However, these require [`PyICU`](https://pypi.org/project/PyICU/) in order to work and installing this on Windows is a pain."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}