tutorial/semantic_search_data_upload.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true,
    "pycharm": {
     "is_executing": true
    }
   },
   "outputs": [],
   "source": [
    "!pip install nomic pandas langchain"
   ]
  },
  {
   "cell_type": "markdown",
   "source": [
    "### Load a demo dataset of 25k news articles"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "outputs": [],
   "source": [
    "from nomic import AtlasProject\n",
    "import pandas\n",
    "\n",
    "#load a demo dataset of 25k news articles\n",
    "news_articles = pandas.read_csv('https://raw.githubusercontent.com/nomic-ai/maps/main/data/ag_news_25k.csv').to_dict('records')\n",
    "\n",
    "#use only the first 10k\n",
    "news_articles = news_articles[:10_000]"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "print(news_articles[0].keys())\n",
    "\n",
    "#create a project in the Atlas Embedding Database.\n",
    "#By specifying modality='embedding' you are saying you will upload your own embeddings.\n",
    "project = AtlasProject(name='10k News Articles', unique_id_field='id', modality='embedding')"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "outputs": [],
   "source": [
    "from langchain.embeddings import OpenAIEmbeddings\n",
    "openai_key= ''\n",
    "\n",
    "openai = OpenAIEmbeddings(openai_api_key=openai_key, model='text-embedding-ada-002')\n",
    "# embed the news articles with OpenAI\n",
    "embeddings = openai.embed_documents(texts=[article['text'] for article in news_articles])"
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "\n",
    "#add your OpenAI embeddings and metadata to the Atlas DB project\n",
    "project.add_embeddings(\n",
    "    embeddings=np.array(embeddings),\n",
    "    data=news_articles\n",
    ")\n",
    "project.create_index(name=project.name, build_topic_model=True, topic_label_field='text')\n",
    "print(project.maps[0])"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "By running the next cell, you will be shown the Atlas Embedding DB inspector (Atlas calls it The Map). Points close to each are semantically similar. It is useful for getting an idea of the types of search results your app will serve for any query."
   ],
   "metadata": {
    "collapsed": false
   }
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "outputs": [],
   "source": [
    "project.maps[0]"
   ],
   "metadata": {
    "collapsed": false,
    "pycharm": {
     "is_executing": true
    }
   }
  },
  {
   "cell_type": "markdown",
   "source": [
    "# Semantic Search in your app\n",
    "To get semantic search running in your app, you need to simply paste your Atlas Embedding DB project name (in this case '10k News Articles') into the settings.py environment variable of the FastAPI app!"
   ],
   "metadata": {
    "collapsed": false
   }
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 0
}