notebooks/2_semantic_search.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fa447fa7-6f85-4c6c-8bf0-d5f5ad9f9e97",
   "metadata": {},
   "outputs": [],
   "source": [
    "# This notebook has been updated to use the latest openai package version! At the time, 1.6.1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "d4ef440a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\nChapter 2: Launching an Application with Proprietary Models \\n    Overview of Proprietary Modelsa\\n    Introduction to OpenAI + Embeddings / GPT3 / ChatGPT\\n    Introduction to Vector Databases\\n    Building a Neural/Semantic Information Retrieval System with Vector Databases, BERT & GPT3\\n\\n'"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "Chapter 2: Launching an Application with Proprietary Models \n",
    "    Overview of Proprietary Models\n",
    "    Introduction to OpenAI + Embeddings / GPT3 / ChatGPT\n",
    "    Introduction to Vector Databases\n",
    "    Building a Neural/Semantic Information Retrieval System with Vector Databases, BERT & GPT3\n",
    "\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "c12182ab",
   "metadata": {},
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "from datetime import datetime\n",
    "import hashlib\n",
    "import re\n",
    "import os\n",
    "from tqdm import tqdm\n",
    "import numpy as np\n",
    "\n",
    "import logging\n",
    "\n",
    "logger = logging.getLogger()\n",
    "logger.setLevel(logging.DEBUG)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "f0289c54",
   "metadata": {},
   "outputs": [],
   "source": [
    "pinecone_key = os.environ.get('PINECONE_API_KEY')\n",
    "client = OpenAI(\n",
    "    api_key=os.environ.get(\"OPENAI_API_KEY\")\n",
    ")\n",
    "\n",
    "INDEX_NAME = 'semantic-search'\n",
    "NAMESPACE = 'default'\n",
    "ENGINE = 'text-embedding-ada-002'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "b9befee1",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pinecone\n",
    "\n",
    "pinecone.init(api_key=pinecone_key, environment=\"us-west1-gcp\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "3cf993a3-6d51-49f4-8968-60f654d6202d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1536, 2)"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# helper functions to get lists of embeddings from the OpenAI API\n",
    "def get_embeddings(texts, engine=ENGINE):\n",
    "    response = client.embeddings.create(\n",
    "        input=texts,\n",
    "        model=engine\n",
    "    )\n",
    "    \n",
    "    return [d.embedding for d in list(response.data)]\n",
    "\n",
    "def get_embedding(text):\n",
    "    return get_embeddings([text])[0]\n",
    "    \n",
    "len(get_embedding('hi')), len(get_embeddings(['hi', 'hello']))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "ea70672a",
   "metadata": {},
   "outputs": [],
   "source": [
    "if not INDEX_NAME in pinecone.list_indexes():\n",
    "    pinecone.create_index(\n",
    "        INDEX_NAME,  # The name of the index\n",
    "        dimension=1536,  # The dimensionality of the vectors\n",
    "        metric='cosine',  # The similarity metric to use when searching the index\n",
    "        pod_type=\"p1\"  # The type of Pinecone pod\n",
    "    )\n",
    "\n",
    "# Store the index as a variable\n",
    "index = pinecone.Index(INDEX_NAME)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8088892",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b6103d6c",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "7f2fdfe7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'ae76cc4dfd345ecaeea9b8ba0d5c3437'"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def my_hash(s):\n",
    "    # Return the MD5 hash of the input string as a hexadecimal string\n",
    "    return hashlib.md5(s.encode()).hexdigest()\n",
    "\n",
    "my_hash('I love to hash it')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "id": "ecd86f51",
   "metadata": {},
   "outputs": [],
   "source": [
    "def prepare_for_pinecone(texts, engine=ENGINE):\n",
    "    # Get the current UTC date and time\n",
    "    now = datetime.utcnow()\n",
    "    \n",
    "    # Generate vector embeddings for each string in the input list, using the specified engine\n",
    "    embeddings = get_embeddings(texts, engine=engine)\n",
    "    \n",
    "    # Create tuples of (hash, embedding, metadata) for each input string and its corresponding vector embedding\n",
    "    # The my_hash() function is used to generate a unique hash for each string, and the datetime.utcnow() function is used to generate the current UTC date and time\n",
    "    return [\n",
    "        (\n",
    "            my_hash(text),  # A unique ID for each string, generated using the my_hash() function\n",
    "            embedding,  # The vector embedding of the string\n",
    "            dict(text=text, date_uploaded=now)  # A dictionary of metadata, including the original text and the current UTC date and time\n",
    "        ) \n",
    "        for text, embedding in zip(texts, embeddings)  # Iterate over each input string and its corresponding vector embedding\n",
    "    ]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4c40d99a",
   "metadata": {},
   "outputs": [],
   "source": [
    "texts = ['hi']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "e065c037",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('49f68a5c8493ec2c0bf489821c21fc3b',\n",
       " [-0.035158250480890274,\n",
       "  -0.02061408758163452,\n",
       "  -0.015398157760500908,\n",
       "  -0.03980923071503639,\n",
       "  -0.027459172531962395,\n",
       "  0.02111334539949894,\n",
       "  -0.022059306502342224,\n",
       "  -0.019418496638536453,\n",
       "  -0.009446481242775917,\n",
       "  -0.013131790794432163,\n",
       "  0.02958758734166622,\n",
       "  -0.004680540878325701,\n",
       "  -0.015227359719574451,\n",
       "  -0.014058045111596584,\n",
       "  0.008966931141912937,\n",
       "  0.015187944285571575,\n",
       "  0.03833773359656334,\n",
       "  -0.005708617623895407,\n",
       "  0.023951230570673943,\n",
       "  -0.012783624231815338,\n",
       "  -0.014951453544199467,\n",
       "  -0.0031351412180811167,\n",
       "  -0.006910777185112238,\n",
       "  -0.008474241942167282,\n",
       "  -0.022729363292455673,\n",
       "  -0.00014329023542813957,\n",
       "  0.013479957357048988,\n",
       "  -0.01696162298321724,\n",
       "  0.0044933189637959,\n",
       "  -0.02238776534795761,\n",
       "  0.014544164761900902,\n",
       "  -0.0009976942092180252,\n",
       "  -0.044880639761686325,\n",
       "  -0.009676402434706688,\n",
       "  -0.009788078255951405,\n",
       "  -0.01576603204011917,\n",
       "  0.00986033957451582,\n",
       "  -0.021218450739979744,\n",
       "  0.015069698914885521,\n",
       "  -0.005547672510147095,\n",
       "  0.00826402846723795,\n",
       "  -0.005048414692282677,\n",
       "  0.006950192619115114,\n",
       "  -0.013342004269361496,\n",
       "  -0.01830173470079899,\n",
       "  -0.008395412005484104,\n",
       "  -0.0022959283087402582,\n",
       "  0.006312982179224491,\n",
       "  -0.01013624481856823,\n",
       "  0.020863715559244156,\n",
       "  0.02180967852473259,\n",
       "  0.005245490465313196,\n",
       "  -0.028037261217832565,\n",
       "  -0.01087856199592352,\n",
       "  -0.01629156619310379,\n",
       "  -0.006644725799560547,\n",
       "  -0.01436022762209177,\n",
       "  0.014412781223654747,\n",
       "  -0.011213590390980244,\n",
       "  -0.01861705631017685,\n",
       "  -0.01013624481856823,\n",
       "  0.003954646177589893,\n",
       "  -0.002704859944060445,\n",
       "  0.012113568373024464,\n",
       "  0.006158606149256229,\n",
       "  0.0013072668807581067,\n",
       "  0.02230893447995186,\n",
       "  -0.00036356307100504637,\n",
       "  0.0008248426602222025,\n",
       "  0.006877931300550699,\n",
       "  0.030139397829771042,\n",
       "  0.030008014291524887,\n",
       "  -0.004631272051483393,\n",
       "  -0.013545649126172066,\n",
       "  0.024200858548283577,\n",
       "  -0.011640586890280247,\n",
       "  -0.0002886333386413753,\n",
       "  0.004315951373428106,\n",
       "  -0.006772824563086033,\n",
       "  -0.008185198530554771,\n",
       "  0.026408104225993156,\n",
       "  -0.026697147637605667,\n",
       "  -0.014806931838393211,\n",
       "  0.015660924836993217,\n",
       "  0.0025652647018432617,\n",
       "  0.014439057558774948,\n",
       "  0.0004586108843795955,\n",
       "  0.02000972256064415,\n",
       "  -0.012422319501638412,\n",
       "  -0.03192621469497681,\n",
       "  0.006335974205285311,\n",
       "  0.02576432377099991,\n",
       "  0.01408432237803936,\n",
       "  0.01030704379081726,\n",
       "  -0.011410665698349476,\n",
       "  0.020903130993247032,\n",
       "  -0.003102295333519578,\n",
       "  0.01324346661567688,\n",
       "  -0.008947223424911499,\n",
       "  0.005196221638470888,\n",
       "  0.0015667494153603911,\n",
       "  0.008020969107747078,\n",
       "  -0.007147267926484346,\n",
       "  -0.006188167724758387,\n",
       "  -0.036839962005615234,\n",
       "  -0.01559523306787014,\n",
       "  -0.01464927103370428,\n",
       "  -0.007731924764811993,\n",
       "  0.01710614562034607,\n",
       "  -0.009538449347019196,\n",
       "  -0.006976469419896603,\n",
       "  0.03678740933537483,\n",
       "  -0.0025554110761731863,\n",
       "  -0.03232036530971527,\n",
       "  -0.010812870226800442,\n",
       "  -0.007114422041922808,\n",
       "  0.00987347774207592,\n",
       "  1.12458719740971e-05,\n",
       "  -0.019444772973656654,\n",
       "  -0.022374628111720085,\n",
       "  0.006877931300550699,\n",
       "  -0.00469696382060647,\n",
       "  0.008789562620222569,\n",
       "  0.0018902815645560622,\n",
       "  0.017579125240445137,\n",
       "  0.008132644928991795,\n",
       "  -0.02956131100654602,\n",
       "  -0.007061868440359831,\n",
       "  0.008198336698114872,\n",
       "  0.0016948484117165208,\n",
       "  0.04314637556672096,\n",
       "  0.012304074130952358,\n",
       "  0.006227582693099976,\n",
       "  -0.008139214478433132,\n",
       "  -0.03260941058397293,\n",
       "  0.030034290626645088,\n",
       "  -0.024292828515172005,\n",
       "  0.027721939608454704,\n",
       "  -0.021743986755609512,\n",
       "  -0.03295100852847099,\n",
       "  -0.010596087202429771,\n",
       "  0.03720783442258835,\n",
       "  -0.002254870953038335,\n",
       "  -0.02131042070686817,\n",
       "  -0.013782139867544174,\n",
       "  -0.01078659389168024,\n",
       "  0.017710508778691292,\n",
       "  -0.002658875659108162,\n",
       "  -0.0038396858144551516,\n",
       "  -0.002254870953038335,\n",
       "  0.00283131655305624,\n",
       "  -0.0008043139823712409,\n",
       "  0.022808192297816277,\n",
       "  -0.0015240497887134552,\n",
       "  0.02874673157930374,\n",
       "  0.022571703419089317,\n",
       "  -0.007948707789182663,\n",
       "  -0.010582949034869671,\n",
       "  -0.009531879797577858,\n",
       "  -0.006303128320723772,\n",
       "  0.0008556356769986451,\n",
       "  -0.004325805231928825,\n",
       "  0.023123513907194138,\n",
       "  -0.006335974205285311,\n",
       "  -0.005110822152346373,\n",
       "  0.02027248963713646,\n",
       "  0.02085057832300663,\n",
       "  0.02953503280878067,\n",
       "  0.011890215799212456,\n",
       "  -0.0006606131792068481,\n",
       "  0.010727470740675926,\n",
       "  0.02204616740345955,\n",
       "  -0.030586102977395058,\n",
       "  0.005022138357162476,\n",
       "  -0.0033469973132014275,\n",
       "  0.014951453544199467,\n",
       "  0.010471273213624954,\n",
       "  0.015805447474122047,\n",
       "  -0.031610894948244095,\n",
       "  -0.04304126650094986,\n",
       "  -0.0165280569344759,\n",
       "  0.014412781223654747,\n",
       "  0.035736341029405594,\n",
       "  0.03886326774954796,\n",
       "  -0.006897639017552137,\n",
       "  0.01056981086730957,\n",
       "  0.03145323321223259,\n",
       "  0.005245490465313196,\n",
       "  0.01861705631017685,\n",
       "  -0.025685494765639305,\n",
       "  0.016422949731349945,\n",
       "  0.026552625000476837,\n",
       "  0.012468303553760052,\n",
       "  -0.00901948381215334,\n",
       "  -0.6991709470748901,\n",
       "  -0.02663145586848259,\n",
       "  -0.007771340198814869,\n",
       "  -0.005094399210065603,\n",
       "  0.015069698914885521,\n",
       "  0.04997832328081131,\n",
       "  0.029613863676786423,\n",
       "  0.022611116990447044,\n",
       "  -0.023675324395298958,\n",
       "  0.018196629360318184,\n",
       "  -0.01084571611136198,\n",
       "  0.006227582693099976,\n",
       "  0.014005491510033607,\n",
       "  -0.013190913945436478,\n",
       "  1.6025207514758222e-05,\n",
       "  0.0036918791010975838,\n",
       "  0.0019379081204533577,\n",
       "  -0.003711586818099022,\n",
       "  -0.011292420327663422,\n",
       "  0.008513657376170158,\n",
       "  -0.005186367779970169,\n",
       "  -0.005041845608502626,\n",
       "  -0.025619802996516228,\n",
       "  0.02622416615486145,\n",
       "  0.009834062308073044,\n",
       "  -0.005603510420769453,\n",
       "  0.0004836558655370027,\n",
       "  -0.010681486688554287,\n",
       "  -0.024529317393898964,\n",
       "  0.013769001699984074,\n",
       "  -0.009702678769826889,\n",
       "  0.007935569621622562,\n",
       "  0.003954646177589893,\n",
       "  -0.015516403131186962,\n",
       "  0.055916860699653625,\n",
       "  0.0018015976529568434,\n",
       "  -0.009774940088391304,\n",
       "  0.022164413705468178,\n",
       "  0.0006967436638660729,\n",
       "  0.056442394852638245,\n",
       "  0.0013499665074050426,\n",
       "  -0.016212737187743187,\n",
       "  9.051867891685106e-06,\n",
       "  0.0017950284527614713,\n",
       "  0.005238920915871859,\n",
       "  0.010037707164883614,\n",
       "  -0.009242836385965347,\n",
       "  -0.0016948484117165208,\n",
       "  0.011476357467472553,\n",
       "  -0.003212329000234604,\n",
       "  0.0076728020794689655,\n",
       "  -0.009255974553525448,\n",
       "  -0.004470326937735081,\n",
       "  0.015411295928061008,\n",
       "  0.004243690520524979,\n",
       "  -0.0028148936107754707,\n",
       "  0.015490126796066761,\n",
       "  -0.022703086957335472,\n",
       "  -0.0027623402420431376,\n",
       "  0.022716224193572998,\n",
       "  0.0011249721283093095,\n",
       "  0.002791901584714651,\n",
       "  -0.01505656074732542,\n",
       "  -0.01957615651190281,\n",
       "  -0.014123736880719662,\n",
       "  0.016239013522863388,\n",
       "  -0.020955683663487434,\n",
       "  -0.014741240069270134,\n",
       "  0.0040630376897752285,\n",
       "  -0.008224613033235073,\n",
       "  -0.010175660252571106,\n",
       "  0.018367426469922066,\n",
       "  -0.029955461621284485,\n",
       "  -0.011469787918031216,\n",
       "  -3.817052913746011e-07,\n",
       "  0.024358520284295082,\n",
       "  0.0213498342782259,\n",
       "  -0.0010420362232252955,\n",
       "  -0.005344028118997812,\n",
       "  0.007896154187619686,\n",
       "  -0.0034323965664952993,\n",
       "  -0.007350912317633629,\n",
       "  0.002169471699744463,\n",
       "  -0.009426773525774479,\n",
       "  0.02535703405737877,\n",
       "  -0.02195419929921627,\n",
       "  -0.033003561198711395,\n",
       "  0.0165806096047163,\n",
       "  -0.0035407880786806345,\n",
       "  -0.006342543289065361,\n",
       "  0.01181795448064804,\n",
       "  0.028667902573943138,\n",
       "  0.01366389449685812,\n",
       "  -0.021967338398098946,\n",
       "  0.0030480995774269104,\n",
       "  0.007278651464730501,\n",
       "  -0.014583579264581203,\n",
       "  0.006753116846084595,\n",
       "  -0.0006700563826598227,\n",
       "  -0.01852508820593357,\n",
       "  -0.004660833161324263,\n",
       "  0.0033864122815430164,\n",
       "  -0.003856108756735921,\n",
       "  -0.008881531655788422,\n",
       "  0.01861705631017685,\n",
       "  -0.004335658624768257,\n",
       "  -0.013729586265981197,\n",
       "  0.02874673157930374,\n",
       "  0.026933638378977776,\n",
       "  -0.017907584086060524,\n",
       "  0.0025833300314843655,\n",
       "  -0.005334174260497093,\n",
       "  -0.0035276496782898903,\n",
       "  0.010970531031489372,\n",
       "  0.00939392764121294,\n",
       "  -0.032110150903463364,\n",
       "  -0.013414265587925911,\n",
       "  0.03250430151820183,\n",
       "  0.023767292499542236,\n",
       "  -0.01664630137383938,\n",
       "  0.0032205404713749886,\n",
       "  -0.004289674572646618,\n",
       "  0.019944030791521072,\n",
       "  -0.012317212298512459,\n",
       "  -0.0019871769472956657,\n",
       "  0.021560048684477806,\n",
       "  -0.009157436899840832,\n",
       "  -0.003570349421352148,\n",
       "  -0.025304481387138367,\n",
       "  0.006276851519942284,\n",
       "  0.008283736184239388,\n",
       "  -0.0032780207693576813,\n",
       "  0.021730847656726837,\n",
       "  -0.020377596840262413,\n",
       "  0.013414265587925911,\n",
       "  -0.008027537725865841,\n",
       "  0.003452104050666094,\n",
       "  -0.01559523306787014,\n",
       "  -0.00355064170435071,\n",
       "  -0.011561756953597069,\n",
       "  -0.00779761653393507,\n",
       "  -0.0009221486398018897,\n",
       "  0.004667402245104313,\n",
       "  -0.0055903722532093525,\n",
       "  0.007488865405321121,\n",
       "  -0.028063537552952766,\n",
       "  -0.030139397829771042,\n",
       "  -0.012612825259566307,\n",
       "  -0.007252374663949013,\n",
       "  0.003645894816145301,\n",
       "  0.0026358834002166986,\n",
       "  -0.0035440726205706596,\n",
       "  -0.009682971052825451,\n",
       "  0.020311905071139336,\n",
       "  0.03197876736521721,\n",
       "  -0.017237529158592224,\n",
       "  -0.0012169405817985535,\n",
       "  -0.02175712399184704,\n",
       "  -0.009242836385965347,\n",
       "  -3.0151510145515203e-05,\n",
       "  0.001246501924470067,\n",
       "  0.025159958750009537,\n",
       "  -0.021875370293855667,\n",
       "  0.001926411991007626,\n",
       "  0.0035834878217428923,\n",
       "  -0.030034290626645088,\n",
       "  -0.004407919943332672,\n",
       "  0.021047653630375862,\n",
       "  0.001418942934833467,\n",
       "  -0.03957274183630943,\n",
       "  0.01710614562034607,\n",
       "  -0.017605401575565338,\n",
       "  -0.005242205690592527,\n",
       "  0.002985692350193858,\n",
       "  0.021060790866613388,\n",
       "  0.000683194724842906,\n",
       "  -0.0063064126297831535,\n",
       "  0.006247290410101414,\n",
       "  0.002841170411556959,\n",
       "  -0.032740794122219086,\n",
       "  0.005064837634563446,\n",
       "  0.0033617778681218624,\n",
       "  -0.009137729182839394,\n",
       "  0.009788078255951405,\n",
       "  0.019944030791521072,\n",
       "  0.009604141116142273,\n",
       "  0.016856515780091286,\n",
       "  0.02764310874044895,\n",
       "  -0.014583579264581203,\n",
       "  0.006198021583259106,\n",
       "  0.008132644928991795,\n",
       "  0.014176290482282639,\n",
       "  -0.01638353429734707,\n",
       "  -0.003915231209248304,\n",
       "  0.0015733186155557632,\n",
       "  0.01151577290147543,\n",
       "  0.0028395280241966248,\n",
       "  0.008408550173044205,\n",
       "  0.005186367779970169,\n",
       "  0.022440319880843163,\n",
       "  0.03471154719591141,\n",
       "  0.008270598016679287,\n",
       "  0.03928369656205177,\n",
       "  -0.005659348797053099,\n",
       "  -0.002721282886341214,\n",
       "  -0.03197876736521721,\n",
       "  0.0035769185051321983,\n",
       "  -0.009301958605647087,\n",
       "  0.021179037168622017,\n",
       "  0.020141106098890305,\n",
       "  0.0003194263845216483,\n",
       "  -0.018984930589795113,\n",
       "  0.004920315928757191,\n",
       "  0.0031433526892215014,\n",
       "  0.019011206924915314,\n",
       "  0.018130937591195107,\n",
       "  -0.008126075379550457,\n",
       "  0.013085806742310524,\n",
       "  -0.01974695548415184,\n",
       "  0.006523195654153824,\n",
       "  0.009006345644593239,\n",
       "  -0.014425919391214848,\n",
       "  0.012179260142147541,\n",
       "  -0.002391181420534849,\n",
       "  0.011502633802592754,\n",
       "  0.01739518903195858,\n",
       "  0.018551364541053772,\n",
       "  0.03129557520151138,\n",
       "  0.01338798925280571,\n",
       "  -0.0033010130282491446,\n",
       "  -0.007252374663949013,\n",
       "  -0.003317435970529914,\n",
       "  0.00307109160348773,\n",
       "  0.001276884344406426,\n",
       "  -0.011003376916050911,\n",
       "  0.0031679871026426554,\n",
       "  0.006710417568683624,\n",
       "  -0.022755639627575874,\n",
       "  0.027275236323475838,\n",
       "  -0.00820490624755621,\n",
       "  0.003616333706304431,\n",
       "  0.028037261217832565,\n",
       "  0.03279334679245949,\n",
       "  -0.024384796619415283,\n",
       "  0.0057677398435771465,\n",
       "  0.024818362668156624,\n",
       "  0.014215705916285515,\n",
       "  0.005964815616607666,\n",
       "  0.02027248963713646,\n",
       "  0.024647563695907593,\n",
       "  0.005485265515744686,\n",
       "  0.011213590390980244,\n",
       "  -0.0056297872215509415,\n",
       "  0.011377819813787937,\n",
       "  0.012579979375004768,\n",
       "  -0.012061014771461487,\n",
       "  -0.01698789931833744,\n",
       "  -0.0034915192518383265,\n",
       "  0.009420203976333141,\n",
       "  0.018551364541053772,\n",
       "  0.00010972583550028503,\n",
       "  0.019904615357518196,\n",
       "  0.01965498737990856,\n",
       "  -0.023110374808311462,\n",
       "  0.01583172380924225,\n",
       "  -0.003154848702251911,\n",
       "  -0.0028444549534469843,\n",
       "  -0.001528976601548493,\n",
       "  0.0007768055656924844,\n",
       "  0.0012046233750879765,\n",
       "  -0.003902093041688204,\n",
       "  -0.0028641624376177788,\n",
       "  0.025659216567873955,\n",
       "  -0.02018052153289318,\n",
       "  0.028589071705937386,\n",
       "  0.015240497887134552,\n",
       "  -0.005137098953127861,\n",
       "  -0.0007283578161150217,\n",
       "  -0.00540971988812089,\n",
       "  -0.004381643142551184,\n",
       "  -0.02224324271082878,\n",
       "  -0.03536846488714218,\n",
       "  0.016948483884334564,\n",
       "  0.01844625733792782,\n",
       "  -0.004775793757289648,\n",
       "  -0.02509426698088646,\n",
       "  -0.034054629504680634,\n",
       "  0.007837031967937946,\n",
       "  0.0037312943022698164,\n",
       "  0.007212959695607424,\n",
       "  -0.006526480428874493,\n",
       "  0.01713242195546627,\n",
       "  0.03121674433350563,\n",
       "  -0.023110374808311462,\n",
       "  -0.003148279618471861,\n",
       "  0.0039349389262497425,\n",
       "  0.04227924346923828,\n",
       "  -0.008802700787782669,\n",
       "  0.00283131655305624,\n",
       "  0.010471273213624954,\n",
       "  -0.0021316988859325647,\n",
       "  -0.00023443761165253818,\n",
       "  0.001714556012302637,\n",
       "  -0.00494330795481801,\n",
       "  -0.006125760264694691,\n",
       "  0.021533772349357605,\n",
       "  -0.008362566120922565,\n",
       "  -0.02276877872645855,\n",
       "  -0.007291789632290602,\n",
       "  0.023622771725058556,\n",
       "  -0.0012506076600402594,\n",
       "  0.026237305253744125,\n",
       "  -0.013979215174913406,\n",
       "  0.010372735559940338,\n",
       "  0.001501878839917481,\n",
       "  0.00994573812931776,\n",
       "  -0.006191452033817768,\n",
       "  -0.0007796795689500868,\n",
       "  0.02358335629105568,\n",
       "  -0.011719416826963425,\n",
       "  -0.020811162889003754,\n",
       "  -0.0011840946972370148,\n",
       "  -0.014452195726335049,\n",
       "  0.0003266114217694849,\n",
       "  0.053683340549468994,\n",
       "  0.02125786617398262,\n",
       "  0.001563464873470366,\n",
       "  -0.0024847923777997494,\n",
       "  0.008717302232980728,\n",
       "  0.020141106098890305,\n",
       "  0.005344028118997812,\n",
       "  -0.020771747455000877,\n",
       "  -0.01434708945453167,\n",
       "  -0.013900385238230228,\n",
       "  0.009183713234961033,\n",
       "  -0.01350623369216919,\n",
       "  -0.0058005861937999725,\n",
       "  -0.0009853770025074482,\n",
       "  0.009545018896460533,\n",
       "  -0.0008622048771940172,\n",
       "  0.02236148901283741,\n",
       "  0.0006154500297270715,\n",
       "  0.0074954344891011715,\n",
       "  -0.007508572656661272,\n",
       "  -0.003931654151529074,\n",
       "  0.011161036789417267,\n",
       "  0.010701194405555725,\n",
       "  0.023649048060178757,\n",
       "  0.011601172387599945,\n",
       "  0.015805447474122047,\n",
       "  0.024292828515172005,\n",
       "  0.02442421205341816,\n",
       "  0.00704873027279973,\n",
       "  -0.02732778899371624,\n",
       "  -0.005120676010847092,\n",
       "  0.0026457372587174177,\n",
       "  0.014110598713159561,\n",
       "  0.009853770025074482,\n",
       "  -0.004775793757289648,\n",
       "  0.028667902573943138,\n",
       "  -0.010937685146927834,\n",
       "  0.009840631857514381,\n",
       "  0.017986414954066277,\n",
       "  -0.0064279427751898766,\n",
       "  0.0038396858144551516,\n",
       "  0.015490126796066761,\n",
       "  0.00731149734929204,\n",
       "  -0.014333951286971569,\n",
       "  0.02142866514623165,\n",
       "  -0.021034514531493187,\n",
       "  -0.016501780599355698,\n",
       "  0.03061237931251526,\n",
       "  -0.006519911345094442,\n",
       "  -0.010714332573115826,\n",
       "  0.028589071705937386,\n",
       "  0.0005698763416148722,\n",
       "  -0.010385873727500439,\n",
       "  -0.01103622280061245,\n",
       "  -0.006033791694790125,\n",
       "  0.010635502636432648,\n",
       "  -0.01844625733792782,\n",
       "  -0.005192936863750219,\n",
       "  -0.005238920915871859,\n",
       "  0.014517887495458126,\n",
       "  -0.015214220620691776,\n",
       "  -0.016593748703598976,\n",
       "  0.012126706540584564,\n",
       "  -0.008047245442867279,\n",
       "  -0.020062275230884552,\n",
       "  -0.03116418980062008,\n",
       "  -0.02979779988527298,\n",
       "  -0.012409181334078312,\n",
       "  -0.015936831012368202,\n",
       "  0.003366704797372222,\n",
       "  -0.014767516404390335,\n",
       "  -0.0008026717114262283,\n",
       "  -0.018275458365678787,\n",
       "  -0.010977099649608135,\n",
       "  0.006069922354072332,\n",
       "  0.0113712502643466,\n",
       "  0.00469039473682642,\n",
       "  -0.00848081149160862,\n",
       "  0.023832984268665314,\n",
       "  0.00871073268353939,\n",
       "  -0.013006976805627346,\n",
       "  -0.03192621469497681,\n",
       "  -0.001965827075764537,\n",
       "  -0.01907689869403839,\n",
       "  0.01367703266441822,\n",
       "  0.012087291106581688,\n",
       "  -0.011088775470852852,\n",
       "  -0.008224613033235073,\n",
       "  -0.003412689082324505,\n",
       "  0.0014953096397221088,\n",
       "  0.026171613484621048,\n",
       "  0.010142814368009567,\n",
       "  0.003117075888440013,\n",
       "  -0.007436311803758144,\n",
       "  -0.0011307201348245144,\n",
       "  -0.011450081132352352,\n",
       "  0.016068214550614357,\n",
       "  0.005534534342586994,\n",
       "  -0.012816470116376877,\n",
       "  -0.013131790794432163,\n",
       "  0.006779393646866083,\n",
       "  -0.0005238921148702502,\n",
       "  0.0042732516303658485,\n",
       "  -0.008953792043030262,\n",
       "  0.01571347936987877,\n",
       "  0.00037136394530534744,\n",
       "  0.008172060362994671,\n",
       "  0.01664630137383938,\n",
       "  -0.018196629360318184,\n",
       "  -0.013742724433541298,\n",
       "  0.03150578588247299,\n",
       "  -0.010871993377804756,\n",
       "  0.009288820438086987,\n",
       "  -0.0030021152924746275,\n",
       "  -0.0003873763489536941,\n",
       "  0.017198113724589348,\n",
       "  0.00023998036340344697,\n",
       "  0.019930891692638397,\n",
       "  0.0009591002599336207,\n",
       "  -0.030559824779629707,\n",
       "  0.004677256103605032,\n",
       "  -0.03980923071503639,\n",
       "  0.03400207683444023,\n",
       "  -0.009255974553525448,\n",
       "  0.004017053637653589,\n",
       "  0.001775320852175355,\n",
       "  0.0131383603438735,\n",
       "  -0.01684337668120861,\n",
       "  -0.007488865405321121,\n",
       "  -0.0008991564973257482,\n",
       "  -0.008007830008864403,\n",
       "  0.02317606657743454,\n",
       "  0.01106249913573265,\n",
       "  -0.01233035046607256,\n",
       "  -0.02288702316582203,\n",
       "  -0.012783624231815338,\n",
       "  -0.022519148886203766,\n",
       "  0.006779393646866083,\n",
       "  -0.00017664935148786753,\n",
       "  -0.03203132003545761,\n",
       "  -0.01830173470079899,\n",
       "  -0.006720270961523056,\n",
       "  0.0018262320663779974,\n",
       "  -0.006664433050900698,\n",
       "  -0.022926438599824905,\n",
       "  -0.045090850442647934,\n",
       "  -0.01408432237803936,\n",
       "  0.01531932782381773,\n",
       "  -0.013158068060874939,\n",
       "  0.006976469419896603,\n",
       "  -0.005120676010847092,\n",
       "  0.0020019575022161007,\n",
       "  -0.02509426698088646,\n",
       "  -0.03505314514040947,\n",
       "  -0.012527426704764366,\n",
       "  -0.0317685529589653,\n",
       "  -0.01576603204011917,\n",
       "  -0.0004935096367262304,\n",
       "  0.03523708134889603,\n",
       "  0.026867946609854698,\n",
       "  0.021730847656726837,\n",
       "  0.005570664536207914,\n",
       "  0.007173544727265835,\n",
       "  0.005035276524722576,\n",
       "  0.013952938839793205,\n",
       "  0.013217190280556679,\n",
       "  -0.006858224049210548,\n",
       "  0.009689540602266788,\n",
       "  -0.01489889994263649,\n",
       "  0.024778947234153748,\n",
       "  0.02634241245687008,\n",
       "  0.0179469995200634,\n",
       "  0.004838201217353344,\n",
       "  0.002680225530639291,\n",
       "  0.018682748079299927,\n",
       "  0.01271793246269226,\n",
       "  -0.005077976267784834,\n",
       "  -0.01600252278149128,\n",
       "  -0.008369135670363903,\n",
       "  -0.029193436726927757,\n",
       "  -0.008428257890045643,\n",
       "  -0.012573410756886005,\n",
       "  -0.01614704541862011,\n",
       "  0.011075637303292751,\n",
       "  0.006680855993181467,\n",
       "  -0.012698224745690823,\n",
       "  0.03250430151820183,\n",
       "  0.018012691289186478,\n",
       "  0.03967784717679024,\n",
       "  0.005481980741024017,\n",
       "  0.0220724456012249,\n",
       "  0.005951676983386278,\n",
       "  0.02625044248998165,\n",
       "  0.006316266488283873,\n",
       "  0.010300474241375923,\n",
       "  0.015949968248605728,\n",
       "  -0.005853139329701662,\n",
       "  -0.00565606402233243,\n",
       "  -0.0037871322128921747,\n",
       "  0.01693534664809704,\n",
       "  -0.0003071091778110713,\n",
       "  0.006319551263004541,\n",
       "  -0.013479957357048988,\n",
       "  0.007258943747729063,\n",
       "  -0.015963107347488403,\n",
       "  0.016711993142962456,\n",
       "  -0.0041845678351819515,\n",
       "  -0.01806524582207203,\n",
       "  -0.01302668359130621,\n",
       "  -0.007298359181731939,\n",
       "  -0.0165543332695961,\n",
       "  -0.01436022762209177,\n",
       "  -0.0010280766291543841,\n",
       "  -0.024673840031027794,\n",
       "  0.012106998823583126,\n",
       "  -0.00496629998087883,\n",
       "  -0.018406841903924942,\n",
       "  0.02221696637570858,\n",
       "  -0.016751408576965332,\n",
       "  -0.03142695873975754,\n",
       "  0.004095883574336767,\n",
       "  -0.014504749327898026,\n",
       "  0.030586102977395058,\n",
       "  -0.010017999447882175,\n",
       "  0.01614704541862011,\n",
       "  0.012711362913250923,\n",
       "  -0.026447519659996033,\n",
       "  -0.026184750720858574,\n",
       "  -0.00542285805568099,\n",
       "  0.0026933636981993914,\n",
       "  -0.00040482572512701154,\n",
       "  0.017460880801081657,\n",
       "  0.0018623624928295612,\n",
       "  -0.004256828688085079,\n",
       "  -0.021678294986486435,\n",
       "  0.00871073268353939,\n",
       "  0.000617092358879745,\n",
       "  0.007120991125702858,\n",
       "  -0.01667257957160473,\n",
       "  0.00829030480235815,\n",
       "  0.012152982875704765,\n",
       "  -0.008717302232980728,\n",
       "  -0.023570217192173004,\n",
       "  -0.003179483115673065,\n",
       "  -0.02430596575140953,\n",
       "  0.010799732059240341,\n",
       "  0.00868445634841919,\n",
       "  0.005015568807721138,\n",
       "  -0.009367650374770164,\n",
       "  -0.012560272589325905,\n",
       "  -0.004992576781660318,\n",
       "  0.022138137370347977,\n",
       "  -0.02979779988527298,\n",
       "  0.01244202721863985,\n",
       "  0.022492872551083565,\n",
       "  -0.0165806096047163,\n",
       "  -0.011410665698349476,\n",
       "  0.011200452223420143,\n",
       "  -0.023044683039188385,\n",
       "  0.02230893447995186,\n",
       "  0.009026053361594677,\n",
       "  0.011890215799212456,\n",
       "  -0.01907689869403839,\n",
       "  0.023596495389938354,\n",
       "  0.01109534502029419,\n",
       "  -0.0021678293123841286,\n",
       "  -0.0221249982714653,\n",
       "  -0.0011553545482456684,\n",
       "  0.00033030658960342407,\n",
       "  0.029351096600294113,\n",
       "  -0.007580833975225687,\n",
       "  0.00013928714906796813,\n",
       "  -0.0019329811912029982,\n",
       "  0.012192398309707642,\n",
       "  0.006739978678524494,\n",
       "  -0.0016570757143199444,\n",
       "  -0.010694624856114388,\n",
       "  -0.02262425608932972,\n",
       "  -0.025343896821141243,\n",
       "  -0.0016751409275457263,\n",
       "  0.03923114389181137,\n",
       "  0.019812647253274918,\n",
       "  -0.0179732758551836,\n",
       "  -0.01960243284702301,\n",
       "  -0.01660688780248165,\n",
       "  -0.024910330772399902,\n",
       "  -0.02639496512711048,\n",
       "  0.003948077093809843,\n",
       "  0.0017047022702172399,\n",
       "  0.0025948260445147753,\n",
       "  -0.02226952090859413,\n",
       "  -0.02584315463900566,\n",
       "  0.0035802030470222235,\n",
       "  0.017434604465961456,\n",
       "  0.017841892316937447,\n",
       "  -0.009650125168263912,\n",
       "  -0.011331835761666298,\n",
       "  0.01727694272994995,\n",
       "  -0.020745471119880676,\n",
       "  0.010898269712924957,\n",
       "  -0.006487065460532904,\n",
       "  -0.0018607202218845487,\n",
       "  -0.04367190971970558,\n",
       "  0.018840407952666283,\n",
       "  -0.005229067523032427,\n",
       "  -0.018603917211294174,\n",
       "  0.008132644928991795,\n",
       "  -0.00853993371129036,\n",
       "  -0.03481665253639221,\n",
       "  -0.007580833975225687,\n",
       "  0.010024568997323513,\n",
       "  0.015017145313322544,\n",
       "  0.0234913881868124,\n",
       "  -0.008053814992308617,\n",
       "  0.021901646628975868,\n",
       "  0.008559641428291798,\n",
       "  -0.003461957909166813,\n",
       "  -0.003425827482715249,\n",
       "  -0.022295797243714333,\n",
       "  0.009091745130717754,\n",
       "  -0.02445048838853836,\n",
       "  0.0067465477623045444,\n",
       "  -0.00869759451597929,\n",
       "  0.00041467949631623924,\n",
       "  0.018853547051548958,\n",
       "  -0.014990868978202343,\n",
       "  0.02404319867491722,\n",
       "  -0.00305959559045732,\n",
       "  -0.02046956494450569,\n",
       "  -0.02012796700000763,\n",
       "  -0.0070355916395783424,\n",
       "  0.021612603217363358,\n",
       "  -0.011739124543964863,\n",
       "  0.01756598800420761,\n",
       "  -0.003429112024605274,\n",
       "  -0.006700563710182905,\n",
       "  -0.022230105474591255,\n",
       "  0.010937685146927834,\n",
       "  0.004903892986476421,\n",
       "  -0.008612195029854774,\n",
       "  0.016357257962226868,\n",
       "  0.003619618248194456,\n",
       "  -0.011942769400775433,\n",
       "  0.005885985214263201,\n",
       "  -0.0051797982305288315,\n",
       "  -0.017789339646697044,\n",
       "  0.0013064456870779395,\n",
       "  -0.029167158529162407,\n",
       "  -0.00562650291249156,\n",
       "  0.002599752973765135,\n",
       "  -0.005744747817516327,\n",
       "  0.018866684287786484,\n",
       "  0.013887246139347553,\n",
       "  -0.03331888094544411,\n",
       "  -0.02178340032696724,\n",
       "  0.006976469419896603,\n",
       "  -0.02238776534795761,\n",
       "  -0.006785963196307421,\n",
       "  0.0019083467777818441,\n",
       "  0.015516403131186962,\n",
       "  0.02985035441815853,\n",
       "  0.006349112372845411,\n",
       "  0.006230867467820644,\n",
       "  0.01989147625863552,\n",
       "  0.007081576157361269,\n",
       "  0.01684337668120861,\n",
       "  -0.0037772785872220993,\n",
       "  -0.017145559191703796,\n",
       "  0.00710128340870142,\n",
       "  -0.004917031154036522,\n",
       "  0.015476987697184086,\n",
       "  0.0029889768920838833,\n",
       "  -0.035210803151130676,\n",
       "  -0.020088553428649902,\n",
       "  0.01972067914903164,\n",
       "  -0.003524365136399865,\n",
       "  -0.012974130921065807,\n",
       "  0.023386280983686447,\n",
       "  -0.006910777185112238,\n",
       "  -0.0024765809066593647,\n",
       "  -0.015660924836993217,\n",
       "  0.010550103150308132,\n",
       "  0.0007345164194703102,\n",
       "  0.0004462936776690185,\n",
       "  0.008559641428291798,\n",
       "  -0.00991946179419756,\n",
       "  -0.0072852205485105515,\n",
       "  0.02219069004058838,\n",
       "  -0.024082614108920097,\n",
       "  -0.000259277323493734,\n",
       "  -0.008270598016679287,\n",
       "  0.000587941613048315,\n",
       "  -0.006263713352382183,\n",
       "  0.0033732738811522722,\n",
       "  -0.027485448867082596,\n",
       "  -0.006864793132990599,\n",
       "  0.01600252278149128,\n",
       "  -0.02535703405737877,\n",
       "  0.013874107971787453,\n",
       "  0.005564095452427864,\n",
       "  -0.02663145586848259,\n",
       "  0.03839028626680374,\n",
       "  -0.017171837389469147,\n",
       "  -0.0030168958473950624,\n",
       "  -0.026762839406728745,\n",
       "  0.007212959695607424,\n",
       "  -0.009472757577896118,\n",
       "  -0.015844862908124924,\n",
       "  0.021507496014237404,\n",
       "  -0.0089734997600317,\n",
       "  -0.00916400644928217,\n",
       "  -0.0099785840138793,\n",
       "  -0.005353881511837244,\n",
       "  -0.0331612192094326,\n",
       "  -0.0006515805725939572,\n",
       "  -0.0332137756049633,\n",
       "  -0.0007698257686570287,\n",
       "  -0.019536741077899933,\n",
       "  0.03266196325421333,\n",
       "  -0.002535703592002392,\n",
       "  0.002622745232656598,\n",
       "  0.0025734761729836464,\n",
       "  -0.008296874351799488,\n",
       "  -0.01725066639482975,\n",
       "  -0.00846767332404852,\n",
       "  -0.007410035002976656,\n",
       "  0.020193658769130707,\n",
       "  0.013269743882119656,\n",
       "  -0.010596087202429771,\n",
       "  -0.0010518899653106928,\n",
       "  -0.003708302043378353,\n",
       "  -0.02622416615486145,\n",
       "  -0.006818809080868959,\n",
       "  0.01826232112944126,\n",
       "  -0.0007102926028892398,\n",
       "  0.020219936966896057,\n",
       "  0.19654986262321472,\n",
       "  -0.0039250850677490234,\n",
       "  0.0016997753409668803,\n",
       "  0.04333031177520752,\n",
       "  0.01229750458151102,\n",
       "  0.002148121828213334,\n",
       "  0.016015660017728806,\n",
       "  0.022374628111720085,\n",
       "  -0.016751408576965332,\n",
       "  0.022177550941705704,\n",
       "  -0.02613219805061817,\n",
       "  0.010044276714324951,\n",
       "  -0.019944030791521072,\n",
       "  0.003796986071392894,\n",
       "  0.01460985653102398,\n",
       "  0.0014452196191996336,\n",
       "  -0.03429111838340759,\n",
       "  -0.006450934801250696,\n",
       "  -0.02651321142911911,\n",
       "  -0.007160406094044447,\n",
       "  -0.011614310555160046,\n",
       "  -0.006891069933772087,\n",
       "  -0.014386503957211971,\n",
       "  -0.003222182858735323,\n",
       "  0.03063865564763546,\n",
       "  0.007350912317633629,\n",
       "  0.005741463508456945,\n",
       "  0.006414804141968489,\n",
       "  0.015910554677248,\n",
       "  0.00562650291249156,\n",
       "  -0.011923061683773994,\n",
       "  -0.03140068054199219,\n",
       "  0.016396673396229744,\n",
       "  0.00876985490322113,\n",
       "  -0.0001806524523999542,\n",
       "  2.9561309929704294e-05,\n",
       "  -0.008237752132117748,\n",
       "  0.004056468605995178,\n",
       "  0.015923691913485527,\n",
       "  -0.004260113462805748,\n",
       "  -0.004151721950620413,\n",
       "  0.010326750576496124,\n",
       "  0.014938315376639366,\n",
       "  -0.014531026594340801,\n",
       "  -0.017040453851222992,\n",
       "  0.005110822152346373,\n",
       "  ...],\n",
       " {'text': 'hi',\n",
       "  'date_uploaded': datetime.datetime(2023, 7, 31, 13, 9, 38, 861790)})"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "prepare_for_pinecone(texts)[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "3e1b73f3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "ID:   49f68a5c8493ec2c0bf489821c21fc3b \n",
      "LEN:  1536 \n",
      "META: {'text': 'hi', 'date_uploaded': datetime.datetime(2023, 7, 31, 13, 9, 40, 102999)}\n"
     ]
    }
   ],
   "source": [
    "_id, embedding, metadata = prepare_for_pinecone(texts)[0]\n",
    "\n",
    "print('ID:  ',_id, '\\nLEN: ', len(embedding), '\\nMETA:', metadata)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b49debd5",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "bf47aabd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def upload_texts_to_pinecone(texts, namespace=NAMESPACE, batch_size=None, show_progress_bar=False):\n",
    "    # Call the prepare_for_pinecone function to prepare the input texts for indexing\n",
    "    total_upserted = 0\n",
    "    if not batch_size:\n",
    "        batch_size = len(texts)\n",
    "\n",
    "    _range = range(0, len(texts), batch_size)\n",
    "    for i in tqdm(_range) if show_progress_bar else _range:\n",
    "        batch = texts[i: i + batch_size]\n",
    "        prepared_texts = prepare_for_pinecone(batch)\n",
    "\n",
    "        # Use the upsert() method of the index object to upload the prepared texts to Pinecone\n",
    "        total_upserted += index.upsert(\n",
    "            prepared_texts,\n",
    "            namespace=namespace\n",
    "        )['upserted_count']\n",
    "\n",
    "    return total_upserted\n",
    "\n",
    "\n",
    "# Call the upload_texts_to_pinecone() function with the input texts\n",
    "upload_texts_to_pinecone(texts)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "e8ed7e8e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[{'id': '093601540a641d12a6f734a9fa624ce5',\n",
       "  'metadata': {'date_uploaded': datetime.datetime(2023, 7, 31, 13, 17, 57, 390314),\n",
       "               'text': \"Alexander Graham Bell originally suggested 'ahoy-hoy' \"\n",
       "                       'be adopted as the standard greeting when answering a '\n",
       "                       \"telephone, before 'hello' (suggested by Thomas Edison) \"\n",
       "                       'became common.'},\n",
       "  'score': 0.780635893,\n",
       "  'values': []},\n",
       " {'id': '9588c26cecaaf486eae14858827a6699',\n",
       "  'metadata': {'date_uploaded': datetime.datetime(2023, 7, 31, 13, 17, 45, 374073),\n",
       "               'text': 'The Abbott family -- wife Evelyn, husband Lee, '\n",
       "                       'congenitally deaf daughter Regan, and sons Marcus and '\n",
       "                       'Beau -- silently scavenge for supplies in a deserted '\n",
       "                       'town. While out in the open, the family communicates '\n",
       "                       'with American Sign Language (ASL). Four-year-old Beau '\n",
       "                       'is drawn to a battery-powered space shuttle toy, but '\n",
       "                       'Lee takes it away due to the noise it makes. Regan '\n",
       "                       'returns the toy to Beau, who also takes the batteries '\n",
       "                       'that his father removed from it. Beau activates the toy '\n",
       "                       'when the family is walking home and crossing a bridge, '\n",
       "                       'giving away his location to a nearby creature which '\n",
       "                       'kills him before Lee can save him.'},\n",
       "  'score': 0.772896409,\n",
       "  'values': []},\n",
       " {'id': 'b19c9b4fb43bb3327d1e108e51b79e24',\n",
       "  'metadata': {'date_uploaded': datetime.datetime(2023, 7, 31, 13, 17, 45, 374073),\n",
       "               'text': \"The second part of the novel begins with Pi's family \"\n",
       "                       'aboard the Tsimtsum, a Japanese freighter that is '\n",
       "                       'transporting animals from their zoo to North America. A '\n",
       "                       'few days out of port from Manila, the ship encounters a '\n",
       "                       'storm and sinks. Pi manages to escape in a small '\n",
       "                       'lifeboat, only to learn that the boat also holds a '\n",
       "                       \"spotted hyena, an injured Grant's zebra, and an \"\n",
       "                       \"orangutan named Orange Juice. Much to the boy's \"\n",
       "                       'distress, the hyena kills the zebra and then Orange '\n",
       "                       \"Juice. A tiger has been hiding under the boat's \"\n",
       "                       \"tarpaulin: it's Richard Parker, who had boarded the \"\n",
       "                       'lifeboat with ambivalent assistance from Pi himself '\n",
       "                       'some time before the hyena attack. Suddenly emerging '\n",
       "                       'from his hideaway, Richard Parker kills and eats the '\n",
       "                       'hyena.'},\n",
       "  'score': 0.769768238,\n",
       "  'values': []}]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def query_from_pinecone(query, top_k=3):\n",
    "    # get embedding from THE SAME embedder as the documents\n",
    "    query_embedding = get_embedding(query, engine=ENGINE)\n",
    "\n",
    "    return index.query(\n",
    "      vector=query_embedding,\n",
    "      top_k=top_k,\n",
    "      namespace=NAMESPACE,\n",
    "      include_metadata=True   # gets the metadata (dates, text, etc)\n",
    "    ).get('matches')\n",
    "\n",
    "query_from_pinecone('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "84a0871f",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import hashlib\n",
    "\n",
    "def delete_texts_from_pinecone(texts, namespace=NAMESPACE):\n",
    "    # Compute the hash (id) for each text\n",
    "    hashes = [hashlib.md5(text.encode()).hexdigest() for text in texts]\n",
    "    \n",
    "    # The ids parameter is used to specify the list of IDs (hashes) to delete\n",
    "    return index.delete(ids=hashes, namespace=namespace)\n",
    "\n",
    "# delete our text\n",
    "delete_texts_from_pinecone(texts)\n",
    "\n",
    "# test that the index is empty\n",
    "query_from_pinecone('hello')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "6a1bac4f",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "72729603",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[36661, 1070]"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Importing the tiktoken library\n",
    "import tiktoken\n",
    "\n",
    "# Initializing a tokenizer for the 'cl100k_base' model\n",
    "# This tokenizer is designed to work with the 'ada-002' embedding model\n",
    "tokenizer = tiktoken.get_encoding(\"cl100k_base\")\n",
    "\n",
    "# Using the tokenizer to encode the text 'hey there'\n",
    "# The resulting output is a list of integers representing the encoded text\n",
    "# This is the input format required for embedding using the 'ada-002' model\n",
    "tokenizer.encode('hey there')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bbc147d8",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "84f34d65",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to split the text into chunks of a maximum number of tokens. Inspired by OpenAI\n",
    "def overlapping_chunks(text, max_tokens = 500, overlapping_factor = 5):\n",
    "    '''\n",
    "    max_tokens: tokens we want per chunk\n",
    "    overlapping_factor: number of sentences to start each chunk with that overlaps with the previous chunk\n",
    "    '''\n",
    "\n",
    "    # Split the text using punctuation\n",
    "    sentences = re.split(r'[.?!]', text)\n",
    "\n",
    "    # Get the number of tokens for each sentence\n",
    "    n_tokens = [len(tokenizer.encode(\" \" + sentence)) for sentence in sentences]\n",
    "    \n",
    "    chunks, tokens_so_far, chunk = [], 0, []\n",
    "\n",
    "    # Loop through the sentences and tokens joined together in a tuple\n",
    "    for sentence, token in zip(sentences, n_tokens):\n",
    "\n",
    "        # If the number of tokens so far plus the number of tokens in the current sentence is greater \n",
    "        # than the max number of tokens, then add the chunk to the list of chunks and reset\n",
    "        # the chunk and tokens so far\n",
    "        if tokens_so_far + token > max_tokens:\n",
    "            chunks.append(\". \".join(chunk) + \".\")\n",
    "            if overlapping_factor > 0:\n",
    "                chunk = chunk[-overlapping_factor:]\n",
    "                tokens_so_far = sum([len(tokenizer.encode(c)) for c in chunk])\n",
    "            else:\n",
    "                chunk = []\n",
    "                tokens_so_far = 0\n",
    "\n",
    "        # If the number of tokens in the current sentence is greater than the max number of \n",
    "        # tokens, go to the next sentence\n",
    "        if token > max_tokens:\n",
    "            continue\n",
    "\n",
    "        # Otherwise, add the sentence to the chunk and add the number of tokens to the total\n",
    "        chunk.append(sentence)\n",
    "        tokens_so_far += token + 1\n",
    "    if chunk:\n",
    "        chunks.append(\". \".join(chunk) + \".\")\n",
    "\n",
    "    return chunks"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "3164e0ba",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████| 428/428 [07:01<00:00,  1.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "575490\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "import PyPDF2\n",
    "\n",
    "# Open the PDF file in read-binary mode\n",
    "with open('../data/pds2.pdf', 'rb') as file:\n",
    "\n",
    "    # Create a PDF reader object\n",
    "    reader = PyPDF2.PdfReader(file)\n",
    "\n",
    "    # Initialize an empty string to hold the text\n",
    "    principles_of_ds = ''\n",
    "    # Loop through each page in the PDF file\n",
    "    for page in tqdm(reader.pages):\n",
    "        text = page.extract_text()\n",
    "        principles_of_ds += '\\n\\n' + text[text.find(' ]')+2:]\n",
    "\n",
    "# Print the final string containing all the text from the PDF file\n",
    "principles_of_ds = principles_of_ds.strip()\n",
    "\n",
    "print(len(principles_of_ds))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "43e01a07",
   "metadata": {},
   "outputs": [],
   "source": [
    "from urllib.request import urlopen\n",
    "\n",
    "#\n",
    "\n",
    "# A textbook about insects\n",
    "text = urlopen('https://www.gutenberg.org/cache/epub/10834/pg10834.txt').read().decode()\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "dfdac6ff",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "non-overlapping chunking approach has 286 documents with average length 474.1 tokens\n"
     ]
    }
   ],
   "source": [
    "split = overlapping_chunks(principles_of_ds, overlapping_factor=0)\n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in split]) / len(split)\n",
    "print(f'non-overlapping chunking approach has {len(split)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "1730b34b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "overlapping chunking approach has 392 documents with average length 485.2 tokens\n"
     ]
    }
   ],
   "source": [
    "split = overlapping_chunks(principles_of_ds)\n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in split]) / len(split)\n",
    "print(f'overlapping chunking approach has {len(split)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d1381cc8",
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "758cb1ff",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "7fc7dc10",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[(' ', 82259), ('\\n', 9220), ('  ', 1592), ('\\n\\n', 333), ('\\n   ', 250), ('\\n\\n\\n', 82), ('\\n    ', 73), ('\\n ', 46), (' \\n', 39), ('     ', 34)]\n"
     ]
    }
   ],
   "source": [
    "# Importing the Counter and re libraries\n",
    "from collections import Counter\n",
    "import re\n",
    "\n",
    "# Find all occurrences of one or more spaces in 'principles_of_ds'\n",
    "matches = re.findall(r'[\\s]{1,}', principles_of_ds)\n",
    "\n",
    "# The 10 most frequent spaces that occur in the document\n",
    "most_common_spaces = Counter(matches).most_common(10)\n",
    "\n",
    "# Print the most common spaces and their frequencies\n",
    "print(most_common_spaces)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "63ccc253",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "custom delimiter approach has 426 documents with average length 316.3 tokens\n"
     ]
    }
   ],
   "source": [
    "# Only keep documents of at least 100 characters split by a custom delimiter\n",
    "split = list(filter(lambda x: len(x) > 50, principles_of_ds.split('\\n\\n')))\n",
    "\n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in split]) / len(split)\n",
    "print(f'custom delimiter approach has {len(split)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "2480fe3c",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|█████████████████████████████████████████████| 5/5 [00:05<00:00,  1.12s/it]\n"
     ]
    }
   ],
   "source": [
    "embeddings = None\n",
    "for s in tqdm(range(0, len(split), 100)):\n",
    "    if embeddings is None:\n",
    "        embeddings = np.array(get_embeddings(split[s:s+100], engine=ENGINE))\n",
    "    else:\n",
    "        embeddings = np.vstack([embeddings, np.array(get_embeddings(split[s:s+100], engine=ENGINE))])\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "id": "1cfe8fd2",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/sklearn/cluster/_agglomerative.py:983: FutureWarning: Attribute `affinity` was deprecated in version 1.2 and will be removed in 1.4. Use `metric` instead\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cluster 0: 2 embeddings\n",
      "Cluster 1: 3 embeddings\n",
      "Cluster 2: 4 embeddings\n",
      "Cluster 3: 2 embeddings\n",
      "Cluster 4: 2 embeddings\n",
      "Cluster 5: 3 embeddings\n",
      "Cluster 6: 3 embeddings\n",
      "Cluster 7: 2 embeddings\n",
      "Cluster 8: 2 embeddings\n",
      "Cluster 9: 2 embeddings\n",
      "Cluster 10: 2 embeddings\n",
      "Cluster 11: 2 embeddings\n",
      "Cluster 12: 3 embeddings\n",
      "Cluster 13: 2 embeddings\n",
      "Cluster 14: 3 embeddings\n",
      "Cluster 15: 4 embeddings\n",
      "Cluster 16: 2 embeddings\n",
      "Cluster 17: 2 embeddings\n",
      "Cluster 18: 2 embeddings\n",
      "Cluster 19: 2 embeddings\n",
      "Cluster 20: 2 embeddings\n",
      "Cluster 21: 2 embeddings\n",
      "Cluster 22: 2 embeddings\n",
      "Cluster 23: 2 embeddings\n",
      "Cluster 24: 2 embeddings\n",
      "Cluster 25: 2 embeddings\n",
      "Cluster 26: 2 embeddings\n",
      "Cluster 27: 2 embeddings\n",
      "Cluster 28: 2 embeddings\n",
      "Cluster 29: 2 embeddings\n",
      "Cluster 30: 2 embeddings\n",
      "Cluster 31: 2 embeddings\n",
      "Cluster 32: 2 embeddings\n",
      "Cluster 33: 1 embeddings\n",
      "Cluster 34: 2 embeddings\n",
      "Cluster 35: 2 embeddings\n",
      "Cluster 36: 1 embeddings\n",
      "Cluster 37: 2 embeddings\n",
      "Cluster 38: 2 embeddings\n",
      "Cluster 39: 2 embeddings\n",
      "Cluster 40: 1 embeddings\n",
      "Cluster 41: 2 embeddings\n",
      "Cluster 42: 1 embeddings\n",
      "Cluster 43: 3 embeddings\n",
      "Cluster 44: 2 embeddings\n",
      "Cluster 45: 2 embeddings\n",
      "Cluster 46: 2 embeddings\n",
      "Cluster 47: 1 embeddings\n",
      "Cluster 48: 2 embeddings\n",
      "Cluster 49: 1 embeddings\n",
      "Cluster 50: 2 embeddings\n",
      "Cluster 51: 2 embeddings\n",
      "Cluster 52: 1 embeddings\n",
      "Cluster 53: 1 embeddings\n",
      "Cluster 54: 1 embeddings\n",
      "Cluster 55: 2 embeddings\n",
      "Cluster 56: 2 embeddings\n",
      "Cluster 57: 2 embeddings\n",
      "Cluster 58: 3 embeddings\n",
      "Cluster 59: 2 embeddings\n",
      "Cluster 60: 2 embeddings\n",
      "Cluster 61: 2 embeddings\n",
      "Cluster 62: 2 embeddings\n",
      "Cluster 63: 3 embeddings\n",
      "Cluster 64: 2 embeddings\n",
      "Cluster 65: 2 embeddings\n",
      "Cluster 66: 2 embeddings\n",
      "Cluster 67: 1 embeddings\n",
      "Cluster 68: 1 embeddings\n",
      "Cluster 69: 2 embeddings\n",
      "Cluster 70: 1 embeddings\n",
      "Cluster 71: 2 embeddings\n",
      "Cluster 72: 2 embeddings\n",
      "Cluster 73: 1 embeddings\n",
      "Cluster 74: 1 embeddings\n",
      "Cluster 75: 1 embeddings\n",
      "Cluster 76: 1 embeddings\n",
      "Cluster 77: 1 embeddings\n",
      "Cluster 78: 1 embeddings\n",
      "Cluster 79: 2 embeddings\n",
      "Cluster 80: 1 embeddings\n",
      "Cluster 81: 1 embeddings\n",
      "Cluster 82: 1 embeddings\n",
      "Cluster 83: 1 embeddings\n",
      "Cluster 84: 2 embeddings\n",
      "Cluster 85: 1 embeddings\n",
      "Cluster 86: 1 embeddings\n",
      "Cluster 87: 2 embeddings\n",
      "Cluster 88: 1 embeddings\n",
      "Cluster 89: 1 embeddings\n",
      "Cluster 90: 1 embeddings\n",
      "Cluster 91: 1 embeddings\n",
      "Cluster 92: 1 embeddings\n",
      "Cluster 93: 1 embeddings\n",
      "Cluster 94: 1 embeddings\n",
      "Cluster 95: 1 embeddings\n",
      "Cluster 96: 1 embeddings\n",
      "Cluster 97: 1 embeddings\n",
      "Cluster 98: 1 embeddings\n",
      "Cluster 99: 1 embeddings\n",
      "Cluster 100: 1 embeddings\n",
      "Cluster 101: 2 embeddings\n",
      "Cluster 102: 1 embeddings\n",
      "Cluster 103: 1 embeddings\n",
      "Cluster 104: 1 embeddings\n",
      "Cluster 105: 1 embeddings\n",
      "Cluster 106: 1 embeddings\n",
      "Cluster 107: 1 embeddings\n",
      "Cluster 108: 1 embeddings\n",
      "Cluster 109: 1 embeddings\n",
      "Cluster 110: 1 embeddings\n",
      "Cluster 111: 1 embeddings\n",
      "Cluster 112: 2 embeddings\n",
      "Cluster 113: 2 embeddings\n",
      "Cluster 114: 2 embeddings\n",
      "Cluster 115: 2 embeddings\n",
      "Cluster 116: 2 embeddings\n",
      "Cluster 117: 2 embeddings\n",
      "Cluster 118: 1 embeddings\n",
      "Cluster 119: 2 embeddings\n",
      "Cluster 120: 1 embeddings\n",
      "Cluster 121: 1 embeddings\n",
      "Cluster 122: 2 embeddings\n",
      "Cluster 123: 2 embeddings\n",
      "Cluster 124: 2 embeddings\n",
      "Cluster 125: 1 embeddings\n",
      "Cluster 126: 1 embeddings\n",
      "Cluster 127: 1 embeddings\n",
      "Cluster 128: 1 embeddings\n",
      "Cluster 129: 1 embeddings\n",
      "Cluster 130: 2 embeddings\n",
      "Cluster 131: 1 embeddings\n",
      "Cluster 132: 1 embeddings\n",
      "Cluster 133: 1 embeddings\n",
      "Cluster 134: 1 embeddings\n",
      "Cluster 135: 1 embeddings\n",
      "Cluster 136: 1 embeddings\n",
      "Cluster 137: 1 embeddings\n",
      "Cluster 138: 1 embeddings\n",
      "Cluster 139: 1 embeddings\n",
      "Cluster 140: 1 embeddings\n",
      "Cluster 141: 1 embeddings\n",
      "Cluster 142: 1 embeddings\n",
      "Cluster 143: 1 embeddings\n",
      "Cluster 144: 1 embeddings\n",
      "Cluster 145: 1 embeddings\n",
      "Cluster 146: 1 embeddings\n",
      "Cluster 147: 1 embeddings\n",
      "Cluster 148: 1 embeddings\n",
      "Cluster 149: 1 embeddings\n",
      "Cluster 150: 1 embeddings\n",
      "Cluster 151: 1 embeddings\n",
      "Cluster 152: 1 embeddings\n",
      "Cluster 153: 1 embeddings\n",
      "Cluster 154: 1 embeddings\n",
      "Cluster 155: 1 embeddings\n",
      "Cluster 156: 1 embeddings\n",
      "Cluster 157: 1 embeddings\n",
      "Cluster 158: 1 embeddings\n",
      "Cluster 159: 1 embeddings\n",
      "Cluster 160: 1 embeddings\n",
      "Cluster 161: 1 embeddings\n",
      "Cluster 162: 1 embeddings\n",
      "Cluster 163: 1 embeddings\n",
      "Cluster 164: 1 embeddings\n",
      "Cluster 165: 1 embeddings\n",
      "Cluster 166: 1 embeddings\n",
      "Cluster 167: 1 embeddings\n",
      "Cluster 168: 1 embeddings\n",
      "Cluster 169: 1 embeddings\n",
      "Cluster 170: 1 embeddings\n",
      "Cluster 171: 1 embeddings\n",
      "Cluster 172: 1 embeddings\n",
      "Cluster 173: 1 embeddings\n",
      "Cluster 174: 1 embeddings\n",
      "Cluster 175: 1 embeddings\n",
      "Cluster 176: 1 embeddings\n",
      "Cluster 177: 1 embeddings\n",
      "Cluster 178: 1 embeddings\n",
      "Cluster 179: 1 embeddings\n",
      "Cluster 180: 1 embeddings\n",
      "Cluster 181: 1 embeddings\n",
      "Cluster 182: 1 embeddings\n",
      "Cluster 183: 1 embeddings\n",
      "Cluster 184: 1 embeddings\n",
      "Cluster 185: 1 embeddings\n",
      "Cluster 186: 1 embeddings\n",
      "Cluster 187: 1 embeddings\n",
      "Cluster 188: 1 embeddings\n",
      "Cluster 189: 1 embeddings\n",
      "Cluster 190: 1 embeddings\n",
      "Cluster 191: 1 embeddings\n",
      "Cluster 192: 1 embeddings\n",
      "Cluster 193: 1 embeddings\n",
      "Cluster 194: 1 embeddings\n",
      "Cluster 195: 1 embeddings\n",
      "Cluster 196: 1 embeddings\n",
      "Cluster 197: 1 embeddings\n",
      "Cluster 198: 1 embeddings\n",
      "Cluster 199: 1 embeddings\n",
      "Cluster 200: 1 embeddings\n",
      "Cluster 201: 1 embeddings\n",
      "Cluster 202: 1 embeddings\n",
      "Cluster 203: 1 embeddings\n",
      "Cluster 204: 1 embeddings\n",
      "Cluster 205: 1 embeddings\n",
      "Cluster 206: 1 embeddings\n",
      "Cluster 207: 1 embeddings\n",
      "Cluster 208: 1 embeddings\n",
      "Cluster 209: 1 embeddings\n",
      "Cluster 210: 1 embeddings\n",
      "Cluster 211: 1 embeddings\n",
      "Cluster 212: 1 embeddings\n",
      "Cluster 213: 1 embeddings\n",
      "Cluster 214: 1 embeddings\n",
      "Cluster 215: 1 embeddings\n",
      "Cluster 216: 1 embeddings\n",
      "Cluster 217: 1 embeddings\n",
      "Cluster 218: 1 embeddings\n",
      "Cluster 219: 1 embeddings\n",
      "Cluster 220: 1 embeddings\n",
      "Cluster 221: 1 embeddings\n",
      "Cluster 222: 1 embeddings\n",
      "Cluster 223: 1 embeddings\n",
      "Cluster 224: 1 embeddings\n",
      "Cluster 225: 1 embeddings\n",
      "Cluster 226: 1 embeddings\n",
      "Cluster 227: 1 embeddings\n",
      "Cluster 228: 1 embeddings\n",
      "Cluster 229: 1 embeddings\n",
      "Cluster 230: 1 embeddings\n",
      "Cluster 231: 1 embeddings\n",
      "Cluster 232: 1 embeddings\n",
      "Cluster 233: 1 embeddings\n",
      "Cluster 234: 2 embeddings\n",
      "Cluster 235: 1 embeddings\n",
      "Cluster 236: 1 embeddings\n",
      "Cluster 237: 1 embeddings\n",
      "Cluster 238: 1 embeddings\n",
      "Cluster 239: 1 embeddings\n",
      "Cluster 240: 1 embeddings\n",
      "Cluster 241: 1 embeddings\n",
      "Cluster 242: 1 embeddings\n",
      "Cluster 243: 1 embeddings\n",
      "Cluster 244: 1 embeddings\n",
      "Cluster 245: 1 embeddings\n",
      "Cluster 246: 1 embeddings\n",
      "Cluster 247: 1 embeddings\n",
      "Cluster 248: 1 embeddings\n",
      "Cluster 249: 1 embeddings\n",
      "Cluster 250: 1 embeddings\n",
      "Cluster 251: 1 embeddings\n",
      "Cluster 252: 1 embeddings\n",
      "Cluster 253: 1 embeddings\n",
      "Cluster 254: 1 embeddings\n",
      "Cluster 255: 1 embeddings\n",
      "Cluster 256: 1 embeddings\n",
      "Cluster 257: 1 embeddings\n",
      "Cluster 258: 1 embeddings\n",
      "Cluster 259: 1 embeddings\n",
      "Cluster 260: 1 embeddings\n",
      "Cluster 261: 1 embeddings\n",
      "Cluster 262: 1 embeddings\n",
      "Cluster 263: 1 embeddings\n",
      "Cluster 264: 1 embeddings\n",
      "Cluster 265: 1 embeddings\n",
      "Cluster 266: 1 embeddings\n",
      "Cluster 267: 1 embeddings\n",
      "Cluster 268: 1 embeddings\n",
      "Cluster 269: 1 embeddings\n",
      "Cluster 270: 1 embeddings\n",
      "Cluster 271: 1 embeddings\n",
      "Cluster 272: 1 embeddings\n",
      "Cluster 273: 1 embeddings\n",
      "Cluster 274: 1 embeddings\n",
      "Cluster 275: 1 embeddings\n",
      "Cluster 276: 1 embeddings\n",
      "Cluster 277: 1 embeddings\n",
      "Cluster 278: 1 embeddings\n",
      "Cluster 279: 1 embeddings\n",
      "Cluster 280: 1 embeddings\n",
      "Cluster 281: 1 embeddings\n",
      "Cluster 282: 1 embeddings\n",
      "Cluster 283: 1 embeddings\n",
      "Cluster 284: 1 embeddings\n",
      "Cluster 285: 1 embeddings\n",
      "Cluster 286: 1 embeddings\n",
      "Cluster 287: 1 embeddings\n",
      "Cluster 288: 1 embeddings\n",
      "Cluster 289: 1 embeddings\n",
      "Cluster 290: 1 embeddings\n",
      "Cluster 291: 1 embeddings\n",
      "Cluster 292: 1 embeddings\n",
      "Cluster 293: 1 embeddings\n",
      "Cluster 294: 1 embeddings\n",
      "Cluster 295: 1 embeddings\n",
      "Cluster 296: 1 embeddings\n",
      "Cluster 297: 1 embeddings\n",
      "Cluster 298: 1 embeddings\n",
      "Cluster 299: 1 embeddings\n",
      "Cluster 300: 1 embeddings\n",
      "Cluster 301: 1 embeddings\n",
      "Cluster 302: 1 embeddings\n",
      "Cluster 303: 1 embeddings\n",
      "Cluster 304: 1 embeddings\n",
      "Cluster 305: 1 embeddings\n",
      "Cluster 306: 1 embeddings\n",
      "Cluster 307: 1 embeddings\n",
      "Cluster 308: 1 embeddings\n",
      "Cluster 309: 1 embeddings\n",
      "Cluster 310: 1 embeddings\n",
      "Cluster 311: 1 embeddings\n",
      "Cluster 312: 1 embeddings\n",
      "Cluster 313: 1 embeddings\n",
      "Cluster 314: 1 embeddings\n",
      "Cluster 315: 1 embeddings\n",
      "Cluster 316: 1 embeddings\n",
      "Cluster 317: 1 embeddings\n",
      "Cluster 318: 1 embeddings\n",
      "Cluster 319: 1 embeddings\n",
      "Cluster 320: 1 embeddings\n",
      "Cluster 321: 1 embeddings\n",
      "Cluster 322: 1 embeddings\n",
      "Cluster 323: 1 embeddings\n",
      "Cluster 324: 1 embeddings\n",
      "Cluster 325: 1 embeddings\n",
      "Cluster 326: 1 embeddings\n",
      "Cluster 327: 1 embeddings\n",
      "Cluster 328: 1 embeddings\n",
      "Cluster 329: 1 embeddings\n",
      "Cluster 330: 1 embeddings\n",
      "Cluster 331: 1 embeddings\n",
      "Cluster 332: 1 embeddings\n",
      "Cluster 333: 1 embeddings\n",
      "Cluster 334: 1 embeddings\n",
      "Cluster 335: 1 embeddings\n",
      "Cluster 336: 1 embeddings\n"
     ]
    }
   ],
   "source": [
    "from sklearn.cluster import AgglomerativeClustering\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import numpy as np\n",
    "\n",
    "# Assume you have a list of text embeddings called `embeddings`\n",
    "# First, compute the cosine similarity matrix between all pairs of embeddings\n",
    "cosine_sim_matrix = cosine_similarity(embeddings)\n",
    "\n",
    "# Instantiate the AgglomerativeClustering model\n",
    "agg_clustering = AgglomerativeClustering(\n",
    "    n_clusters=None,         # the algorithm will determine the optimal number of clusters based on the data\n",
    "    distance_threshold=0.1,  # clusters will be formed until all pairwise distances between clusters are greater than 0.1\n",
    "    affinity='precomputed',  # we are providing a precomputed distance matrix (1 - similarity matrix) as input\n",
    "    linkage='complete'       # form clusters by iteratively merging the smallest clusters based on the maximum distance between their components\n",
    ")\n",
    "\n",
    "# Fit the model to the cosine distance matrix (1 - similarity matrix)\n",
    "agg_clustering.fit(1 - cosine_sim_matrix)\n",
    "\n",
    "# Get the cluster labels for each embedding\n",
    "cluster_labels = agg_clustering.labels_\n",
    "\n",
    "# Print the number of embeddings in each cluster\n",
    "unique_labels, counts = np.unique(cluster_labels, return_counts=True)\n",
    "for label, count in zip(unique_labels, counts):\n",
    "    print(f'Cluster {label}: {count} embeddings')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "id": "0727319b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Our pruning approach has 337 documents with average length 399.9 tokens\n"
     ]
    }
   ],
   "source": [
    "pruned_documents = []\n",
    "for _label, count in zip(unique_labels, counts):\n",
    "    pruned_documents.append('\\n\\n'.join([text for text, label in zip(split, cluster_labels) if label == _label]))\n",
    "\n",
    "    \n",
    "avg_length = sum([len(tokenizer.encode(t)) for t in pruned_documents]) / len(pruned_documents)\n",
    "print(f'Our pruning approach has {len(pruned_documents)} documents with average length {avg_length:.1f} tokens')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "id": "97f80b78",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "We get the following table:\n",
      "In the preceding table, each row represents a single passenger on the ship, and, for now, we\n",
      "are looking at two specific features: the sex of the individual and whether or not they\n",
      "survived the sinking. For example, the first row represents a man who did not survive,\n",
      "while the fourth row (with index 3, remember how Python indexes lists) represents a\n",
      "female who did survive.\n",
      "Let's start with some basics. We'll start by calculating the probability that any given person\n",
      "on the ship survived, regardless of their gender. To do this, we'll count the number of yeses\n",
      "in the Survived  column and divide this figure by the total number of rows, as shown here:\n",
      "num_rows = float(titanic.shape[0]) # == 891 rows\n",
      "p_survived = (titanic.Survived==\"yes\").sum() / num_rows # == .38\n",
      "p_notsurvived = 1 - p_survived                          # == .61\n",
      "Note that I only had to calculate P(Survived) , and I used the law of conjugate probabilities to\n",
      "calculate P(Died)  because those two events are complementary. Now, let's calculate the\n",
      "probability that any single passenger is male or female:\n",
      "p_male = (titanic.Sex==\"male\").sum() / num_rows     # == .65\n",
      "p_female = 1 - p_male # == .35\n",
      "Now, let's ask ourselves a question: did having a certain gender affect the survival rate? For\n",
      "this, we can estimate P(Survived|Female)  or the chance that someone survived given that\n",
      "they were a female. For this, we need to divide the number of women who survived by the\n",
      "total number of women, as shown here:\n",
      "\n",
      "\n",
      "Here's the code for that calculation:\n",
      "number_of_women = titanic[titanic.Sex=='female'].shape[0] # == 314\n",
      "women_who_lived = titanic[(titanic.Sex=='female') &\n",
      "(titanic.Survived=='yes')].shape[0]                       # == 233\n",
      "p_survived_given_woman = women_who_lived / float(number_of_women)\n",
      "p_survived_given_woman            # == .74\n",
      "That's a pretty big difference. It seems that gender plays a big part in this dataset.\n",
      "Example  – medical studies\n",
      "A classic use of Bayes' theorem is the interpretation of medical trials. Routine testing for\n",
      "illegal drug use is increasingly common in workplaces and schools. The companies that\n",
      "perform these tests maintain that the tests have a high sensitivity, which means that they\n",
      "are likely to produce a positive result if there are drugs in their system . They claim that these\n",
      "tests are also highly specific, which means that they are likely to yield a negative result if\n",
      "there are no drugs .\n",
      "On average, let's assume that the sensitivity of common drug tests is about 60% and the\n",
      "specificity is about 99%. It means that if an employee is using drugs, the test has a 60%\n",
      "chance of being positive, while if an employee is not on drugs, the test has a 99% chance of\n",
      "being negative. Now, suppose these tests are applied to a workforce where the actual rate\n",
      "of drug use is 5%.\n",
      "The real question here is of the people who test positive, how many actually use drugs?\n",
      "In Bayesian terms, we want to compute the probability of drug use, given a positive test:\n",
      "Let D = the event that drugs are in use\n",
      "Let E = the event that the test is positive\n",
      "Let N = the event that drugs are NOT  in use\n",
      "We are looking for P(D|E) .\n",
      "By using Bayes' theorem, we can extrapolate it as follows:\n"
     ]
    }
   ],
   "source": [
    "print(pruned_documents[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "34bafae2",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "690ebbb9",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3a46ca7",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "3a3364dd",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "337"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "upload_texts_to_pinecone(pruned_documents, batch_size=128)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8511c945",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "13cc8bdc",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "fe05666e4653e007201c336eba0b0c89\t0.85\tLet's begin by learning a very  important value  i\n",
      "28c9f36450ba7f7a78fcf29f2e4e9909\t0.85\tZ-scores are an effective way to standardize  data\n",
      "268922260e7eabb5ed6170d07ec4585a\t0.83\tThe preceding code gives us this graph:\n",
      "Now, our d\n",
      "7da124bb8c927e0c392cf75d4bfb2c76\t0.82\t\n",
      "Basic Statistics\n",
      "This chapter will focus on the s\n",
      "f90c995652f29ac9c2a4ca94f0e6809c\t0.81\t\n",
      "We can think of this problem like as follows:\n",
      "The\n"
     ]
    }
   ],
   "source": [
    "query = 'How do z scores work?'\n",
    "\n",
    "results_from_pinecone = query_from_pinecone(query, top_k=5)\n",
    "\n",
    "for result_from_pinecone in results_from_pinecone:\n",
    "    print(f\"{result_from_pinecone['id']}\\t{result_from_pinecone['score']:.2f}\\t{result_from_pinecone['metadata']['text'][:50]}\")\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "id": "074fab6f",
   "metadata": {},
   "outputs": [],
   "source": [
    "\"\"\"\n",
    "This example computes the score between a query and all possible\n",
    "sentences in a corpus using a Cross-Encoder for semantic textual similarity (STS).\n",
    "It output then the most similar sentences for the given query.\n",
    "\"\"\"\n",
    "from sentence_transformers.cross_encoder import CrossEncoder\n",
    "import numpy as np\n",
    "from torch import nn\n",
    "\n",
    "# Pre-trained cross encoder\n",
    "cross_encoder = CrossEncoder('cross-encoder/mmarco-mMiniLMv2-L12-H384-v1')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "25b8e942",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "id": "414fc2d5",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_results_from_pinecone(query, top_k=3, re_rank=False, verbose=True):\n",
    "\n",
    "    results_from_pinecone = query_from_pinecone(query, top_k=top_k)\n",
    "    if not results_from_pinecone:\n",
    "        return []\n",
    "\n",
    "    if verbose:\n",
    "        print(\"Query:\", query)\n",
    "    \n",
    "    \n",
    "    final_results = []\n",
    "\n",
    "    if re_rank:\n",
    "        if verbose:\n",
    "            print('Document ID (Hash)\\t\\tRetrieval Score\\tCE Score\\tText')\n",
    "\n",
    "        sentence_combinations = [[query, result_from_pinecone['metadata']['text']] for result_from_pinecone in results_from_pinecone]\n",
    "\n",
    "        # Compute the similarity scores for these combinations\n",
    "        similarity_scores = cross_encoder.predict(sentence_combinations, activation_fct=nn.Sigmoid())\n",
    "\n",
    "        # Sort the scores in decreasing order\n",
    "        sim_scores_argsort = reversed(np.argsort(similarity_scores))\n",
    "\n",
    "        # Print the scores\n",
    "        for idx in sim_scores_argsort:\n",
    "            result_from_pinecone = results_from_pinecone[idx]\n",
    "            final_results.append(result_from_pinecone)\n",
    "            if verbose:\n",
    "                print(f\"{result_from_pinecone['id']}\\t{result_from_pinecone['score']:.2f}\\t{similarity_scores[idx]:.2f}\\t{result_from_pinecone['metadata']['text'][:50]}\")\n",
    "        return final_results\n",
    "\n",
    "    if verbose:\n",
    "        print('Document ID (Hash)\\t\\tRetrieval Score\\tText')\n",
    "    for result_from_pinecone in results_from_pinecone:\n",
    "        final_results.append(result_from_pinecone)\n",
    "        if verbose:\n",
    "            print(f\"{result_from_pinecone['id']}\\t{result_from_pinecone['score']:.2f}\\t{result_from_pinecone['metadata']['text'][:50]}\")\n",
    "\n",
    "    return final_results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "id": "d89bcc77",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query: How do z scores work?\n",
      "Document ID (Hash)\t\tRetrieval Score\tCE Score\tText\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d1545d1a8ce0494d9a3ed11a4aa6c59c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28c9f36450ba7f7a78fcf29f2e4e9909\t0.85\t0.99\tZ-scores are an effective way to standardize  data\n",
      "fe05666e4653e007201c336eba0b0c89\t0.85\t0.99\tLet's begin by learning a very  important value  i\n",
      "268922260e7eabb5ed6170d07ec4585a\t0.83\t0.02\tThe preceding code gives us this graph:\n",
      "Now, our d\n"
     ]
    }
   ],
   "source": [
    "final_results = get_results_from_pinecone(query, top_k=3, re_rank=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "id": "a3c9922f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Query: How do z scores work?\n",
      "Document ID (Hash)\t\tRetrieval Score\tCE Score\tText\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "fb8b171a629342518b1721d849272794",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "28c9f36450ba7f7a78fcf29f2e4e9909\t0.85\t0.99\tZ-scores are an effective way to standardize  data\n",
      "fe05666e4653e007201c336eba0b0c89\t0.85\t0.99\tLet's begin by learning a very  important value  i\n",
      "7da124bb8c927e0c392cf75d4bfb2c76\t0.82\t0.38\t\n",
      "Basic Statistics\n",
      "This chapter will focus on the s\n",
      "268922260e7eabb5ed6170d07ec4585a\t0.83\t0.02\tThe preceding code gives us this graph:\n",
      "Now, our d\n",
      "75cf0bcd29070579356e3c50e9440d94\t0.80\t0.01\tThe following is the probability distribution of o\n",
      "c41c22f849c21729d2ccad5aa5da0047\t0.79\t0.01\tChoose a significance level (usually called alpha \n",
      "e9ec4bdefba2f80157ffce04812a1cf4\t0.80\t0.01\t\n",
      "Let's look at each of the elements in this formul\n",
      "5bf0b775d7e41ee065e79ef1d2ac9f2c\t0.80\t0.00\tThe empirical rule\n",
      "Recall that a normal distributi\n",
      "00c617088f8b57db6cc63331603a47ed\t0.79\t0.00\tWe can see that our p-value is quite large; theref\n",
      "f90c995652f29ac9c2a4ca94f0e6809c\t0.81\t0.00\t\n",
      "We can think of this problem like as follows:\n",
      "The\n"
     ]
    }
   ],
   "source": [
    "final_results = get_results_from_pinecone(query, top_k=10, re_rank=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "id": "c018b64d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{}"
      ]
     },
     "execution_count": 43,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "delete_texts_from_pinecone(pruned_documents)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "33c716a0",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "id": "79e1caf7",
   "metadata": {},
   "source": [
    "# BoolQ"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "62e28784",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Found cached dataset boolq (/Users/sinanozdemir/.cache/huggingface/datasets/boolq/default/0.1.0/bf0dd57da941c50de94ae3ce3cef7fea48c08f337a4b7aac484e9dddc5aa24e5)\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a2ab254f8a6c4b6d93f45873e5e071d8",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "  0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from datasets import load_dataset\n",
    "from evaluate import load\n",
    "\n",
    "\n",
    "dataset = load_dataset(\"boolq\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "id": "e6773457",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'does ethanol take more energy make that produces',\n",
       " 'answer': False,\n",
       " 'passage': \"All biomass goes through at least some of these steps: it needs to be grown, collected, dried, fermented, distilled, and burned. All of these steps require resources and an infrastructure. The total amount of energy input into the process compared to the energy released by burning the resulting ethanol fuel is known as the energy balance (or ``energy returned on energy invested''). Figures compiled in a 2007 report by National Geographic Magazine point to modest results for corn ethanol produced in the US: one unit of fossil-fuel energy is required to create 1.3 energy units from the resulting ethanol. The energy balance for sugarcane ethanol produced in Brazil is more favorable, with one unit of fossil-fuel energy required to create 8 from the ethanol. Energy balance estimates are not easily produced, thus numerous such reports have been generated that are contradictory. For instance, a separate survey reports that production of ethanol from sugarcane, which requires a tropical climate to grow productively, returns from 8 to 9 units of energy for each unit expended, as compared to corn, which only returns about 1.34 units of fuel energy for each unit of energy expended. A 2006 University of California Berkeley study, after analyzing six separate studies, concluded that producing ethanol from corn uses much less petroleum than producing gasoline.\"}"
      ]
     },
     "execution_count": 45,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['validation'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "id": "0221343a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|███████████████████████████████████████████| 13/13 [00:39<00:00,  3.04s/it]\n"
     ]
    }
   ],
   "source": [
    "for idx in tqdm(range(0, len(dataset['validation']), 256)):\n",
    "    data_sample = dataset['validation'][idx:idx + 256]\n",
    "\n",
    "    passages = data_sample['passage']\n",
    "\n",
    "    upload_texts_to_pinecone(passages)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "9802a9ee",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "c20f66e1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "is the barber of seville a true story\n",
      "Query: is the barber of seville a true story\n",
      "Document ID (Hash)\t\tRetrieval Score\tCE Score\tText\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e37b9b97d63d4bfd9102a18b62ab06e9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "22433164547c459ac32cd575eb78a917\t0.74\t0.01\tSweeney Todd: The Demon Barber of Fleet Street is \n",
      "c3225f28c8c4cbfb81f3b01715f6d3f4\t0.76\t0.00\tRossini's opera recounts the events of the first o\n",
      "50b66c4c465aa18ad8e8589c7ba15728\t0.73\t0.00\tWhile the film was a box office hit and raised Str\n"
     ]
    }
   ],
   "source": [
    "from random import sample\n",
    "\n",
    "query = sample(dataset['validation']['question'], 1)[0]\n",
    "print(query)\n",
    "final_results = get_results_from_pinecone(query, top_k=3, re_rank=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "e1e27222",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'c3225f28c8c4cbfb81f3b01715f6d3f4'"
      ]
     },
     "execution_count": 48,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "q_to_hash = {data['question']: my_hash(data['passage']) for data in dataset['validation']}\n",
    "\n",
    "q_to_hash[query]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "a63ec53e",
   "metadata": {},
   "outputs": [],
   "source": [
    "# super_glue_metric = load('super_glue', 'boolq')  # just accuracy\n",
    "\n",
    "# Let's test the performance re-ranking against 1000 of our validation datapoints\n",
    "# Note we could not use Pinecone here to speed things up\n",
    "#  but it's also a good time to test latency of the pipeline with Pinecone\n",
    "val_sample = dataset['validation']#[:1000]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "21a3a147",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 98,
   "id": "bfea2ee7",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [17:41<00:00,  3.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy without re-ranking: 0.8522935779816514\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "predictions = []\n",
    "\n",
    "# Note we will keep top_k the same so latency from Pinecone is consistent\n",
    "#  and the only major time difference will be in the re-ranking\n",
    "\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash = get_results_from_pinecone(question, top_k=1, re_rank=False, verbose=False)[0]['id']\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append(retrieved_hash == correct_hash)\n",
    "    \n",
    "accuracy = sum(predictions)/len(predictions)\n",
    "\n",
    "print(f'Accuracy without re-ranking: {accuracy}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 99,
   "id": "b98413a3",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [27:20<00:00,  1.99it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Accuracy with re-ranking: 0.8373088685015291\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "predictions = []\n",
    "\n",
    "# Note we will keep top_k the same so latency from Pinecone is consistent\n",
    "#  and the only major time difference will be in the re-ranking\n",
    "\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash = get_results_from_pinecone(question, top_k=3, re_rank=True, verbose=False)[0]['id']\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append(retrieved_hash == correct_hash)\n",
    "    \n",
    "accuracy = sum(predictions)/len(predictions)\n",
    "\n",
    "print(f'Accuracy with re-ranking: {accuracy}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 100,
   "id": "d5acf707",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Note the time differences between with and without re-ranking\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "0cfebd73",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "id": "086ca98e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def eval_ranking(query, cross_encoder, top_k=3):\n",
    "    results_from_pinecone = query_from_pinecone(query, top_k=top_k)\n",
    "    sentence_combinations = [[query, result_from_pinecone['metadata']['text']] for result_from_pinecone in results_from_pinecone]\n",
    "    similarity_scores = cross_encoder.predict(sentence_combinations)\n",
    "    sim_scores_argsort = list(reversed(np.argsort(similarity_scores)))\n",
    "    re_ranked_final_result = results_from_pinecone[sim_scores_argsort[0]]\n",
    "    return results_from_pinecone[0]['id'], re_ranked_final_result['id']\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "id": "b4010029",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Trying another pre-trained cross encoder\n",
    "# sentence-transformers/multi-qa-mpnet-base-cos-v1\n",
    "newer_cross_encoder = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-12-v2')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f34ac11",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 103,
   "id": "104305e0",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█                                                                       | 50/3270 [00:23<21:10,  2.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 50\n",
      "Accuracy without re-ranking: 0.88\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▏                                                                    | 100/3270 [00:45<22:31,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 100\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▎                                                                   | 150/3270 [01:07<25:32,  2.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 150\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8466666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▎                                                                  | 200/3270 [01:27<23:56,  2.14it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 200\n",
      "Accuracy without re-ranking: 0.865\n",
      "Accuracy with re-ranking: 0.845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|█████▍                                                                 | 250/3270 [01:48<20:02,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 250\n",
      "Accuracy without re-ranking: 0.872\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▌                                                                | 300/3270 [02:09<19:23,  2.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 300\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|███████▌                                                               | 350/3270 [02:31<17:17,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 350\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8457142857142858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|████████▋                                                              | 400/3270 [02:53<18:28,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 400\n",
      "Accuracy without re-ranking: 0.8625\n",
      "Accuracy with re-ranking: 0.845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█████████▊                                                             | 450/3270 [03:20<19:07,  2.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 450\n",
      "Accuracy without re-ranking: 0.8577777777777778\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|██████████▊                                                            | 500/3270 [03:41<20:36,  2.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 500\n",
      "Accuracy without re-ranking: 0.852\n",
      "Accuracy with re-ranking: 0.838\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|███████████▉                                                           | 550/3270 [04:12<23:15,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 550\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8381818181818181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████                                                          | 600/3270 [04:32<16:10,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 600\n",
      "Accuracy without re-ranking: 0.8383333333333334\n",
      "Accuracy with re-ranking: 0.8316666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██████████████                                                         | 650/3270 [04:55<18:55,  2.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 650\n",
      "Accuracy without re-ranking: 0.8369230769230769\n",
      "Accuracy with re-ranking: 0.8276923076923077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▏                                                       | 700/3270 [05:17<18:02,  2.37it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 700\n",
      "Accuracy without re-ranking: 0.8385714285714285\n",
      "Accuracy with re-ranking: 0.8285714285714286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|████████████████▎                                                      | 750/3270 [05:38<17:34,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 750\n",
      "Accuracy without re-ranking: 0.832\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|█████████████████▎                                                     | 800/3270 [05:59<17:45,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 800\n",
      "Accuracy without re-ranking: 0.835\n",
      "Accuracy with re-ranking: 0.82875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██████████████████▍                                                    | 850/3270 [06:21<19:12,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 850\n",
      "Accuracy without re-ranking: 0.8352941176470589\n",
      "Accuracy with re-ranking: 0.8282352941176471\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|███████████████████▌                                                   | 900/3270 [06:41<14:34,  2.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 900\n",
      "Accuracy without re-ranking: 0.8344444444444444\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|████████████████████▋                                                  | 950/3270 [07:04<20:37,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 950\n",
      "Accuracy without re-ranking: 0.8378947368421052\n",
      "Accuracy with re-ranking: 0.8252631578947368\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|█████████████████████▍                                                | 1000/3270 [07:27<14:24,  2.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1000\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.826\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|██████████████████████▍                                               | 1050/3270 [07:50<15:45,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1050\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.8257142857142857\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███████████████████████▌                                              | 1100/3270 [08:14<14:23,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1100\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8272727272727273\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|████████████████████████▌                                             | 1150/3270 [08:35<21:58,  1.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1150\n",
      "Accuracy without re-ranking: 0.8443478260869566\n",
      "Accuracy with re-ranking: 0.8304347826086956\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|█████████████████████████▋                                            | 1200/3270 [08:56<14:30,  2.38it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1200\n",
      "Accuracy without re-ranking: 0.8458333333333333\n",
      "Accuracy with re-ranking: 0.8325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|██████████████████████████▊                                           | 1250/3270 [09:21<14:05,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1250\n",
      "Accuracy without re-ranking: 0.8488\n",
      "Accuracy with re-ranking: 0.8352\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███████████████████████████▊                                          | 1300/3270 [09:46<22:47,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1300\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8361538461538461\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████████████████████████████▉                                         | 1350/3270 [10:10<13:00,  2.46it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1350\n",
      "Accuracy without re-ranking: 0.8511111111111112\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|█████████████████████████████▉                                        | 1400/3270 [10:37<28:41,  1.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1400\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8385714285714285\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|███████████████████████████████                                       | 1450/3270 [11:10<24:11,  1.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1450\n",
      "Accuracy without re-ranking: 0.8475862068965517\n",
      "Accuracy with re-ranking: 0.8344827586206897\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████████████████████████████████                                      | 1500/3270 [11:32<10:30,  2.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1500\n",
      "Accuracy without re-ranking: 0.846\n",
      "Accuracy with re-ranking: 0.8313333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|█████████████████████████████████▏                                    | 1550/3270 [11:58<16:05,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1550\n",
      "Accuracy without re-ranking: 0.8464516129032258\n",
      "Accuracy with re-ranking: 0.832258064516129\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|██████████████████████████████████▎                                   | 1600/3270 [12:27<17:12,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1600\n",
      "Accuracy without re-ranking: 0.845625\n",
      "Accuracy with re-ranking: 0.831875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|███████████████████████████████████▎                                  | 1650/3270 [12:50<13:20,  2.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1650\n",
      "Accuracy without re-ranking: 0.8460606060606061\n",
      "Accuracy with re-ranking: 0.8321212121212122\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|████████████████████████████████████▍                                 | 1700/3270 [13:21<16:55,  1.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1700\n",
      "Accuracy without re-ranking: 0.8482352941176471\n",
      "Accuracy with re-ranking: 0.8335294117647059\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████████████████████████████████████▍                                | 1750/3270 [13:51<14:25,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1750\n",
      "Accuracy without re-ranking: 0.848\n",
      "Accuracy with re-ranking: 0.8331428571428572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|██████████████████████████████████████▌                               | 1800/3270 [14:22<12:56,  1.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1800\n",
      "Accuracy without re-ranking: 0.8483333333333334\n",
      "Accuracy with re-ranking: 0.8355555555555556\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|███████████████████████████████████████▌                              | 1850/3270 [14:51<14:33,  1.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1850\n",
      "Accuracy without re-ranking: 0.8475675675675676\n",
      "Accuracy with re-ranking: 0.8356756756756757\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|████████████████████████████████████████▋                             | 1900/3270 [15:20<10:16,  2.22it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1900\n",
      "Accuracy without re-ranking: 0.8494736842105263\n",
      "Accuracy with re-ranking: 0.8378947368421052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████████████████████████████████████████▋                            | 1950/3270 [15:51<11:43,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1950\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8384615384615385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████████████████████████████████████████▊                           | 2000/3270 [16:20<12:50,  1.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.839\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|███████████████████████████████████████████▉                          | 2050/3270 [16:52<11:14,  1.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2050\n",
      "Accuracy without re-ranking: 0.8526829268292683\n",
      "Accuracy with re-ranking: 0.8414634146341463\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|████████████████████████████████████████████▉                         | 2100/3270 [17:18<09:55,  1.96it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2100\n",
      "Accuracy without re-ranking: 0.8514285714285714\n",
      "Accuracy with re-ranking: 0.8404761904761905\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████████████████████████████████████████████                        | 2150/3270 [17:43<08:29,  2.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2150\n",
      "Accuracy without re-ranking: 0.8502325581395349\n",
      "Accuracy with re-ranking: 0.8418604651162791\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|███████████████████████████████████████████████                       | 2200/3270 [18:12<08:40,  2.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2200\n",
      "Accuracy without re-ranking: 0.8504545454545455\n",
      "Accuracy with re-ranking: 0.8409090909090909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|████████████████████████████████████████████████▏                     | 2250/3270 [18:36<07:34,  2.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2250\n",
      "Accuracy without re-ranking: 0.8506666666666667\n",
      "Accuracy with re-ranking: 0.8404444444444444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|█████████████████████████████████████████████████▏                    | 2300/3270 [18:58<06:26,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2300\n",
      "Accuracy without re-ranking: 0.8504347826086956\n",
      "Accuracy with re-ranking: 0.8386956521739131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|██████████████████████████████████████████████████▎                   | 2350/3270 [19:22<05:50,  2.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2350\n",
      "Accuracy without re-ranking: 0.8506382978723405\n",
      "Accuracy with re-ranking: 0.8395744680851064\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████████████████████████████████████████████████▍                  | 2400/3270 [19:43<05:22,  2.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2400\n",
      "Accuracy without re-ranking: 0.85125\n",
      "Accuracy with re-ranking: 0.8395833333333333\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████▍                 | 2450/3270 [20:07<05:54,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2450\n",
      "Accuracy without re-ranking: 0.8526530612244898\n",
      "Accuracy with re-ranking: 0.8420408163265306\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████████▌                | 2500/3270 [20:29<06:34,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2500\n",
      "Accuracy without re-ranking: 0.8524\n",
      "Accuracy with re-ranking: 0.842\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|██████████████████████████████████████████████████████▌               | 2550/3270 [20:57<08:21,  1.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2550\n",
      "Accuracy without re-ranking: 0.8509803921568627\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████▋              | 2600/3270 [21:22<05:57,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2600\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8396153846153847\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████████████████████████████████████████████████████▋             | 2650/3270 [21:48<04:08,  2.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2650\n",
      "Accuracy without re-ranking: 0.849811320754717\n",
      "Accuracy with re-ranking: 0.8392452830188679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|█████████████████████████████████████████████████████████▊            | 2700/3270 [22:10<04:05,  2.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2700\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.8396296296296296\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████▊           | 2750/3270 [22:32<05:25,  1.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2750\n",
      "Accuracy without re-ranking: 0.8501818181818181\n",
      "Accuracy with re-ranking: 0.8389090909090909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|███████████████████████████████████████████████████████████▉          | 2800/3270 [23:01<04:11,  1.87it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2800\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8389285714285715\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████         | 2850/3270 [23:26<03:56,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2850\n",
      "Accuracy without re-ranking: 0.8501754385964913\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|██████████████████████████████████████████████████████████████        | 2900/3270 [23:56<03:23,  1.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2900\n",
      "Accuracy without re-ranking: 0.8510344827586207\n",
      "Accuracy with re-ranking: 0.8396551724137931\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████▏      | 2950/3270 [24:25<02:41,  1.98it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2950\n",
      "Accuracy without re-ranking: 0.8501694915254238\n",
      "Accuracy with re-ranking: 0.8396610169491525\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████▏     | 3000/3270 [24:52<02:02,  2.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.8413333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████▎    | 3050/3270 [25:15<01:22,  2.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3050\n",
      "Accuracy without re-ranking: 0.8511475409836066\n",
      "Accuracy with re-ranking: 0.8422950819672131\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|██████████████████████████████████████████████████████████████████▎   | 3100/3270 [25:38<01:08,  2.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3100\n",
      "Accuracy without re-ranking: 0.8522580645161291\n",
      "Accuracy with re-ranking: 0.8422580645161291\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|███████████████████████████████████████████████████████████████████▍  | 3150/3270 [26:01<00:53,  2.25it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3150\n",
      "Accuracy without re-ranking: 0.8526984126984127\n",
      "Accuracy with re-ranking: 0.8428571428571429\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|████████████████████████████████████████████████████████████████████▌ | 3200/3270 [26:23<00:23,  2.93it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3200\n",
      "Accuracy without re-ranking: 0.8525\n",
      "Accuracy with re-ranking: 0.8421875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████████████████████████████████████████████████████████████████▌| 3250/3270 [26:43<00:07,  2.75it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3250\n",
      "Accuracy without re-ranking: 0.8526153846153847\n",
      "Accuracy with re-ranking: 0.8415384615384616\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [26:53<00:00,  2.03it/s]\n"
     ]
    }
   ],
   "source": [
    "i = 0\n",
    "print_every = 50\n",
    "predictions = []\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash, reranked_hash = eval_ranking(question, newer_cross_encoder, top_k=3)\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append((retrieved_hash == correct_hash, reranked_hash == correct_hash))\n",
    "    i += 1\n",
    "    if i % print_every == 0:\n",
    "        print(f'Step {i}')\n",
    "        raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "        reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "        print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "        print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "id": "502d5e33",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cross-encoder: <sentence_transformers.cross_encoder.CrossEncoder.CrossEncoder object at 0x158c9cc70>\n",
      "Accuracy without re-ranking: 0.8522935779816514\n",
      "Accuracy with re-ranking: 0.8418960244648318\n"
     ]
    }
   ],
   "source": [
    "raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "print(f'Using cross-encoder: {newer_cross_encoder.config._name_or_path}')\n",
    "print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "ea30153c",
   "metadata": {},
   "source": [
    "# Fine-tuning re-ranker"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 105,
   "id": "9918c97d",
   "metadata": {},
   "outputs": [],
   "source": [
    "# https://github.com/UKPLab/sentence-transformers/blob/master/examples/training/ms_marco/train_cross-encoder_scratch.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 52,
   "id": "508edaf6",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'do iran and afghanistan speak the same language',\n",
       " 'answer': True,\n",
       " 'passage': 'Persian (/ˈpɜːrʒən, -ʃən/), also known by its endonym Farsi (فارسی fārsi (fɒːɾˈsiː) ( listen)), is one of the Western Iranian languages within the Indo-Iranian branch of the Indo-European language family. It is primarily spoken in Iran, Afghanistan (officially known as Dari since 1958), and Tajikistan (officially known as Tajiki since the Soviet era), and some other regions which historically were Persianate societies and considered part of Greater Iran. It is written in the Persian alphabet, a modified variant of the Arabic script, which itself evolved from the Aramaic alphabet.'}"
      ]
     },
     "execution_count": 52,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train'][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "id": "2e7e68f1",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'question': 'do good samaritan laws protect those who help at an accident',\n",
       " 'answer': True,\n",
       " 'passage': \"Good Samaritan laws offer legal protection to people who give reasonable assistance to those who are, or who they believe to be, injured, ill, in peril, or otherwise incapacitated. The protection is intended to reduce bystanders' hesitation to assist, for fear of being sued or prosecuted for unintentional injury or wrongful death. An example of such a law in common-law areas of Canada: a good Samaritan doctrine is a legal principle that prevents a rescuer who has voluntarily helped a victim in distress from being successfully sued for wrongdoing. Its purpose is to keep people from being reluctant to help a stranger in need for fear of legal repercussions should they make some mistake in treatment. By contrast, a duty to rescue law requires people to offer assistance and holds those who fail to do so liable.\"}"
      ]
     },
     "execution_count": 54,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "dataset['train'][1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 349,
   "id": "51bf1607",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sentence_transformers import InputExample, losses, evaluation\n",
    "from torch.utils.data import DataLoader\n",
    "from random import shuffle\n",
    "\n",
    "shuffled_training_passages = dataset['train']['passage'].copy()\n",
    "shuffle(shuffled_training_passages)\n",
    "\n",
    "\n",
    "train_samples = [\n",
    "  InputExample(texts=[d['question'], d['passage']], label=1) for d in dataset['train']\n",
    "]\n",
    "\n",
    "# add some negative examples\n",
    "train_samples += [\n",
    "  InputExample(texts=[d['question'], shuffled_training_passages[i]], label=0) for i, d in enumerate(dataset['train'])\n",
    "]\n",
    "\n",
    "shuffle(train_samples)\n",
    "\n",
    "# running the risk of overfitting on my data but maybe I want that. \n",
    "#  Combined with sufficient input and output validation, we can make a viable product with a model overfit to my data\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 350,
   "id": "9e7826e7",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "18854"
      ]
     },
     "execution_count": 350,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train_samples)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 351,
   "id": "6b58d2d0",
   "metadata": {},
   "outputs": [],
   "source": [
    "model = CrossEncoder('cross-encoder/ms-marco-MiniLM-L-12-v2', num_labels=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 352,
   "id": "8e7eaa40",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{'guid': '',\n",
       " 'texts': ['is 44 special the same as 44 mag',\n",
       "  'Sleeping while on duty or sleeping on the job refers to falling asleep while on the time clock or equivalent, or else while responsible for performing some active or passive job duty. While in some jobs, this is a minor transgression or not even worthy of sanctioning, in other workplaces, this is considered gross misconduct and may be grounds for disciplinary action, including possible termination of employment. Recently however, there has been a movement in support of sleeping, or napping at work, with scientific studies highlighting health and productivity benefits, and over 6% of employers in some countries providing facilities to do so. In some types of work, such as firefighting or live-in caregiving, sleeping at least part of the shift may be an expected part of paid work time. While some employees who sleep while on duty in violation do so intentionally and hope not to get caught, others intend in good faith to stay awake, and accidentally doze.'],\n",
       " 'label': 0}"
      ]
     },
     "execution_count": 352,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train_samples[0].__dict__"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 353,
   "id": "2ede1b89",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "409fb8754a734ca6a13584d792aed82b",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "1.3383447e-05"
      ]
     },
     "execution_count": 353,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "model.predict(train_samples[0].texts, activation_fct=nn.Sigmoid())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 354,
   "id": "bc2bc9d1",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Warmup-steps: 95\n"
     ]
    }
   ],
   "source": [
    "from sentence_transformers.cross_encoder.evaluation import CECorrelationEvaluator, CEBinaryClassificationEvaluator\n",
    "import math\n",
    "import torch\n",
    "from random import sample\n",
    "\n",
    "logger.setLevel(logging.DEBUG)  # just to get some logs\n",
    "\n",
    "num_epochs = 2\n",
    "\n",
    "model_save_path = './fine_tuned_ir_cross_encoder'\n",
    "\n",
    "# train_samples = sample(train_samples, 1000)\n",
    "\n",
    "# int(len(train_samples)*.8)\n",
    "train_dataloader = DataLoader(train_samples[:int(len(train_samples)*.8)], shuffle=True, batch_size=32)\n",
    "\n",
    "# An evaluator for training performance\n",
    "evaluator = CEBinaryClassificationEvaluator.from_input_examples(train_samples[-int(len(train_samples)*.8):], name='test')\n",
    "\n",
    "# Rule of thumb for warmup steps\n",
    "warmup_steps = math.ceil(len(train_dataloader) * num_epochs * 0.1)  # 10% of train data for warm-up\n",
    "print(f\"Warmup-steps: {warmup_steps}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 358,
   "id": "0de4dbc8",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/torch/cuda/amp/grad_scaler.py:115: UserWarning: torch.cuda.amp.GradScaler is enabled, but CUDA is not available.  Disabling.\n",
      "  warnings.warn(\"torch.cuda.amp.GradScaler is enabled, but CUDA is not available.  Disabling.\")\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "8f7b781eb87b4d93b999db18ffca5265",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Epoch:   0%|          | 0/2 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "70faf81c2db046d780aeb34fae81005c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "cbaa59552a5249f2801385ce90359c93",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Iteration:   0%|          | 0/472 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/opt/homebrew/lib/python3.10/site-packages/torch/amp/autocast_mode.py:198: UserWarning: User provided device_type of 'cuda', but CUDA is not available. Disabling\n",
      "  warnings.warn('User provided device_type of \\'cuda\\', but CUDA is not available. Disabling')\n"
     ]
    }
   ],
   "source": [
    "# # ##### Load model and eval on test set\n",
    "# print(evaluator(model))\n",
    "\n",
    "# Train the model\n",
    "model.fit(\n",
    "    train_dataloader=train_dataloader,\n",
    "    loss_fct=losses.nn.CrossEntropyLoss(),\n",
    "    activation_fct=nn.Sigmoid(),\n",
    "    evaluator=evaluator,\n",
    "    epochs=num_epochs,\n",
    "    warmup_steps=warmup_steps,\n",
    "    output_path=model_save_path,\n",
    "    use_amp=True\n",
    ")\n",
    "\n",
    "# ##### Load model and eval on test set\n",
    "# print(evaluator(model))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8ca59b71",
   "metadata": {},
   "outputs": [],
   "source": [
    "# run the more fine tuned version on open source as well to match??\n",
    "# depends if it does better here"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "id": "2f747537",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62d987a5eedb4188a3f5d98cc084687d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.9999926\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "62b3396e14b646068224f55e0913bfab",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "11.819853\n"
     ]
    }
   ],
   "source": [
    "finetuned = CrossEncoder(model_save_path)\n",
    "\n",
    "print(finetuned.predict(['hello', 'hi'], activation_fct=nn.Sigmoid()))\n",
    "print(finetuned.predict(['hello', 'hi'], activation_fct=nn.Identity()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e3c1f756",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "id": "94cf736e",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█                                                                       | 51/3270 [00:44<19:37,  2.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 50\n",
      "Accuracy without re-ranking: 0.88\n",
      "Accuracy with re-ranking: 0.82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▏                                                                    | 100/3270 [01:07<23:44,  2.23it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 100\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▎                                                                   | 150/3270 [01:32<24:54,  2.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 150\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8466666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▎                                                                  | 200/3270 [02:06<30:03,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 200\n",
      "Accuracy without re-ranking: 0.865\n",
      "Accuracy with re-ranking: 0.85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|█████▍                                                                 | 250/3270 [02:32<22:50,  2.20it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 250\n",
      "Accuracy without re-ranking: 0.872\n",
      "Accuracy with re-ranking: 0.832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▌                                                                | 300/3270 [02:56<25:25,  1.95it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 300\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|███████▌                                                               | 350/3270 [03:24<27:41,  1.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 350\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8342857142857143\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|████████▋                                                              | 400/3270 [03:58<28:05,  1.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 400\n",
      "Accuracy without re-ranking: 0.8625\n",
      "Accuracy with re-ranking: 0.835\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█████████▊                                                             | 450/3270 [04:23<29:22,  1.60it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 450\n",
      "Accuracy without re-ranking: 0.8577777777777778\n",
      "Accuracy with re-ranking: 0.8333333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|██████████▊                                                            | 500/3270 [04:53<29:04,  1.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 500\n",
      "Accuracy without re-ranking: 0.852\n",
      "Accuracy with re-ranking: 0.832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|███████████▉                                                           | 550/3270 [05:17<20:42,  2.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 550\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8290909090909091\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████                                                          | 600/3270 [05:39<17:47,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 600\n",
      "Accuracy without re-ranking: 0.8383333333333334\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██████████████                                                         | 650/3270 [06:06<23:15,  1.88it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 650\n",
      "Accuracy without re-ranking: 0.8369230769230769\n",
      "Accuracy with re-ranking: 0.8276923076923077\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▏                                                       | 700/3270 [06:31<16:14,  2.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 700\n",
      "Accuracy without re-ranking: 0.8385714285714285\n",
      "Accuracy with re-ranking: 0.8285714285714286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|████████████████▎                                                      | 750/3270 [07:09<21:55,  1.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 750\n",
      "Accuracy without re-ranking: 0.832\n",
      "Accuracy with re-ranking: 0.8226666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|█████████████████▎                                                     | 800/3270 [07:28<15:52,  2.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 800\n",
      "Accuracy without re-ranking: 0.835\n",
      "Accuracy with re-ranking: 0.82625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██████████████████▍                                                    | 851/3270 [07:56<14:30,  2.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 850\n",
      "Accuracy without re-ranking: 0.8352941176470589\n",
      "Accuracy with re-ranking: 0.8270588235294117\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|███████████████████▌                                                   | 900/3270 [08:19<16:32,  2.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 900\n",
      "Accuracy without re-ranking: 0.8344444444444444\n",
      "Accuracy with re-ranking: 0.8233333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|████████████████████▋                                                  | 950/3270 [08:48<16:46,  2.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 950\n",
      "Accuracy without re-ranking: 0.8378947368421052\n",
      "Accuracy with re-ranking: 0.8210526315789474\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|█████████████████████▍                                                | 1000/3270 [09:09<15:09,  2.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1000\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.822\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|██████████████████████▍                                               | 1050/3270 [09:33<13:29,  2.74it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1050\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.82\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███████████████████████▌                                              | 1100/3270 [09:56<14:04,  2.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1100\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8209090909090909\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|████████████████████████▌                                             | 1150/3270 [10:16<19:39,  1.80it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1150\n",
      "Accuracy without re-ranking: 0.8443478260869566\n",
      "Accuracy with re-ranking: 0.8234782608695652\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|█████████████████████████▋                                            | 1200/3270 [10:38<21:26,  1.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1200\n",
      "Accuracy without re-ranking: 0.8458333333333333\n",
      "Accuracy with re-ranking: 0.8266666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|██████████████████████████▊                                           | 1250/3270 [11:02<09:46,  3.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1250\n",
      "Accuracy without re-ranking: 0.8488\n",
      "Accuracy with re-ranking: 0.8288\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███████████████████████████▊                                          | 1300/3270 [11:20<10:53,  3.01it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1300\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8292307692307692\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████████████████████████████▉                                         | 1350/3270 [11:37<19:39,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1350\n",
      "Accuracy without re-ranking: 0.8511111111111112\n",
      "Accuracy with re-ranking: 0.8318518518518518\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|█████████████████████████████▉                                        | 1400/3270 [12:26<15:02,  2.07it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1400\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8307142857142857\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|███████████████████████████████                                       | 1450/3270 [12:48<09:38,  3.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1450\n",
      "Accuracy without re-ranking: 0.8475862068965517\n",
      "Accuracy with re-ranking: 0.8289655172413793\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████████████████████████████████                                      | 1500/3270 [13:21<12:32,  2.35it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1500\n",
      "Accuracy without re-ranking: 0.846\n",
      "Accuracy with re-ranking: 0.828\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|█████████████████████████████████▏                                    | 1550/3270 [13:54<13:40,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1550\n",
      "Accuracy without re-ranking: 0.8464516129032258\n",
      "Accuracy with re-ranking: 0.8316129032258065\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|██████████████████████████████████▎                                   | 1600/3270 [14:17<13:13,  2.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1600\n",
      "Accuracy without re-ranking: 0.845625\n",
      "Accuracy with re-ranking: 0.83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|███████████████████████████████████▎                                  | 1650/3270 [14:43<08:55,  3.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1650\n",
      "Accuracy without re-ranking: 0.8460606060606061\n",
      "Accuracy with re-ranking: 0.8303030303030303\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|████████████████████████████████████▍                                 | 1700/3270 [15:05<06:57,  3.76it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1700\n",
      "Accuracy without re-ranking: 0.8482352941176471\n",
      "Accuracy with re-ranking: 0.8305882352941176\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████████████████████████████████████▍                                | 1750/3270 [15:24<07:15,  3.49it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1750\n",
      "Accuracy without re-ranking: 0.848\n",
      "Accuracy with re-ranking: 0.832\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|██████████████████████████████████████▌                               | 1800/3270 [15:44<08:02,  3.05it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1800\n",
      "Accuracy without re-ranking: 0.8483333333333334\n",
      "Accuracy with re-ranking: 0.8333333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|███████████████████████████████████████▌                              | 1850/3270 [16:06<09:23,  2.52it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1850\n",
      "Accuracy without re-ranking: 0.8475675675675676\n",
      "Accuracy with re-ranking: 0.8345945945945946\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|████████████████████████████████████████▋                             | 1900/3270 [16:25<08:27,  2.70it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1900\n",
      "Accuracy without re-ranking: 0.8494736842105263\n",
      "Accuracy with re-ranking: 0.8373684210526315\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████████████████████████████████████████▋                            | 1950/3270 [16:42<08:18,  2.65it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1950\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8374358974358974\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████████████████████████████████████████▊                           | 2000/3270 [17:03<07:24,  2.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.838\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|███████████████████████████████████████████▉                          | 2050/3270 [17:18<06:03,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2050\n",
      "Accuracy without re-ranking: 0.8526829268292683\n",
      "Accuracy with re-ranking: 0.8409756097560975\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|████████████████████████████████████████████▉                         | 2100/3270 [17:50<08:31,  2.29it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2100\n",
      "Accuracy without re-ranking: 0.8514285714285714\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████████████████████████████████████████████                        | 2150/3270 [18:25<11:22,  1.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2150\n",
      "Accuracy without re-ranking: 0.8502325581395349\n",
      "Accuracy with re-ranking: 0.8404651162790697\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|███████████████████████████████████████████████                       | 2200/3270 [18:56<06:31,  2.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2200\n",
      "Accuracy without re-ranking: 0.8504545454545455\n",
      "Accuracy with re-ranking: 0.8413636363636363\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|██████████████████████████████████████████████▊                     | 2250/3270 [19:53<1:04:46,  3.81s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2250\n",
      "Accuracy without re-ranking: 0.8506666666666667\n",
      "Accuracy with re-ranking: 0.8408888888888889\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|█████████████████████████████████████████████████▏                    | 2300/3270 [20:55<10:53,  1.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2300\n",
      "Accuracy without re-ranking: 0.8504347826086956\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|██████████████████████████████████████████████████▎                   | 2350/3270 [21:17<04:13,  3.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2350\n",
      "Accuracy without re-ranking: 0.8506382978723405\n",
      "Accuracy with re-ranking: 0.8408510638297872\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████████████████████████████████████████████████▍                  | 2400/3270 [21:36<03:47,  3.82it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2400\n",
      "Accuracy without re-ranking: 0.85125\n",
      "Accuracy with re-ranking: 0.8416666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████▍                 | 2450/3270 [21:55<05:10,  2.64it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2450\n",
      "Accuracy without re-ranking: 0.8526530612244898\n",
      "Accuracy with re-ranking: 0.843265306122449\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████████▌                | 2500/3270 [22:18<03:46,  3.40it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2500\n",
      "Accuracy without re-ranking: 0.8524\n",
      "Accuracy with re-ranking: 0.8428\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|██████████████████████████████████████████████████████▌               | 2550/3270 [22:49<05:34,  2.15it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2550\n",
      "Accuracy without re-ranking: 0.8509803921568627\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████▋              | 2600/3270 [23:17<10:46,  1.04it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2600\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████████████████████████████████████████████████████▋             | 2650/3270 [23:44<04:25,  2.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2650\n",
      "Accuracy without re-ranking: 0.849811320754717\n",
      "Accuracy with re-ranking: 0.8392452830188679\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|█████████████████████████████████████████████████████████▊            | 2701/3270 [24:09<03:46,  2.51it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2700\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.8392592592592593\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████▊           | 2750/3270 [24:32<03:55,  2.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2750\n",
      "Accuracy without re-ranking: 0.8501818181818181\n",
      "Accuracy with re-ranking: 0.8385454545454546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|███████████████████████████████████████████████████████████▉          | 2800/3270 [24:59<03:24,  2.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2800\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8382142857142857\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████         | 2850/3270 [25:23<03:55,  1.78it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2850\n",
      "Accuracy without re-ranking: 0.8501754385964913\n",
      "Accuracy with re-ranking: 0.8378947368421052\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|██████████████████████████████████████████████████████████████        | 2900/3270 [25:42<03:53,  1.59it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2900\n",
      "Accuracy without re-ranking: 0.8510344827586207\n",
      "Accuracy with re-ranking: 0.8379310344827586\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████▏      | 2950/3270 [26:01<02:22,  2.24it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2950\n",
      "Accuracy without re-ranking: 0.8501694915254238\n",
      "Accuracy with re-ranking: 0.8372881355932204\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████▏     | 3000/3270 [26:15<01:27,  3.08it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.8393333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████▎    | 3050/3270 [26:31<00:55,  3.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3050\n",
      "Accuracy without re-ranking: 0.8511475409836066\n",
      "Accuracy with re-ranking: 0.839344262295082\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|██████████████████████████████████████████████████████████████████▎   | 3100/3270 [26:50<00:49,  3.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3100\n",
      "Accuracy without re-ranking: 0.8522580645161291\n",
      "Accuracy with re-ranking: 0.8390322580645161\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|███████████████████████████████████████████████████████████████████▍  | 3150/3270 [27:07<00:33,  3.57it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3150\n",
      "Accuracy without re-ranking: 0.8526984126984127\n",
      "Accuracy with re-ranking: 0.8396825396825397\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|████████████████████████████████████████████████████████████████████▌ | 3201/3270 [27:31<00:20,  3.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3200\n",
      "Accuracy without re-ranking: 0.8525\n",
      "Accuracy with re-ranking: 0.83875\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████████████████████████████████████████████████████████████████▌| 3250/3270 [27:50<00:05,  3.50it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3250\n",
      "Accuracy without re-ranking: 0.8526153846153847\n",
      "Accuracy with re-ranking: 0.8384615384615385\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [27:57<00:00,  1.95it/s]\n"
     ]
    }
   ],
   "source": [
    "# Trying our fine-tuned cross encoder\n",
    "logger.setLevel(logging.CRITICAL)  # just to suppress some logs\n",
    "from tqdm import tqdm\n",
    "\n",
    "i = 0\n",
    "print_every = 50\n",
    "predictions = []\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash, reranked_hash = eval_ranking(question, finetuned, top_k=3)\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append((retrieved_hash == correct_hash, reranked_hash == correct_hash))\n",
    "    i += 1\n",
    "    if i % print_every == 0:\n",
    "        print(f'Step {i}')\n",
    "        raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "        reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "        print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "        print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "id": "bb46e242",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Re-ranking got slightly better after 2 epochs."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 120,
   "id": "2b6e0a9e",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cross-encoder: <sentence_transformers.cross_encoder.CrossEncoder.CrossEncoder object at 0x158c9cc70>\n",
      "Accuracy without re-ranking: 0.8522935779816514\n",
      "Accuracy with re-ranking: 0.8495412844036697\n"
     ]
    }
   ],
   "source": [
    "raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "print(f'Using cross-encoder: {finetuned.config._name_or_path}')\n",
    "print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "92875bca",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "75b414c6",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "id": "2535cbae",
   "metadata": {},
   "outputs": [],
   "source": [
    "# pinecone.delete_index(INDEX_NAME)  # delete the index"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "3f14809e",
   "metadata": {},
   "source": [
    "# OPEN SOURCE ALTERNATIVE TO EMBEDDING"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "id": "99138055",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "Batches:   0%|          | 0/1 [00:00<?, ?it/s]"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/plain": [
       "(2, 768)"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sentence_transformers import SentenceTransformer\n",
    "\n",
    "model = SentenceTransformer('sentence-transformers/multi-qa-mpnet-base-cos-v1')\n",
    "\n",
    "docs = [\"Around 9 Million people live in London\", \"London is known for its financial district\"]\n",
    "\n",
    "doc_emb = model.encode(docs, batch_size=32, show_progress_bar=True)\n",
    "\n",
    "doc_emb.shape#  == ('2, 768')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "a1140c18",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 379,
   "id": "be37c48b",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████████| 33/33 [00:29<00:00,  1.11it/s]\n"
     ]
    }
   ],
   "source": [
    "#Encode query and documents\n",
    "docs = dataset['validation']['passage']\n",
    "doc_emb = model.encode(docs, batch_size=32, show_progress_bar=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 380,
   "id": "ae0afee7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "did greg oden win a championship with miami\n",
      "Query: did greg oden win a championship with miami\n",
      "Document ID (Hash)\t\tRetrieval Score\tCE Score\tText\n",
      "dc737eb4be84a308b945fb697a67f922\t0.82\t0.35\tOn January 15, 2014, Oden made his long-awaited re\n",
      "275270d2171ffe4d024015ab4ea3343b\t0.78\t0.05\tSelected 5th overall in the 2003 NBA draft by the \n",
      "ed060ec9bb1129f14f06186a2c5d29b4\t0.78\t0.02\tThe 2012 NBA Finals was the championship series of\n"
     ]
    }
   ],
   "source": [
    "from random import sample\n",
    "\n",
    "query = sample(dataset['validation']['question'], 1)[0]\n",
    "print(query)\n",
    "final_results = get_results_from_pinecone(query, top_k=3, re_rank=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 381,
   "id": "45c71fb0",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.8174281120809472 On January 15, 2014, Oden made his long-awaited return to the court. In his first regular season game since December 2009, he recorded 6 points and 2 rebounds in 8 minutes of game time in a 114--97 loss to the Washington Wizards. On February 23, 2014, Oden made his first start since December 2009 in the Heat's 93--79 win over the Chicago Bulls. The Heat made the 2014 NBA Finals where they faced the San Antonio Spurs. They went on to lose the series in five games.\n",
      "0.7841596747078865 Selected 5th overall in the 2003 NBA draft by the Miami Heat, Wade quickly emerged as a productive player on a youthful Miami Heat team and averaged 16.2 points on 46.5% shooting with averages of 4.0 rebounds and 4.5 assists per game. Wade is one of only four Marquette University players to be drafted in the first round; his is the highest draft selection in school history. After a 5--15 start, the Heat would gradually improve and finish 42--40 to qualify for the NBA playoffs. He further distinguished himself with outstanding performances in the playoffs, particularly against the Indiana Pacers in the Eastern Conference Semi-finals. In the end, however, Wade's successful rookie season was somewhat overshadowed by the success of fellow rookies Carmelo Anthony and LeBron James. Wade did earn unanimous selection to the 2004 NBA All-Rookie Team, and also finished third in rookie of the year voting (behind James and Anthony). He was ranked in the top five among rookies in several major statistical categories, including second in field goal percentage, second in steals, third in scoring, fourth in assists, and fourth in minutes played. In the playoffs Wade hit a game-winning shot in Game 1 of the Heat's first round series against the New Orleans Hornets. The Heat won the series 4--3 and advanced to the second round to face the top-seeded and best record team in the NBA, the Indiana Pacers, in a very entertaining series that almost pushed the 61-win Pacers to the edge, though Miami would eventually lose the series in six games. He became the fourth rookie since the shot clock era began to lead his team in scoring and assist average in the postseason.\n",
      "0.7836699729450296 The 2012 NBA Finals was the championship series of the 2011--12 season of the National Basketball Association (NBA), and the conclusion of the season's playoffs. The Eastern Conference champion Miami Heat defeated the Western Conference champion Oklahoma City Thunder 4 games to 1 to win their second NBA title. Heat Small forward LeBron James was named the Finals MVP.\n"
     ]
    }
   ],
   "source": [
    "query_emb = model.encode(query)\n",
    "\n",
    "#Compute dot score between query and all document embeddings\n",
    "scores = util.dot_score(query_emb, doc_emb)[0].cpu().tolist()\n",
    "\n",
    "#Combine docs & scores\n",
    "doc_score_pairs = list(zip(docs, scores))\n",
    "\n",
    "#Sort by decreasing score\n",
    "doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True)\n",
    "\n",
    "#Output passages & scores\n",
    "for doc, score in doc_score_pairs[:3]:\n",
    "    print(score, doc)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "8533d9cb",
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 382,
   "id": "93280be8",
   "metadata": {},
   "outputs": [],
   "source": [
    "logger.setLevel(logging.CRITICAL)  # just to suppress some logs\n",
    "\n",
    "\n",
    "def eval_ranking_open_source(query, cross_encoder, top_k=3):\n",
    "    query_emb = np.array(get_embedding(query, engine=ENGINE))\n",
    "\n",
    "    #Compute dot score between query and all document embeddings\n",
    "    scores = util.dot_score(query_emb, doc_emb)[0].cpu().tolist()\n",
    "\n",
    "    #Combine docs & scores\n",
    "    doc_score_pairs = list(zip(docs, scores))\n",
    "\n",
    "    #Sort by decreasing score\n",
    "    doc_score_pairs = sorted(doc_score_pairs, key=lambda x: x[1], reverse=True)[:top_k]\n",
    "\n",
    "    retrieved_hash = my_hash(doc_score_pairs[0][0])\n",
    "    if cross_encoder:\n",
    "        sentence_combinations = [[query, doc_score_pair[0]] for doc_score_pair in doc_score_pairs]\n",
    "        similarity_scores = cross_encoder.predict(sentence_combinations)\n",
    "        sim_scores_argsort = list(reversed(np.argsort(similarity_scores)))\n",
    "        reranked_hash = my_hash(doc_score_pairs[sim_scores_argsort[0]][0])\n",
    "    else:\n",
    "        reranked_hash = None\n",
    "    return retrieved_hash, reranked_hash\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 383,
   "id": "06f198cd",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('dc737eb4be84a308b945fb697a67f922', 'dc737eb4be84a308b945fb697a67f922')"
      ]
     },
     "execution_count": 383,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "eval_ranking_open_source(query, finetuned)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 384,
   "id": "8faf0c1a",
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  2%|█                                                                       | 51/3270 [00:16<11:01,  4.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 50\n",
      "Accuracy without re-ranking: 0.88\n",
      "Accuracy with re-ranking: 0.84\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  3%|██▏                                                                    | 100/3270 [00:31<14:50,  3.56it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 100\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.85\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  5%|███▎                                                                   | 151/3270 [00:49<10:45,  4.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 150\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8533333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  6%|████▎                                                                  | 200/3270 [01:09<15:19,  3.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 200\n",
      "Accuracy without re-ranking: 0.865\n",
      "Accuracy with re-ranking: 0.855\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  8%|█████▍                                                                 | 250/3270 [01:21<13:01,  3.86it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 250\n",
      "Accuracy without re-ranking: 0.872\n",
      "Accuracy with re-ranking: 0.844\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "  9%|██████▌                                                                | 301/3270 [01:35<11:34,  4.27it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 300\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.8366666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 11%|███████▌                                                               | 350/3270 [01:49<16:06,  3.02it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 350\n",
      "Accuracy without re-ranking: 0.86\n",
      "Accuracy with re-ranking: 0.8457142857142858\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 12%|████████▋                                                              | 401/3270 [02:00<08:31,  5.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 400\n",
      "Accuracy without re-ranking: 0.8625\n",
      "Accuracy with re-ranking: 0.8475\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 14%|█████████▊                                                             | 451/3270 [02:16<09:10,  5.12it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 450\n",
      "Accuracy without re-ranking: 0.8577777777777778\n",
      "Accuracy with re-ranking: 0.8466666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 15%|██████████▊                                                            | 500/3270 [02:34<12:59,  3.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 500\n",
      "Accuracy without re-ranking: 0.852\n",
      "Accuracy with re-ranking: 0.844\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 17%|███████████▉                                                           | 551/3270 [02:51<12:22,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 550\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8381818181818181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 18%|█████████████                                                          | 600/3270 [03:03<09:46,  4.55it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 600\n",
      "Accuracy without re-ranking: 0.8383333333333334\n",
      "Accuracy with re-ranking: 0.8366666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 20%|██████████████▏                                                        | 651/3270 [03:17<11:08,  3.92it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 650\n",
      "Accuracy without re-ranking: 0.8369230769230769\n",
      "Accuracy with re-ranking: 0.8369230769230769\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 21%|███████████████▏                                                       | 701/3270 [03:30<15:07,  2.83it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 700\n",
      "Accuracy without re-ranking: 0.8385714285714285\n",
      "Accuracy with re-ranking: 0.8371428571428572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 23%|████████████████▎                                                      | 751/3270 [03:42<08:56,  4.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 750\n",
      "Accuracy without re-ranking: 0.832\n",
      "Accuracy with re-ranking: 0.8306666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 24%|█████████████████▎                                                     | 800/3270 [03:58<09:16,  4.44it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 800\n",
      "Accuracy without re-ranking: 0.835\n",
      "Accuracy with re-ranking: 0.83375\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 26%|██████████████████▍                                                    | 851/3270 [04:13<12:11,  3.31it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 850\n",
      "Accuracy without re-ranking: 0.8352941176470589\n",
      "Accuracy with re-ranking: 0.8341176470588235\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 28%|███████████████████▌                                                   | 900/3270 [04:25<15:00,  2.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 900\n",
      "Accuracy without re-ranking: 0.8344444444444444\n",
      "Accuracy with re-ranking: 0.83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 29%|████████████████████▋                                                  | 950/3270 [04:39<09:11,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 950\n",
      "Accuracy without re-ranking: 0.8378947368421052\n",
      "Accuracy with re-ranking: 0.8294736842105264\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 31%|█████████████████████▍                                                | 1001/3270 [04:53<12:52,  2.94it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1000\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.83\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 32%|██████████████████████▍                                               | 1050/3270 [05:06<08:25,  4.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1050\n",
      "Accuracy without re-ranking: 0.84\n",
      "Accuracy with re-ranking: 0.8276190476190476\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 34%|███████████████████████▌                                              | 1100/3270 [05:19<10:14,  3.53it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1100\n",
      "Accuracy without re-ranking: 0.8418181818181818\n",
      "Accuracy with re-ranking: 0.8281818181818181\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 35%|████████████████████████▌                                             | 1150/3270 [05:33<21:36,  1.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1150\n",
      "Accuracy without re-ranking: 0.8443478260869566\n",
      "Accuracy with re-ranking: 0.8295652173913044\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 37%|█████████████████████████▋                                            | 1200/3270 [05:51<12:57,  2.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1200\n",
      "Accuracy without re-ranking: 0.8458333333333333\n",
      "Accuracy with re-ranking: 0.8325\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 38%|██████████████████████████▊                                           | 1251/3270 [06:08<06:36,  5.09it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1250\n",
      "Accuracy without re-ranking: 0.8488\n",
      "Accuracy with re-ranking: 0.8344\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 40%|███████████████████████████▊                                          | 1301/3270 [06:21<08:24,  3.90it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1300\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8369230769230769\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 41%|████████████████████████████▉                                         | 1351/3270 [06:35<07:35,  4.21it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1350\n",
      "Accuracy without re-ranking: 0.8511111111111112\n",
      "Accuracy with re-ranking: 0.8385185185185186\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 43%|█████████████████████████████▉                                        | 1401/3270 [06:49<09:22,  3.32it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1400\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8371428571428572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 44%|███████████████████████████████                                       | 1450/3270 [07:05<06:45,  4.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1450\n",
      "Accuracy without re-ranking: 0.8475862068965517\n",
      "Accuracy with re-ranking: 0.8358620689655173\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 46%|████████████████████████████████                                      | 1500/3270 [07:16<05:52,  5.03it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1500\n",
      "Accuracy without re-ranking: 0.846\n",
      "Accuracy with re-ranking: 0.834\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 47%|█████████████████████████████████▏                                    | 1550/3270 [07:29<06:51,  4.18it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1550\n",
      "Accuracy without re-ranking: 0.8464516129032258\n",
      "Accuracy with re-ranking: 0.8374193548387097\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 49%|██████████████████████████████████▎                                   | 1600/3270 [07:43<05:22,  5.19it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1600\n",
      "Accuracy without re-ranking: 0.845625\n",
      "Accuracy with re-ranking: 0.835625\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 50%|███████████████████████████████████▎                                  | 1650/3270 [07:58<21:24,  1.26it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1650\n",
      "Accuracy without re-ranking: 0.8460606060606061\n",
      "Accuracy with re-ranking: 0.8357575757575758\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 52%|████████████████████████████████████▍                                 | 1700/3270 [08:13<07:15,  3.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1700\n",
      "Accuracy without re-ranking: 0.8482352941176471\n",
      "Accuracy with re-ranking: 0.8358823529411765\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 54%|█████████████████████████████████████▍                                | 1751/3270 [08:30<05:48,  4.36it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1750\n",
      "Accuracy without re-ranking: 0.848\n",
      "Accuracy with re-ranking: 0.8371428571428572\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 55%|██████████████████████████████████████▌                               | 1800/3270 [08:45<06:40,  3.67it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1800\n",
      "Accuracy without re-ranking: 0.8483333333333334\n",
      "Accuracy with re-ranking: 0.8383333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 57%|███████████████████████████████████████▌                              | 1850/3270 [08:58<11:07,  2.13it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1850\n",
      "Accuracy without re-ranking: 0.8475675675675676\n",
      "Accuracy with re-ranking: 0.8394594594594594\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 58%|████████████████████████████████████████▋                             | 1901/3270 [09:12<05:18,  4.30it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1900\n",
      "Accuracy without re-ranking: 0.8494736842105263\n",
      "Accuracy with re-ranking: 0.8415789473684211\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 60%|█████████████████████████████████████████▊                            | 1951/3270 [09:28<03:51,  5.69it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 1950\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.841025641025641\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 61%|██████████████████████████████████████████▊                           | 2001/3270 [09:41<04:26,  4.77it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.8415\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 63%|███████████████████████████████████████████▉                          | 2050/3270 [09:52<04:36,  4.41it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2050\n",
      "Accuracy without re-ranking: 0.8526829268292683\n",
      "Accuracy with re-ranking: 0.844390243902439\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 64%|████████████████████████████████████████████▉                         | 2101/3270 [10:10<04:09,  4.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2100\n",
      "Accuracy without re-ranking: 0.8514285714285714\n",
      "Accuracy with re-ranking: 0.8433333333333334\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 66%|██████████████████████████████████████████████                        | 2150/3270 [10:20<04:18,  4.33it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2150\n",
      "Accuracy without re-ranking: 0.8502325581395349\n",
      "Accuracy with re-ranking: 0.8437209302325581\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 67%|███████████████████████████████████████████████                       | 2200/3270 [10:34<03:47,  4.71it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2200\n",
      "Accuracy without re-ranking: 0.8504545454545455\n",
      "Accuracy with re-ranking: 0.8445454545454546\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 69%|████████████████████████████████████████████████▏                     | 2251/3270 [10:49<03:06,  5.45it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2250\n",
      "Accuracy without re-ranking: 0.8506666666666667\n",
      "Accuracy with re-ranking: 0.8444444444444444\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 70%|█████████████████████████████████████████████████▏                    | 2300/3270 [11:03<04:24,  3.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2300\n",
      "Accuracy without re-ranking: 0.8504347826086956\n",
      "Accuracy with re-ranking: 0.8434782608695652\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 72%|██████████████████████████████████████████████████▎                   | 2350/3270 [11:20<05:43,  2.68it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2350\n",
      "Accuracy without re-ranking: 0.8506382978723405\n",
      "Accuracy with re-ranking: 0.8442553191489361\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 73%|███████████████████████████████████████████████████▍                  | 2400/3270 [11:38<03:53,  3.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2400\n",
      "Accuracy without re-ranking: 0.85125\n",
      "Accuracy with re-ranking: 0.845\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 75%|████████████████████████████████████████████████████▍                 | 2450/3270 [11:49<03:46,  3.62it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2450\n",
      "Accuracy without re-ranking: 0.8526530612244898\n",
      "Accuracy with re-ranking: 0.846530612244898\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 76%|█████████████████████████████████████████████████████▌                | 2501/3270 [12:03<08:41,  1.47it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2500\n",
      "Accuracy without re-ranking: 0.8524\n",
      "Accuracy with re-ranking: 0.846\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 78%|██████████████████████████████████████████████████████▌               | 2550/3270 [12:15<02:05,  5.73it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2550\n",
      "Accuracy without re-ranking: 0.8509803921568627\n",
      "Accuracy with re-ranking: 0.8431372549019608\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 80%|███████████████████████████████████████████████████████▋              | 2600/3270 [12:29<03:24,  3.28it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2600\n",
      "Accuracy without re-ranking: 0.8492307692307692\n",
      "Accuracy with re-ranking: 0.8430769230769231\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 81%|████████████████████████████████████████████████████████▋             | 2650/3270 [12:43<02:06,  4.89it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2650\n",
      "Accuracy without re-ranking: 0.849811320754717\n",
      "Accuracy with re-ranking: 0.8422641509433962\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 83%|█████████████████████████████████████████████████████████▊            | 2700/3270 [13:02<03:49,  2.48it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2700\n",
      "Accuracy without re-ranking: 0.85\n",
      "Accuracy with re-ranking: 0.8422222222222222\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 84%|██████████████████████████████████████████████████████████▊           | 2750/3270 [13:15<02:16,  3.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2750\n",
      "Accuracy without re-ranking: 0.8501818181818181\n",
      "Accuracy with re-ranking: 0.8414545454545455\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 86%|███████████████████████████████████████████████████████████▉          | 2801/3270 [13:36<01:48,  4.34it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2800\n",
      "Accuracy without re-ranking: 0.8492857142857143\n",
      "Accuracy with re-ranking: 0.8410714285714286\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 87%|█████████████████████████████████████████████████████████████         | 2851/3270 [13:47<01:30,  4.61it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2850\n",
      "Accuracy without re-ranking: 0.8501754385964913\n",
      "Accuracy with re-ranking: 0.8417543859649123\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 89%|██████████████████████████████████████████████████████████████        | 2900/3270 [14:00<01:12,  5.10it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2900\n",
      "Accuracy without re-ranking: 0.8510344827586207\n",
      "Accuracy with re-ranking: 0.8420689655172414\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 90%|███████████████████████████████████████████████████████████████▏      | 2951/3270 [14:14<01:44,  3.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 2950\n",
      "Accuracy without re-ranking: 0.8501694915254238\n",
      "Accuracy with re-ranking: 0.8410169491525423\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 92%|████████████████████████████████████████████████████████████████▏     | 3001/3270 [14:26<00:49,  5.39it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3000\n",
      "Accuracy without re-ranking: 0.851\n",
      "Accuracy with re-ranking: 0.8426666666666667\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 93%|█████████████████████████████████████████████████████████████████▎    | 3051/3270 [14:39<00:47,  4.66it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3050\n",
      "Accuracy without re-ranking: 0.8511475409836066\n",
      "Accuracy with re-ranking: 0.8426229508196721\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 95%|██████████████████████████████████████████████████████████████████▎   | 3100/3270 [14:53<00:36,  4.63it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3100\n",
      "Accuracy without re-ranking: 0.8522580645161291\n",
      "Accuracy with re-ranking: 0.8425806451612903\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 96%|███████████████████████████████████████████████████████████████████▍  | 3150/3270 [15:06<00:24,  4.81it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3150\n",
      "Accuracy without re-ranking: 0.8526984126984127\n",
      "Accuracy with re-ranking: 0.8431746031746031\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 98%|████████████████████████████████████████████████████████████████████▌ | 3201/3270 [15:21<00:17,  4.06it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3200\n",
      "Accuracy without re-ranking: 0.8525\n",
      "Accuracy with re-ranking: 0.8425\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      " 99%|█████████████████████████████████████████████████████████████████████▌| 3251/3270 [15:31<00:03,  5.11it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Step 3250\n",
      "Accuracy without re-ranking: 0.8529230769230769\n",
      "Accuracy with re-ranking: 0.8427692307692307\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "100%|██████████████████████████████████████████████████████████████████████| 3270/3270 [15:35<00:00,  3.49it/s]\n"
     ]
    }
   ],
   "source": [
    "logger.setLevel(logging.CRITICAL)\n",
    "\n",
    "i = 0\n",
    "print_every = 50\n",
    "predictions = []\n",
    "for question in tqdm(val_sample['question']):\n",
    "    retrieved_hash, reranked_hash = eval_ranking_open_source(question, finetuned, top_k=3)\n",
    "    correct_hash = q_to_hash[question]\n",
    "    predictions.append((retrieved_hash == correct_hash, reranked_hash == correct_hash))\n",
    "    i += 1\n",
    "    if i % print_every == 0:\n",
    "        print(f'Step {i}')\n",
    "        raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "        reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "        print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "        print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 385,
   "id": "6eaeb9b4",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Using cross-encoder: cross-encoder/mmarco-mMiniLMv2-L12-H384-v1\n",
      "Accuracy without re-ranking: 0.8525993883792049\n",
      "Accuracy with re-ranking: 0.8434250764525993\n"
     ]
    }
   ],
   "source": [
    "raw_accuracy = sum([p[0] for p in predictions])/len(predictions)\n",
    "reranked_accuracy = sum([p[1] for p in predictions])/len(predictions)\n",
    "\n",
    "print(f'Using cross-encoder: {finetuned.config._name_or_path}')\n",
    "print(f'Accuracy without re-ranking: {raw_accuracy}')\n",
    "print(f'Accuracy with re-ranking: {reranked_accuracy}')\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "1eca7d89",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}