question_chunks.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f7ebe218-db7b-4189-8a38-f52f2ee66347",
   "metadata": {},
   "outputs": [],
   "source": [
    "import os\n",
    "import openai\n",
    "import json\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "def load_json(filename):\n",
    "    with open(filename, 'r', encoding='utf-8') as file:\n",
    "        data = json.load(file)\n",
    "    return data\n",
    "\n",
    "\n",
    "load_dotenv()\n",
    "openai.organization = os.getenv(\"OPENAI_ORG\")\n",
    "openai.api_key = os.getenv(\"OPENAI_API_KEY\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "55314073-200b-4b70-8c55-17fb47a6c47e",
   "metadata": {},
   "outputs": [],
   "source": [
    "def load_json(filename):\n",
    "    with open(filename, 'r', encoding='utf-8') as file:\n",
    "        data = json.load(file)\n",
    "    return data\n",
    "\n",
    "def save_json(filename,data):\n",
    "    with open(filename, 'w', encoding='utf-8') as f:\n",
    "        json.dump(data, f, indent=4)\n",
    "    return"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "2c3dd0af-02de-4c4e-a598-5ac62a531d01",
   "metadata": {},
   "outputs": [],
   "source": [
    "from IPython.display import JSON"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e7b756d7-4142-4d18-b4f5-480be7ed9b69",
   "metadata": {},
   "source": [
    "# Load chunks\n",
    "`chunks.json` is a list of strings with max tokens and overlap"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "2324cf30-0c00-4b2d-bcf8-76ed46bf1165",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "419"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "json_data = load_json(\"website_data/export/chunks.json\")\n",
    "#JSON(json_data)\n",
    "len(json_data)"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3d71fc8-651f-43b6-8324-34103e5c6442",
   "metadata": {},
   "source": [
    "## chunks truncation\n",
    "for demo purpose, only first 10 chunks used"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8ab595e5-c862-44bf-91dd-858225e1cf11",
   "metadata": {},
   "outputs": [],
   "source": [
    "text_input = json_data[:10]"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "756e574d-5c9c-4414-ba0c-a823c4bfd729",
   "metadata": {},
   "source": [
    "# Embeddings API call\n",
    "trandforms Chunks into Embeddings"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "d1573806-ee66-464a-a533-a90dc8f29c64",
   "metadata": {},
   "outputs": [],
   "source": [
    "response = openai.Embedding.create(\n",
    "    input=text_input,\n",
    "    model=\"text-embedding-ada-002\",\n",
    "    user=\"website_embeddings.ipynb\"\n",
    ")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "fcb76488-a77a-4d4d-a930-e284454b81af",
   "metadata": {},
   "outputs": [],
   "source": [
    "#JSON(response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "df9837a1-0c19-4262-abad-f9415abf123a",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json(\"website_data/embeddings/response.json\",response)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "6722e739-a8be-4a14-88ad-d256916809a0",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = [entry[\"embedding\"] for entry in response[\"data\"] ]\n",
    "#JSON(embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "a7c17629-45b1-4bbf-9db6-919c6337530b",
   "metadata": {},
   "outputs": [],
   "source": [
    "save_json(\"website_data/embeddings/embeddings.json\",embeddings)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "116cdbf3-16df-4ac9-b19d-c8025c6bef74",
   "metadata": {},
   "outputs": [],
   "source": [
    "embeddings = load_json(\"website_data/embeddings/embeddings.json\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "a325d63f-2856-48f5-b49d-78e8afc1b73d",
   "metadata": {},
   "source": [
    "# Vector Store"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "4f592993-e5fd-41de-a15a-77b183e0e361",
   "metadata": {},
   "outputs": [],
   "source": [
    "import faiss\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "6090b116-f007-428d-b870-f37c5be546cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[ 0.0071328 ,  0.00672303,  0.00615699, ...,  0.00047228,\n",
       "        -0.01272376, -0.05311751],\n",
       "       [ 0.0031525 ,  0.02858355,  0.00448695, ...,  0.00307107,\n",
       "        -0.02127148, -0.03515112],\n",
       "       [ 0.00095427,  0.02871195,  0.00421623, ..., -0.00827817,\n",
       "        -0.01112924, -0.03348833],\n",
       "       ...,\n",
       "       [ 0.00623995,  0.02721228, -0.02536341, ..., -0.0013834 ,\n",
       "        -0.0033462 , -0.04978936],\n",
       "       [ 0.01363436,  0.03848915, -0.02468762, ...,  0.00132983,\n",
       "         0.00347379, -0.04359785],\n",
       "       [ 0.00854609,  0.04833328, -0.01710539, ...,  0.006563  ,\n",
       "        -0.01306661, -0.04492804]], dtype=float32)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "embeddings_np = np.array(embeddings).astype('float32')\n",
    "embeddings_np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "id": "0b6bd1ff-17df-4b64-9b1f-cb147107da69",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "vectors dimension 1536\n"
     ]
    }
   ],
   "source": [
    "dimension = embeddings_np.shape[1]\n",
    "faiss_index = faiss.IndexFlatL2(dimension)\n",
    "faiss_index.add(embeddings_np)\n",
    "print(f\"vectors dimension {dimension}\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "id": "313ac7aa-b183-4ac6-938c-86e807be229d",
   "metadata": {},
   "outputs": [],
   "source": [
    "faiss.write_index(faiss_index,\"website_data/embeddings/faiss_index.index\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a3a4fea8-dd65-4ef9-879a-7b3e9b0fbe97",
   "metadata": {},
   "outputs": [],
   "source": [
    "faiss_index = faiss.read_index(\"website_data/embeddings/faiss_index.index\")"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "59f44e37-cf71-4dc5-89ef-8e7220191152",
   "metadata": {},
   "source": [
    "# Query"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "5c36e0a7-c8cb-4399-9fad-e4d73b24ff7d",
   "metadata": {},
   "outputs": [],
   "source": [
    "question = \"is an Astro island interactive or not ?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "id": "8e55b3db-7210-495e-964d-f3bad0a195ae",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"prompt_tokens\": 8,\n",
      "  \"total_tokens\": 8\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "response = openai.Embedding.create(\n",
    "    input=question,\n",
    "    model=\"text-embedding-ada-002\",\n",
    "    user=\"website_embeddings.ipynb\"\n",
    ")\n",
    "print(response[\"usage\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "c54975cb-8a7f-47b2-864e-8fc11edbf41c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([-0.01427942, -0.01084952, -0.01550539, ..., -0.01527863,\n",
       "       -0.01619988, -0.05011616], dtype=float32)"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_embedding = np.array(response[\"data\"][0][\"embedding\"]).astype('float32')\n",
    "query_embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "871e6230-811c-4176-9a82-e69dd33904b8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[-0.01427942, -0.01084952, -0.01550539, ..., -0.01527863,\n",
       "        -0.01619988, -0.05011616]], dtype=float32)"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "query_embedding_2d = query_embedding.reshape(1,-1)\n",
    "query_embedding_2d"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c3587210-2da6-470a-87bd-904e88f16430",
   "metadata": {},
   "source": [
    "## search"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "efbd4dfe-6148-45af-bdc1-4251e9872579",
   "metadata": {},
   "outputs": [],
   "source": [
    "top_k = 2\n",
    "distances, indices = faiss_index.search(query_embedding_2d, top_k)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "id": "4f1ab491-4090-4915-867f-4038fe386d90",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.3331605"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "distances[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "id": "53c516a2-2049-4b31-a70f-7297a7b23074",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0 4]\n"
     ]
    }
   ],
   "source": [
    "print(indices[0])"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "28c871a7-b752-446e-921e-5f4bcda35725",
   "metadata": {},
   "source": [
    "## Show references"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "id": "aa58d7ff-57c0-4ce9-b82b-af9c896d49e8",
   "metadata": {},
   "outputs": [],
   "source": [
    "chunks_info = load_json(\"website_data/export/chunks_infos.json\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "id": "33ca7889-b868-49e7-ae3b-693463fad82c",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      " * chunk 0 with distance (0.33316048979759216) from concepts/islands \n",
      " * chunk 4 with distance (0.4124778211116791) from concepts/why-astro \n"
     ]
    }
   ],
   "source": [
    "messages = []\n",
    "for i in range(len(indices[0])):\n",
    "    print(f\" * chunk {indices[0][i]} with distance ({distances[0][i]}) from {chunks_info[indices[0][i]]['file_path']} \")\n",
    "    messages.append({\"role\":\"system\", \"content\":chunks_info[indices[0][i]]['content']})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "id": "103d2720-585c-4266-a71e-25c5f748edb3",
   "metadata": {},
   "outputs": [],
   "source": [
    "messages.append({\"role\":\"user\", \"content\":question})"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "0698a46e-d79b-42f4-9f64-5b97edfdc3d7",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"prompt_tokens\": 1577,\n",
      "  \"completion_tokens\": 69,\n",
      "  \"total_tokens\": 1646\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "completion = openai.ChatCompletion.create(\n",
    "  model=\"gpt-3.5-turbo\",\n",
    "  messages=messages\n",
    ")\n",
    "print(completion.usage)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "id": "bb6f7c75-bfed-4224-831e-3f3c7ee98e28",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{\n",
      "  \"role\": \"assistant\",\n",
      "  \"content\": \"An Astro island is interactive. It refers to an interactive UI component on an otherwise static page of HTML. The island itself is rendered in isolation, separate from the rest of the static content on the page. You can use any supported UI framework (React, Svelte, Vue, etc.) to render islands in Astro, making them fully interactive.\"\n",
      "}\n"
     ]
    }
   ],
   "source": [
    "print(completion.choices[0].message)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "feb2ff59-d632-440b-9202-e00474d350dc",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.11"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}