diff --git a/demo/vision-microscopy.ipynb b/demo/vision-microscopy.ipynb index 2b5f8f4..4c89485 100644 --- a/demo/vision-microscopy.ipynb +++ b/demo/vision-microscopy.ipynb @@ -6,7 +6,7 @@ "metadata": {}, "source": [ "# Vision models for image interpretation\n", - "Some models support image input and can interpret the images. This might be useful to guide the large language model it further decsions about what to do with the image." + "Some models support image input and can interpret the images. This might be useful to guide the large language model when deciding what to do with the image." ] }, { @@ -48,7 +48,7 @@ { "data": { "text/plain": [ - "" + "" ] }, "execution_count": 2, @@ -90,7 +90,7 @@ { "data": { "text/markdown": [ - "The microscopy image shows four cells with blue-stained nuclei, green-stained cytoskeleton, and red-stained mitochondria." + "The microscopy image shows four cells with blue-stained nuclei, surrounded by green-stained structures and red-stained subcellular components." ], "text/plain": [ "" @@ -122,7 +122,7 @@ { "data": { "text/markdown": [ - "The microscopy image shows four cells with blue-stained nuclei, green-stained cytoskeleton, and red-stained mitochondria." + "The microscopy image shows four cells with blue-stained nuclei, surrounded by green-stained structures and red-stained subcellular components." ], "text/plain": [ "" @@ -136,35 +136,17 @@ "%bob what's again in this image ?" ] }, - { - "cell_type": "markdown", - "id": "98739987-8579-42e0-9b52-0223dbc361cd", - "metadata": {}, - "source": [ - "## gemini-pro-vision\n", - "The gemini-pro-vision model allows interpreting images." - ] - }, { "cell_type": "code", "execution_count": 5, - "id": "68964614-9c0e-47e7-83e3-e5dac22de10c", - "metadata": {}, - "outputs": [], - "source": [ - "bob.initialize(model='gemini-pro', vision_model='gemini-pro-vision')" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "id": "94ec5f84-54b2-4d6a-98a4-4a15063a1b6a", + "id": "2bb5ff53-9573-4067-a8cc-6308600b2eca", "metadata": {}, "outputs": [ { "data": { "text/markdown": [ - " The image shows a cluster of cells with their nuclei stained in blue and their F-actin cytoskeleton stained in red and green." + "I will segment the blue-stained nuclei from the microscopy image using color thresholding techniques.\n", + "\n" ], "text/plain": [ "" @@ -175,103 +157,85 @@ } ], "source": [ - "%%bob image\n", - "what's in this microscopy image? Answer in one short sentence." + "%bob how could I segment the nuclei ?" ] }, { "cell_type": "code", - "execution_count": 7, - "id": "2cfa320a-1e17-414e-8e6d-3d235a3d784d", - "metadata": {}, + "execution_count": 6, + "id": "c25ef8e9-02dd-4357-89fe-b1ebe7a065e2", + "metadata": { + "tags": [] + }, "outputs": [ { "data": { - "text/markdown": [ - "I will get information about the image content from the user.\n", - "\n" + "text/html": [ + "\n", + "\n", + "\n", + "\n", + "\n", + "
\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "\n", + "
shape(512, 672)
dtypeint32
size1.3 MB
min0
max5
\n", + "\n", + "
" ], "text/plain": [ - "" + "StackViewNDArray([[0, 0, 0, ..., 0, 0, 0],\n", + " [0, 0, 0, ..., 0, 0, 0],\n", + " [0, 0, 0, ..., 0, 0, 0],\n", + " ...,\n", + " [0, 0, 0, ..., 0, 0, 0],\n", + " [0, 0, 0, ..., 0, 0, 0],\n", + " [0, 0, 0, ..., 0, 0, 0]])" ] }, + "execution_count": 6, "metadata": {}, - "output_type": "display_data" + "output_type": "execute_result" } ], "source": [ - "%bob what's again in this image ?" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "7ef5ae76-29d9-426b-b17b-2723792ef1e7", - "metadata": {}, - "outputs": [], - "source": [ - "def summarize_image_content(image_filename):\n", - " \"\"\"\n", - " Summarizes the content of an image.\n", + "from skimage.io import imread\n", + "from skimage.color import rgb2hsv\n", + "from skimage import morphology, measure\n", + "import numpy as np\n", "\n", - " Args:\n", - " image_filename: The filename of the image.\n", + "# Load the image\n", + "#image = imread('path_to_your_image')\n", "\n", - " Returns:\n", - " A string summarizing the content of the image.\n", - " \"\"\"\n", + "# Convert RGB image to HSV\n", + "hsv_image = rgb2hsv(image)\n", "\n", - " # Load the image\n", - " image = imread(image_filename)\n", + "# Isolate the blue channel in HSV. Typically, blue has a hue range of around 0.5 to 0.7\n", + "blue_mask = np.logical_and(hsv_image[:, :, 0] >= 0.5, hsv_image[:, :, 0] <= 0.7)\n", "\n", - " # Get the dimensions of the image\n", - " height, width, channels = image.shape\n", + "# Apply morphological operations to clean segmentation\n", + "blue_mask = morphology.remove_small_objects(blue_mask, min_size=50)\n", + "blue_mask = morphology.remove_small_holes(blue_mask, area_threshold=64)\n", + "blue_mask = morphology.dilation(blue_mask, morphology.disk(3))\n", "\n", - " # Determine the image type\n", - " if channels == 1:\n", - " image_type = \"grayscale\"\n", - " else:\n", - " image_type = \"color\"\n", + "# Label connected components\n", + "labeled_nuclei = measure.label(blue_mask)\n", "\n", - " # Summarize the image content\n", - " summary = f\"The image is {height} pixels high, {width} pixels wide, and is a {image_type} image.\"\n", - "\n", - " return summary\n", - "\n", - "def get_image_content_from_user():\n", - " \"\"\"\n", - " Gets the image content from the user.\n", - "\n", - " Returns:\n", - " A string containing the image content.\n", - " \"\"\"\n", - "\n", - " image_content = input(\"Please describe the image content: \")\n", - "\n", - " return image_content\n", - "\n", - "def main():\n", - " \"\"\"\n", - " The main function.\n", - " \"\"\"\n", - "\n", - " # Get the image content from the user\n", - " image_content = get_image_content_from_user()\n", - "\n", - " # Summarize the image content\n", - " summary = summarize_image_content(image_content)\n", - "\n", - " # Print the summary\n", - " print(summary)\n", - "\n", - "if __name__ == \"__main__\":\n", - " main()" + "# Show the result using stackview\n", + "import stackview\n", + "stackview.insight(labeled_nuclei)" ] }, { "cell_type": "code", "execution_count": null, - "id": "2bb5ff53-9573-4067-a8cc-6308600b2eca", + "id": "d160b8ba-e0cd-4a54-b4e2-e838a98c7a4f", "metadata": {}, "outputs": [], "source": []