diff --git a/.github/workflows/java_master_only.yml b/.github/workflows/java_master_only.yml index 2475321f706..b7f49d14544 100644 --- a/.github/workflows/java_master_only.yml +++ b/.github/workflows/java_master_only.yml @@ -72,13 +72,13 @@ jobs: java-version: '11' java-package: jdk architecture: x64 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} diff --git a/.github/workflows/java_pr.yml b/.github/workflows/java_pr.yml index 0391e6fde9f..3aea4d275e8 100644 --- a/.github/workflows/java_pr.yml +++ b/.github/workflows/java_pr.yml @@ -53,13 +53,13 @@ jobs: java-version: '11' java-package: jdk architecture: x64 - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-ut-maven-${{ hashFiles('**/pom.xml') }} @@ -97,11 +97,11 @@ jobs: python-version: "3.11" architecture: x64 - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' + uses: google-github-actions/auth@v2 with: credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - run: gcloud auth configure-docker --quiet @@ -137,18 +137,18 @@ jobs: with: python-version: '3.11' architecture: 'x64' - - uses: actions/cache@v2 + - uses: actions/cache@v4 with: path: ~/.m2/repository key: ${{ runner.os }}-it-maven-${{ hashFiles('**/pom.xml') }} restore-keys: | ${{ runner.os }}-it-maven- - name: Authenticate to Google Cloud - uses: 'google-github-actions/auth@v1' + uses: google-github-actions/auth@v2 with: credentials_json: '${{ secrets.GCP_SA_KEY }}' - name: Set up gcloud SDK - uses: google-github-actions/setup-gcloud@v1 + uses: google-github-actions/setup-gcloud@v2 with: project_id: ${{ secrets.GCP_PROJECT_ID }} - name: Use gcloud CLI diff --git a/.github/workflows/lint_pr.yml b/.github/workflows/lint_pr.yml index 81732258455..33fafdcd23d 100644 --- a/.github/workflows/lint_pr.yml +++ b/.github/workflows/lint_pr.yml @@ -14,7 +14,7 @@ jobs: name: Validate PR title runs-on: ubuntu-latest steps: - - uses: amannn/action-semantic-pull-request@v4 + - uses: amannn/action-semantic-pull-request@v5 with: # Must use uppercase subjectPattern: ^(?=[A-Z]).+$ diff --git a/.github/workflows/operator_pr.yml b/.github/workflows/operator_pr.yml index e4d371b9454..232ccf7d339 100644 --- a/.github/workflows/operator_pr.yml +++ b/.github/workflows/operator_pr.yml @@ -7,7 +7,7 @@ jobs: steps: - uses: actions/checkout@v4 - name: Install Go - uses: actions/setup-go@v2 + uses: actions/setup-go@v5 with: go-version: 1.21.x - name: Operator tests diff --git a/.github/workflows/pr_local_integration_tests.yml b/.github/workflows/pr_local_integration_tests.yml index e6a9e3e8bde..2825b96f482 100644 --- a/.github/workflows/pr_local_integration_tests.yml +++ b/.github/workflows/pr_local_integration_tests.yml @@ -45,7 +45,7 @@ jobs: - name: Get uv cache dir id: uv-cache run: | - echo "::set-output name=dir::$(uv cache dir)" + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: diff --git a/.github/workflows/smoke_tests.yml b/.github/workflows/smoke_tests.yml index 9a898dd4c54..a7eb1966269 100644 --- a/.github/workflows/smoke_tests.yml +++ b/.github/workflows/smoke_tests.yml @@ -31,7 +31,7 @@ jobs: - name: Get uv cache dir id: uv-cache run: | - echo "::set-output name=dir::$(uv cache dir)" + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index 6f46d129638..443f40270ff 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -33,8 +33,8 @@ jobs: curl -LsSf https://astral.sh/uv/install.sh | sh - name: Get uv cache dir id: uv-cache - run: | - echo "::set-output name=dir::$(uv cache dir)" + run: | + echo "dir=$(uv cache dir)" >> $GITHUB_OUTPUT - name: uv cache uses: actions/cache@v4 with: @@ -52,7 +52,7 @@ jobs: NPM_TOKEN: ${{ secrets.NPM_TOKEN }} steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version-file: './ui/.nvmrc' registry-url: 'https://registry.npmjs.org' diff --git a/Makefile b/Makefile index d7374d347c8..de2ee568b68 100644 --- a/Makefile +++ b/Makefile @@ -96,14 +96,14 @@ test-python-unit: python -m pytest -n 8 --color=yes sdk/python/tests test-python-integration: - python -m pytest -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ + python -m pytest --tb=short -v -n 8 --integration --color=yes --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "(not snowflake or not test_historical_features_main)" \ sdk/python/tests test-python-integration-local: FEAST_IS_LOCAL_TEST=True \ FEAST_LOCAL_ONLINE_CONTAINER=True \ - python -m pytest -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ + python -m pytest --tb=short -v -n 8 --color=yes --integration --durations=10 --timeout=1200 --timeout_method=thread --dist loadgroup \ -k "not test_lambda_materialization and not test_snowflake_materialization" \ sdk/python/tests diff --git a/docs/README.md b/docs/README.md index 5e36e1ce40a..36c83ed177a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -42,12 +42,15 @@ serving system must make a request to the feature store to retrieve feature valu ## Who is Feast for? -Feast helps ML platform/MLOps teams with DevOps experience productionize real-time models. Feast also helps these teams -build a feature platform that improves collaboration between data engineers, software engineers, machine learning -engineers, and data scientists. +Feast helps ML platform/MLOps teams with DevOps experience productionize real-time models. Feast also helps these teams build a feature platform that improves collaboration between data engineers, software engineers, machine learning engineers, and data scientists. -Feast is likely **not** the right tool if you -* are in an organization that’s just getting started with ML and is not yet sure what the business impact of ML is +* *For Data Scientists*: Feast is a a tool where you can easily define, store, and retrieve your features for both model development and model deployment. By using Feast, you can focus on what you do best: build features that power your AI/ML models and maximize the value of your data. + +* *For MLOps Engineers*: Feast is a library that allows you to connect your existing infrastructure (e.g., online database, application server, microservice, analytical database, and orchestration tooling) that enables your Data Scientists to ship features for their models to production using a friendly SDK without having to be concerned with software engineering challenges that occur from serving real-time production systems. By using Feast, you can focus on maintaining a resilient system, instead of implementing features for Data Scientists. + +* *For Data Engineers*: Feast provides a centralized catalog for storing feature definitions allowing one to maintain a single source of truth for feature data. It provides the abstraction for reading and writing to many different types of offline and online data stores. Using either the provided python SDK or the feature server service, users can write data to the online and/or offline stores and then read that data out again in either low-latency online scenarios for model inference, or in batch scenarios for model training. + +* *For AI Engineers*: Feast provides a platform designed to scale your AI applications by enabling seamless integration of richer data and facilitating fine-tuning. With Feast, you can optimize the performance of your AI models while ensuring a scalable and efficient data pipeline. ## What Feast is not? diff --git a/docs/SUMMARY.md b/docs/SUMMARY.md index 7aad7a94428..e24e15fb5cb 100644 --- a/docs/SUMMARY.md +++ b/docs/SUMMARY.md @@ -32,6 +32,7 @@ * [Registry](getting-started/components/registry.md) * [Offline store](getting-started/components/offline-store.md) * [Online store](getting-started/components/online-store.md) + * [Feature server](getting-started/components/feature-server.md) * [Batch Materialization Engine](getting-started/components/batch-materialization-engine.md) * [Provider](getting-started/components/provider.md) * [Authorization Manager](getting-started/components/authz_manager.md) diff --git a/docs/getting-started/components/README.md b/docs/getting-started/components/README.md index e1c000abced..4c6f3a54dfc 100644 --- a/docs/getting-started/components/README.md +++ b/docs/getting-started/components/README.md @@ -12,6 +12,10 @@ [online-store.md](online-store.md) {% endcontent-ref %} +{% content-ref url="feature-server.md" %} +[feature-server.md](feature-server.md) +{% endcontent-ref %} + {% content-ref url="batch-materialization-engine.md" %} [batch-materialization-engine.md](batch-materialization-engine.md) {% endcontent-ref %} diff --git a/docs/getting-started/components/feature-server.md b/docs/getting-started/components/feature-server.md new file mode 100644 index 00000000000..90e6d25e5a2 --- /dev/null +++ b/docs/getting-started/components/feature-server.md @@ -0,0 +1,40 @@ +# Feature Server + +The Feature Server is a core architectural component in Feast, designed to provide low-latency feature retrieval and updates for machine learning applications. + +It is a REST API server built using [FastAPI](https://fastapi.tiangolo.com/) and exposes a limited set of endpoints to serve features, push data, and support materialization operations. The server is scalable, flexible, and designed to work seamlessly with various deployment environments, including local setups and cloud-based systems. + +## Motivation + +In machine learning workflows, real-time access to feature values is critical for enabling low-latency predictions. The Feature Server simplifies this requirement by: + +1. **Serving Features:** Allowing clients to retrieve feature values for specific entities in real-time, reducing the complexity of direct interactions with the online store. +2. **Data Integration:** Providing endpoints to push feature data directly into the online or offline store, ensuring data freshness and consistency. +3. **Scalability:** Supporting horizontal scaling to handle high request volumes efficiently. +4. **Standardized API:** Exposing HTTP/JSON endpoints that integrate seamlessly with various programming languages and ML pipelines. +5. **Secure Communication:** Supporting TLS (SSL) for secure data transmission in production environments. + +## Architecture + +The Feature Server operates as a stateless service backed by two key components: + +- **[Online Store](./online-store.md):** The primary data store used for low-latency feature retrieval. +- **[Registry](./registry.md):** The metadata store that defines feature sets, feature views, and their relationships to entities. + +## Key Features + +1. **RESTful API:** Provides standardized endpoints for feature retrieval and data pushing. +2. **CLI Integration:** Easily managed through the Feast CLI with commands like `feast serve`. +3. **Flexible Deployment:** Can be deployed locally, via Docker, or on Kubernetes using Helm charts. +4. **Scalability:** Designed for distributed deployments to handle large-scale workloads. +5. **TLS Support:** Ensures secure communication in production setups. + +## Endpoints Overview + +| Endpoint | Description | +| -------------------------- | ----------------------------------------------------------------------- | +| `/get-online-features` | Retrieves feature values for specified entities and feature references. | +| `/push` | Pushes feature data to the online and/or offline store. | +| `/materialize` | Materializes features within a specific time range to the online store. | +| `/materialize-incremental` | Incrementally materializes features up to the current timestamp. | + diff --git a/docs/getting-started/components/overview.md b/docs/getting-started/components/overview.md index ac0b99de8ab..05c7503d842 100644 --- a/docs/getting-started/components/overview.md +++ b/docs/getting-started/components/overview.md @@ -13,6 +13,7 @@ * **Deploy Model:** The trained model binary (and list of features) are deployed into a model serving system. This step is not executed by Feast. * **Prediction:** A backend system makes a request for a prediction from the model serving service. * **Get Online Features:** The model serving service makes a request to the Feast Online Serving service for online features using a Feast SDK. +* **Feature Retrieval:** The online serving service retrieves the latest feature values from the online store and returns them to the model serving service. ## Components @@ -24,6 +25,7 @@ A complete Feast deployment contains the following components: * Materialize (load) feature values into the online store. * Build and retrieve training datasets from the offline store. * Retrieve online features. +* **Feature Server:** The Feature Server is a REST API server that serves feature values for a given entity key and feature reference. The Feature Server is designed to be horizontally scalable and can be deployed in a distributed manner. * **Stream Processor:** The Stream Processor can be used to ingest feature data from streams and write it into the online or offline stores. Currently, there's an experimental Spark processor that's able to consume data from Kafka. * **Batch Materialization Engine:** The [Batch Materialization Engine](batch-materialization-engine.md) component launches a process which loads data into the online store from the offline store. By default, Feast uses a local in-process engine implementation to materialize data. However, additional infrastructure can be used for a more scalable materialization process. * **Online Store:** The online store is a database that stores only the latest feature values for each entity. The online store is either populated through materialization jobs or through [stream ingestion](../../reference/data-sources/push.md). diff --git a/docs/getting-started/quickstart.md b/docs/getting-started/quickstart.md index d35446ce7f0..a83897005fd 100644 --- a/docs/getting-started/quickstart.md +++ b/docs/getting-started/quickstart.md @@ -10,6 +10,9 @@ Feast (Feature Store) is an open-source feature store designed to facilitate the * *For Data Engineers*: Feast provides a centralized catalog for storing feature definitions allowing one to maintain a single source of truth for feature data. It provides the abstraction for reading and writing to many different types of offline and online data stores. Using either the provided python SDK or the feature server service, users can write data to the online and/or offline stores and then read that data out again in either low-latency online scenarios for model inference, or in batch scenarios for model training. +* *For AI Engineers*: Feast provides a platform designed to scale your AI applications by enabling seamless integration of richer data and facilitating fine-tuning. With Feast, you can optimize the performance of your AI models while ensuring a scalable and efficient data pipeline. + + For more info refer to [Introduction to feast](../README.md) ## Prerequisites diff --git a/docs/how-to-guides/starting-feast-servers-tls-mode.md b/docs/how-to-guides/starting-feast-servers-tls-mode.md index e1ddbc08be5..a868e17cf96 100644 --- a/docs/how-to-guides/starting-feast-servers-tls-mode.md +++ b/docs/how-to-guides/starting-feast-servers-tls-mode.md @@ -189,3 +189,8 @@ INFO: Waiting for application startup. INFO: Application startup complete. INFO: Uvicorn running on https://0.0.0.0:8888 (Press CTRL+C to quit) ``` + + +## Adding public key to CA trust store and configuring the feast to use the trust store. +You can pass the public key for SSL verification using the `cert` parameter, however, it is sometimes difficult to maintain individual certificates and pass them individually. +The alternative recommendation is to add the public certificate to CA trust store and set the path as an environment variable (e.g., `FEAST_CA_CERT_FILE_PATH`). Feast will use the trust store path in the `FEAST_CA_CERT_FILE_PATH` environment variable. \ No newline at end of file diff --git a/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb b/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb new file mode 100644 index 00000000000..a345ec8ca46 --- /dev/null +++ b/examples/credit-risk-end-to-end/01_Credit_Risk_Data_Prep.ipynb @@ -0,0 +1,757 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "a52c80c4-1ea2-4d1e-b582-fac51081e76d", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "576a8e30-fe4c-4eda-bc56-9edd7fde3385", + "metadata": {}, + "source": [ + "# Credit Risk Data Preparation" + ] + }, + { + "cell_type": "markdown", + "id": "1f3fbd5a-1587-4b4e-9263-a57490657337", + "metadata": {}, + "source": [ + "Predicting credit risk is an important task for financial institutions. If a bank can accurately determine the probability that a borrower will pay back a future loan, then they can make better decisions on loan terms and approvals. Getting credit risk right is critical to offering good financial services, and getting credit risk wrong could mean going out of business.\n", + "\n", + "AI models have played a central role in modern credit risk assessment systems. In this example, we develop a credit risk model to predict whether a future loan will be good or bad, given some context data (presumably supplied from the loan application). We use the modeling process to demonstrate how Feast can be used to facilitate the serving of data for training and inference use-cases.\n", + "\n", + "In this notebook, we prepare the data." + ] + }, + { + "cell_type": "markdown", + "id": "4d05715f-ddb8-42de-8f0c-212dcbad9e0e", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "6fba29f9-db1f-4ceb-b066-5b2df2c95d33", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8a897b19-6f82-4631-ae51-8a23182ff267", + "metadata": {}, + "outputs": [], + "source": [ + "# Import Python libraries\n", + "import os\n", + "import warnings\n", + "import datetime as dt\n", + "import pandas as pd\n", + "import numpy as np\n", + "from sklearn.datasets import fetch_openml" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b944ed48-54b3-43fa-8373-ce788d7e71af", + "metadata": {}, + "outputs": [], + "source": [ + "# suppress warning messages for example flow (don't run if you want to see warnings)\n", + "warnings.filterwarnings('ignore')" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "70788c73-144f-4ecf-b370-c5669c538d93", + "metadata": {}, + "outputs": [], + "source": [ + "# Seed for reproducibility\n", + "SEED = 142" + ] + }, + { + "cell_type": "markdown", + "id": "cfb4dfd0-f583-4aa0-bd39-3ff9fbb80db0", + "metadata": {}, + "source": [ + "### Pull the Data" + ] + }, + { + "cell_type": "markdown", + "id": "3c206dfc-d551-4002-ae63-ccbb981768fa", + "metadata": {}, + "source": [ + "The data we will use to train the model is from the [OpenML](https://www.openml.org/) dataset [credit-g](https://www.openml.org/search?type=data&sort=runs&status=active&id=31), obtained from a 1994 German study. More details on the data can be found in the `DESC` attribute and `details` map (see below)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "31a9e964-bdb3-4ae4-b2b4-64bbe0ab93a3", + "metadata": {}, + "outputs": [], + "source": [ + "data = fetch_openml(name=\"credit-g\", version=1, parser='auto')" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "58dbf7c2-f40b-4965-baac-6903a27ef622", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "**Author**: Dr. Hans Hofmann \n", + "**Source**: [UCI](https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)) - 1994 \n", + "**Please cite**: [UCI](https://archive.ics.uci.edu/ml/citation_policy.html)\n", + "\n", + "**German Credit dataset** \n", + "This dataset classifies people described by a set of attributes as good or bad credit risks.\n", + "\n", + "This dataset comes with a cost matrix: \n", + "``` \n", + "Good Bad (predicted) \n", + "Good 0 1 (actual) \n", + "Bad 5 0 \n", + "```\n", + "\n", + "It is worse to class a customer as good when they are bad (5), than it is to class a customer as bad when they are good (1). \n", + "\n", + "### Attribute description \n", + "\n", + "1. Status of existing checking account, in Deutsche Mark. \n", + "2. Duration in months \n", + "3. Credit history (credits taken, paid back duly, delays, critical accounts) \n", + "4. Purpose of the credit (car, television,...) \n", + "5. Credit amount \n", + "6. Status of savings account/bonds, in Deutsche Mark. \n", + "7. Present employment, in number of years. \n", + "8. Installment rate in percentage of disposable income \n", + "9. Personal status (married, single,...) and sex \n", + "10. Other debtors / guarantors \n", + "11. Present residence since X years \n", + "12. Property (e.g. real estate) \n", + "13. Age in years \n", + "14. Other installment plans (banks, stores) \n", + "15. Housing (rent, own,...) \n", + "16. Number of existing credits at this bank \n", + "17. Job \n", + "18. Number of people being liable to provide maintenance for \n", + "19. Telephone (yes,no) \n", + "20. Foreign worker (yes,no)\n", + "\n", + "Downloaded from openml.org.\n" + ] + } + ], + "source": [ + "print(data.DESCR)" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "53de57ec-0fb6-4b51-9c27-696b059a1847", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Original data url: https://archive.ics.uci.edu/ml/datasets/statlog+(german+credit+data)\n", + "Paper url: https://dl.acm.org/doi/abs/10.1145/967900.968104\n" + ] + } + ], + "source": [ + "print(\"Original data url: \".ljust(20), data.details[\"original_data_url\"])\n", + "print(\"Paper url: \".ljust(20), data.details[\"paper_url\"])" + ] + }, + { + "cell_type": "markdown", + "id": "6b2c2514-484e-46cb-aedc-89a301266f44", + "metadata": {}, + "source": [ + "### High-Level Data Inspection" + ] + }, + { + "cell_type": "markdown", + "id": "a76af306-caba-403d-a9cb-b5de12573075", + "metadata": {}, + "source": [ + "Let's inspect the data to see high level details like data types and size. We also want to make sure there are no glaring issues (like a large number of null values)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "20fb82c4-ed8d-42f8-b386-c7ebdc9bf786", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 1000 entries, 0 to 999\n", + "Data columns (total 21 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 checking_status 1000 non-null category\n", + " 1 duration 1000 non-null int64 \n", + " 2 credit_history 1000 non-null category\n", + " 3 purpose 1000 non-null category\n", + " 4 credit_amount 1000 non-null int64 \n", + " 5 savings_status 1000 non-null category\n", + " 6 employment 1000 non-null category\n", + " 7 installment_commitment 1000 non-null int64 \n", + " 8 personal_status 1000 non-null category\n", + " 9 other_parties 1000 non-null category\n", + " 10 residence_since 1000 non-null int64 \n", + " 11 property_magnitude 1000 non-null category\n", + " 12 age 1000 non-null int64 \n", + " 13 other_payment_plans 1000 non-null category\n", + " 14 housing 1000 non-null category\n", + " 15 existing_credits 1000 non-null int64 \n", + " 16 job 1000 non-null category\n", + " 17 num_dependents 1000 non-null int64 \n", + " 18 own_telephone 1000 non-null category\n", + " 19 foreign_worker 1000 non-null category\n", + " 20 class 1000 non-null category\n", + "dtypes: category(14), int64(7)\n", + "memory usage: 71.0 KB\n" + ] + } + ], + "source": [ + "df = data.frame\n", + "df.info()" + ] + }, + { + "cell_type": "markdown", + "id": "a384932a-40df-45f6-bfbc-a9cf6c708f1b", + "metadata": {}, + "source": [ + "We see that there are 21 columns, each with 1000 non-null values. The first 20 columns are contextual fields with `Dtype` of `category` or `int64`, while the last field is actually the target variable, `class`, which we wish to predict. \n", + "\n", + "From the description (above), the `class` tells us whether a loan to a customer was \"good\" or \"bad\". We are anticipating that patterns in the contextual data, as well as their relationship to the class outcomes, can give insight into loan classification. In the following notebooks, we will build a loan classification model that seeks to encode these patterns and relationships in its weights, such that given a new loan application (context data), the model can predict whether the loan (if approved) will be good or bad in the future." + ] + }, + { + "cell_type": "markdown", + "id": "a451c9a3-0390-4d5a-b687-c59f52445eb1", + "metadata": {}, + "source": [ + "### Data Preparation For Demonstrating Feast" + ] + }, + { + "cell_type": "markdown", + "id": "dc4e7653-b118-44c3-ade3-f1b217b112fc", + "metadata": {}, + "source": [ + "At this point, it's important to bring up that Feast was developed primarily to work with production data. Feast requires datasets to have entities (in our case, IDs) and timestamps, which it uses in joins. Feast can support joining data on multiple entities (like primary keys in SQL), as well as \"created\" timestamps and \"event\" timestamps. However, in this example, we'll keep things more simple.\n", + "\n", + "In a real loan application scenario, the application fields (in a database) would be associated with a timestamp, while the actual loan outcome (label) would be determined much later and recorded separately with a different timestamp.\n", + "\n", + "In order to demonstrate Feast capabilities, such as point-in-time joins, we will mock IDs and timestamps for this data. For IDs, we will use the original dataframe index values. For the timestamps, we will generate random values between \"Tue Sep 24 12:00:00 2023\" and \"Wed Oct 9 12:00:00 2023\"." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "9d6ec4f6-9410-4858-a440-45dccaa0896b", + "metadata": {}, + "outputs": [], + "source": [ + "# Make index into \"ID\" column\n", + "df = df.reset_index(names=[\"ID\"])" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "055f2cb7-3abf-4d01-be60-e4c7b8ad1988", + "metadata": {}, + "outputs": [], + "source": [ + "# Add mock timestamps\n", + "time_format = \"%a %b %d %H:%M:%S %Y\"\n", + "date = dt.datetime.strptime(\"Wed Oct 9 12:00:00 2023\", time_format)\n", + "end = int(date.timestamp())\n", + "start = int((date - dt.timedelta(days=15)).timestamp()) # 'Tue Sep 24 12:00:00 2023'\n", + "\n", + "def make_tstamp(date):\n", + " dtime = dt.datetime.fromtimestamp(date).ctime()\n", + " return dtime\n", + " \n", + "# (seed set for reproducibility)\n", + "np.random.seed(SEED)\n", + "df[\"application_timestamp\"] = pd.to_datetime([\n", + " make_tstamp(d) for d in np.random.randint(start, end, len(df))\n", + "])" + ] + }, + { + "cell_type": "markdown", + "id": "f7800ea9-de9a-4aab-9d77-c4276e7db5f9", + "metadata": {}, + "source": [ + "Verify that the newly created \"ID\" and \"application_timestamp\" fields were added to the data as expected." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "9516fc5c-7c25-4e60-acba-7400ab6bab42", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
012
ID012
checking_status<00<=X<200no checking
duration64812
credit_historycritical/other existing creditexisting paidcritical/other existing credit
purposeradio/tvradio/tveducation
credit_amount116959512096
savings_statusno known savings<100<100
employment>=71<=X<44<=X<7
installment_commitment422
personal_statusmale singlefemale div/dep/marmale single
other_partiesnonenonenone
residence_since423
property_magnitudereal estatereal estatereal estate
age672249
other_payment_plansnonenonenone
housingownownown
existing_credits211
jobskilledskilledunskilled resident
num_dependents112
own_telephoneyesnonenone
foreign_workeryesyesyes
classgoodbadgood
application_timestamp2023-10-04 17:50:132023-09-28 18:10:132023-10-03 23:06:03
\n", + "
" + ], + "text/plain": [ + " 0 1 \\\n", + "ID 0 1 \n", + "checking_status <0 0<=X<200 \n", + "duration 6 48 \n", + "credit_history critical/other existing credit existing paid \n", + "purpose radio/tv radio/tv \n", + "credit_amount 1169 5951 \n", + "savings_status no known savings <100 \n", + "employment >=7 1<=X<4 \n", + "installment_commitment 4 2 \n", + "personal_status male single female div/dep/mar \n", + "other_parties none none \n", + "residence_since 4 2 \n", + "property_magnitude real estate real estate \n", + "age 67 22 \n", + "other_payment_plans none none \n", + "housing own own \n", + "existing_credits 2 1 \n", + "job skilled skilled \n", + "num_dependents 1 1 \n", + "own_telephone yes none \n", + "foreign_worker yes yes \n", + "class good bad \n", + "application_timestamp 2023-10-04 17:50:13 2023-09-28 18:10:13 \n", + "\n", + " 2 \n", + "ID 2 \n", + "checking_status no checking \n", + "duration 12 \n", + "credit_history critical/other existing credit \n", + "purpose education \n", + "credit_amount 2096 \n", + "savings_status <100 \n", + "employment 4<=X<7 \n", + "installment_commitment 2 \n", + "personal_status male single \n", + "other_parties none \n", + "residence_since 3 \n", + "property_magnitude real estate \n", + "age 49 \n", + "other_payment_plans none \n", + "housing own \n", + "existing_credits 1 \n", + "job unskilled resident \n", + "num_dependents 2 \n", + "own_telephone none \n", + "foreign_worker yes \n", + "class good \n", + "application_timestamp 2023-10-03 23:06:03 " + ] + }, + "execution_count": 10, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check data (first few records, transposed for readability)\n", + "df.head(3).T" + ] + }, + { + "cell_type": "markdown", + "id": "72b2105a-b459-4715-aa53-6fe69fc4a210", + "metadata": {}, + "source": [ + "We'll also generate counterpart IDs and timestamps on the label data. In a real-life scenario, the label data would come separate and later relative to the loan application data. To mimic this, let's create a labels dataset with an \"outcome_timestamp\" column with a variable lag from the application timestamp of 30 to 90 days." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "e214478b-ed9b-4354-ba6f-4117813c56c3", + "metadata": {}, + "outputs": [], + "source": [ + "# Add (lagged) label timestamps (30 to 90 days)\n", + "def lag_delta(data, seed):\n", + " np.random.seed(seed)\n", + " delta_days = np.random.randint(30, 90, len(data))\n", + " delta_hours = np.random.randint(0, 24, len(data))\n", + " delta = np.array([dt.timedelta(days=int(delta_days[i]), hours=int(delta_hours[i])) for i in range(len(data))])\n", + " return delta\n", + "\n", + "labels = df[[\"ID\", \"class\"]]\n", + "labels[\"outcome_timestamp\"] = pd.to_datetime(df.application_timestamp + lag_delta(df, SEED))" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "356a7225-db20-4c15-87a3-4a0eb3127475", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestamp
00good2023-11-24 22:50:13
11bad2023-11-03 12:10:13
22good2023-11-30 22:06:03
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp\n", + "0 0 good 2023-11-24 22:50:13\n", + "1 1 bad 2023-11-03 12:10:13\n", + "2 2 good 2023-11-30 22:06:03" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check labels\n", + "labels.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "4a29f754-f758-402b-ac42-2dcfcee3b7fc", + "metadata": {}, + "source": [ + "You can verify that the `outcome timestamp` has a difference of 30 to 90 days from the \"application_timestamp\" (above)." + ] + }, + { + "cell_type": "markdown", + "id": "e720ce24-e092-4fcd-be3e-68bb18f4d2a7", + "metadata": {}, + "source": [ + "### Save Data" + ] + }, + { + "cell_type": "markdown", + "id": "5cae0578-8431-46c7-8d64-e52146f47d46", + "metadata": {}, + "source": [ + "Now that we have our data prepared, let's save it to local parquet files in the `data` directory (parquet is one of the file formats supported by Feast).\n", + "\n", + "One more step we will add is splitting the context data column-wise and saving it in two files. This step is contrived--we don't usually split data when we don't need to--but it will allow us to demonstrate later how Feast can easily join datasets (a common need in Data Science projects)." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "cebef56c-1f54-4d31-a545-75d708d38579", + "metadata": {}, + "outputs": [], + "source": [ + "# Create the data directory if it doesn't exist\n", + "os.makedirs(\"Feature_Store/data\", exist_ok=True)\n", + "\n", + "# Split columns and save context data\n", + "a_cols = [\n", + " 'ID', 'checking_status', 'duration', 'credit_history', 'purpose',\n", + " 'credit_amount', 'savings_status', 'employment', 'application_timestamp',\n", + " 'installment_commitment', 'personal_status', 'other_parties',\n", + "]\n", + "b_cols = [\n", + " 'ID', 'residence_since', 'property_magnitude', 'age', 'other_payment_plans',\n", + " 'housing', 'existing_credits', 'job', 'num_dependents', 'own_telephone',\n", + " 'foreign_worker', 'application_timestamp'\n", + "]\n", + "\n", + "df[a_cols].to_parquet(\"Feature_Store/data/data_a.parquet\", engine=\"pyarrow\")\n", + "df[b_cols].to_parquet(\"Feature_Store/data/data_b.parquet\", engine=\"pyarrow\")\n", + "\n", + "# Save label data\n", + "labels.to_parquet(\"Feature_Store/data/labels.parquet\", engine=\"pyarrow\")" + ] + }, + { + "cell_type": "markdown", + "id": "d8d5de9f-bd27-4e95-802c-b121743dd1b0", + "metadata": {}, + "source": [ + "We have saved the following files to the `Feature_Store/data` directory: \n", + "- `data_a.parquet` (training data, a columns)\n", + "- `data_b.parquet` (training data, b columns)\n", + "- `labels.parquet` (label outcomes)" + ] + }, + { + "cell_type": "markdown", + "id": "af6355dc-ff5b-4b3f-b0bd-3c4020ef67e8", + "metadata": {}, + "source": [ + "With the feature data prepared, we are ready to setup and deploy the feature store. \n", + "\n", + "Continue with the [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) notebook." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb b/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb new file mode 100644 index 00000000000..f736cdaed93 --- /dev/null +++ b/examples/credit-risk-end-to-end/02_Deploying_the_Feature_Store.ipynb @@ -0,0 +1,801 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "08d9e060-d455-43e2-b1ec-51e2a53e3169", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "93095241-3886-44a2-83b1-2a9537c21bc8", + "metadata": {}, + "source": [ + "# Deploying the Feature Store" + ] + }, + { + "cell_type": "markdown", + "id": "465783da-18eb-4945-98e7-bb1058a7af1b", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "11961d1b-72db-48dc-a07d-dcea9ba223b4", + "metadata": {}, + "source": [ + "Feast enables AI/ML teams to serve (and consume) features via feature stores. In this notebook, we will configure the feature stores and feature definitions, and deploy a Feast feature store server. We will also materialize (move) data from the offline store to the online store.\n", + "\n", + "In Feast, offline stores support pulling large amounts of data for model training using tools like Redshift, Snowflake, Bigquery, and Spark. In contrast, the focus of Feast online stores is feature serving in support of model inference, using tools like Redis, Snowflake, PostgreSQL, and SQLite.\n", + "\n", + "In this notebook, we will setup a file-based (Dask) offline store and SQLite online store. The online store will be made available through the Feast server." + ] + }, + { + "cell_type": "markdown", + "id": "dfed8ccf-0d7d-46a1-82f0-5765f8796088", + "metadata": {}, + "source": [ + "This notebook assumes that you have prepared the data by running the notebook [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb). " + ] + }, + { + "cell_type": "markdown", + "id": "e66b7a08-5d15-4804-a82a-8bc571777496", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "1c1e87a4-900b-48f3-a400-ce6608046ce3", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "8bd21689-4a8e-4b0c-937d-0911df9db1d3", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import re\n", + "import sys\n", + "import time\n", + "import signal\n", + "import sqlite3\n", + "import subprocess\n", + "import datetime as dt\n", + "from feast import FeatureStore" + ] + }, + { + "cell_type": "markdown", + "id": "471db4b0-ea93-47a1-9d55-a80e4d2bdc1e", + "metadata": {}, + "source": [ + "### Feast Feature Store Configuration" + ] + }, + { + "cell_type": "markdown", + "id": "0a307490-4121-4bf3-a5c4-77a8885a4f6a", + "metadata": {}, + "source": [ + "For model training, we usually don't need (or want) a constantly running feature server. All we need is the ability to efficiently query and pull all of the training data at training time. In contrast, during model serving we need servers that are always ready to supply feature records in response to application requests. \n", + "\n", + "This training-serving dichotomy is reflected in Feast using \"offline\" and \"online\" stores. Offline stores are configured to work with database technologies typically used for training, while online stores are configured to use storage and streaming technologies that are popular for feature serving.\n", + "\n", + "We need to create a `feature_store.yaml` config file to tell feast the structure we want in our offline and online feature stores. Below, we write the configuration for a local \"Dask\" offline store and local SQLite online store. We give the feature store a project name of `loan_applications`, and provider `local`. The registry is where the feature store will keep track of feature definitions and online store updates; we choose a file location in this case.\n", + "\n", + "See the [feature_store.yaml](https://docs.feast.dev/reference/feature-repository/feature-store-yaml) documentation for further details. " + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "b3757221-2037-49eb-867f-b9529fec06e2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing Feature_Store/feature_store.yaml\n" + ] + } + ], + "source": [ + "%%writefile Feature_Store/feature_store.yaml\n", + "\n", + "project: loan_applications\n", + "registry: data/registry.db\n", + "provider: local\n", + "offline_store:\n", + " type: dask\n", + "online_store:\n", + " type: sqlite\n", + " path: data/online_store.db\n", + "entity_key_serialization_version: 2" + ] + }, + { + "cell_type": "markdown", + "id": "180038f3-e5ce-4cce-bdf0-118eee7a822d", + "metadata": {}, + "source": [ + "### Feature Definitions" + ] + }, + { + "cell_type": "markdown", + "id": "dd44b206-1f5c-4f55-bbab-41ba2d3f5202", + "metadata": {}, + "source": [ + "We also need to create feature definitions and other feature constructs in a python file, which we name `feature_definitions.py`. For our purposes, we define the following:\n", + "\n", + "- Data Source: connections to data storage or data-producing endpoints\n", + "- Entity: primary key fields which can be used for joining data\n", + "- FeatureView: collections of features from a data source\n", + "\n", + "For more information on these, see the [Concepts](https://docs.feast.dev/getting-started/concepts) section of the Feast documentation." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "d3e8fd80-0bee-463c-b3fb-bd0d1ee83a9c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Writing Feature_Store/feature_definitions.py\n" + ] + } + ], + "source": [ + "%%writefile Feature_Store/feature_definitions.py\n", + "\n", + "# Imports\n", + "import os\n", + "from pathlib import Path\n", + "from feast import (\n", + " FileSource,\n", + " Entity,\n", + " FeatureView,\n", + " Field,\n", + " FeatureService\n", + ")\n", + "from feast.types import Float32, String\n", + "from feast.data_format import ParquetFormat\n", + "\n", + "CURRENT_DIR = os.path.abspath(os.curdir)\n", + "\n", + "# Data Sources\n", + "# A data source tells Feast where the data lives\n", + "data_a = FileSource(\n", + " file_format=ParquetFormat(),\n", + " path=Path(CURRENT_DIR,\"data/data_a.parquet\").as_uri()\n", + ")\n", + "data_b = FileSource(\n", + " file_format=ParquetFormat(),\n", + " path=Path(CURRENT_DIR,\"data/data_b.parquet\").as_uri()\n", + ")\n", + "\n", + "# Entity\n", + "# An entity tells Feast the column it can use to join tables\n", + "loan_id = Entity(\n", + " name = \"loan_id\",\n", + " join_keys = [\"ID\"]\n", + ")\n", + "\n", + "# Feature views\n", + "# A feature view is how Feast groups features\n", + "features_a = FeatureView(\n", + " name=\"data_a\",\n", + " entities=[loan_id],\n", + " schema=[\n", + " Field(name=\"checking_status\", dtype=String),\n", + " Field(name=\"duration\", dtype=Float32),\n", + " Field(name=\"credit_history\", dtype=String),\n", + " Field(name=\"purpose\", dtype=String),\n", + " Field(name=\"credit_amount\", dtype=Float32),\n", + " Field(name=\"savings_status\", dtype=String),\n", + " Field(name=\"employment\", dtype=String),\n", + " Field(name=\"installment_commitment\", dtype=Float32),\n", + " Field(name=\"personal_status\", dtype=String),\n", + " Field(name=\"other_parties\", dtype=String),\n", + " ],\n", + " source=data_a\n", + ")\n", + "features_b = FeatureView(\n", + " name=\"data_b\",\n", + " entities=[loan_id],\n", + " schema=[\n", + " Field(name=\"residence_since\", dtype=Float32),\n", + " Field(name=\"property_magnitude\", dtype=String),\n", + " Field(name=\"age\", dtype=Float32),\n", + " Field(name=\"other_payment_plans\", dtype=String),\n", + " Field(name=\"housing\", dtype=String),\n", + " Field(name=\"existing_credits\", dtype=Float32),\n", + " Field(name=\"job\", dtype=String),\n", + " Field(name=\"num_dependents\", dtype=Float32),\n", + " Field(name=\"own_telephone\", dtype=String),\n", + " Field(name=\"foreign_worker\", dtype=String),\n", + " ],\n", + " source=data_b\n", + ")\n", + "\n", + "# Feature Service\n", + "# a feature service in Feast represents a logical group of features\n", + "loan_fs = FeatureService(\n", + " name=\"loan_fs\",\n", + " features=[features_a, features_b]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "4b47c1b5-849e-43f3-8043-60466aaed69f", + "metadata": {}, + "outputs": [], + "source": [] + }, + { + "cell_type": "markdown", + "id": "be9723eb-8fa0-4338-b50c-f9f1ff6bb13a", + "metadata": {}, + "source": [ + "### Applying the Configuration and Definitions" + ] + }, + { + "cell_type": "markdown", + "id": "c796d45f-28c0-4875-bbb1-71e5a15dcb96", + "metadata": {}, + "source": [ + "Now that we have our feature store configuration (`feature_store.yaml`) and feature definitions (`feature_definitions.py`), we are ready to \"apply\" them. The `feast apply` command creates a registry file (`Feature_Store/data/registry.db`) and sets up data connections; in this case, it creates a SQLite database (`Feature_Store/data/online_store.db`)." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "394467f3-4ced-492a-9379-105aea9d4a6d", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "Created entity \u001b[1m\u001b[32mloan_id\u001b[0m\n", + "Created feature view \u001b[1m\u001b[32mdata_a\u001b[0m\n", + "Created feature view \u001b[1m\u001b[32mdata_b\u001b[0m\n", + "Created feature service \u001b[1m\u001b[32mloan_fs\u001b[0m\n", + "\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "10/27/2024 02:19:03 PM root WARNING: Cannot use sqlite_vec for vector search\n", + "Created sqlite table \u001b[1m\u001b[32mloan_applications_data_a\u001b[0m\n", + "Created sqlite table \u001b[1m\u001b[32mloan_applications_data_b\u001b[0m\n", + "\n" + ] + } + ], + "source": [ + "# Run 'feast apply' in the Feature_Store directory\n", + "!feast --chdir ./Feature_Store apply" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "e32f40eb-a31a-4877-8f40-2d8515302f39", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "total 232\n", + "-rw-r--r-- 1 501 20 33K Oct 27 14:17 data_a.parquet\n", + "-rw-r--r-- 1 501 20 27K Oct 27 14:17 data_b.parquet\n", + "-rw-r--r-- 1 501 20 17K Oct 27 14:17 labels.parquet\n", + "-rw-r--r-- 1 501 20 28K Oct 27 14:19 online_store.db\n", + "-rw-r--r-- 1 501 20 2.8K Oct 27 14:19 registry.db\n" + ] + } + ], + "source": [ + "# List the Feature_Store/data/ directory to see newly created files\n", + "!ls -nlh Feature_Store/data/" + ] + }, + { + "cell_type": "markdown", + "id": "31014885-ce6a-4007-8bdb-d74d3b44781b", + "metadata": {}, + "source": [ + "Note that while `feast apply` set up the `sqlite` online database, `online_store.db`, no data has been added to the online database as of yet. We can verify this by connecting with the `sqlite3` library." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "107ca856-af06-40c4-8339-70daf59cdf37", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online Store Tables: [('loan_applications_data_a',), ('loan_applications_data_b',)]\n", + "loan_applications_data_a data: []\n", + "loan_applications_data_b data: []\n" + ] + } + ], + "source": [ + "# Connect to sqlite database\n", + "conn = sqlite3.connect(\"Feature_Store/data/online_store.db\")\n", + "cursor = conn.cursor()\n", + "# Query table data (3 tables)\n", + "print(\n", + " \"Online Store Tables: \",\n", + " cursor.execute(\"SELECT name FROM sqlite_master WHERE type='table';\").fetchall()\n", + ")\n", + "print(\n", + " \"loan_applications_data_a data: \",\n", + " cursor.execute(\"SELECT * FROM loan_applications_data_a\").fetchall()\n", + ")\n", + "print(\n", + " \"loan_applications_data_b data: \",\n", + " cursor.execute(\"SELECT * FROM loan_applications_data_b\").fetchall()\n", + ")\n", + "conn.close()" + ] + }, + { + "cell_type": "markdown", + "id": "03b927ee-7913-4a8a-b17b-9bee361d8d94", + "metadata": {}, + "source": [ + "Since we have used `feast apply` to create the registry, we can now use the Feast Python SDK to interact with our new feature store. To see other possible commands see the [Feast Python SDK documentation](https://rtd.feast.dev/en/master/)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "c764a60a-b911-41a8-ba8f-7ef0a0bc7257", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "RepoConfig(project='loan_applications', provider='local', registry_config='data/registry.db', online_config={'type': 'sqlite', 'path': 'data/online_store.db'}, offline_config={'type': 'dask'}, batch_engine_config='local', feature_server=None, flags=None, repo_path=PosixPath('Feature_Store'), entity_key_serialization_version=2, coerce_tz_aware=True)" + ] + }, + "execution_count": 7, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Get feature store config\n", + "store = FeatureStore(repo_path=\"./Feature_Store\")\n", + "store.config" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "fc572976-6ce9-44f6-8b67-28ee6157e29c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Feature view: data_a | Features: [checking_status-String, duration-Float32, credit_history-String, purpose-String, credit_amount-Float32, savings_status-String, employment-String, installment_commitment-Float32, personal_status-String, other_parties-String]\n", + "Feature view: data_b | Features: [residence_since-Float32, property_magnitude-String, age-Float32, other_payment_plans-String, housing-String, existing_credits-Float32, job-String, num_dependents-Float32, own_telephone-String, foreign_worker-String]\n" + ] + } + ], + "source": [ + "# List feature views\n", + "feature_views = store.list_batch_feature_views()\n", + "for fv in feature_views:\n", + " print(f\"Feature view: {fv.name} | Features: {fv.features}\")" + ] + }, + { + "cell_type": "markdown", + "id": "027edcfe-58d7-4dcb-92e2-5a5514c0f1f0", + "metadata": {}, + "source": [ + "### Deploying the Feature Store Servers" + ] + }, + { + "cell_type": "markdown", + "id": "c9aab68d-395f-421e-ba11-ad8c4acc9d6f", + "metadata": {}, + "source": [ + "If you wish to share a feature store with your team, Feast provides feature servers. To spin up an offline feature server process, we can use the `feast serve_offline` command, while to spin up a Feast online feature server, we use the `feast serve` command.\n", + "\n", + "Let's spin up an offline and an online server that we can use in the subsequent notebooks to get features during model training and model serving. We will run both servers as background processes, that we can communicate with in the other notebooks.\n", + "\n", + "First, we write a helper function to extract the first few printed log lines (so we can print it in the notebook cell output)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "568f81b8-df34-4b06-8a3f-1a6bdc2e6cff", + "metadata": {}, + "outputs": [], + "source": [ + "# TimeoutError class\n", + "class TimeoutError(Exception):\n", + " pass\n", + "\n", + "# TimeoutError raise function\n", + "def timeout():\n", + " raise TimeoutError(\"timeout\")\n", + "\n", + "# Get first few log lines function\n", + "def print_first_proc_lines(proc, wait):\n", + " '''Given a process, `proc`, read and print output lines until they stop \n", + " comming (waiting up to `wait` seconds for new lines to appear)'''\n", + " lines = \"\"\n", + " while True:\n", + " signal.signal(signal.SIGALRM, timeout)\n", + " signal.alarm(wait)\n", + " try:\n", + " lines += proc.stderr.readline()\n", + " except:\n", + " break\n", + " if lines:\n", + " print(lines, file=sys.stderr)" + ] + }, + { + "cell_type": "markdown", + "id": "88d25a87-241a-46c6-9ca7-d035959c5f74", + "metadata": {}, + "source": [ + "Launch the offline server with the command `feast --chdir ./Feature_Store serve_offline`." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "ce965dd4-652b-4c36-a064-fd0fd97d3ef7", + "metadata": {}, + "outputs": [], + "source": [ + "# Feast offline server process\n", + "offline_server_proc = subprocess.Popen(\n", + " \"feast --chdir ./Feature_Store serve_offline 2>&2 & echo $! > server_proc.txt\",\n", + " shell=True,\n", + " text=True,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " bufsize=0\n", + ")\n", + "print_first_proc_lines(offline_server_proc, 2)" + ] + }, + { + "cell_type": "markdown", + "id": "59958d64-8e68-45ff-9549-556cbf46908c", + "metadata": {}, + "source": [ + "The tail end of the command above, `2>&2 & echo $! > server_proc.txt`, captures log messages (in the offline case there are none), and writes the process PID to the file `server_proc.txt` (we will use this in the cleanup notebook, [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb))." + ] + }, + { + "cell_type": "markdown", + "id": "cfed4334-9e62-4f3f-be96-3f7db2f06ada", + "metadata": {}, + "source": [ + "Next, launch the online server with the command `feast --chdir ./Feature_Store serve`." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "a581fbe2-13ba-433e-8e76-dc82cc22af74", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/Users/ddowler/Code/Feast/feast/examples/credit-risk-end-to-end/venv-py3.11/lib/python3.11/site-packages/uvicorn/workers.py:16: DeprecationWarning: The `uvicorn.workers` module is deprecated. Please use `uvicorn-worker` package instead.\n", + "For more details, see https://github.com/Kludex/uvicorn-worker.\n", + " warnings.warn(\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Starting gunicorn 23.0.0\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Listening at: http://127.0.0.1:6566 (44621)\n", + "[2024-10-27 14:19:07 -0600] [44621] [INFO] Using worker: uvicorn.workers.UvicornWorker\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Booting worker with pid: 44623\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Started server process [44623]\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Waiting for application startup.\n", + "[2024-10-27 14:19:07 -0600] [44623] [INFO] Application startup complete.\n", + "\n" + ] + } + ], + "source": [ + "# Feast online server (master and worker) processes\n", + "online_server_proc = subprocess.Popen(\n", + " \"feast --chdir ./Feature_Store serve 2>&2 & echo $! >> server_proc.txt\",\n", + " shell=True,\n", + " text=True,\n", + " stdout=subprocess.PIPE,\n", + " stderr=subprocess.PIPE,\n", + " bufsize=0\n", + ")\n", + "print_first_proc_lines(online_server_proc, 3)" + ] + }, + { + "cell_type": "markdown", + "id": "0e778173-f58a-4074-b63f-107e1f39577b", + "metadata": {}, + "source": [ + "Note that the output helpfully let's us know that the online server is \"Listening at: http://127.0.0.1:6566\" (the default host:port).\n", + "\n", + "List the running processes to verify they are up." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9b1a224d-884d-45c5-9711-2e2eb4351710", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " 501 44594 1 0 2:19PM ?? 0:03.66 **/python **/feast --chdir ./Feature_Store serve_offline\n", + " 501 44621 1 0 2:19PM ?? 0:03.58 **/python **/feast --chdir ./Feature_Store serve\n", + " 501 44623 44621 0 2:19PM ?? 0:00.03 **/python **/feast --chdir ./Feature_Store serve\n", + " 501 44662 44542 0 2:19PM ?? 0:00.01 /bin/zsh -c ps -ef | grep **/feast | grep serve\n" + ] + } + ], + "source": [ + "# List running Feast processes (paths redacted)\n", + "running_procs = !ps -ef | grep feast | grep serve\n", + "\n", + "for line in running_procs:\n", + " redacted = re.sub(r'/*[^\\s]*(?P(python )|(feast ))', r'**/\\g', line)\n", + " print(redacted)" + ] + }, + { + "cell_type": "markdown", + "id": "fd52eeb4-948c-472b-9111-8549fda955a1", + "metadata": {}, + "source": [ + "Note that there are two process for the online server (master and worker)." + ] + }, + { + "cell_type": "markdown", + "id": "8258e7a8-5f6e-4737-93ee-63591518b169", + "metadata": {}, + "source": [ + "### Materialize Features to the Online Store" + ] + }, + { + "cell_type": "markdown", + "id": "21b354ab-ec22-476d-8fd9-6ffe0f3fbacb", + "metadata": {}, + "source": [ + "At this point, there is no data in the online store yet. Let's use the SDK feature store object (that we created above) to \"materialize\" data; this is Feast lingo for moving/updating data from the offline store to the online store." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "ff6146df-03a7-4ac2-a665-ee5f440c3605", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Materializing \u001b[1m\u001b[32m2\u001b[0m feature views from \u001b[1m\u001b[32m2023-09-24 12:00:00-06:00\u001b[0m to \u001b[1m\u001b[32m2024-01-07 12:00:00-07:00\u001b[0m into the \u001b[1m\u001b[32msqlite\u001b[0m online store.\n", + "\n", + "\u001b[1m\u001b[32mdata_a\u001b[0m:\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + " 0%| | 0/1000 [00:00=7\",\"4<=X<7\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"0<=X<200\",\"no checking\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"existing paid\",\"critical/other existing credit\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"female div/dep/mar\",\"male mar/wid\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[12579.0,2463.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"none\",\"none\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"used car\",\"new car\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[24.0,24.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[4.0,4.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"yes\",\"yes\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[2.0,3.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[1.0,1.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[44.0,27.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"none\",\"none\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"for free\",\"own\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[1.0,2.0],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"high qualif/self emp/mgmt\",\"skilled\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"no known property\",\"life insurance\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]},{\"values\":[\"yes\",\"yes\"],\"statuses\":[\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"2023-09-25T01:03:47Z\",\"2023-09-29T03:17:24Z\"]}]}']" + ] + }, + "execution_count": 16, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "response" + ] + }, + { + "cell_type": "markdown", + "id": "01d20196-1d42-486d-a0bd-97193c953785", + "metadata": {}, + "source": [ + "The `curl` command gave us a quick validation. In the [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb) notebook, we'll use the Python `requests` library to handle the query better." + ] + }, + { + "cell_type": "markdown", + "id": "d74a5117-dd34-4dde-93a8-ea6e8c4c545a", + "metadata": {}, + "source": [ + "Now that the feature stores and their respective servers have been configured and deployed, we can proceed to train an AI model in [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb b/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb new file mode 100644 index 00000000000..ca0d0e29d95 --- /dev/null +++ b/examples/credit-risk-end-to-end/03_Credit_Risk_Model_Training.ipynb @@ -0,0 +1,1541 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "54f2ab19-68e1-4725-b6e7-efd8eedebe1a", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "69a40de4-65cf-4b45-b321-2b7ce571f8cb", + "metadata": {}, + "source": [ + "# Credit Risk Model Training" + ] + }, + { + "cell_type": "markdown", + "id": "fe641d83-1e28-4f7f-895c-8ca038f6cc53", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "8f04f635-401b-47b6-b807-df61d42ec752", + "metadata": {}, + "source": [ + "AI models have played a central role in modern credit risk assessment systems. In this example, we develop a credit risk model to predict whether a future loan will be good or bad, given some context data (presumably supplied from the loan application process). We use the modeling process to demonstrate how Feast can be used to facilitate the serving of data for training and inference use-cases.\n", + "\n", + "In this notebook, we train our AI model. We will use the popular scikit-learn library (sklearn) to train a RandomForestClassifier, as this is a relatively easy choice for a baseline model." + ] + }, + { + "cell_type": "markdown", + "id": "a96bf1aa-c450-4201-83a4-e25b08bdd12d", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "a47b33bc-bc06-4de0-8f3a-beea8179035c", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "c66a3dab-fdbf-40be-8227-6180dc314a84", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import warnings\n", + "import datetime\n", + "import feast\n", + "import joblib\n", + "import pandas as pd\n", + "import seaborn as sns\n", + "\n", + "from feast import FeatureStore, RepoConfig\n", + "from sklearn.model_selection import train_test_split\n", + "from sklearn.preprocessing import OrdinalEncoder\n", + "from sklearn.compose import ColumnTransformer\n", + "from sklearn.pipeline import Pipeline\n", + "from sklearn.ensemble import RandomForestClassifier\n", + "from sklearn.metrics import classification_report" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "2a841445-fa47-4826-a874-28ac0e4ea57f", + "metadata": {}, + "outputs": [], + "source": [ + "# Ignore warnings\n", + "warnings.filterwarnings(action=\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "23579727-7797-4101-a70d-b0d4c24b0fdf", + "metadata": {}, + "outputs": [], + "source": [ + "# Random seed\n", + "SEED = 142" + ] + }, + { + "cell_type": "markdown", + "id": "fc5be519-7733-449b-8dc3-411e86371315", + "metadata": {}, + "source": [ + "This notebook assumes that you have already done the following:\n", + "\n", + "1. Run the [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb) notebook to prepare the data.\n", + "2. Run the [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) notebook to configure the feature stores and launch the feature store servers.\n", + "\n", + "If you have not completed the above steps, please go back and do so before continuing. This notebook relies on the data prepared by 1, and it uses the Feast offline server stood up by 2." + ] + }, + { + "cell_type": "markdown", + "id": "1ca99047-e508-4b1f-9f4c-f11e38587d70", + "metadata": {}, + "source": [ + "### Load Label (Outcome) Data" + ] + }, + { + "cell_type": "markdown", + "id": "89b49268-b7a5-4abc-8d82-1cdbf9bb4473", + "metadata": {}, + "source": [ + "From our previous data exploration, remember that the label data represents whether the loan was classed as \"good\" (1) or \"bad\" (0). Let's pull the labels for training, as we will use them as our \"entity dataframe\" when pulling features.\n", + "\n", + "This is also a good time to remember that the label timestamps are lagged by 30-90 days from the context data records." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "6a227a12-7b3e-462a-8f6e-38a7690df1c4", + "metadata": {}, + "outputs": [], + "source": [ + "labels = pd.read_parquet(\"Feature_Store/data/labels.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "31a39cad-0a85-4d98-ad95-008c81bb6fe0", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestamp
00good2023-11-24 22:50:13
11bad2023-11-03 12:10:13
22good2023-11-30 22:06:03
33good2023-11-17 07:37:19
44bad2023-12-01 05:01:48
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp\n", + "0 0 good 2023-11-24 22:50:13\n", + "1 1 bad 2023-11-03 12:10:13\n", + "2 2 good 2023-11-30 22:06:03\n", + "3 3 good 2023-11-17 07:37:19\n", + "4 4 bad 2023-12-01 05:01:48" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "labels.head()" + ] + }, + { + "cell_type": "markdown", + "id": "857f29fd-46d3-444b-b24f-eaccd82ab7d3", + "metadata": {}, + "source": [ + "### Pull Feature Data from Feast Offline Store" + ] + }, + { + "cell_type": "markdown", + "id": "07c13b69-3d26-484c-97cd-97734cc812bd", + "metadata": {}, + "source": [ + "In order to pull feature data from the offline store, we create a FeatureStore object that connects to the offline server (continuously running in the previous notebook)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "9e9828f8-f210-4586-ac36-3f7e17f4f1e8", + "metadata": {}, + "outputs": [], + "source": [ + "# Create FeatureStore object\n", + "# (connects to the offline server deployed in 02_Deploying_the_Feature_Store.ipynb) \n", + "store = FeatureStore(config=RepoConfig(\n", + " project=\"loan_applications\",\n", + " provider=\"local\",\n", + " registry=\"Feature_Store/data/registry.db\",\n", + " offline_store={\n", + " \"type\": \"remote\",\n", + " \"host\": \"localhost\",\n", + " \"port\": 8815\n", + " },\n", + " entity_key_serialization_version=2\n", + "))" + ] + }, + { + "cell_type": "markdown", + "id": "c007e7ca-40c1-4850-abed-73b6171ad08d", + "metadata": {}, + "source": [ + "Now, we can retrieve feature data by supplying our entity dataframe and feature specifications to the `get_historical_features` function. Note that this function performs a fuzzy lookback (\"point-in-time\") join, matching the lagged outcome timestamp to the closest application timestamp (per ID) in the context data; it also joins the \"a\" and \"b\" features that we had previously split into two tables.\n", + "\n", + "To keep this example simple, we will limit our feature set to the numerical features plus two categorical features." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "dd2e3cb5-c865-48f4-80b6-8a14a1ff09ab", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Using outcome_timestamp as the event timestamp. To specify a column explicitly, please name it event_timestamp.\n" + ] + } + ], + "source": [ + "# Get feature data\n", + "# (Joins a and b data, and selects records with the right timestamps)\n", + "df = store.get_historical_features(\n", + " entity_df=labels,\n", + " features=[\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\"\n", + " ]\n", + ").to_df()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "c72f6cb1-bbbf-4512-98cd-0abe5ff0c24b", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "RangeIndex: 1000 entries, 0 to 999\n", + "Data columns (total 12 columns):\n", + " # Column Non-Null Count Dtype \n", + "--- ------ -------------- ----- \n", + " 0 ID 1000 non-null int64 \n", + " 1 class 1000 non-null category \n", + " 2 outcome_timestamp 1000 non-null datetime64[ns, UTC]\n", + " 3 duration 1000 non-null int64 \n", + " 4 credit_amount 1000 non-null int64 \n", + " 5 installment_commitment 1000 non-null int64 \n", + " 6 checking_status 1000 non-null category \n", + " 7 residence_since 1000 non-null int64 \n", + " 8 age 1000 non-null int64 \n", + " 9 existing_credits 1000 non-null int64 \n", + " 10 num_dependents 1000 non-null int64 \n", + " 11 housing 1000 non-null category \n", + "dtypes: category(3), datetime64[ns, UTC](1), int64(8)\n", + "memory usage: 73.8 KB\n" + ] + } + ], + "source": [ + "# Check the data info\n", + "df.info()" + ] + }, + { + "cell_type": "markdown", + "id": "110ea48c-0a5a-4642-aaba-a9eeb4a7da48", + "metadata": {}, + "source": [ + "### Split the Data" + ] + }, + { + "cell_type": "markdown", + "id": "f6669dce-a8b0-4d80-9a15-70b7dfd2d718", + "metadata": {}, + "source": [ + "Next, we split the data into a `train` and `validate` set, which we will use to train and then validate a model. The validation set will allow us to more accurately assess the model's performance on data that it has not seen during the training phase." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "036b0a54-48e4-4414-bb8c-0c30b6ab7469", + "metadata": {}, + "outputs": [], + "source": [ + "# Split data into train and validate datasets\n", + "train, validate = train_test_split(df, test_size=0.2, random_state=SEED)" + ] + }, + { + "cell_type": "markdown", + "id": "4b65cbf7-5981-4f51-97aa-a3ff7027f2f3", + "metadata": {}, + "source": [ + "### Exploratory Data Analysis" + ] + }, + { + "cell_type": "markdown", + "id": "e516ded8-10ad-4274-a736-f288290b5883", + "metadata": {}, + "source": [ + "Before building a model, a data scientist needs to gain understanding of the data to make sure it meets important statistical assumptions, and to identify potential opportunities and issues. As the purpose of this particular example is to show working with Feast, we will take the view of a data scientist looking to build a quick baseline model to establish some low-end metrics.\n", + "\n", + "Note that this data set is very \"clean\", as it has already been prepared. In real-life, production credit risk data can be much more complex, and have many issues that need to be understood and addressed before modeling." + ] + }, + { + "cell_type": "markdown", + "id": "553986a0-c804-4ab4-a4b9-48b16c72fd4f", + "metadata": {}, + "source": [ + "Let's look at counts for the target variable `class`, which tells us whether a (historical) loan was good or bad. We can see that there were many more good loans than bad, making the dataset imbalanced." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "607bd29b-eaf4-41a6-aaca-a8eaaf37e2d2", + "metadata": {}, + "outputs": [ + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAjsAAAHHCAYAAABZbpmkAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMiwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy8hTgPZAAAACXBIWXMAAA9hAAAPYQGoP6dpAAAysElEQVR4nO3deVxV5d7///dmRmWDoIKWqJWKOHZw2k1qkWRkeWvllKlHG8FKy2PcOWLedqwcQ6tTqWWm2WBq5kRZHcVSTFNT1MqwFCgVtnoUBNbvj37sb/ugpQhsvHw9H4/1yHVd11rrc+3d1jdr2Ngsy7IEAABgKC9PFwAAAFCRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIO8AlyGazafz48Z4u45LD6wZcngg7QDmYN2+ebDab21KnTh116dJFn3zyiafLq3T5+fmaNWuWbrjhBtWsWVN+fn6qV6+e7rzzTr3zzjsqKirydInndODAAdlsNr3wwgueLuWCOZ1OTZgwQa1bt1aNGjUUGBioFi1aaNSoUTp06JCny5MkrVy5ksCJSufj6QIAkyQnJ6tRo0ayLEvZ2dmaN2+ebr/9di1fvlx33HGHp8urFL/++qu6deum9PR0xcXFafTo0QoNDVVWVpbWrVunfv36af/+/RozZoynSzXKDz/8oNjYWGVmZuqee+7Rgw8+KD8/P3377bd6/fXX9eGHH2rv3r2eLlMrV65USkoKgQeVirADlKNu3bqpbdu2rvUhQ4YoPDxc77zzzmUTdgYMGKBvvvlG77//vnr27OnWl5SUpC1btigjI8ND1ZmpsLBQPXv2VHZ2ttavX68bbrjBrX/SpEn65z//6aHqAM/jMhZQgUJCQhQYGCgfH/efK1544QVdd911CgsLU2BgoGJiYvTee++V2j4/P1/Dhw9X7dq1FRQUpDvvvFM///zzXx43OztbPj4+mjBhQqm+jIwM2Ww2vfTSS5KkM2fOaMKECWrcuLECAgIUFhamG264QWvXrr3g+aalpWn16tV68MEHSwWdEm3btlX//v3d2nJyclzBMCAgQK1bt9b8+fNLbXvy5Ek9+eSTql+/vvz9/dW0aVO98MILsizLbVxZX7cLcb41n+97bbPZlJiYqKVLl6pFixby9/dX8+bNtWrVqr+s5f3339f27dv1zDPPlAo6kmS32zVp0iS3tiVLligmJkaBgYGqVauW7rvvPv3yyy9uYzp37qzOnTuX2t+gQYPUsGFD1/ofL/29+uqruvrqq+Xv76927dpp8+bNbtulpKS45luyABWNMztAOcrLy9Nvv/0my7KUk5OjWbNm6cSJE7rvvvvcxs2YMUN33nmn+vfvr4KCAi1atEj33HOPVqxYofj4eNe4oUOHasGCBerXr5+uu+46ffrpp2795xIeHq5OnTrp3Xff1bhx49z6Fi9eLG9vb91zzz2SpPHjx2vy5MkaOnSo2rdvL6fTqS1btmjr1q269dZbL2j+y5cvl6RS8/0zp06dUufOnbV//34lJiaqUaNGWrJkiQYNGqTc3Fw9/vjjkiTLsnTnnXfqs88+05AhQ9SmTRutXr1aI0eO1C+//KJp06a59lnW1628a5bO/72WpH//+9/64IMP9OijjyooKEgzZ85Ur169lJmZqbCwsHPWs2zZMkm/n1U7H/PmzdPgwYPVrl07TZ48WdnZ2ZoxY4Y2bNigb775RiEhIRf+okhauHChjh8/roceekg2m01TpkxRz5499cMPP8jX11cPPfSQDh06pLVr1+qtt94q0zGAMrEAXLS5c+dakkot/v7+1rx580qN/89//uO2XlBQYLVo0cK6+eabXW3btm2zJFmPPvqo29h+/fpZkqxx48b9aU2vvPKKJcnasWOHW3t0dLTbcVq3bm3Fx8ef71T/1P/8z/9Ykqzc3Fy39lOnTlm//vqrazl27Jirb/r06ZYka8GCBa62goICy+FwWDVq1LCcTqdlWZa1dOlSS5L17LPPuu377rvvtmw2m7V//37Lsi7+dfvxxx8tSdbzzz9/zjHnW7Nlnd97bVmWJcny8/NzzcOyLGv79u2WJGvWrFl/WvO1115rBQcH/+mYPx6/Tp06VosWLaxTp0652lesWGFJssaOHetq69Spk9WpU6dS+xg4cKDVoEED13rJaxYWFmYdPXrU1f7RRx9Zkqzly5e72hISEiz+6UFl4zIWUI5SUlK0du1arV27VgsWLFCXLl00dOhQffDBB27jAgMDXX8+duyY8vLydOONN2rr1q2u9pUrV0qSHnvsMbdtn3jiifOqpWfPnvLx8dHixYtdbTt37tR3332n3r17u9pCQkK0a9cu7du377zneS5Op1OSVKNGDbf2l19+WbVr13Ytf7zUsnLlSkVERKhv376uNl9fXz322GM6ceKEPv/8c9c4b2/vUq/Hk08+KcuyXE+9Xezrdj7Ot2bp/N7rErGxsbr66qtd661atZLdbtcPP/zwp/U4nU4FBQWdV+1btmxRTk6OHn30UQUEBLja4+PjFRUVpY8//vi89nM2vXv3Vs2aNV3rN954oyT9Zf1ARSPsAOWoffv2io2NVWxsrPr376+PP/5Y0dHRSkxMVEFBgWvcihUr1LFjRwUEBCg0NFS1a9fWnDlzlJeX5xrz008/ycvLy+0fP0lq2rTpedVSq1Yt3XLLLXr33XddbYsXL5aPj4/b/TTJycnKzc1VkyZN1LJlS40cOVLffvttmeZf8g/uiRMn3Np79erlCoGtWrVy6/vpp5/UuHFjeXm5/3XUrFkzV3/Jf+vVq1fqH/WzjbuY1+18nG/N0vm91yUiIyNLtdWsWVPHjh3703rsdruOHz9+3rVLZ389oqKi3Gq/UP9df0nw+av6gYpG2AEqkJeXl7p06aLDhw+7zpx8+eWXuvPOOxUQEKDZs2dr5cqVWrt2rfr161fqRtuL1adPH+3du1fbtm2TJL377ru65ZZbVKtWLdeYm266Sd9//73eeOMNtWjRQq+99pr+9re/6bXXXrvg40VFRUn6/QzSH9WvX98VAv/4k7/pLvS99vb2Put+/ur/i6ioKOXl5engwYPlUneJc908fK7vSSpr/UBFI+wAFaywsFDS/zvb8f777ysgIECrV6/W3//+d3Xr1k2xsbGltmvQoIGKi4v1/fffu7VfyGPbPXr0kJ+fnxYvXqxt27Zp79696tOnT6lxoaGhGjx4sN555x0dPHhQrVq1KtP3oJQ8Xv/222+f9zYNGjTQvn37VFxc7Na+Z88eV3/Jfw8dOlTqDMbZxl3s61ZeNZ/ve32xunfvLklasGDBX44tqe1sr0dGRoarX/r9zExubm6pcRdz9oenr+AJhB2gAp05c0Zr1qyRn5+f6xKHt7e3bDab20/HBw4c0NKlS9227datmyRp5syZbu3Tp08/7+OHhIQoLi5O7777rhYtWiQ/Pz/16NHDbcyRI0fc1mvUqKFrrrlG+fn5rra8vDzt2bPnrJde/uj666/XrbfeqldffVUfffTRWcf890/5t99+u7KystzuLSosLNSsWbNUo0YNderUyTWuqKjI9ch8iWnTpslms7ler/J43f7K+dZ8vu/1xbr77rvVsmVLTZo0SWlpaaX6jx8/rmeeeUbS74/+16lTRy+//LLbe/zJJ59o9+7dbk+IXX311dqzZ49+/fVXV9v27du1YcOGMtdavXp1STpriAIqCo+eA+Xok08+cf10n5OTo4ULF2rfvn16+umnZbfbJf1+I+jUqVN12223qV+/fsrJyVFKSoquueYat3tl2rRpo759+2r27NnKy8vTddddp9TUVO3fv/+Caurdu7fuu+8+zZ49W3FxcaUeK46Ojlbnzp0VExOj0NBQbdmyRe+9954SExNdYz788EMNHjxYc+fO1aBBg/70eAsWLNBtt92mHj16uM5k1KxZ0/UNyl988YUrkEjSgw8+qFdeeUWDBg1Senq6GjZsqPfee08bNmzQ9OnTXffodO/eXV26dNEzzzyjAwcOqHXr1lqzZo0++ugjPfHEE657dMrrdUtNTdXp06dLtffo0eO8az7f9/pi+fr66oMPPlBsbKxuuukm3Xvvvbr++uvl6+urXbt2aeHChapZs6YmTZokX19f/fOf/9TgwYPVqVMn9e3b1/XoecOGDTV8+HDXfv/+979r6tSpiouL05AhQ5STk6OXX35ZzZs3d92MfqFiYmIk/X4DeVxcnLy9vc96thEoV558FAwwxdkePQ8ICLDatGljzZkzxyouLnYb//rrr1uNGze2/P39raioKGvu3LnWuHHjSj2Se+rUKeuxxx6zwsLCrOrVq1vdu3e3Dh48eF6PUJdwOp1WYGBgqUelSzz77LNW+/btrZCQECswMNCKioqyJk2aZBUUFJSa39y5c8/rmKdOnbKmT59uORwOy263Wz4+PlZERIR1xx13WG+//bZVWFjoNj47O9saPHiwVatWLcvPz89q2bLlWY91/Phxa/jw4Va9evUsX19fq3Hjxtbzzz9f6vW9mNet5DHqcy1vvfXWBdV8vu+1JCshIaHU9g0aNLAGDhz4pzWXOHbsmDV27FirZcuWVrVq1ayAgACrRYsWVlJSknX48GG3sYsXL7auvfZay9/f3woNDbX69+9v/fzzz6X2uWDBAuuqq66y/Pz8rDZt2lirV68+56PnZ3tc/79f88LCQmvYsGFW7dq1LZvNxmPoqBQ2y+LOMQAAYC7u2QEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphR79/o6vT6eT3twAAYCDCjn7/KvXg4ODz/q3BAADg0kHYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNF8PF0AAFSW02eKlHn0P54uA7gsRIZWU4Cvt6fLkETYAXAZyTz6H439aKenywAuC8l3tVCT8CBPlyGJy1gAAMBwhB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDSPhp3x48fLZrO5LVFRUa7+06dPKyEhQWFhYapRo4Z69eql7Oxst31kZmYqPj5e1apVU506dTRy5EgVFhZW9lQAAEAV5ePpApo3b65169a51n18/l9Jw4cP18cff6wlS5YoODhYiYmJ6tmzpzZs2CBJKioqUnx8vCIiIrRx40YdPnxY999/v3x9ffV///d/lT4XAABQ9Xg87Pj4+CgiIqJUe15enl5//XUtXLhQN998syRp7ty5atasmTZt2qSOHTtqzZo1+u6777Ru3TqFh4erTZs2mjhxokaNGqXx48fLz8+vsqcDAACqGI/fs7Nv3z7Vq1dPV111lfr376/MzExJUnp6us6cOaPY2FjX2KioKEVGRiotLU2SlJaWppYtWyo8PNw1Ji4uTk6nU7t27TrnMfPz8+V0Ot0WAABgJo+GnQ4dOmjevHlatWqV5syZox9//FE33nijjh8/rqysLPn5+SkkJMRtm/DwcGVlZUmSsrKy3IJOSX9J37lMnjxZwcHBrqV+/frlOzEAAFBlePQyVrdu3Vx/btWqlTp06KAGDRro3XffVWBgYIUdNykpSSNGjHCtO51OAg8AAIby+GWsPwoJCVGTJk20f/9+RUREqKCgQLm5uW5jsrOzXff4RERElHo6q2T9bPcBlfD395fdbndbAACAmapU2Dlx4oS+//571a1bVzExMfL19VVqaqqrPyMjQ5mZmXI4HJIkh8OhHTt2KCcnxzVm7dq1stvtio6OrvT6AQBA1ePRy1hPPfWUunfvrgYNGujQoUMaN26cvL291bdvXwUHB2vIkCEaMWKEQkNDZbfbNWzYMDkcDnXs2FGS1LVrV0VHR2vAgAGaMmWKsrKyNHr0aCUkJMjf39+TUwMAAFWER8POzz//rL59++rIkSOqXbu2brjhBm3atEm1a9eWJE2bNk1eXl7q1auX8vPzFRcXp9mzZ7u29/b21ooVK/TII4/I4XCoevXqGjhwoJKTkz01JQAAUMXYLMuyPF2EpzmdTgUHBysvL4/7dwCD7c0+rrEf7fR0GcBlIfmuFmoSHuTpMiRVsXt2AAAAyhthBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjFZlws5zzz0nm82mJ554wtV2+vRpJSQkKCwsTDVq1FCvXr2UnZ3ttl1mZqbi4+NVrVo11alTRyNHjlRhYWElVw8AAKqqKhF2Nm/erFdeeUWtWrVyax8+fLiWL1+uJUuW6PPPP9ehQ4fUs2dPV39RUZHi4+NVUFCgjRs3av78+Zo3b57Gjh1b2VMAAABVlMfDzokTJ9S/f3/961//Us2aNV3teXl5ev311zV16lTdfPPNiomJ0dy5c7Vx40Zt2rRJkrRmzRp99913WrBggdq0aaNu3bpp4sSJSklJUUFBgaemBAAAqhCPh52EhATFx8crNjbWrT09PV1nzpxxa4+KilJkZKTS0tIkSWlpaWrZsqXCw8NdY+Li4uR0OrVr165zHjM/P19Op9NtAQAAZvLx5MEXLVqkrVu3avPmzaX6srKy5Ofnp5CQELf28PBwZWVlucb8MeiU9Jf0ncvkyZM1YcKEi6weAABcCjx2ZufgwYN6/PHH9fbbbysgIKBSj52UlKS8vDzXcvDgwUo9PgAAqDweCzvp6enKycnR3/72N/n4+MjHx0eff/65Zs6cKR8fH4WHh6ugoEC5ublu22VnZysiIkKSFBERUerprJL1kjFn4+/vL7vd7rYAAAAzeSzs3HLLLdqxY4e2bdvmWtq2bav+/fu7/uzr66vU1FTXNhkZGcrMzJTD4ZAkORwO7dixQzk5Oa4xa9euld1uV3R0dKXPCQAAVD0eu2cnKChILVq0cGurXr26wsLCXO1DhgzRiBEjFBoaKrvdrmHDhsnhcKhjx46SpK5duyo6OloDBgzQlClTlJWVpdGjRyshIUH+/v6VPicAAFD1ePQG5b8ybdo0eXl5qVevXsrPz1dcXJxmz57t6vf29taKFSv0yCOPyOFwqHr16ho4cKCSk5M9WDUAAKhKbJZlWZ4uwtOcTqeCg4OVl5fH/TuAwfZmH9fYj3Z6ugzgspB8Vws1CQ/ydBmSqsD37AAAAFQkwg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAo5Up7Fx11VU6cuRIqfbc3FxdddVVF10UAABAeSlT2Dlw4ICKiopKtefn5+uXX3656KIAAADKi8+FDF62bJnrz6tXr1ZwcLBrvaioSKmpqWrYsGG5FQcAAHCxLijs9OjRQ5Jks9k0cOBAtz5fX181bNhQL774YrkVBwAAcLEuKOwUFxdLkho1aqTNmzerVq1aFVIUAABAebmgsFPixx9/LO86AAAAKkSZwo4kpaamKjU1VTk5Oa4zPiXeeOONiy4MAACgPJQp7EyYMEHJyclq27at6tatK5vNVt51AQAAlIsyhZ2XX35Z8+bN04ABA8q7HgAAgHJVpu/ZKSgo0HXXXVfetQAAAJS7MoWdoUOHauHCheVdCwAAQLkr02Ws06dP69VXX9W6devUqlUr+fr6uvVPnTq1XIoDAAC4WGUKO99++63atGkjSdq5c6dbHzcrAwCAqqRMl7E+++yzcy6ffvrpee9nzpw5atWqlex2u+x2uxwOhz755BNX/+nTp5WQkKCwsDDVqFFDvXr1UnZ2tts+MjMzFR8fr2rVqqlOnToaOXKkCgsLyzItAABgoDKFnfJy5ZVX6rnnnlN6erq2bNmim2++WXfddZd27dolSRo+fLiWL1+uJUuW6PPPP9ehQ4fUs2dP1/ZFRUWKj49XQUGBNm7cqPnz52vevHkaO3asp6YEAACqGJtlWdaFbtSlS5c/vVx1IWd3/ltoaKief/553X333apdu7YWLlyou+++W5K0Z88eNWvWTGlpaerYsaM++eQT3XHHHTp06JDCw8Ml/f5Y/KhRo/Trr7/Kz8/vvI7pdDoVHBysvLw82e32MtcOoGrbm31cYz/a+dcDAVy05LtaqEl4kKfLkFTGMztt2rRR69atXUt0dLQKCgq0detWtWzZskyFFBUVadGiRTp58qQcDofS09N15swZxcbGusZERUUpMjJSaWlpkqS0tDS1bNnSFXQkKS4uTk6n03V26Gzy8/PldDrdFgAAYKYy3aA8bdq0s7aPHz9eJ06cuKB97dixQw6HQ6dPn1aNGjX04YcfKjo6Wtu2bZOfn59CQkLcxoeHhysrK0uSlJWV5RZ0SvpL+s5l8uTJmjBhwgXVCQAALk3les/Offfdd8G/F6tp06batm2bvvrqKz3yyCMaOHCgvvvuu/Isq5SkpCTl5eW5loMHD1bo8QAAgOeU+ReBnk1aWpoCAgIuaBs/Pz9dc801kqSYmBht3rxZM2bMUO/evVVQUKDc3Fy3szvZ2dmKiIiQJEVEROjrr79221/J01olY87G399f/v7+F1QnAAC4NJUp7PzxiShJsixLhw8f1pYtWzRmzJiLKqi4uFj5+fmKiYmRr6+vUlNT1atXL0lSRkaGMjMz5XA4JEkOh0OTJk1STk6O6tSpI0lau3at7Ha7oqOjL6oOAABghjKFneDgYLd1Ly8vNW3aVMnJyeratet57ycpKUndunVTZGSkjh8/roULF2r9+vVavXq1goODNWTIEI0YMUKhoaGy2+0aNmyYHA6HOnbsKEnq2rWroqOjNWDAAE2ZMkVZWVkaPXq0EhISOHMDAAAklTHszJ07t1wOnpOTo/vvv1+HDx9WcHCwWrVqpdWrV+vWW2+V9PuN0F5eXurVq5fy8/MVFxen2bNnu7b39vbWihUr9Mgjj8jhcKh69eoaOHCgkpOTy6U+AABw6SvT9+yUSE9P1+7duyVJzZs317XXXltuhVUmvmcHuDzwPTtA5alK37NTpjM7OTk56tOnj9avX++6eTg3N1ddunTRokWLVLt27fKsEQAAoMzK9Oj5sGHDdPz4ce3atUtHjx7V0aNHtXPnTjmdTj322GPlXSMAAECZlenMzqpVq7Ru3To1a9bM1RYdHa2UlJQLukEZAACgopXpzE5xcbF8fX1Ltfv6+qq4uPiiiwIAACgvZQo7N998sx5//HEdOnTI1fbLL79o+PDhuuWWW8qtOAAAgItVprDz0ksvyel0qmHDhrr66qt19dVXq1GjRnI6nZo1a1Z51wgAAFBmZbpnp379+tq6davWrVunPXv2SJKaNWvm9hvKAQAAqoILOrPz6aefKjo6Wk6nUzabTbfeequGDRumYcOGqV27dmrevLm+/PLLiqoVAADggl1Q2Jk+fboeeOCBs37xXnBwsB566CFNnTq13IoDAAC4WBcUdrZv367bbrvtnP1du3ZVenr6RRcFAABQXi4o7GRnZ5/1kfMSPj4++vXXXy+6KAAAgPJyQWHniiuu0M6d5/69Mt9++63q1q170UUBAACUlwsKO7fffrvGjBmj06dPl+o7deqUxo0bpzvuuKPcigMAALhYF/To+ejRo/XBBx+oSZMmSkxMVNOmTSVJe/bsUUpKioqKivTMM89USKEAAABlcUFhJzw8XBs3btQjjzyipKQkWZYlSbLZbIqLi1NKSorCw8MrpFAAAICyuOAvFWzQoIFWrlypY8eOaf/+/bIsS40bN1bNmjUroj4AAICLUqZvUJakmjVrql27duVZCwAAQLkr0+/GAgAAuFQQdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0H08XcLk4faZImUf/4+kygMvGNbVryMvL5ukyAFQBhJ1Kknn0Pxr70U5PlwFcNuYOaq9AP29PlwGgCuAyFgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACM5tGwM3nyZLVr105BQUGqU6eOevTooYyMDLcxp0+fVkJCgsLCwlSjRg316tVL2dnZbmMyMzMVHx+vatWqqU6dOho5cqQKCwsrcyoAAKCK8mjY+fzzz5WQkKBNmzZp7dq1OnPmjLp27aqTJ0+6xgwfPlzLly/XkiVL9Pnnn+vQoUPq2bOnq7+oqEjx8fEqKCjQxo0bNX/+fM2bN09jx471xJQAAEAV49Hfer5q1Sq39Xnz5qlOnTpKT0/XTTfdpLy8PL3++utauHChbr75ZknS3Llz1axZM23atEkdO3bUmjVr9N1332ndunUKDw9XmzZtNHHiRI0aNUrjx4+Xn5+fJ6YGAACqiCp1z05eXp4kKTQ0VJKUnp6uM2fOKDY21jUmKipKkZGRSktLkySlpaWpZcuWCg8Pd42Ji4uT0+nUrl27KrF6AABQFXn0zM4fFRcX64knntD111+vFi1aSJKysrLk5+enkJAQt7Hh4eHKyspyjflj0CnpL+k7m/z8fOXn57vWnU5neU0DAABUMVXmzE5CQoJ27typRYsWVfixJk+erODgYNdSv379Cj8mAADwjCoRdhITE7VixQp99tlnuvLKK13tERERKigoUG5urtv47OxsRUREuMb899NZJeslY/5bUlKS8vLyXMvBgwfLcTYAAKAq8WjYsSxLiYmJ+vDDD/Xpp5+qUaNGbv0xMTHy9fVVamqqqy0jI0OZmZlyOBySJIfDoR07dignJ8c1Zu3atbLb7YqOjj7rcf39/WW3290WAABgJo/es5OQkKCFCxfqo48+UlBQkOsem+DgYAUGBio4OFhDhgzRiBEjFBoaKrvdrmHDhsnhcKhjx46SpK5duyo6OloDBgzQlClTlJWVpdGjRyshIUH+/v6enB4AAKgCPBp25syZI0nq3LmzW/vcuXM1aNAgSdK0adPk5eWlXr16KT8/X3FxcZo9e7ZrrLe3t1asWKFHHnlEDodD1atX18CBA5WcnFxZ0wAAAFWYR8OOZVl/OSYgIEApKSlKSUk555gGDRpo5cqV5VkaAAAwRJW4QRkAAKCiEHYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBoHg07X3zxhbp376569erJZrNp6dKlbv2WZWns2LGqW7euAgMDFRsbq3379rmNOXr0qPr37y+73a6QkBANGTJEJ06cqMRZAACAqsyjYefkyZNq3bq1UlJSzto/ZcoUzZw5Uy+//LK++uorVa9eXXFxcTp9+rRrTP/+/bVr1y6tXbtWK1as0BdffKEHH3ywsqYAAACqOB9PHrxbt27q1q3bWfssy9L06dM1evRo3XXXXZKkN998U+Hh4Vq6dKn69Omj3bt3a9WqVdq8ebPatm0rSZo1a5Zuv/12vfDCC6pXr16lzQUAAFRNVfaenR9//FFZWVmKjY11tQUHB6tDhw5KS0uTJKWlpSkkJMQVdCQpNjZWXl5e+uqrryq9ZgAAUPV49MzOn8nKypIkhYeHu7WHh4e7+rKyslSnTh23fh8fH4WGhrrGnE1+fr7y8/Nd606ns7zKBgAAVUyVPbNTkSZPnqzg4GDXUr9+fU+XBAAAKkiVDTsRERGSpOzsbLf27OxsV19ERIRycnLc+gsLC3X06FHXmLNJSkpSXl6eazl48GA5Vw8AAKqKKht2GjVqpIiICKWmprranE6nvvrqKzkcDkmSw+FQbm6u0tPTXWM+/fRTFRcXq0OHDufct7+/v+x2u9sCAADM5NF7dk6cOKH9+/e71n/88Udt27ZNoaGhioyM1BNPPKFnn31WjRs3VqNGjTRmzBjVq1dPPXr0kCQ1a9ZMt912mx544AG9/PLLOnPmjBITE9WnTx+exAIAAJI8HHa2bNmiLl26uNZHjBghSRo4cKDmzZunf/zjHzp58qQefPBB5ebm6oYbbtCqVasUEBDg2ubtt99WYmKibrnlFnl5ealXr16aOXNmpc8FAABUTR4NO507d5ZlWefst9lsSk5OVnJy8jnHhIaGauHChRVRHgAAMECVvWcHAACgPBB2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBohB0AAGA0wg4AADAaYQcAABiNsAMAAIxG2AEAAEYj7AAAAKMRdgAAgNEIOwAAwGiEHQAAYDTCDgAAMBphBwAAGI2wAwAAjEbYAQAARiPsAAAAoxF2AACA0Qg7AADAaMaEnZSUFDVs2FABAQHq0KGDvv76a0+XBAAAqgAjws7ixYs1YsQIjRs3Tlu3blXr1q0VFxennJwcT5cGAAA8zGZZluXpIi5Whw4d1K5dO7300kuSpOLiYtWvX1/Dhg3T008//ZfbO51OBQcHKy8vT3a7vUJqPH2mSJlH/1Mh+wZQ2jW1a8jLy+bWxucQqDyRodUU4Ovt6TIkST6eLuBiFRQUKD09XUlJSa42Ly8vxcbGKi0tzYOVuQvw9VaT8CBPlwFc1vgcApenSz7s/PbbbyoqKlJ4eLhbe3h4uPbs2XPWbfLz85Wfn+9az8vLk/T7GR4AAHBpCQoKks1mO2f/JR92ymLy5MmaMGFCqfb69et7oBoAAHAx/uo2lEs+7NSqVUve3t7Kzs52a8/OzlZERMRZt0lKStKIESNc68XFxTp69KjCwsL+NBni8uN0OlW/fn0dPHiwwu7nAnBufAZxPoKC/vzy9CUfdvz8/BQTE6PU1FT16NFD0u/hJTU1VYmJiWfdxt/fX/7+/m5tISEhFVwpLmV2u52/aAEP4jOIi3HJhx1JGjFihAYOHKi2bduqffv2mj59uk6ePKnBgwd7ujQAAOBhRoSd3r1769dff9XYsWOVlZWlNm3aaNWqVaVuWgYAAJcfI8KOJCUmJp7zshVQVv7+/ho3blypy54AKgefQZQHI75UEAAA4FyM+HURAAAA50LYAQAARiPsAAAAoxF2cNnp3LmznnjiiXLd5/r162Wz2ZSbm1uu+wVwcRo2bKjp06d7ugx4GGEHAAAYjbADAACMRtjBZamwsFCJiYkKDg5WrVq1NGbMGJV8C8Nbb72ltm3bKigoSBEREerXr59ycnLctl+5cqWaNGmiwMBAdenSRQcOHPDALIBLx/Hjx9W/f39Vr15ddevW1bRp09wuKR87dkz333+/atasqWrVqqlbt27at2+f2z7ef/99NW/eXP7+/mrYsKFefPFFt/6cnBx1795dgYGBatSokd5+++3Kmh6qOMIOLkvz58+Xj4+Pvv76a82YMUNTp07Va6+9Jkk6c+aMJk6cqO3bt2vp0qU6cOCABg0a5Nr24MGD6tmzp7p3765t27Zp6NChevrppz00E+DSMGLECG3YsEHLli3T2rVr9eWXX2rr1q2u/kGDBmnLli1atmyZ0tLSZFmWbr/9dp05c0aSlJ6ernvvvVd9+vTRjh07NH78eI0ZM0bz5s1z28fBgwf12Wef6b333tPs2bNL/aCCy5QFXGY6depkNWvWzCouLna1jRo1ymrWrNlZx2/evNmSZB0/ftyyLMtKSkqyoqOj3caMGjXKkmQdO3aswuoGLlVOp9Py9fW1lixZ4mrLzc21qlWrZj3++OPW3r17LUnWhg0bXP2//fabFRgYaL377ruWZVlWv379rFtvvdVtvyNHjnR9FjMyMixJ1tdff+3q3717tyXJmjZtWgXODpcCzuzgstSxY0fZbDbXusPh0L59+1RUVKT09HR1795dkZGRCgoKUqdOnSRJmZmZkqTdu3erQ4cObvtzOByVVzxwifnhhx905swZtW/f3tUWHByspk2bSvr9M+Xj4+P2uQoLC1PTpk21e/du15jrr7/ebb/XX3+963Nbso+YmBhXf1RUlEJCQipwZrhUEHaAPzh9+rTi4uJkt9v19ttva/Pmzfrwww8lSQUFBR6uDgBQFoQdXJa++uort/VNmzapcePG2rNnj44cOaLnnntON954o6Kiokpd82/WrJm+/vrrUtsDOLurrrpKvr6+2rx5s6stLy9Pe/fulfT7Z6qwsNDtc3nkyBFlZGQoOjraNWbDhg1u+92wYYOaNGkib29vRUVFqbCwUOnp6a7+jIwMvvsKkgg7uExlZmZqxIgRysjI0DvvvKNZs2bp8ccfV2RkpPz8/DRr1iz98MMPWrZsmSZOnOi27cMPP6x9+/Zp5MiRysjI0MKFC91ukgTgLigoSAMHDtTIkSP12WefadeuXRoyZIi8vLxks9nUuHFj3XXXXXrggQf073//W9u3b9d9992nK664QnfddZck6cknn1RqaqomTpyovXv3av78+XrppZf01FNPSZKaNm2q2267TQ899JC++uorpaena+jQoQoMDPTk1FFVePqmIaCyderUyXr00Uethx9+2LLb7VbNmjWt//3f/3XdsLxw4UKrYcOGlr+/v+VwOKxly5ZZkqxvvvnGtY/ly5db11xzjeXv72/deOON1htvvMENysCfcDqdVr9+/axq1apZERER1tSpU6327dtbTz/9tGVZlnX06FFrwIABVnBwsBUYGGjFxcVZe/fuddvHe++9Z0VHR1u+vr5WZGSk9fzzz7v1Hz582IqPj7f8/f2tyMhI680337QaNGjADcqwbJb1/3+5CAAAleTkyZO64oor9OKLL2rIkCGeLgeG8/F0AQAA833zzTfas2eP2rdvr7y8PCUnJ0uS6zIVUJEIOwCASvHCCy8oIyNDfn5+iomJ0ZdffqlatWp5uixcBriMBQAAjMbTWAAAwGiEHQAAYDTCDgAAMBphBwAAGI2wA+CSdeDAAdlsNm3bts3TpQCowgg7AADAaIQdAABgNMIOgCqvuLhYU6ZM0TXXXCN/f39FRkZq0qRJpcYVFRVpyJAhatSokQIDA9W0aVPNmDHDbcz69evVvn17Va9eXSEhIbr++uv1008/SZK2b9+uLl26KCgoSHa7XTExMdqyZUulzBFAxeEblAFUeUlJSfrXv/6ladOm6YYbbtDhw4e1Z8+eUuOKi4t15ZVXasmSJQoLC9PGjRv14IMPqm7durr33ntVWFioHj166IEHHtA777yjgoICff3117LZbJKk/v3769prr9WcOXPk7e2tbdu2ydfXt7KnC6Cc8Q3KAKq048ePq3bt2nrppZc0dOhQt74DBw6oUaNG+uabb9SmTZuzbp+YmKisrCy99957Onr0qMLCwrR+/Xp16tSp1Fi73a5Zs2Zp4MCBFTEVAB7CZSwAVdru3buVn5+vW2655bzGp6SkKCYmRrVr11aNGjX06quvKjMzU5IUGhqqQYMGKS4uTt27d9eMGTN0+PBh17YjRozQ0KFDFRsbq+eee07ff/99hcwJQOUi7ACo0gIDA8977KJFi/TUU09pyJAhWrNmjbZt26bBgweroKDANWbu3LlKS0vTddddp8WLF6tJkybatGmTJGn8+PHatWuX4uPj9emnnyo6Oloffvhhuc8JQOXiMhaAKu306dMKDQ3VzJkz//Iy1rBhw/Tdd98pNTXVNSY2Nla//fbbOb+Lx+FwqF27dpo5c2apvr59++rkyZNatmxZuc4JQOXizA6AKi0gIECjRo3SP/7xD7355pv6/vvvtWnTJr3++uulxjZu3FhbtmzR6tWrtXfvXo0ZM0abN2929f/4449KSkpSWlqafvrpJ61Zs0b79u1Ts2bNdOrUKSUmJmr9+vX66aeftGHDBm3evFnNmjWrzOkCqAA8jQWgyhszZox8fHw0duxYHTp0SHXr1tXDDz9catxDDz2kb775Rr1795bNZlPfvn316KOP6pNPPpEkVatWTXv27NH8+fN15MgR1a1bVwkJCXrooYdUWFioI0eO6P7771d2drZq1aqlnj17asKECZU9XQDljMtYAADAaFzGAgAARiPsAAAAoxF2AACA0Qg7AADAaIQdAABgNMIOAAAwGmEHAAAYjbADAACMRtgBAABGI+wAAACjEXYAAIDRCDsAAMBo/x9dWkm/NZ32CAAAAABJRU5ErkJggg==", + "text/plain": [ + "
" + ] + }, + "metadata": {}, + "output_type": "display_data" + } + ], + "source": [ + "# plot the target variable \"class\"\n", + "p = sns.histplot(train[\"class\"], ec=\"w\", lw=4)\n", + "_ = p.set_title(\"Bad vs. Good Loan Count\")\n", + "_ = p.spines[\"top\"].set_visible(False)\n", + "_ = p.spines[\"right\"].set_visible(False)" + ] + }, + { + "cell_type": "markdown", + "id": "c6a697a5-5709-4a69-b644-62779b4f8bc5", + "metadata": {}, + "source": [ + "Now, view the first few records of the context data." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "79424785-129d-4007-84a5-041b6d38457d", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDclassoutcome_timestampdurationcredit_amountinstallment_commitmentchecking_statusresidence_sinceageexisting_creditsnum_dependentshousing
18473good2023-12-16 03:29:12+00:00612384no checking43612own
764894good2023-11-15 23:19:35+00:001811694no checking32921own
504318good2023-11-23 13:03:53+00:00127014no checking23221own
454340good2023-12-26 17:59:37+00:0024574320<=X<20042421for free
453605good2023-12-18 11:27:02+00:002428284<042211own
\n", + "
" + ], + "text/plain": [ + " ID class outcome_timestamp duration credit_amount \\\n", + "18 473 good 2023-12-16 03:29:12+00:00 6 1238 \n", + "764 894 good 2023-11-15 23:19:35+00:00 18 1169 \n", + "504 318 good 2023-11-23 13:03:53+00:00 12 701 \n", + "454 340 good 2023-12-26 17:59:37+00:00 24 5743 \n", + "453 605 good 2023-12-18 11:27:02+00:00 24 2828 \n", + "\n", + " installment_commitment checking_status residence_since age \\\n", + "18 4 no checking 4 36 \n", + "764 4 no checking 3 29 \n", + "504 4 no checking 2 32 \n", + "454 2 0<=X<200 4 24 \n", + "453 4 <0 4 22 \n", + "\n", + " existing_credits num_dependents housing \n", + "18 1 2 own \n", + "764 2 1 own \n", + "504 2 1 own \n", + "454 2 1 for free \n", + "453 1 1 own " + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# View first records in training data\n", + "train.head()" + ] + }, + { + "cell_type": "markdown", + "id": "fd52f5bc-aa0f-48db-b356-c52aa7ce3724", + "metadata": {}, + "source": [ + "### Feature Engineering" + ] + }, + { + "cell_type": "markdown", + "id": "3e5b5c02-ad4d-400e-bdac-bfdf2799f575", + "metadata": {}, + "source": [ + "Once data columns have been prepared so that they can be used to train an AI model, it is common to refer to them as \"features\". The process of preparing features is referred to as \"feature engineering\". \n", + "\n", + "Below, we will train a random forest model. Random forests are relatively robust to non-standardized, non-normalized data, making it easier for us to getting started. As such, the numerical columns are ready for a simple baseline training. \n", + "\n", + "We have pulled two categorical columns, wich we will need to engineer into numerical features." + ] + }, + { + "cell_type": "markdown", + "id": "45a6fb27-140c-4f5a-b464-1f5e5d81d086", + "metadata": {}, + "source": [ + "The `checking_status` column tells us roughly how much money the applicant has in their checking account, while the `housing` column shows the applicant's housing status. We presume that more money in checking correlates inversely with credit risk, while owing vs. renting, vs. living for free correlates directly with credit risk. Hence, converting these to ordinal features makes sense. Of course, in a real study we would want to quantitatively verify these presumptions." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "id": "9e374096-b02d-4cbb-8fca-dcc451c90c50", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "checking_status\n", + "no checking 0.39375\n", + "0<=X<200 0.27500\n", + "<0 0.26125\n", + ">=200 0.07000\n", + "Name: proportion, dtype: float64" + ] + }, + "execution_count": 12, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the `checking_status` column distibution\n", + "train.checking_status.value_counts(normalize=True)" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "0144b525-244b-4526-8e4b-d393cb174d06", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "housing\n", + "own 0.7225\n", + "rent 0.1675\n", + "for free 0.1100\n", + "Name: proportion, dtype: float64" + ] + }, + "execution_count": 13, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the `housing` column distribution\n", + "train.housing.value_counts(normalize=True)" + ] + }, + { + "cell_type": "markdown", + "id": "2cb340b4-7d21-4810-8be2-1633da2e4396", + "metadata": {}, + "source": [ + "We define a tranformer that can be used to convert `checking_status` and `housing` to ordinal variables. The transformer will also drop the non-feature columns (`class`, `ID`, and `application_timestamp`) from the feature data." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "id": "27796e23-c12e-4e51-8fb4-090b26aff2ef", + "metadata": {}, + "outputs": [], + "source": [ + "# Feature lists\n", + "cat_features = [\"checking_status\", \"housing\"]\n", + "num_features = [\n", + " \"duration\", \"credit_amount\", \"installment_commitment\",\n", + " \"residence_since\", \"age\", \"existing_credits\", \"num_dependents\"\n", + "]\n", + "\n", + "# Ordinal encoder for cat_features\n", + "# (We use a ColumnTransformer to passthrough numerical feature columns)\n", + "col_transform = ColumnTransformer([\n", + " (\"cat_features\", OrdinalEncoder(), cat_features),\n", + " (\"num_features\", \"passthrough\", num_features),\n", + " ],\n", + " remainder=\"drop\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "id": "318429b9-e008-4cc7-8108-779934f9ac2f", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
checking_statushousingdurationcredit_amountinstallment_commitmentresidence_sinceageexisting_creditsnum_dependents
183.01.06.01238.04.04.036.01.02.0
7643.01.018.01169.04.03.029.02.01.0
5043.01.012.0701.04.02.032.02.01.0
4540.00.024.05743.02.04.024.02.01.0
4531.01.024.02828.04.04.022.01.01.0
\n", + "
" + ], + "text/plain": [ + " checking_status housing duration credit_amount \\\n", + "18 3.0 1.0 6.0 1238.0 \n", + "764 3.0 1.0 18.0 1169.0 \n", + "504 3.0 1.0 12.0 701.0 \n", + "454 0.0 0.0 24.0 5743.0 \n", + "453 1.0 1.0 24.0 2828.0 \n", + "\n", + " installment_commitment residence_since age existing_credits \\\n", + "18 4.0 4.0 36.0 1.0 \n", + "764 4.0 3.0 29.0 2.0 \n", + "504 4.0 2.0 32.0 2.0 \n", + "454 2.0 4.0 24.0 2.0 \n", + "453 4.0 4.0 22.0 1.0 \n", + "\n", + " num_dependents \n", + "18 2.0 \n", + "764 1.0 \n", + "504 1.0 \n", + "454 1.0 \n", + "453 1.0 " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Check the tranform outputs features as expected\n", + "# (Note: transform output is an array, so we convert it\n", + "# back to dataframe for inspection)\n", + "pd.DataFrame(\n", + " index=train.index,\n", + " columns=cat_features + num_features,\n", + " data= col_transform.fit_transform(train)\n", + ").head()" + ] + }, + { + "cell_type": "markdown", + "id": "a3785c93-8830-4fa2-bb9d-31b6e8fecb01", + "metadata": {}, + "source": [ + "Finally, let's separate out the labels, and engineer them from categorical (\"good\" | \"bad\") to float (1.0 | 0.0). We do this for both the training and validation data." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "id": "30ebff90-a193-43a2-86fb-cf09e7d03777", + "metadata": {}, + "outputs": [], + "source": [ + "# Make \"class\" target variable numeric\n", + "train_y = (train[\"class\"] == \"good\").astype(float)\n", + "validate_y = (validate[\"class\"] == \"good\").astype(float)" + ] + }, + { + "cell_type": "markdown", + "id": "b052f6b2-2a34-441d-8a5f-2aad4e4db022", + "metadata": {}, + "source": [ + "### Train the Model" + ] + }, + { + "cell_type": "markdown", + "id": "c4f14590-31f4-4680-b1a1-75755a78513e", + "metadata": {}, + "source": [ + "Now that the features are prepared, we can train (fit) our baseline model on the feature data." + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "id": "0ff48f34-dbb6-4221-aefc-3c9b3f9da3e3", + "metadata": {}, + "outputs": [], + "source": [ + "# Specify the model\n", + "rf_model = RandomForestClassifier(\n", + " n_estimators=400,\n", + " criterion=\"entropy\",\n", + " max_depth=4,\n", + " min_samples_leaf=10,\n", + " class_weight={0:5, 1:1},\n", + " random_state=SEED\n", + ")\n", + "\n", + "# Package transform and model in pipeline\n", + "model = Pipeline([(\"transform\", col_transform), (\"rf_model\", rf_model)])" + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "id": "1d6ef38a-23b0-4056-a108-960495521164", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
Pipeline(steps=[('transform',\n",
+       "                 ColumnTransformer(transformers=[('cat_features',\n",
+       "                                                  OrdinalEncoder(),\n",
+       "                                                  ['checking_status',\n",
+       "                                                   'housing']),\n",
+       "                                                 ('num_features', 'passthrough',\n",
+       "                                                  ['duration', 'credit_amount',\n",
+       "                                                   'installment_commitment',\n",
+       "                                                   'residence_since', 'age',\n",
+       "                                                   'existing_credits',\n",
+       "                                                   'num_dependents'])])),\n",
+       "                ('rf_model',\n",
+       "                 RandomForestClassifier(class_weight={0: 5, 1: 1},\n",
+       "                                        criterion='entropy', max_depth=4,\n",
+       "                                        min_samples_leaf=10, n_estimators=400,\n",
+       "                                        random_state=142))])
In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook.
On GitHub, the HTML representation is unable to render, please try loading this page with nbviewer.org.
" + ], + "text/plain": [ + "Pipeline(steps=[('transform',\n", + " ColumnTransformer(transformers=[('cat_features',\n", + " OrdinalEncoder(),\n", + " ['checking_status',\n", + " 'housing']),\n", + " ('num_features', 'passthrough',\n", + " ['duration', 'credit_amount',\n", + " 'installment_commitment',\n", + " 'residence_since', 'age',\n", + " 'existing_credits',\n", + " 'num_dependents'])])),\n", + " ('rf_model',\n", + " RandomForestClassifier(class_weight={0: 5, 1: 1},\n", + " criterion='entropy', max_depth=4,\n", + " min_samples_leaf=10, n_estimators=400,\n", + " random_state=142))])" + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Fit the model\n", + "model.fit(train, train_y)" + ] + }, + { + "cell_type": "markdown", + "id": "73c45c39-9d8e-4f76-aca5-9f0c1568d263", + "metadata": {}, + "source": [ + "### Evaluate the Model" + ] + }, + { + "cell_type": "markdown", + "id": "ef58d432-80ba-428f-b59f-621a9e53b331", + "metadata": {}, + "source": [ + "Let's evaluate our baseline model performance. With credit risk, recall is going to be an important measure to look at. We compare the performance on the training data, with the performance on the validation data through a classification report." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "id": "8c5472f6-2ddc-437d-8102-4d5bd2c9f39c", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " 0.0 0.42 0.92 0.58 232\n", + " 1.0 0.94 0.49 0.64 568\n", + "\n", + " accuracy 0.61 800\n", + " macro avg 0.68 0.70 0.61 800\n", + "weighted avg 0.79 0.61 0.63 800\n", + "\n" + ] + } + ], + "source": [ + "# Evaluate training set performance\n", + "train_preds = model.predict(train)\n", + "print(classification_report(train_y, train_preds))" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "id": "c296bbd3-603e-4615-abbe-2689ebcf5d8c", + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + " precision recall f1-score support\n", + "\n", + " 0.0 0.46 0.87 0.61 68\n", + " 1.0 0.88 0.48 0.62 132\n", + "\n", + " accuracy 0.61 200\n", + " macro avg 0.67 0.68 0.61 200\n", + "weighted avg 0.74 0.61 0.62 200\n", + "\n" + ] + } + ], + "source": [ + "# Evaluate validation data performance\n", + "print(classification_report(validate_y, model.predict(validate)))" + ] + }, + { + "cell_type": "markdown", + "id": "d57ffbdc-f0b3-4fb6-9575-5acd983082cf", + "metadata": {}, + "source": [ + "The recall on the validation set for bad loans (0 class) is 0.87, meaning that the model correctly identified close to 90% of the bad loans. However, the precision of 0.46 tells us that the model is also classifying many loans that were actually good as bad. Precision and recall are technical metrics. In order to truly assess the models value, we would need feedback from the business side on the impact of misclassifications (for both good and bad loans).\n", + "\n", + "The difference in performance on the training vs. validation data, tells us that the model is slightly overfitting the data. Remember that this is just a quick baseline model. To improve further, we could do things like:\n", + "- gather more data\n", + "- engineer features\n", + "- experiment with hyperparameter settings\n", + "- experiment with other model types\n", + "\n", + "In fact, this is just a start. Creating AI models that meet business needs often requires a lot of guided experimentation." + ] + }, + { + "cell_type": "markdown", + "id": "0378d21a-d6db-42f9-851a-ce71f68c6802", + "metadata": {}, + "source": [ + "### Save the Model" + ] + }, + { + "cell_type": "markdown", + "id": "4450a328-f00c-4579-8e08-b2ebe5046961", + "metadata": {}, + "source": [ + "The last thing we do is save our trained model, so that we can pick it up later in the serving environment." + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "id": "da7a7906-d54f-4f2d-9803-6c82c86b28ad", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "['rf_model.pkl']" + ] + }, + "execution_count": 21, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Save the model to a pickle file\n", + "joblib.dump(model, \"rf_model.pkl\")" + ] + }, + { + "cell_type": "markdown", + "id": "299588b8-ab67-4155-97a9-770e8e4a7476", + "metadata": {}, + "source": [ + "In the next notebook, [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb), we will load the trained model and request predictions, with input features provided by the Feast online feature server." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb b/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb new file mode 100644 index 00000000000..f263dd6cd7b --- /dev/null +++ b/examples/credit-risk-end-to-end/04_Credit_Risk_Model_Serving.ipynb @@ -0,0 +1,697 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "9c870dcb-c66d-454d-a3fa-5f9a723bf8af", + "metadata": {}, + "source": [ + "
" + ] + }, + { + "cell_type": "markdown", + "id": "339ab741-ac90-4763-9971-3b274f6a90b4", + "metadata": {}, + "source": [ + "# Credit Risk Model Serving" + ] + }, + { + "cell_type": "markdown", + "id": "31d29794-4c33-4bc1-9bb4-e238c59f882d", + "metadata": {}, + "source": [ + "### Introduction" + ] + }, + { + "cell_type": "markdown", + "id": "d6553fe7-5427-4ecc-b638-615b47acf1a8", + "metadata": {}, + "source": [ + "Model serving is an exciting part of AI/ML. All of our previous work was building to this phase where we can actually serve loan predictions. \n", + "\n", + "So what role does Feast play in model serving? We've already seen that Feast can \"materialize\" data from the training offline store to the serving online store. This comes in handy because many models need contextual features at inference time. \n", + "\n", + "With this example, we can imagine a scenario something like this:\n", + "1. A bank customer submits a loan application on a website. \n", + "2. The website backend requests features, supplying the customer's ID as input.\n", + "3. The backend retrieves feature data for the ID in question.\n", + "4. The backend submits the feature data to the model to obtain a prediction.\n", + "5. The backend uses the prediction to make a decision.\n", + "6. The response is recorded and made available to the user.\n", + "\n", + "With online requests like this, time and resource usage often matter a lot. Feast facilitates quickly retrieving the correct feature data.\n", + "\n", + "In real-life, some of the contextual feature data points could be requested from the user, while others are retrieved from data sources. While outside the scope of this example, Feast does facilitate retrieving request data, and joining it with feature data. (See [Request Source](https://rtd.feast.dev/en/master/#request-source)).\n", + "\n", + "In this notebook, we request feature data from the online store for a small batch of users. We then get outcome predictions from our trained model. This notebook is a continuation of the work done in the previous notebooks; it comes as the step after [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + }, + { + "cell_type": "markdown", + "id": "53818109-c357-435f-8a8b-2a62982fa9a8", + "metadata": {}, + "source": [ + "### Setup" + ] + }, + { + "cell_type": "markdown", + "id": "92b5ab1b-186d-4b76-aac7-9b5110f8673e", + "metadata": {}, + "source": [ + "*The following code assumes that you have read the example README.md file, and that you have setup an environment where the code can be run. Please make sure you have addressed the prerequisite needs.*" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "378189ed-e967-4b2b-b591-aab980a685b3", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import os\n", + "import joblib\n", + "import json\n", + "import requests\n", + "import warnings\n", + "import pandas as pd\n", + "\n", + "from feast import FeatureStore" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "ea90edb2-16f0-4d40-a280-4e6ea79ea5be", + "metadata": {}, + "outputs": [], + "source": [ + "# ingnore warnings\n", + "warnings.filterwarnings(action=\"ignore\")" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "55f8ed91-7c13-44f7-a294-b6cacd43f8db", + "metadata": {}, + "outputs": [], + "source": [ + "# Load the model\n", + "model = joblib.load(\"rf_model.pkl\")" + ] + }, + { + "cell_type": "markdown", + "id": "3093e1b6-66d9-4936-b197-d853631914db", + "metadata": {}, + "source": [ + "### Query Feast Online Server for Feature Data" + ] + }, + { + "cell_type": "markdown", + "id": "2b5bbc4a-e2d3-4b7b-8309-434ff3b3e2cf", + "metadata": {}, + "source": [ + "Here, we show two different ways to retrieve data from the online feature server. The first is using the Python `requests` library, and the second is using the Feast Python SDK.\n", + "\n", + "We can use the Python requests library to request feature data from the online feature server (that we deployed in notebook [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)). The request takes the form of an HTTP POST command sent to the server endpoint (`url`). We request the data we need by supplying the entity and feature information in the data payload. We also need to specify an `application/json` content type in the request header." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "c6fd4f1a-917b-4a98-9bf6-101b4a074b64", + "metadata": {}, + "outputs": [], + "source": [ + "# ID examples\n", + "ids = [18, 764, 504, 454, 453, 0, 1, 2, 3, 4, 5, 6, 7, 8]\n", + "\n", + "# Submit get_online_features request to Feast online store server\n", + "response = requests.post(\n", + " url=\"http://localhost:6566/get-online-features\",\n", + " headers = {'Content-Type': 'application/json'},\n", + " data=json.dumps({\n", + " \"entities\": {\"ID\": ids},\n", + " \"features\": [\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\"\n", + " ]\n", + " })\n", + ")" + ] + }, + { + "cell_type": "markdown", + "id": "8e616a52-c18c-44a9-9e63-3aba071d7e79", + "metadata": {}, + "source": [ + "The response is returned as JSON, with feature values for each of the IDs." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "cf8948b7-4ed7-4c45-8acf-462331d9e4d2", + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "'{\"metadata\":{\"feature_names\":[\"ID\",\"checking_status\",\"duration\",\"installment_commitment\",\"credit_amount\",\"residence_since\",\"num_dependents\",\"age\",\"housing\",\"existing_credits\"]},\"results\":[{\"values\":[18,764,504,454,453,0,1,2,3,4,5,6,7,8],\"statuses\":[\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\"],\"event_timestamps\":[\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\",\"1970-01-01T00:00:00Z\"]},{\"values\":[\"0<=X<200\",\"no checking\",\"<0\",\"<0\",\"no checking\",\"<0\",\"0<=X<200\",\"no checking\",\"<0\",\"<0\",\"no checking\",\"no checking\",\"0<=X<200\",\"no checking\"],\"statuses\":[\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",\"PRESENT\",'" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Show first 1000 characters of response\n", + "response.text[:1000]" + ] + }, + { + "cell_type": "markdown", + "id": "c719f702-578a-4f35-b8ff-e41707cda23e", + "metadata": {}, + "source": [ + "As the response data comes in JSON format, there is a little formatting required to organize the data into a dataframe with one record per row (and features as columns)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "b992063d-8d83-4bf7-8153-f690b0410359", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDchecking_statusdurationinstallment_commitmentcredit_amountresidence_sincenum_dependentsagehousingexisting_credits
0180<=X<20024.04.012579.02.01.044.0for free1.0
1764no checking24.04.02463.03.01.027.0own2.0
2504<024.04.01207.04.01.024.0rent1.0
\n", + "
" + ], + "text/plain": [ + " ID checking_status duration installment_commitment credit_amount \\\n", + "0 18 0<=X<200 24.0 4.0 12579.0 \n", + "1 764 no checking 24.0 4.0 2463.0 \n", + "2 504 <0 24.0 4.0 1207.0 \n", + "\n", + " residence_since num_dependents age housing existing_credits \n", + "0 2.0 1.0 44.0 for free 1.0 \n", + "1 3.0 1.0 27.0 own 2.0 \n", + "2 4.0 1.0 24.0 rent 1.0 " + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Inspect the response\n", + "resp_data = json.loads(response.text)\n", + "\n", + "# Transform JSON into dataframe\n", + "records = pd.DataFrame(\n", + " columns=resp_data[\"metadata\"][\"feature_names\"], \n", + " data=[[r[\"values\"][i] for r in resp_data[\"results\"]] for i in range(len(ids))]\n", + ")\n", + "records.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "6db9b8ac-146e-40d3-b35a-cf4f4b6bbc8a", + "metadata": {}, + "source": [ + "Now, let's see how we can do the same with the Feast Python SDK. Note that we instantiate our `FeatureStore` object with the configuration that we set up in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb), by pointing to the `./Feature_Store` directory." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "765dc62b-e1e7-45fe-88b4-cc0235519ff8", + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future.\n", + "WARNING:root:Cannot use sqlite_vec for vector search\n" + ] + } + ], + "source": [ + "# Instantiate FeatureStore object\n", + "store = FeatureStore(repo_path=\"./Feature_Store\")\n", + "\n", + "# Retrieve features\n", + "records = store.get_online_features(\n", + " entity_rows=[{\"ID\":v} for v in ids],\n", + " features=[\n", + " \"data_a:duration\",\n", + " \"data_a:credit_amount\",\n", + " \"data_a:installment_commitment\",\n", + " \"data_a:checking_status\",\n", + " \"data_b:residence_since\",\n", + " \"data_b:age\",\n", + " \"data_b:existing_credits\",\n", + " \"data_b:num_dependents\",\n", + " \"data_b:housing\" \n", + " ]\n", + ").to_df()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "1d214e55-df0b-460d-936c-8951f7365a93", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDcredit_amountinstallment_commitmentchecking_statusdurationnum_dependentshousingageresidence_sinceexisting_credits
01812579.04.00<=X<20024.01.0for free44.02.01.0
17642463.04.0no checking24.01.0own27.03.02.0
25041207.04.0<024.01.0rent24.04.01.0
\n", + "
" + ], + "text/plain": [ + " ID credit_amount installment_commitment checking_status duration \\\n", + "0 18 12579.0 4.0 0<=X<200 24.0 \n", + "1 764 2463.0 4.0 no checking 24.0 \n", + "2 504 1207.0 4.0 <0 24.0 \n", + "\n", + " num_dependents housing age residence_since existing_credits \n", + "0 1.0 for free 44.0 2.0 1.0 \n", + "1 1.0 own 27.0 3.0 2.0 \n", + "2 1.0 rent 24.0 4.0 1.0 " + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "records.head(3)" + ] + }, + { + "cell_type": "markdown", + "id": "fd828758-6c57-4f9e-bbda-3983b6579da2", + "metadata": {}, + "source": [ + "### Get Predictions from the Model" + ] + }, + { + "cell_type": "markdown", + "id": "f446d7ec-0dae-409a-82a2-c0d7016c2001", + "metadata": {}, + "source": [ + "Now we can request predictions from our trained model. \n", + "\n", + "For convenience, we output the predictions along with the implied loan designations. Remember that these are predictions on loan outcomes, given context data from the loan application process. Since we have access to the actual `class` outcomes, we display those as well to see how the model did.|" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "70203f7b-f1e5-46ba-8623-f10bf3a5abf8", + "metadata": {}, + "outputs": [], + "source": [ + "# Get predictions from the model\n", + "preds = model.predict(records)" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "27001dde-8bdb-4de1-8c33-a76f030748e0", + "metadata": {}, + "outputs": [], + "source": [ + "# Load labels\n", + "labels = pd.read_parquet(\"Feature_Store/data/labels.parquet\")" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "ddc958e8-8ff8-49b1-ac10-fc965f3bf21c", + "metadata": {}, + "outputs": [ + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
IDPredictionLoan_DesignationTrue_Value
18180.0badbad
7647641.0goodgood
5045040.0badbad
4544540.0badbad
4534531.0goodgood
001.0goodgood
110.0badbad
221.0goodgood
330.0badgood
440.0badbad
551.0goodgood
661.0goodgood
770.0badgood
881.0goodgood
\n", + "
" + ], + "text/plain": [ + " ID Prediction Loan_Designation True_Value\n", + "18 18 0.0 bad bad\n", + "764 764 1.0 good good\n", + "504 504 0.0 bad bad\n", + "454 454 0.0 bad bad\n", + "453 453 1.0 good good\n", + "0 0 1.0 good good\n", + "1 1 0.0 bad bad\n", + "2 2 1.0 good good\n", + "3 3 0.0 bad good\n", + "4 4 0.0 bad bad\n", + "5 5 1.0 good good\n", + "6 6 1.0 good good\n", + "7 7 0.0 bad good\n", + "8 8 1.0 good good" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# Show preds\n", + "pd.DataFrame({\n", + " \"ID\": ids,\n", + " \"Prediction\": preds,\n", + " \"Loan_Designation\": [\"bad\" if i==0.0 else \"good\" for i in preds],\n", + " \"True_Value\": labels.loc[ids, \"class\"]\n", + "})" + ] + }, + { + "cell_type": "markdown", + "id": "87cd592a-61fc-4553-b84a-941d1785910d", + "metadata": {}, + "source": [ + "It's important to remember that the model's predictions are like educated guesses based on learned patterns. The model will get some predictions right, and other wrong. With the example records above, it looks like the model did pretty good! An AI/ML team's task is generally to make the model's predictions as useful as possible in helping the organization make decisions (for example, on loan approvals).\n", + "\n", + "In this case, we have a baseline model. While not ready for production, this model has set a low bar by which other models can be measured. Teams can also use a model like this to help with early testing, and with proving out things like pipelines and infrastructure before more sophisticated models are available.\n", + "\n", + "We have used Feast to query the feature data in support of model serving. The next notebook, [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb), cleans up resources created in this and previous notebooks." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb b/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb new file mode 100644 index 00000000000..846748dc425 --- /dev/null +++ b/examples/credit-risk-end-to-end/05_Credit_Risk_Cleanup.ipynb @@ -0,0 +1,296 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "id": "cf46ec61-7914-4677-b12b-a9e478e88d3f", + "metadata": {}, + "source": [ + "# Credit Risk Cleanup" + ] + }, + { + "cell_type": "markdown", + "id": "6ae8aaec-e01d-48d3-b768-98661ad1ec85", + "metadata": {}, + "source": [ + "Run this notebook if you are done experimenting with this demo, or if you wish to start again with a clean slate.\n", + "\n", + "**RUNNING THE FOLLOWING CODE WILL REMOVE FILES AND PROCESSES CREATED BY THE PREVIOUS EXAMPLE NOTEBOOKS.**\n", + "\n", + "The notebook progresses in reverse order of how the files and processes were added. (The reverse order makes it possible to partially revert changes by running cells up to a certain point.)" + ] + }, + { + "cell_type": "markdown", + "id": "6feaa771-4226-459f-b6dd-214024cb5c7c", + "metadata": {}, + "source": [ + "#### Setup" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "id": "20a39e94-920d-4108-aa6b-1e29d2224f71", + "metadata": {}, + "outputs": [], + "source": [ + "# Imports\n", + "import os\n", + "import time\n", + "import psutil" + ] + }, + { + "cell_type": "markdown", + "id": "3f124260-a8b2-475d-9103-8d336c543fce", + "metadata": {}, + "source": [ + "#### Remove Trained Model File" + ] + }, + { + "cell_type": "markdown", + "id": "f7a05a2b-9a26-4722-a526-84da99fc0b29", + "metadata": {}, + "source": [ + "This removes the model that was created and saved in [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "id": "a6b21063-ea43-4329-be0c-c1644c705db2", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove the model file that was saved in model training.\n", + "model_path = \"./rf_model.pkl\"\n", + "os.remove(model_path)" + ] + }, + { + "cell_type": "markdown", + "id": "ed97c24a-8f25-4e77-9037-f9cf4ad68dfa", + "metadata": {}, + "source": [ + "#### Shutdown Servers" + ] + }, + { + "cell_type": "markdown", + "id": "2f825d10-c13d-4701-b102-e15ad1c0bd3b", + "metadata": {}, + "source": [ + "Shut down the servers that were launched in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb); also remove the `server_proc.txt` that held the process PIDs." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "id": "66db4d46-a895-4041-ad87-ab0a77f13211", + "metadata": {}, + "outputs": [], + "source": [ + "# Load server process objects\n", + "server_pids = open(\"server_proc.txt\").readlines()\n", + "offline_server_proc = psutil.Process(int(server_pids[0].strip()))\n", + "online_server_proc = psutil.Process(int(server_pids[1].strip()))" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "id": "001fd472-2e28-499e-9eac-0a16ad8187a0", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online server : psutil.Process(pid=44621, name='python3.11', status='running', started='14:19:05')\n", + "Online server is running: True\n", + "\n", + "Offline server PID: psutil.Process(pid=44594, name='python3.11', status='running', started='14:19:03')\n", + "Offline server is running: True\n" + ] + } + ], + "source": [ + "# Verify if servers are running\n", + "def verify_servers():\n", + " # online server\n", + " print(f\"Online server : {online_server_proc}\")\n", + " print(f\"Online server is running: {online_server_proc.is_running()}\", end='\\n\\n')\n", + " # offline server\n", + " print(f\"Offline server PID: {offline_server_proc}\")\n", + " print(f\"Offline server is running: {offline_server_proc.is_running()}\")\n", + " \n", + "verify_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "id": "68376350-790a-4e7e-9325-c7de4d22e54b", + "metadata": {}, + "outputs": [], + "source": [ + "# Terminate offline server\n", + "offline_server_proc.terminate()" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "id": "446b6bf9-aef2-4873-b477-8bf595a8eabf", + "metadata": {}, + "outputs": [], + "source": [ + "# Terminate online server (master and worker)\n", + "for child in online_server_proc.children(recursive=True):\n", + " child.terminate()\n", + "online_server_proc.terminate()\n", + "time.sleep(2)" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "id": "774827f6-4dcd-495b-b5c5-186b97148619", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Online server : psutil.Process(pid=44621, name='python3.11', status='terminated', started='14:19:05')\n", + "Online server is running: False\n", + "\n", + "Offline server PID: psutil.Process(pid=44594, name='python3.11', status='terminated', started='14:19:03')\n", + "Offline server is running: False\n" + ] + } + ], + "source": [ + "# Verify termination\n", + "verify_servers()" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "id": "f8a155e4-23b3-4fb3-b868-02ba2e0a4a31", + "metadata": {}, + "outputs": [], + "source": [ + "# Remove server_proc.txt (file for keeping track of pids)\n", + "os.remove(\"server_proc.txt\")" + ] + }, + { + "cell_type": "markdown", + "id": "ed7d6f25-d255-4986-9cf2-9876f6c558cc", + "metadata": {}, + "source": [ + "#### Remove Feast Applied Configuration Files" + ] + }, + { + "cell_type": "markdown", + "id": "d73efe15-a1d9-459b-8142-835dc2bf1c9f", + "metadata": {}, + "source": [ + "Remove the registry and online store (SQLite) files created on`feast apply` created in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "id": "0f13a4ac-d2ad-462b-b65e-4266b7cb4922", + "metadata": {}, + "outputs": [], + "source": [ + "os.remove(\"Feature_Store/data/online_store.db\")\n", + "os.remove(\"Feature_Store/data/registry.db\")" + ] + }, + { + "cell_type": "markdown", + "id": "eb0494cd-0143-4f5f-b7d6-9675e1403d9f", + "metadata": {}, + "source": [ + "#### Remove Feast Configuration Files" + ] + }, + { + "cell_type": "markdown", + "id": "86c33ac7-9e1f-4798-9f14-77773a1c13bd", + "metadata": {}, + "source": [ + "Remove the configution and feature definition files created in [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "id": "a747043f-05fe-4b44-979d-9b30565074ee", + "metadata": {}, + "outputs": [], + "source": [ + "os.remove(\"Feature_Store/feature_store.yaml\")\n", + "os.remove(\"Feature_Store/feature_definitions.py\")" + ] + }, + { + "cell_type": "markdown", + "id": "81975a0f-7fd6-4ed3-91cf-812946df4713", + "metadata": {}, + "source": [ + "#### Remove Data Files" + ] + }, + { + "cell_type": "markdown", + "id": "8182dc1e-d5c1-4739-b7c7-0620e93c5b64", + "metadata": {}, + "source": [ + "Remove the data files created in [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb)." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "id": "4ddb4fb2-fea1-4b70-8978-732af9a1cd3f", + "metadata": {}, + "outputs": [], + "source": [ + "for f in [\"data_a.parquet\", \"data_b.parquet\", \"labels.parquet\"]:\n", + " os.remove(f\"Feature_Store/data/{f}\")\n", + "os.rmdir(\"Feature_Store/data\")\n", + "os.rmdir(\"Feature_Store\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/examples/credit-risk-end-to-end/README.md b/examples/credit-risk-end-to-end/README.md new file mode 100644 index 00000000000..5f59c750784 --- /dev/null +++ b/examples/credit-risk-end-to-end/README.md @@ -0,0 +1,39 @@ + +![Feast_Logo](https://raw.githubusercontent.com/feast-dev/feast/master/docs/assets/feast_logo.png) + +# Feast Credit Risk Classification End-to-End Example + +This example starts with an [OpenML](https://openml.org) credit risk dataset, and walks through the steps of preparing the data, setting up feature store resources, and serving features; this is all done inside the paradigm of an ML workflow, with the goal of helping users understand how Feast fits in the progression from data preparation, to model training and model serving. + +The example is organized in five notebooks: +1. [01_Credit_Risk_Data_Prep.ipynb](01_Credit_Risk_Data_Prep.ipynb) +2. [02_Deploying_the_Feature_Store.ipynb](02_Deploying_the_Feature_Store.ipynb) +3. [03_Credit_Risk_Model_Training.ipynb](03_Credit_Risk_Model_Training.ipynb) +4. [04_Credit_Risk_Model_Serving.ipynb](04_Credit_Risk_Model_Serving.ipynb) +5. [05_Credit_Risk_Cleanup.ipynb](05_Credit_Risk_Cleanup.ipynb) + +Run the notebooks in order to progress through the example. See below for prerequisite setup steps. + +### Preparing your Environment +To run the example, install the Python dependencies. You may wish to do so inside a virtual environment. Open a command terminal, and run the following: + +``` +# create venv-example virtual environment +python -m venv venv-example +# activate environment +source venv-example/bin/activate +``` + +Install the Python dependencies: +``` +pip install -r requirements.txt +``` + +Note that this example was tested with Python 3.11, but it should also work with other similar versions. + +### Running the Notebooks +Once you have installed the Python dependencies, you can run the example notebooks. To run the notebooks locally, execute the following command in a terminal window: + +```jupyter notebook``` + +You should see a browser window open a page where you can navigate to the example notebook (.ipynb) files and open them. diff --git a/examples/credit-risk-end-to-end/requirements.txt b/examples/credit-risk-end-to-end/requirements.txt new file mode 100644 index 00000000000..8b9b1313e78 --- /dev/null +++ b/examples/credit-risk-end-to-end/requirements.txt @@ -0,0 +1,6 @@ +feast +jupyter==1.1.1 +scikit-learn==1.5.2 +pandas==2.2.3 +matplotlib==3.9.2 +seaborn==0.13.2 \ No newline at end of file diff --git a/examples/python-helm-demo/README.md b/examples/python-helm-demo/README.md index 90469e746d4..078550ae392 100644 --- a/examples/python-helm-demo/README.md +++ b/examples/python-helm-demo/README.md @@ -3,87 +3,168 @@ For this tutorial, we set up Feast with Redis. -We use the Feast CLI to register and materialize features, and then retrieving via a Feast Python feature server deployed in Kubernetes +We use the Feast CLI to register and materialize features from the current machine, and then retrieving via a +Feast Python feature server deployed in Kubernetes ## First, let's set up a Redis cluster 1. Start minikube (`minikube start`) -2. Use helm to install a default Redis cluster +1. Use helm to install a default Redis cluster ```bash helm repo add bitnami https://charts.bitnami.com/bitnami helm repo update helm install my-redis bitnami/redis ``` ![](redis-screenshot.png) -3. Port forward Redis so we can materialize features to it +1. Port forward Redis so we can materialize features to it ```bash kubectl port-forward --namespace default svc/my-redis-master 6379:6379 ``` -4. Get your Redis password using the command (pasted below for convenience). We'll need this to tell Feast how to communicate with the cluster. +1. Get your Redis password using the command (pasted below for convenience). We'll need this to tell Feast how to communicate with the cluster. ```bash export REDIS_PASSWORD=$(kubectl get secret --namespace default my-redis -o jsonpath="{.data.redis-password}" | base64 --decode) echo $REDIS_PASSWORD ``` +## Then, let's set up a MinIO S3 store +Manifests have been taken from [Deploy Minio in your project](https://ai-on-openshift.io/tools-and-applications/minio/minio/#deploy-minio-in-your-project). + +1. Deploy MinIO instance: + ``` + kubectl apply -f minio-dev.yaml + ``` + +1. Forward the UI port: + ```console + kubectl port-forward svc/minio-service 9090:9090 + ``` +1. Login to (localhost:9090)[http://localhost:9090] as `minio`/`minio123` and create bucket called `feast-demo`. +1. Stop previous port forwarding and forward the API port instead: + ```console + kubectl port-forward svc/minio-service 9000:9000 + ``` + ## Next, we setup a local Feast repo -1. Install Feast with Redis dependencies `pip install "feast[redis]"` -2. Make a bucket in GCS (or S3) -3. The feature repo is already setup here, so you just need to swap in your GCS bucket and Redis credentials. - We need to modify the `feature_store.yaml`, which has two fields for you to replace: +1. Install Feast with Redis dependencies `pip install "feast[redis,aws]"` +1. The feature repo is already setup here, so you just need to swap in your Redis credentials. + We need to modify the `feature_store.yaml`, which has one field for you to replace: + ```console + sed "s/_REDIS_PASSWORD_/${REDIS_PASSWORD}/" feature_repo/feature_store.yaml.template > feature_repo/feature_store.yaml + cat feature_repo/feature_store.yaml + ``` + + Example repo: ```yaml - registry: gs://[YOUR GCS BUCKET]/demo-repo/registry.db + registry: s3://localhost:9000/feast-demo/registry.db project: feast_python_demo - provider: gcp + provider: local online_store: type: redis - # Note: this would normally be using instance URL's to access Redis - connection_string: localhost:6379,password=[YOUR PASSWORD] + connection_string: localhost:6379,password=**** offline_store: type: file entity_key_serialization_version: 2 ``` -4. Run `feast apply` from within the `feature_repo` directory to apply your local features to the remote registry - - Note: you may need to authenticate to gcloud first with `gcloud auth login` -5. Materialize features to the online store: +1. To run `feast apply` from the current machine we need to define the AWS credentials to connect the MinIO S3 store, which +are defined in [minio.env](./minio.env): + ```console + source minio.env + cd feature_repo + feast apply + ``` +1. Let's validate the setup by running some queries + ```console + feast entities list + feast feature-views list + ``` +1. Materialize features to the online store: ```bash + cd feature_repo CURRENT_TIME=$(date -u +"%Y-%m-%dT%H:%M:%S") feast materialize-incremental $CURRENT_TIME ``` ## Now let's setup the Feast Server -1. Add the gcp-auth addon to mount GCP credentials: - ```bash - minikube addons enable gcp-auth - ``` -2. Add Feast's Python/Go feature server chart repo +1. Add Feast's Python feature server chart repo ```bash helm repo add feast-charts https://feast-helm-charts.storage.googleapis.com helm repo update ``` -3. For this tutorial, because we don't have a direct hosted endpoint into Redis, we need to change `feature_store.yaml` to talk to the Kubernetes Redis service - ```bash - sed -i '' 's/localhost:6379/my-redis-master:6379/g' feature_store.yaml - ``` -4. Install the Feast helm chart: `helm install feast-release feast-charts/feast-feature-server --set feature_store_yaml_base64=$(base64 feature_store.yaml)` - > **Dev instructions**: if you're changing the java logic or chart, you can do - 1. `eval $(minikube docker-env)` - 2. `make build-feature-server-dev` - 3. `helm install feast-release ../../../infra/charts/feast-feature-server --set image.tag=dev --set feature_store_yaml_base64=$(base64 feature_store.yaml)` -5. (Optional): check logs of the server to make sure it’s working +1. For this tutorial, we'll use a predefined configuration where we just needs to inject the Redis service password: + ```console + sed "s/_REDIS_PASSWORD_/$REDIS_PASSWORD/" online_feature_store.yaml.template > online_feature_store.yaml + cat online_feature_store.yaml + ``` + As you see, the connection points to `my-redis-master:6379` instead of `localhost:6379`. + +1. Install the Feast helm chart: + ```console + helm upgrade --install feast-online feast-charts/feast-feature-server \ + --set fullnameOverride=online-server --set feast_mode=online \ + --set feature_store_yaml_base64=$(base64 -i 'online_feature_store.yaml') + ``` +1. Patch the deployment to include MinIO settings: + ```console + kubectl patch deployment online-server --type='json' -p='[ + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_ACCESS_KEY_ID", + "value": "minio" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_SECRET_ACCESS_KEY", + "value": "minio123" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "AWS_DEFAULT_REGION", + "value": "default" + } + }, + { + "op": "add", + "path": "/spec/template/spec/containers/0/env/-", + "value": { + "name": "FEAST_S3_ENDPOINT_URL", + "value": "http://minio-service:9000" + } + } + ]' + kubectl wait --for=condition=available deployment/online-server --timeout=2m + ``` +1. (Optional): check logs of the server to make sure it’s working ```bash - kubectl logs svc/feast-release-feast-feature-server + kubectl logs svc/online-server ``` -6. Port forward to expose the grpc endpoint: +1. Port forward to expose the grpc endpoint: ```bash - kubectl port-forward svc/feast-release-feast-feature-server 6566:80 + kubectl port-forward svc/online-server 6566:80 ``` -7. Run test fetches for online features:8. - - First: change back the Redis connection string to allow localhost connections to Redis +1. Run test fetches for online features:8. ```bash - sed -i '' 's/my-redis-master:6379/localhost:6379/g' feature_store.yaml + source minio.env + cd test + python test_python_fetch.py ``` - - Then run the included fetch script, which fetches both via the HTTP endpoint and for comparison, via the Python SDK - ```bash - python test_python_fetch.py + + Output example: + ```console + --- Online features with SDK --- + WARNING:root:_list_feature_views will make breaking changes. Please use _list_batch_feature_views instead. _list_feature_views will behave like _list_all_feature_views in the future. + conv_rate : [0.6799587607383728, 0.9761165976524353] + driver_id : [1001, 1002] + + --- Online features with HTTP endpoint --- + conv_rate : [0.67995876 0.9761166 ] + driver_id : [1001 1002] ``` \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet b/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet index 83b8c31aa51..ae8f17e45d3 100644 Binary files a/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet and b/examples/python-helm-demo/feature_repo/data/driver_stats_with_string.parquet differ diff --git a/examples/python-helm-demo/feature_repo/feature_store.yaml b/examples/python-helm-demo/feature_repo/feature_store.yaml deleted file mode 100644 index d49c0cbd0eb..00000000000 --- a/examples/python-helm-demo/feature_repo/feature_store.yaml +++ /dev/null @@ -1,10 +0,0 @@ -registry: gs://[YOUR GCS BUCKET]/demo-repo/registry.db -project: feast_python_demo -provider: gcp -online_store: - type: redis - # Note: this would normally be using instance URL's to access Redis - connection_string: localhost:6379,password=[YOUR PASSWORD] -offline_store: - type: file -entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/feature_store.yaml.template b/examples/python-helm-demo/feature_repo/feature_store.yaml.template new file mode 100644 index 00000000000..585ba23e63b --- /dev/null +++ b/examples/python-helm-demo/feature_repo/feature_store.yaml.template @@ -0,0 +1,9 @@ +registry: s3://feast-demo/registry.db +project: feast_python_demo +provider: local +online_store: + type: redis + connection_string: localhost:6379,password=_REDIS_PASSWORD_ +offline_store: + type: file +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/minio-dev.yaml b/examples/python-helm-demo/minio-dev.yaml new file mode 100644 index 00000000000..9285cbca983 --- /dev/null +++ b/examples/python-helm-demo/minio-dev.yaml @@ -0,0 +1,128 @@ +--- +kind: PersistentVolumeClaim +apiVersion: v1 +metadata: + name: minio-pvc +spec: + accessModes: + - ReadWriteOnce + resources: + requests: + storage: 20Gi + volumeMode: Filesystem +--- +kind: Secret +apiVersion: v1 +metadata: + name: minio-secret +stringData: + # change the username and password to your own values. + # ensure that the user is at least 3 characters long and the password at least 8 + minio_root_user: minio + minio_root_password: minio123 +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: minio +spec: + replicas: 1 + selector: + matchLabels: + app: minio + template: + metadata: + labels: + app: minio + spec: + volumes: + - name: data + persistentVolumeClaim: + claimName: minio-pvc + containers: + - resources: + limits: + cpu: 250m + memory: 1Gi + requests: + cpu: 20m + memory: 100Mi + readinessProbe: + tcpSocket: + port: 9000 + initialDelaySeconds: 5 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + terminationMessagePath: /dev/termination-log + name: minio + livenessProbe: + tcpSocket: + port: 9000 + initialDelaySeconds: 30 + timeoutSeconds: 1 + periodSeconds: 5 + successThreshold: 1 + failureThreshold: 3 + env: + - name: MINIO_ROOT_USER + valueFrom: + secretKeyRef: + name: minio-secret + key: minio_root_user + - name: MINIO_ROOT_PASSWORD + valueFrom: + secretKeyRef: + name: minio-secret + key: minio_root_password + ports: + - containerPort: 9000 + protocol: TCP + - containerPort: 9090 + protocol: TCP + imagePullPolicy: IfNotPresent + volumeMounts: + - name: data + mountPath: /data + subPath: minio + terminationMessagePolicy: File + image: >- + quay.io/minio/minio:RELEASE.2023-06-19T19-52-50Z + args: + - server + - /data + - --console-address + - :9090 + restartPolicy: Always + terminationGracePeriodSeconds: 30 + dnsPolicy: ClusterFirst + securityContext: {} + schedulerName: default-scheduler + strategy: + type: Recreate + revisionHistoryLimit: 10 + progressDeadlineSeconds: 600 +--- +kind: Service +apiVersion: v1 +metadata: + name: minio-service +spec: + ipFamilies: + - IPv4 + ports: + - name: api + protocol: TCP + port: 9000 + targetPort: 9000 + - name: ui + protocol: TCP + port: 9090 + targetPort: 9090 + internalTrafficPolicy: Cluster + type: ClusterIP + ipFamilyPolicy: SingleStack + sessionAffinity: None + selector: + app: minio \ No newline at end of file diff --git a/examples/python-helm-demo/minio.env b/examples/python-helm-demo/minio.env new file mode 100644 index 00000000000..b19ec5083f5 --- /dev/null +++ b/examples/python-helm-demo/minio.env @@ -0,0 +1,7 @@ +export AWS_ACCESS_KEY_ID=minio +export AWS_DEFAULT_REGION=default +#export AWS_S3_BUCKET=feast-demo +#export AWS_S3_ENDPOINT=http://localhost:9000 +export FEAST_S3_ENDPOINT_URL=http://localhost:9000 +export AWS_SECRET_ACCESS_KEY=minio123 + diff --git a/examples/python-helm-demo/online_feature_store.yaml.template b/examples/python-helm-demo/online_feature_store.yaml.template new file mode 100644 index 00000000000..7acb9582c51 --- /dev/null +++ b/examples/python-helm-demo/online_feature_store.yaml.template @@ -0,0 +1,7 @@ +project: feast_python_demo +provider: local +registry: s3://feast-demo/registry.db +online_store: + type: redis + connection_string: my-redis-master:6379,password=_REDIS_PASSWORD_ +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/test/feature_store.yaml b/examples/python-helm-demo/test/feature_store.yaml new file mode 100644 index 00000000000..13e99873ee7 --- /dev/null +++ b/examples/python-helm-demo/test/feature_store.yaml @@ -0,0 +1,7 @@ +registry: s3://feast-demo/registry.db +project: feast_python_demo +provider: local +online_store: + path: http://localhost:6566 + type: remote +entity_key_serialization_version: 2 \ No newline at end of file diff --git a/examples/python-helm-demo/feature_repo/test_python_fetch.py b/examples/python-helm-demo/test/test_python_fetch.py similarity index 73% rename from examples/python-helm-demo/feature_repo/test_python_fetch.py rename to examples/python-helm-demo/test/test_python_fetch.py index f9c7c62f4fd..715912422f3 100644 --- a/examples/python-helm-demo/feature_repo/test_python_fetch.py +++ b/examples/python-helm-demo/test/test_python_fetch.py @@ -1,6 +1,7 @@ from feast import FeatureStore import requests import json +import pandas as pd def run_demo_http(): @@ -14,7 +15,14 @@ def run_demo_http(): r = requests.post( "http://localhost:6566/get-online-features", data=json.dumps(online_request) ) - print(json.dumps(r.json(), indent=4, sort_keys=True)) + + resp_data = json.loads(r.text) + records = pd.DataFrame.from_records( + columns=resp_data["metadata"]["feature_names"], + data=[[r["values"][i] for r in resp_data["results"]] for i in range(len(resp_data["results"]))] + ) + for col in sorted(records.columns): + print(col, " : ", records[col].values) def run_demo_sdk(): diff --git a/infra/feast-operator/api/v1alpha1/featurestore_types.go b/infra/feast-operator/api/v1alpha1/featurestore_types.go index 635912a1b5f..f73c7fc6a40 100644 --- a/infra/feast-operator/api/v1alpha1/featurestore_types.go +++ b/infra/feast-operator/api/v1alpha1/featurestore_types.go @@ -65,7 +65,7 @@ type FeatureStoreSpec struct { AuthzConfig *AuthzConfig `json:"authz,omitempty"` } -// FeatureStoreServices defines the desired feast service deployments. ephemeral registry is deployed by default. +// FeatureStoreServices defines the desired feast services. An ephemeral registry is deployed by default. type FeatureStoreServices struct { OfflineStore *OfflineStore `json:"offlineStore,omitempty"` OnlineStore *OnlineStore `json:"onlineStore,omitempty"` @@ -74,22 +74,15 @@ type FeatureStoreServices struct { // OfflineStore configures the deployed offline store service type OfflineStore struct { - StoreServiceConfigs `json:",inline"` - Persistence *OfflineStorePersistence `json:"persistence,omitempty"` - TLS *OfflineTlsConfigs `json:"tls,omitempty"` + ServiceConfigs `json:",inline"` + Persistence *OfflineStorePersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the offline store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical LogLevel string `json:"logLevel,omitempty"` } -// OfflineTlsConfigs configures server TLS for the offline feast service. in an openshift cluster, this is configured by default using service serving certificates. -type OfflineTlsConfigs struct { - TlsConfigs `json:",inline"` - // verify the client TLS certificate. - VerifyClient *bool `json:"verifyClient,omitempty"` -} - // OfflineStorePersistence configures the persistence settings for the offline store service // +kubebuilder:validation:XValidation:rule="[has(self.file), has(self.store)].exists_one(c, c)",message="One selection required between file or store." type OfflineStorePersistence struct { @@ -134,9 +127,9 @@ var ValidOfflineStoreDBStorePersistenceTypes = []string{ // OnlineStore configures the deployed online store service type OnlineStore struct { - StoreServiceConfigs `json:",inline"` - Persistence *OnlineStorePersistence `json:"persistence,omitempty"` - TLS *TlsConfigs `json:"tls,omitempty"` + ServiceConfigs `json:",inline"` + Persistence *OnlineStorePersistence `json:"persistence,omitempty"` + TLS *TlsConfigs `json:"tls,omitempty"` // LogLevel sets the logging level for the online store service // Allowed values: "debug", "info", "warning", "error", "critical". // +kubebuilder:validation:Enum=debug;info;warning;error;critical @@ -153,7 +146,7 @@ type OnlineStorePersistence struct { // OnlineStoreFilePersistence configures the file-based persistence for the offline store service // +kubebuilder:validation:XValidation:rule="(!has(self.pvc) && has(self.path)) ? self.path.startsWith('/') : true",message="Ephemeral stores must have absolute paths." // +kubebuilder:validation:XValidation:rule="(has(self.pvc) && has(self.path)) ? !self.path.startsWith('/') : true",message="PVC path must be a file name only, with no slashes." -// +kubebuilder:validation:XValidation:rule="has(self.path) && !self.path.startsWith('s3://') && !self.path.startsWith('gs://')",message="Online store does not support S3 or GS buckets." +// +kubebuilder:validation:XValidation:rule="has(self.path) ? !(self.path.startsWith('s3://') || self.path.startsWith('gs://')) : true",message="Online store does not support S3 or GS buckets." type OnlineStoreFilePersistence struct { Path string `json:"path,omitempty"` PvcConfig *PvcConfig `json:"pvc,omitempty"` @@ -242,13 +235,15 @@ type PvcConfig struct { Create *PvcCreate `json:"create,omitempty"` // MountPath within the container at which the volume should be mounted. // Must start by "/" and cannot contain ':'. - MountPath string `json:"mountPath,omitempty"` + MountPath string `json:"mountPath"` } // PvcCreate defines the immutable settings to create a new PVC mounted at the given path. -// The PVC name is the same as the associated deployment name. +// The PVC name is the same as the associated deployment & feast service name. // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="PvcCreate is immutable" type PvcCreate struct { + // AccessModes k8s persistent volume access modes. Defaults to ["ReadWriteOnce"]. + AccessModes []corev1.PersistentVolumeAccessMode `json:"accessModes,omitempty"` // StorageClassName is the name of an existing StorageClass to which this persistent volume belongs. Empty value // means that this volume does not belong to any StorageClass and the cluster default will be used. StorageClassName *string `json:"storageClassName,omitempty"` @@ -297,14 +292,6 @@ type DefaultConfigs struct { Image *string `json:"image,omitempty"` } -// StoreServiceConfigs k8s deployment settings -type StoreServiceConfigs struct { - // Replicas determines the number of pods for the feast service. - // When Replicas > 1, persistence is recommended. - Replicas *int32 `json:"replicas,omitempty"` - ServiceConfigs `json:",inline"` -} - // OptionalConfigs k8s container settings that are optional type OptionalConfigs struct { Env *[]corev1.EnvVar `json:"env,omitempty"` diff --git a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go index bccf9ec5378..f1e05030880 100644 --- a/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/infra/feast-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -273,7 +273,7 @@ func (in *LocalRegistryConfig) DeepCopy() *LocalRegistryConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OfflineStore) DeepCopyInto(out *OfflineStore) { *out = *in - in.StoreServiceConfigs.DeepCopyInto(&out.StoreServiceConfigs) + in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(OfflineStorePersistence) @@ -281,7 +281,7 @@ func (in *OfflineStore) DeepCopyInto(out *OfflineStore) { } if in.TLS != nil { in, out := &in.TLS, &out.TLS - *out = new(OfflineTlsConfigs) + *out = new(TlsConfigs) (*in).DeepCopyInto(*out) } } @@ -357,27 +357,6 @@ func (in *OfflineStorePersistence) DeepCopy() *OfflineStorePersistence { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *OfflineTlsConfigs) DeepCopyInto(out *OfflineTlsConfigs) { - *out = *in - in.TlsConfigs.DeepCopyInto(&out.TlsConfigs) - if in.VerifyClient != nil { - in, out := &in.VerifyClient, &out.VerifyClient - *out = new(bool) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new OfflineTlsConfigs. -func (in *OfflineTlsConfigs) DeepCopy() *OfflineTlsConfigs { - if in == nil { - return nil - } - out := new(OfflineTlsConfigs) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OidcAuthz) DeepCopyInto(out *OidcAuthz) { *out = *in @@ -397,7 +376,7 @@ func (in *OidcAuthz) DeepCopy() *OidcAuthz { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *OnlineStore) DeepCopyInto(out *OnlineStore) { *out = *in - in.StoreServiceConfigs.DeepCopyInto(&out.StoreServiceConfigs) + in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) if in.Persistence != nil { in, out := &in.Persistence, &out.Persistence *out = new(OnlineStorePersistence) @@ -545,6 +524,11 @@ func (in *PvcConfig) DeepCopy() *PvcConfig { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PvcCreate) DeepCopyInto(out *PvcCreate) { *out = *in + if in.AccessModes != nil { + in, out := &in.AccessModes, &out.AccessModes + *out = make([]v1.PersistentVolumeAccessMode, len(*in)) + copy(*out, *in) + } if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName *out = new(string) @@ -737,27 +721,6 @@ func (in *ServiceHostnames) DeepCopy() *ServiceHostnames { return out } -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *StoreServiceConfigs) DeepCopyInto(out *StoreServiceConfigs) { - *out = *in - if in.Replicas != nil { - in, out := &in.Replicas, &out.Replicas - *out = new(int32) - **out = **in - } - in.ServiceConfigs.DeepCopyInto(&out.ServiceConfigs) -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StoreServiceConfigs. -func (in *StoreServiceConfigs) DeepCopy() *StoreServiceConfigs { - if in == nil { - return nil - } - out := new(StoreServiceConfigs) - in.DeepCopyInto(out) - return out -} - // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *TlsConfigs) DeepCopyInto(out *TlsConfigs) { *out = *in diff --git a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml index 74f0fd059e3..2cab2d8c5d8 100644 --- a/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml +++ b/infra/feast-operator/config/crd/bases/feast.dev_featurestores.yaml @@ -101,8 +101,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline store @@ -254,6 +254,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -312,6 +318,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -369,12 +377,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -432,9 +434,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for the - offline feast service. in an openshift cluster, this is - configured by default using service serving certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. properties: disable: description: will disable TLS for the feast service. useful @@ -464,9 +466,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -625,6 +624,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -683,6 +688,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -702,8 +709,8 @@ spec: rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -751,12 +758,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1009,6 +1010,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1067,6 +1074,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1354,8 +1363,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline @@ -1509,6 +1518,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1567,6 +1582,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1626,12 +1643,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1690,10 +1701,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for - the offline feast service. in an openshift cluster, - this is configured by default using service serving - certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. properties: disable: description: will disable TLS for the feast service. @@ -1723,9 +1733,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -1886,6 +1893,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1944,6 +1957,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1964,8 +1979,8 @@ spec: : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -2015,12 +2030,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -2279,6 +2288,13 @@ spec: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -2338,6 +2354,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between diff --git a/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml b/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml index b7c7412c0f0..15aa46c456c 100644 --- a/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml +++ b/infra/feast-operator/config/samples/v1alpha1_featurestore_pvc_persistence.yaml @@ -5,6 +5,7 @@ metadata: spec: feastProject: my_project services: + # demonstrates using a pre-existing PVC onlineStore: persistence: file: @@ -13,6 +14,7 @@ spec: ref: name: online-pvc mountPath: /data/online + # demonstrates specifying a storageClassName and storage size offlineStore: persistence: file: @@ -24,6 +26,7 @@ spec: requests: storage: 5Gi mountPath: /data/offline + # demonstrates letting the Operator create a PVC w/ defaults set registry: local: persistence: @@ -39,7 +42,7 @@ metadata: name: online-pvc spec: accessModes: - - ReadWriteMany + - ReadWriteOnce resources: requests: storage: 5Gi diff --git a/infra/feast-operator/dist/install.yaml b/infra/feast-operator/dist/install.yaml index f40c5caebb7..cd63b3df8d0 100644 --- a/infra/feast-operator/dist/install.yaml +++ b/infra/feast-operator/dist/install.yaml @@ -109,8 +109,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline store @@ -262,6 +262,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -320,6 +326,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -377,12 +385,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -440,9 +442,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for the - offline feast service. in an openshift cluster, this is - configured by default using service serving certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured by + default using service serving certificates. properties: disable: description: will disable TLS for the feast service. useful @@ -472,9 +474,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -633,6 +632,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent volume + access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -691,6 +696,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref and @@ -710,8 +717,8 @@ spec: rule: '(has(self.pvc) && has(self.path)) ? !self.path.startsWith(''/'') : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -759,12 +766,6 @@ spec: x-kubernetes-validations: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1017,6 +1018,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1075,6 +1082,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1362,8 +1371,8 @@ spec: pattern: ^[A-Za-z0-9][A-Za-z0-9_]*$ type: string services: - description: FeatureStoreServices defines the desired feast service - deployments. ephemeral registry is deployed by default. + description: FeatureStoreServices defines the desired feast services. + An ephemeral registry is deployed by default. properties: offlineStore: description: OfflineStore configures the deployed offline @@ -1517,6 +1526,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1575,6 +1590,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1634,12 +1651,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -1698,10 +1709,9 @@ spec: type: object type: object tls: - description: OfflineTlsConfigs configures server TLS for - the offline feast service. in an openshift cluster, - this is configured by default using service serving - certificates. + description: TlsConfigs configures server TLS for a feast + service. in an openshift cluster, this is configured + by default using service serving certificates. properties: disable: description: will disable TLS for the feast service. @@ -1731,9 +1741,6 @@ spec: type: string type: object x-kubernetes-map-type: atomic - verifyClient: - description: verify the client TLS certificate. - type: boolean type: object x-kubernetes-validations: - message: '`secretRef` required if `disable` is false.' @@ -1894,6 +1901,12 @@ spec: create: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -1952,6 +1965,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between ref @@ -1972,8 +1987,8 @@ spec: : true' - message: Online store does not support S3 or GS buckets. - rule: has(self.path) && !self.path.startsWith('s3://') - && !self.path.startsWith('gs://') + rule: 'has(self.path) ? !(self.path.startsWith(''s3://'') + || self.path.startsWith(''gs://'')) : true' store: description: OnlineStoreDBStorePersistence configures the DB store persistence for the offline store service @@ -2023,12 +2038,6 @@ spec: - message: One selection required between file or store. rule: '[has(self.file), has(self.store)].exists_one(c, c)' - replicas: - description: |- - Replicas determines the number of pods for the feast service. - When Replicas > 1, persistence is recommended. - format: int32 - type: integer resources: description: ResourceRequirements describes the compute resource requirements. @@ -2287,6 +2296,13 @@ spec: description: Settings for creating a new PVC properties: + accessModes: + description: AccessModes k8s persistent + volume access modes. Defaults to + ["ReadWriteOnce"]. + items: + type: string + type: array resources: description: |- Resources describes the storage resource requirements for a volume. @@ -2346,6 +2362,8 @@ spec: type: string type: object x-kubernetes-map-type: atomic + required: + - mountPath type: object x-kubernetes-validations: - message: One selection is required between diff --git a/infra/feast-operator/internal/controller/authz/authz.go b/infra/feast-operator/internal/controller/authz/authz.go index efcae23a4b0..8596d993899 100644 --- a/infra/feast-operator/internal/controller/authz/authz.go +++ b/infra/feast-operator/internal/controller/authz/authz.go @@ -134,28 +134,11 @@ func (authz *FeastAuthorization) initFeastRoleBinding() *rbacv1.RoleBinding { func (authz *FeastAuthorization) setFeastRoleBinding(roleBinding *rbacv1.RoleBinding) error { roleBinding.Labels = authz.getLabels() - roleBinding.Subjects = []rbacv1.Subject{} - if authz.Handler.FeatureStore.Status.Applied.Services.OfflineStore != nil { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.OfflineFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } - if authz.Handler.FeatureStore.Status.Applied.Services.OnlineStore != nil { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.OnlineFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } - if services.IsLocalRegistry(authz.Handler.FeatureStore) { - roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: services.GetFeastServiceName(authz.Handler.FeatureStore, services.RegistryFeastType), - Namespace: authz.Handler.FeatureStore.Namespace, - }) - } + roleBinding.Subjects = append(roleBinding.Subjects, rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: services.GetFeastName(authz.Handler.FeatureStore), + Namespace: authz.Handler.FeatureStore.Namespace, + }) roleBinding.RoleRef = rbacv1.RoleRef{ APIGroup: rbacv1.GroupName, Kind: "Role", diff --git a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go index 0ee269bda17..0bde0dfd7b9 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_db_store_test.go @@ -127,7 +127,6 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Context("When deploying a resource with all db storage services", func() { const resourceName = "cr-name" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -206,7 +205,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ DBPersistence: &feastdevv1alpha1.OfflineStoreDBStorePersistence{ Type: string(offlineType), @@ -430,16 +429,17 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -529,7 +529,7 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -550,20 +550,20 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { }, } - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -579,29 +579,29 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineDBPersistenceSnowflakeConfigType, + DBParameters: unmarshallYamlString(snowflakeYamlString), + }, Registry: services.RegistryConfig{ Path: copyMap["path"].(string), RegistryType: services.RegistryDBPersistenceSQLConfigType, DBParameters: dbParametersMap, }, + OnlineStore: services.OnlineStoreConfig{ + Type: onlineType, + DBParameters: unmarshallYamlString(cassandraYamlString), + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -610,38 +610,16 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineDBPersistenceSnowflakeConfigType, - DBParameters: unmarshallYamlString(snowflakeYamlString), - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) - // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.VolumeMounts).To(HaveLen(1)) + Expect(onlineContainer.Env).To(HaveLen(1)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -650,25 +628,9 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Type: onlineType, - DBParameters: unmarshallYamlString(cassandraYamlString), - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) + Expect(repoConfigOnline).To(Equal(testConfig)) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(1)) // check client config cm := &corev1.ConfigMap{} @@ -682,6 +644,15 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -716,17 +687,16 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { feast.Handler.FeatureStore = resource // check online config - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -736,9 +706,9 @@ var _ = Describe("FeatureStore Controller - db storage services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Type = services.OnlineDBPersistenceSnowflakeConfigType - onlineConfig.OnlineStore.DBParameters = unmarshallYamlString(snowflakeYamlString) - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Type = services.OnlineDBPersistenceSnowflakeConfigType + testConfig.OnlineStore.DBParameters = unmarshallYamlString(snowflakeYamlString) + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go index a762faa5a21..70ac81a056d 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_ephemeral_test.go @@ -48,7 +48,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { const resourceName = "services-ephemeral" const offlineType = "duckdb" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -66,7 +65,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ @@ -199,16 +198,17 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -244,7 +244,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -265,20 +265,21 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -291,28 +292,27 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDuckDbConfigType, + }, Registry: services.RegistryConfig{ RegistryType: services.RegistryFileConfigType, Path: registryPath, }, + OnlineStore: services.OnlineStoreConfig{ + Path: onlineStorePath, + Type: services.OnlineSqliteConfigType, + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -321,37 +321,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDuckDbConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) - // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -360,25 +338,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: onlineStorePath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -396,12 +356,19 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, + OfflineStore: services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + }, OnlineStore: services.OnlineStoreConfig{ Path: fmt.Sprintf("http://feast-%s-online.default.svc.cluster.local:80", resourceName), Type: services.OnlineRemoteConfigType, }, - Registry: regRemote, + Registry: services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfigClient).To(Equal(clientConfig)) @@ -424,17 +391,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(err).NotTo(HaveOccurred()) feast.Handler.FeatureStore = resource - // check registry config - deploy = &appsv1.Deployment{} + // check registry err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -443,21 +409,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) + testConfig.OnlineStore.Path = newOnlineStorePath testConfig.Registry.Path = newRegistryPath Expect(repoConfig).To(Equal(testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -466,20 +427,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -489,8 +444,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Path = newOnlineStorePath - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Path = newOnlineStorePath + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go index 57dd3a290df..57a73eb0eb2 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_kubernetes_auth_test.go @@ -48,7 +48,6 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Context("When deploying a resource with all ephemeral services and Kubernetes authorization", func() { const resourceName = "kubernetes-authorization" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -63,7 +62,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{ Roles: roles, }} @@ -125,7 +124,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -134,7 +133,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -188,47 +187,19 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) // check configured Roles for _, roleName := range roles { @@ -277,23 +248,21 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Kind: "Role", Name: feastRole.Name, } - for _, serviceType := range []services.FeastServiceType{services.RegistryFeastType, services.OnlineFeastType, services.OfflineFeastType} { - sa := &corev1.ServiceAccount{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(serviceType), - Namespace: resource.Namespace, - }, - sa) - Expect(err).NotTo(HaveOccurred()) + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: services.GetFeastName(feast.Handler.FeatureStore), + Namespace: resource.Namespace, + }, + sa) + Expect(err).NotTo(HaveOccurred()) - expectedSubject := rbacv1.Subject{ - Kind: rbacv1.ServiceAccountKind, - Name: sa.Name, - Namespace: sa.Namespace, - } - Expect(roleBinding.Subjects).To(ContainElement(expectedSubject)) - Expect(roleBinding.RoleRef).To(Equal(expectedRoleRef)) + expectedSubject := rbacv1.Subject{ + Kind: rbacv1.ServiceAccountKind, + Name: sa.Name, + Namespace: sa.Namespace, } + Expect(roleBinding.Subjects).To(ContainElement(expectedSubject)) + Expect(roleBinding.RoleRef).To(Equal(expectedRoleRef)) By("Updating the user roled and reconciling") resourceNew := resource.DeepCopy() @@ -394,7 +363,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -415,19 +384,19 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { }, } - // check registry deployment + // check registry deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -436,34 +405,22 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - S3AdditionalKwargs: nil, - }, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + } + testConfig.Registry.RegistryType = services.RegistryFileConfigType + testConfig.AuthzConfig = services.AuthzConfig{ + Type: services.KubernetesAuthType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check offline + env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -472,37 +429,14 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, - } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check online + env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check online config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -511,26 +445,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: services.AuthzConfig{ - Type: services.KubernetesAuthType, - }, - } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -544,6 +459,15 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -553,10 +477,7 @@ var _ = Describe("FeatureStore Controller-Kubernetes authorization", func() { Path: fmt.Sprintf("http://feast-%s-online.default.svc.cluster.local:80", resourceName), Type: services.OnlineRemoteConfigType, }, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - }, + Registry: regRemote, AuthzConfig: services.AuthzConfig{ Type: services.KubernetesAuthType, }, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go index 70f33486fce..5139e14dd38 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_loglevel_test.go @@ -154,43 +154,26 @@ var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { Expect(cond.Message).To(Equal(feastdevv1alpha1.OnlineStoreReadyMessage)) Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command := deploy.Spec.Template.Spec.Containers[0].Command + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + command := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("ERROR")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("INFO")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).To(ContainElement("--log-level")) Expect(command).To(ContainElement("DEBUG")) }) @@ -229,32 +212,22 @@ var _ = Describe("FeatureStore Controller - Feast service LogLevel", func() { }, } + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command := deploy.Spec.Template.Spec.Containers[0].Command + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + command := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - command = deploy.Spec.Template.Spec.Containers[0].Command + command = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Command Expect(command).NotTo(ContainElement("--log-level")) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go index f4a21a28f17..aff36f338e7 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_objectstore_test.go @@ -46,7 +46,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-object-store" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -68,7 +67,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OnlineStore = nil resource.Spec.Services.OfflineStore = nil @@ -176,39 +175,23 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) // update S3 additional args and reconcile resourceNew := resource.DeepCopy() @@ -234,16 +217,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.S3AdditionalKwargs).To(Equal(&newS3AdditionalKwargs)) // check registry deployment - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.VolumeMounts).To(HaveLen(1)) }) It("should properly encode a feature_store.yaml config", func() { @@ -290,21 +271,28 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) + Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers)).To(BeNil()) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -313,38 +301,13 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: registryPath, - S3AdditionalKwargs: &s3AdditionalKwargs, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryFileConfigType, + Path: registryPath, + S3AdditionalKwargs: &s3AdditionalKwargs, } - Expect(repoConfig).To(Equal(testConfig)) - - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -358,17 +321,12 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) - clientConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - }, - AuthzConfig: noAuthzConfig(), + clientConfig := feast.GetInitRepoConfig() + clientConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), } - Expect(repoConfigClient).To(Equal(clientConfig)) + Expect(repoConfigClient).To(Equal(&clientConfig)) // remove S3 additional keywords and reconcile resourceNew := resource.DeepCopy() @@ -386,16 +344,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { feast.Handler.FeatureStore = resource // check registry config - deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -405,27 +361,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) testConfig.Registry.S3AdditionalKwargs = nil - Expect(repoConfig).To(Equal(testConfig)) - - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(errors.IsNotFound(err)).To(BeTrue()) + Expect(repoConfig).To(Equal(&testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go index eb320c5bb39..08c92a88a97 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_oidc_auth_test.go @@ -49,7 +49,6 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { const resourceName = "oidc-authorization" const oidcSecretName = "oidc-secret" var pullPolicy = corev1.PullAlways - var replicas = int32(1) ctx := context.Background() @@ -74,7 +73,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { By("creating the custom resource for the Kind FeatureStore") err = k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{}) + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{}) resource.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{OidcAuthz: &feastdevv1alpha1.OidcAuthz{ SecretRef: corev1.LocalObjectReference{ Name: oidcSecretName, @@ -147,7 +146,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -156,7 +155,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -206,47 +205,22 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) - - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) // check Feast Role feastRole := &rbacv1.Role{} @@ -268,16 +242,14 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Expect(err).To(HaveOccurred()) Expect(errors.IsNotFound(err)).To(BeTrue()) - // check ServiceAccounts - for _, serviceType := range []services.FeastServiceType{services.RegistryFeastType, services.OnlineFeastType, services.OfflineFeastType} { - sa := &corev1.ServiceAccount{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(serviceType), - Namespace: resource.Namespace, - }, - sa) - Expect(err).NotTo(HaveOccurred()) - } + // check ServiceAccount + sa := &corev1.ServiceAccount{} + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: services.GetFeastName(feast.Handler.FeatureStore), + Namespace: resource.Namespace, + }, + sa) + Expect(err).NotTo(HaveOccurred()) By("Clearing the OIDC authorization and reconciling") resourceNew := resource.DeepCopy() @@ -328,7 +300,7 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -349,19 +321,19 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + env := getFeatureStoreYamlEnvVar(services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -374,28 +346,27 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { Project: feastProject, Provider: services.LocalProviderType, EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, + }, Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - S3AdditionalKwargs: nil, + RegistryType: services.RegistryFileConfigType, + Path: services.EphemeralPath + "/" + services.DefaultRegistryPath, + }, + OnlineStore: services.OnlineStoreConfig{ + Path: services.EphemeralPath + "/" + services.DefaultOnlineStorePath, + Type: services.OnlineSqliteConfigType, }, AuthzConfig: expectedServerOidcAuthorizConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check offline + env = getFeatureStoreYamlEnvVar(services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -404,35 +375,14 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: expectedServerOidcAuthorizConfig(), - } Expect(repoConfig).To(Equal(testConfig)) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + // check online + env = getFeatureStoreYamlEnvVar(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).Env) Expect(env).NotTo(BeNil()) // check online config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -441,23 +391,6 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - testConfig = &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: expectedServerOidcAuthorizConfig(), - } Expect(repoConfig).To(Equal(testConfig)) // check client config @@ -472,6 +405,11 @@ var _ = Describe("FeatureStore Controller-OIDC authorization", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, diff --git a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go index fe0caa38e63..887d9070efb 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_pvc_test.go @@ -50,7 +50,6 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Context("When deploying a resource with all ephemeral services", func() { const resourceName = "services-pvc" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -69,6 +68,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { onlineStoreMountPath := "/online" registryMountPath := "/registry" + accessModes := []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce, corev1.ReadWriteMany} storageClassName := "test" onlineStoreMountedPath := path.Join(onlineStoreMountPath, onlineStorePath) @@ -78,13 +78,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) resource.Spec.Services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{ FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ Type: offlineType, PvcConfig: &feastdevv1alpha1.PvcConfig{ Create: &feastdevv1alpha1.PvcCreate{ + AccessModes: accessModes, StorageClassName: &storageClassName, }, MountPath: offlineStoreMountPath, @@ -162,6 +163,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.Type).To(Equal(offlineType)) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(accessModes)) Expect(resource.Status.Applied.Services.OfflineStore.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(Equal(&storageClassName)) expectedResources := corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -179,6 +181,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(onlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(BeNil()) expectedResources = corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -198,6 +201,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(registryPath)) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create).NotTo(BeNil()) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.PvcConfig.Create.StorageClassName).To(BeNil()) expectedResources = corev1.VolumeResourceRequirements{ Requests: corev1.ResourceList{ @@ -255,94 +259,88 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) - // check offline deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(offlineStoreMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + name := feast.GetFeastServiceName(services.RegistryFeastType) + regVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(regVol.Name).To(Equal(name)) + Expect(regVol.PersistentVolumeClaim.ClaimName).To(Equal(name)) + + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.VolumeMounts).To(HaveLen(3)) + offlineVolMount := services.GetOfflineVolumeMount(feast.Handler.FeatureStore, offlineContainer.VolumeMounts) + Expect(offlineVolMount.MountPath).To(Equal(offlineStoreMountPath)) + offlinePvcName := feast.GetFeastServiceName(services.OfflineFeastType) + Expect(offlineVolMount.Name).To(Equal(offlinePvcName)) // check offline pvc pvc := &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: offlinePvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) Expect(pvc.Spec.StorageClassName).To(Equal(&storageClassName)) + Expect(pvc.Spec.AccessModes).To(Equal(accessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOfflineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) - // check online deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(onlineStoreMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + // check online + onlinePvcName := feast.GetFeastServiceName(services.OnlineFeastType) + onlineVol := services.GetOnlineVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(onlineVol.Name).To(Equal(onlinePvcName)) + Expect(onlineVol.PersistentVolumeClaim.ClaimName).To(Equal(onlinePvcName)) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.VolumeMounts).To(HaveLen(3)) + onlineVolMount := services.GetOnlineVolumeMount(feast.Handler.FeatureStore, onlineContainer.VolumeMounts) + Expect(onlineVolMount.MountPath).To(Equal(onlineStoreMountPath)) + Expect(onlineVolMount.Name).To(Equal(onlinePvcName)) // check online pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: onlinePvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Name).To(Equal(onlinePvcName)) + Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultOnlineStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) - // check registry deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) - Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Volumes[0].Name).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Volumes[0].PersistentVolumeClaim.ClaimName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].MountPath).To(Equal(registryMountPath)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts[0].Name).To(Equal(deploy.Name)) + // check registry + registryPvcName := feast.GetFeastServiceName(services.RegistryFeastType) + registryVol := services.GetRegistryVolume(feast.Handler.FeatureStore, deploy.Spec.Template.Spec.Volumes) + Expect(registryVol.Name).To(Equal(registryPvcName)) + Expect(registryVol.PersistentVolumeClaim.ClaimName).To(Equal(registryPvcName)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.VolumeMounts).To(HaveLen(3)) + registryVolMount := services.GetRegistryVolumeMount(feast.Handler.FeatureStore, registryContainer.VolumeMounts) + Expect(registryVolMount.MountPath).To(Equal(registryMountPath)) + Expect(registryVolMount.Name).To(Equal(registryPvcName)) // check registry pvc pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: registryPvcName, Namespace: resource.Namespace, }, pvc) Expect(err).NotTo(HaveOccurred()) - Expect(pvc.Name).To(Equal(deploy.Name)) + Expect(pvc.Name).To(Equal(registryPvcName)) + Expect(pvc.Spec.AccessModes).To(Equal(services.DefaultPVCAccessModes)) Expect(pvc.Spec.Resources.Requests.Storage().String()).To(Equal(services.DefaultRegistryStorageRequest)) Expect(pvc.DeletionTimestamp).To(BeNil()) @@ -367,19 +365,18 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { // check online deployment deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(0)) - Expect(deploy.Spec.Template.Spec.Containers[0].VolumeMounts).To(HaveLen(0)) + Expect(deploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) + Expect(services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(2)) // check online pvc is deleted log.FromContext(feast.Handler.Context).Info("Checking deletion of", "PersistentVolumeClaim", deploy.Name) pvc = &corev1.PersistentVolumeClaim{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: deploy.Name, + Name: onlinePvcName, Namespace: resource.Namespace, }, pvc) @@ -413,7 +410,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -434,21 +431,22 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { }, } - // check registry deployment + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) // check registry config - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -465,25 +463,24 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { RegistryType: services.RegistryFileConfigType, Path: registryMountedPath, }, + OfflineStore: services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDuckDbConfigType, + }, + OnlineStore: services.OnlineStoreConfig{ + Path: onlineStoreMountedPath, + Type: services.OnlineSqliteConfigType, + }, AuthzConfig: noAuthzConfig(), } Expect(repoConfig).To(Equal(testConfig)) - // check offline deployment - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) // check offline config - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -492,37 +489,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDuckDbConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -531,25 +507,7 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: onlineStoreMountedPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -563,6 +521,15 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:80", resourceName), + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -602,14 +569,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { // check registry config deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + registryContainer = services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -618,21 +585,16 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfig = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) + testConfig.OnlineStore.Path = newOnlineStoreMountedPath testConfig.Registry.Path = newRegistryMountedPath Expect(repoConfig).To(Equal(testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -641,20 +603,14 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOffline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -664,8 +620,8 @@ var _ = Describe("FeatureStore Controller-Ephemeral services", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.OnlineStore.Path = newOnlineStoreMountedPath - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.OnlineStore.Path = newOnlineStoreMountedPath + Expect(repoConfigOnline).To(Equal(testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/featurestore_controller_test.go b/infra/feast-operator/internal/controller/featurestore_controller_test.go index debd63300b2..71b5d400f87 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_test.go @@ -172,15 +172,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) svc := &corev1.Service{} @@ -220,11 +221,11 @@ var _ = Describe("FeatureStore Controller", func() { } deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) @@ -232,7 +233,7 @@ var _ = Describe("FeatureStore Controller", func() { env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -241,17 +242,8 @@ var _ = Describe("FeatureStore Controller", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfig).To(Equal(testConfig)) + testConfig := feast.GetDefaultRepoConfig() + Expect(repoConfig).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -265,17 +257,12 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) - clientConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: "feast-test-resource-registry.default.svc.cluster.local:80", - }, - AuthzConfig: noAuthzConfig(), + clientConfig := feast.GetInitRepoConfig() + clientConfig.Registry = services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: "feast-test-resource-registry.default.svc.cluster.local:80", } - Expect(repoConfigClient).To(Equal(clientConfig)) + Expect(repoConfigClient).To(Equal(&clientConfig)) // change feast project and reconcile resourceNew := resource.DeepCopy() @@ -291,8 +278,8 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) @@ -302,7 +289,7 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -310,7 +297,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) }) It("should error on reconcile", func() { @@ -339,11 +326,11 @@ var _ = Describe("FeatureStore Controller", func() { } deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) @@ -378,17 +365,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) Expect(cond.Status).To(Equal(metav1.ConditionFalse)) Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + deploy.Name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType) Expect(cond).To(BeNil()) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType) Expect(cond).ToNot(BeNil()) - Expect(cond.Status).To(Equal(metav1.ConditionFalse)) - Expect(cond.Reason).To(Equal(feastdevv1alpha1.RegistryFailedReason)) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) Expect(cond.Type).To(Equal(feastdevv1alpha1.RegistryReadyType)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.ClientReadyType) Expect(cond).ToNot(BeNil()) @@ -404,7 +390,6 @@ var _ = Describe("FeatureStore Controller", func() { Context("When reconciling a resource with all services enabled", func() { const resourceName = "services" var pullPolicy = corev1.PullAlways - var replicas = int32(1) var testEnvVarName = "testEnvVarName" var testEnvVarValue = "testEnvVarValue" @@ -420,7 +405,7 @@ var _ = Describe("FeatureStore Controller", func() { By("creating the custom resource for the Kind FeatureStore") err := k8sClient.Get(ctx, typeNamespacedName, featurestore) if err != nil && errors.IsNotFound(err) { - resource := createFeatureStoreResource(resourceName, image, pullPolicy, replicas, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, + resource := createFeatureStoreResource(resourceName, image, pullPolicy, &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}}) Expect(k8sClient.Create(ctx, resource)).To(Succeed()) } @@ -474,7 +459,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.OnlineStore).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.DefaultOnlineStoreEphemeralPath)) + Expect(resource.Status.Applied.Services.OnlineStore.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultOnlineStorePath)) Expect(resource.Status.Applied.Services.OnlineStore.Env).To(Equal(&[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})) Expect(resource.Status.Applied.Services.OnlineStore.ImagePullPolicy).To(Equal(&pullPolicy)) Expect(resource.Status.Applied.Services.OnlineStore.Resources).NotTo(BeNil()) @@ -483,7 +468,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Applied.Services.Registry.Local).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence).NotTo(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence).NotTo(BeNil()) - Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.DefaultRegistryEphemeralPath)) + Expect(resource.Status.Applied.Services.Registry.Local.Persistence.FilePersistence.Path).To(Equal(services.EphemeralPath + "/" + services.DefaultRegistryPath)) Expect(resource.Status.Applied.Services.Registry.Local.ImagePullPolicy).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Resources).To(BeNil()) Expect(resource.Status.Applied.Services.Registry.Local.Image).To(Equal(&services.DefaultImage)) @@ -534,16 +519,16 @@ var _ = Describe("FeatureStore Controller", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -579,12 +564,12 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) saList := corev1.ServiceAccountList{} err = k8sClient.List(ctx, &saList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(saList.Items).To(HaveLen(3)) + Expect(saList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -607,19 +592,20 @@ var _ = Describe("FeatureStore Controller", func() { // check registry config deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -628,33 +614,20 @@ var _ = Describe("FeatureStore Controller", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -663,38 +636,16 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: "feast-services-registry.default.svc.cluster.local:80", - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -703,25 +654,7 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: "feast-services-offline.default.svc.cluster.local", - Type: services.OfflineRemoteConfigType, - Port: services.HttpPort, - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) + Expect(repoConfigOnline).To(Equal(&testConfig)) // check client config cm := &corev1.ConfigMap{} @@ -735,6 +668,15 @@ var _ = Describe("FeatureStore Controller", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: "feast-services-offline.default.svc.cluster.local", + Type: services.OfflineRemoteConfigType, + Port: services.HttpPort, + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: "feast-services-registry.default.svc.cluster.local:80", + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -763,8 +705,8 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) Expect(resource.Spec.FeastProject).To(Equal(resourceNew.Spec.FeastProject)) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, + Name: objMeta.Name, + Namespace: objMeta.Namespace, }, deploy) Expect(err).NotTo(HaveOccurred()) @@ -774,7 +716,7 @@ var _ = Describe("FeatureStore Controller", func() { env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -782,7 +724,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).NotTo(HaveOccurred()) err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) }) It("should properly set container env variables", func() { @@ -808,7 +750,7 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -830,129 +772,27 @@ var _ = Describe("FeatureStore Controller", func() { } fsYamlStr := "" - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) // check online config deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy.Name)) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - - // change feast project and reconcile - resourceNew := resource.DeepCopy() - resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}} - err = k8sClient.Update(ctx, resourceNew) - Expect(err).NotTo(HaveOccurred()) - _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue()) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) - }) - - It("Should scale online/offline store service", func() { - By("Reconciling the created resource") - controllerReconciler := &FeatureStoreReconciler{ - Client: k8sClient, - Scheme: k8sClient.Scheme(), - } - - _, err := controllerReconciler.Reconcile(ctx, reconcile.Request{ - NamespacedName: typeNamespacedName, - }) - Expect(err).NotTo(HaveOccurred()) - - resource := &feastdevv1alpha1.FeatureStore{} - err = k8sClient.Get(ctx, typeNamespacedName, resource) - Expect(err).NotTo(HaveOccurred()) - - req, err := labels.NewRequirement(services.NameLabelKey, selection.Equals, []string{resource.Name}) - Expect(err).NotTo(HaveOccurred()) - labelSelector := labels.NewSelector().Add(*req) - listOpts := &client.ListOptions{Namespace: resource.Namespace, LabelSelector: labelSelector} - deployList := appsv1.DeploymentList{} - err = k8sClient.List(ctx, &deployList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) - - svcList := corev1.ServiceList{} - err = k8sClient.List(ctx, &svcList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(svcList.Items).To(HaveLen(3)) - - cmList := corev1.ConfigMapList{} - err = k8sClient.List(ctx, &cmList, listOpts) - Expect(err).NotTo(HaveOccurred()) - Expect(cmList.Items).To(HaveLen(1)) - - feast := services.FeastServices{ - Handler: handler.FeastHandler{ - Client: controllerReconciler.Client, - Context: ctx, - Scheme: controllerReconciler.Scheme, - FeatureStore: resource, - }, - } - - fsYamlStr := "" - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) - Expect(err).NotTo(HaveOccurred()) - - // check online config - deploy_online := &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy_online) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy_online.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy_online.Name)) - Expect(deploy_online.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy_online.Spec.Template.Spec.Containers[0].Env).To(HaveLen(3)) - Expect(areEnvVarArraysEqual(deploy_online.Spec.Template.Spec.Containers[0].Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.FeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) - Expect(deploy_online.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullAlways)) - - // check offline config - deploy_offline := &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy_offline) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy_offline.Spec.Template.Spec.ServiceAccountName).To(Equal(deploy_offline.Name)) - Expect(deploy_offline.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy_offline.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - Expect(deploy_offline.Spec.Template.Spec.Containers[0].ImagePullPolicy).To(Equal(corev1.PullIfNotPresent)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.namespace"}}}})).To(BeTrue()) + Expect(onlineContainer.ImagePullPolicy).To(Equal(corev1.PullAlways)) // change feast project and reconcile - // scale online replicas to 2 resourceNew := resource.DeepCopy() - new_replicas := int32(2) - resourceNew.Spec.Services.OnlineStore.Replicas = &new_replicas - resourceNew.Spec.Services.OfflineStore.Replicas = &new_replicas - + resourceNew.Spec.Services.OnlineStore.Env = &[]corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}} err = k8sClient.Update(ctx, resourceNew) Expect(err).NotTo(HaveOccurred()) _, err = controllerReconciler.Reconcile(ctx, reconcile.Request{ @@ -962,21 +802,16 @@ var _ = Describe("FeatureStore Controller", func() { err = k8sClient.Get(ctx, typeNamespacedName, resource) Expect(err).NotTo(HaveOccurred()) + Expect(areEnvVarArraysEqual(*resource.Status.Applied.Services.OnlineStore.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{FieldPath: "metadata.name"}}}})).To(BeTrue()) err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy_online) - Expect(err).NotTo(HaveOccurred()) - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy_offline) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) - Expect(deploy_online.Spec.Replicas).To(Equal(&new_replicas)) - Expect(deploy_offline.Spec.Replicas).To(Equal(&new_replicas)) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(3)) + Expect(areEnvVarArraysEqual(onlineContainer.Env, []corev1.EnvVar{{Name: testEnvVarName, Value: testEnvVarValue + "1"}, {Name: services.TmpFeatureStoreYamlEnvVar, Value: fsYamlStr}, {Name: "fieldRefName", ValueFrom: &corev1.EnvVarSource{FieldRef: &corev1.ObjectFieldSelector{APIVersion: "v1", FieldPath: "metadata.name"}}}})).To(BeTrue()) }) It("Should delete k8s objects owned by the FeatureStore CR", func() { @@ -1002,7 +837,7 @@ var _ = Describe("FeatureStore Controller", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -1021,7 +856,7 @@ var _ = Describe("FeatureStore Controller", func() { err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(2)) + Expect(deployList.Items).To(HaveLen(1)) err = k8sClient.List(ctx, &svcList, listOpts) Expect(err).NotTo(HaveOccurred()) @@ -1096,6 +931,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(err).To(HaveOccurred()) err = k8sClient.Get(ctx, nsName, resource) Expect(err).NotTo(HaveOccurred()) + Expect(resource.Status.Applied.Services.Registry.Remote.FeastRef.Namespace).NotTo(BeEmpty()) Expect(apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType)).To(BeNil()) Expect(apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.RegistryReadyType)).To(BeNil()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.ReadyType)).To(BeFalse()) @@ -1135,7 +971,6 @@ var _ = Describe("FeatureStore Controller", func() { Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.ReadyType)).To(BeTrue()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType)).To(BeTrue()) Expect(apimeta.IsStatusConditionTrue(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType)).To(BeTrue()) - Expect(resource.Status.Applied.Services.Registry.Remote.FeastRef.Namespace).To(Equal(resource.Namespace)) Expect(resource.Status.ServiceHostnames.Registry).ToNot(BeEmpty()) Expect(resource.Status.ServiceHostnames.Registry).To(Equal(referencedRegistry.Status.ServiceHostnames.Registry)) feast := services.FeastServices{ @@ -1147,6 +982,16 @@ var _ = Describe("FeatureStore Controller", func() { }, } + // check deployment + deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() + err = k8sClient.Get(ctx, types.NamespacedName{ + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.InitContainers).To(HaveLen(2)) + // check client config cm := &corev1.ConfigMap{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -1235,12 +1080,13 @@ var _ = Describe("FeatureStore Controller", func() { }, } + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) err = controllerutil.RemoveControllerReference(resource, deploy, controllerReconciler.Scheme) @@ -1275,7 +1121,7 @@ var _ = Describe("FeatureStore Controller", func() { Expect(cond.Type).To(Equal(feastdevv1alpha1.ReadyType)) Expect(cond.Status).To(Equal(metav1.ConditionFalse)) Expect(cond.Reason).To(Equal(feastdevv1alpha1.FailedReason)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) + Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + deploy.Name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.AuthorizationReadyType) Expect(cond).To(BeNil()) @@ -1296,10 +1142,9 @@ var _ = Describe("FeatureStore Controller", func() { cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OfflineStoreReadyType) Expect(cond).ToNot(BeNil()) - Expect(cond.Status).To(Equal(metav1.ConditionFalse)) - Expect(cond.Reason).To(Equal(feastdevv1alpha1.OfflineStoreFailedReason)) + Expect(cond.Status).To(Equal(metav1.ConditionTrue)) + Expect(cond.Reason).To(Equal(feastdevv1alpha1.ReadyReason)) Expect(cond.Type).To(Equal(feastdevv1alpha1.OfflineStoreReadyType)) - Expect(cond.Message).To(Equal("Error: Object " + resource.Namespace + "/" + name + " is already owned by another Service controller " + name)) cond = apimeta.FindStatusCondition(resource.Status.Conditions, feastdevv1alpha1.OnlineStoreReadyType) Expect(cond).ToNot(BeNil()) @@ -1362,7 +1207,7 @@ var _ = Describe("FeatureStore Controller", func() { }) }) -func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, replicas int32, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { +func createFeatureStoreResource(resourceName string, image string, pullPolicy corev1.PullPolicy, envVars *[]corev1.EnvVar) *feastdevv1alpha1.FeatureStore { return &feastdevv1alpha1.FeatureStore{ ObjectMeta: metav1.ObjectMeta{ Name: resourceName, @@ -1373,17 +1218,14 @@ func createFeatureStoreResource(resourceName string, image string, pullPolicy co Services: &feastdevv1alpha1.FeatureStoreServices{ OfflineStore: &feastdevv1alpha1.OfflineStore{}, OnlineStore: &feastdevv1alpha1.OnlineStore{ - StoreServiceConfigs: feastdevv1alpha1.StoreServiceConfigs{ - Replicas: &replicas, - ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ - DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ - Image: &image, - }, - OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ - Env: envVars, - ImagePullPolicy: &pullPolicy, - Resources: &corev1.ResourceRequirements{}, - }, + ServiceConfigs: feastdevv1alpha1.ServiceConfigs{ + DefaultConfigs: feastdevv1alpha1.DefaultConfigs{ + Image: &image, + }, + OptionalConfigs: feastdevv1alpha1.OptionalConfigs{ + Env: envVars, + ImagePullPolicy: &pullPolicy, + Resources: &corev1.ResourceRequirements{}, }, }, }, @@ -1394,7 +1236,7 @@ func createFeatureStoreResource(resourceName string, image string, pullPolicy co func getFeatureStoreYamlEnvVar(envs []corev1.EnvVar) *corev1.EnvVar { for _, e := range envs { - if e.Name == services.FeatureStoreYamlEnvVar { + if e.Name == services.TmpFeatureStoreYamlEnvVar { return &e } } diff --git a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go index 45cda317409..0b7fe84d22a 100644 --- a/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go +++ b/infra/feast-operator/internal/controller/featurestore_controller_tls_test.go @@ -27,7 +27,6 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" - apierrors "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -56,7 +55,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { } featurestore := &feastdevv1alpha1.FeatureStore{} localRef := corev1.LocalObjectReference{Name: "test"} - tlsConfigs := feastdevv1alpha1.TlsConfigs{ + tlsConfigs := &feastdevv1alpha1.TlsConfigs{ SecretRef: &localRef, } BeforeEach(func() { @@ -72,16 +71,14 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { FeastProject: feastProject, Services: &feastdevv1alpha1.FeatureStoreServices{ OnlineStore: &feastdevv1alpha1.OnlineStore{ - TLS: &tlsConfigs, + TLS: tlsConfigs, }, OfflineStore: &feastdevv1alpha1.OfflineStore{ - TLS: &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: tlsConfigs, - }, + TLS: tlsConfigs, }, Registry: &feastdevv1alpha1.Registry{ Local: &feastdevv1alpha1.LocalRegistryConfig{ - TLS: &tlsConfigs, + TLS: tlsConfigs, }, }, }, @@ -174,16 +171,17 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(resource.Status.Phase).To(Equal(feastdevv1alpha1.ReadyPhase)) + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) Expect(err).NotTo(HaveOccurred()) Expect(deploy.Spec.Replicas).To(Equal(&services.DefaultReplicas)) Expect(controllerutil.HasControllerReference(deploy)).To(BeTrue()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) svc := &corev1.Service{} err = k8sClient.Get(ctx, types.NamespacedName{ @@ -227,7 +225,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { deployList := appsv1.DeploymentList{} err = k8sClient.List(ctx, &deployList, listOpts) Expect(err).NotTo(HaveOccurred()) - Expect(deployList.Items).To(HaveLen(3)) + Expect(deployList.Items).To(HaveLen(1)) svcList := corev1.ServiceList{} err = k8sClient.List(ctx, &svcList, listOpts) @@ -239,20 +237,21 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Expect(err).NotTo(HaveOccurred()) Expect(cmList.Items).To(HaveLen(1)) - // check registry config + // check deployment deploy := &appsv1.Deployment{} + objMeta := feast.GetObjectMeta() err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env := getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + registryContainer := services.GetRegistryContainer(deploy.Spec.Template.Spec.Containers) + Expect(registryContainer.Env).To(HaveLen(1)) + env := getFeatureStoreYamlEnvVar(registryContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64(services.RegistryFeastType) + fsYamlStr, err := feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -261,32 +260,19 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfig := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfig) Expect(err).NotTo(HaveOccurred()) - testConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - Registry: services.RegistryConfig{ - RegistryType: services.RegistryFileConfigType, - Path: services.DefaultRegistryEphemeralPath, - }, - AuthzConfig: noAuthzConfig(), + testConfig := feast.GetDefaultRepoConfig() + testConfig.OfflineStore = services.OfflineStoreConfig{ + Type: services.OfflineFilePersistenceDaskConfigType, } - Expect(repoConfig).To(Equal(testConfig)) + Expect(repoConfig).To(Equal(&testConfig)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer := services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + Expect(offlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -295,37 +281,15 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOffline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOffline) Expect(err).NotTo(HaveOccurred()) - regRemote := services.RegistryConfig{ - RegistryType: services.RegistryRemoteConfigType, - Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), - Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", - } - offlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: services.OfflineStoreConfig{ - Type: services.OfflineFilePersistenceDaskConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOffline).To(Equal(offlineConfig)) + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer := services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + Expect(onlineContainer.Env).To(HaveLen(1)) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -334,26 +298,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOnline := &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - offlineRemote := services.OfflineStoreConfig{ - Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), - Type: services.OfflineRemoteConfigType, - Port: services.HttpsPort, - Scheme: services.HttpsScheme, - Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", - } - onlineConfig := &services.RepoConfig{ - Project: feastProject, - Provider: services.LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, - OfflineStore: offlineRemote, - OnlineStore: services.OnlineStoreConfig{ - Path: services.DefaultOnlineStoreEphemeralPath, - Type: services.OnlineSqliteConfigType, - }, - Registry: regRemote, - AuthzConfig: noAuthzConfig(), - } - Expect(repoConfigOnline).To(Equal(onlineConfig)) + Expect(repoConfigOnline).To(Equal(&testConfig)) Expect(deploy.Spec.Template.Spec.Containers[0].Env).To(HaveLen(1)) // check client config @@ -368,6 +313,18 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigClient := &services.RepoConfig{} err = yaml.Unmarshal([]byte(cm.Data[services.FeatureStoreYamlCmKey]), repoConfigClient) Expect(err).NotTo(HaveOccurred()) + offlineRemote := services.OfflineStoreConfig{ + Host: fmt.Sprintf("feast-%s-offline.default.svc.cluster.local", resourceName), + Type: services.OfflineRemoteConfigType, + Port: services.HttpsPort, + Scheme: services.HttpsScheme, + Cert: services.GetTlsPath(services.OfflineFeastType) + "tls.crt", + } + regRemote := services.RegistryConfig{ + RegistryType: services.RegistryRemoteConfigType, + Path: fmt.Sprintf("feast-%s-registry.default.svc.cluster.local:443", resourceName), + Cert: services.GetTlsPath(services.RegistryFeastType) + "tls.crt", + } clientConfig := &services.RepoConfig{ Project: feastProject, Provider: services.LocalProviderType, @@ -396,9 +353,7 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { }, }, OfflineStore: &feastdevv1alpha1.OfflineStore{ - TLS: &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: tlsConfigs, - }, + TLS: tlsConfigs, }, Registry: &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ @@ -426,25 +381,18 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { // check registry deploy = &appsv1.Deployment{} err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.RegistryFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).To(HaveOccurred()) - Expect(apierrors.IsNotFound(err)).To(BeTrue()) + Name: objMeta.Name, + Namespace: objMeta.Namespace, + }, deploy) + Expect(err).NotTo(HaveOccurred()) + Expect(deploy.Spec.Template.Spec.Containers).To(HaveLen(2)) // check offline config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OfflineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + offlineContainer = services.GetOfflineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(offlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OfflineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -458,21 +406,15 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { Path: remoteRegHost, Cert: services.GetTlsPath(services.RegistryFeastType) + "remote.crt", } - offlineConfig.Registry = regRemote - Expect(repoConfigOffline).To(Equal(offlineConfig)) + testConfig.Registry = regRemote + Expect(repoConfigOffline).To(Equal(&testConfig)) // check online config - deploy = &appsv1.Deployment{} - err = k8sClient.Get(ctx, types.NamespacedName{ - Name: feast.GetFeastServiceName(services.OnlineFeastType), - Namespace: resource.Namespace, - }, - deploy) - Expect(err).NotTo(HaveOccurred()) - env = getFeatureStoreYamlEnvVar(deploy.Spec.Template.Spec.Containers[0].Env) + onlineContainer = services.GetOnlineContainer(deploy.Spec.Template.Spec.Containers) + env = getFeatureStoreYamlEnvVar(onlineContainer.Env) Expect(env).NotTo(BeNil()) - fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64(services.OnlineFeastType) + fsYamlStr, err = feast.GetServiceFeatureStoreYamlBase64() Expect(err).NotTo(HaveOccurred()) Expect(fsYamlStr).To(Equal(env.Value)) @@ -482,8 +424,8 @@ var _ = Describe("FeatureStore Controller - Feast service TLS", func() { repoConfigOnline = &services.RepoConfig{} err = yaml.Unmarshal(envByte, repoConfigOnline) Expect(err).NotTo(HaveOccurred()) - onlineConfig.Registry = regRemote - Expect(repoConfigOnline).To(Equal(onlineConfig)) + testConfig.Registry = regRemote + Expect(repoConfigOnline).To(Equal(&testConfig)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/client.go b/infra/feast-operator/internal/controller/services/client.go index d4b78e2611e..89e22f7be6d 100644 --- a/infra/feast-operator/internal/controller/services/client.go +++ b/infra/feast-operator/internal/controller/services/client.go @@ -32,7 +32,7 @@ func (feast *FeastServices) deployClient() error { func (feast *FeastServices) createClientConfigMap() error { logger := log.FromContext(feast.Handler.Context) cm := &corev1.ConfigMap{ - ObjectMeta: feast.GetObjectMeta(ClientFeastType), + ObjectMeta: feast.GetObjectMetaType(ClientFeastType), } cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, cm, controllerutil.MutateFn(func() error { @@ -46,7 +46,7 @@ func (feast *FeastServices) createClientConfigMap() error { } func (feast *FeastServices) setClientConfigMap(cm *corev1.ConfigMap) error { - cm.Labels = feast.getLabels(ClientFeastType) + cm.Labels = feast.getFeastTypeLabels(ClientFeastType) clientYaml, err := feast.getClientFeatureStoreYaml(feast.extractConfigFromSecret) if err != nil { return err @@ -81,7 +81,7 @@ func (feast *FeastServices) setCaConfigMap(cm *corev1.ConfigMap) error { func (feast *FeastServices) initCaConfigMap() *corev1.ConfigMap { cm := &corev1.ConfigMap{ - ObjectMeta: feast.GetObjectMeta(ClientCaFeastType), + ObjectMeta: feast.GetObjectMetaType(ClientCaFeastType), } cm.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ConfigMap")) return cm diff --git a/infra/feast-operator/internal/controller/services/repo_config.go b/infra/feast-operator/internal/controller/services/repo_config.go index c70996ab867..0a5dd11544e 100644 --- a/infra/feast-operator/internal/controller/services/repo_config.go +++ b/infra/feast-operator/internal/controller/services/repo_config.go @@ -27,37 +27,75 @@ import ( ) // GetServiceFeatureStoreYamlBase64 returns a base64 encoded feature_store.yaml config for the feast service -func (feast *FeastServices) GetServiceFeatureStoreYamlBase64(feastType FeastServiceType) (string, error) { - fsYaml, err := feast.getServiceFeatureStoreYaml(feastType) +func (feast *FeastServices) GetServiceFeatureStoreYamlBase64() (string, error) { + fsYaml, err := feast.getServiceFeatureStoreYaml() if err != nil { return "", err } return base64.StdEncoding.EncodeToString(fsYaml), nil } -func (feast *FeastServices) getServiceFeatureStoreYaml(feastType FeastServiceType) ([]byte, error) { - repoConfig, err := feast.getServiceRepoConfig(feastType) +func (feast *FeastServices) getServiceFeatureStoreYaml() ([]byte, error) { + repoConfig, err := feast.getServiceRepoConfig() if err != nil { return nil, err } return yaml.Marshal(repoConfig) } -func (feast *FeastServices) getServiceRepoConfig(feastType FeastServiceType) (RepoConfig, error) { - return getServiceRepoConfig(feastType, feast.Handler.FeatureStore, feast.extractConfigFromSecret) +func (feast *FeastServices) getServiceRepoConfig() (RepoConfig, error) { + return getServiceRepoConfig(feast.Handler.FeatureStore, feast.extractConfigFromSecret) } func getServiceRepoConfig( - feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + repoConfig, err := getBaseServiceRepoConfig(featureStore, secretExtractionFunc) + if err != nil { + return repoConfig, err + } + appliedSpec := featureStore.Status.Applied + if appliedSpec.Services != nil { + services := appliedSpec.Services + if services.OfflineStore != nil { + err := setRepoConfigOffline(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + if services.OnlineStore != nil { + err := setRepoConfigOnline(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + if IsLocalRegistry(featureStore) { + err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) + if err != nil { + return repoConfig, err + } + } + } + + return repoConfig, nil +} - repoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) +func getBaseServiceRepoConfig( + featureStore *feastdevv1alpha1.FeatureStore, + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + + repoConfig := defaultRepoConfig(featureStore) + clientRepoConfig, err := getClientRepoConfig(featureStore, secretExtractionFunc) if err != nil { return repoConfig, err } + if isRemoteRegistry(featureStore) { + repoConfig.Registry = clientRepoConfig.Registry + } + repoConfig.AuthzConfig = clientRepoConfig.AuthzConfig + appliedSpec := featureStore.Status.Applied if appliedSpec.AuthzConfig != nil && appliedSpec.AuthzConfig.OidcAuthz != nil { propertiesMap, authSecretErr := secretExtractionFunc("", appliedSpec.AuthzConfig.OidcAuthz.SecretRef.Name, "") if authSecretErr != nil { @@ -75,43 +113,10 @@ func getServiceRepoConfig( repoConfig.AuthzConfig.OidcParameters = oidcServerProperties } - if appliedSpec.Services != nil { - services := appliedSpec.Services - - switch feastType { - case OfflineFeastType: - // Offline server has an `offline_store` section and a remote `registry` - if services.OfflineStore != nil { - err := setRepoConfigOffline(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - case OnlineFeastType: - // Online server has an `online_store` section, a remote `registry` and a remote `offline_store` - if services.OnlineStore != nil { - err := setRepoConfigOnline(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - case RegistryFeastType: - // Registry server only has a `registry` section - if IsLocalRegistry(featureStore) { - err := setRepoConfigRegistry(services, secretExtractionFunc, &repoConfig) - if err != nil { - return repoConfig, err - } - } - } - } - return repoConfig, nil } func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.Registry = RegistryConfig{} - repoConfig.Registry.Path = DefaultRegistryEphemeralPath registryPersistence := services.Registry.Local.Persistence if registryPersistence != nil { @@ -142,18 +147,10 @@ func setRepoConfigRegistry(services *feastdevv1alpha1.FeatureStoreServices, secr repoConfig.Registry.DBParameters = parametersMap } } - - repoConfig.OfflineStore = OfflineStoreConfig{} - repoConfig.OnlineStore = OnlineStoreConfig{} - return nil } func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.OnlineStore = OnlineStoreConfig{} - - repoConfig.OnlineStore.Path = DefaultOnlineStoreEphemeralPath - repoConfig.OnlineStore.Type = OnlineSqliteConfigType onlineStorePersistence := services.OnlineStore.Persistence if onlineStorePersistence != nil { @@ -183,13 +180,11 @@ func setRepoConfigOnline(services *feastdevv1alpha1.FeatureStoreServices, secret repoConfig.OnlineStore.DBParameters = parametersMap } } - return nil } func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error), repoConfig *RepoConfig) error { - repoConfig.OfflineStore = OfflineStoreConfig{} - repoConfig.OfflineStore.Type = OfflineFilePersistenceDaskConfigType + repoConfig.OfflineStore = defaultOfflineStoreConfig offlineStorePersistence := services.OfflineStore.Persistence if offlineStorePersistence != nil { @@ -218,9 +213,6 @@ func setRepoConfigOffline(services *feastdevv1alpha1.FeatureStoreServices, secre repoConfig.OfflineStore.DBParameters = parametersMap } } - - repoConfig.OnlineStore = OnlineStoreConfig{} - return nil } @@ -237,10 +229,9 @@ func getClientRepoConfig( secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { status := featureStore.Status appliedServices := status.Applied.Services - clientRepoConfig := RepoConfig{ - Project: status.Applied.FeastProject, - Provider: LocalProviderType, - EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + clientRepoConfig, err := getRepoConfig(featureStore, secretExtractionFunc) + if err != nil { + return clientRepoConfig, err } if len(status.ServiceHostnames.OfflineStore) > 0 { clientRepoConfig.OfflineStore = OfflineStoreConfig{ @@ -248,9 +239,8 @@ func getClientRepoConfig( Host: strings.Split(status.ServiceHostnames.OfflineStore, ":")[0], Port: HttpPort, } - if appliedServices.OfflineStore != nil && appliedServices.OfflineStore.TLS != nil && - (&appliedServices.OfflineStore.TLS.TlsConfigs).IsTLS() { - clientRepoConfig.OfflineStore.Cert = GetTlsPath(OfflineFeastType) + appliedServices.OfflineStore.TLS.TlsConfigs.SecretKeyNames.TlsCrt + if appliedServices.OfflineStore != nil && appliedServices.OfflineStore.TLS.IsTLS() { + clientRepoConfig.OfflineStore.Cert = GetTlsPath(OfflineFeastType) + appliedServices.OfflineStore.TLS.SecretKeyNames.TlsCrt clientRepoConfig.OfflineStore.Port = HttpsPort clientRepoConfig.OfflineStore.Scheme = HttpsScheme } @@ -278,23 +268,27 @@ func getClientRepoConfig( } } - if status.Applied.AuthzConfig == nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ - Type: NoAuthAuthType, - } - } else { + return clientRepoConfig, nil +} + +func getRepoConfig( + featureStore *feastdevv1alpha1.FeatureStore, + secretExtractionFunc func(storeType string, secretRef string, secretKeyName string) (map[string]interface{}, error)) (RepoConfig, error) { + status := featureStore.Status + repoConfig := initRepoConfig(status.Applied.FeastProject) + if status.Applied.AuthzConfig != nil { if status.Applied.AuthzConfig.KubernetesAuthz != nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ + repoConfig.AuthzConfig = AuthzConfig{ Type: KubernetesAuthType, } } else if status.Applied.AuthzConfig.OidcAuthz != nil { - clientRepoConfig.AuthzConfig = AuthzConfig{ + repoConfig.AuthzConfig = AuthzConfig{ Type: OidcAuthType, } propertiesMap, err := secretExtractionFunc("", status.Applied.AuthzConfig.OidcAuthz.SecretRef.Name, "") if err != nil { - return clientRepoConfig, err + return repoConfig, err } oidcClientProperties := map[string]interface{}{} @@ -302,13 +296,13 @@ func getClientRepoConfig( if val, exists := propertiesMap[string(oidcClientProperty)]; exists { oidcClientProperties[string(oidcClientProperty)] = val } else { - return clientRepoConfig, missingOidcSecretProperty(oidcClientProperty) + return repoConfig, missingOidcSecretProperty(oidcClientProperty) } } - clientRepoConfig.AuthzConfig.OidcParameters = oidcClientProperties + repoConfig.AuthzConfig.OidcParameters = oidcClientProperties } } - return clientRepoConfig, nil + return repoConfig, nil } func getActualPath(filePath string, pvcConfig *feastdevv1alpha1.PvcConfig) string { @@ -373,3 +367,49 @@ func mergeStructWithDBParametersMap(parametersMap *map[string]interface{}, s int return nil } + +func (feast *FeastServices) GetDefaultRepoConfig() RepoConfig { + return defaultRepoConfig(feast.Handler.FeatureStore) +} + +func defaultRepoConfig(featureStore *feastdevv1alpha1.FeatureStore) RepoConfig { + repoConfig := initRepoConfig(featureStore.Status.Applied.FeastProject) + repoConfig.OnlineStore = defaultOnlineStoreConfig(featureStore) + repoConfig.Registry = defaultRegistryConfig(featureStore) + return repoConfig +} + +func (feast *FeastServices) GetInitRepoConfig() RepoConfig { + return initRepoConfig(feast.Handler.FeatureStore.Status.Applied.FeastProject) +} + +func initRepoConfig(feastProject string) RepoConfig { + return RepoConfig{ + Project: feastProject, + Provider: LocalProviderType, + EntityKeySerializationVersion: feastdevv1alpha1.SerializationVersion, + AuthzConfig: defaultAuthzConfig, + } +} + +func defaultOnlineStoreConfig(featureStore *feastdevv1alpha1.FeatureStore) OnlineStoreConfig { + return OnlineStoreConfig{ + Type: OnlineSqliteConfigType, + Path: defaultOnlineStorePath(featureStore), + } +} + +func defaultRegistryConfig(featureStore *feastdevv1alpha1.FeatureStore) RegistryConfig { + return RegistryConfig{ + RegistryType: RegistryFileConfigType, + Path: defaultRegistryPath(featureStore), + } +} + +var defaultOfflineStoreConfig = OfflineStoreConfig{ + Type: OfflineFilePersistenceDaskConfigType, +} + +var defaultAuthzConfig = AuthzConfig{ + Type: NoAuthAuthType, +} diff --git a/infra/feast-operator/internal/controller/services/repo_config_test.go b/infra/feast-operator/internal/controller/services/repo_config_test.go index 7f017f4d102..9138f00f2f6 100644 --- a/infra/feast-operator/internal/controller/services/repo_config_test.go +++ b/infra/feast-operator/internal/controller/services/repo_config_test.go @@ -32,35 +32,25 @@ var projectName = "test-project" var _ = Describe("Repo Config", func() { Context("When creating the RepoConfig of a FeatureStore", func() { - It("should successfully create the repo configs", func() { By("Having the minimal created resource") featureStore := minimalFeatureStore() ApplyDefaultsToStatus(featureStore) - var repoConfig RepoConfig - repoConfig, err := getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig := RegistryConfig{ RegistryType: "file", - Path: DefaultRegistryEphemeralPath, + Path: EphemeralPath + "/" + DefaultRegistryPath, } + expectedOnlineConfig := OnlineStoreConfig{ + Type: "sqlite", + Path: EphemeralPath + "/" + DefaultOnlineStorePath, + } + + repoConfig, err := getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having the local registry resource") @@ -77,29 +67,17 @@ var _ = Describe("Repo Config", func() { }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig = RegistryConfig{ RegistryType: "file", Path: "file.db", } + + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) + Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having the remote registry resource") @@ -108,33 +86,47 @@ var _ = Describe("Repo Config", func() { Registry: &feastdevv1alpha1.Registry{ Remote: &feastdevv1alpha1.RemoteRegistryConfig{ FeastRef: &feastdevv1alpha1.FeatureStoreRef{ - Name: "registry", - Namespace: "remoteNS", + Name: "registry", }, }, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) + Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) + Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig)) - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) + By("Having an offlineStore with PVC") + mountPath := "/testing" + expectedOnlineConfig.Path = mountPath + "/" + DefaultOnlineStorePath + expectedRegistryConfig.Path = mountPath + "/" + DefaultRegistryPath + + featureStore = minimalFeatureStore() + featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ + OfflineStore: &feastdevv1alpha1.OfflineStore{ + Persistence: &feastdevv1alpha1.OfflineStorePersistence{ + FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{ + PvcConfig: &feastdevv1alpha1.PvcConfig{ + MountPath: mountPath, + }, + }, + }, + }, + } + ApplyDefaultsToStatus(featureStore) + appliedServices := featureStore.Status.Applied.Services + Expect(appliedServices.OnlineStore).To(BeNil()) + Expect(appliedServices.Registry.Local.Persistence.FilePersistence.Path).To(Equal(expectedRegistryConfig.Path)) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) + Expect(repoConfig.OfflineStore).To(Equal(defaultOfflineStoreConfig)) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) + Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) By("Having the all the file services") featureStore = minimalFeatureStore() @@ -164,36 +156,24 @@ var _ = Describe("Repo Config", func() { }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) + expectedOfflineConfig := OfflineStoreConfig{ Type: "duckdb", } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, emptyMockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig := OnlineStoreConfig{ + expectedRegistryConfig = RegistryConfig{ + RegistryType: "file", + Path: "/data/registry.db", + } + expectedOnlineConfig = OnlineStoreConfig{ Type: "sqlite", Path: "/data/online.db", } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, emptyMockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, emptyMockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(NoAuthAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: "/data/registry.db", - } + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) By("Having kubernetes authorization") @@ -202,56 +182,24 @@ var _ = Describe("Repo Config", func() { KubernetesAuthz: &feastdevv1alpha1.KubernetesAuthz{}, } featureStore.Spec.Services = &feastdevv1alpha1.FeatureStoreServices{ - OfflineStore: &feastdevv1alpha1.OfflineStore{ - Persistence: &feastdevv1alpha1.OfflineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OfflineStoreFilePersistence{}, - }, - }, - OnlineStore: &feastdevv1alpha1.OnlineStore{ - Persistence: &feastdevv1alpha1.OnlineStorePersistence{ - FilePersistence: &feastdevv1alpha1.OnlineStoreFilePersistence{}, - }, - }, + OfflineStore: &feastdevv1alpha1.OfflineStore{}, + OnlineStore: &feastdevv1alpha1.OnlineStore{}, Registry: &feastdevv1alpha1.Registry{ - Local: &feastdevv1alpha1.LocalRegistryConfig{ - Persistence: &feastdevv1alpha1.RegistryPersistence{ - FilePersistence: &feastdevv1alpha1.RegistryFilePersistence{}, - }, - }, + Local: &feastdevv1alpha1.LocalRegistryConfig{}, }, } ApplyDefaultsToStatus(featureStore) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) + expectedOfflineConfig = OfflineStoreConfig{ Type: "dask", } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig = OnlineStoreConfig{ - Type: "sqlite", - Path: DefaultOnlineStoreEphemeralPath, - } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, mockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, mockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(KubernetesAuthType)) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: DefaultRegistryEphemeralPath, - } - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) + Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) By("Having oidc authorization") featureStore.Spec.AuthzConfig = &feastdevv1alpha1.AuthzConfig{ @@ -269,46 +217,15 @@ var _ = Describe("Repo Config", func() { string(OidcClientSecret): "client-secret", string(OidcUsername): "username", string(OidcPassword): "password"}) - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + repoConfig, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).NotTo(HaveOccurred()) Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - expectedOfflineConfig = OfflineStoreConfig{ - Type: "dask", - } Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - expectedOnlineConfig = OnlineStoreConfig{ - Type: "sqlite", - Path: DefaultOnlineStoreEphemeralPath, - } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.AuthzConfig.Type).To(Equal(OidcAuthType)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveLen(2)) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcClientId))) - Expect(repoConfig.AuthzConfig.OidcParameters).To(HaveKey(string(OidcAuthDiscoveryUrl))) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - expectedRegistryConfig = RegistryConfig{ - RegistryType: "file", - Path: DefaultRegistryEphemeralPath, - } - Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) + Expect(repoConfig.OnlineStore).To(Equal(defaultOnlineStoreConfig(featureStore))) + Expect(repoConfig.Registry).To(Equal(defaultRegistryConfig(featureStore))) repoConfig, err = getClientRepoConfig(featureStore, secretExtractionFunc) Expect(err).NotTo(HaveOccurred()) @@ -359,7 +276,7 @@ var _ = Describe("Repo Config", func() { featureStore.Spec.Services.OfflineStore.Persistence.FilePersistence = nil featureStore.Spec.Services.OnlineStore.Persistence.FilePersistence = nil featureStore.Spec.Services.Registry.Local.Persistence.FilePersistence = nil - repoConfig, err = getServiceRepoConfig(OfflineFeastType, featureStore, mockExtractConfigFromSecret) + repoConfig, err = getServiceRepoConfig(featureStore, mockExtractConfigFromSecret) Expect(err).NotTo(HaveOccurred()) newMap := CopyMap(parameterMap) port := parameterMap["port"].(int) @@ -369,29 +286,16 @@ var _ = Describe("Repo Config", func() { Port: port, DBParameters: newMap, } - Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(OnlineFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - newMap = CopyMap(parameterMap) expectedOnlineConfig = OnlineStoreConfig{ Type: OnlineDBPersistenceSnowflakeConfigType, - DBParameters: newMap, + DBParameters: CopyMap(parameterMap), } - Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) - Expect(repoConfig.Registry).To(Equal(emptyRegistryConfig())) - - repoConfig, err = getServiceRepoConfig(RegistryFeastType, featureStore, mockExtractConfigFromSecret) - Expect(err).NotTo(HaveOccurred()) - Expect(repoConfig.OfflineStore).To(Equal(emptyOfflineStoreConfig())) - Expect(repoConfig.OnlineStore).To(Equal(emptyOnlineStoreConfig())) expectedRegistryConfig = RegistryConfig{ RegistryType: RegistryDBPersistenceSnowflakeConfigType, DBParameters: parameterMap, } + Expect(repoConfig.OfflineStore).To(Equal(expectedOfflineConfig)) + Expect(repoConfig.OnlineStore).To(Equal(expectedOnlineConfig)) Expect(repoConfig.Registry).To(Equal(expectedRegistryConfig)) }) }) @@ -413,13 +317,13 @@ var _ = Describe("Repo Config", func() { string(OidcClientSecret): "client-secret", string(OidcUsername): "username", string(OidcPassword): "password"}) - _, err := getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + _, err := getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) _, err = getClientRepoConfig(featureStore, secretExtractionFunc) @@ -440,13 +344,13 @@ var _ = Describe("Repo Config", func() { string(OidcClientId): "client-id", string(OidcUsername): "username", string(OidcPassword): "password"}) - _, err = getServiceRepoConfig(OfflineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(OnlineFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) - _, err = getServiceRepoConfig(RegistryFeastType, featureStore, secretExtractionFunc) + _, err = getServiceRepoConfig(featureStore, secretExtractionFunc) Expect(err).To(HaveOccurred()) Expect(err.Error()).To(ContainSubstring("missing OIDC secret")) _, err = getClientRepoConfig(featureStore, secretExtractionFunc) @@ -455,17 +359,8 @@ var _ = Describe("Repo Config", func() { }) }) -func emptyOnlineStoreConfig() OnlineStoreConfig { - return OnlineStoreConfig{} -} - -func emptyOfflineStoreConfig() OfflineStoreConfig { - return OfflineStoreConfig{} -} - -func emptyRegistryConfig() RegistryConfig { - return RegistryConfig{} -} +var emptyOfflineStoreConfig = OfflineStoreConfig{} +var emptyRegistryConfig = RegistryConfig{} func minimalFeatureStore() *feastdevv1alpha1.FeatureStore { return &feastdevv1alpha1.FeatureStore{ diff --git a/infra/feast-operator/internal/controller/services/services.go b/infra/feast-operator/internal/controller/services/services.go index 0f18cc55224..32cf91d09ba 100644 --- a/infra/feast-operator/internal/controller/services/services.go +++ b/infra/feast-operator/internal/controller/services/services.go @@ -106,7 +106,12 @@ func (feast *FeastServices) Deploy() error { return err } } - + if err := feast.createServiceAccount(); err != nil { + return err + } + if err := feast.createDeployment(); err != nil { + return err + } if err := feast.deployClient(); err != nil { return err } @@ -194,12 +199,6 @@ func (feast *FeastServices) deployFeastServiceByType(feastType FeastServiceType) if err := feast.createService(feastType); err != nil { return feast.setFeastServiceCondition(err, feastType) } - if err := feast.createServiceAccount(feastType); err != nil { - return feast.setFeastServiceCondition(err, feastType) - } - if err := feast.createDeployment(feastType); err != nil { - return feast.setFeastServiceCondition(err, feastType) - } return feast.setFeastServiceCondition(nil, feastType) } @@ -207,12 +206,6 @@ func (feast *FeastServices) removeFeastServiceByType(feastType FeastServiceType) if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastSvc(feastType)); err != nil { return err } - if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastDeploy(feastType)); err != nil { - return err - } - if err := feast.Handler.DeleteOwnedFeastObj(feast.initFeastSA(feastType)); err != nil { - return err - } if err := feast.Handler.DeleteOwnedFeastObj(feast.initPVC(feastType)); err != nil { return err } @@ -233,11 +226,11 @@ func (feast *FeastServices) createService(feastType FeastServiceType) error { return nil } -func (feast *FeastServices) createServiceAccount(feastType FeastServiceType) error { +func (feast *FeastServices) createServiceAccount() error { logger := log.FromContext(feast.Handler.Context) - sa := feast.initFeastSA(feastType) + sa := feast.initFeastSA() if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, sa, controllerutil.MutateFn(func() error { - return feast.setServiceAccount(sa, feastType) + return feast.setServiceAccount(sa) })); err != nil { return err } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { @@ -246,11 +239,11 @@ func (feast *FeastServices) createServiceAccount(feastType FeastServiceType) err return nil } -func (feast *FeastServices) createDeployment(feastType FeastServiceType) error { +func (feast *FeastServices) createDeployment() error { logger := log.FromContext(feast.Handler.Context) - deploy := feast.initFeastDeploy(feastType) + deploy := feast.initFeastDeploy() if op, err := controllerutil.CreateOrUpdate(feast.Handler.Context, feast.Handler.Client, deploy, controllerutil.MutateFn(func() error { - return feast.setDeployment(deploy, feastType) + return feast.setDeployment(deploy) })); err != nil { return err } else if op == controllerutil.OperationResultCreated || op == controllerutil.OperationResultUpdated { @@ -280,78 +273,122 @@ func (feast *FeastServices) createPVC(pvcCreate *feastdevv1alpha1.PvcCreate, fea return nil } -func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment, feastType FeastServiceType) error { - fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64(feastType) - if err != nil { - return err - } - deploy.Labels = feast.getLabels(feastType) - sa := feast.initFeastSA(feastType) - tls := feast.getTlsConfigs(feastType) - serviceConfigs := feast.getServiceConfigs(feastType) - defaultServiceConfigs := serviceConfigs.DefaultConfigs - probeHandler := getProbeHandler(feastType, tls) - +func (feast *FeastServices) setDeployment(deploy *appsv1.Deployment) error { + deploy.Labels = feast.getLabels() deploy.Spec = appsv1.DeploymentSpec{ - Replicas: feast.getServiceReplicas(feastType), + Replicas: &DefaultReplicas, Selector: metav1.SetAsLabelSelector(deploy.GetLabels()), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, Template: corev1.PodTemplateSpec{ ObjectMeta: metav1.ObjectMeta{ Labels: deploy.GetLabels(), }, Spec: corev1.PodSpec{ - ServiceAccountName: sa.Name, - Containers: []corev1.Container{ - { - Name: string(feastType), - Image: *defaultServiceConfigs.Image, - Command: feast.getContainerCommand(feastType), - Ports: []corev1.ContainerPort{ - { - Name: string(feastType), - ContainerPort: getTargetPort(feastType, tls), - Protocol: corev1.ProtocolTCP, - }, - }, - Env: []corev1.EnvVar{ - { - Name: FeatureStoreYamlEnvVar, - Value: fsYamlB64, - }, - }, - LivenessProbe: &corev1.Probe{ - ProbeHandler: probeHandler, - InitialDelaySeconds: 30, - PeriodSeconds: 30, - }, - ReadinessProbe: &corev1.Probe{ - ProbeHandler: probeHandler, - InitialDelaySeconds: 20, - PeriodSeconds: 30, - }, - }, - }, + ServiceAccountName: feast.initFeastSA().Name, }, }, } + if err := feast.setPod(&deploy.Spec.Template.Spec); err != nil { + return err + } + return controllerutil.SetControllerReference(feast.Handler.FeatureStore, deploy, feast.Handler.Scheme) +} - // configs are applied here - podSpec := &deploy.Spec.Template.Spec - applyOptionalContainerConfigs(&podSpec.Containers[0], serviceConfigs.OptionalConfigs) - feast.mountTlsConfig(feastType, podSpec) - if pvcConfig, hasPvcConfig := hasPvcConfig(feast.Handler.FeatureStore, feastType); hasPvcConfig { - mountPvcConfig(podSpec, pvcConfig, deploy.Name) +func (feast *FeastServices) setPod(podSpec *corev1.PodSpec) error { + if err := feast.setContainers(podSpec); err != nil { + return err } + feast.setRegistryClientInitContainer(podSpec) + feast.mountTlsConfigs(podSpec) + feast.mountPvcConfigs(podSpec) + feast.mountEmptyDirVolumes(podSpec) - switch feastType { - case OfflineFeastType: - feast.registryClientPodConfigs(podSpec) - case OnlineFeastType: - feast.registryClientPodConfigs(podSpec) - feast.offlineClientPodConfigs(podSpec) + return nil +} + +func (feast *FeastServices) setContainers(podSpec *corev1.PodSpec) error { + fsYamlB64, err := feast.GetServiceFeatureStoreYamlBase64() + if err != nil { + return err + } + feastProject := feast.Handler.FeatureStore.Status.Applied.FeastProject + workingDir := getOfflineMountPath(feast.Handler.FeatureStore) + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + Name: "feast-init", + Image: DefaultImage, + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + }, + Command: []string{"/bin/sh", "-c"}, + Args: []string{"echo \"Starting feast initialization job...\";\n[ -d " + + feastProject + " ] || feast init " + feastProject + ";\necho $" + + TmpFeatureStoreYamlEnvVar + " | base64 -d \u003e " + workingDir + "/" + feastProject + + "/feature_repo/feature_store.yaml;\necho \"Feast initialization complete\";\n"}, + WorkingDir: workingDir, + }) + if feast.isLocalRegistry() { + feast.setContainer(&podSpec.Containers, RegistryFeastType, fsYamlB64) + } + if feast.isOfflinStore() { + feast.setContainer(&podSpec.Containers, OfflineFeastType, fsYamlB64) } + if feast.isOnlinStore() { + feast.setContainer(&podSpec.Containers, OnlineFeastType, fsYamlB64) + } + return nil +} - return controllerutil.SetControllerReference(feast.Handler.FeatureStore, deploy, feast.Handler.Scheme) +func (feast *FeastServices) setContainer(containers *[]corev1.Container, feastType FeastServiceType, fsYamlB64 string) { + tls := feast.getTlsConfigs(feastType) + serviceConfigs := feast.getServiceConfigs(feastType) + defaultServiceConfigs := serviceConfigs.DefaultConfigs + probeHandler := getProbeHandler(feastType, tls) + container := &corev1.Container{ + Name: string(feastType), + Image: *defaultServiceConfigs.Image, + WorkingDir: getOfflineMountPath(feast.Handler.FeatureStore) + "/" + feast.Handler.FeatureStore.Status.Applied.FeastProject + "/feature_repo", + Command: feast.getContainerCommand(feastType), + Ports: []corev1.ContainerPort{ + { + Name: string(feastType), + ContainerPort: getTargetPort(feastType, tls), + Protocol: corev1.ProtocolTCP, + }, + }, + Env: []corev1.EnvVar{ + { + Name: TmpFeatureStoreYamlEnvVar, + Value: fsYamlB64, + }, + /* + { + Name: mlpConfigVar, + Value: DefaultMlpConfigPath, + }, + */ + }, + StartupProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 3, + FailureThreshold: 40, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 20, + FailureThreshold: 6, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: probeHandler, + PeriodSeconds: 10, + }, + } + applyOptionalContainerConfigs(container, serviceConfigs.OptionalConfigs) + *containers = append(*containers, *container) } func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []string { @@ -373,13 +410,6 @@ func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []st } deploySettings.Args = append(deploySettings.Args, []string{"-p", strconv.Itoa(int(targetPort))}...) - if feastType == OfflineFeastType { - if tls.IsTLS() && feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient != nil { - deploySettings.Args = append(deploySettings.Args, - []string{"--verify_client", strconv.FormatBool(*feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.VerifyClient)}...) - } - } - // Combine base command, options, and arguments feastCommand := append([]string{baseCommand}, options...) feastCommand = append(feastCommand, deploySettings.Args...) @@ -387,38 +417,28 @@ func (feast *FeastServices) getContainerCommand(feastType FeastServiceType) []st return feastCommand } -func (feast *FeastServices) offlineClientPodConfigs(podSpec *corev1.PodSpec) { - feast.mountTlsConfig(OfflineFeastType, podSpec) -} - -func (feast *FeastServices) registryClientPodConfigs(podSpec *corev1.PodSpec) { - feast.setRegistryClientInitContainer(podSpec) - feast.mountRegistryClientTls(podSpec) -} - func (feast *FeastServices) setRegistryClientInitContainer(podSpec *corev1.PodSpec) { hostname := feast.Handler.FeatureStore.Status.ServiceHostnames.Registry - if len(hostname) > 0 { + // add grpc init container if remote registry reference (feastRef) is configured + if len(hostname) > 0 && feast.IsRemoteRefRegistry() { grpcurlFlag := "-plaintext" hostSplit := strings.Split(hostname, ":") if len(hostSplit) > 1 && hostSplit[1] == "443" { grpcurlFlag = "-insecure" } - podSpec.InitContainers = []corev1.Container{ - { - Name: "init-registry", - Image: "fullstorydev/grpcurl:v1.9.1-alpine", - Command: []string{ - "sh", "-c", - "until grpcurl " + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", - }, + podSpec.InitContainers = append(podSpec.InitContainers, corev1.Container{ + Name: "init-registry", + Image: "fullstorydev/grpcurl:v1.9.1-alpine", + Command: []string{ + "sh", "-c", + "until grpcurl -H \"authorization: Bearer $(cat /var/run/secrets/kubernetes.io/serviceaccount/token)\" " + + grpcurlFlag + " -d '' -format text " + hostname + " grpc.health.v1.Health/Check; do echo waiting for registry; sleep 2; done", }, - } + }) } } - func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServiceType) error { - svc.Labels = feast.getLabels(feastType) + svc.Labels = feast.getFeastTypeLabels(feastType) if feast.isOpenShiftTls(feastType) { svc.Annotations = map[string]string{ "service.beta.openshift.io/serving-cert-secret-name": svc.Name + tlsNameSuffix, @@ -433,7 +453,7 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi scheme = HttpsScheme } svc.Spec = corev1.ServiceSpec{ - Selector: svc.GetLabels(), + Selector: feast.getLabels(), Type: corev1.ServiceTypeClusterIP, Ports: []corev1.ServicePort{ { @@ -448,8 +468,8 @@ func (feast *FeastServices) setService(svc *corev1.Service, feastType FeastServi return controllerutil.SetControllerReference(feast.Handler.FeatureStore, svc, feast.Handler.Scheme) } -func (feast *FeastServices) setServiceAccount(sa *corev1.ServiceAccount, feastType FeastServiceType) error { - sa.Labels = feast.getLabels(feastType) +func (feast *FeastServices) setServiceAccount(sa *corev1.ServiceAccount) error { + sa.Labels = feast.getLabels() return controllerutil.SetControllerReference(feast.Handler.FeatureStore, sa, feast.Handler.Scheme) } @@ -457,7 +477,7 @@ func (feast *FeastServices) createNewPVC(pvcCreate *feastdevv1alpha1.PvcCreate, pvc := feast.initPVC(feastType) pvc.Spec = corev1.PersistentVolumeClaimSpec{ - AccessModes: []corev1.PersistentVolumeAccessMode{corev1.ReadWriteMany}, + AccessModes: pvcCreate.AccessModes, Resources: pvcCreate.Resources, } if pvcCreate.StorageClassName != nil { @@ -485,21 +505,6 @@ func (feast *FeastServices) getServiceConfigs(feastType FeastServiceType) feastd return feastdevv1alpha1.ServiceConfigs{} } -func (feast *FeastServices) getServiceReplicas(feastType FeastServiceType) *int32 { - appliedServices := feast.Handler.FeatureStore.Status.Applied.Services - switch feastType { - case OfflineFeastType: - if feast.isOfflinStore() { - return appliedServices.OfflineStore.Replicas - } - case OnlineFeastType: - if feast.isOnlinStore() { - return appliedServices.OnlineStore.Replicas - } - } - return &DefaultReplicas -} - func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *string { services := feast.Handler.FeatureStore.Status.Applied.Services switch feastType { @@ -519,8 +524,13 @@ func (feast *FeastServices) getLogLevelForType(feastType FeastServiceType) *stri return nil } -// GetObjectMeta returns the feast k8s object metadata -func (feast *FeastServices) GetObjectMeta(feastType FeastServiceType) metav1.ObjectMeta { +// GetObjectMeta returns the feast k8s object metadata with type +func (feast *FeastServices) GetObjectMeta() metav1.ObjectMeta { + return metav1.ObjectMeta{Name: GetFeastName(feast.Handler.FeatureStore), Namespace: feast.Handler.FeatureStore.Namespace} +} + +// GetObjectMeta returns the feast k8s object metadata with type +func (feast *FeastServices) GetObjectMetaType(feastType FeastServiceType) metav1.ObjectMeta { return metav1.ObjectMeta{Name: feast.GetFeastServiceName(feastType), Namespace: feast.Handler.FeatureStore.Namespace} } @@ -537,10 +547,15 @@ func GetFeastName(featureStore *feastdevv1alpha1.FeatureStore) string { return handler.FeastPrefix + featureStore.Name } -func (feast *FeastServices) getLabels(feastType FeastServiceType) map[string]string { +func (feast *FeastServices) getFeastTypeLabels(feastType FeastServiceType) map[string]string { + labels := feast.getLabels() + labels[ServiceTypeLabelKey] = string(feastType) + return labels +} + +func (feast *FeastServices) getLabels() map[string]string { return map[string]string{ - NameLabelKey: feast.Handler.FeatureStore.Name, - ServiceTypeLabelKey: string(feastType), + NameLabelKey: feast.Handler.FeatureStore.Name, } } @@ -548,20 +563,17 @@ func (feast *FeastServices) setServiceHostnames() error { feast.Handler.FeatureStore.Status.ServiceHostnames = feastdevv1alpha1.ServiceHostnames{} domain := svcDomain + ":" if feast.isOfflinStore() { - objMeta := feast.GetObjectMeta(OfflineFeastType) - port := strconv.Itoa(HttpPort) - if feast.offlineTls() { - port = strconv.Itoa(HttpsPort) - } - feast.Handler.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + port + objMeta := feast.initFeastSvc(OfflineFeastType) + feast.Handler.FeatureStore.Status.ServiceHostnames.OfflineStore = objMeta.Name + "." + objMeta.Namespace + domain + + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS) } if feast.isOnlinStore() { - objMeta := feast.GetObjectMeta(OnlineFeastType) + objMeta := feast.initFeastSvc(OnlineFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.OnlineStore = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.OnlineStore.TLS) } if feast.isLocalRegistry() { - objMeta := feast.GetObjectMeta(RegistryFeastType) + objMeta := feast.initFeastSvc(RegistryFeastType) feast.Handler.FeatureStore.Status.ServiceHostnames.Registry = objMeta.Name + "." + objMeta.Namespace + domain + getPortStr(feast.Handler.FeatureStore.Status.Applied.Services.Registry.Local.TLS) } else if feast.isRemoteRegistry() { @@ -608,10 +620,6 @@ func (feast *FeastServices) setRemoteRegistryURL() error { func (feast *FeastServices) getRemoteRegistryFeastHandler() (*FeastServices, error) { if feast.IsRemoteRefRegistry() { feastRemoteRef := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef - // default to FeatureStore namespace if not set - if len(feastRemoteRef.Namespace) == 0 { - feastRemoteRef.Namespace = feast.Handler.FeatureStore.Namespace - } nsName := types.NamespacedName{Name: feastRemoteRef.Name, Namespace: feastRemoteRef.Namespace} crNsName := client.ObjectKeyFromObject(feast.Handler.FeatureStore) if nsName == crNsName { @@ -645,19 +653,13 @@ func (feast *FeastServices) isRemoteRegistry() bool { } func (feast *FeastServices) IsRemoteRefRegistry() bool { - if feast.isRemoteRegistry() { - remote := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote - return remote != nil && remote.FeastRef != nil - } - return false + return feast.isRemoteRegistry() && + feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.FeastRef != nil } func (feast *FeastServices) isRemoteHostnameRegistry() bool { - if feast.isRemoteRegistry() { - remote := feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote - return remote != nil && remote.Hostname != nil - } - return false + return feast.isRemoteRegistry() && + feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.Hostname != nil } func (feast *FeastServices) isOfflinStore() bool { @@ -670,9 +672,9 @@ func (feast *FeastServices) isOnlinStore() bool { return appliedServices != nil && appliedServices.OnlineStore != nil } -func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1.Deployment { +func (feast *FeastServices) initFeastDeploy() *appsv1.Deployment { deploy := &appsv1.Deployment{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMeta(), } deploy.SetGroupVersionKind(appsv1.SchemeGroupVersion.WithKind("Deployment")) return deploy @@ -680,15 +682,15 @@ func (feast *FeastServices) initFeastDeploy(feastType FeastServiceType) *appsv1. func (feast *FeastServices) initFeastSvc(feastType FeastServiceType) *corev1.Service { svc := &corev1.Service{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMetaType(feastType), } svc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("Service")) return svc } -func (feast *FeastServices) initFeastSA(feastType FeastServiceType) *corev1.ServiceAccount { +func (feast *FeastServices) initFeastSA() *corev1.ServiceAccount { sa := &corev1.ServiceAccount{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMeta(), } sa.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("ServiceAccount")) return sa @@ -696,7 +698,7 @@ func (feast *FeastServices) initFeastSA(feastType FeastServiceType) *corev1.Serv func (feast *FeastServices) initPVC(feastType FeastServiceType) *corev1.PersistentVolumeClaim { pvc := &corev1.PersistentVolumeClaim{ - ObjectMeta: feast.GetObjectMeta(feastType), + ObjectMeta: feast.GetObjectMetaType(feastType), } pvc.SetGroupVersionKind(corev1.SchemeGroupVersion.WithKind("PersistentVolumeClaim")) return pvc @@ -714,28 +716,73 @@ func applyOptionalContainerConfigs(container *corev1.Container, optionalConfigs } } -func mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, deployName string) { +func (feast *FeastServices) mountPvcConfigs(podSpec *corev1.PodSpec) { + for _, feastType := range feastServerTypes { + if pvcConfig, hasPvcConfig := hasPvcConfig(feast.Handler.FeatureStore, feastType); hasPvcConfig { + feast.mountPvcConfig(podSpec, pvcConfig, feastType) + } + } +} + +func (feast *FeastServices) mountPvcConfig(podSpec *corev1.PodSpec, pvcConfig *feastdevv1alpha1.PvcConfig, feastType FeastServiceType) { if podSpec != nil && pvcConfig != nil { - container := &podSpec.Containers[0] - var pvcName string - if pvcConfig.Create != nil { - pvcName = deployName - } else { + volName := feast.initPVC(feastType).Name + pvcName := volName + if pvcConfig.Ref != nil { pvcName = pvcConfig.Ref.Name } - podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ - Name: pvcName, + Name: volName, VolumeSource: corev1.VolumeSource{ PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ ClaimName: pvcName, }, }, }) - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: pvcName, - MountPath: pvcConfig.MountPath, + if feastType == OfflineFeastType { + for i := range podSpec.InitContainers { + podSpec.InitContainers[i].VolumeMounts = append(podSpec.InitContainers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: pvcConfig.MountPath, + }) + } + } + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: pvcConfig.MountPath, + }) + } + } +} + +func (feast *FeastServices) mountEmptyDirVolumes(podSpec *corev1.PodSpec) { + if shouldMountEmptyDir(feast.Handler.FeatureStore) { + mountEmptyDirVolume(podSpec) + } +} + +func mountEmptyDirVolume(podSpec *corev1.PodSpec) { + if podSpec != nil { + volName := strings.TrimPrefix(EphemeralPath, "/") + podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ + Name: volName, + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, }) + for i := range podSpec.InitContainers { + podSpec.InitContainers[i].VolumeMounts = append(podSpec.InitContainers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: EphemeralPath, + }) + } + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: EphemeralPath, + }) + } } } diff --git a/infra/feast-operator/internal/controller/services/services_types.go b/infra/feast-operator/internal/controller/services/services_types.go index b7c0f5f048b..882839a429f 100644 --- a/infra/feast-operator/internal/controller/services/services_types.go +++ b/infra/feast-operator/internal/controller/services/services_types.go @@ -20,17 +20,17 @@ import ( "github.com/feast-dev/feast/infra/feast-operator/api/feastversion" feastdevv1alpha1 "github.com/feast-dev/feast/infra/feast-operator/api/v1alpha1" handler "github.com/feast-dev/feast/infra/feast-operator/internal/controller/handler" + corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) const ( - FeatureStoreYamlEnvVar = "FEATURE_STORE_YAML_BASE64" - FeatureStoreYamlCmKey = "feature_store.yaml" - DefaultRegistryEphemeralPath = "/tmp/registry.db" - DefaultRegistryPvcPath = "registry.db" - DefaultOnlineStoreEphemeralPath = "/tmp/online_store.db" - DefaultOnlineStorePvcPath = "online_store.db" - svcDomain = ".svc.cluster.local" + TmpFeatureStoreYamlEnvVar = "TMP_FEATURE_STORE_YAML_BASE64" + FeatureStoreYamlCmKey = "feature_store.yaml" + EphemeralPath = "/feast-data" + DefaultRegistryPath = "registry.db" + DefaultOnlineStorePath = "online_store.db" + svcDomain = ".svc.cluster.local" HttpPort = 80 HttpsPort = 443 @@ -80,10 +80,11 @@ const ( ) var ( - DefaultImage = "feastdev/feature-server:" + feastversion.FeastVersion - DefaultReplicas = int32(1) - NameLabelKey = feastdevv1alpha1.GroupVersion.Group + "/name" - ServiceTypeLabelKey = feastdevv1alpha1.GroupVersion.Group + "/service-type" + DefaultImage = "feastdev/feature-server:" + feastversion.FeastVersion + DefaultReplicas = int32(1) + DefaultPVCAccessModes = []corev1.PersistentVolumeAccessMode{corev1.ReadWriteOnce} + NameLabelKey = feastdevv1alpha1.GroupVersion.Group + "/name" + ServiceTypeLabelKey = feastdevv1alpha1.GroupVersion.Group + "/service-type" FeastServiceConstants = map[FeastServiceType]deploymentSettings{ OfflineFeastType: { @@ -162,6 +163,13 @@ var ( OidcClientProperties = []OidcPropertyType{OidcClientSecret, OidcUsername, OidcPassword} ) +// feast server types, not the client types +var feastServerTypes = []FeastServiceType{ + RegistryFeastType, + OfflineFeastType, + OnlineFeastType, +} + // AuthzType defines the authorization type type AuthzType string diff --git a/infra/feast-operator/internal/controller/services/tls.go b/infra/feast-operator/internal/controller/services/tls.go index c92c4d8de23..6dcca7edea1 100644 --- a/infra/feast-operator/internal/controller/services/tls.go +++ b/infra/feast-operator/internal/controller/services/tls.go @@ -29,7 +29,7 @@ func (feast *FeastServices) setTlsDefaults() error { } appliedServices := feast.Handler.FeatureStore.Status.Applied.Services if feast.isOfflinStore() && appliedServices.OfflineStore.TLS != nil { - tlsDefaults(&appliedServices.OfflineStore.TLS.TlsConfigs) + tlsDefaults(appliedServices.OfflineStore.TLS) } if feast.isOnlinStore() { tlsDefaults(appliedServices.OnlineStore.TLS) @@ -43,11 +43,9 @@ func (feast *FeastServices) setTlsDefaults() error { func (feast *FeastServices) setOpenshiftTls() error { appliedServices := feast.Handler.FeatureStore.Status.Applied.Services if feast.offlineOpenshiftTls() { - appliedServices.OfflineStore.TLS = &feastdevv1alpha1.OfflineTlsConfigs{ - TlsConfigs: feastdevv1alpha1.TlsConfigs{ - SecretRef: &corev1.LocalObjectReference{ - Name: feast.initFeastSvc(OfflineFeastType).Name + tlsNameSuffix, - }, + appliedServices.OfflineStore.TLS = &feastdevv1alpha1.TlsConfigs{ + SecretRef: &corev1.LocalObjectReference{ + Name: feast.initFeastSvc(OfflineFeastType).Name + tlsNameSuffix, }, } } @@ -103,8 +101,8 @@ func (feast *FeastServices) getTlsConfigs(feastType FeastServiceType) (tls *feas appliedServices := feast.Handler.FeatureStore.Status.Applied.Services switch feastType { case OfflineFeastType: - if feast.isOfflinStore() && appliedServices.OfflineStore.TLS != nil { - tls = &appliedServices.OfflineStore.TLS.TlsConfigs + if feast.isOfflinStore() { + tls = appliedServices.OfflineStore.TLS } case OnlineFeastType: if feast.isOnlinStore() { @@ -154,12 +152,6 @@ func (feast *FeastServices) remoteRegistryOpenshiftTls() (bool, error) { return false, nil } -func (feast *FeastServices) offlineTls() bool { - return feast.isOfflinStore() && - feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS != nil && - (&feast.Handler.FeatureStore.Status.Applied.Services.OfflineStore.TLS.TlsConfigs).IsTLS() -} - func (feast *FeastServices) localRegistryTls() bool { return localRegistryTls(feast.Handler.FeatureStore) } @@ -173,12 +165,19 @@ func (feast *FeastServices) mountRegistryClientTls(podSpec *corev1.PodSpec) { if feast.localRegistryTls() { feast.mountTlsConfig(RegistryFeastType, podSpec) } else if feast.remoteRegistryTls() { - mountTlsRemoteRegistryConfig(RegistryFeastType, podSpec, + mountTlsRemoteRegistryConfig(podSpec, feast.Handler.FeatureStore.Status.Applied.Services.Registry.Remote.TLS) } } } +func (feast *FeastServices) mountTlsConfigs(podSpec *corev1.PodSpec) { + // how deal w/ client deployment tls mounts when the time comes? new function? + feast.mountRegistryClientTls(podSpec) + feast.mountTlsConfig(OfflineFeastType, podSpec) + feast.mountTlsConfig(OnlineFeastType, podSpec) +} + func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec *corev1.PodSpec) { tls := feast.getTlsConfigs(feastType) if tls.IsTLS() && podSpec != nil { @@ -191,18 +190,19 @@ func (feast *FeastServices) mountTlsConfig(feastType FeastServiceType, podSpec * }, }, }) - container := &podSpec.Containers[0] - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: volName, - MountPath: GetTlsPath(feastType), - ReadOnly: true, - }) + if i, container := getContainerByType(feastType, podSpec.Containers); container != nil { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(feastType), + ReadOnly: true, + }) + } } } -func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { +func mountTlsRemoteRegistryConfig(podSpec *corev1.PodSpec, tls *feastdevv1alpha1.TlsRemoteRegistryConfigs) { if tls != nil { - volName := string(feastType) + tlsNameSuffix + volName := string(RegistryFeastType) + tlsNameSuffix podSpec.Volumes = append(podSpec.Volumes, corev1.Volume{ Name: volName, VolumeSource: corev1.VolumeSource{ @@ -211,12 +211,13 @@ func mountTlsRemoteRegistryConfig(feastType FeastServiceType, podSpec *corev1.Po }, }, }) - container := &podSpec.Containers[0] - container.VolumeMounts = append(container.VolumeMounts, corev1.VolumeMount{ - Name: volName, - MountPath: GetTlsPath(feastType), - ReadOnly: true, - }) + for i := range podSpec.Containers { + podSpec.Containers[i].VolumeMounts = append(podSpec.Containers[i].VolumeMounts, corev1.VolumeMount{ + Name: volName, + MountPath: GetTlsPath(RegistryFeastType), + ReadOnly: true, + }) + } } } diff --git a/infra/feast-operator/internal/controller/services/tls_test.go b/infra/feast-operator/internal/controller/services/tls_test.go index 2a66d8a4fdd..522eb2265b5 100644 --- a/infra/feast-operator/internal/controller/services/tls_test.go +++ b/infra/feast-operator/internal/controller/services/tls_test.go @@ -58,7 +58,6 @@ var _ = Describe("TLS Config", func() { Expect(tls.IsTLS()).To(BeFalse()) Expect(getPortStr(tls)).To(Equal("80")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeFalse()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) @@ -87,7 +86,6 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("443")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) @@ -127,7 +125,6 @@ var _ = Describe("TLS Config", func() { Expect(tls.SecretKeyNames).To(Equal(secretKeyNames)) Expect(tls.IsTLS()).To(BeTrue()) - Expect(feast.offlineTls()).To(BeTrue()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeTrue()) @@ -138,23 +135,15 @@ var _ = Describe("TLS Config", func() { Expect(openshiftTls).To(BeTrue()) // check k8s deployment objects - offlineDeploy := feast.initFeastDeploy(OfflineFeastType) - err = feast.setDeployment(offlineDeploy, OfflineFeastType) + feastDeploy := feast.initFeastDeploy() + err = feast.setDeployment(feastDeploy) Expect(err).To(BeNil()) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) - Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) - onlineDeploy := feast.initFeastDeploy(OnlineFeastType) - err = feast.setDeployment(onlineDeploy, OnlineFeastType) - Expect(err).To(BeNil()) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-insecure"))) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(3)) + Expect(feastDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) + Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(feastDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Containers[1].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Containers[2].Command).To(ContainElements(ContainSubstring("--key"))) + Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(4)) // registry service w/ tls and in an openshift cluster feast.Handler.FeatureStore = minimalFeatureStore() @@ -189,7 +178,6 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("443")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeFalse()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeTrue()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeFalse()) @@ -238,7 +226,6 @@ var _ = Describe("TLS Config", func() { Expect(getPortStr(tls)).To(Equal("80")) Expect(GetTlsPath(RegistryFeastType)).To(Equal("/tls/registry/")) - Expect(feast.offlineTls()).To(BeTrue()) Expect(feast.remoteRegistryTls()).To(BeFalse()) Expect(feast.localRegistryTls()).To(BeFalse()) Expect(feast.isOpenShiftTls(OfflineFeastType)).To(BeTrue()) @@ -263,22 +250,19 @@ var _ = Describe("TLS Config", func() { Expect(onlineSvc.Spec.Ports[0].Name).To(Equal(HttpScheme)) // check k8s deployment objects - offlineDeploy = feast.initFeastDeploy(OfflineFeastType) - err = feast.setDeployment(offlineDeploy, OfflineFeastType) - Expect(err).To(BeNil()) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) - Expect(offlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(offlineDeploy.Spec.Template.Spec.Containers[0].Command).To(ContainElements(ContainSubstring("--key"))) - Expect(offlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) - onlineDeploy = feast.initFeastDeploy(OnlineFeastType) - err = feast.setDeployment(onlineDeploy, OnlineFeastType) + feastDeploy = feast.initFeastDeploy() + err = feast.setDeployment(feastDeploy) Expect(err).To(BeNil()) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.InitContainers[0].Command).To(ContainElements(ContainSubstring("-plaintext"))) - Expect(onlineDeploy.Spec.Template.Spec.Containers).To(HaveLen(1)) - Expect(onlineDeploy.Spec.Template.Spec.Containers[0].Command).NotTo(ContainElements(ContainSubstring("--key"))) - Expect(onlineDeploy.Spec.Template.Spec.Volumes).To(HaveLen(1)) + Expect(feastDeploy.Spec.Template.Spec.Containers).To(HaveLen(3)) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers)).NotTo(BeNil()) + Expect(feastDeploy.Spec.Template.Spec.Volumes).To(HaveLen(2)) + + Expect(GetRegistryContainer(feastDeploy.Spec.Template.Spec.Containers).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetRegistryContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers).Command).To(ContainElements(ContainSubstring("--key"))) + Expect(GetOfflineContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(2)) + Expect(GetOnlineContainer(feastDeploy.Spec.Template.Spec.Containers).Command).NotTo(ContainElements(ContainSubstring("--key"))) + Expect(GetOnlineContainer(feastDeploy.Spec.Template.Spec.Containers).VolumeMounts).To(HaveLen(1)) }) }) }) diff --git a/infra/feast-operator/internal/controller/services/util.go b/infra/feast-operator/internal/controller/services/util.go index 631709d6ba0..7b9c177c89d 100644 --- a/infra/feast-operator/internal/controller/services/util.go +++ b/infra/feast-operator/internal/controller/services/util.go @@ -32,20 +32,25 @@ func isRemoteRegistry(featureStore *feastdevv1alpha1.FeatureStore) bool { } func hasPvcConfig(featureStore *feastdevv1alpha1.FeatureStore, feastType FeastServiceType) (*feastdevv1alpha1.PvcConfig, bool) { + var pvcConfig *feastdevv1alpha1.PvcConfig services := featureStore.Status.Applied.Services - var pvcConfig *feastdevv1alpha1.PvcConfig = nil - switch feastType { - case OnlineFeastType: - if services.OnlineStore != nil && services.OnlineStore.Persistence.FilePersistence != nil { - pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig - } - case OfflineFeastType: - if services.OfflineStore != nil && services.OfflineStore.Persistence.FilePersistence != nil { - pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig - } - case RegistryFeastType: - if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence.FilePersistence != nil { - pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig + if services != nil { + switch feastType { + case OnlineFeastType: + if services.OnlineStore != nil && services.OnlineStore.Persistence != nil && + services.OnlineStore.Persistence.FilePersistence != nil { + pvcConfig = services.OnlineStore.Persistence.FilePersistence.PvcConfig + } + case OfflineFeastType: + if services.OfflineStore != nil && services.OfflineStore.Persistence != nil && + services.OfflineStore.Persistence.FilePersistence != nil { + pvcConfig = services.OfflineStore.Persistence.FilePersistence.PvcConfig + } + case RegistryFeastType: + if IsLocalRegistry(featureStore) && services.Registry.Local.Persistence != nil && + services.Registry.Local.Persistence.FilePersistence != nil { + pvcConfig = services.Registry.Local.Persistence.FilePersistence.PvcConfig + } } } return pvcConfig, pvcConfig != nil @@ -58,10 +63,24 @@ func shouldCreatePvc(featureStore *feastdevv1alpha1.FeatureStore, feastType Feas return nil, false } +func shouldMountEmptyDir(featureStore *feastdevv1alpha1.FeatureStore) bool { + _, ok := hasPvcConfig(featureStore, OfflineFeastType) + return !ok +} + +func getOfflineMountPath(featureStore *feastdevv1alpha1.FeatureStore) string { + if pvcConfig, ok := hasPvcConfig(featureStore, OfflineFeastType); ok { + return pvcConfig.MountPath + } + return EphemeralPath +} + func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { + // overwrite status.applied with every reconcile + cr.Spec.DeepCopyInto(&cr.Status.Applied) cr.Status.FeastVersion = feastversion.FeastVersion - applied := cr.Spec.DeepCopy() + applied := &cr.Status.Applied if applied.Services == nil { applied.Services = &feastdevv1alpha1.FeatureStoreServices{} } @@ -87,19 +106,17 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } if len(services.Registry.Local.Persistence.FilePersistence.Path) == 0 { - services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(services.Registry.Local.Persistence.FilePersistence) + services.Registry.Local.Persistence.FilePersistence.Path = defaultRegistryPath(cr) } - if services.Registry.Local.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.Registry.Local.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultRegistryStorageRequest) - } - } + ensurePVCDefaults(services.Registry.Local.Persistence.FilePersistence.PvcConfig, RegistryFeastType) } setServiceDefaultConfigs(&services.Registry.Local.ServiceConfigs.DefaultConfigs) + } else if services.Registry.Remote.FeastRef != nil && len(services.Registry.Remote.FeastRef.Namespace) == 0 { + services.Registry.Remote.FeastRef.Namespace = cr.Namespace } + if services.OfflineStore != nil { if services.OfflineStore.Persistence == nil { services.OfflineStore.Persistence = &feastdevv1alpha1.OfflineStorePersistence{} @@ -114,15 +131,10 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { services.OfflineStore.Persistence.FilePersistence.Type = string(OfflineFilePersistenceDaskConfigType) } - if services.OfflineStore.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.OfflineStore.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultOfflineStorageRequest) - } - } + ensurePVCDefaults(services.OfflineStore.Persistence.FilePersistence.PvcConfig, OfflineFeastType) } - setStoreServiceDefaultConfigs(&services.OfflineStore.StoreServiceConfigs) + setServiceDefaultConfigs(&services.OfflineStore.ServiceConfigs.DefaultConfigs) } if services.OnlineStore != nil { @@ -136,21 +148,14 @@ func ApplyDefaultsToStatus(cr *feastdevv1alpha1.FeatureStore) { } if len(services.OnlineStore.Persistence.FilePersistence.Path) == 0 { - services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(services.OnlineStore.Persistence.FilePersistence) + services.OnlineStore.Persistence.FilePersistence.Path = defaultOnlineStorePath(cr) } - if services.OnlineStore.Persistence.FilePersistence.PvcConfig != nil { - pvc := services.OnlineStore.Persistence.FilePersistence.PvcConfig - if pvc.Create != nil { - ensureRequestedStorage(&pvc.Create.Resources, DefaultOnlineStorageRequest) - } - } + ensurePVCDefaults(services.OnlineStore.Persistence.FilePersistence.PvcConfig, OnlineFeastType) } - setStoreServiceDefaultConfigs(&services.OnlineStore.StoreServiceConfigs) + setServiceDefaultConfigs(&services.OnlineStore.ServiceConfigs.DefaultConfigs) } - // overwrite status.applied with every reconcile - applied.DeepCopyInto(&cr.Status.Applied) } func setServiceDefaultConfigs(defaultConfigs *feastdevv1alpha1.DefaultConfigs) { @@ -159,13 +164,6 @@ func setServiceDefaultConfigs(defaultConfigs *feastdevv1alpha1.DefaultConfigs) { } } -func setStoreServiceDefaultConfigs(storeServiceConfigs *feastdevv1alpha1.StoreServiceConfigs) { - if storeServiceConfigs.Replicas == nil { - storeServiceConfigs.Replicas = &DefaultReplicas - } - setServiceDefaultConfigs(&storeServiceConfigs.ServiceConfigs.DefaultConfigs) -} - func checkOfflineStoreFilePersistenceType(value string) error { if slices.Contains(feastdevv1alpha1.ValidOfflineStoreFilePersistenceTypes, value) { return nil @@ -182,18 +180,40 @@ func ensureRequestedStorage(resources *v1.VolumeResourceRequirements, requestedS } } -func defaultOnlineStorePath(persistence *feastdevv1alpha1.OnlineStoreFilePersistence) string { - if persistence.PvcConfig == nil { - return DefaultOnlineStoreEphemeralPath +func ensurePVCDefaults(pvc *feastdevv1alpha1.PvcConfig, feastType FeastServiceType) { + if pvc != nil { + var storageRequest string + switch feastType { + case OnlineFeastType: + storageRequest = DefaultOnlineStorageRequest + case OfflineFeastType: + storageRequest = DefaultOfflineStorageRequest + case RegistryFeastType: + storageRequest = DefaultRegistryStorageRequest + } + if pvc.Create != nil { + ensureRequestedStorage(&pvc.Create.Resources, storageRequest) + if pvc.Create.AccessModes == nil { + pvc.Create.AccessModes = DefaultPVCAccessModes + } + } + } +} + +func defaultOnlineStorePath(featureStore *feastdevv1alpha1.FeatureStore) string { + if _, ok := hasPvcConfig(featureStore, OnlineFeastType); ok { + return DefaultOnlineStorePath } - return DefaultOnlineStorePvcPath + // if online pvc not set, use offline's mount path. + return getOfflineMountPath(featureStore) + "/" + DefaultOnlineStorePath } -func defaultRegistryPath(persistence *feastdevv1alpha1.RegistryFilePersistence) string { - if persistence.PvcConfig == nil { - return DefaultRegistryEphemeralPath +func defaultRegistryPath(featureStore *feastdevv1alpha1.FeatureStore) string { + if _, ok := hasPvcConfig(featureStore, RegistryFeastType); ok { + return DefaultRegistryPath } - return DefaultRegistryPvcPath + // if registry pvc not set, use offline's mount path. + return getOfflineMountPath(featureStore) + "/" + DefaultRegistryPath } func checkOfflineStoreDBStorePersistenceType(value string) error { @@ -345,3 +365,69 @@ func envOverride(dst, src []corev1.EnvVar) []corev1.EnvVar { } return dst } + +func GetRegistryContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(RegistryFeastType, containers) + return container +} + +func GetOfflineContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(OfflineFeastType, containers) + return container +} + +func GetOnlineContainer(containers []corev1.Container) *corev1.Container { + _, container := getContainerByType(OnlineFeastType, containers) + return container +} + +func getContainerByType(feastType FeastServiceType, containers []corev1.Container) (int, *corev1.Container) { + for i, c := range containers { + if c.Name == string(feastType) { + return i, &c + } + } + return -1, nil +} + +func GetRegistryVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(RegistryFeastType, featureStore, volumes) +} + +func GetOnlineVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(OnlineFeastType, featureStore, volumes) +} + +func GetOfflineVolume(featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + return getVolumeByType(OfflineFeastType, featureStore, volumes) +} + +func getVolumeByType(feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, volumes []corev1.Volume) *corev1.Volume { + for _, v := range volumes { + if v.Name == GetFeastServiceName(featureStore, feastType) { + return &v + } + } + return nil +} + +func GetRegistryVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(RegistryFeastType, featureStore, volumeMounts) +} + +func GetOnlineVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(OnlineFeastType, featureStore, volumeMounts) +} + +func GetOfflineVolumeMount(featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + return getVolumeMountByType(OfflineFeastType, featureStore, volumeMounts) +} + +func getVolumeMountByType(feastType FeastServiceType, featureStore *feastdevv1alpha1.FeatureStore, volumeMounts []corev1.VolumeMount) *corev1.VolumeMount { + for _, vm := range volumeMounts { + if vm.Name == GetFeastServiceName(featureStore, feastType) { + return &vm + } + } + return nil +} diff --git a/infra/feast-operator/test/e2e/e2e_test.go b/infra/feast-operator/test/e2e/e2e_test.go index fdf58d8f3b7..bab57a3c006 100644 --- a/infra/feast-operator/test/e2e/e2e_test.go +++ b/infra/feast-operator/test/e2e/e2e_test.go @@ -27,9 +27,12 @@ import ( "github.com/feast-dev/feast/infra/feast-operator/test/utils" ) -const feastControllerNamespace = "feast-operator-system" -const timeout = 2 * time.Minute -const controllerDeploymentName = "feast-operator-controller-manager" +const ( + feastControllerNamespace = "feast-operator-system" + timeout = 2 * time.Minute + controllerDeploymentName = "feast-operator-controller-manager" + feastPrefix = "feast-" +) var _ = Describe("controller", Ordered, func() { BeforeAll(func() { @@ -159,15 +162,18 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st "Error occurred while checking FeatureStore %s is having remote registry or not. \nError: %v\n", featureStoreName, err)) - k8ResourceNames := []string{fmt.Sprintf("feast-%s-online", featureStoreName), - fmt.Sprintf("feast-%s-offline", featureStoreName), + feastResourceName := feastPrefix + featureStoreName + k8sResourceNames := []string{feastResourceName} + feastK8sResourceNames := []string{ + feastResourceName + "-online", + feastResourceName + "-offline", } if !hasRemoteRegistry { - k8ResourceNames = append(k8ResourceNames, fmt.Sprintf("feast-%s-registry", featureStoreName)) + feastK8sResourceNames = append(feastK8sResourceNames, feastResourceName+"-registry") } - for _, deploymentName := range k8ResourceNames { + for _, deploymentName := range k8sResourceNames { By(fmt.Sprintf("validate the feast deployment: %s is up and in availability state.", deploymentName)) err = checkIfDeploymentExistsAndAvailable(namespace, deploymentName, timeout) Expect(err).To(BeNil(), fmt.Sprintf( @@ -178,7 +184,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st } By("Check if the feast client - kubernetes config map exists.") - configMapName := fmt.Sprintf("feast-%s-client", featureStoreName) + configMapName := feastResourceName + "-client" err = checkIfConfigMapExists(namespace, configMapName) Expect(err).To(BeNil(), fmt.Sprintf( "config map %s is not available but expected to be available. \nError: %v\n", @@ -186,7 +192,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st )) fmt.Printf("Feast Deployment client config map %s is available\n", configMapName) - for _, serviceAccountName := range k8ResourceNames { + for _, serviceAccountName := range k8sResourceNames { By(fmt.Sprintf("validate the feast service account: %s is available.", serviceAccountName)) err = checkIfServiceAccountExists(namespace, serviceAccountName) Expect(err).To(BeNil(), fmt.Sprintf( @@ -196,7 +202,7 @@ func validateTheFeatureStoreCustomResource(namespace string, featureStoreName st fmt.Printf("Service account %s exists in namespace %s\n", serviceAccountName, namespace) } - for _, serviceName := range k8ResourceNames { + for _, serviceName := range feastK8sResourceNames { By(fmt.Sprintf("validate the kubernetes service name: %s is available.", serviceName)) err = checkIfKubernetesServiceExists(namespace, serviceName) Expect(err).To(BeNil(), fmt.Sprintf( diff --git a/infra/feast-operator/test/e2e/test_util.go b/infra/feast-operator/test/e2e/test_util.go index d92f719fb97..017690a0ec9 100644 --- a/infra/feast-operator/test/e2e/test_util.go +++ b/infra/feast-operator/test/e2e/test_util.go @@ -192,9 +192,5 @@ func isFeatureStoreHavingRemoteRegistry(namespace, featureStoreName string) (boo hasValidFeastRef := registryConfig.Remote.FeastRef != nil && registryConfig.Remote.FeastRef.Name != "" - if hasHostname || hasValidFeastRef { - return true, nil - } - - return false, nil + return hasHostname || hasValidFeastRef, nil } diff --git a/java/datatypes/pom.xml b/java/datatypes/pom.xml index b0ba049c575..967262d0e01 100644 --- a/java/datatypes/pom.xml +++ b/java/datatypes/pom.xml @@ -118,6 +118,11 @@ grpc-stub ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + javax.annotation javax.annotation-api diff --git a/java/pom.xml b/java/pom.xml index 6b18588923a..82c0a00ba22 100644 --- a/java/pom.xml +++ b/java/pom.xml @@ -41,9 +41,9 @@ UTF-8 UTF-8 - 1.30.2 + 1.63.0 3.12.2 - 3.16.1 + 3.25.5 1.111.1 0.8.0 1.9.10 @@ -61,15 +61,15 @@ 1.5.24 3.14.7 3.10 - 2.14.0 + 2.15.0 2.3.1 1.3.2 2.0.1.Final 0.21.0 1.6.6 - 30.1-jre + 32.0.0-jre 3.4.34 - 4.1.101.Final + 4.1.96.Final src/main/java/**/BatchLoadsWithResult.java - + @@ -365,7 +365,7 @@ [11.0,) - + @@ -376,7 +376,7 @@ - + diff --git a/java/serving-client/pom.xml b/java/serving-client/pom.xml index 7b8838a009c..dc611b4a76e 100644 --- a/java/serving-client/pom.xml +++ b/java/serving-client/pom.xml @@ -50,6 +50,11 @@ grpc-testing ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + com.google.protobuf protobuf-java-util diff --git a/java/serving/pom.xml b/java/serving/pom.xml index ca7f8a73b5f..1be4da1b622 100644 --- a/java/serving/pom.xml +++ b/java/serving/pom.xml @@ -126,7 +126,7 @@ com.azure azure-storage-blob - 12.25.2 + 12.26.1 com.azure @@ -164,6 +164,11 @@ grpc-stub ${grpc.version} + + io.grpc + grpc-api + ${grpc.version} + io.grpc grpc-netty-shaded @@ -192,7 +197,7 @@ io.jaegertracing jaeger-client - 1.3.2 + 1.8.1 io.opentracing @@ -240,7 +245,7 @@ com.google.cloud google-cloud-storage - 1.118.0 + 2.43.1 @@ -253,13 +258,13 @@ com.amazonaws aws-java-sdk-s3 - 1.12.261 + 1.12.546 com.amazonaws aws-java-sdk-sts - 1.12.476 + 1.12.546 @@ -378,7 +383,7 @@ io.lettuce lettuce-core - 6.0.2.RELEASE + 6.5.1.RELEASE org.apache.commons diff --git a/protos/feast/core/Feature.proto b/protos/feast/core/Feature.proto index 882de47eb9c..8a56d67905a 100644 --- a/protos/feast/core/Feature.proto +++ b/protos/feast/core/Feature.proto @@ -35,6 +35,11 @@ message FeatureSpecV2 { map tags = 3; // Description of the feature. - string description = 4; + + // Field indicating the vector will be indexed for vector similarity search + bool vector_index = 5; + + // Metric used for vector similarity search. + string vector_search_metric = 6; } diff --git a/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst new file mode 100644 index 00000000000..7c5c3d371a7 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.cassandra_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.cassandra\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.cassandra\_online\_store.cassandra\_online\_store module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store.cassandra_online_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.cassandra\_online\_store.cassandra\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store.cassandra_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.cassandra_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst new file mode 100644 index 00000000000..29d51304928 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.couchbase_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.couchbase\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.couchbase\_online\_store.couchbase module +-------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store.couchbase + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.couchbase\_online\_store.couchbase\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store.couchbase_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.couchbase_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst new file mode 100644 index 00000000000..d470e3301d0 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.elasticsearch_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.elasticsearch\_online\_store package +=============================================================== + +Submodules +---------- + +feast.infra.online\_stores.elasticsearch\_online\_store.elasticsearch module +---------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store.elasticsearch + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.elasticsearch\_online\_store.elasticsearch\_repo\_configuration module +------------------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store.elasticsearch_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.elasticsearch_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst new file mode 100644 index 00000000000..9cb565ca132 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.hazelcast_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.hazelcast\_online\_store package +=========================================================== + +Submodules +---------- + +feast.infra.online\_stores.hazelcast\_online\_store.hazelcast\_online\_store module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store.hazelcast_online_store + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.hazelcast\_online\_store.hazelcast\_repo\_configuration module +----------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store.hazelcast_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.hazelcast_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst new file mode 100644 index 00000000000..50ad80e0a9e --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.hbase_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.hbase\_online\_store package +======================================================= + +Submodules +---------- + +feast.infra.online\_stores.hbase\_online\_store.hbase module +------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.hbase_online_store.hbase + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.hbase\_online\_store.hbase\_repo\_configuration module +--------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.hbase_online_store.hbase_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.hbase_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst new file mode 100644 index 00000000000..391af17024f --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.ikv_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.ikv\_online\_store package +===================================================== + +Submodules +---------- + +feast.infra.online\_stores.ikv\_online\_store.ikv module +-------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.ikv_online_store.ikv + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.ikv_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst new file mode 100644 index 00000000000..ee9faa55dc0 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.milvus_online_store.rst @@ -0,0 +1,21 @@ +feast.infra.online\_stores.milvus\_online\_store package +======================================================== + +Submodules +---------- + +feast.infra.online\_stores.milvus\_online\_store.milvus\_repo\_configuration module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.milvus_online_store.milvus_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.milvus_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst new file mode 100644 index 00000000000..b1a9ea4f802 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.mysql_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.mysql\_online\_store package +======================================================= + +Submodules +---------- + +feast.infra.online\_stores.mysql\_online\_store.mysql module +------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.mysql_online_store.mysql + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.mysql\_online\_store.mysql\_repo\_configuration module +--------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.mysql_online_store.mysql_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.mysql_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst new file mode 100644 index 00000000000..9dfd200a4e1 --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.postgres_online_store.rst @@ -0,0 +1,37 @@ +feast.infra.online\_stores.postgres\_online\_store package +========================================================== + +Submodules +---------- + +feast.infra.online\_stores.postgres\_online\_store.pgvector\_repo\_configuration module +--------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store.pgvector_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.postgres\_online\_store.postgres module +------------------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.postgres_online_store.postgres + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.postgres\_online\_store.postgres\_repo\_configuration module +--------------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store.postgres_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.postgres_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst b/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst new file mode 100644 index 00000000000..5c210d4124d --- /dev/null +++ b/sdk/python/docs/source/feast.infra.online_stores.qdrant_online_store.rst @@ -0,0 +1,29 @@ +feast.infra.online\_stores.qdrant\_online\_store package +======================================================== + +Submodules +---------- + +feast.infra.online\_stores.qdrant\_online\_store.qdrant module +-------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store.qdrant + :members: + :undoc-members: + :show-inheritance: + +feast.infra.online\_stores.qdrant\_online\_store.qdrant\_repo\_configuration module +----------------------------------------------------------------------------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store.qdrant_repo_configuration + :members: + :undoc-members: + :show-inheritance: + +Module contents +--------------- + +.. automodule:: feast.infra.online_stores.qdrant_online_store + :members: + :undoc-members: + :show-inheritance: diff --git a/sdk/python/docs/source/feast.infra.online_stores.rst b/sdk/python/docs/source/feast.infra.online_stores.rst index ea714e45c5b..c07c7e0c279 100644 --- a/sdk/python/docs/source/feast.infra.online_stores.rst +++ b/sdk/python/docs/source/feast.infra.online_stores.rst @@ -7,7 +7,16 @@ Subpackages .. toctree:: :maxdepth: 4 - feast.infra.online_stores + feast.infra.online_stores.cassandra_online_store + feast.infra.online_stores.couchbase_online_store + feast.infra.online_stores.elasticsearch_online_store + feast.infra.online_stores.hazelcast_online_store + feast.infra.online_stores.hbase_online_store + feast.infra.online_stores.ikv_online_store + feast.infra.online_stores.milvus_online_store + feast.infra.online_stores.mysql_online_store + feast.infra.online_stores.postgres_online_store + feast.infra.online_stores.qdrant_online_store Submodules ---------- @@ -36,6 +45,14 @@ feast.infra.online\_stores.dynamodb module :undoc-members: :show-inheritance: +feast.infra.online\_stores.faiss\_online\_store module +------------------------------------------------------ + +.. automodule:: feast.infra.online_stores.faiss_online_store + :members: + :undoc-members: + :show-inheritance: + feast.infra.online\_stores.helpers module ----------------------------------------- diff --git a/sdk/python/docs/source/feast.infra.rst b/sdk/python/docs/source/feast.infra.rst index b0046a2719e..791a4ace832 100644 --- a/sdk/python/docs/source/feast.infra.rst +++ b/sdk/python/docs/source/feast.infra.rst @@ -51,6 +51,14 @@ feast.infra.provider module :undoc-members: :show-inheritance: +feast.infra.supported\_async\_methods module +-------------------------------------------- + +.. automodule:: feast.infra.supported_async_methods + :members: + :undoc-members: + :show-inheritance: + Module contents --------------- diff --git a/sdk/python/feast/cli.py b/sdk/python/feast/cli.py index 15b592119ca..165677a843a 100644 --- a/sdk/python/feast/cli.py +++ b/sdk/python/feast/cli.py @@ -982,7 +982,6 @@ def serve_command( raise click.BadParameter( "Please pass --cert and --key args to start the feature server in TLS mode." ) - store = create_feature_store(ctx) store.serve( @@ -1133,15 +1132,6 @@ def serve_registry_command( show_default=False, help="path to TLS certificate public key. You need to pass --key as well to start server in TLS mode", ) -@click.option( - "--verify_client", - "-v", - "tls_verify_client", - type=click.BOOL, - default="True", - show_default=True, - help="Verify the client or not for the TLS client certificate.", -) @click.pass_context def serve_offline_command( ctx: click.Context, @@ -1149,7 +1139,6 @@ def serve_offline_command( port: int, tls_key_path: str, tls_cert_path: str, - tls_verify_client: bool, ): """Start a remote server locally on a given host, port.""" if (tls_key_path and not tls_cert_path) or (not tls_key_path and tls_cert_path): @@ -1158,7 +1147,7 @@ def serve_offline_command( ) store = create_feature_store(ctx) - store.serve_offline(host, port, tls_key_path, tls_cert_path, tls_verify_client) + store.serve_offline(host, port, tls_key_path, tls_cert_path) @cli.command("validate") diff --git a/sdk/python/feast/feature_store.py b/sdk/python/feast/feature_store.py index 79a0d752efb..4564d6abf3c 100644 --- a/sdk/python/feast/feature_store.py +++ b/sdk/python/feast/feature_store.py @@ -86,6 +86,7 @@ from feast.repo_config import RepoConfig, load_repo_config from feast.repo_contents import RepoContents from feast.saved_dataset import SavedDataset, SavedDatasetStorage, ValidationReference +from feast.ssl_ca_trust_store_setup import configure_ca_trust_store_env_variables from feast.stream_feature_view import StreamFeatureView from feast.utils import _utc_now @@ -129,6 +130,8 @@ def __init__( if fs_yaml_file is not None and config is not None: raise ValueError("You cannot specify both fs_yaml_file and config.") + configure_ca_trust_store_env_variables() + if repo_path: self.repo_path = Path(repo_path) else: @@ -1750,9 +1753,10 @@ async def get_online_features_async( def retrieve_online_documents( self, - feature: str, + feature: Optional[str], query: Union[str, List[float]], top_k: int, + features: Optional[List[str]] = None, distance_metric: Optional[str] = None, ) -> OnlineResponse: """ @@ -1762,6 +1766,7 @@ def retrieve_online_documents( feature: The list of document features that should be retrieved from the online document store. These features can be specified either as a list of string document feature references or as a feature service. String feature references must have format "feature_view:feature", e.g, "document_fv:document_embeddings". + features: The list of features that should be retrieved from the online store. query: The query to retrieve the closest document features for. top_k: The number of closest document features to retrieve. distance_metric: The distance metric to use for retrieval. @@ -1770,18 +1775,44 @@ def retrieve_online_documents( raise ValueError( "Using embedding functionality is not supported for document retrieval. Please embed the query before calling retrieve_online_documents." ) + feature_list: List[str] = ( + features + if features is not None + else ([feature] if feature is not None else []) + ) + ( available_feature_views, _, ) = utils._get_feature_views_to_use( registry=self._registry, project=self.project, - features=[feature], + features=feature_list, allow_cache=True, hide_dummy_entity=False, ) + if features: + feature_view_set = set() + for feature in features: + feature_view_name = feature.split(":")[0] + feature_view = self.get_feature_view(feature_view_name) + feature_view_set.add(feature_view.name) + if len(feature_view_set) > 1: + raise ValueError( + "Document retrieval only supports a single feature view." + ) + requested_feature = None + requested_features = [ + f.split(":")[1] for f in features if isinstance(f, str) and ":" in f + ] + else: + requested_feature = ( + feature.split(":")[1] if isinstance(feature, str) else feature + ) + requested_features = [requested_feature] if requested_feature else [] + requested_feature_view_name = ( - feature.split(":")[0] if isinstance(feature, str) else feature + feature.split(":")[0] if feature else list(feature_view_set)[0] ) for feature_view in available_feature_views: if feature_view.name == requested_feature_view_name: @@ -1790,14 +1821,15 @@ def retrieve_online_documents( raise ValueError( f"Feature view {requested_feature_view} not found in the registry." ) - requested_feature = ( - feature.split(":")[1] if isinstance(feature, str) else feature - ) + + requested_feature_view = available_feature_views[0] + provider = self._get_provider() document_features = self._retrieve_from_online_store( provider, requested_feature_view, requested_feature, + requested_features, query, top_k, distance_metric, @@ -1819,6 +1851,7 @@ def retrieve_online_documents( document_feature_vals = [feature[4] for feature in document_features] document_feature_distance_vals = [feature[5] for feature in document_features] online_features_response = GetOnlineFeaturesResponse(results=[]) + requested_feature = requested_feature or requested_features[0] utils._populate_result_rows_from_columnar( online_features_response=online_features_response, data={ @@ -1833,7 +1866,8 @@ def _retrieve_from_online_store( self, provider: Provider, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str], @@ -1849,6 +1883,7 @@ def _retrieve_from_online_store( config=self.config, table=table, requested_feature=requested_feature, + requested_features=requested_features, query=query, top_k=top_k, distance_metric=distance_metric, @@ -1964,14 +1999,11 @@ def serve_offline( port: int, tls_key_path: str = "", tls_cert_path: str = "", - tls_verify_client: bool = True, ) -> None: """Start offline server locally on a given port.""" from feast import offline_server - offline_server.start_server( - self, host, port, tls_key_path, tls_cert_path, tls_verify_client - ) + offline_server.start_server(self, host, port, tls_key_path, tls_cert_path) def serve_transformations(self, port: int) -> None: """Start the feature transformation server locally on a given port.""" diff --git a/sdk/python/feast/field.py b/sdk/python/feast/field.py index a41dcf5d5e6..fda1fbffe54 100644 --- a/sdk/python/feast/field.py +++ b/sdk/python/feast/field.py @@ -32,12 +32,16 @@ class Field: dtype: The type of the field, such as string or float. description: A human-readable description. tags: User-defined metadata in dictionary form. + vector_index: If set to True the field will be indexed for vector similarity search. + vector_search_metric: The metric used for vector similarity search. """ name: str dtype: FeastType description: str tags: Dict[str, str] + vector_index: bool + vector_search_metric: Optional[str] def __init__( self, @@ -46,6 +50,8 @@ def __init__( dtype: FeastType, description: str = "", tags: Optional[Dict[str, str]] = None, + vector_index: bool = False, + vector_search_metric: Optional[str] = None, ): """ Creates a Field object. @@ -55,11 +61,15 @@ def __init__( dtype: The type of the field, such as string or float. description (optional): A human-readable description. tags (optional): User-defined metadata in dictionary form. + vector_index (optional): If set to True the field will be indexed for vector similarity search. + vector_search_metric (optional): The metric used for vector similarity search. """ self.name = name self.dtype = dtype self.description = description self.tags = tags or {} + self.vector_index = vector_index + self.vector_search_metric = vector_search_metric def __eq__(self, other): if type(self) != type(other): @@ -70,6 +80,8 @@ def __eq__(self, other): or self.dtype != other.dtype or self.description != other.description or self.tags != other.tags + # or self.vector_index != other.vector_index + # or self.vector_search_metric != other.vector_search_metric ): return False return True @@ -87,6 +99,8 @@ def __repr__(self): f" dtype={self.dtype!r},\n" f" description={self.description!r},\n" f" tags={self.tags!r}\n" + f" vector_index={self.vector_index!r}\n" + f" vector_search_metric={self.vector_search_metric!r}\n" f")" ) @@ -96,11 +110,14 @@ def __str__(self): def to_proto(self) -> FieldProto: """Converts a Field object to its protobuf representation.""" value_type = self.dtype.to_value_type() + vector_search_metric = self.vector_search_metric or "" return FieldProto( name=self.name, value_type=value_type.value, description=self.description, tags=self.tags, + vector_index=self.vector_index, + vector_search_metric=vector_search_metric, ) @classmethod @@ -112,11 +129,15 @@ def from_proto(cls, field_proto: FieldProto): field_proto: FieldProto protobuf object """ value_type = ValueType(field_proto.value_type) + vector_search_metric = getattr(field_proto, "vector_search_metric", "") + vector_index = getattr(field_proto, "vector_index", False) return cls( name=field_proto.name, dtype=from_value_type(value_type=value_type), tags=dict(field_proto.tags), description=field_proto.description, + vector_index=vector_index, + vector_search_metric=vector_search_metric, ) @classmethod diff --git a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile index c1da48f55d0..b4d7b5e3e9c 100644 --- a/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile +++ b/sdk/python/feast/infra/feature_servers/multicloud/Dockerfile @@ -1,7 +1,7 @@ FROM python:3.11-slim-bullseye RUN pip install --no-cache-dir pip --upgrade -RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s]" +RUN pip install --no-cache-dir "feast[aws,gcp,snowflake,redis,go,mysql,postgres,opentelemetry,grpcio,k8s,duckdb]" RUN apt update && apt install -y -V ca-certificates lsb-release wget && \ diff --git a/sdk/python/feast/infra/key_encoding_utils.py b/sdk/python/feast/infra/key_encoding_utils.py index 1f9ffeef140..18127896bd5 100644 --- a/sdk/python/feast/infra/key_encoding_utils.py +++ b/sdk/python/feast/infra/key_encoding_utils.py @@ -1,5 +1,7 @@ import struct -from typing import List, Tuple +from typing import List, Tuple, Union + +from google.protobuf.internal.containers import RepeatedScalarFieldContainer from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto @@ -163,3 +165,16 @@ def get_list_val_str(val): if val.HasField(accept_type): return str(getattr(val, accept_type).val) return None + + +def serialize_f32( + vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int +) -> bytes: + """serializes a list of floats into a compact "raw bytes" format""" + return struct.pack(f"{vector_length}f", *vector) + + +def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: + """deserializes a list of floats from a compact "raw bytes" format""" + num_floats = vector_length // 4 # 4 bytes per float + return list(struct.unpack(f"{num_floats}f", byte_vector)) diff --git a/sdk/python/feast/infra/offline_stores/remote.py b/sdk/python/feast/infra/offline_stores/remote.py index 6f26e06c6ba..d11fb4673db 100644 --- a/sdk/python/feast/infra/offline_stores/remote.py +++ b/sdk/python/feast/infra/offline_stores/remote.py @@ -74,7 +74,7 @@ def build_arrow_flight_client( scheme: str, host: str, port, auth_config: AuthConfig, cert: str = "" ): arrow_scheme = "grpc+tcp" - if cert: + if scheme == "https": logger.info( "Scheme is https so going to connect offline server in SSL(TLS) mode." ) diff --git a/sdk/python/feast/infra/offline_stores/snowflake.py b/sdk/python/feast/infra/offline_stores/snowflake.py index 3d23682769b..101685cec6f 100644 --- a/sdk/python/feast/infra/offline_stores/snowflake.py +++ b/sdk/python/feast/infra/offline_stores/snowflake.py @@ -716,8 +716,8 @@ def _get_entity_df_event_timestamp_range( MULTIPLE_FEATURE_VIEW_POINT_IN_TIME_JOIN = """ /* - Compute a deterministic hash for the `left_table_query_string` that will be used throughout - all the logic as the field to GROUP BY the data + 0. Compute a deterministic hash for the `left_table_query_string` that will be used throughout + all the logic as the field to GROUP BY the data. */ WITH "entity_dataframe" AS ( SELECT *, @@ -739,6 +739,10 @@ def _get_entity_df_event_timestamp_range( {% for featureview in featureviews %} +/* + 1. Only select the required columns with entities of the featureview. +*/ + "{{ featureview.name }}__entity_dataframe" AS ( SELECT {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} @@ -752,20 +756,7 @@ def _get_entity_df_event_timestamp_range( ), /* - This query template performs the point-in-time correctness join for a single feature set table - to the provided entity table. - - 1. We first join the current feature_view to the entity dataframe that has been passed. - This JOIN has the following logic: - - For each row of the entity dataframe, only keep the rows where the `timestamp_field` - is less than the one provided in the entity dataframe - - If there a TTL for the current feature_view, also keep the rows where the `timestamp_field` - is higher the the one provided minus the TTL - - For each row, Join on the entity key and retrieve the `entity_row_unique_id` that has been - computed previously - - The output of this CTE will contain all the necessary information and already filtered out most - of the data that is not relevant. +2. Use subquery to prepare event_timestamp, created_timestamp, entity columns and feature columns. */ "{{ featureview.name }}__subquery" AS ( @@ -777,94 +768,61 @@ def _get_entity_df_event_timestamp_range( "{{ feature }}" as {% if full_feature_names %}"{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}"{% else %}"{{ featureview.field_mapping.get(feature, feature) }}"{% endif %}{% if loop.last %}{% else %}, {% endif %} {% endfor %} FROM {{ featureview.table_subquery }} - WHERE "{{ featureview.timestamp_field }}" <= '{{ featureview.max_event_timestamp }}' - {% if featureview.ttl == 0 %}{% else %} - AND "{{ featureview.timestamp_field }}" >= '{{ featureview.min_event_timestamp }}' - {% endif %} -), - -"{{ featureview.name }}__base" AS ( - SELECT - "subquery".*, - "entity_dataframe"."entity_timestamp", - "entity_dataframe"."{{featureview.name}}__entity_row_unique_id" - FROM "{{ featureview.name }}__subquery" AS "subquery" - INNER JOIN "{{ featureview.name }}__entity_dataframe" AS "entity_dataframe" - ON TRUE - AND "subquery"."event_timestamp" <= "entity_dataframe"."entity_timestamp" - - {% if featureview.ttl == 0 %}{% else %} - AND "subquery"."event_timestamp" >= TIMESTAMPADD(second,-{{ featureview.ttl }},"entity_dataframe"."entity_timestamp") - {% endif %} - - {% for entity in featureview.entities %} - AND "subquery"."{{ entity }}" = "entity_dataframe"."{{ entity }}" - {% endfor %} ), /* - 2. If the `created_timestamp_column` has been set, we need to - deduplicate the data first. This is done by calculating the - `MAX(created_at_timestamp)` for each event_timestamp. - We then join the data on the next CTE +3. If the `created_timestamp_column` has been set, we need to +deduplicate the data first. This is done by calculating the +`MAX(created_at_timestamp)` for each event_timestamp and joining back on the subquery. +Otherwise, the ASOF JOIN can have unstable side effects +https://docs.snowflake.com/en/sql-reference/constructs/asof-join#expected-behavior-when-ties-exist-in-the-right-table */ + {% if featureview.created_timestamp_column %} "{{ featureview.name }}__dedup" AS ( - SELECT - "{{featureview.name}}__entity_row_unique_id", - "event_timestamp", - MAX("created_timestamp") AS "created_timestamp" - FROM "{{ featureview.name }}__base" - GROUP BY "{{featureview.name}}__entity_row_unique_id", "event_timestamp" + SELECT * + FROM "{{ featureview.name }}__subquery" + INNER JOIN ( + SELECT + {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} + "event_timestamp", + MAX("created_timestamp") AS "created_timestamp" + FROM "{{ featureview.name }}__subquery" + GROUP BY {{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} "event_timestamp" + ) + USING({{ featureview.entities | map('tojson') | join(', ')}}{% if featureview.entities %},{% else %}{% endif %} "event_timestamp", "created_timestamp") ), {% endif %} /* - 3. The data has been filtered during the first CTE "*__base" - Thus we only need to compute the latest timestamp of each feature. +4. Make ASOF JOIN of deduplicated feature CTE on reduced entity dataframe. */ -"{{ featureview.name }}__latest" AS ( + +"{{ featureview.name }}__asof_join" AS ( SELECT - "event_timestamp", - {% if featureview.created_timestamp_column %}"created_timestamp",{% endif %} - "{{featureview.name}}__entity_row_unique_id" - FROM - ( - SELECT *, - ROW_NUMBER() OVER( - PARTITION BY "{{featureview.name}}__entity_row_unique_id" - ORDER BY "event_timestamp" DESC{% if featureview.created_timestamp_column %},"created_timestamp" DESC{% endif %} - ) AS "row_number" - FROM "{{ featureview.name }}__base" - {% if featureview.created_timestamp_column %} - INNER JOIN "{{ featureview.name }}__dedup" - USING ("{{featureview.name}}__entity_row_unique_id", "event_timestamp", "created_timestamp") - {% endif %} - ) - WHERE "row_number" = 1 + e.*, + v.* + FROM "{{ featureview.name }}__entity_dataframe" e + ASOF JOIN {% if featureview.created_timestamp_column %}"{{ featureview.name }}__dedup"{% else %}"{{ featureview.name }}__subquery"{% endif %} v + MATCH_CONDITION (e."entity_timestamp" >= v."event_timestamp") + {% if featureview.entities %} USING({{ featureview.entities | map('tojson') | join(', ')}}) {% endif %} ), /* - 4. Once we know the latest value of each feature for a given timestamp, - we can join again the data back to the original "base" dataset +5. If TTL is configured filter the CTE to remove rows where the feature values are older than the configured ttl. */ -"{{ featureview.name }}__cleaned" AS ( - SELECT "base".* - FROM "{{ featureview.name }}__base" AS "base" - INNER JOIN "{{ featureview.name }}__latest" - USING( - "{{featureview.name}}__entity_row_unique_id", - "event_timestamp" - {% if featureview.created_timestamp_column %} - ,"created_timestamp" - {% endif %} - ) -){% if loop.last %}{% else %}, {% endif %} +"{{ featureview.name }}__ttl" AS ( + SELECT * + FROM "{{ featureview.name }}__asof_join" + {% if featureview.ttl == 0 %}{% else %} + WHERE "event_timestamp" >= TIMESTAMPADD(second,-{{ featureview.ttl }},"entity_timestamp") + {% endif %} +){% if loop.last %}{% else %}, {% endif %} {% endfor %} /* - Joins the outputs of multiple time travel joins to a single table. + Join the outputs of multiple time travel joins to a single table. The entity_dataframe dataset being our source of truth here. */ @@ -877,7 +835,7 @@ def _get_entity_df_event_timestamp_range( {% for feature in featureview.features %} ,{% if full_feature_names %}"{{ featureview.name }}__{{featureview.field_mapping.get(feature, feature)}}"{% else %}"{{ featureview.field_mapping.get(feature, feature) }}"{% endif %} {% endfor %} - FROM "{{ featureview.name }}__cleaned" -) "{{ featureview.name }}__cleaned" USING ("{{featureview.name}}__entity_row_unique_id") + FROM "{{ featureview.name }}__ttl" +) "{{ featureview.name }}__ttl" USING ("{{featureview.name}}__entity_row_unique_id") {% endfor %} """ diff --git a/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py b/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py index 0152ca330c9..af328141520 100644 --- a/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py +++ b/sdk/python/feast/infra/online_stores/elasticsearch_online_store/elasticsearch.py @@ -213,7 +213,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, *args, diff --git a/sdk/python/feast/infra/online_stores/faiss_online_store.py b/sdk/python/feast/infra/online_stores/faiss_online_store.py index cc2e75800e6..fd4d6768abd 100644 --- a/sdk/python/feast/infra/online_stores/faiss_online_store.py +++ b/sdk/python/feast/infra/online_stores/faiss_online_store.py @@ -176,7 +176,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_featres: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py b/sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py new file mode 100644 index 00000000000..e69de29bb2d diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py new file mode 100644 index 00000000000..8e8402862cb --- /dev/null +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py @@ -0,0 +1,12 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.milvus import ( + MilvusOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig( + online_store="milvus", online_store_creator=MilvusOnlineStoreCreator + ), +] diff --git a/sdk/python/feast/infra/online_stores/online_store.py b/sdk/python/feast/infra/online_stores/online_store.py index 789885f82bc..be3128562dc 100644 --- a/sdk/python/feast/infra/online_stores/online_store.py +++ b/sdk/python/feast/infra/online_stores/online_store.py @@ -390,7 +390,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -411,6 +412,7 @@ def retrieve_online_documents( config: The config for the current feature store. table: The feature view whose feature values should be read. requested_feature: The name of the feature whose embeddings should be used for retrieval. + requested_features: The list of features whose embeddings should be used for retrieval. embedding: The embeddings to use for retrieval. top_k: The number of documents to retrieve. @@ -419,6 +421,10 @@ def retrieve_online_documents( where the first item is the event timestamp for the row, and the second item is a dict of feature name to embeddings. """ + if not requested_feature and not requested_features: + raise ValueError( + "Either requested_feature or requested_features must be specified" + ) raise NotImplementedError( f"Online store {self.__class__.__name__} does not support online retrieval" ) diff --git a/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py b/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py index 7c099c80ecc..f43247a5457 100644 --- a/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py +++ b/sdk/python/feast/infra/online_stores/postgres_online_store/postgres.py @@ -347,7 +347,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = "L2", @@ -366,6 +367,7 @@ def retrieve_online_documents( config: Feast configuration object table: FeatureView object as the table to search requested_feature: The requested feature as the column to search + requested_features: The list of features whose embeddings should be used for retrieval. embedding: The query embedding to search for top_k: The number of items to return distance_metric: The distance metric to use for the search.G diff --git a/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py b/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py index 074c52ba5e8..cdbef95348d 100644 --- a/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py +++ b/sdk/python/feast/infra/online_stores/qdrant_online_store/qdrant.py @@ -248,7 +248,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = "cosine", diff --git a/sdk/python/feast/infra/online_stores/sqlite.py b/sdk/python/feast/infra/online_stores/sqlite.py index e2eeb038d00..23b4f6db3a3 100644 --- a/sdk/python/feast/infra/online_stores/sqlite.py +++ b/sdk/python/feast/infra/online_stores/sqlite.py @@ -15,19 +15,20 @@ import logging import os import sqlite3 -import struct import sys from datetime import date, datetime from pathlib import Path -from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple, Union +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple -from google.protobuf.internal.containers import RepeatedScalarFieldContainer from pydantic import StrictStr from feast import Entity from feast.feature_view import FeatureView from feast.infra.infra_object import SQLITE_INFRA_OBJECT_CLASS_TYPE, InfraObject -from feast.infra.key_encoding_utils import serialize_entity_key +from feast.infra.key_encoding_utils import ( + serialize_entity_key, + serialize_f32, +) from feast.infra.online_stores.online_store import OnlineStore from feast.infra.online_stores.vector_store import VectorStoreConfig from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto @@ -330,7 +331,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_featuers: Optional[List[str]], embedding: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -432,6 +434,7 @@ def retrieve_online_documents( _build_retrieve_online_document_record( entity_key, string_value if string_value else b"", + # This may be a bug embedding, distance, event_ts, @@ -459,19 +462,6 @@ def _table_id(project: str, table: FeatureView) -> str: return f"{project}_{table.name}" -def serialize_f32( - vector: Union[RepeatedScalarFieldContainer[float], List[float]], vector_length: int -) -> bytes: - """serializes a list of floats into a compact "raw bytes" format""" - return struct.pack(f"{vector_length}f", *vector) - - -def deserialize_f32(byte_vector: bytes, vector_length: int) -> List[float]: - """deserializes a list of floats from a compact "raw bytes" format""" - num_floats = vector_length // 4 # 4 bytes per float - return list(struct.unpack(f"{num_floats}f", byte_vector)) - - class SqliteTable(InfraObject): """ A Sqlite table managed by Feast. diff --git a/sdk/python/feast/infra/passthrough_provider.py b/sdk/python/feast/infra/passthrough_provider.py index 215b175eb2e..57aa122ae8a 100644 --- a/sdk/python/feast/infra/passthrough_provider.py +++ b/sdk/python/feast/infra/passthrough_provider.py @@ -294,7 +294,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -305,6 +306,7 @@ def retrieve_online_documents( config, table, requested_feature, + requested_features, query, top_k, distance_metric, diff --git a/sdk/python/feast/infra/provider.py b/sdk/python/feast/infra/provider.py index 8351f389ad9..efc806ba2f0 100644 --- a/sdk/python/feast/infra/provider.py +++ b/sdk/python/feast/infra/provider.py @@ -419,7 +419,8 @@ def retrieve_online_documents( self, config: RepoConfig, table: FeatureView, - requested_feature: str, + requested_feature: Optional[str], + requested_features: Optional[List[str]], query: List[float], top_k: int, distance_metric: Optional[str] = None, @@ -440,6 +441,7 @@ def retrieve_online_documents( config: The config for the current feature store. table: The feature view whose embeddings should be searched. requested_feature: the requested document feature name. + requested_features: the requested document feature names. query: The query embedding to search for. top_k: The number of documents to return. diff --git a/sdk/python/feast/infra/registry/remote.py b/sdk/python/feast/infra/registry/remote.py index 6cc80d5dad1..590c0454b73 100644 --- a/sdk/python/feast/infra/registry/remote.py +++ b/sdk/python/feast/infra/registry/remote.py @@ -1,3 +1,4 @@ +import os from datetime import datetime from pathlib import Path from typing import List, Optional, Union @@ -59,6 +60,12 @@ class RemoteRegistryConfig(RegistryConfig): """ str: Path to the public certificate when the registry server starts in TLS(SSL) mode. This may be needed if the registry server started with a self-signed certificate, typically this file ends with `*.crt`, `*.cer`, or `*.pem`. If registry_type is 'remote', then this configuration is needed to connect to remote registry server in TLS mode. If the remote registry started in non-tls mode then this configuration is not needed.""" + is_tls: bool = False + """ bool: Set to `True` if you plan to connect to a registry server running in TLS (SSL) mode. + If you intend to add the public certificate to the trust store instead of passing it via the `cert` parameter, this field must be set to `True`. + If you are planning to add the public certificate as part of the trust store instead of passing it as a `cert` parameters then setting this field to `true` is mandatory. + """ + class RemoteRegistry(BaseRegistry): def __init__( @@ -70,20 +77,32 @@ def __init__( ): self.auth_config = auth_config assert isinstance(registry_config, RemoteRegistryConfig) - if registry_config.cert: - with open(registry_config.cert, "rb") as cert_file: - trusted_certs = cert_file.read() - tls_credentials = grpc.ssl_channel_credentials( - root_certificates=trusted_certs - ) - self.channel = grpc.secure_channel(registry_config.path, tls_credentials) - else: - self.channel = grpc.insecure_channel(registry_config.path) + self.channel = self._create_grpc_channel(registry_config) auth_header_interceptor = GrpcClientAuthHeaderInterceptor(auth_config) self.channel = grpc.intercept_channel(self.channel, auth_header_interceptor) self.stub = RegistryServer_pb2_grpc.RegistryServerStub(self.channel) + def _create_grpc_channel(self, registry_config): + assert isinstance(registry_config, RemoteRegistryConfig) + if registry_config.cert or registry_config.is_tls: + cafile = os.getenv("SSL_CERT_FILE") or os.getenv("REQUESTS_CA_BUNDLE") + if not cafile and not registry_config.cert: + raise EnvironmentError( + "SSL_CERT_FILE or REQUESTS_CA_BUNDLE environment variable must be set to use secure TLS or set the cert parameter in feature_Store.yaml file under remote registry configuration." + ) + with open( + registry_config.cert if registry_config.cert else cafile, "rb" + ) as cert_file: + trusted_certs = cert_file.read() + tls_credentials = grpc.ssl_channel_credentials( + root_certificates=trusted_certs + ) + return grpc.secure_channel(registry_config.path, tls_credentials) + else: + # Create an insecure gRPC channel + return grpc.insecure_channel(registry_config.path) + def close(self): if self.channel: self.channel.close() diff --git a/sdk/python/feast/offline_server.py b/sdk/python/feast/offline_server.py index 8774dea8aed..1b714a45c7e 100644 --- a/sdk/python/feast/offline_server.py +++ b/sdk/python/feast/offline_server.py @@ -45,7 +45,6 @@ def __init__( location: str, host: str = "localhost", tls_certificates: List = [], - verify_client=False, **kwargs, ): super(OfflineServer, self).__init__( @@ -54,7 +53,7 @@ def __init__( str_to_auth_manager_type(store.config.auth_config.type) ), tls_certificates=tls_certificates, - verify_client=verify_client, + verify_client=False, # this is needed for when we don't need mTLS **kwargs, ) self._location = location @@ -568,7 +567,6 @@ def start_server( port: int, tls_key_path: str = "", tls_cert_path: str = "", - tls_verify_client: bool = True, ): _init_auth_manager(store) @@ -591,7 +589,6 @@ def start_server( location=location, host=host, tls_certificates=tls_certificates, - verify_client=tls_verify_client, ) try: logger.info(f"Offline store server serving at: {location}") diff --git a/sdk/python/feast/protos/feast/core/DataSource_pb2.py b/sdk/python/feast/protos/feast/core/DataSource_pb2.py index b58c33a3830..68bee8d7609 100644 --- a/sdk/python/feast/protos/feast/core/DataSource_pb2.py +++ b/sdk/python/feast/protos/feast/core/DataSource_pb2.py @@ -19,7 +19,7 @@ from feast.protos.feast.core import Feature_pb2 as feast_dot_core_dot_Feature__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x66\x65\x61st/core/DataSource.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataFormat.proto\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"\xc0\x16\n\nDataSource\x12\x0c\n\x04name\x18\x14 \x01(\t\x12\x0f\n\x07project\x18\x15 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x17 \x01(\t\x12.\n\x04tags\x18\x18 \x03(\x0b\x32 .feast.core.DataSource.TagsEntry\x12\r\n\x05owner\x18\x19 \x01(\t\x12/\n\x04type\x18\x01 \x01(\x0e\x32!.feast.core.DataSource.SourceType\x12?\n\rfield_mapping\x18\x02 \x03(\x0b\x32(.feast.core.DataSource.FieldMappingEntry\x12\x17\n\x0ftimestamp_field\x18\x03 \x01(\t\x12\x1d\n\x15\x64\x61te_partition_column\x18\x04 \x01(\t\x12 \n\x18\x63reated_timestamp_column\x18\x05 \x01(\t\x12\x1e\n\x16\x64\x61ta_source_class_type\x18\x11 \x01(\t\x12,\n\x0c\x62\x61tch_source\x18\x1a \x01(\x0b\x32\x16.feast.core.DataSource\x12/\n\x04meta\x18\x32 \x01(\x0b\x32!.feast.core.DataSource.SourceMeta\x12:\n\x0c\x66ile_options\x18\x0b \x01(\x0b\x32\".feast.core.DataSource.FileOptionsH\x00\x12\x42\n\x10\x62igquery_options\x18\x0c \x01(\x0b\x32&.feast.core.DataSource.BigQueryOptionsH\x00\x12<\n\rkafka_options\x18\r \x01(\x0b\x32#.feast.core.DataSource.KafkaOptionsH\x00\x12@\n\x0fkinesis_options\x18\x0e \x01(\x0b\x32%.feast.core.DataSource.KinesisOptionsH\x00\x12\x42\n\x10redshift_options\x18\x0f \x01(\x0b\x32&.feast.core.DataSource.RedshiftOptionsH\x00\x12I\n\x14request_data_options\x18\x12 \x01(\x0b\x32).feast.core.DataSource.RequestDataOptionsH\x00\x12\x44\n\x0e\x63ustom_options\x18\x10 \x01(\x0b\x32*.feast.core.DataSource.CustomSourceOptionsH\x00\x12\x44\n\x11snowflake_options\x18\x13 \x01(\x0b\x32\'.feast.core.DataSource.SnowflakeOptionsH\x00\x12:\n\x0cpush_options\x18\x16 \x01(\x0b\x32\".feast.core.DataSource.PushOptionsH\x00\x12<\n\rspark_options\x18\x1b \x01(\x0b\x32#.feast.core.DataSource.SparkOptionsH\x00\x12<\n\rtrino_options\x18\x1e \x01(\x0b\x32#.feast.core.DataSource.TrinoOptionsH\x00\x12>\n\x0e\x61thena_options\x18# \x01(\x0b\x32$.feast.core.DataSource.AthenaOptionsH\x00\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11\x46ieldMappingEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x82\x01\n\nSourceMeta\x12:\n\x16\x65\x61rliestEventTimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x38\n\x14latestEventTimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\x65\n\x0b\x46ileOptions\x12+\n\x0b\x66ile_format\x18\x01 \x01(\x0b\x32\x16.feast.core.FileFormat\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x03 \x01(\t\x1a/\n\x0f\x42igQueryOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a,\n\x0cTrinoOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a\xae\x01\n\x0cKafkaOptions\x12\x1f\n\x17kafka_bootstrap_servers\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x30\n\x0emessage_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x12<\n\x19watermark_delay_threshold\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x66\n\x0eKinesisOptions\x12\x0e\n\x06region\x18\x01 \x01(\t\x12\x13\n\x0bstream_name\x18\x02 \x01(\t\x12/\n\rrecord_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x1aQ\n\x0fRedshiftOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\t\x1aT\n\rAthenaOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x61ta_source\x18\x04 \x01(\t\x1aX\n\x10SnowflakeOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\tJ\x04\x08\x05\x10\x06\x1aO\n\x0cSparkOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x13\n\x0b\x66ile_format\x18\x04 \x01(\t\x1a,\n\x13\x43ustomSourceOptions\x12\x15\n\rconfiguration\x18\x01 \x01(\x0c\x1a\xf7\x01\n\x12RequestDataOptions\x12Z\n\x11\x64\x65precated_schema\x18\x02 \x03(\x0b\x32?.feast.core.DataSource.RequestDataOptions.DeprecatedSchemaEntry\x12)\n\x06schema\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x1aT\n\x15\x44\x65precatedSchemaEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum:\x02\x38\x01J\x04\x08\x01\x10\x02\x1a\x13\n\x0bPushOptionsJ\x04\x08\x01\x10\x02\"\xf8\x01\n\nSourceType\x12\x0b\n\x07INVALID\x10\x00\x12\x0e\n\nBATCH_FILE\x10\x01\x12\x13\n\x0f\x42\x41TCH_SNOWFLAKE\x10\x08\x12\x12\n\x0e\x42\x41TCH_BIGQUERY\x10\x02\x12\x12\n\x0e\x42\x41TCH_REDSHIFT\x10\x05\x12\x10\n\x0cSTREAM_KAFKA\x10\x03\x12\x12\n\x0eSTREAM_KINESIS\x10\x04\x12\x11\n\rCUSTOM_SOURCE\x10\x06\x12\x12\n\x0eREQUEST_SOURCE\x10\x07\x12\x0f\n\x0bPUSH_SOURCE\x10\t\x12\x0f\n\x0b\x42\x41TCH_TRINO\x10\n\x12\x0f\n\x0b\x42\x41TCH_SPARK\x10\x0b\x12\x10\n\x0c\x42\x41TCH_ATHENA\x10\x0c\x42\t\n\x07optionsJ\x04\x08\x06\x10\x0b\x42T\n\x10\x66\x65\x61st.proto.coreB\x0f\x44\x61taSourceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1b\x66\x65\x61st/core/DataSource.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataFormat.proto\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"\xc0\x16\n\nDataSource\x12\x0c\n\x04name\x18\x14 \x01(\t\x12\x0f\n\x07project\x18\x15 \x01(\t\x12\x13\n\x0b\x64\x65scription\x18\x17 \x01(\t\x12.\n\x04tags\x18\x18 \x03(\x0b\x32 .feast.core.DataSource.TagsEntry\x12\r\n\x05owner\x18\x19 \x01(\t\x12/\n\x04type\x18\x01 \x01(\x0e\x32!.feast.core.DataSource.SourceType\x12?\n\rfield_mapping\x18\x02 \x03(\x0b\x32(.feast.core.DataSource.FieldMappingEntry\x12\x17\n\x0ftimestamp_field\x18\x03 \x01(\t\x12\x1d\n\x15\x64\x61te_partition_column\x18\x04 \x01(\t\x12 \n\x18\x63reated_timestamp_column\x18\x05 \x01(\t\x12\x1e\n\x16\x64\x61ta_source_class_type\x18\x11 \x01(\t\x12,\n\x0c\x62\x61tch_source\x18\x1a \x01(\x0b\x32\x16.feast.core.DataSource\x12/\n\x04meta\x18\x32 \x01(\x0b\x32!.feast.core.DataSource.SourceMeta\x12:\n\x0c\x66ile_options\x18\x0b \x01(\x0b\x32\".feast.core.DataSource.FileOptionsH\x00\x12\x42\n\x10\x62igquery_options\x18\x0c \x01(\x0b\x32&.feast.core.DataSource.BigQueryOptionsH\x00\x12<\n\rkafka_options\x18\r \x01(\x0b\x32#.feast.core.DataSource.KafkaOptionsH\x00\x12@\n\x0fkinesis_options\x18\x0e \x01(\x0b\x32%.feast.core.DataSource.KinesisOptionsH\x00\x12\x42\n\x10redshift_options\x18\x0f \x01(\x0b\x32&.feast.core.DataSource.RedshiftOptionsH\x00\x12I\n\x14request_data_options\x18\x12 \x01(\x0b\x32).feast.core.DataSource.RequestDataOptionsH\x00\x12\x44\n\x0e\x63ustom_options\x18\x10 \x01(\x0b\x32*.feast.core.DataSource.CustomSourceOptionsH\x00\x12\x44\n\x11snowflake_options\x18\x13 \x01(\x0b\x32\'.feast.core.DataSource.SnowflakeOptionsH\x00\x12:\n\x0cpush_options\x18\x16 \x01(\x0b\x32\".feast.core.DataSource.PushOptionsH\x00\x12<\n\rspark_options\x18\x1b \x01(\x0b\x32#.feast.core.DataSource.SparkOptionsH\x00\x12<\n\rtrino_options\x18\x1e \x01(\x0b\x32#.feast.core.DataSource.TrinoOptionsH\x00\x12>\n\x0e\x61thena_options\x18# \x01(\x0b\x32$.feast.core.DataSource.AthenaOptionsH\x00\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x33\n\x11\x46ieldMappingEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x1a\x82\x01\n\nSourceMeta\x12:\n\x16\x65\x61rliestEventTimestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x38\n\x14latestEventTimestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x1a\x65\n\x0b\x46ileOptions\x12+\n\x0b\x66ile_format\x18\x01 \x01(\x0b\x32\x16.feast.core.FileFormat\x12\x0b\n\x03uri\x18\x02 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x03 \x01(\t\x1a/\n\x0f\x42igQueryOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a,\n\x0cTrinoOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x1a\xae\x01\n\x0cKafkaOptions\x12\x1f\n\x17kafka_bootstrap_servers\x18\x01 \x01(\t\x12\r\n\x05topic\x18\x02 \x01(\t\x12\x30\n\x0emessage_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x12<\n\x19watermark_delay_threshold\x18\x04 \x01(\x0b\x32\x19.google.protobuf.Duration\x1a\x66\n\x0eKinesisOptions\x12\x0e\n\x06region\x18\x01 \x01(\t\x12\x13\n\x0bstream_name\x18\x02 \x01(\t\x12/\n\rrecord_format\x18\x03 \x01(\x0b\x32\x18.feast.core.StreamFormat\x1aQ\n\x0fRedshiftOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\t\x1aT\n\rAthenaOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x03 \x01(\t\x12\x13\n\x0b\x64\x61ta_source\x18\x04 \x01(\t\x1aX\n\x10SnowflakeOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0e\n\x06schema\x18\x03 \x01(\t\x12\x10\n\x08\x64\x61tabase\x18\x04 \x01(\tJ\x04\x08\x05\x10\x06\x1aO\n\x0cSparkOptions\x12\r\n\x05table\x18\x01 \x01(\t\x12\r\n\x05query\x18\x02 \x01(\t\x12\x0c\n\x04path\x18\x03 \x01(\t\x12\x13\n\x0b\x66ile_format\x18\x04 \x01(\t\x1a,\n\x13\x43ustomSourceOptions\x12\x15\n\rconfiguration\x18\x01 \x01(\x0c\x1a\xf7\x01\n\x12RequestDataOptions\x12Z\n\x11\x64\x65precated_schema\x18\x02 \x03(\x0b\x32?.feast.core.DataSource.RequestDataOptions.DeprecatedSchemaEntry\x12)\n\x06schema\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x1aT\n\x15\x44\x65precatedSchemaEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12*\n\x05value\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum:\x02\x38\x01J\x04\x08\x01\x10\x02\x1a\x13\n\x0bPushOptionsJ\x04\x08\x01\x10\x02\"\xf8\x01\n\nSourceType\x12\x0b\n\x07INVALID\x10\x00\x12\x0e\n\nBATCH_FILE\x10\x01\x12\x13\n\x0f\x42\x41TCH_SNOWFLAKE\x10\x08\x12\x12\n\x0e\x42\x41TCH_BIGQUERY\x10\x02\x12\x12\n\x0e\x42\x41TCH_REDSHIFT\x10\x05\x12\x10\n\x0cSTREAM_KAFKA\x10\x03\x12\x12\n\x0eSTREAM_KINESIS\x10\x04\x12\x11\n\rCUSTOM_SOURCE\x10\x06\x12\x12\n\x0eREQUEST_SOURCE\x10\x07\x12\x0f\n\x0bPUSH_SOURCE\x10\t\x12\x0f\n\x0b\x42\x41TCH_TRINO\x10\n\x12\x0f\n\x0b\x42\x41TCH_SPARK\x10\x0b\x12\x10\n\x0c\x42\x41TCH_ATHENA\x10\x0c\x42\t\n\x07optionsJ\x04\x08\x06\x10\x0b\"=\n\x0e\x44\x61taSourceList\x12+\n\x0b\x64\x61tasources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSourceBT\n\x10\x66\x65\x61st.proto.coreB\x0f\x44\x61taSourceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -69,4 +69,6 @@ _globals['_DATASOURCE_PUSHOPTIONS']._serialized_end=2801 _globals['_DATASOURCE_SOURCETYPE']._serialized_start=2804 _globals['_DATASOURCE_SOURCETYPE']._serialized_end=3052 + _globals['_DATASOURCELIST']._serialized_start=3071 + _globals['_DATASOURCELIST']._serialized_end=3132 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi b/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi index 94336638e19..aadec3fad4c 100644 --- a/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/DataSource_pb2.pyi @@ -557,3 +557,18 @@ class DataSource(google.protobuf.message.Message): def WhichOneof(self, oneof_group: typing_extensions.Literal["options", b"options"]) -> typing_extensions.Literal["file_options", "bigquery_options", "kafka_options", "kinesis_options", "redshift_options", "request_data_options", "custom_options", "snowflake_options", "push_options", "spark_options", "trino_options", "athena_options"] | None: ... global___DataSource = DataSource + +class DataSourceList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + DATASOURCES_FIELD_NUMBER: builtins.int + @property + def datasources(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___DataSource]: ... + def __init__( + self, + *, + datasources: collections.abc.Iterable[global___DataSource] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["datasources", b"datasources"]) -> None: ... + +global___DataSourceList = DataSourceList diff --git a/sdk/python/feast/protos/feast/core/Entity_pb2.py b/sdk/python/feast/protos/feast/core/Entity_pb2.py index 5a192854cab..2b3e7806736 100644 --- a/sdk/python/feast/protos/feast/core/Entity_pb2.py +++ b/sdk/python/feast/protos/feast/core/Entity_pb2.py @@ -16,7 +16,7 @@ from google.protobuf import timestamp_pb2 as google_dot_protobuf_dot_timestamp__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x66\x65\x61st/core/Entity.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"V\n\x06\x45ntity\x12&\n\x04spec\x18\x01 \x01(\x0b\x32\x18.feast.core.EntitySpecV2\x12$\n\x04meta\x18\x02 \x01(\x0b\x32\x16.feast.core.EntityMeta\"\xf3\x01\n\x0c\x45ntitySpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\t \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08join_key\x18\x04 \x01(\t\x12\x30\n\x04tags\x18\x08 \x03(\x0b\x32\".feast.core.EntitySpecV2.TagsEntry\x12\r\n\x05owner\x18\n \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\nEntityMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampBP\n\x10\x66\x65\x61st.proto.coreB\x0b\x45ntityProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x17\x66\x65\x61st/core/Entity.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\x1a\x1fgoogle/protobuf/timestamp.proto\"V\n\x06\x45ntity\x12&\n\x04spec\x18\x01 \x01(\x0b\x32\x18.feast.core.EntitySpecV2\x12$\n\x04meta\x18\x02 \x01(\x0b\x32\x16.feast.core.EntityMeta\"\xf3\x01\n\x0c\x45ntitySpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\t \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x13\n\x0b\x64\x65scription\x18\x03 \x01(\t\x12\x10\n\x08join_key\x18\x04 \x01(\t\x12\x30\n\x04tags\x18\x08 \x03(\x0b\x32\".feast.core.EntitySpecV2.TagsEntry\x12\r\n\x05owner\x18\n \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x7f\n\nEntityMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"2\n\nEntityList\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.EntityBP\n\x10\x66\x65\x61st.proto.coreB\x0b\x45ntityProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -34,4 +34,6 @@ _globals['_ENTITYSPECV2_TAGSENTRY']._serialized_end=429 _globals['_ENTITYMETA']._serialized_start=431 _globals['_ENTITYMETA']._serialized_end=558 + _globals['_ENTITYLIST']._serialized_start=560 + _globals['_ENTITYLIST']._serialized_end=610 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/Entity_pb2.pyi b/sdk/python/feast/protos/feast/core/Entity_pb2.pyi index 732b3e10326..025817edfee 100644 --- a/sdk/python/feast/protos/feast/core/Entity_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/Entity_pb2.pyi @@ -128,3 +128,18 @@ class EntityMeta(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["created_timestamp", b"created_timestamp", "last_updated_timestamp", b"last_updated_timestamp"]) -> None: ... global___EntityMeta = EntityMeta + +class EntityList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ENTITIES_FIELD_NUMBER: builtins.int + @property + def entities(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___Entity]: ... + def __init__( + self, + *, + entities: collections.abc.Iterable[global___Entity] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["entities", b"entities"]) -> None: ... + +global___EntityList = EntityList diff --git a/sdk/python/feast/protos/feast/core/FeatureService_pb2.py b/sdk/python/feast/protos/feast/core/FeatureService_pb2.py index cf6ac46ac54..642d5b010f9 100644 --- a/sdk/python/feast/protos/feast/core/FeatureService_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureService_pb2.py @@ -16,7 +16,7 @@ from feast.protos.feast.core import FeatureViewProjection_pb2 as feast_dot_core_dot_FeatureViewProjection__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/FeatureService.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&feast/core/FeatureViewProjection.proto\"l\n\x0e\x46\x65\x61tureService\x12,\n\x04spec\x18\x01 \x01(\x0b\x32\x1e.feast.core.FeatureServiceSpec\x12,\n\x04meta\x18\x02 \x01(\x0b\x32\x1e.feast.core.FeatureServiceMeta\"\xa4\x02\n\x12\x46\x65\x61tureServiceSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x33\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32!.feast.core.FeatureViewProjection\x12\x36\n\x04tags\x18\x04 \x03(\x0b\x32(.feast.core.FeatureServiceSpec.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05owner\x18\x06 \x01(\t\x12\x31\n\x0elogging_config\x18\x07 \x01(\x0b\x32\x19.feast.core.LoggingConfig\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x87\x01\n\x12\x46\x65\x61tureServiceMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x9a\x07\n\rLoggingConfig\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x45\n\x10\x66ile_destination\x18\x03 \x01(\x0b\x32).feast.core.LoggingConfig.FileDestinationH\x00\x12M\n\x14\x62igquery_destination\x18\x04 \x01(\x0b\x32-.feast.core.LoggingConfig.BigQueryDestinationH\x00\x12M\n\x14redshift_destination\x18\x05 \x01(\x0b\x32-.feast.core.LoggingConfig.RedshiftDestinationH\x00\x12O\n\x15snowflake_destination\x18\x06 \x01(\x0b\x32..feast.core.LoggingConfig.SnowflakeDestinationH\x00\x12I\n\x12\x63ustom_destination\x18\x07 \x01(\x0b\x32+.feast.core.LoggingConfig.CustomDestinationH\x00\x12I\n\x12\x61thena_destination\x18\x08 \x01(\x0b\x32+.feast.core.LoggingConfig.AthenaDestinationH\x00\x1aS\n\x0f\x46ileDestination\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x02 \x01(\t\x12\x14\n\x0cpartition_by\x18\x03 \x03(\t\x1a(\n\x13\x42igQueryDestination\x12\x11\n\ttable_ref\x18\x01 \x01(\t\x1a)\n\x13RedshiftDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\'\n\x11\x41thenaDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a*\n\x14SnowflakeDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\x99\x01\n\x11\x43ustomDestination\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12G\n\x06\x63onfig\x18\x02 \x03(\x0b\x32\x37.feast.core.LoggingConfig.CustomDestination.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x64\x65stinationBX\n\x10\x66\x65\x61st.proto.coreB\x13\x46\x65\x61tureServiceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1f\x66\x65\x61st/core/FeatureService.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a&feast/core/FeatureViewProjection.proto\"l\n\x0e\x46\x65\x61tureService\x12,\n\x04spec\x18\x01 \x01(\x0b\x32\x1e.feast.core.FeatureServiceSpec\x12,\n\x04meta\x18\x02 \x01(\x0b\x32\x1e.feast.core.FeatureServiceMeta\"\xa4\x02\n\x12\x46\x65\x61tureServiceSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x33\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32!.feast.core.FeatureViewProjection\x12\x36\n\x04tags\x18\x04 \x03(\x0b\x32(.feast.core.FeatureServiceSpec.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x05 \x01(\t\x12\r\n\x05owner\x18\x06 \x01(\t\x12\x31\n\x0elogging_config\x18\x07 \x01(\x0b\x32\x19.feast.core.LoggingConfig\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x87\x01\n\x12\x46\x65\x61tureServiceMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\x9a\x07\n\rLoggingConfig\x12\x13\n\x0bsample_rate\x18\x01 \x01(\x02\x12\x45\n\x10\x66ile_destination\x18\x03 \x01(\x0b\x32).feast.core.LoggingConfig.FileDestinationH\x00\x12M\n\x14\x62igquery_destination\x18\x04 \x01(\x0b\x32-.feast.core.LoggingConfig.BigQueryDestinationH\x00\x12M\n\x14redshift_destination\x18\x05 \x01(\x0b\x32-.feast.core.LoggingConfig.RedshiftDestinationH\x00\x12O\n\x15snowflake_destination\x18\x06 \x01(\x0b\x32..feast.core.LoggingConfig.SnowflakeDestinationH\x00\x12I\n\x12\x63ustom_destination\x18\x07 \x01(\x0b\x32+.feast.core.LoggingConfig.CustomDestinationH\x00\x12I\n\x12\x61thena_destination\x18\x08 \x01(\x0b\x32+.feast.core.LoggingConfig.AthenaDestinationH\x00\x1aS\n\x0f\x46ileDestination\x12\x0c\n\x04path\x18\x01 \x01(\t\x12\x1c\n\x14s3_endpoint_override\x18\x02 \x01(\t\x12\x14\n\x0cpartition_by\x18\x03 \x03(\t\x1a(\n\x13\x42igQueryDestination\x12\x11\n\ttable_ref\x18\x01 \x01(\t\x1a)\n\x13RedshiftDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\'\n\x11\x41thenaDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a*\n\x14SnowflakeDestination\x12\x12\n\ntable_name\x18\x01 \x01(\t\x1a\x99\x01\n\x11\x43ustomDestination\x12\x0c\n\x04kind\x18\x01 \x01(\t\x12G\n\x06\x63onfig\x18\x02 \x03(\x0b\x32\x37.feast.core.LoggingConfig.CustomDestination.ConfigEntry\x1a-\n\x0b\x43onfigEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42\r\n\x0b\x64\x65stination\"I\n\x12\x46\x65\x61tureServiceList\x12\x33\n\x0f\x66\x65\x61tureservices\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureServiceBX\n\x10\x66\x65\x61st.proto.coreB\x13\x46\x65\x61tureServiceProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -52,4 +52,6 @@ _globals['_LOGGINGCONFIG_CUSTOMDESTINATION']._serialized_end=1571 _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_start=1526 _globals['_LOGGINGCONFIG_CUSTOMDESTINATION_CONFIGENTRY']._serialized_end=1571 + _globals['_FEATURESERVICELIST']._serialized_start=1588 + _globals['_FEATURESERVICELIST']._serialized_end=1661 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi index b3305b72df9..0b1c0baa871 100644 --- a/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureService_pb2.pyi @@ -264,3 +264,18 @@ class LoggingConfig(google.protobuf.message.Message): def WhichOneof(self, oneof_group: typing_extensions.Literal["destination", b"destination"]) -> typing_extensions.Literal["file_destination", "bigquery_destination", "redshift_destination", "snowflake_destination", "custom_destination", "athena_destination"] | None: ... global___LoggingConfig = LoggingConfig + +class FeatureServiceList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FEATURESERVICES_FIELD_NUMBER: builtins.int + @property + def featureservices(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FeatureService]: ... + def __init__( + self, + *, + featureservices: collections.abc.Iterable[global___FeatureService] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["featureservices", b"featureservices"]) -> None: ... + +global___FeatureServiceList = FeatureServiceList diff --git a/sdk/python/feast/protos/feast/core/FeatureView_pb2.py b/sdk/python/feast/protos/feast/core/FeatureView_pb2.py index f1480593d9a..80d04c1ec3f 100644 --- a/sdk/python/feast/protos/feast/core/FeatureView_pb2.py +++ b/sdk/python/feast/protos/feast/core/FeatureView_pb2.py @@ -18,7 +18,7 @@ from feast.protos.feast.core import Feature_pb2 as feast_dot_core_dot_Feature__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x61st/core/FeatureView.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"c\n\x0b\x46\x65\x61tureView\x12)\n\x04spec\x18\x01 \x01(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12)\n\x04meta\x18\x02 \x01(\x0b\x32\x1b.feast.core.FeatureViewMeta\"\xbd\x03\n\x0f\x46\x65\x61tureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12+\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x31\n\x0e\x65ntity_columns\x18\x0c \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x13\n\x0b\x64\x65scription\x18\n \x01(\t\x12\x33\n\x04tags\x18\x05 \x03(\x0b\x32%.feast.core.FeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x0b \x01(\t\x12&\n\x03ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x0c\x62\x61tch_source\x18\x07 \x01(\x0b\x32\x16.feast.core.DataSource\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0e\n\x06online\x18\x08 \x01(\x08\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x0f\x46\x65\x61tureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x19materialization_intervals\x18\x03 \x03(\x0b\x32#.feast.core.MaterializationInterval\"w\n\x17MaterializationInterval\x12.\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.TimestampBU\n\x10\x66\x65\x61st.proto.coreB\x10\x46\x65\x61tureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1c\x66\x65\x61st/core/FeatureView.proto\x12\nfeast.core\x1a\x1egoogle/protobuf/duration.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\"c\n\x0b\x46\x65\x61tureView\x12)\n\x04spec\x18\x01 \x01(\x0b\x32\x1b.feast.core.FeatureViewSpec\x12)\n\x04meta\x18\x02 \x01(\x0b\x32\x1b.feast.core.FeatureViewMeta\"\xbd\x03\n\x0f\x46\x65\x61tureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x10\n\x08\x65ntities\x18\x03 \x03(\t\x12+\n\x08\x66\x65\x61tures\x18\x04 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x31\n\x0e\x65ntity_columns\x18\x0c \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x13\n\x0b\x64\x65scription\x18\n \x01(\t\x12\x33\n\x04tags\x18\x05 \x03(\x0b\x32%.feast.core.FeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x0b \x01(\t\x12&\n\x03ttl\x18\x06 \x01(\x0b\x32\x19.google.protobuf.Duration\x12,\n\x0c\x62\x61tch_source\x18\x07 \x01(\x0b\x32\x16.feast.core.DataSource\x12-\n\rstream_source\x18\t \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0e\n\x06online\x18\x08 \x01(\x08\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\xcc\x01\n\x0f\x46\x65\x61tureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x46\n\x19materialization_intervals\x18\x03 \x03(\x0b\x32#.feast.core.MaterializationInterval\"w\n\x17MaterializationInterval\x12.\n\nstart_time\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_time\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"@\n\x0f\x46\x65\x61tureViewList\x12-\n\x0c\x66\x65\x61tureviews\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureViewBU\n\x10\x66\x65\x61st.proto.coreB\x10\x46\x65\x61tureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -38,4 +38,6 @@ _globals['_FEATUREVIEWMETA']._serialized_end=918 _globals['_MATERIALIZATIONINTERVAL']._serialized_start=920 _globals['_MATERIALIZATIONINTERVAL']._serialized_end=1039 + _globals['_FEATUREVIEWLIST']._serialized_start=1041 + _globals['_FEATUREVIEWLIST']._serialized_end=1105 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi b/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi index e1d4e2dfee8..57158fc2c6c 100644 --- a/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/FeatureView_pb2.pyi @@ -192,3 +192,18 @@ class MaterializationInterval(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["end_time", b"end_time", "start_time", b"start_time"]) -> None: ... global___MaterializationInterval = MaterializationInterval + +class FeatureViewList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + FEATUREVIEWS_FIELD_NUMBER: builtins.int + @property + def featureviews(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___FeatureView]: ... + def __init__( + self, + *, + featureviews: collections.abc.Iterable[global___FeatureView] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["featureviews", b"featureviews"]) -> None: ... + +global___FeatureViewList = FeatureViewList diff --git a/sdk/python/feast/protos/feast/core/Feature_pb2.py b/sdk/python/feast/protos/feast/core/Feature_pb2.py index dd7c6008ef1..6b1081fe811 100644 --- a/sdk/python/feast/protos/feast/core/Feature_pb2.py +++ b/sdk/python/feast/protos/feast/core/Feature_pb2.py @@ -15,7 +15,7 @@ from feast.protos.feast.types import Value_pb2 as feast_dot_types_dot_Value__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66\x65\x61st/core/Feature.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\"\xc3\x01\n\rFeatureSpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x31\n\x04tags\x18\x03 \x03(\x0b\x32#.feast.core.FeatureSpecV2.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42Q\n\x10\x66\x65\x61st.proto.coreB\x0c\x46\x65\x61tureProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x18\x66\x65\x61st/core/Feature.proto\x12\nfeast.core\x1a\x17\x66\x65\x61st/types/Value.proto\"\xf7\x01\n\rFeatureSpecV2\x12\x0c\n\x04name\x18\x01 \x01(\t\x12/\n\nvalue_type\x18\x02 \x01(\x0e\x32\x1b.feast.types.ValueType.Enum\x12\x31\n\x04tags\x18\x03 \x03(\x0b\x32#.feast.core.FeatureSpecV2.TagsEntry\x12\x13\n\x0b\x64\x65scription\x18\x04 \x01(\t\x12\x14\n\x0cvector_index\x18\x05 \x01(\x08\x12\x1c\n\x14vector_search_metric\x18\x06 \x01(\t\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\x42Q\n\x10\x66\x65\x61st.proto.coreB\x0c\x46\x65\x61tureProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -26,7 +26,7 @@ _globals['_FEATURESPECV2_TAGSENTRY']._options = None _globals['_FEATURESPECV2_TAGSENTRY']._serialized_options = b'8\001' _globals['_FEATURESPECV2']._serialized_start=66 - _globals['_FEATURESPECV2']._serialized_end=261 - _globals['_FEATURESPECV2_TAGSENTRY']._serialized_start=218 - _globals['_FEATURESPECV2_TAGSENTRY']._serialized_end=261 + _globals['_FEATURESPECV2']._serialized_end=313 + _globals['_FEATURESPECV2_TAGSENTRY']._serialized_start=270 + _globals['_FEATURESPECV2_TAGSENTRY']._serialized_end=313 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/Feature_pb2.pyi b/sdk/python/feast/protos/feast/core/Feature_pb2.pyi index f4235b0965b..451f1aa61ce 100644 --- a/sdk/python/feast/protos/feast/core/Feature_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/Feature_pb2.pyi @@ -53,6 +53,8 @@ class FeatureSpecV2(google.protobuf.message.Message): VALUE_TYPE_FIELD_NUMBER: builtins.int TAGS_FIELD_NUMBER: builtins.int DESCRIPTION_FIELD_NUMBER: builtins.int + VECTOR_INDEX_FIELD_NUMBER: builtins.int + VECTOR_SEARCH_METRIC_FIELD_NUMBER: builtins.int name: builtins.str """Name of the feature. Not updatable.""" value_type: feast.types.Value_pb2.ValueType.Enum.ValueType @@ -62,6 +64,10 @@ class FeatureSpecV2(google.protobuf.message.Message): """Tags for user defined metadata on a feature""" description: builtins.str """Description of the feature.""" + vector_index: builtins.bool + """Field indicating the vector will be indexed for vector similarity search""" + vector_search_metric: builtins.str + """Metric used for vector similarity search.""" def __init__( self, *, @@ -69,7 +75,9 @@ class FeatureSpecV2(google.protobuf.message.Message): value_type: feast.types.Value_pb2.ValueType.Enum.ValueType = ..., tags: collections.abc.Mapping[builtins.str, builtins.str] | None = ..., description: builtins.str = ..., + vector_index: builtins.bool = ..., + vector_search_metric: builtins.str = ..., ) -> None: ... - def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "name", b"name", "tags", b"tags", "value_type", b"value_type"]) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["description", b"description", "name", b"name", "tags", b"tags", "value_type", b"value_type", "vector_index", b"vector_index", "vector_search_metric", b"vector_search_metric"]) -> None: ... global___FeatureSpecV2 = FeatureSpecV2 diff --git a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py index 020515a6b89..926b54df288 100644 --- a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py +++ b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.py @@ -20,7 +20,7 @@ from feast.protos.feast.core import Transformation_pb2 as feast_dot_core_dot_Transformation__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$feast/core/OnDemandFeatureView.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a&feast/core/FeatureViewProjection.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"{\n\x13OnDemandFeatureView\x12\x31\n\x04spec\x18\x01 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewSpec\x12\x31\n\x04meta\x18\x02 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewMeta\"\x90\x05\n\x17OnDemandFeatureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12+\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x41\n\x07sources\x18\x04 \x03(\x0b\x32\x30.feast.core.OnDemandFeatureViewSpec.SourcesEntry\x12\x42\n\x15user_defined_function\x18\x05 \x01(\x0b\x32\x1f.feast.core.UserDefinedFunctionB\x02\x18\x01\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\n \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12;\n\x04tags\x18\x07 \x03(\x0b\x32-.feast.core.OnDemandFeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x08 \x01(\t\x12\x0c\n\x04mode\x18\x0b \x01(\t\x12\x1d\n\x15write_to_online_store\x18\x0c \x01(\x08\x12\x10\n\x08\x65ntities\x18\r \x03(\t\x12\x31\n\x0e\x65ntity_columns\x18\x0e \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x11\n\tsingleton\x18\x0f \x01(\x08\x1aJ\n\x0cSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.feast.core.OnDemandSource:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x17OnDemandFeatureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xc8\x01\n\x0eOnDemandSource\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x44\n\x17\x66\x65\x61ture_view_projection\x18\x03 \x01(\x0b\x32!.feast.core.FeatureViewProjectionH\x00\x12\x35\n\x13request_data_source\x18\x02 \x01(\x0b\x32\x16.feast.core.DataSourceH\x00\x42\x08\n\x06source\"H\n\x13UserDefinedFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t:\x02\x18\x01\x42]\n\x10\x66\x65\x61st.proto.coreB\x18OnDemandFeatureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n$feast/core/OnDemandFeatureView.proto\x12\nfeast.core\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a&feast/core/FeatureViewProjection.proto\x1a\x18\x66\x65\x61st/core/Feature.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1f\x66\x65\x61st/core/Transformation.proto\"{\n\x13OnDemandFeatureView\x12\x31\n\x04spec\x18\x01 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewSpec\x12\x31\n\x04meta\x18\x02 \x01(\x0b\x32#.feast.core.OnDemandFeatureViewMeta\"\x90\x05\n\x17OnDemandFeatureViewSpec\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12+\n\x08\x66\x65\x61tures\x18\x03 \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x41\n\x07sources\x18\x04 \x03(\x0b\x32\x30.feast.core.OnDemandFeatureViewSpec.SourcesEntry\x12\x42\n\x15user_defined_function\x18\x05 \x01(\x0b\x32\x1f.feast.core.UserDefinedFunctionB\x02\x18\x01\x12\x43\n\x16\x66\x65\x61ture_transformation\x18\n \x01(\x0b\x32#.feast.core.FeatureTransformationV2\x12\x13\n\x0b\x64\x65scription\x18\x06 \x01(\t\x12;\n\x04tags\x18\x07 \x03(\x0b\x32-.feast.core.OnDemandFeatureViewSpec.TagsEntry\x12\r\n\x05owner\x18\x08 \x01(\t\x12\x0c\n\x04mode\x18\x0b \x01(\t\x12\x1d\n\x15write_to_online_store\x18\x0c \x01(\x08\x12\x10\n\x08\x65ntities\x18\r \x03(\t\x12\x31\n\x0e\x65ntity_columns\x18\x0e \x03(\x0b\x32\x19.feast.core.FeatureSpecV2\x12\x11\n\tsingleton\x18\x0f \x01(\x08\x1aJ\n\x0cSourcesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12)\n\x05value\x18\x02 \x01(\x0b\x32\x1a.feast.core.OnDemandSource:\x02\x38\x01\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x8c\x01\n\x17OnDemandFeatureViewMeta\x12\x35\n\x11\x63reated_timestamp\x18\x01 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12:\n\x16last_updated_timestamp\x18\x02 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\"\xc8\x01\n\x0eOnDemandSource\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x44\n\x17\x66\x65\x61ture_view_projection\x18\x03 \x01(\x0b\x32!.feast.core.FeatureViewProjectionH\x00\x12\x35\n\x13request_data_source\x18\x02 \x01(\x0b\x32\x16.feast.core.DataSourceH\x00\x42\x08\n\x06source\"H\n\x13UserDefinedFunction\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0c\n\x04\x62ody\x18\x02 \x01(\x0c\x12\x11\n\tbody_text\x18\x03 \x01(\t:\x02\x18\x01\"X\n\x17OnDemandFeatureViewList\x12=\n\x14ondemandfeatureviews\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureViewB]\n\x10\x66\x65\x61st.proto.coreB\x18OnDemandFeatureViewProtoZ/github.com/feast-dev/feast/go/protos/feast/coreb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) @@ -50,4 +50,6 @@ _globals['_ONDEMANDSOURCE']._serialized_end=1371 _globals['_USERDEFINEDFUNCTION']._serialized_start=1373 _globals['_USERDEFINEDFUNCTION']._serialized_end=1445 + _globals['_ONDEMANDFEATUREVIEWLIST']._serialized_start=1447 + _globals['_ONDEMANDFEATUREVIEWLIST']._serialized_end=1535 # @@protoc_insertion_point(module_scope) diff --git a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi index 3380779c97e..c9fca2f550d 100644 --- a/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi +++ b/sdk/python/feast/protos/feast/core/OnDemandFeatureView_pb2.pyi @@ -233,3 +233,18 @@ class UserDefinedFunction(google.protobuf.message.Message): def ClearField(self, field_name: typing_extensions.Literal["body", b"body", "body_text", b"body_text", "name", b"name"]) -> None: ... global___UserDefinedFunction = UserDefinedFunction + +class OnDemandFeatureViewList(google.protobuf.message.Message): + DESCRIPTOR: google.protobuf.descriptor.Descriptor + + ONDEMANDFEATUREVIEWS_FIELD_NUMBER: builtins.int + @property + def ondemandfeatureviews(self) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[global___OnDemandFeatureView]: ... + def __init__( + self, + *, + ondemandfeatureviews: collections.abc.Iterable[global___OnDemandFeatureView] | None = ..., + ) -> None: ... + def ClearField(self, field_name: typing_extensions.Literal["ondemandfeatureviews", b"ondemandfeatureviews"]) -> None: ... + +global___OnDemandFeatureViewList = OnDemandFeatureViewList diff --git a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py index e0cae3da4b7..2d5f7b020ab 100644 --- a/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py +++ b/sdk/python/feast/protos/feast/registry/RegistryServer_pb2.py @@ -28,13 +28,14 @@ from feast.protos.feast.core import Project_pb2 as feast_dot_core_dot_Project__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#feast/registry/RegistryServer.proto\x12\x0e\x66\x65\x61st.registry\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/core/Registry.proto\x1a\x17\x66\x65\x61st/core/Entity.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a\"feast/core/StreamFeatureView.proto\x1a$feast/core/OnDemandFeatureView.proto\x1a\x1f\x66\x65\x61st/core/FeatureService.proto\x1a\x1d\x66\x65\x61st/core/SavedDataset.proto\x1a\"feast/core/ValidationProfile.proto\x1a\x1c\x66\x65\x61st/core/InfraObject.proto\x1a\x1b\x66\x65\x61st/core/Permission.proto\x1a\x18\x66\x65\x61st/core/Project.proto\"!\n\x0eRefreshRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\"W\n\x12UpdateInfraRequest\x12 \n\x05infra\x18\x01 \x01(\x0b\x32\x11.feast.core.Infra\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"7\n\x0fGetInfraRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"B\n\x1aListProjectMetadataRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"T\n\x1bListProjectMetadataResponse\x12\x35\n\x10project_metadata\x18\x01 \x03(\x0b\x32\x1b.feast.core.ProjectMetadata\"\xcb\x01\n\x1b\x41pplyMaterializationRequest\x12-\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureView\x12\x0f\n\x07project\x18\x02 \x01(\t\x12.\n\nstart_date\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_date\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\"Y\n\x12\x41pplyEntityRequest\x12\"\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x12.feast.core.Entity\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"F\n\x10GetEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa5\x01\n\x13ListEntitiesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12;\n\x04tags\x18\x03 \x03(\x0b\x32-.feast.registry.ListEntitiesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"<\n\x14ListEntitiesResponse\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.Entity\"D\n\x13\x44\x65leteEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"f\n\x16\x41pplyDataSourceRequest\x12+\n\x0b\x64\x61ta_source\x18\x01 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListDataSourcesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListDataSourcesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"G\n\x17ListDataSourcesResponse\x12,\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSource\"H\n\x17\x44\x65leteDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x02\n\x17\x41pplyFeatureViewRequest\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12\x0f\n\x07project\x18\x04 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\x42\x13\n\x11\x62\x61se_feature_view\"K\n\x15GetFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xad\x01\n\x17ListFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12?\n\x04tags\x18\x03 \x03(\x0b\x32\x31.feast.registry.ListFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"J\n\x18ListFeatureViewsResponse\x12.\n\rfeature_views\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureView\"I\n\x18\x44\x65leteFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xd6\x01\n\x0e\x41nyFeatureView\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x42\x12\n\x10\x61ny_feature_view\"N\n\x18GetAnyFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"U\n\x19GetAnyFeatureViewResponse\x12\x38\n\x10\x61ny_feature_view\x18\x01 \x01(\x0b\x32\x1e.feast.registry.AnyFeatureView\"\xb3\x01\n\x1aListAllFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListAllFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"T\n\x1bListAllFeatureViewsResponse\x12\x35\n\rfeature_views\x18\x01 \x03(\x0b\x32\x1e.feast.registry.AnyFeatureView\"Q\n\x1bGetStreamFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb9\x01\n\x1dListStreamFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x45\n\x04tags\x18\x03 \x03(\x0b\x32\x37.feast.registry.ListStreamFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x1eListStreamFeatureViewsResponse\x12;\n\x14stream_feature_views\x18\x01 \x03(\x0b\x32\x1d.feast.core.StreamFeatureView\"S\n\x1dGetOnDemandFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListOnDemandFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListOnDemandFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"d\n ListOnDemandFeatureViewsResponse\x12@\n\x17on_demand_feature_views\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureView\"r\n\x1a\x41pplyFeatureServiceRequest\x12\x33\n\x0f\x66\x65\x61ture_service\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureService\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"N\n\x18GetFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb3\x01\n\x1aListFeatureServicesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListFeatureServicesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"S\n\x1bListFeatureServicesResponse\x12\x34\n\x10\x66\x65\x61ture_services\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureService\"L\n\x1b\x44\x65leteFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"l\n\x18\x41pplySavedDatasetRequest\x12/\n\rsaved_dataset\x18\x01 \x01(\x0b\x32\x18.feast.core.SavedDataset\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"L\n\x16GetSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xaf\x01\n\x18ListSavedDatasetsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12@\n\x04tags\x18\x03 \x03(\x0b\x32\x32.feast.registry.ListSavedDatasetsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"M\n\x19ListSavedDatasetsResponse\x12\x30\n\x0esaved_datasets\x18\x01 \x03(\x0b\x32\x18.feast.core.SavedDataset\"J\n\x19\x44\x65leteSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x01\n\x1f\x41pplyValidationReferenceRequest\x12=\n\x14validation_reference\x18\x01 \x01(\x0b\x32\x1f.feast.core.ValidationReference\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"S\n\x1dGetValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListValidationReferencesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListValidationReferencesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"b\n ListValidationReferencesResponse\x12>\n\x15validation_references\x18\x01 \x03(\x0b\x32\x1f.feast.core.ValidationReference\"Q\n DeleteValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"e\n\x16\x41pplyPermissionRequest\x12*\n\npermission\x18\x01 \x01(\x0b\x32\x16.feast.core.Permission\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetPermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListPermissionsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListPermissionsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"F\n\x17ListPermissionsResponse\x12+\n\x0bpermissions\x18\x01 \x03(\x0b\x32\x16.feast.core.Permission\"H\n\x17\x44\x65letePermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"K\n\x13\x41pplyProjectRequest\x12$\n\x07project\x18\x01 \x01(\x0b\x32\x13.feast.core.Project\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"6\n\x11GetProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"\x94\x01\n\x13ListProjectsRequest\x12\x13\n\x0b\x61llow_cache\x18\x01 \x01(\x08\x12;\n\x04tags\x18\x02 \x03(\x0b\x32-.feast.registry.ListProjectsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"=\n\x14ListProjectsResponse\x12%\n\x08projects\x18\x01 \x03(\x0b\x32\x13.feast.core.Project\"4\n\x14\x44\x65leteProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\x32\xcb \n\x0eRegistryServer\x12K\n\x0b\x41pplyEntity\x12\".feast.registry.ApplyEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\tGetEntity\x12 .feast.registry.GetEntityRequest\x1a\x12.feast.core.Entity\"\x00\x12[\n\x0cListEntities\x12#.feast.registry.ListEntitiesRequest\x1a$.feast.registry.ListEntitiesResponse\"\x00\x12M\n\x0c\x44\x65leteEntity\x12#.feast.registry.DeleteEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyDataSource\x12&.feast.registry.ApplyDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetDataSource\x12$.feast.registry.GetDataSourceRequest\x1a\x16.feast.core.DataSource\"\x00\x12\x64\n\x0fListDataSources\x12&.feast.registry.ListDataSourcesRequest\x1a\'.feast.registry.ListDataSourcesResponse\"\x00\x12U\n\x10\x44\x65leteDataSource\x12\'.feast.registry.DeleteDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x10\x41pplyFeatureView\x12\'.feast.registry.ApplyFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x44\x65leteFeatureView\x12(.feast.registry.DeleteFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x11GetAnyFeatureView\x12(.feast.registry.GetAnyFeatureViewRequest\x1a).feast.registry.GetAnyFeatureViewResponse\"\x00\x12p\n\x13ListAllFeatureViews\x12*.feast.registry.ListAllFeatureViewsRequest\x1a+.feast.registry.ListAllFeatureViewsResponse\"\x00\x12R\n\x0eGetFeatureView\x12%.feast.registry.GetFeatureViewRequest\x1a\x17.feast.core.FeatureView\"\x00\x12g\n\x10ListFeatureViews\x12\'.feast.registry.ListFeatureViewsRequest\x1a(.feast.registry.ListFeatureViewsResponse\"\x00\x12\x64\n\x14GetStreamFeatureView\x12+.feast.registry.GetStreamFeatureViewRequest\x1a\x1d.feast.core.StreamFeatureView\"\x00\x12y\n\x16ListStreamFeatureViews\x12-.feast.registry.ListStreamFeatureViewsRequest\x1a..feast.registry.ListStreamFeatureViewsResponse\"\x00\x12j\n\x16GetOnDemandFeatureView\x12-.feast.registry.GetOnDemandFeatureViewRequest\x1a\x1f.feast.core.OnDemandFeatureView\"\x00\x12\x7f\n\x18ListOnDemandFeatureViews\x12/.feast.registry.ListOnDemandFeatureViewsRequest\x1a\x30.feast.registry.ListOnDemandFeatureViewsResponse\"\x00\x12[\n\x13\x41pplyFeatureService\x12*.feast.registry.ApplyFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12[\n\x11GetFeatureService\x12(.feast.registry.GetFeatureServiceRequest\x1a\x1a.feast.core.FeatureService\"\x00\x12p\n\x13ListFeatureServices\x12*.feast.registry.ListFeatureServicesRequest\x1a+.feast.registry.ListFeatureServicesResponse\"\x00\x12]\n\x14\x44\x65leteFeatureService\x12+.feast.registry.DeleteFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x41pplySavedDataset\x12(.feast.registry.ApplySavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x0fGetSavedDataset\x12&.feast.registry.GetSavedDatasetRequest\x1a\x18.feast.core.SavedDataset\"\x00\x12j\n\x11ListSavedDatasets\x12(.feast.registry.ListSavedDatasetsRequest\x1a).feast.registry.ListSavedDatasetsResponse\"\x00\x12Y\n\x12\x44\x65leteSavedDataset\x12).feast.registry.DeleteSavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x65\n\x18\x41pplyValidationReference\x12/.feast.registry.ApplyValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x16GetValidationReference\x12-.feast.registry.GetValidationReferenceRequest\x1a\x1f.feast.core.ValidationReference\"\x00\x12\x7f\n\x18ListValidationReferences\x12/.feast.registry.ListValidationReferencesRequest\x1a\x30.feast.registry.ListValidationReferencesResponse\"\x00\x12g\n\x19\x44\x65leteValidationReference\x12\x30.feast.registry.DeleteValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyPermission\x12&.feast.registry.ApplyPermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetPermission\x12$.feast.registry.GetPermissionRequest\x1a\x16.feast.core.Permission\"\x00\x12\x64\n\x0fListPermissions\x12&.feast.registry.ListPermissionsRequest\x1a\'.feast.registry.ListPermissionsResponse\"\x00\x12U\n\x10\x44\x65letePermission\x12\'.feast.registry.DeletePermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12M\n\x0c\x41pplyProject\x12#.feast.registry.ApplyProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x46\n\nGetProject\x12!.feast.registry.GetProjectRequest\x1a\x13.feast.core.Project\"\x00\x12[\n\x0cListProjects\x12#.feast.registry.ListProjectsRequest\x1a$.feast.registry.ListProjectsResponse\"\x00\x12O\n\rDeleteProject\x12$.feast.registry.DeleteProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12]\n\x14\x41pplyMaterialization\x12+.feast.registry.ApplyMaterializationRequest\x1a\x16.google.protobuf.Empty\"\x00\x12p\n\x13ListProjectMetadata\x12*.feast.registry.ListProjectMetadataRequest\x1a+.feast.registry.ListProjectMetadataResponse\"\x00\x12K\n\x0bUpdateInfra\x12\".feast.registry.UpdateInfraRequest\x1a\x16.google.protobuf.Empty\"\x00\x12@\n\x08GetInfra\x12\x1f.feast.registry.GetInfraRequest\x1a\x11.feast.core.Infra\"\x00\x12:\n\x06\x43ommit\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\x07Refresh\x12\x1e.feast.registry.RefreshRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x37\n\x05Proto\x12\x16.google.protobuf.Empty\x1a\x14.feast.core.Registry\"\x00\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n#feast/registry/RegistryServer.proto\x12\x0e\x66\x65\x61st.registry\x1a\x1bgoogle/protobuf/empty.proto\x1a\x1fgoogle/protobuf/timestamp.proto\x1a\x19\x66\x65\x61st/core/Registry.proto\x1a\x17\x66\x65\x61st/core/Entity.proto\x1a\x1b\x66\x65\x61st/core/DataSource.proto\x1a\x1c\x66\x65\x61st/core/FeatureView.proto\x1a\"feast/core/StreamFeatureView.proto\x1a$feast/core/OnDemandFeatureView.proto\x1a\x1f\x66\x65\x61st/core/FeatureService.proto\x1a\x1d\x66\x65\x61st/core/SavedDataset.proto\x1a\"feast/core/ValidationProfile.proto\x1a\x1c\x66\x65\x61st/core/InfraObject.proto\x1a\x1b\x66\x65\x61st/core/Permission.proto\x1a\x18\x66\x65\x61st/core/Project.proto\"!\n\x0eRefreshRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\"W\n\x12UpdateInfraRequest\x12 \n\x05infra\x18\x01 \x01(\x0b\x32\x11.feast.core.Infra\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"7\n\x0fGetInfraRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"B\n\x1aListProjectMetadataRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"T\n\x1bListProjectMetadataResponse\x12\x35\n\x10project_metadata\x18\x01 \x03(\x0b\x32\x1b.feast.core.ProjectMetadata\"\xcb\x01\n\x1b\x41pplyMaterializationRequest\x12-\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureView\x12\x0f\n\x07project\x18\x02 \x01(\t\x12.\n\nstart_date\x18\x03 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12,\n\x08\x65nd_date\x18\x04 \x01(\x0b\x32\x1a.google.protobuf.Timestamp\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\"Y\n\x12\x41pplyEntityRequest\x12\"\n\x06\x65ntity\x18\x01 \x01(\x0b\x32\x12.feast.core.Entity\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"F\n\x10GetEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xa5\x01\n\x13ListEntitiesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12;\n\x04tags\x18\x03 \x03(\x0b\x32-.feast.registry.ListEntitiesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"<\n\x14ListEntitiesResponse\x12$\n\x08\x65ntities\x18\x01 \x03(\x0b\x32\x12.feast.core.Entity\"D\n\x13\x44\x65leteEntityRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"f\n\x16\x41pplyDataSourceRequest\x12+\n\x0b\x64\x61ta_source\x18\x01 \x01(\x0b\x32\x16.feast.core.DataSource\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListDataSourcesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListDataSourcesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"G\n\x17ListDataSourcesResponse\x12,\n\x0c\x64\x61ta_sources\x18\x01 \x03(\x0b\x32\x16.feast.core.DataSource\"H\n\x17\x44\x65leteDataSourceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x02\n\x17\x41pplyFeatureViewRequest\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x12\x0f\n\x07project\x18\x04 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x05 \x01(\x08\x42\x13\n\x11\x62\x61se_feature_view\"K\n\x15GetFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xad\x01\n\x17ListFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12?\n\x04tags\x18\x03 \x03(\x0b\x32\x31.feast.registry.ListFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"J\n\x18ListFeatureViewsResponse\x12.\n\rfeature_views\x18\x01 \x03(\x0b\x32\x17.feast.core.FeatureView\"I\n\x18\x44\x65leteFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\xd6\x01\n\x0e\x41nyFeatureView\x12/\n\x0c\x66\x65\x61ture_view\x18\x01 \x01(\x0b\x32\x17.feast.core.FeatureViewH\x00\x12\x41\n\x16on_demand_feature_view\x18\x02 \x01(\x0b\x32\x1f.feast.core.OnDemandFeatureViewH\x00\x12<\n\x13stream_feature_view\x18\x03 \x01(\x0b\x32\x1d.feast.core.StreamFeatureViewH\x00\x42\x12\n\x10\x61ny_feature_view\"N\n\x18GetAnyFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"U\n\x19GetAnyFeatureViewResponse\x12\x38\n\x10\x61ny_feature_view\x18\x01 \x01(\x0b\x32\x1e.feast.registry.AnyFeatureView\"\xb3\x01\n\x1aListAllFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListAllFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"T\n\x1bListAllFeatureViewsResponse\x12\x35\n\rfeature_views\x18\x01 \x03(\x0b\x32\x1e.feast.registry.AnyFeatureView\"Q\n\x1bGetStreamFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb9\x01\n\x1dListStreamFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x45\n\x04tags\x18\x03 \x03(\x0b\x32\x37.feast.registry.ListStreamFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"]\n\x1eListStreamFeatureViewsResponse\x12;\n\x14stream_feature_views\x18\x01 \x03(\x0b\x32\x1d.feast.core.StreamFeatureView\"S\n\x1dGetOnDemandFeatureViewRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListOnDemandFeatureViewsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListOnDemandFeatureViewsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"d\n ListOnDemandFeatureViewsResponse\x12@\n\x17on_demand_feature_views\x18\x01 \x03(\x0b\x32\x1f.feast.core.OnDemandFeatureView\"r\n\x1a\x41pplyFeatureServiceRequest\x12\x33\n\x0f\x66\x65\x61ture_service\x18\x01 \x01(\x0b\x32\x1a.feast.core.FeatureService\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"N\n\x18GetFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xb3\x01\n\x1aListFeatureServicesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12\x42\n\x04tags\x18\x03 \x03(\x0b\x32\x34.feast.registry.ListFeatureServicesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"S\n\x1bListFeatureServicesResponse\x12\x34\n\x10\x66\x65\x61ture_services\x18\x01 \x03(\x0b\x32\x1a.feast.core.FeatureService\"L\n\x1b\x44\x65leteFeatureServiceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"l\n\x18\x41pplySavedDatasetRequest\x12/\n\rsaved_dataset\x18\x01 \x01(\x0b\x32\x18.feast.core.SavedDataset\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"L\n\x16GetSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xaf\x01\n\x18ListSavedDatasetsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12@\n\x04tags\x18\x03 \x03(\x0b\x32\x32.feast.registry.ListSavedDatasetsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"M\n\x19ListSavedDatasetsResponse\x12\x30\n\x0esaved_datasets\x18\x01 \x03(\x0b\x32\x18.feast.core.SavedDataset\"J\n\x19\x44\x65leteSavedDatasetRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"\x81\x01\n\x1f\x41pplyValidationReferenceRequest\x12=\n\x14validation_reference\x18\x01 \x01(\x0b\x32\x1f.feast.core.ValidationReference\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"S\n\x1dGetValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xbd\x01\n\x1fListValidationReferencesRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12G\n\x04tags\x18\x03 \x03(\x0b\x32\x39.feast.registry.ListValidationReferencesRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"b\n ListValidationReferencesResponse\x12>\n\x15validation_references\x18\x01 \x03(\x0b\x32\x1f.feast.core.ValidationReference\"Q\n DeleteValidationReferenceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"e\n\x16\x41pplyPermissionRequest\x12*\n\npermission\x18\x01 \x01(\x0b\x32\x16.feast.core.Permission\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"J\n\x14GetPermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x03 \x01(\x08\"\xab\x01\n\x16ListPermissionsRequest\x12\x0f\n\x07project\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\x12>\n\x04tags\x18\x03 \x03(\x0b\x32\x30.feast.registry.ListPermissionsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"F\n\x17ListPermissionsResponse\x12+\n\x0bpermissions\x18\x01 \x03(\x0b\x32\x16.feast.core.Permission\"H\n\x17\x44\x65letePermissionRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0f\n\x07project\x18\x02 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x03 \x01(\x08\"K\n\x13\x41pplyProjectRequest\x12$\n\x07project\x18\x01 \x01(\x0b\x32\x13.feast.core.Project\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\"6\n\x11GetProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x13\n\x0b\x61llow_cache\x18\x02 \x01(\x08\"\x94\x01\n\x13ListProjectsRequest\x12\x13\n\x0b\x61llow_cache\x18\x01 \x01(\x08\x12;\n\x04tags\x18\x02 \x03(\x0b\x32-.feast.registry.ListProjectsRequest.TagsEntry\x1a+\n\tTagsEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"=\n\x14ListProjectsResponse\x12%\n\x08projects\x18\x01 \x03(\x0b\x32\x13.feast.core.Project\"4\n\x14\x44\x65leteProjectRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x0e\n\x06\x63ommit\x18\x02 \x01(\x08\x32\xcb \n\x0eRegistryServer\x12K\n\x0b\x41pplyEntity\x12\".feast.registry.ApplyEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\tGetEntity\x12 .feast.registry.GetEntityRequest\x1a\x12.feast.core.Entity\"\x00\x12[\n\x0cListEntities\x12#.feast.registry.ListEntitiesRequest\x1a$.feast.registry.ListEntitiesResponse\"\x00\x12M\n\x0c\x44\x65leteEntity\x12#.feast.registry.DeleteEntityRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyDataSource\x12&.feast.registry.ApplyDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetDataSource\x12$.feast.registry.GetDataSourceRequest\x1a\x16.feast.core.DataSource\"\x00\x12\x64\n\x0fListDataSources\x12&.feast.registry.ListDataSourcesRequest\x1a\'.feast.registry.ListDataSourcesResponse\"\x00\x12U\n\x10\x44\x65leteDataSource\x12\'.feast.registry.DeleteDataSourceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x10\x41pplyFeatureView\x12\'.feast.registry.ApplyFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x44\x65leteFeatureView\x12(.feast.registry.DeleteFeatureViewRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x11GetAnyFeatureView\x12(.feast.registry.GetAnyFeatureViewRequest\x1a).feast.registry.GetAnyFeatureViewResponse\"\x00\x12p\n\x13ListAllFeatureViews\x12*.feast.registry.ListAllFeatureViewsRequest\x1a+.feast.registry.ListAllFeatureViewsResponse\"\x00\x12R\n\x0eGetFeatureView\x12%.feast.registry.GetFeatureViewRequest\x1a\x17.feast.core.FeatureView\"\x00\x12g\n\x10ListFeatureViews\x12\'.feast.registry.ListFeatureViewsRequest\x1a(.feast.registry.ListFeatureViewsResponse\"\x00\x12\x64\n\x14GetStreamFeatureView\x12+.feast.registry.GetStreamFeatureViewRequest\x1a\x1d.feast.core.StreamFeatureView\"\x00\x12y\n\x16ListStreamFeatureViews\x12-.feast.registry.ListStreamFeatureViewsRequest\x1a..feast.registry.ListStreamFeatureViewsResponse\"\x00\x12j\n\x16GetOnDemandFeatureView\x12-.feast.registry.GetOnDemandFeatureViewRequest\x1a\x1f.feast.core.OnDemandFeatureView\"\x00\x12\x7f\n\x18ListOnDemandFeatureViews\x12/.feast.registry.ListOnDemandFeatureViewsRequest\x1a\x30.feast.registry.ListOnDemandFeatureViewsResponse\"\x00\x12[\n\x13\x41pplyFeatureService\x12*.feast.registry.ApplyFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12[\n\x11GetFeatureService\x12(.feast.registry.GetFeatureServiceRequest\x1a\x1a.feast.core.FeatureService\"\x00\x12p\n\x13ListFeatureServices\x12*.feast.registry.ListFeatureServicesRequest\x1a+.feast.registry.ListFeatureServicesResponse\"\x00\x12]\n\x14\x44\x65leteFeatureService\x12+.feast.registry.DeleteFeatureServiceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12W\n\x11\x41pplySavedDataset\x12(.feast.registry.ApplySavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12U\n\x0fGetSavedDataset\x12&.feast.registry.GetSavedDatasetRequest\x1a\x18.feast.core.SavedDataset\"\x00\x12j\n\x11ListSavedDatasets\x12(.feast.registry.ListSavedDatasetsRequest\x1a).feast.registry.ListSavedDatasetsResponse\"\x00\x12Y\n\x12\x44\x65leteSavedDataset\x12).feast.registry.DeleteSavedDatasetRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x65\n\x18\x41pplyValidationReference\x12/.feast.registry.ApplyValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12j\n\x16GetValidationReference\x12-.feast.registry.GetValidationReferenceRequest\x1a\x1f.feast.core.ValidationReference\"\x00\x12\x7f\n\x18ListValidationReferences\x12/.feast.registry.ListValidationReferencesRequest\x1a\x30.feast.registry.ListValidationReferencesResponse\"\x00\x12g\n\x19\x44\x65leteValidationReference\x12\x30.feast.registry.DeleteValidationReferenceRequest\x1a\x16.google.protobuf.Empty\"\x00\x12S\n\x0f\x41pplyPermission\x12&.feast.registry.ApplyPermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12O\n\rGetPermission\x12$.feast.registry.GetPermissionRequest\x1a\x16.feast.core.Permission\"\x00\x12\x64\n\x0fListPermissions\x12&.feast.registry.ListPermissionsRequest\x1a\'.feast.registry.ListPermissionsResponse\"\x00\x12U\n\x10\x44\x65letePermission\x12\'.feast.registry.DeletePermissionRequest\x1a\x16.google.protobuf.Empty\"\x00\x12M\n\x0c\x41pplyProject\x12#.feast.registry.ApplyProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x46\n\nGetProject\x12!.feast.registry.GetProjectRequest\x1a\x13.feast.core.Project\"\x00\x12[\n\x0cListProjects\x12#.feast.registry.ListProjectsRequest\x1a$.feast.registry.ListProjectsResponse\"\x00\x12O\n\rDeleteProject\x12$.feast.registry.DeleteProjectRequest\x1a\x16.google.protobuf.Empty\"\x00\x12]\n\x14\x41pplyMaterialization\x12+.feast.registry.ApplyMaterializationRequest\x1a\x16.google.protobuf.Empty\"\x00\x12p\n\x13ListProjectMetadata\x12*.feast.registry.ListProjectMetadataRequest\x1a+.feast.registry.ListProjectMetadataResponse\"\x00\x12K\n\x0bUpdateInfra\x12\".feast.registry.UpdateInfraRequest\x1a\x16.google.protobuf.Empty\"\x00\x12@\n\x08GetInfra\x12\x1f.feast.registry.GetInfraRequest\x1a\x11.feast.core.Infra\"\x00\x12:\n\x06\x43ommit\x12\x16.google.protobuf.Empty\x1a\x16.google.protobuf.Empty\"\x00\x12\x43\n\x07Refresh\x12\x1e.feast.registry.RefreshRequest\x1a\x16.google.protobuf.Empty\"\x00\x12\x37\n\x05Proto\x12\x16.google.protobuf.Empty\x1a\x14.feast.core.Registry\"\x00\x42\x35Z3github.com/feast-dev/feast/go/protos/feast/registryb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'feast.registry.RegistryServer_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'Z3github.com/feast-dev/feast/go/protos/feast/registry' _globals['_LISTENTITIESREQUEST_TAGSENTRY']._options = None _globals['_LISTENTITIESREQUEST_TAGSENTRY']._serialized_options = b'8\001' _globals['_LISTDATASOURCESREQUEST_TAGSENTRY']._options = None diff --git a/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py b/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py index 8e40630cfff..ce4db37a658 100644 --- a/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py +++ b/sdk/python/feast/protos/feast/serving/GrpcServer_pb2.py @@ -15,13 +15,14 @@ from feast.protos.feast.serving import ServingService_pb2 as feast_dot_serving_dot_ServingService__pb2 -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1e\x66\x65\x61st/serving/GrpcServer.proto\x1a\"feast/serving/ServingService.proto\"\xb3\x01\n\x0bPushRequest\x12,\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x1a.PushRequest.FeaturesEntry\x12\x1b\n\x13stream_feature_view\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x12\n\n\x02to\x18\x04 \x01(\t\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1e\n\x0cPushResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\xc1\x01\n\x19WriteToOnlineStoreRequest\x12:\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32(.WriteToOnlineStoreRequest.FeaturesEntry\x12\x19\n\x11\x66\x65\x61ture_view_name\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\",\n\x1aWriteToOnlineStoreResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x32\xf1\x01\n\x11GrpcFeatureServer\x12%\n\x04Push\x12\x0c.PushRequest\x1a\r.PushResponse\"\x00\x12M\n\x12WriteToOnlineStore\x12\x1a.WriteToOnlineStoreRequest\x1a\x1b.WriteToOnlineStoreResponse\x12\x66\n\x11GetOnlineFeatures\x12\'.feast.serving.GetOnlineFeaturesRequest\x1a(.feast.serving.GetOnlineFeaturesResponseb\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x1e\x66\x65\x61st/serving/GrpcServer.proto\x1a\"feast/serving/ServingService.proto\"\xb3\x01\n\x0bPushRequest\x12,\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32\x1a.PushRequest.FeaturesEntry\x12\x1b\n\x13stream_feature_view\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x12\n\n\x02to\x18\x04 \x01(\t\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\"\x1e\n\x0cPushResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\"\xc1\x01\n\x19WriteToOnlineStoreRequest\x12:\n\x08\x66\x65\x61tures\x18\x01 \x03(\x0b\x32(.WriteToOnlineStoreRequest.FeaturesEntry\x12\x19\n\x11\x66\x65\x61ture_view_name\x18\x02 \x01(\t\x12\x1c\n\x14\x61llow_registry_cache\x18\x03 \x01(\x08\x1a/\n\rFeaturesEntry\x12\x0b\n\x03key\x18\x01 \x01(\t\x12\r\n\x05value\x18\x02 \x01(\t:\x02\x38\x01\",\n\x1aWriteToOnlineStoreResponse\x12\x0e\n\x06status\x18\x01 \x01(\x08\x32\xf1\x01\n\x11GrpcFeatureServer\x12%\n\x04Push\x12\x0c.PushRequest\x1a\r.PushResponse\"\x00\x12M\n\x12WriteToOnlineStore\x12\x1a.WriteToOnlineStoreRequest\x1a\x1b.WriteToOnlineStoreResponse\x12\x66\n\x11GetOnlineFeatures\x12\'.feast.serving.GetOnlineFeaturesRequest\x1a(.feast.serving.GetOnlineFeaturesResponseB4Z2github.com/feast-dev/feast/go/protos/feast/servingb\x06proto3') _globals = globals() _builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) _builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'feast.serving.GrpcServer_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: - DESCRIPTOR._options = None + _globals['DESCRIPTOR']._options = None + _globals['DESCRIPTOR']._serialized_options = b'Z2github.com/feast-dev/feast/go/protos/feast/serving' _globals['_PUSHREQUEST_FEATURESENTRY']._options = None _globals['_PUSHREQUEST_FEATURESENTRY']._serialized_options = b'8\001' _globals['_WRITETOONLINESTOREREQUEST_FEATURESENTRY']._options = None diff --git a/sdk/python/feast/ssl_ca_trust_store_setup.py b/sdk/python/feast/ssl_ca_trust_store_setup.py new file mode 100644 index 00000000000..72e84132187 --- /dev/null +++ b/sdk/python/feast/ssl_ca_trust_store_setup.py @@ -0,0 +1,22 @@ +import logging +import os + +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) + + +def configure_ca_trust_store_env_variables(): + """ + configures the environment variable so that other libraries or servers refer to the TLS ca file path. + :param ca_file_path: + :return: + """ + if ( + "FEAST_CA_CERT_FILE_PATH" in os.environ + and os.environ["FEAST_CA_CERT_FILE_PATH"] + ): + logger.info( + f"Feast CA Cert file path found in environment variable FEAST_CA_CERT_FILE_PATH={os.environ['FEAST_CA_CERT_FILE_PATH']}. Going to refer this path." + ) + os.environ["SSL_CERT_FILE"] = os.environ["FEAST_CA_CERT_FILE_PATH"] + os.environ["REQUESTS_CA_BUNDLE"] = os.environ["FEAST_CA_CERT_FILE_PATH"] diff --git a/sdk/python/feast/ui/package.json b/sdk/python/feast/ui/package.json index 0382cfafee6..f1e28382da6 100644 --- a/sdk/python/feast/ui/package.json +++ b/sdk/python/feast/ui/package.json @@ -4,7 +4,7 @@ "private": true, "dependencies": { "@elastic/datemath": "^5.0.3", - "@elastic/eui": "^55.0.1", + "@elastic/eui": "^72.0.0", "@emotion/react": "^11.9.0", "@feast-dev/feast-ui": "0.42.0", "@testing-library/jest-dom": "^5.16.4", diff --git a/sdk/python/feast/ui/yarn.lock b/sdk/python/feast/ui/yarn.lock index 2e6af4dc7ca..6578b4a34f7 100644 --- a/sdk/python/feast/ui/yarn.lock +++ b/sdk/python/feast/ui/yarn.lock @@ -1272,10 +1272,10 @@ dependencies: tslib "^1.9.3" -"@elastic/eui@^55.0.1": - version "55.1.2" - resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-55.1.2.tgz#dd0b42f5b26c5800d6a9cb2d4c2fe1afce9d3f07" - integrity sha512-wwZz5KxMIMFlqEsoCRiQBJDc4CrluS1d0sCOmQ5lhIzKhYc91MdxnqCk2i6YkhL4sSDf2Y9KAEuMXa+uweOWUA== +"@elastic/eui@^72.0.0": + version "72.2.0" + resolved "https://registry.yarnpkg.com/@elastic/eui/-/eui-72.2.0.tgz#0d89ec4c6d8a677ba41d086abd509c5a5ea09180" + integrity sha512-3JHKLWqbU1A6qMVkw0n1VZ5PaL07sd3N44tWsRCn+DEaDv9jq68ilEmY1wdYqKXw8VyFwcPbd8ZYZpdzBD2nPA== dependencies: "@types/chroma-js" "^2.0.0" "@types/lodash" "^4.14.160" @@ -1296,7 +1296,7 @@ react-beautiful-dnd "^13.1.0" react-dropzone "^11.5.3" react-element-to-jsx-string "^14.3.4" - react-focus-on "^3.5.4" + react-focus-on "^3.7.0" react-input-autosize "^3.0.0" react-is "^17.0.2" react-virtualized-auto-sizer "^1.0.6" @@ -1307,7 +1307,7 @@ rehype-stringify "^8.0.0" remark-breaks "^2.0.2" remark-emoji "^2.1.0" - remark-parse "^8.0.3" + remark-parse-no-trim "^8.0.4" remark-rehype "^8.0.0" tabbable "^5.2.1" text-diff "^1.0.1" @@ -3363,13 +3363,6 @@ argparse@^2.0.1: resolved "https://registry.yarnpkg.com/argparse/-/argparse-2.0.1.tgz#246f50f3ca78a3240f6c997e8a9bd1eac49e4b38" integrity sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q== -aria-hidden@^1.1.3: - version "1.1.3" - resolved "https://registry.yarnpkg.com/aria-hidden/-/aria-hidden-1.1.3.tgz#bb48de18dc84787a3c6eee113709c473c64ec254" - integrity sha512-RhVWFtKH5BiGMycI72q2RAFMLQi8JP9bLuQXgR5a8Znp7P5KOIADSJeyfI8PCVxLEp067B2HbP5JIiI/PXIZeA== - dependencies: - tslib "^1.0.0" - aria-hidden@^1.2.2: version "1.2.4" resolved "https://registry.yarnpkg.com/aria-hidden/-/aria-hidden-1.2.4.tgz#b78e383fdbc04d05762c78b4a25a501e736c4522" @@ -5724,13 +5717,6 @@ flatted@^3.1.0: resolved "https://registry.yarnpkg.com/flatted/-/flatted-3.2.5.tgz#76c8584f4fc843db64702a6bd04ab7a8bd666da3" integrity sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg== -focus-lock@^0.11.2: - version "0.11.2" - resolved "https://registry.yarnpkg.com/focus-lock/-/focus-lock-0.11.2.tgz#aeef3caf1cea757797ac8afdebaec8fd9ab243ed" - integrity sha512-pZ2bO++NWLHhiKkgP1bEXHhR1/OjVcSvlCJ98aNJDFeb7H5OOQaO+SKOZle6041O9rv2tmbrO4JzClAvDUHf0g== - dependencies: - tslib "^2.0.3" - focus-lock@^1.3.5: version "1.3.5" resolved "https://registry.yarnpkg.com/focus-lock/-/focus-lock-1.3.5.tgz#aa644576e5ec47d227b57eb14e1efb2abf33914c" @@ -7785,15 +7771,10 @@ nano-time@1.0.0: dependencies: big-integer "^1.6.16" -nanoid@^3.3.3: - version "3.3.4" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.4.tgz#730b67e3cd09e2deacf03c027c81c9d9dbc5e8ab" - integrity sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw== - -nanoid@^3.3.7: - version "3.3.7" - resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.7.tgz#d0c301a691bc8d54efa0a2226ccf3fe2fd656bd8" - integrity sha512-eSRppjcPIatRIMC1U6UngP8XFcz8MQWGQdt1MTBQ7NaAmvXDfvNxbvWV3x2y6CdEUciCSsDHDQZbhYaB8QEo2g== +nanoid@^3.3.3, nanoid@^3.3.7: + version "3.3.8" + resolved "https://registry.yarnpkg.com/nanoid/-/nanoid-3.3.8.tgz#b1be3030bee36aaff18bacb375e5cce521684baf" + integrity sha512-WNLf5Sd8oZxOm+TzppcYk8gVOgP+l58xNy58D0nbUnOxOWRWvlcCV4kUF7ltmI6PsrLl/BgKEyS4mqsGChFN0w== natural-compare@^1.4.0: version "1.4.0" @@ -9103,32 +9084,7 @@ react-focus-lock@^2.11.3: use-callback-ref "^1.3.2" use-sidecar "^1.1.2" -react-focus-lock@^2.9.0: - version "2.9.1" - resolved "https://registry.yarnpkg.com/react-focus-lock/-/react-focus-lock-2.9.1.tgz#094cfc19b4f334122c73bb0bff65d77a0c92dd16" - integrity sha512-pSWOQrUmiKLkffPO6BpMXN7SNKXMsuOakl652IBuALAu1esk+IcpJyM+ALcYzPTTFz1rD0R54aB9A4HuP5t1Wg== - dependencies: - "@babel/runtime" "^7.0.0" - focus-lock "^0.11.2" - prop-types "^15.6.2" - react-clientside-effect "^1.2.6" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - -react-focus-on@^3.5.4: - version "3.6.0" - resolved "https://registry.yarnpkg.com/react-focus-on/-/react-focus-on-3.6.0.tgz#159e13082dad4ea1f07abe11254f0e981d5a7b79" - integrity sha512-onIRjpd9trAUenXNdDcvjc8KJUSklty4X/Gr7hAm/MzM7ekSF2pg9D8KBKL7ipige22IAPxLRRf/EmJji9KD6Q== - dependencies: - aria-hidden "^1.1.3" - react-focus-lock "^2.9.0" - react-remove-scroll "^2.5.2" - react-style-singleton "^2.2.0" - tslib "^2.3.1" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - -react-focus-on@^3.9.1: +react-focus-on@^3.7.0, react-focus-on@^3.9.1: version "3.9.4" resolved "https://registry.yarnpkg.com/react-focus-on/-/react-focus-on-3.9.4.tgz#0b6c13273d86243c330d1aa53af39290f543da7b" integrity sha512-NFKmeH6++wu8e7LJcbwV8TTd4L5w/U5LMXTMOdUcXhCcZ7F5VOvgeTHd4XN1PD7TNmdvldDu/ENROOykUQ4yQg== @@ -9209,14 +9165,6 @@ react-refresh@^0.11.0: resolved "https://registry.yarnpkg.com/react-refresh/-/react-refresh-0.11.0.tgz#77198b944733f0f1f1a90e791de4541f9f074046" integrity sha512-F27qZr8uUqwhWZboondsPx8tnC3Ct3SxZA3V5WyEvujRyyNv0VYPhoBg1gZ8/MV5tubQp76Trw8lTv9hzRBa+A== -react-remove-scroll-bar@^2.3.1: - version "2.3.1" - resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.1.tgz#9f13b05b249eaa57c8d646c1ebb83006b3581f5f" - integrity sha512-IvGX3mJclEF7+hga8APZczve1UyGMkMG+tjS0o/U1iLgvZRpjFAQEUBJ4JETfvbNlfNnZnoDyWJCICkA15Mghg== - dependencies: - react-style-singleton "^2.2.0" - tslib "^2.0.0" - react-remove-scroll-bar@^2.3.4, react-remove-scroll-bar@^2.3.6: version "2.3.6" resolved "https://registry.yarnpkg.com/react-remove-scroll-bar/-/react-remove-scroll-bar-2.3.6.tgz#3e585e9d163be84a010180b18721e851ac81a29c" @@ -9225,17 +9173,6 @@ react-remove-scroll-bar@^2.3.4, react-remove-scroll-bar@^2.3.6: react-style-singleton "^2.2.1" tslib "^2.0.0" -react-remove-scroll@^2.5.2: - version "2.5.3" - resolved "https://registry.yarnpkg.com/react-remove-scroll/-/react-remove-scroll-2.5.3.tgz#a152196e710e8e5811be39dc352fd8a90b05c961" - integrity sha512-NQ1bXrxKrnK5pFo/GhLkXeo3CrK5steI+5L+jynwwIemvZyfXqaL0L5BzwJd7CSwNCU723DZaccvjuyOdoy3Xw== - dependencies: - react-remove-scroll-bar "^2.3.1" - react-style-singleton "^2.2.0" - tslib "^2.0.0" - use-callback-ref "^1.3.0" - use-sidecar "^1.1.2" - react-remove-scroll@^2.6.0: version "2.6.0" resolved "https://registry.yarnpkg.com/react-remove-scroll/-/react-remove-scroll-2.6.0.tgz#fb03a0845d7768a4f1519a99fdb84983b793dc07" @@ -9317,15 +9254,6 @@ react-scripts@^5.0.0: optionalDependencies: fsevents "^2.3.2" -react-style-singleton@^2.2.0: - version "2.2.0" - resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.0.tgz#70f45f5fef97fdb9a52eed98d1839fa6b9032b22" - integrity sha512-nK7mN92DMYZEu3cQcAhfwE48NpzO5RpxjG4okbSqRRbfal9Pk+fG2RdQXTMp+f6all1hB9LIJSt+j7dCYrU11g== - dependencies: - get-nonce "^1.0.0" - invariant "^2.2.4" - tslib "^2.0.0" - react-style-singleton@^2.2.1: version "2.2.1" resolved "https://registry.yarnpkg.com/react-style-singleton/-/react-style-singleton-2.2.1.tgz#f99e420492b2d8f34d38308ff660b60d0b1205b4" @@ -9589,28 +9517,6 @@ remark-parse-no-trim@^8.0.4: vfile-location "^3.0.0" xtend "^4.0.1" -remark-parse@^8.0.3: - version "8.0.3" - resolved "https://registry.yarnpkg.com/remark-parse/-/remark-parse-8.0.3.tgz#9c62aa3b35b79a486454c690472906075f40c7e1" - integrity sha512-E1K9+QLGgggHxCQtLt++uXltxEprmWzNfg+MxpfHsZlrddKzZ/hZyWHDbK3/Ap8HJQqYJRXP+jHczdL6q6i85Q== - dependencies: - ccount "^1.0.0" - collapse-white-space "^1.0.2" - is-alphabetical "^1.0.0" - is-decimal "^1.0.0" - is-whitespace-character "^1.0.0" - is-word-character "^1.0.0" - markdown-escapes "^1.0.0" - parse-entities "^2.0.0" - repeat-string "^1.5.4" - state-toggle "^1.0.0" - trim "0.0.1" - trim-trailing-lines "^1.0.0" - unherit "^1.0.4" - unist-util-remove-position "^2.0.0" - vfile-location "^3.0.0" - xtend "^4.0.1" - remark-rehype@^8.0.0, remark-rehype@^8.1.0: version "8.1.0" resolved "https://registry.yarnpkg.com/remark-rehype/-/remark-rehype-8.1.0.tgz#610509a043484c1e697437fa5eb3fd992617c945" @@ -10643,11 +10549,6 @@ trim-trailing-lines@^1.0.0: resolved "https://registry.yarnpkg.com/trim-trailing-lines/-/trim-trailing-lines-1.1.4.tgz#bd4abbec7cc880462f10b2c8b5ce1d8d1ec7c2c0" integrity sha512-rjUWSqnfTNrjbB9NQWfPMH/xRK1deHeGsHoVfpxJ++XeYXE0d6B1En37AHfw3jtfTU7dzMzZL2jjpe8Qb5gLIQ== -trim@0.0.1: - version "0.0.1" - resolved "https://registry.yarnpkg.com/trim/-/trim-0.0.1.tgz#5858547f6b290757ee95cccc666fb50084c460dd" - integrity sha1-WFhUf2spB1fulczMZm+1AITEYN0= - trough@^1.0.0: version "1.0.5" resolved "https://registry.yarnpkg.com/trough/-/trough-1.0.5.tgz#b8b639cefad7d0bb2abd37d433ff8293efa5f406" @@ -10673,7 +10574,7 @@ tslib@2.6.2: resolved "https://registry.yarnpkg.com/tslib/-/tslib-2.6.2.tgz#703ac29425e7b37cd6fd456e92404d46d1f3e4ae" integrity sha512-AEYxH93jGFPn/a2iVAwW87VuUIkR1FVUKB77NwMF7nBTDkDrrT/Hpt/IrCJ0QXhW27jTBDcf5ZY7w6RiqTMw2Q== -tslib@^1.0.0, tslib@^1.8.1, tslib@^1.9.3: +tslib@^1.8.1, tslib@^1.9.3: version "1.14.1" resolved "https://registry.yarnpkg.com/tslib/-/tslib-1.14.1.tgz#cf2d38bdc34a134bcaf1091c41f6619e2f672d00" integrity sha512-Xni35NKzjgMrwevysHTCArtLDpPvye8zV/0E4EyYn43P7/7qvQwPh9BGkHewbMulVntbigmcT7rdX3BNo9wRJg== diff --git a/sdk/python/feast/utils.py b/sdk/python/feast/utils.py index 51d4bf4f2cc..cfc19e37ca4 100644 --- a/sdk/python/feast/utils.py +++ b/sdk/python/feast/utils.py @@ -1192,6 +1192,10 @@ def _utc_now() -> datetime: return datetime.now(tz=timezone.utc) +def _serialize_vector_to_float_list(vector: List[float]) -> ValueProto: + return ValueProto(float_list_val=FloatListProto(val=vector)) + + def _build_retrieve_online_document_record( entity_key: Union[str, bytes], feature_value: Union[str, bytes], diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 54a64f5b1ce..c0b1c348a3a 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.10 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.10-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -46,7 +48,7 @@ async-timeout==5.0.1 # redis atpublic==5.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -69,11 +71,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -88,7 +90,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -128,9 +130,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -146,21 +148,21 @@ cryptography==42.0.8 # types-redis cython==3.0.11 # via thriftpy2 -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -176,10 +178,12 @@ duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus exceptiongroup==1.2.2 # via # anyio @@ -191,9 +195,9 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -211,7 +215,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -220,7 +224,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -242,9 +246,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -264,7 +268,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -275,6 +279,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -344,7 +349,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.29.0 +ipython==8.30.0 # via # great-expectations # ipykernel @@ -372,7 +377,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -404,7 +409,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -417,7 +422,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -442,15 +447,19 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 - # via great-expectations +marshmallow==3.23.2 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 +milvus-lite==2.4.10 + # via pymilvus +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -480,7 +489,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -494,7 +503,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -548,6 +557,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -582,13 +592,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -614,6 +624,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -658,13 +669,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -673,19 +686,21 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -738,8 +753,10 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn -python-json-logger==2.0.7 + # via + # environs + # uvicorn +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -815,7 +832,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -825,7 +842,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -839,12 +856,12 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -859,7 +876,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -883,7 +900,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -909,7 +926,7 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via # build # coverage @@ -939,6 +956,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -954,7 +972,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -968,7 +986,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -984,7 +1002,7 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests @@ -1000,6 +1018,7 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # multidict # mypy # psycopg @@ -1018,6 +1037,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==2.2.3 @@ -1033,7 +1054,7 @@ urllib3==2.2.3 # requests # responses # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1045,7 +1066,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1075,7 +1096,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index 9a087b4a8eb..87b9cf04c91 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.10 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.10-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -25,19 +25,19 @@ cloudpickle==3.1.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -85,25 +85,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -122,11 +122,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -140,7 +140,7 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via mypy toolz==1.0.0 # via @@ -164,7 +164,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -172,7 +172,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 43637fd2067..37bcbdb2c9c 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.11 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.11-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -44,7 +46,7 @@ async-timeout==5.0.1 # via redis atpublic==5.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -67,11 +69,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -86,7 +88,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -126,9 +128,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -144,21 +146,21 @@ cryptography==42.0.8 # types-redis cython==3.0.11 # via thriftpy2 -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -174,19 +176,21 @@ duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus execnet==2.1.1 # via pytest-xdist executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -204,7 +208,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -213,7 +217,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -235,9 +239,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -257,7 +261,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -268,6 +272,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -335,7 +340,7 @@ iniconfig==2.0.0 # via pytest ipykernel==6.29.5 # via jupyterlab -ipython==8.29.0 +ipython==8.30.0 # via # great-expectations # ipykernel @@ -363,7 +368,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -395,7 +400,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -408,7 +413,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -433,15 +438,19 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 - # via great-expectations +marshmallow==3.23.2 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 +milvus-lite==2.4.10 + # via pymilvus +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -471,7 +480,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -485,7 +494,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -539,6 +548,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -573,13 +583,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -605,6 +615,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -649,13 +660,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -664,19 +677,21 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -729,8 +744,10 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn -python-json-logger==2.0.7 + # via + # environs + # uvicorn +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -806,7 +823,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -816,7 +833,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -830,12 +847,12 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -850,7 +867,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -874,7 +891,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==25.20.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -920,6 +937,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -935,7 +953,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -949,7 +967,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -965,12 +983,13 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests typing-extensions==4.12.2 # via + # anyio # azure-core # azure-identity # azure-storage-blob @@ -979,6 +998,7 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # mypy # psycopg # psycopg-pool @@ -994,6 +1014,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==2.2.3 @@ -1009,7 +1031,7 @@ urllib3==2.2.3 # requests # responses # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1021,7 +1043,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1051,7 +1073,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 8f776fdc457..c536ef91ae3 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.11 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.11-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -25,17 +25,17 @@ cloudpickle==3.1.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2024.11.2 +dask[dataframe]==2024.12.1 # via # feast (setup.py) # dask-expr -dask-expr==1.1.19 +dask-expr==1.1.21 # via dask dill==0.3.9 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -83,25 +83,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -120,11 +120,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -148,6 +148,7 @@ typeguard==4.4.1 # via feast (setup.py) typing-extensions==4.12.2 # via + # anyio # fastapi # mypy # pydantic @@ -158,7 +159,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -166,7 +167,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 3deb441827c..da914388de2 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -1,14 +1,14 @@ # This file was autogenerated by uv via the following command: # uv pip compile -p 3.9 --system --no-strip-extras setup.py --extra ci --output-file sdk/python/requirements/py3.9-ci-requirements.txt -aiobotocore==2.15.2 +aiobotocore==2.16.0 # via feast (setup.py) -aiohappyeyeballs==2.4.3 +aiohappyeyeballs==2.4.4 # via aiohttp -aiohttp==3.11.7 +aiohttp==3.11.11 # via aiobotocore aioitertools==0.12.0 # via aiobotocore -aiosignal==1.3.1 +aiosignal==1.3.2 # via aiohttp alabaster==0.7.16 # via sphinx @@ -16,7 +16,7 @@ altair==4.2.2 # via great-expectations annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # httpx # jupyter-server @@ -25,7 +25,9 @@ anyio==4.6.2.post1 appnope==0.1.4 # via ipykernel argon2-cffi==23.1.0 - # via jupyter-server + # via + # jupyter-server + # minio argon2-cffi-bindings==21.2.0 # via argon2-cffi arrow==1.3.0 @@ -34,7 +36,7 @@ asn1crypto==1.5.1 # via snowflake-connector-python assertpy==1.1 # via feast (setup.py) -asttokens==2.4.1 +asttokens==3.0.0 # via stack-data async-lru==2.0.4 # via jupyterlab @@ -46,7 +48,7 @@ async-timeout==5.0.1 # redis atpublic==4.1.0 # via ibis-framework -attrs==24.2.0 +attrs==24.3.0 # via # aiohttp # jsonschema @@ -71,11 +73,11 @@ bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert -boto3==1.35.36 +boto3==1.35.81 # via # feast (setup.py) # moto -botocore==1.35.36 +botocore==1.35.81 # via # aiobotocore # boto3 @@ -90,7 +92,7 @@ cachetools==5.5.0 # via google-auth cassandra-driver==3.29.2 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via # elastic-transport # httpcore @@ -130,9 +132,9 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.8 +coverage[toml]==7.6.9 # via pytest-cov -cryptography==42.0.8 +cryptography==43.0.3 # via # feast (setup.py) # azure-identity @@ -156,13 +158,13 @@ dask-expr==1.1.10 # via dask db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.9 +debugpy==1.8.11 # via ipykernel decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.22.0 +deltalake==0.22.3 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -178,10 +180,12 @@ duckdb==0.10.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.16.0 +elasticsearch==8.17.0 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus exceptiongroup==1.2.2 # via # anyio @@ -193,9 +197,9 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0.post1 # via feast (setup.py) -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fastjsonschema==2.20.0 +fastjsonschema==2.21.1 # via nbformat filelock==3.16.1 # via @@ -213,7 +217,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.23.0 +google-api-core[grpc]==2.24.0 # via # feast (setup.py) # google-cloud-bigquery @@ -222,7 +226,7 @@ google-api-core[grpc]==2.23.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.36.0 +google-auth==2.37.0 # via # google-api-core # google-cloud-bigquery @@ -244,9 +248,9 @@ google-cloud-core==2.4.1 # google-cloud-bigtable # google-cloud-datastore # google-cloud-storage -google-cloud-datastore==2.20.1 +google-cloud-datastore==2.20.2 # via feast (setup.py) -google-cloud-storage==2.18.2 +google-cloud-storage==2.19.0 # via feast (setup.py) google-crc32c==1.6.0 # via @@ -266,7 +270,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.68.0 +grpcio==1.68.1 # via # feast (setup.py) # google-api-core @@ -277,6 +281,7 @@ grpcio==1.68.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -381,7 +386,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.28 +json5==0.10.0 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -413,7 +418,7 @@ jupyter-core==5.7.2 # nbclient # nbconvert # nbformat -jupyter-events==0.10.0 +jupyter-events==0.11.0 # via jupyter-server jupyter-lsp==2.2.5 # via jupyterlab @@ -426,7 +431,7 @@ jupyter-server==2.14.2 # notebook-shim jupyter-server-terminals==0.5.3 # via jupyter-server -jupyterlab==4.2.6 +jupyterlab==4.3.4 # via notebook jupyterlab-pygments==0.3.0 # via nbconvert @@ -451,15 +456,19 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.1 - # via great-expectations +marshmallow==3.23.2 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py -minio==7.1.0 +milvus-lite==2.4.10 + # via pymilvus +minio==7.2.11 # via feast (setup.py) mistune==3.0.2 # via @@ -489,7 +498,7 @@ mypy-extensions==1.0.0 # via mypy mypy-protobuf==3.3.0 # via feast (setup.py) -nbclient==0.10.0 +nbclient==0.10.2 # via nbconvert nbconvert==7.16.4 # via jupyter-server @@ -503,7 +512,7 @@ nest-asyncio==1.6.0 # via ipykernel nodeenv==1.9.1 # via pre-commit -notebook==7.2.2 +notebook==7.3.1 # via great-expectations notebook-shim==0.2.4 # via @@ -556,6 +565,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -590,13 +600,13 @@ portalocker==2.10.1 # qdrant-client pre-commit==3.3.1 # via feast (setup.py) -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via # feast (setup.py) # jupyter-server prompt-toolkit==3.0.48 # via ipython -propcache==0.2.0 +propcache==0.2.1 # via # aiohttp # yarl @@ -622,6 +632,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -666,13 +677,15 @@ pybindgen==0.22.1 # via feast (setup.py) pycparser==2.22 # via cffi -pydantic==2.10.1 +pycryptodome==3.21.0 + # via minio +pydantic==2.10.4 # via # feast (setup.py) # fastapi # great-expectations # qdrant-client -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via @@ -681,19 +694,21 @@ pygments==2.18.0 # nbconvert # rich # sphinx -pyjwt[crypto]==2.10.0 +pyjwt[crypto]==2.10.1 # via # feast (setup.py) # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.2 # via feast (setup.py) pymysql==1.1.1 # via feast (setup.py) pyodbc==5.2.0 # via feast (setup.py) -pyopenssl==24.2.1 +pyopenssl==24.3.0 # via snowflake-connector-python pyparsing==3.2.0 # via great-expectations @@ -746,8 +761,10 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn -python-json-logger==2.0.7 + # via + # environs + # uvicorn +python-json-logger==3.2.1 # via jupyter-events python-keycloak==4.2.2 # via feast (setup.py) @@ -823,7 +840,7 @@ rfc3986-validator==0.1.1 # jupyter-events rich==13.9.4 # via ibis-framework -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing @@ -833,7 +850,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.8.0 +ruff==0.8.4 # via feast (setup.py) s3transfer==0.10.4 # via boto3 @@ -847,12 +864,12 @@ setuptools==75.6.0 # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) -six==1.16.0 +six==1.17.0 # via - # asttokens # azure-core # geomet # happybase @@ -867,7 +884,7 @@ sniffio==1.3.1 # httpx snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python[pandas]==3.12.3 +snowflake-connector-python[pandas]==3.12.4 # via feast (setup.py) sortedcontainers==2.4.0 # via snowflake-connector-python @@ -891,7 +908,7 @@ sqlalchemy[mypy]==2.0.36 # via feast (setup.py) sqlglot==23.12.2 # via ibis-framework -sqlite-vec==0.1.1 +sqlite-vec==0.1.3 # via feast (setup.py) sqlparams==6.1.0 # via singlestoredb @@ -917,7 +934,7 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via # build # coverage @@ -947,6 +964,7 @@ tqdm==4.67.1 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -962,7 +980,7 @@ traitlets==5.14.3 # nbclient # nbconvert # nbformat -trino==0.330.0 +trino==0.331.0 # via feast (setup.py) typeguard==4.4.1 # via feast (setup.py) @@ -976,7 +994,7 @@ types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis -types-python-dateutil==2.9.0.20241003 +types-python-dateutil==2.9.0.20241206 # via # feast (setup.py) # arrow @@ -992,7 +1010,7 @@ types-setuptools==75.6.0.20241126 # via # feast (setup.py) # types-cffi -types-tabulate==0.9.0.20240106 +types-tabulate==0.9.0.20241207 # via feast (setup.py) types-urllib3==1.26.25.14 # via types-requests @@ -1009,12 +1027,14 @@ typing-extensions==4.12.2 # ibis-framework # ipython # jwcrypto + # minio # multidict # mypy # psycopg # psycopg-pool # pydantic # pydantic-core + # python-json-logger # rich # snowflake-connector-python # sqlalchemy @@ -1028,6 +1048,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==1.26.20 @@ -1044,7 +1066,7 @@ urllib3==1.26.20 # responses # snowflake-connector-python # testcontainers -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -1056,7 +1078,7 @@ virtualenv==20.23.0 # via # feast (setup.py) # pre-commit -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit @@ -1086,7 +1108,7 @@ wrapt==1.17.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.18.0 +yarl==1.18.3 # via aiohttp zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 8c9fc036433..80f1e499e1f 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -2,17 +2,17 @@ # uv pip compile -p 3.9 --system --no-strip-extras setup.py --output-file sdk/python/requirements/py3.9-requirements.txt annotated-types==0.7.0 # via pydantic -anyio==4.6.2.post1 +anyio==4.7.0 # via # starlette # watchfiles -attrs==24.2.0 +attrs==24.3.0 # via # jsonschema # referencing bigtree==0.22.3 # via feast (setup.py) -certifi==2024.8.30 +certifi==2024.12.14 # via requests charset-normalizer==3.4.0 # via requests @@ -35,9 +35,9 @@ dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.5 +fastapi==0.115.6 # via feast (setup.py) -fsspec==2024.10.0 +fsspec==2024.12.0 # via dask gunicorn==23.0.0 # via @@ -87,25 +87,25 @@ pandas==2.2.3 # dask-expr partd==1.4.2 # via dask -prometheus-client==0.21.0 +prometheus-client==0.21.1 # via feast (setup.py) protobuf==4.25.5 # via feast (setup.py) -psutil==6.1.0 +psutil==6.1.1 # via feast (setup.py) pyarrow==18.0.0 # via # feast (setup.py) # dask-expr -pydantic==2.10.1 +pydantic==2.10.4 # via # feast (setup.py) # fastapi -pydantic-core==2.27.1 +pydantic-core==2.27.2 # via pydantic pygments==2.18.0 # via feast (setup.py) -pyjwt==2.10.0 +pyjwt==2.10.1 # via feast (setup.py) python-dateutil==2.9.0.post0 # via pandas @@ -124,11 +124,11 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.21.0 +rpds-py==0.22.3 # via # jsonschema # referencing -six==1.16.0 +six==1.17.0 # via python-dateutil sniffio==1.3.1 # via anyio @@ -142,7 +142,7 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.1.0 +tomli==2.2.1 # via mypy toolz==1.0.0 # via @@ -167,7 +167,7 @@ tzdata==2024.2 # via pandas urllib3==2.2.3 # via requests -uvicorn[standard]==0.32.1 +uvicorn[standard]==0.34.0 # via # feast (setup.py) # uvicorn-worker @@ -175,7 +175,7 @@ uvicorn-worker==0.2.0 # via feast (setup.py) uvloop==0.21.0 # via uvicorn -watchfiles==1.0.0 +watchfiles==1.0.3 # via uvicorn websockets==14.1 # via uvicorn diff --git a/sdk/python/tests/conftest.py b/sdk/python/tests/conftest.py index 24c8f40f742..6e5f1e14870 100644 --- a/sdk/python/tests/conftest.py +++ b/sdk/python/tests/conftest.py @@ -57,8 +57,12 @@ location, ) from tests.utils.auth_permissions_util import default_store -from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert from tests.utils.http_server import check_port_open, free_port # noqa: E402 +from tests.utils.ssl_certifcates_util import ( + combine_trust_stores, + create_ca_trust_store, + generate_self_signed_cert, +) logger = logging.getLogger(__name__) @@ -514,17 +518,36 @@ def auth_config(request, is_integration_test): return auth_configuration -@pytest.fixture(params=[True, False], scope="module") +@pytest.fixture(scope="module") def tls_mode(request): - is_tls_mode = request.param + is_tls_mode = request.param[0] + output_combined_truststore_path = "" if is_tls_mode: certificates_path = tempfile.mkdtemp() tls_key_path = os.path.join(certificates_path, "key.pem") tls_cert_path = os.path.join(certificates_path, "cert.pem") + generate_self_signed_cert(cert_path=tls_cert_path, key_path=tls_key_path) + is_ca_trust_store_set = request.param[1] + if is_ca_trust_store_set: + # Paths + feast_ca_trust_store_path = os.path.join( + certificates_path, "feast_trust_store.pem" + ) + create_ca_trust_store( + public_key_path=tls_cert_path, + private_key_path=tls_key_path, + output_trust_store_path=feast_ca_trust_store_path, + ) + + # Combine trust stores + output_combined_path = os.path.join( + certificates_path, "combined_trust_store.pem" + ) + combine_trust_stores(feast_ca_trust_store_path, output_combined_path) else: tls_key_path = "" tls_cert_path = "" - return is_tls_mode, tls_key_path, tls_cert_path + return is_tls_mode, tls_key_path, tls_cert_path, output_combined_truststore_path diff --git a/sdk/python/tests/data/data_creator.py b/sdk/python/tests/data/data_creator.py index 6b0984f799d..dfe94913e97 100644 --- a/sdk/python/tests/data/data_creator.py +++ b/sdk/python/tests/data/data_creator.py @@ -84,6 +84,8 @@ def get_feature_values_for_dtype( def create_document_dataset() -> pd.DataFrame: data = { "item_id": [1, 2, 3], + "string_feature": ["a", "b", "c"], + "float_feature": [1.0, 2.0, 3.0], "embedding_float": [[4.0, 5.0], [1.0, 2.0], [3.0, 4.0]], "embedding_double": [[4.0, 5.0], [1.0, 2.0], [3.0, 4.0]], "ts": [ diff --git a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py index dc716f45e1e..1d33402e012 100644 --- a/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py +++ b/sdk/python/tests/integration/feature_repos/universal/data_sources/file.py @@ -34,8 +34,8 @@ DataSourceCreator, ) from tests.utils.auth_permissions_util import include_auth_config -from tests.utils.generate_self_signed_certifcate_util import generate_self_signed_cert from tests.utils.http_server import check_port_open, free_port # noqa: E402 +from tests.utils.ssl_certifcates_util import generate_self_signed_cert logger = logging.getLogger(__name__) @@ -452,9 +452,6 @@ def setup(self, registry: RegistryConfig): str(tls_key_path), "--cert", str(self.tls_cert_path), - # This is needed for the self-signed certificate, disabled verify_client for integration tests. - "--verify_client", - str(False), ] self.proc = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL diff --git a/sdk/python/tests/integration/feature_repos/universal/feature_views.py b/sdk/python/tests/integration/feature_repos/universal/feature_views.py index 11ddcb0ecc6..47e5746e61a 100644 --- a/sdk/python/tests/integration/feature_repos/universal/feature_views.py +++ b/sdk/python/tests/integration/feature_repos/universal/feature_views.py @@ -17,7 +17,7 @@ from feast.data_source import DataSource, RequestSource from feast.feature_view_projection import FeatureViewProjection from feast.on_demand_feature_view import PandasTransformation, SubstraitTransformation -from feast.types import Array, FeastType, Float32, Float64, Int32, Int64 +from feast.types import Array, FeastType, Float32, Float64, Int32, Int64, String from tests.integration.feature_repos.universal.entities import ( customer, driver, @@ -160,8 +160,20 @@ def create_item_embeddings_feature_view(source, infer_features: bool = False): schema=None if infer_features else [ - Field(name="embedding_double", dtype=Array(Float64)), - Field(name="embedding_float", dtype=Array(Float32)), + Field( + name="embedding_double", + dtype=Array(Float64), + vector_index=True, + vector_search_metric="L2", + ), + Field( + name="embedding_float", + dtype=Array(Float32), + vector_index=True, + vector_search_metric="L2", + ), + Field(name="string_feature", dtype=String), + Field(name="float_feature", dtype=Float32), ], source=source, ttl=timedelta(hours=2), diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py new file mode 100644 index 00000000000..8ffee04c12f --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -0,0 +1,35 @@ +from typing import Any, Dict + +from testcontainers.milvus import MilvusContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class MilvusOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.fixed_port = 19530 + self.container = MilvusContainer("milvusdb/milvus:v2.4.4").with_exposed_ports( + self.fixed_port + ) + + def create_online_store(self) -> Dict[str, Any]: + self.container.start() + # Wait for Milvus server to be ready + host = "localhost" + port = self.container.get_exposed_port(self.fixed_port) + return { + "type": "milvus", + "host": host, + "port": int(port), + "index_type": "IVF_FLAT", + "metric_type": "L2", + "embedding_dim": 2, + "vector_enabled": True, + "nlist": 1, + } + + def teardown(self): + self.container.stop() diff --git a/sdk/python/tests/integration/offline_store/test_feature_logging.py b/sdk/python/tests/integration/offline_store/test_feature_logging.py index 32f506f90b2..53147d242ef 100644 --- a/sdk/python/tests/integration/offline_store/test_feature_logging.py +++ b/sdk/python/tests/integration/offline_store/test_feature_logging.py @@ -106,7 +106,15 @@ def retrieve(): ) persisted_logs = persisted_logs[expected_columns] + logs_df = logs_df[expected_columns] + + # Convert timezone-aware datetime values to naive datetime values + logs_df[LOG_TIMESTAMP_FIELD] = logs_df[LOG_TIMESTAMP_FIELD].dt.tz_localize(None) + persisted_logs[LOG_TIMESTAMP_FIELD] = persisted_logs[ + LOG_TIMESTAMP_FIELD + ].dt.tz_localize(None) + pd.testing.assert_frame_equal( logs_df.sort_values(REQUEST_ID_FIELD).reset_index(drop=True), persisted_logs.sort_values(REQUEST_ID_FIELD).reset_index(drop=True), diff --git a/sdk/python/tests/integration/online_store/test_remote_online_store.py b/sdk/python/tests/integration/online_store/test_remote_online_store.py index 10f1180d8e6..285253dfaaf 100644 --- a/sdk/python/tests/integration/online_store/test_remote_online_store.py +++ b/sdk/python/tests/integration/online_store/test_remote_online_store.py @@ -22,6 +22,9 @@ @pytest.mark.integration +@pytest.mark.parametrize( + "tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True +) def test_remote_online_store_read(auth_config, tls_mode): with ( tempfile.TemporaryDirectory() as remote_server_tmp_dir, @@ -56,13 +59,13 @@ def test_remote_online_store_read(auth_config, tls_mode): ) ) assert None not in (server_store, server_url, registry_path) - _, _, tls_cert_path = tls_mode + client_store = _create_remote_client_feature_store( temp_dir=remote_client_tmp_dir, server_registry_path=str(registry_path), feature_server_url=server_url, auth_config=auth_config, - tls_cert_path=tls_cert_path, + tls_mode=tls_mode, ) assert client_store is not None _assert_non_existing_entity_feature_views_entity( @@ -172,7 +175,7 @@ def _create_server_store_spin_feature_server( ): store = default_store(str(temp_dir), auth_config, permissions_list) feast_server_port = free_port() - is_tls_mode, tls_key_path, tls_cert_path = tls_mode + is_tls_mode, tls_key_path, tls_cert_path, ca_trust_store_path = tls_mode server_url = next( start_feature_server( @@ -180,6 +183,7 @@ def _create_server_store_spin_feature_server( server_port=feast_server_port, tls_key_path=tls_key_path, tls_cert_path=tls_cert_path, + ca_trust_store_path=ca_trust_store_path, ) ) if is_tls_mode: @@ -203,20 +207,33 @@ def _create_remote_client_feature_store( server_registry_path: str, feature_server_url: str, auth_config: str, - tls_cert_path: str = "", + tls_mode, ) -> FeatureStore: project_name = "REMOTE_ONLINE_CLIENT_PROJECT" runner = CliRunner() result = runner.run(["init", project_name], cwd=temp_dir) assert result.returncode == 0 repo_path = os.path.join(temp_dir, project_name, "feature_repo") - _overwrite_remote_client_feature_store_yaml( - repo_path=str(repo_path), - registry_path=server_registry_path, - feature_server_url=feature_server_url, - auth_config=auth_config, - tls_cert_path=tls_cert_path, - ) + is_tls_mode, _, tls_cert_path, ca_trust_store_path = tls_mode + if is_tls_mode and not ca_trust_store_path: + _overwrite_remote_client_feature_store_yaml( + repo_path=str(repo_path), + registry_path=server_registry_path, + feature_server_url=feature_server_url, + auth_config=auth_config, + tls_cert_path=tls_cert_path, + ) + else: + _overwrite_remote_client_feature_store_yaml( + repo_path=str(repo_path), + registry_path=server_registry_path, + feature_server_url=feature_server_url, + auth_config=auth_config, + ) + + if is_tls_mode and ca_trust_store_path: + # configure trust store path only when is_tls_mode and ca_trust_store_path exists. + os.environ["FEAST_CA_CERT_FILE_PATH"] = ca_trust_store_path return FeatureStore(repo_path=repo_path) diff --git a/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py b/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py index 25c5fe3eb8c..0395f995410 100644 --- a/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py +++ b/sdk/python/tests/unit/permissions/auth/server/test_auth_registry_server.py @@ -44,7 +44,7 @@ def start_registry_server( assertpy.assert_that(server_port).is_not_equal_to(0) - is_tls_mode, tls_key_path, tls_cert_path = tls_mode + is_tls_mode, tls_key_path, tls_cert_path, tls_ca_file_path = tls_mode if is_tls_mode: print(f"Starting Registry in TLS mode at {server_port}") server = start_server( @@ -74,6 +74,9 @@ def start_registry_server( server.stop(grace=None) # Teardown server +@pytest.mark.parametrize( + "tls_mode", [("True", "True"), ("True", "False"), ("False", "")], indirect=True +) def test_registry_apis( auth_config, tls_mode, diff --git a/sdk/python/tests/utils/auth_permissions_util.py b/sdk/python/tests/utils/auth_permissions_util.py index 6f0a3c8eeac..8a1e7b7c4d7 100644 --- a/sdk/python/tests/utils/auth_permissions_util.py +++ b/sdk/python/tests/utils/auth_permissions_util.py @@ -60,6 +60,7 @@ def start_feature_server( metrics: bool = False, tls_key_path: str = "", tls_cert_path: str = "", + ca_trust_store_path: str = "", ): host = "0.0.0.0" cmd = [ @@ -127,18 +128,30 @@ def start_feature_server( def get_remote_registry_store(server_port, feature_store, tls_mode): - is_tls_mode, _, tls_cert_path = tls_mode + is_tls_mode, _, tls_cert_path, ca_trust_store_path = tls_mode if is_tls_mode: - registry_config = RemoteRegistryConfig( - registry_type="remote", - path=f"localhost:{server_port}", - cert=tls_cert_path, - ) + if ca_trust_store_path: + registry_config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{server_port}", + is_tls=True, + ) + else: + registry_config = RemoteRegistryConfig( + registry_type="remote", + path=f"localhost:{server_port}", + is_tls=True, + cert=tls_cert_path, + ) else: registry_config = RemoteRegistryConfig( registry_type="remote", path=f"localhost:{server_port}" ) + if is_tls_mode and ca_trust_store_path: + # configure trust store path only when is_tls_mode and ca_trust_store_path exists. + os.environ["FEAST_CA_CERT_FILE_PATH"] = ca_trust_store_path + store = FeatureStore( config=RepoConfig( project=PROJECT_NAME, diff --git a/sdk/python/tests/utils/generate_self_signed_certifcate_util.py b/sdk/python/tests/utils/generate_self_signed_certifcate_util.py deleted file mode 100644 index 559ee18cde7..00000000000 --- a/sdk/python/tests/utils/generate_self_signed_certifcate_util.py +++ /dev/null @@ -1,79 +0,0 @@ -import ipaddress -import logging -from datetime import datetime, timedelta - -from cryptography import x509 -from cryptography.hazmat.backends import default_backend -from cryptography.hazmat.primitives import hashes, serialization -from cryptography.hazmat.primitives.asymmetric import rsa -from cryptography.x509.oid import NameOID - -logger = logging.getLogger(__name__) - - -def generate_self_signed_cert( - cert_path="cert.pem", key_path="key.pem", common_name="localhost" -): - """ - Generate a self-signed certificate and save it to the specified paths. - - :param cert_path: Path to save the certificate (PEM format) - :param key_path: Path to save the private key (PEM format) - :param common_name: Common name (CN) for the certificate, defaults to 'localhost' - """ - # Generate private key - key = rsa.generate_private_key( - public_exponent=65537, key_size=2048, backend=default_backend() - ) - - # Create a self-signed certificate - subject = issuer = x509.Name( - [ - x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), - x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "California"), - x509.NameAttribute(NameOID.LOCALITY_NAME, "San Francisco"), - x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Feast"), - x509.NameAttribute(NameOID.COMMON_NAME, common_name), - ] - ) - - # Define the certificate's Subject Alternative Names (SANs) - alt_names = [ - x509.DNSName("localhost"), # Hostname - x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")), # Localhost IP - x509.IPAddress(ipaddress.IPv4Address("0.0.0.0")), # Bind-all IP (optional) - ] - san = x509.SubjectAlternativeName(alt_names) - - certificate = ( - x509.CertificateBuilder() - .subject_name(subject) - .issuer_name(issuer) - .public_key(key.public_key()) - .serial_number(x509.random_serial_number()) - .not_valid_before(datetime.utcnow()) - .not_valid_after( - # Certificate valid for 1 year - datetime.utcnow() + timedelta(days=365) - ) - .add_extension(san, critical=False) - .sign(key, hashes.SHA256(), default_backend()) - ) - - # Write the private key to a file - with open(key_path, "wb") as f: - f.write( - key.private_bytes( - encoding=serialization.Encoding.PEM, - format=serialization.PrivateFormat.TraditionalOpenSSL, - encryption_algorithm=serialization.NoEncryption(), - ) - ) - - # Write the certificate to a file - with open(cert_path, "wb") as f: - f.write(certificate.public_bytes(serialization.Encoding.PEM)) - - logger.info( - f"Self-signed certificate and private key have been generated at {cert_path} and {key_path}." - ) diff --git a/sdk/python/tests/utils/ssl_certifcates_util.py b/sdk/python/tests/utils/ssl_certifcates_util.py new file mode 100644 index 00000000000..53a56e04f3d --- /dev/null +++ b/sdk/python/tests/utils/ssl_certifcates_util.py @@ -0,0 +1,174 @@ +import ipaddress +import logging +import os +import shutil +from datetime import datetime, timedelta + +import certifi +from cryptography import x509 +from cryptography.hazmat.backends import default_backend +from cryptography.hazmat.primitives import hashes, serialization +from cryptography.hazmat.primitives.asymmetric import rsa +from cryptography.x509 import load_pem_x509_certificate +from cryptography.x509.oid import NameOID + +logger = logging.getLogger(__name__) + + +def generate_self_signed_cert( + cert_path="cert.pem", key_path="key.pem", common_name="localhost" +): + """ + Generate a self-signed certificate and save it to the specified paths. + + :param cert_path: Path to save the certificate (PEM format) + :param key_path: Path to save the private key (PEM format) + :param common_name: Common name (CN) for the certificate, defaults to 'localhost' + """ + # Generate private key + key = rsa.generate_private_key( + public_exponent=65537, key_size=2048, backend=default_backend() + ) + + # Create a self-signed certificate + subject = issuer = x509.Name( + [ + x509.NameAttribute(NameOID.COUNTRY_NAME, "US"), + x509.NameAttribute(NameOID.STATE_OR_PROVINCE_NAME, "California"), + x509.NameAttribute(NameOID.LOCALITY_NAME, "San Francisco"), + x509.NameAttribute(NameOID.ORGANIZATION_NAME, "Feast"), + x509.NameAttribute(NameOID.COMMON_NAME, common_name), + ] + ) + + # Define the certificate's Subject Alternative Names (SANs) + alt_names = [ + x509.DNSName("localhost"), # Hostname + x509.IPAddress(ipaddress.IPv4Address("127.0.0.1")), # Localhost IP + x509.IPAddress(ipaddress.IPv4Address("0.0.0.0")), # Bind-all IP (optional) + ] + san = x509.SubjectAlternativeName(alt_names) + + certificate = ( + x509.CertificateBuilder() + .subject_name(subject) + .issuer_name(issuer) + .public_key(key.public_key()) + .serial_number(x509.random_serial_number()) + .not_valid_before(datetime.utcnow()) + .not_valid_after( + # Certificate valid for 1 year + datetime.utcnow() + timedelta(days=365) + ) + .add_extension(san, critical=False) + .sign(key, hashes.SHA256(), default_backend()) + ) + + # Write the private key to a file + with open(key_path, "wb") as f: + f.write( + key.private_bytes( + encoding=serialization.Encoding.PEM, + format=serialization.PrivateFormat.TraditionalOpenSSL, + encryption_algorithm=serialization.NoEncryption(), + ) + ) + + # Write the certificate to a file + with open(cert_path, "wb") as f: + f.write(certificate.public_bytes(serialization.Encoding.PEM)) + + logger.info( + f"Self-signed certificate and private key have been generated at {cert_path} and {key_path}." + ) + + +def create_ca_trust_store( + public_key_path: str, private_key_path: str, output_trust_store_path: str +): + """ + Create a new CA trust store as a copy of the existing one (if available), + and add the provided public certificate to it. + + :param public_key_path: Path to the public certificate (e.g., PEM file). + :param private_key_path: Path to the private key (optional, to verify signing authority). + :param output_trust_store_path: Path to save the new trust store. + """ + try: + # Step 1: Identify the existing trust store (if available via environment variables) + existing_trust_store = os.environ.get("SSL_CERT_FILE") or os.environ.get( + "REQUESTS_CA_BUNDLE" + ) + + # Step 2: Copy the existing trust store to the new location (if it exists) + if existing_trust_store and os.path.exists(existing_trust_store): + shutil.copy(existing_trust_store, output_trust_store_path) + logger.info( + f"Copied existing trust store from {existing_trust_store} to {output_trust_store_path}" + ) + else: + # Log the creation of a new trust store (without opening a file unnecessarily) + logger.info( + f"No existing trust store found. Creating a new trust store at {output_trust_store_path}" + ) + + # Step 3: Load and validate the public certificate + with open(public_key_path, "rb") as pub_file: + public_cert_data = pub_file.read() + public_cert = load_pem_x509_certificate( + public_cert_data, backend=default_backend() + ) + + # Verify the private key matches (optional, adds validation) + if private_key_path: + with open(private_key_path, "rb") as priv_file: + private_key_data = priv_file.read() + private_key = serialization.load_pem_private_key( + private_key_data, password=None, backend=default_backend() + ) + # Check the public/private key match + if ( + private_key.public_key().public_numbers() + != public_cert.public_key().public_numbers() + ): + raise ValueError( + "Public certificate does not match the private key." + ) + + # Step 4: Add the public certificate to the new trust store + with open(output_trust_store_path, "ab") as trust_store_file: + trust_store_file.write(public_cert.public_bytes(serialization.Encoding.PEM)) + + logger.info( + f"Trust store created/updated successfully at: {output_trust_store_path}" + ) + + except Exception as e: + logger.error(f"Error creating CA trust store: {e}") + + +def combine_trust_stores(custom_cert_path: str, output_combined_path: str): + """ + Combine the default certifi CA bundle with a custom certificate file. + + :param custom_cert_path: Path to the custom certificate PEM file. + :param output_combined_path: Path where the combined CA bundle will be saved. + """ + try: + # Get the default certifi CA bundle + certifi_ca_bundle = certifi.where() + + with open(output_combined_path, "wb") as combined_file: + # Write the default CA bundle + with open(certifi_ca_bundle, "rb") as default_file: + combined_file.write(default_file.read()) + + # Append the custom certificates + with open(custom_cert_path, "rb") as custom_file: + combined_file.write(custom_file.read()) + + logger.info(f"Combined trust store created at: {output_combined_path}") + + except Exception as e: + logger.error(f"Error combining trust stores: {e}") + raise e diff --git a/setup.py b/setup.py index 815d1b23229..59b881c9715 100644 --- a/setup.py +++ b/setup.py @@ -156,18 +156,20 @@ GO_REQUIRED = ["cffi~=1.15.0"] +MILVUS_REQUIRED = ["pymilvus"] + CI_REQUIRED = ( [ "build", "virtualenv==20.23.0", - "cryptography>=35.0,<43", + "cryptography>=43.0,<44", "ruff>=0.8.0", "mypy-protobuf>=3.1", "grpcio-tools>=1.56.2,<2", "grpcio-testing>=1.56.2,<2", # FastAPI does not correctly pull starlette dependency on httpx see thread(https://github.com/tiangolo/fastapi/issues/5656). - "httpx>=0.23.3", - "minio==7.1.0", + "httpx==0.27.2", + "minio==7.2.11", "mock==2.0.0", "moto<5", "mypy>=1.4.1,<1.11.3", @@ -226,6 +228,7 @@ + OPENTELEMETRY + FAISS_REQUIRED + QDRANT_REQUIRED + + MILVUS_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -355,6 +358,7 @@ def run(self): "faiss": FAISS_REQUIRED, "qdrant": QDRANT_REQUIRED, "go": GO_REQUIRED, + "milvus": MILVUS_REQUIRED, }, include_package_data=True, license="Apache",