Feat: Add classes and objects to enable PANGEA support

google/cloud/bigquery/dataset.py

@@ -27,6 +27,8 @@
 from google.cloud.bigquery.routine import Routine, RoutineReference
 from google.cloud.bigquery.table import Table, TableReference
 from google.cloud.bigquery.encryption_configuration import EncryptionConfiguration
+from google.cloud.bigquery.external_config import ExternalCatalogDatasetOptions
+
 
 from typing import Optional, List, Dict, Any, Union
 
@@ -530,6 +532,7 @@ class Dataset(object):
         "storage_billing_model": "storageBillingModel",
         "max_time_travel_hours": "maxTimeTravelHours",
         "default_rounding_mode": "defaultRoundingMode",
+        "external_catalog_dataset_options": "externalCatalogDatasetOptions",
     }
 
     def __init__(self, dataset_ref) -> None:
@@ -937,6 +940,24 @@ def _build_resource(self, filter_fields):
         """Generate a resource for ``update``."""
         return _helpers._build_resource_from_properties(self, filter_fields)
 
+    @property
+    def external_catalog_dataset_options(self):
+        """Options defining open source compatible datasets living in the
+        BigQuery catalog. Contains metadata of open source database, schema
+        or namespace represented by the current dataset."""
+
+        return self._properties.get("externalCatalogDatasetOptions")
+
+    @external_catalog_dataset_options.setter
+    def external_catalog_dataset_options(self, value):
+        if not isinstance(value, ExternalCatalogDatasetOptions) and value is not None:
+            raise ValueError(
+                "external_catalog_dataset_options must be an "
+                "ExternalCatalogDatasetOptions object or None. "
+                f"Got {repr(value)}."
+            )
+        self._properties["externalCatalogDatasetOptions"] = value
+
     table = _get_table_reference
 
     model = _get_model_reference

google/cloud/bigquery/external_config.py

@@ -1003,3 +1003,95 @@ def from_api_repr(cls, resource: dict) -> "ExternalConfig":
         config = cls(resource["sourceFormat"])
         config._properties = copy.deepcopy(resource)
         return config
+
+
+class ExternalCatalogDatasetOptions(object):
+    """Options defining open source compatible datasets living in the BigQuery catalog.
+    Contains metadata of open source database, schema or namespace represented
+    by the current dataset.
+
+    Args:
+        defaultStorageLocationUri: Optional. The storage location URI for all
+            tables in the dataset. Equivalent to hive metastore's database
+            locationUri. Maximum length of 1024 characters. (str)
+        parameters: Optional. A map of key value pairs defining the parameters
+            and properties of the open source schema. Maximum size of 2Mib.
+    """
+
+    def __init__(self, defaultStorageLocationUri: Optional[str] = None, parameters: Optional[dict] = None):
+        self._properties = {}
+        if not isinstance(defaultStorageLocationUri, (str, None)):
+            raise ValueError(
+                "Pass defaultStorageLocationUri as a 'str'."
+                f"Got {repr(dtype)}."
+            )
+        if not isinstance(parameters, (dict, None)):
+            raise ValueError(
+                "Pass parameters as a ''."
+                f"Got {repr(dtype)}."
+            )
+        self._properties["defaultStorageLocationUri"] = defaultStorageLocationUri
+        self._properties["parameters"] = parameters
+
+    def to_api_repr(self) -> dict:
+        """Build an API representation of this object.
+
+        Returns:
+            Dict[str, Any]:
+                A dictionary in the format used by the BigQuery API.
+        """
+        config = copy.deepcopy(self._properties)
+        return config
+
+class ExternalCatalogTableOptions(object):
+    """Metadata about open source compatible table. The fields contained in these
+    options correspond to hive metastore's table level properties.
+
+    Args:
+        connectionId: Optional. The connection specifying the credentials to be
+            used to read external storage, such as Azure Blob, Cloud Storage, or
+            S3. The connection is needed to read the open source table from
+            BigQuery Engine. The connection_id can have the form `..` or
+            `projects//locations//connections/`. (str)
+        parameters: Optional. A map of key value pairs defining the parameters
+            and properties of the open source table. Corresponds with hive meta
+            store table parameters. Maximum size of 4Mib. (dict)
+        storageDescriptor: Optional. A storage descriptor containing information
+            about the physical storage of this table. (StorageDescriptor)
+    """
+
+    def __init__(
+            self, 
+            connectionId: Optional[str] = None,
+            parameters: Optional[dict] = None,
+            storageDescriptor: Optional[str] = None # TODO implement StorageDescriptor, correct this type hint 
+        ):
+        self._properties = {}
+        if not isinstance(connectionId, str):
+            raise ValueError(
+                "Pass connectionId as a 'str'."
+                f"Got {repr(dtype)}."
+            )
+        if not isinstance(parameters, dict):
+            raise ValueError(
+                "Pass parameters as a 'dict'."
+                f"Got {repr(dtype)}."
+            )
+        if not isinstance(storageDescriptor, str): # TODO implement StorageDescriptor, correct this type hint 
+            raise ValueError(
+                "Pass storageDescriptor as a 'StorageDescriptor'."
+                f"Got {repr(dtype)}."
+            )
+        self._properties["connectionId"] = connectionId
+        self._properties["parameters"] = parameters
+        self._properties["storageDescriptor"] = storageDescriptor        
+
+    def to_api_repr(self) -> dict:
+        """Build an API representation of this object.
+
+        Returns:
+            Dict[str, Any]:
+                A dictionary in the format used by the BigQuery API.
+        """
+        config = copy.deepcopy(self._properties)
+        return config

google/cloud/bigquery/table.py

@@ -69,6 +69,7 @@
 from google.cloud.bigquery.schema import _build_schema_resource
 from google.cloud.bigquery.schema import _parse_schema_resource
 from google.cloud.bigquery.schema import _to_schema_fields
+from google.cloud.bigquery.external_config import ExternalCatalogTableOptions
 
 if typing.TYPE_CHECKING:  # pragma: NO COVER
     # Unconditionally import optional dependencies again to tell pytype that
@@ -999,6 +1000,24 @@ def table_constraints(self) -> Optional["TableConstraints"]:
             table_constraints = TableConstraints.from_api_repr(table_constraints)
         return table_constraints
 
+    @property
+    def external_catalog_table_options(self):
+        """Options defining open source compatible datasets living in the
+        BigQuery catalog. Contains metadata of open source database, schema
+        or namespace represented by the current dataset."""
+
+        return self._properties.get("externalCatalogTableOptions")
+
+    @external_catalog_table_options.setter
+    def external_catalog_table_options(self, value):
+        if not isinstance(value, ExternalCatalogTableOptions) and value is not None:
+            raise ValueError(
+                "external_catalog_table_options must be an "
+                "ExternalCatalogTableOptions object or None. "
+                f"Got {repr(value)}."
+            )
+        self._properties["externalCatalogTableOptions"] = value
+
     @classmethod
     def from_string(cls, full_table_id: str) -> "Table":
         """Construct a table from fully-qualified table ID.
@@ -3308,3 +3327,43 @@ def _table_arg_to_table(value, default_project=None) -> Table:
         value = newvalue
 
     return value
+
+
+class ExternalCatalogTableOptions
+    """Metadata about open source compatible table. The fields contained in these options correspond to hive metastore's table level properties.
+    Args:
+        connectionId: Optional. The connection specifying the credentials to be
+            used to read external storage, such as Azure Blob, Cloud Storage, or
+            S3. The connection is needed to read the open source table from
+            BigQuery Engine. The connection_id can have the form `..` or
+            `projects//locations//connections/`. (str)
+        parameters: Optional. A map of key value pairs defining the parameters
+            and properties of the open source table. Corresponds with hive meta
+            store table parameters. Maximum size of 4Mib.
+        storageDescriptor: Optional. A storage descriptor containing information
+            about the physical storage of this table.
+    """
+
+    def __init__(self, connectionId=None, parameters=None, storageDescriptor=None):
+        self._properties = {}
+        self.connectionId = connectionId
+        self.parameters = parameters
+        self.storageDescriptor = storageDescriptor
+
+        if not isinstance(connectionId, str):
+            raise ValueError(
+                connectionId must be a str.
+                Got 'connectionId'.
+            )
+
+        if not isinstance(parameters, ):
+            raise ValueError(
+                parameters must be a .
+                Got 'parameters'.
+            )
+
+        if not isinstance(storageDescriptor, ):
+            raise ValueError(
+                storageDescriptor must be a .
+                Got 'storageDescriptor'.
+            )