diff --git a/.gitignore b/.gitignore index 83ac8a7..14b7c8a 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +### Python template # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] @@ -20,7 +21,6 @@ parts/ sdist/ var/ wheels/ -pip-wheel-metadata/ share/python-wheels/ *.egg-info/ .installed.cfg @@ -50,7 +50,7 @@ coverage.xml *.py,cover .hypothesis/ .pytest_cache/ -test.env +cover/ # Translations *.mo @@ -73,6 +73,7 @@ instance/ docs/_build/ # PyBuilder +.pybuilder/ target/ # Jupyter Notebook @@ -83,7 +84,9 @@ profile_default/ ipython_config.py # pyenv -.python-version +# For a library or package, you might want to ignore these files since the code is +# intended to run in multiple environments; otherwise, check them in: +# .python-version # pipenv # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. @@ -128,3 +131,9 @@ dmypy.json # Pyre type checker .pyre/ + +# pytype static type analyzer +.pytype/ + +# Cython debug symbols +cython_debug/ diff --git a/.idea/.gitignore b/.idea/.gitignore new file mode 100644 index 0000000..26d3352 --- /dev/null +++ b/.idea/.gitignore @@ -0,0 +1,3 @@ +# Default ignored files +/shelf/ +/workspace.xml diff --git a/.idea/dbnavigator.xml b/.idea/dbnavigator.xml new file mode 100644 index 0000000..0f64700 --- /dev/null +++ b/.idea/dbnavigator.xml @@ -0,0 +1,413 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/dbt-doris.iml b/.idea/dbt-doris.iml new file mode 100644 index 0000000..a3166ab --- /dev/null +++ b/.idea/dbt-doris.iml @@ -0,0 +1,13 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/.idea/inspectionProfiles/profiles_settings.xml b/.idea/inspectionProfiles/profiles_settings.xml new file mode 100644 index 0000000..105ce2d --- /dev/null +++ b/.idea/inspectionProfiles/profiles_settings.xml @@ -0,0 +1,6 @@ + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml new file mode 100644 index 0000000..d235ddd --- /dev/null +++ b/.idea/misc.xml @@ -0,0 +1,4 @@ + + + + \ No newline at end of file diff --git a/.idea/modules.xml b/.idea/modules.xml new file mode 100644 index 0000000..d735fe2 --- /dev/null +++ b/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/.idea/vcs.xml b/.idea/vcs.xml new file mode 100644 index 0000000..94a25f7 --- /dev/null +++ b/.idea/vcs.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 750d525..594650a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,3 +1,4 @@ + # For more on configuring pre-commit hooks (see https://pre-commit.com/) # TODO: remove global exclusion of tests when testing overhaul is complete @@ -52,4 +53,3 @@ repos: args: [--show-error-codes, --pretty, --ignore-missing-imports] files: ^dbt/adapters language: system - diff --git a/README.md b/README.md index 32690d4..e701f32 100644 --- a/README.md +++ b/README.md @@ -6,8 +6,8 @@ dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis. -## SelectDB -This repo contains the base code to help you start to build out your dbt-selectdb adapter plugin, for more information on how to build out the adapter please follow the [docs](https://docs.getdbt.com/docs/contributing/building-a-new-adapter) +## Apache Doris +This repo contains the base code to help you start to build out your dbt-doris adapter plugin, for more information on how to build out the adapter please follow the [docs](https://docs.getdbt.com/docs/contributing/building-a-new-adapter) ** Note ** this `README` is meant to be replaced with what information would be required to use your adpater once your at a point todo so. @@ -15,8 +15,6 @@ This repo contains the base code to help you start to build out your dbt-selectd ### Adapter Scaffold default Versioning This adapter plugin follows [semantic versioning](https://semver.org/). The first version of this plugin is v0.1.0, in order to be compatible with dbt Core v1.3.0. -It's also brand new! For selectdb-specific functionality, we will aim for backwards-compatibility wherever possible. We are likely to be iterating more quickly than most major-version-1 software projects. To that end, backwards-incompatible changes will be clearly communicated and limited to minor versions (once every three months). - ## Getting Started #### Setting up Locally @@ -30,7 +28,7 @@ It's also brand new! For selectdb-specific functionality, we will aim for backwa ## Join the dbt Community - Be part of the conversation in the [dbt Community Slack](http://community.getdbt.com/) -- If one doesn't exist feel free to request a #db-selectdb channel be made in the [#channel-requests](https://getdbt.slack.com/archives/C01D8J8AJDA) on dbt community slack channel. +- If one doesn't exist feel free to request a #db-doris channel be made in the [#channel-requests](https://getdbt.slack.com/archives/C01D8J8AJDA) on dbt community slack channel. - Read more on the [dbt Community Discourse](https://discourse.getdbt.com) ## Reporting bugs and contributing code @@ -40,4 +38,4 @@ It's also brand new! For selectdb-specific functionality, we will aim for backwa ## Code of Conduct -Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct).# dbt-selectdb +Everyone interacting in the dbt project's codebases, issue trackers, chat rooms, and mailing lists is expected to follow the [dbt Code of Conduct](https://community.getdbt.com/code-of-conduct).# dbt-doris diff --git a/dbt/adapters/doris/__init__.py b/dbt/adapters/doris/__init__.py new file mode 100644 index 0000000..4e2721c --- /dev/null +++ b/dbt/adapters/doris/__init__.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dbt.adapters.doris.connections import DorisConnectionManager # noqa +from dbt.adapters.doris.connections import DorisCredentials +from dbt.adapters.doris.impl import DorisAdapter + +from dbt.adapters.base import AdapterPlugin +from dbt.include import doris + + +Plugin = AdapterPlugin( + adapter=DorisAdapter, + credentials=DorisCredentials, + include_path=doris.PACKAGE_PATH, + ) diff --git a/dbt/adapters/doris/__version__.py b/dbt/adapters/doris/__version__.py new file mode 100644 index 0000000..ba5a23c --- /dev/null +++ b/dbt/adapters/doris/__version__.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +# this 'version' must be set !!! +# otherwise the adapters will not be found after the 'dbt init xxx' command + +version = "0.3.4" diff --git a/dbt/adapters/doris/column.py b/dbt/adapters/doris/column.py new file mode 100644 index 0000000..e5f8da0 --- /dev/null +++ b/dbt/adapters/doris/column.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass + +from dbt.adapters.base.column import Column + + +@dataclass +class DorisColumn(Column): + @property + def quoted(self) -> str: + return "`{}`".format(self.column) + + def __repr__(self) -> str: + return f"" diff --git a/dbt/adapters/selectdb/connections.py b/dbt/adapters/doris/connections.py similarity index 66% rename from dbt/adapters/selectdb/connections.py rename to dbt/adapters/doris/connections.py index c5e298c..f19878b 100644 --- a/dbt/adapters/selectdb/connections.py +++ b/dbt/adapters/doris/connections.py @@ -1,3 +1,23 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + from contextlib import contextmanager from dataclasses import dataclass from typing import ContextManager, Optional, Union @@ -10,11 +30,11 @@ from dbt.contracts.connection import AdapterResponse, Connection, ConnectionState from dbt.events import AdapterLogger -logger = AdapterLogger("selectdb") +logger = AdapterLogger("doris") @dataclass -class SelectdbCredentials(Credentials): +class DorisCredentials(Credentials): host: str = "127.0.0.1" port: int = 9030 username: str = "root" @@ -25,7 +45,7 @@ class SelectdbCredentials(Credentials): @property def type(self): - return "selectdb" + return "doris" def _connection_keys(self): return "host", "port", "user", "schema" @@ -36,16 +56,16 @@ def unique_field(self) -> str: def __post_init__(self): if self.database is not None and self.database != self.schema: - raise exceptions.RuntimeException( + raise exceptions.DbtRuntimeError( f" schema: {self.schema} \n" f" database: {self.database} \n" - f"On SelectDB, database must be omitted or have the same value as" + f"On Doris, database must be omitted or have the same value as" f" schema." ) -class SelectdbConnectionManager(SQLConnectionManager): - TYPE = "selectdb" +class DorisConnectionManager(SQLConnectionManager): + TYPE = "doris" @classmethod def open(cls, connection: Connection) -> Connection: @@ -82,7 +102,7 @@ def open(cls, connection: Connection) -> Connection: connection.handle = None connection.state = 'fail' - raise dbt.exceptions.FailedToConnectException(str(e)) + raise exceptions.FailedToConnectError(str(e)) return connection @classmethod @@ -106,23 +126,24 @@ def get_response(cls, cursor) -> Union[AdapterResponse, str]: rows_affected=num_rows, ) - @contextmanager - def exception_handler(self, sql: str) -> ContextManager: + @contextmanager + def exception_handler(self, sql: str) -> ContextManager: try: yield except mysql.connector.DatabaseError as e: - logger.debug(f"SelectDB database error: {e}, sql: {sql}") - raise exceptions.DatabaseException(str(e)) from e + logger.debug(f"Doris database error: {e}, sql: {sql}") + raise exceptions.DbtDatabaseError(str(e)) from e except Exception as e: logger.debug(f"Error running SQL: {sql}") - if isinstance(e, exceptions.RuntimeException): + if isinstance(e, exceptions.DbtRuntimeError): raise e - raise exceptions.RuntimeException(str(e)) from e + raise exceptions.DbtRuntimeError(str(e)) from e @classmethod def begin(self): """ - SelectDB's inserting always transaction, ignore it + https://doris.apache.org/docs/data-operate/import/import-scenes/load-atomicity/ + Doris's inserting always transaction, ignore it """ pass diff --git a/dbt/adapters/doris/doris_column_item.py b/dbt/adapters/doris/doris_column_item.py new file mode 100644 index 0000000..1c73d67 --- /dev/null +++ b/dbt/adapters/doris/doris_column_item.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +class DorisColumnItem: + def __init__(self, col_name, col_type, col_comment, col_default): + self._col_name = col_name + self._col_type = col_type + self._col_comment = col_comment + self._col_default = col_default + + def get_col_name(self): + return self._col_name + + def get_col_type(self): + return self._col_type + + def get_col_comment(self): + return self._col_comment + + def get_col_default(self): + return self._col_default + + def get_view_column_constraint(self): + res = "" + if self._col_comment != "": + res = f"`{self._col_name}` COMMENT '{self._col_comment}'" + else: + res = f"`{self._col_name}`" + return res + + def get_table_column_constraint(self): + res = "" + if self._col_type is not None: + res = f"cast(`{self._col_name}` as {self._col_type}) as `{self._col_name}`" + else: + res = f"`{self._col_name}`" + return res diff --git a/dbt/adapters/selectdb/impl.py b/dbt/adapters/doris/impl.py similarity index 67% rename from dbt/adapters/selectdb/impl.py rename to dbt/adapters/doris/impl.py index 77057c8..ba6406d 100644 --- a/dbt/adapters/selectdb/impl.py +++ b/dbt/adapters/doris/impl.py @@ -1,23 +1,56 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. from dbt.adapters.sql import SQLAdapter from concurrent.futures import Future from enum import Enum -from typing import Callable, Dict, List, Optional, Set, Tuple +from typing import ( + Any, + Callable, + Dict, + Iterable, + Iterator, + List, + Mapping, + Optional, + Set, + Tuple, + Type, + Union, +) import agate import dbt.exceptions from dbt.adapters.base.impl import _expect_row_value, catch_as_completed from dbt.adapters.base.relation import InformationSchema, BaseRelation -from dbt.adapters.selectdb.column import SelectdbColumn -from dbt.adapters.selectdb.connections import SelectdbConnectionManager -from dbt.adapters.selectdb.relation import SelectdbRelation +from dbt.adapters.doris.column import DorisColumn +from dbt.adapters.doris.connections import DorisConnectionManager +from dbt.adapters.doris.relation import DorisRelation from dbt.adapters.protocol import AdapterConfig from dbt.adapters.sql.impl import LIST_RELATIONS_MACRO_NAME, LIST_SCHEMAS_MACRO_NAME from dbt.clients.agate_helper import table_from_rows from dbt.contracts.graph.manifest import Manifest from dbt.contracts.relation import RelationType from dbt.utils import executor +from dbt.adapters.doris.doris_column_item import DorisColumnItem class Engine(str, Enum): @@ -32,7 +65,8 @@ class PartitionType(str, Enum): list = "LIST" range = "RANGE" -class SelectdbConfig(AdapterConfig): + +class DorisConfig(AdapterConfig): engine: Engine duplicate_key: Tuple[str] partition_by: Tuple[str] @@ -42,11 +76,12 @@ class SelectdbConfig(AdapterConfig): buckets: int properties: Dict[str, str] -class SelectdbAdapter(SQLAdapter): - ConnectionManager = SelectdbConnectionManager - Relation = SelectdbRelation - AdapterSpecificConfigs = SelectdbConfig - Column = SelectdbColumn + +class DorisAdapter(SQLAdapter): + ConnectionManager = DorisConnectionManager + Relation = DorisRelation + AdapterSpecificConfigs = DorisConfig + Column = DorisColumn @classmethod def date_function(cls) -> str: @@ -70,10 +105,7 @@ def check_schema_exists(self, database, schema): return exists def get_relation(self, database: Optional[str], schema: str, identifier: str): - if not self.Relation.include_policy.database: - database = None - - return super().get_relation(database, schema, identifier) + return super().get_relation(None, schema, identifier) def drop_schema(self, relation: BaseRelation): relations = self.list_relations( @@ -84,14 +116,14 @@ def drop_schema(self, relation: BaseRelation): self.drop_relation(relation) super().drop_schema(relation) - def list_relations_without_caching(self, schema_relation: SelectdbRelation) -> List[SelectdbRelation]: + def list_relations_without_caching(self, schema_relation: DorisRelation) -> List[DorisRelation]: kwargs = {"schema_relation": schema_relation} results = self.execute_macro(LIST_RELATIONS_MACRO_NAME, kwargs=kwargs) relations = [] for row in results: if len(row) != 4: - raise dbt.exceptions.RuntimeException( + raise dbt.exceptions.DbtRuntimeError( f"Invalid value from 'show table extended ...', " f"got {len(row)} values, expected 4" ) @@ -109,7 +141,7 @@ def list_relations_without_caching(self, schema_relation: SelectdbRelation) -> L def get_catalog(self, manifest): schema_map = self._get_catalog_schemas(manifest) - + with executor(self.config) as tpe: futures: List[Future[agate.Table]] = [] for info, schemas in schema_map.items(): @@ -150,14 +182,14 @@ def _catalog_filter_table(cls, table: agate.Table, manifest: Manifest) -> agate. return table.where(cls._catalog_filter_schemas(manifest)) def _get_one_catalog( - self, - information_schema: InformationSchema, - schemas: Set[str], - manifest: Manifest, + self, + information_schema: InformationSchema, + schemas: Set[str], + manifest: Manifest, ) -> agate.Table: if len(schemas) != 1: dbt.exceptions.raise_compiler_error( - f"Expected only one schema in SelectDB _get_one_catalog, found " f"{schemas}" + f"Expected only one schema in Doris _get_one_catalog, found " f"{schemas}" ) return super()._get_one_catalog(information_schema, schemas, manifest) @@ -168,4 +200,18 @@ def timestamp_add_sql(self, add_to: str, number: int = 1, interval: str = "hour" # default. A lot of searching has lead me to believe that the # '+ interval' syntax used in postgres/redshift is relatively common # and might even be the SQL standard's intention. - return f"{add_to} + interval {number} {interval}" + return f"{add_to} + interval {number} {interval}" + + + @classmethod + def render_raw_columns_constraints(cls, raw_columns: Dict[str, Dict[str, Any]]) -> List: + rendered_column_constraints = [] + for v in raw_columns.values(): + cols_name = cls.quote(v["name"]) if v.get("quote") else v["name"] + data_type = v.get('data_type') + comment = v.get('description') + + column = DorisColumnItem(cols_name, data_type, comment, "") + rendered_column_constraints.append(column) + + return rendered_column_constraints \ No newline at end of file diff --git a/dbt/adapters/doris/relation.py b/dbt/adapters/doris/relation.py new file mode 100644 index 0000000..17956d1 --- /dev/null +++ b/dbt/adapters/doris/relation.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +from dataclasses import dataclass, field + +from dbt.adapters.base.relation import BaseRelation, Policy +from dbt.exceptions import DbtRuntimeError + + +@dataclass +class DorisQuotePolicy(Policy): + database: bool = False + schema: bool = True + identifier: bool = True + + +@dataclass +class DorisIncludePolicy(Policy): + database: bool = False + schema: bool = True + identifier: bool = True + + +@dataclass(frozen=True, eq=False, repr=False) +class DorisRelation(BaseRelation): + quote_policy: DorisQuotePolicy = field(default_factory=lambda: DorisQuotePolicy()) + include_policy: DorisIncludePolicy = field(default_factory=lambda: DorisIncludePolicy()) + quote_character: str = "`" + + def __post_init__(self): + if self.database != self.schema and self.database: + raise DbtRuntimeError(f"Cannot set database {self.database} in Doris!") + + def render(self): + if self.include_policy.database and self.include_policy.schema: + raise DbtRuntimeError( + "Got a Doris relation with schema and database set to include, but only one can be set" + ) + return super().render() diff --git a/dbt/adapters/selectdb/__init__.py b/dbt/adapters/selectdb/__init__.py deleted file mode 100644 index a559345..0000000 --- a/dbt/adapters/selectdb/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from dbt.adapters.selectdb.connections import SelectdbConnectionManager # noqa -from dbt.adapters.selectdb.connections import SelectdbCredentials -from dbt.adapters.selectdb.impl import SelectdbAdapter - -from dbt.adapters.base import AdapterPlugin -from dbt.include import selectdb - - -Plugin = AdapterPlugin( - adapter=SelectdbAdapter, - credentials=SelectdbCredentials, - include_path=selectdb.PACKAGE_PATH - ) diff --git a/dbt/adapters/selectdb/__version__.py b/dbt/adapters/selectdb/__version__.py deleted file mode 100644 index 9f7a875..0000000 --- a/dbt/adapters/selectdb/__version__.py +++ /dev/null @@ -1 +0,0 @@ -version = "0.1.0" diff --git a/dbt/adapters/selectdb/column.py b/dbt/adapters/selectdb/column.py deleted file mode 100644 index 6df2a05..0000000 --- a/dbt/adapters/selectdb/column.py +++ /dev/null @@ -1,13 +0,0 @@ -from dataclasses import dataclass - -from dbt.adapters.base.column import Column - - -@dataclass -class SelectdbColumn(Column): - @property - def quoted(self) -> str: - return "`{}`".format(self.column) - - def __repr__(self) -> str: - return f"" diff --git a/dbt/adapters/selectdb/relation.py b/dbt/adapters/selectdb/relation.py deleted file mode 100644 index f4a778f..0000000 --- a/dbt/adapters/selectdb/relation.py +++ /dev/null @@ -1,35 +0,0 @@ -from dataclasses import dataclass - -from dbt.adapters.base.relation import BaseRelation, Policy -from dbt.exceptions import RuntimeException - -@dataclass -class SelectdbQuotePolicy(Policy): - database: bool = False - schema: bool = True - identifier: bool = True - - -@dataclass -class SelectdbIncludePolicy(Policy): - database: bool = False - schema: bool = True - identifier: bool = True - - -@dataclass(frozen=True, eq=False, repr=False) -class SelectdbRelation(BaseRelation): - quote_policy: SelectdbQuotePolicy = SelectdbQuotePolicy() - include_policy: SelectdbIncludePolicy = SelectdbIncludePolicy() - quote_character: str = "`" - - def __post_init__(self): - if self.database != self.schema and self.database: - raise RuntimeException(f"Cannot set database {self.database} in Selectdb!") - - def render(self): - if self.include_policy.database and self.include_policy.schema: - raise RuntimeException( - "Got a Selectdb relation with schema and database set to include, but only one can be set" - ) - return super().render() diff --git a/dbt/include/doris/__init__.py b/dbt/include/doris/__init__.py new file mode 100644 index 0000000..2dc63b8 --- /dev/null +++ b/dbt/include/doris/__init__.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os + +PACKAGE_PATH = os.path.dirname(__file__) diff --git a/dbt/include/doris/dbt_project.yml b/dbt/include/doris/dbt_project.yml new file mode 100644 index 0000000..337002a --- /dev/null +++ b/dbt/include/doris/dbt_project.yml @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: dbt_doris +version: 0.3.4 +config-version: 2 + +macro-paths: ["macros"] diff --git a/dbt/include/doris/macros/adapters/columns.sql b/dbt/include/doris/macros/adapters/columns.sql new file mode 100644 index 0000000..b1a834c --- /dev/null +++ b/dbt/include/doris/macros/adapters/columns.sql @@ -0,0 +1,40 @@ +{% macro doris__get_columns_in_relation(relation) -%} + {% call statement('get_columns_in_relation', fetch_result=True) %} + select column_name as `column`, + data_type as 'dtype', + character_maximum_length as char_size, + numeric_precision, + numeric_scale +from information_schema.columns +where table_schema = '{{ relation.schema }}' + and table_name = '{{ relation.identifier }}' + {% endcall %} + {% set table = load_result('get_columns_in_relation').table %} + {{ return(sql_convert_columns_in_relation(table)) }} +{%- endmacro %} + +{% macro doris__alter_column_type(relation,column_name,new_column_type) -%} +'''Changes column name or data type''' +{% endmacro %} + +{% macro columns_and_constraints(table_type="table") %} + {# loop through user_provided_columns to create DDL with data types and constraints #} + {%- set raw_column_constraints = adapter.render_raw_columns_constraints(raw_columns=model['columns']) -%} + {% for c in raw_column_constraints -%} + {% if table_type == "table" %} + {{ c.get_table_column_constraint() }}{{ "," if not loop.last or raw_model_constraints }} + {% else %} + {{ c.get_view_column_constraint() }}{{ "," if not loop.last or raw_model_constraints }} + {% endif %} + {% endfor %} +{% endmacro %} + +{% macro doris__get_table_columns_and_constraints() -%} + {{ return(columns_and_constraints("table")) }} +{%- endmacro %} + + +{% macro doris__get_view_columns_comment() -%} + {{ return(columns_and_constraints("view")) }} +{%- endmacro %} + diff --git a/dbt/include/doris/macros/adapters/freshness.sql b/dbt/include/doris/macros/adapters/freshness.sql new file mode 100644 index 0000000..2e09787 --- /dev/null +++ b/dbt/include/doris/macros/adapters/freshness.sql @@ -0,0 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro doris__current_timestamp() -%} + current_timestamp() +{%- endmacro %} diff --git a/dbt/include/selectdb/macros/adapters/metadata.sql b/dbt/include/doris/macros/adapters/metadata.sql similarity index 69% rename from dbt/include/selectdb/macros/adapters/metadata.sql rename to dbt/include/doris/macros/adapters/metadata.sql index 5046ca9..9927c3d 100644 --- a/dbt/include/selectdb/macros/adapters/metadata.sql +++ b/dbt/include/doris/macros/adapters/metadata.sql @@ -1,4 +1,21 @@ -{% macro selectdb__list_relations_without_caching(schema_relation) -%} +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro doris__list_relations_without_caching(schema_relation) -%} {% call statement('list_relations_without_caching', fetch_result=True) %} select null as "database", @@ -13,7 +30,7 @@ {{ return(load_result('list_relations_without_caching').table) }} {%- endmacro %} -{% macro selectdb__get_catalog(information_schema, schemas) -%} +{% macro doris__get_catalog(information_schema, schemas) -%} {%- call statement('catalog', fetch_result=True) -%} with tables as ( select @@ -65,10 +82,10 @@ {%- endmacro %} -{% macro selectdb__check_schema_exists(database, schema) -%} +{% macro doris__check_schema_exists(database, schema) -%} {%- endmacro %} -{% macro selectdb__list_schemas(database) -%} +{% macro doris__list_schemas(database) -%} {% call statement('list_schemas', fetch_result=True, auto_begin=False) -%} select distinct schema_name from information_schema.schemata {%- endcall %} diff --git a/dbt/include/doris/macros/adapters/relation.sql b/dbt/include/doris/macros/adapters/relation.sql new file mode 100644 index 0000000..3b2950d --- /dev/null +++ b/dbt/include/doris/macros/adapters/relation.sql @@ -0,0 +1,217 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro doris__engine() -%} + {% set label = 'ENGINE' %} + {% set engine = config.get('engine', 'OLAP') %} + {{ label }} = {{ engine }} +{%- endmacro %} + +{% macro doris__partition_by() -%} + {% set cols = config.get('partition_by', validator=validation.any[list, basestring]) %} + {% set partition_type = config.get('partition_type', 'RANGE') %} + {% if cols is not none %} + {%- if cols is string -%} + {%- set cols = [cols] -%} + {%- endif -%} + PARTITION BY {{ partition_type }} ( + {% for col in cols %} + {{ col }}{% if not loop.last %},{% endif %} + {% endfor %} + )( + {% set init = config.get('partition_by_init', validator=validation.any[list]) %} + {% if init is not none %} + {% for row in init %} + {{ row }}{% if not loop.last %},{% endif %} + {% endfor %} + {% endif %} + ) + {% endif %} +{%- endmacro %} + +{% macro doris__duplicate_key() -%} + {% set cols = config.get('duplicate_key', validator=validation.any[list, basestring]) %} + {% if cols is not none %} + {%- if cols is string -%} + {%- set cols = [cols] -%} + {%- endif -%} + DUPLICATE KEY ( + {% for item in cols %} + {{ item }} + {% if not loop.last %},{% endif %} + {% endfor %} + ) + {% endif %} +{%- endmacro %} + +{% macro doris__table_comment() -%} + {% set description = model.get('description', "") %} + COMMENT '{{description}}' +{%- endmacro %} + +{% macro doris__unique_key() -%} + {% set cols = config.get('unique_key', validator=validation.any[list, basestring]) %} + + {% if cols is not none %} + {%- if cols is string -%} + {%- set cols = [cols] -%} + {%- endif -%} + + UNIQUE KEY ( + {% for item in cols %} + {{ item }} + {% if not loop.last %},{% endif %} + {% endfor %} + ) + {% endif %} +{%- endmacro %} + +{% macro doris__distributed_by(column_names) -%} + {% set engine = config.get('engine', validator=validation.any[basestring]) %} + {% set cols = config.get('distributed_by', validator=validation.any[list, basestring]) %} + {% if cols is none and engine in [none,'OLAP'] %} + {% set cols = column_names %} + {% endif %} + + {% if cols %} + {%- if cols is string -%} + {%- set cols = [cols] -%} + {%- endif -%} + DISTRIBUTED BY HASH ( + {% for item in cols %} + {{ item }}{% if not loop.last %},{% endif %} + {% endfor %} + ) BUCKETS {{ config.get('buckets', validator=validation.any[int]) or 10 }} + {% endif %} +{%- endmacro %} + +{% macro doris__properties() -%} + {% set properties = config.get('properties', validator=validation.any[dict]) %} + {% set replice_num = config.get('replication_num') %} + + {% if replice_num is not none %} + {% if properties is none %} + {% set properties = {} %} + {% endif %} + {% do properties.update({'replication_num': replice_num}) %} + {% endif %} + + {% if properties is not none %} + PROPERTIES ( + {% for key, value in properties.items() %} + "{{ key }}" = "{{ value }}"{% if not loop.last %},{% endif %} + {% endfor %} + ) + {% endif %} +{%- endmacro%} + +{% macro doris__drop_relation(relation) -%} + {% if relation is not none %} + {% set relation_type = relation.type %} + {% if not relation_type or relation_type is none %} + {% set relation_type = 'table' %} + {% endif %} + {% call statement('drop_relation', auto_begin=False) %} + drop {{ relation_type }} if exists {{ relation }} + {% endcall %} + {% endif %} + +{%- endmacro %} + +{% macro doris__truncate_relation(relation) -%} + {% call statement('truncate_relation') %} + truncate table {{ relation }} + {% endcall %} +{%- endmacro %} + +{% macro doris__rename_relation(from_relation, to_relation) -%} + {% call statement('drop_relation') %} + drop {{ to_relation.type }} if exists {{ to_relation }} + {% endcall %} + {% call statement('rename_relation') %} + {% if to_relation.is_view %} + {% set results = run_query('show create view ' + from_relation.render() ) %} + create view {{ to_relation }} as {{ results[0]['Create View'].split('AS',1)[1] }} + {% else %} + alter table {{ from_relation }} rename {{ to_relation.table }} + {% endif %} + {% endcall %} + + {% if to_relation.is_view %} + {% call statement('rename_relation_end_drop_old') %} + drop view if exists {{ from_relation }} + {% endcall %} + {% endif %} + +{%- endmacro %} + + +{% macro exchange_relation(relation1, relation2, is_drop_r1=false) -%} + + {% if relation2.is_view %} + {% set from_results = run_query('show create view ' + relation1.render() ) %} + {% set to_results = run_query('show create view ' + relation2.render() ) %} + {% call statement('exchange_view_relation') %} + alter view {{ relation1 }} as {{ to_results[0]['Create View'].split('AS',1)[1] }} + {% endcall %} + {% if is_drop_r1 %} + {% do doris__drop_relation(relation2) %} + {% else %} + {% call statement('exchange_view_relation') %} + alter view {{ relation2 }} as {{ from_results[0]['Create View'].split('AS',1)[1] }} + {% endcall %} + {% endif %} + {% else %} + {% call statement('exchange_relation') %} + ALTER TABLE {{ relation1 }} REPLACE WITH TABLE `{{ relation2.table }}` PROPERTIES('swap' = '{{not is_drop_r1}}'); + {% endcall %} + {% endif %} + +{%- endmacro %} + +{% macro doris__timestimp_id() -%} + {{ return( (modules.datetime.datetime.now() ~ "").replace('-','').replace(':','').replace('.','').replace(' ','') ) }} +{%- endmacro %} + +{% macro doris__with_label() -%} + {% set lable_suffix_id = config.get('label_id', validator=validation.any[basestring]) %} + {% if lable_suffix_id in [none,'DEFAULT'] %} + WITH LABEL dbt_doris_label_{{doris__timestimp_id()}} + {% else %} + WITH LABEL dbt_doris_label_{{ lable_suffix_id }} + {% endif %} +{%- endmacro %} + +{% macro doris__get_or_create_relation(database, schema, identifier, type) %} + {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} + + {% if target_relation %} + {% do return([true, target_relation]) %} + {% endif %} + + {%- set new_relation = api.Relation.create( + database=none, + schema=schema, + identifier=identifier, + type=type + ) -%} + {% do return([false, new_relation]) %} +{% endmacro %} + +{% macro catalog_source(catalog,database,table) -%} + `{{catalog}}`.`{{database}}`.`{{table}}` +{%- endmacro %} diff --git a/dbt/include/doris/macros/adapters/schema.sql b/dbt/include/doris/macros/adapters/schema.sql new file mode 100644 index 0000000..c71ee58 --- /dev/null +++ b/dbt/include/doris/macros/adapters/schema.sql @@ -0,0 +1,32 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + + +-- doris have not 'schema' to make a collection of table or view + +{% macro doris__drop_schema(relation) -%} + {%- call statement('drop_schema') -%} + drop database if exists {{ relation.without_identifier().include(database=False) }} + {%- endcall -%} +{% endmacro %} + + +{% macro doris__create_schema(relation) -%} + {%- call statement('create_schema') -%} + create database if not exists {{ relation.without_identifier().include(database=False) }} + {% endcall %} +{% endmacro %} diff --git a/dbt/include/selectdb/macros/materializations/incremental/help.sql b/dbt/include/doris/macros/materializations/incremental/help.sql similarity index 59% rename from dbt/include/selectdb/macros/materializations/incremental/help.sql rename to dbt/include/doris/macros/materializations/incremental/help.sql index bfce986..be8af0b 100644 --- a/dbt/include/selectdb/macros/materializations/incremental/help.sql +++ b/dbt/include/doris/macros/materializations/incremental/help.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + {% macro is_incremental() %} {% if not execute %} {{ return(False) }} @@ -39,7 +56,12 @@ show create table {{ target_relation }} {%- endmacro %} -{% macro is_unique_model( table_create_obj ) %} +{% macro is_unique_model( target_relation ) %} + {% set build_show_create = show_create( target_relation, statement_name='table_model') %} + {% call statement('table_model' , fetch_result=True) %} + {{ build_show_create }} + {% endcall %} + {%- set table_create_obj = load_result('table_model') -%} {% set create_table = table_create_obj['data'][0][1]%} {{ return('\nUNIQUE KEY(' in create_table and '\nDUPLICATE KEY(' not in create_table and '\nAGGREGATE KEY(' not in create_table) }} {%- endmacro %} diff --git a/dbt/include/doris/macros/materializations/incremental/incremental.sql b/dbt/include/doris/macros/materializations/incremental/incremental.sql new file mode 100644 index 0000000..60b0a34 --- /dev/null +++ b/dbt/include/doris/macros/materializations/incremental/incremental.sql @@ -0,0 +1,113 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% materialization incremental, adapter='doris' %} + {% set unique_key = config.get('unique_key', validator=validation.any[list]) %} + {% set strategy = dbt_doris_validate_get_incremental_strategy(config) %} + {% set full_refresh_mode = (should_full_refresh()) %} + {% set target_relation = this.incorporate(type='table') %} + {% set existing_relation = load_relation(this) %} + {% set tmp_relation = make_temp_relation(this) %} + {{ run_hooks(pre_hooks, inside_transaction=False) }} + {{ run_hooks(pre_hooks, inside_transaction=True) }} + {% set to_drop = [] %} + {#-- append or no unique key --#} + + + + {% if not unique_key or strategy == 'append' %} + {#-- create table first --#} + {% if existing_relation is none %} + {% set build_sql = doris__create_table_as(False, target_relation, sql) %} + {% elif existing_relation.is_view or full_refresh_mode %} + {#-- backup table is new table ,exchange table backup and old table #} + {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %} + {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %} + {% do adapter.drop_relation(backup_relation) %} {#-- likes 'drop table if exists ... ' --#} + {% set run_sql = doris__create_table_as(False, backup_relation, sql) %} + {% call statement("run_sql") %} + {{ run_sql }} + {% endcall %} + {% do exchange_relation(target_relation, backup_relation, True) %} + {% set build_sql = "select 'hello doris'" %} + {#-- append data --#} + {% else %} + {% do to_drop.append(tmp_relation) %} + {% do run_query(create_table_as(True, tmp_relation, sql)) %} + {% set build_sql = tmp_insert(tmp_relation, target_relation, unique_key=none) %} + {% endif %} + {#-- insert overwrite --#} + {% elif strategy == 'insert_overwrite' %} + {#-- create table first --#} + {% if existing_relation is none %} + {% set build_sql = doris__create_unique_table_as(False, target_relation, sql) %} + {#-- insert data refresh --#} + {% elif existing_relation.is_view or full_refresh_mode %} + {#-- backup table is new table ,exchange table backup and old table #} + {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %} + {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %} + {% do adapter.drop_relation(backup_relation) %} {#-- likes 'drop table if exists ... ' --#} + {% set run_sql = doris__create_unique_table_as(False, backup_relation, sql) %} + {% call statement("run_sql") %} + {{ run_sql }} + {% endcall %} + {% do exchange_relation(target_relation, backup_relation, True) %} + {% set build_sql = "select 'hello doris'" %} + {#-- append data --#} + {% else %} + {#-- check doris unique table --#} + {% if not is_unique_model(target_relation) %} + {% do exceptions.raise_compiler_error("doris table:"~ target_relation ~ ", model must be 'UNIQUE'" ) %} + {% endif %} + {#-- create temp duplicate table for this incremental task --#} + {% do run_query(create_table_as(True, tmp_relation, sql)) %} + {% do to_drop.append(tmp_relation) %} + {% do adapter.expand_target_column_types( + from_relation=tmp_relation, + to_relation=target_relation) %} + {% set build_sql = tmp_insert(tmp_relation, target_relation, unique_key=unique_key) %} + {% endif %} + {% else %} + {#-- never --#} + {% endif %} + + {% call statement("main") %} + {{ build_sql }} + {% endcall %} + + {#-- {% do persist_docs(target_relation, model) %} #} + {{ run_hooks(post_hooks, inside_transaction=True) }} + {% do adapter.commit() %} + {% for rel in to_drop %} + {% do doris__drop_relation(rel) %} + {% endfor %} + {{ run_hooks(post_hooks, inside_transaction=False) }} + {{ return({'relations': [target_relation]}) }} +{%- endmaterialization %} + +{% macro dbt_doris_validate_get_incremental_strategy(config) %} + {#-- Find and validate the incremental strategy #} + {%- set strategy = config.get('incremental_strategy') or 'insert_overwrite' -%} + {% set invalid_strategy_msg -%} + Invalid incremental strategy provided: {{ strategy }} + Expected one of: 'append', 'insert_overwrite' + {%- endset %} + {% if strategy not in ['append', 'insert_overwrite'] %} + {% do exceptions.raise_compiler_error(invalid_strategy_msg) %} + {% endif %} + {% do return (strategy) %} +{% endmacro %} diff --git a/dbt/include/selectdb/macros/materializations/partition/helpers.sql b/dbt/include/doris/macros/materializations/partition/helpers.sql similarity index 69% rename from dbt/include/selectdb/macros/materializations/partition/helpers.sql rename to dbt/include/doris/macros/materializations/partition/helpers.sql index d9d617e..aecf153 100644 --- a/dbt/include/selectdb/macros/materializations/partition/helpers.sql +++ b/dbt/include/doris/macros/materializations/partition/helpers.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + {% macro get_distinct_partitions(relation, partition_by) %} {% set sql %} select distinct {{ ','.join(partition_by) }} from {{ relation }} order by {{ ','.join(partition_by) }} diff --git a/dbt/include/selectdb/macros/materializations/partition/partition.sql b/dbt/include/doris/macros/materializations/partition/partition.sql similarity index 83% rename from dbt/include/selectdb/macros/materializations/partition/partition.sql rename to dbt/include/doris/macros/materializations/partition/partition.sql index d39191d..3892c3e 100644 --- a/dbt/include/selectdb/macros/materializations/partition/partition.sql +++ b/dbt/include/doris/macros/materializations/partition/partition.sql @@ -1,3 +1,20 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + {% materialization partition, default -%} {% set partition_by = config.get('partition_by') %} diff --git a/dbt/include/doris/macros/materializations/partition/replace.sql b/dbt/include/doris/macros/materializations/partition/replace.sql new file mode 100644 index 0000000..bce087c --- /dev/null +++ b/dbt/include/doris/macros/materializations/partition/replace.sql @@ -0,0 +1,26 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro get_partition_replace_sql(relation, partitions) %} + {% for partition in partitions %} + {% set items = get_partition_items(partition) %} + {% set p = ''.join(items) %} + alter table {{ relation }} replace partition (p{{ p }}) with temporary partition (tp{{ p }}) properties ( + "strict_range" = "false" + ); + {% endfor %} +{% endmacro %} diff --git a/dbt/include/doris/macros/materializations/seed/helpers.sql b/dbt/include/doris/macros/materializations/seed/helpers.sql new file mode 100644 index 0000000..bb39990 --- /dev/null +++ b/dbt/include/doris/macros/materializations/seed/helpers.sql @@ -0,0 +1,55 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro doris__create_csv_table(model, agate_table) -%} + {% set column_override = model['config'].get('column_types', {}) %} + {% set quote_seed_column = model['config'].get('quote_columns', None) %} + {% set col = model.get('columns', None) %} + + {# + {{print(column_override)}} + {{print(quote_seed_column)}} + {{print(col)}} + #} + + {% set sql %} + create table {{ this.render() }} ( + {% for col_name in agate_table.column_names %} + {% set inferred_type = adapter.convert_type(agate_table, loop.index0) %} + {% set col_type = column_override.get(col_name, inferred_type) %} + {% set column_name = (col_name | string) %} + {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ col_type }}{% if not loop.last %},{% endif %} + {% endfor %} + ) + {{ doris__engine() }} + {{ doris__duplicate_key() }} + {{ doris__partition_by() }} + {{ doris__distributed_by(agate_table.column_names[0:1]) }} + {{ doris__properties() }} + {% endset %} + + {# + {{print(sql)}} + #} + + {% call statement('run_ddl_create_seed') %} + {{ sql }} + {% endcall %} + + {{ return(sql) }} + +{%- endmacro %} diff --git a/dbt/include/selectdb/macros/materializations/snapshot/snapshot.sql b/dbt/include/doris/macros/materializations/snapshot/snapshot.sql similarity index 70% rename from dbt/include/selectdb/macros/materializations/snapshot/snapshot.sql rename to dbt/include/doris/macros/materializations/snapshot/snapshot.sql index 6066d73..104e31b 100644 --- a/dbt/include/selectdb/macros/materializations/snapshot/snapshot.sql +++ b/dbt/include/doris/macros/materializations/snapshot/snapshot.sql @@ -1,9 +1,26 @@ -{% macro selectdb__snapshot_string_as_time(timestamp) -%} +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro doris__snapshot_string_as_time(timestamp) -%} {%- set result = "str_to_date('" ~ timestamp ~ "', '%Y-%m-%d %T')" -%} {{ return(result) }} {%- endmacro %} -{% macro selectdb__snapshot_merge_sql(target, source, insert_cols) -%} +{% macro doris__snapshot_merge_sql(target, source, insert_cols) -%} {%- set insert_cols_csv = insert_cols | join(', ') -%} {%- set valid_to_col = adapter.quote('dbt_valid_to') -%} diff --git a/dbt/include/doris/macros/materializations/snapshot/strategies.sql b/dbt/include/doris/macros/materializations/snapshot/strategies.sql new file mode 100644 index 0000000..a985a41 --- /dev/null +++ b/dbt/include/doris/macros/materializations/snapshot/strategies.sql @@ -0,0 +1,23 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro doris__snapshot_hash_arguments(args) -%} + md5(concat_ws('|', {%- for arg in args -%} + coalesce(cast({{ arg }} as char), '') + {% if not loop.last %}, {% endif %} + {%- endfor -%})) +{%- endmacro %} diff --git a/dbt/include/doris/macros/materializations/table/create_table_as.sql b/dbt/include/doris/macros/materializations/table/create_table_as.sql new file mode 100644 index 0000000..6db7099 --- /dev/null +++ b/dbt/include/doris/macros/materializations/table/create_table_as.sql @@ -0,0 +1,57 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro doris__create_table_as(temporary, relation, sql) -%} + {% set sql_header = config.get('sql_header', none) %} + {% set table = relation.include(database=False) %} + {{ sql_header if sql_header is not none }} + {%if temporary %} + {{doris__drop_relation(relation)}} + {% endif %} + create table {{ table }} + {{ doris__duplicate_key() }} + {{ doris__table_comment()}} + {{ doris__partition_by() }} + {{ doris__distributed_by() }} + {{ doris__properties() }} as {{ doris__table_colume_type(sql) }}; + +{%- endmacro %} + +{% macro doris__create_unique_table_as(temporary, relation, sql) -%} + {% set sql_header = config.get('sql_header', none) %} + {% set table = relation.include(database=False) %} + {{ sql_header if sql_header is not none }} + create table {{ table }} + {{ doris__unique_key() }} + {{ doris__table_comment()}} + {{ doris__partition_by() }} + {{ doris__distributed_by() }} + {{ doris__properties() }} as {{ doris__table_colume_type(sql) }}; + +{%- endmacro %} + + +{% macro doris__table_colume_type(sql) -%} + {% set cols = model.get('columns') %} + {% if cols %} + select {{get_table_columns_and_constraints()}} from ( + {{sql}} + ) `_table_colume_type_name` + {% else %} + {{sql}} + {%- endif -%} +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/doris/macros/materializations/table/table.sql b/dbt/include/doris/macros/materializations/table/table.sql new file mode 100644 index 0000000..051e54d --- /dev/null +++ b/dbt/include/doris/macros/materializations/table/table.sql @@ -0,0 +1,53 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% materialization table, adapter='doris' %} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='table') %} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + + -- grab current tables grants config for comparision later on + {% set grant_config = config.get('grants') %} + + -- drop the temp relations if they exist already in the database + {{ doris__drop_relation(preexisting_intermediate_relation) }} + + -- build model + {% call statement('main') -%} + {{ get_create_table_as_sql(False, intermediate_relation, sql) }} + {%- endcall %} + + {% if existing_relation -%} + {% do exchange_relation(target_relation, intermediate_relation, True) %} + {% else %} + {{ adapter.rename_relation(intermediate_relation, target_relation) }} + {% endif %} + + + {% set should_revoke = should_revoke(existing_relation, full_refresh_mode=True) %} + {% do apply_grants(target_relation, grant_config, should_revoke=should_revoke) %} + + -- alter relation comment + {% do persist_docs(target_relation, model) %} + + -- finally, drop the existing/backup relation after the commit + {{ doris__drop_relation(intermediate_relation) }} + + {{ return({'relations': [target_relation]}) }} +{% endmaterialization %} diff --git a/dbt/include/doris/macros/materializations/view/create_view_as.sql b/dbt/include/doris/macros/materializations/view/create_view_as.sql new file mode 100644 index 0000000..8bb845e --- /dev/null +++ b/dbt/include/doris/macros/materializations/view/create_view_as.sql @@ -0,0 +1,30 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{% macro doris__create_view_as(relation, sql) -%} + {%- set sql_header = config.get('sql_header', none) -%} + + {{ sql_header if sql_header is not none }} + create View {{ relation }} {{doris__view_colume_comment()}} as {{ sql }}; +{%- endmacro %} + +{% macro doris__view_colume_comment() -%} + {% set cols = model.get('columns') %} + {% if cols %} + ( {{doris__get_view_columns_comment()}} ) + {%- endif -%} +{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/doris/macros/materializations/view/view.sql b/dbt/include/doris/macros/materializations/view/view.sql new file mode 100644 index 0000000..fc0baa2 --- /dev/null +++ b/dbt/include/doris/macros/materializations/view/view.sql @@ -0,0 +1,49 @@ +-- Licensed to the Apache Software Foundation (ASF) under one +-- or more contributor license agreements. See the NOTICE file +-- distributed with this work for additional information +-- regarding copyright ownership. The ASF licenses this file +-- to you under the Apache License, Version 2.0 (the +-- "License"); you may not use this file except in compliance +-- with the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, +-- software distributed under the License is distributed on an +-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +-- KIND, either express or implied. See the License for the +-- specific language governing permissions and limitations +-- under the License. + +{%- materialization view, adapter='doris' -%} + + {%- set existing_relation = load_cached_relation(this) -%} + {%- set target_relation = this.incorporate(type='view') -%} + {%- set intermediate_relation = make_intermediate_relation(target_relation) -%} + {%- set preexisting_intermediate_relation = load_cached_relation(intermediate_relation) -%} + + + {{ drop_relation_if_exists(intermediate_relation) }} + + + {% if existing_relation is not none %} + --todo: exchange + {% call statement('main_test') -%} + {{ get_create_view_as_sql(intermediate_relation, sql) }} + {%- endcall %} + {{ drop_relation_if_exists(intermediate_relation) }} + {{ drop_relation_if_exists(target_relation) }} + {% call statement('main') -%} + {{ get_create_view_as_sql(target_relation, sql) }} + {%- endcall %} + {# {{ adapter.rename_relation(intermediate_relation, target_relation) }} #} + {% else %} + {% call statement('main') -%} + {{ get_create_view_as_sql(target_relation, sql) }} + {%- endcall %} + {% endif %} + + + {{ return({'relations': [target_relation]}) }} + +{%- endmaterialization -%} diff --git a/dbt/include/doris/profile_template.yml b/dbt/include/doris/profile_template.yml new file mode 100644 index 0000000..71c1e14 --- /dev/null +++ b/dbt/include/doris/profile_template.yml @@ -0,0 +1,25 @@ +#!/usr/bin/env python +# encoding: utf-8 + +fixed: + type: doris +prompts: + host: + hint: 'hostname for your instance(your doris fe host)' + port: + default: 9030 + type: 'int' + hint: 'port for your instance(your doris fe query_port)' + schema: + default: 'dbt' + hint: 'the schema name as stored in the database,doris have not schema to make a collection of table or view' + username: + hint: 'your doris username' + password: + hint: 'your doris password, if no password, just Enter' + hide_input: true + default: '' + threads: + hint: "1 or more" + type: "int" + default: 1 \ No newline at end of file diff --git a/dbt/include/selectdb/sample_profiles.yml b/dbt/include/doris/sample_profiles.yml similarity index 73% rename from dbt/include/selectdb/sample_profiles.yml rename to dbt/include/doris/sample_profiles.yml index 1f73aa0..e9e7676 100644 --- a/dbt/include/selectdb/sample_profiles.yml +++ b/dbt/include/doris/sample_profiles.yml @@ -1,7 +1,10 @@ +#!/usr/bin/env python +# encoding: utf-8 + default: outputs: dev: - type: selectdb + type: doris host: port: username: diff --git a/dbt/include/selectdb/__init__.py b/dbt/include/selectdb/__init__.py deleted file mode 100644 index b177e5d..0000000 --- a/dbt/include/selectdb/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -import os - -PACKAGE_PATH = os.path.dirname(__file__) diff --git a/dbt/include/selectdb/dbt_project.yml b/dbt/include/selectdb/dbt_project.yml deleted file mode 100644 index 1348a67..0000000 --- a/dbt/include/selectdb/dbt_project.yml +++ /dev/null @@ -1,6 +0,0 @@ -name: dbt_selectdb -version: 0.1.0 -config-version: 2 - -macro-paths: ["macros"] -clean-targets: [target, dbt_packages, logs] diff --git a/dbt/include/selectdb/macros/adapters/columns.sql b/dbt/include/selectdb/macros/adapters/columns.sql deleted file mode 100644 index 2c71862..0000000 --- a/dbt/include/selectdb/macros/adapters/columns.sql +++ /dev/null @@ -1,19 +0,0 @@ -{% macro selectdb__get_columns_in_relation(relation) -%} - {% call statement('get_columns_in_relation', fetch_result=True) %} - select column_name as `column`, - data_type as 'dtype', - character_maximum_length as char_size, - numeric_precision, - numeric_scale -from information_schema.columns -where table_schema = '{{ relation.schema }}' - and table_name = '{{ relation.identifier }}' - {% endcall %} - {% set table = load_result('get_columns_in_relation').table %} - {{ return(sql_convert_columns_in_relation(table)) }} -{%- endmacro %} - - -{% macro selectdb__alter_column_type(relation,column_name,new_column_type) -%} -'''Changes column name or data type''' -{% endmacro %} \ No newline at end of file diff --git a/dbt/include/selectdb/macros/adapters/freshness.sql b/dbt/include/selectdb/macros/adapters/freshness.sql deleted file mode 100644 index 7fa909b..0000000 --- a/dbt/include/selectdb/macros/adapters/freshness.sql +++ /dev/null @@ -1,3 +0,0 @@ -{% macro selectdb__current_timestamp() -%} - current_timestamp() -{%- endmacro %} diff --git a/dbt/include/selectdb/macros/adapters/relation.sql b/dbt/include/selectdb/macros/adapters/relation.sql deleted file mode 100644 index 9f15794..0000000 --- a/dbt/include/selectdb/macros/adapters/relation.sql +++ /dev/null @@ -1,149 +0,0 @@ -{% macro selectdb__engine() -%} - {% set label = 'ENGINE' %} - {% set engine = config.get('engine', 'OLAP') %} - {{ label }} = {{ engine }} -{%- endmacro %} - -{% macro selectdb__partition_by() -%} - {% set cols = config.get('partition_by') %} - {% set partition_type = config.get('partition_type', 'RANGE') %} - {% if cols is not none %} - PARTITION BY {{ partition_type }} ( - {% for col in cols %} - {{ col }}{% if not loop.last %},{% endif %} - {% endfor %} - )( - {% set init = config.get('partition_by_init', validator=validation.any[list]) %} - {% if init is not none %} - {% for row in init %} - {{ row }}{% if not loop.last %},{% endif %} - {% endfor %} - {% endif %} - ) - {% endif %} -{%- endmacro %} - -{% macro selectdb__duplicate_key() -%} - {% set cols = config.get('duplicate_key', validator=validation.any[list]) %} - {% if cols is not none %} - DUPLICATE KEY ( - {% for item in cols %} - {{ item }} - {% if not loop.last %},{% endif %} - {% endfor %} - ) - {% endif %} -{%- endmacro %} - -{% macro selectdb__unique_key() -%} - {% set cols = config.get('unique_key', validator=validation.any[list]) %} - {% if cols is not none %} - UNIQUE KEY ( - {% for item in cols %} - {{ item }} - {% if not loop.last %},{% endif %} - {% endfor %} - ) - {% endif %} -{%- endmacro %} - -{% macro selectdb__distributed_by(column_names) -%} - {% set label = 'DISTRIBUTED BY HASH' %} - {% set engine = config.get('engine', validator=validation.any[basestring]) %} - {% set cols = config.get('distributed_by', validator=validation.any[list]) %} - {% set buckets = config.get('buckets', validator=validation.any[int]) %} - {% if cols is none and engine in [none,'OLAP'] %} - {% set cols = column_names %} - {% endif %} - {% if cols %} - {{ label }} ( - {% for item in cols %} - {{ item }}{% if not loop.last %},{% endif %} - {% endfor %} - ) - {% if buckets is not none %} - BUCKETS {{ buckets }} - {% endif %} - {% endif %} -{%- endmacro %} - -{% macro selectdb__properties() -%} - {% set properties = config.get('properties', validator=validation.any[dict]) %} - {% if properties is not none %} - PROPERTIES ( - {% for key, value in properties.items() %} - "{{ key }}" = "{{ value }}"{% if not loop.last %},{% endif %} - {% endfor %} - ) - {% endif %} -{%- endmacro%} - -{% macro selectdb__drop_relation(relation) -%} - {% set relation_type = relation.type %} - {% if relation_type is none %} - {% set relation_type = 'table' %} - {% endif %} - {% call statement('drop_relation', auto_begin=False) %} - drop {{ relation_type }} if exists {{ relation }} - {% endcall %} -{%- endmacro %} - -{% macro selectdb__truncate_relation(relation) -%} - {% call statement('truncate_relation') %} - truncate table {{ relation }} - {% endcall %} -{%- endmacro %} - -{% macro selectdb__rename_relation(from_relation, to_relation) -%} - {% call statement('drop_relation') %} - drop {{ to_relation.type }} if exists {{ to_relation }} - {% endcall %} - {% call statement('rename_relation') %} - {% if to_relation.is_view %} - {% set results = run_query('show create view ' + from_relation.render() ) %} - create view {{ to_relation }} as {{ results[0]['Create View'].replace(from_relation.table, to_relation.table).split('AS',1)[1] }} - {% else %} - alter table {{ from_relation }} rename {{ to_relation.table }} - {% endif %} - {% endcall %} - - {% if to_relation.is_view %} - {% call statement('rename_relation_end_drop_old') %} - drop view if exists {{ from_relation }} - {% endcall %} - {% endif %} - - {%- endmacro %} - -{% macro selectdb__timestimp_id() -%} - {{ return( (modules.datetime.datetime.now() ~ "").replace('-','').replace(':','').replace('.','').replace(' ','') ) }} -{%- endmacro %} - -{% macro selectdb__with_label() -%} - {% set lable_suffix_id = config.get('label_id', validator=validation.any[basestring]) %} - {% if lable_suffix_id in [none,'DEFAULT'] %} - WITH LABEL dbt_selectdb_label_{{selectdb__timestimp_id()}} - {% else %} - WITH LABEL dbt_selectdb_label_{{ lable_suffix_id }} - {% endif %} -{%- endmacro %} - -{% macro selectdb__get_or_create_relation(database, schema, identifier, type) %} - {%- set target_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %} - - {% if target_relation %} - {% do return([true, target_relation]) %} - {% endif %} - - {%- set new_relation = api.Relation.create( - database=none, - schema=schema, - identifier=identifier, - type=type - ) -%} - {% do return([false, new_relation]) %} -{% endmacro %} - -{% macro catalog_source(catalog,database,table) -%} - `{{catalog}}`.`{{database}}`.`{{table}}` -{%- endmacro %} diff --git a/dbt/include/selectdb/macros/adapters/schema.sql b/dbt/include/selectdb/macros/adapters/schema.sql deleted file mode 100644 index 2950a4e..0000000 --- a/dbt/include/selectdb/macros/adapters/schema.sql +++ /dev/null @@ -1,13 +0,0 @@ --- selectdb have not 'schema' to make a collection of table or view -{% macro selectdb__drop_schema(relation) -%} - {%- call statement('drop_schema') -%} - drop database if exists {{ relation.without_identifier().include(database=False) }} - {%- endcall -%} -{% endmacro %} - - -{% macro selectdb__create_schema(relation) -%} - {%- call statement('create_schema') -%} - create database if not exists {{ relation.without_identifier().include(database=False) }} - {% endcall %} -{% endmacro %} diff --git a/dbt/include/selectdb/macros/materializations/incremental/incremental.sql b/dbt/include/selectdb/macros/materializations/incremental/incremental.sql deleted file mode 100644 index b5468e0..0000000 --- a/dbt/include/selectdb/macros/materializations/incremental/incremental.sql +++ /dev/null @@ -1,64 +0,0 @@ -{% materialization incremental, adapter='selectdb' %} - {% set unique_key = config.get('unique_key', validator=validation.any[list]) %} - {%- set inserts_only = config.get('inserts_only') -%} - - {% set target_relation = this.incorporate(type='table') %} - - - {% set existing_relation = load_relation(this) %} - {% set tmp_relation = make_temp_relation(this) %} - - {{ run_hooks(pre_hooks, inside_transaction=False) }} - - {{ run_hooks(pre_hooks, inside_transaction=True) }} - - {% set to_drop = [] %} - - {% if unique_key is none or inserts_only %} - {% set build_sql = tmp_insert(tmp_relation, target_relation, unique_key=none) %} - {% elif existing_relation is none %} - {% set build_sql = selectdb__create_unique_table_as(False, target_relation, sql) %} - {% elif existing_relation.is_view or should_full_refresh() %} - {#-- Make sure the backup doesn't exist so we don't encounter issues with the rename below #} - {% set backup_identifier = existing_relation.identifier ~ "__dbt_backup" %} - {% set backup_relation = existing_relation.incorporate(path={"identifier": backup_identifier}) %} - {% do adapter.drop_relation(backup_relation) %} - {% do adapter.rename_relation(target_relation, backup_relation) %} - {% set build_sql = selectdb__create_unique_table_as(False, target_relation, sql) %} - {% do to_drop.append(backup_relation) %} - {% else %} - {% set build_show_create = show_create( target_relation, statement_name="table_model") %} - {% call statement('table_model' , fetch_result=True) %} - {{ build_show_create }} - {% endcall %} - {%- set table_create_obj = load_result('table_model') -%} - {% if not is_unique_model(table_create_obj) %} - {% do exceptions.raise_compiler_error("selectdb table:"~ target_relation ~ ", model must be 'UNIQUE'" ) %} - {% endif %} - {% do run_query(create_table_as(True, tmp_relation, sql)) %} - {% do to_drop.append(tmp_relation) %} - - {% do adapter.expand_target_column_types( - from_relation=tmp_relation, - to_relation=target_relation) %} - {% set build_sql = tmp_insert(tmp_relation, target_relation, unique_key=unique_key) %} - {% endif %} - - {% call statement("main") %} - {{ build_sql }} - {% endcall %} - - - {% do persist_docs(target_relation, model) %} - {{ run_hooks(post_hooks, inside_transaction=True) }} - {% do adapter.commit() %} - {% for rel in to_drop %} - {% do selectdb__drop_relation(rel) %} - {% endfor %} - {{ run_hooks(post_hooks, inside_transaction=False) }} - {{ return({'relations': [target_relation]}) }} - -{%- endmaterialization %} - - - diff --git a/dbt/include/selectdb/macros/materializations/partition/replace.sql b/dbt/include/selectdb/macros/materializations/partition/replace.sql deleted file mode 100644 index 9188e16..0000000 --- a/dbt/include/selectdb/macros/materializations/partition/replace.sql +++ /dev/null @@ -1,9 +0,0 @@ -{% macro get_partition_replace_sql(relation, partitions) %} - {% for partition in partitions %} - {% set items = get_partition_items(partition) %} - {% set p = ''.join(items) %} - alter table {{ relation }} replace partition (p{{ p }}) with temporary partition (tp{{ p }}) properties ( - "strict_range" = "false" - ); - {% endfor %} -{% endmacro %} diff --git a/dbt/include/selectdb/macros/materializations/seed/helpers.sql b/dbt/include/selectdb/macros/materializations/seed/helpers.sql deleted file mode 100644 index 407d834..0000000 --- a/dbt/include/selectdb/macros/materializations/seed/helpers.sql +++ /dev/null @@ -1,29 +0,0 @@ - -{% macro selectdb__create_csv_table(model, agate_table) -%} - {% set column_override = model['config'].get('column_types', {}) %} - {% set quote_seed_column = model['config'].get('quote_columns', None) %} - - {% set sql %} - create table {{ this.render() }} - ( - {% for col_name in agate_table.column_names %} - {% set inferred_type = adapter.convert_type(agate_table, loop.index0) %} - {% set type = column_override.get(col_name, inferred_type) %} - {% set column_name = (col_name | string) %} - {{ adapter.quote_seed_column(column_name, quote_seed_column) }} {{ type }}{% if not loop.last %},{% endif %} - {% endfor %} - ) - {{ selectdb__engine() }} - {{ selectdb__duplicate_key() }} - {{ selectdb__partition_by() }} - {{ selectdb__distributed_by(agate_table.column_names[0:1]) }} - {{ selectdb__properties() }} - {% endset %} - - {% call statement('_') %} - {{ sql }} - {% endcall %} - - {{ return(sql) }} - -{%- endmacro %} diff --git a/dbt/include/selectdb/macros/materializations/snapshot/strategies.sql b/dbt/include/selectdb/macros/materializations/snapshot/strategies.sql deleted file mode 100644 index 67aad91..0000000 --- a/dbt/include/selectdb/macros/materializations/snapshot/strategies.sql +++ /dev/null @@ -1,6 +0,0 @@ -{% macro selectdb__snapshot_hash_arguments(args) -%} - md5(concat_ws('|', {%- for arg in args -%} - coalesce(cast({{ arg }} as char), '') - {% if not loop.last %}, {% endif %} - {%- endfor -%})) -{%- endmacro %} diff --git a/dbt/include/selectdb/macros/materializations/table/create_table_as.sql b/dbt/include/selectdb/macros/materializations/table/create_table_as.sql deleted file mode 100644 index 2346469..0000000 --- a/dbt/include/selectdb/macros/materializations/table/create_table_as.sql +++ /dev/null @@ -1,26 +0,0 @@ -{% macro selectdb__create_table_as(temporary, relation, sql) -%} - {% set sql_header = config.get('sql_header', none) %} - {% set table = relation.include(database=False) %} - {{ sql_header if sql_header is not none }} - {%if temporary %} - {{selectdb__drop_relation(relation)}} - {% endif %} - create table {{ table }} - {{ selectdb__duplicate_key() }} - {{ selectdb__partition_by() }} - {{ selectdb__distributed_by() }} - {{ selectdb__properties() }} as {{ sql }}; - -{%- endmacro %} - -{% macro selectdb__create_unique_table_as(temporary, relation, sql) -%} - {% set sql_header = config.get('sql_header', none) %} - {% set table = relation.include(database=False) %} - {{ sql_header if sql_header is not none }} - create table {{ table }} - {{ selectdb__unique_key() }} - {{ selectdb__partition_by() }} - {{ selectdb__distributed_by() }} - {{ selectdb__properties() }} as {{ sql }}; - -{%- endmacro %} \ No newline at end of file diff --git a/dbt/include/selectdb/macros/materializations/view/create_view_as.sql b/dbt/include/selectdb/macros/materializations/view/create_view_as.sql deleted file mode 100644 index 90873f1..0000000 --- a/dbt/include/selectdb/macros/materializations/view/create_view_as.sql +++ /dev/null @@ -1,6 +0,0 @@ -{% macro selectdb__create_view_as(relation, sql) -%} - {%- set sql_header = config.get('sql_header', none) -%} - - {{ sql_header if sql_header is not none }} - create view {{ relation }} as {{ sql }}; -{%- endmacro %} diff --git a/dbt/include/selectdb/profile_template.yml b/dbt/include/selectdb/profile_template.yml deleted file mode 100644 index a2d87f4..0000000 --- a/dbt/include/selectdb/profile_template.yml +++ /dev/null @@ -1,20 +0,0 @@ -fixed: - type: selectdb -prompts: - host: - hint: 'endpoint for your instance(your SelectDB Endpoint Details)' - port: - type: 'int' - hint: 'port for your instance(your SelectDB MySQL Protocol Port)' - schema: - default: 'dbt' - hint: 'the schema name as stored in the database,selectdb have not schema to make a collection of table or view' - username: - hint: 'your selectdb username(not your selectdb account login user)' - password: - hint: 'your selectdb password' - hide_input: true - threads: - hint: "1 or more" - type: "int" - default: 1 diff --git a/setup.py b/setup.py index 97ff5af..b76ee20 100644 --- a/setup.py +++ b/setup.py @@ -1,25 +1,26 @@ #!/usr/bin/env python from setuptools import find_namespace_packages, setup -package_name = "dbt-selectdb" +package_name = "dbt-doris" # make sure this always matches dbt/adapters/{adapter}/__version__.py -package_version = "0.1.0" -dbt_core_version = "1.3.0" -description = """The selectdb cloud adapter plugin for dbt, Original code fork from Apache Doris""" +package_version = "0.3.4" +dbt_core_version = "1.5.0" +description = """The doris adapter plugin for dbt """ setup( name=package_name, version=package_version, description=description, long_description=description, - author="catpineapple,JNSimba", + author="long2ice,catpineapple,JNSimba", author_email="1391869588@qq.com", - url="https://github.com/selectdb/dbt-selectdb.git", + url="https://github.com/selectdb/dbt-doris", packages=find_namespace_packages(include=["dbt", "dbt.*"]), include_package_data=True, install_requires=[ "dbt-core~={}".format(dbt_core_version), "mysql-connector-python>=8.0.0,<8.1", + "urllib3~=1.0", ], - python_requires=">=3.8,<=3.10", + python_requires=">=3.7.2", ) diff --git a/test/__init__.py b/test/__init__.py new file mode 100644 index 0000000..5542f6c --- /dev/null +++ b/test/__init__.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. diff --git a/test/conftest.py b/test/conftest.py new file mode 100644 index 0000000..e12af32 --- /dev/null +++ b/test/conftest.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import pytest + +import os +import json + +# Import the fuctional fixtures as a plugin +# Note: fixtures with session scope need to be local + +pytest_plugins = ["dbt.tests.fixtures.project"] + + +# The profile dictionary, used to write out profiles.yml +@pytest.fixture(scope="class") +def dbt_profile_target(): + return { + "type": "doris", + "threads": 1, + "host": os.getenv("DORIS_TEST_HOST", "127.0.0.1"), + "user": os.getenv("DORIS_TEST_USER", "root"), + "password": os.getenv("DORIS_TEST_PASSWORD", ""), + "port": os.getenv("DORIS_TEST_PORT", 9030), + } diff --git a/tests/functional/adapter/test_basic.py b/test/functional/adapter/test_basic.py similarity index 66% rename from tests/functional/adapter/test_basic.py rename to test/functional/adapter/test_basic.py index 1339183..2f6ee1d 100644 --- a/tests/functional/adapter/test_basic.py +++ b/test/functional/adapter/test_basic.py @@ -1,3 +1,23 @@ +#!/usr/bin/env python +# encoding: utf-8 + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + import pytest from dbt.tests.adapter.basic.test_base import BaseSimpleMaterializations @@ -15,7 +35,7 @@ from dbt.tests.adapter.basic.test_adapter_methods import BaseAdapterMethod -class TestSimpleMaterializationsselectdb(BaseSimpleMaterializations): +class TestSimpleMaterializationsdoris(BaseSimpleMaterializations): def test_base(self, project): results = run_dbt(["seed"]) assert len(results) == 1 @@ -39,16 +59,16 @@ def test_base(self, project): check_relations_equal(project.adapter, ["base", "view_model", "table_model", "swappable"]) -class TestSingularTestsselectdb(BaseSingularTests): +class TestSingularTestsdoris(BaseSingularTests): pass -class TestSingularTestsEphemeralselectdb(BaseSingularTestsEphemeral): +class TestSingularTestsEphemeraldoris(BaseSingularTestsEphemeral): pass -class TestEmptyselectdb(BaseEmpty): +class TestEmptydoris(BaseEmpty): pass -class TestEphemeralselectdb(BaseEphemeral): +class TestEphemeraldoris(BaseEphemeral): def test_ephemeral(self, project): results = run_dbt(["seed"]) assert len(results) == 1 @@ -62,8 +82,8 @@ def test_ephemeral(self, project): check_relations_equal(project.adapter, ["base", "view_model", "table_model"]) -@pytest.mark.skip(reason="Incremental for selectdb table model bust be 'unique' ") -class TestIncrementalselectdb(BaseIncremental): +@pytest.mark.skip(reason="Incremental for doris table model bust be 'unique' ") +class TestIncrementaldoris(BaseIncremental): def test_incremental(self, project): results = run_dbt(["seed"]) assert len(results) == 2 @@ -77,18 +97,18 @@ def test_incremental(self, project): assert len(results) == 1 check_relations_equal(project.adapter, ["base", "incremental"]) -class TestGenericTestsselectdb(BaseGenericTests): +class TestGenericTestsdoris(BaseGenericTests): pass -@pytest.mark.skip(reason="Snapshot for selectdb table model bust be 'unique'") -class TestSnapshotCheckColsselectdb(BaseSnapshotCheckCols): +@pytest.mark.skip(reason="Snapshot for doris table model bust be 'unique'") +class TestSnapshotCheckColsdoris(BaseSnapshotCheckCols): pass -@pytest.mark.skip(reason="Snapshot for selectdb table model bust be 'unique'") -class TestSnapshotTimestampselectdb(BaseSnapshotTimestamp): +@pytest.mark.skip(reason="Snapshot for doris table model bust be 'unique'") +class TestSnapshotTimestampdoris(BaseSnapshotTimestamp): pass -class TestBaseAdapterMethodselectdb(BaseAdapterMethod): +class TestBaseAdapterMethoddoris(BaseAdapterMethod): def test_adapter_methods(self, project, equal_tables): result = run_dbt() assert len(result) == 3 diff --git a/tests/__init__.py b/tests/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index ffbe162..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,22 +0,0 @@ -import pytest - -import os -import json - -# Import the fuctional fixtures as a plugin -# Note: fixtures with session scope need to be local - -pytest_plugins = ["dbt.tests.fixtures.project"] - - -# The profile dictionary, used to write out profiles.yml -@pytest.fixture(scope="class") -def dbt_profile_target(): - return { - "type": "selectdb", - "threads": 1, - "host": os.getenv("SELECTDB_TEST_HOST", "127.0.0.1"), - "user": os.getenv("SELECTDB_TEST_USER", "root"), - "password": os.getenv("SELECTDB_TEST_PASSWORD", ""), - "port": os.getenv("SELECTDB_TEST_PORT", 9030), - } diff --git a/tox.ini b/tox.ini index fec16bd..470681d 100644 --- a/tox.ini +++ b/tox.ini @@ -12,13 +12,13 @@ deps = -e. -[testenv:{integration,py37,py38,py39,py}-{ selectdb }] +[testenv:{integration,py37,py38,py39,py}-{ doris }] description = adapter plugin integration testing skip_install = true -passenv = DBT_* SELECTDB_TEST_* PYTEST_ADOPTS +passenv = DBT_* DORIS_TEST_* PYTEST_ADOPTS commands = - selectdb: {envpython} -m pytest -m profile_selectdb {posargs:test/integration} - selectdb: {envpython} -m pytest {posargs} tests/functional + doris: {envpython} -m pytest -m profile_doris {posargs:test/integration} + doris: {envpython} -m pytest {posargs} tests/functional deps = -rdev_requirements.txt -e.