add kettle plugins

JNSimba · Sep 25, 2024 · d106fad · d106fad
1 parent f8a40ad
commit d106fad
Show file tree

Hide file tree

Showing 38 changed files with 3,816 additions and 0 deletions.
diff --git a/extension/kettle/README.md b/extension/kettle/README.md
@@ -0,0 +1,42 @@
+# kettle-plugin
+
+1. Download and install kettle
+Download: https://pentaho.com/download/#download-pentaho
+
+After downloading, unzip it and run spoon.sh to start kettle.
+
+You can also compile it yourself, refer to the [compilation section](https://github.com/pentaho/pentaho-kettle?tab=readme-ov-file#how-to-build)
+
+2. Compile doris-stream-loader
+```shell
+cd doris-stream-loader
+mvn clean package -DskipTests
+```
+After the compilation is complete, unzip the plugin package and copy it to the plugins directory of kettle
+```shell
+cd assemblies/plugin/target
+unzip doris-stream-loader-plugins-9.4.0.0-343.zip 
+cp -r doris-stream-loader ${KETTLE_HOME}/plugins/
+mvn clean package -DskipTests
+```
+3. Build job
+Find Doris Stream Loader in Batch Loading in Kettle and build the job
+![create.png](images/create.png)
+
+4. Click Start to run the job to complete data synchronization
+![running.png](images/running.png)
+
+5. Configuration
+
+| Key                           | Default Value  | Required | Comment                                                                 |
+|-------------------------------|----------------| -------- |-------------------------------------------------------------------------|
+| Step name                     | --             | Y        | Step name                                                               |
+| fenodes                       | --             | Y        | Doris FE http address, supports multiple addresses, separated by commas |
+| DataBase                      | --             | Y        | Doris's write database                                                  |
+| Table                         | --             | Y        | Doris's write table                                                     |
+| User                          | --             | Y        | Username to access Doris                                                |
+| Password                      | --             | N        | Password to access Doris                                                |
+| Maximum rows for load | 10000          | N        | Maximum number of rows to load at a time                                |
+| Maximum bytes for load | 10485760(10MB) | N        | Maximum size in bytes of a single load                                  |
+| Load retries                  | 3              | N        | Number of retries after load failure                                    |
+| StreamLoad Properties         | --             | N        | Streamload http header for request                                      |
diff --git a/extension/kettle/README_zh.md b/extension/kettle/README_zh.md
@@ -0,0 +1,42 @@
+# kettle-plugin
+
+1. 下载安装kettle
+下载地址： https://pentaho.com/download/#download-pentaho
+
+下载后解压，运行spoon.sh即可启动kettle
+
+也可以自行编译，参考[编译章节](https://github.com/pentaho/pentaho-kettle?tab=readme-ov-file#how-to-build)
+
+2. 编译doris-stream-loader
+```shell
+cd doris-stream-loader
+mvn clean package -DskipTests
+```
+编译完成后，将插件包解压后拷贝到kettle的plugins目录下
+```shell
+cd assemblies/plugin/target
+unzip doris-stream-loader-plugins-9.4.0.0-343.zip 
+cp -r doris-stream-loader ${KETTLE_HOME}/plugins/
+mvn clean package -DskipTests
+```
+3. 构建作业
+在Kettle中的批量加载中找到Doris Stream Loader，构建作业
+![create.png](images/create.png)
+
+4. 点击开始运行作业即可完成数据同步
+![running.png](images/running.png)
+
+5. 参数说明
+
+| Key          | Default Value  | Required | Comment                        |
+|--------------|----------------| -------- |--------------------------------|
+| Step name         | --             | Y        | 步骤名称                           |
+| fenodes      | --             | Y        | Doris FE http 地址，支持多个地址，使用逗号分隔 |
+| 数据库          | --             | Y        | Doris 的写入数据库                   |
+| 目标表          | --             | Y        | Doris 的写入表                     |
+| 用户名          | --             | Y        | 访问 Doris 的用户名                  |
+| 密码           | --             | N        | 访问 Doris 的密码                   |
+| 单次导入最大行数     | 10000          | N        | 单次导入的最大行数                      |
+| 单次导入最大字节     | 10485760(10MB) | N        | 单次导入的最大字节大小                    |
+| 导入重试次数       | 3              | N        | 导入失败之后的重试次数                    |
+| StreamLoad属性 | --             | N        | Streamload的请求头                 |
diff --git a/extension/kettle/assemblies/plugin/pom.xml b/extension/kettle/assemblies/plugin/pom.xml
@@ -0,0 +1,73 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.pentaho.di.plugins</groupId>
+    <artifactId>doris-stream-loader-assemblies</artifactId>
+    <version>9.4.0.0-343</version>
+  </parent>
+
+  <artifactId>doris-stream-loader-plugins</artifactId>
+  <version>9.4.0.0-343</version>
+  <packaging>pom</packaging>
+
+  <name>PDI Doris Stream Loader Plugins Distribution</name>
+  <description></description>
+
+  <dependencies>
+    <dependency>
+      <groupId>org.pentaho.di.plugins</groupId>
+      <artifactId>doris-stream-loader-impl</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+    <dependency>
+      <groupId>org.pentaho.di.plugins</groupId>
+      <artifactId>doris-stream-loader-ui</artifactId>
+      <version>${project.version}</version>
+    </dependency>
+  </dependencies>
+
+  <build>
+    <plugins>
+      <plugin>
+        <artifactId>maven-dependency-plugin</artifactId>
+        <executions>
+          <execution>
+            <id>add-libs</id>
+            <phase>generate-resources</phase>
+            <goals>
+              <goal>copy-dependencies</goal>
+            </goals>
+            <configuration>
+              <outputDirectory>${project.build.directory}/assembly/lib</outputDirectory>
+              <includeScope>compile</includeScope>
+              <excludeArtifactIds>doris-stream-loader-impl,doris-stream-loader-ui</excludeArtifactIds>
+            </configuration>
+          </execution>
+        </executions>
+      </plugin>
+    </plugins>
+  </build>
+
+</project>
diff --git a/extension/kettle/assemblies/plugin/src/assembly/assembly.xml b/extension/kettle/assemblies/plugin/src/assembly/assembly.xml
@@ -0,0 +1,37 @@
+<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+          xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.0.0 http://maven.apache.org/xsd/assembly-2.0.0.xsd">
+  <id>doris-bulk-loader</id>
+  <formats>
+    <format>zip</format>
+  </formats>
+  <baseDirectory>doris-stream-loader</baseDirectory>
+
+  <files>
+    <file>
+      <source>${project.basedir}/src/main/resources/version.xml</source>
+      <outputDirectory>.</outputDirectory>
+      <filtered>true</filtered>
+    </file>
+  </files>
+
+  <fileSets>
+    <!-- the staging dir -->
+    <fileSet>
+      <directory>${project.build.directory}/assembly</directory>
+      <outputDirectory>.</outputDirectory>
+    </fileSet>
+  </fileSets>
+
+  <dependencySets>
+    <dependencySet>
+      <outputDirectory>.</outputDirectory>
+      <includes>
+        <include>org.pentaho.di.plugins:doris-stream-loader-impl:jar</include>
+        <include>org.pentaho.di.plugins:doris-stream-loader-ui:jar</include>
+      </includes>
+      <useProjectArtifact>false</useProjectArtifact>
+    </dependencySet>
+  </dependencySets>
+</assembly>
+
+
diff --git a/extension/kettle/assemblies/plugin/src/main/resources/version.xml b/extension/kettle/assemblies/plugin/src/main/resources/version.xml
@@ -0,0 +1,2 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<version>${project.version}</version>
diff --git a/extension/kettle/assemblies/pom.xml b/extension/kettle/assemblies/pom.xml
@@ -0,0 +1,40 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.pentaho.di.plugins</groupId>
+    <artifactId>doris-stream-loader</artifactId>
+    <version>9.4.0.0-343</version>
+  </parent>
+
+  <artifactId>doris-stream-loader-assemblies</artifactId>
+  <version>9.4.0.0-343</version>
+  <packaging>pom</packaging>
+
+  <name>PDI Doris  Stream Loader Assemblies</name>
+
+  <modules>
+    <module>plugin</module>
+  </modules>
+</project>
diff --git a/extension/kettle/images/create.png b/extension/kettle/images/create.png
diff --git a/extension/kettle/images/running.png b/extension/kettle/images/running.png
diff --git a/extension/kettle/impl/pom.xml b/extension/kettle/impl/pom.xml
@@ -0,0 +1,122 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+Licensed to the Apache Software Foundation (ASF) under one
+or more contributor license agreements.  See the NOTICE file
+distributed with this work for additional information
+regarding copyright ownership.  The ASF licenses this file
+to you under the Apache License, Version 2.0 (the
+"License"); you may not use this file except in compliance
+with the License.  You may obtain a copy of the License at
+
+  http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing,
+software distributed under the License is distributed on an
+"AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+KIND, either express or implied.  See the License for the
+specific language governing permissions and limitations
+under the License.
+-->
+<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"
+         xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+  <modelVersion>4.0.0</modelVersion>
+
+  <parent>
+    <groupId>org.pentaho.di.plugins</groupId>
+    <artifactId>doris-stream-loader</artifactId>
+    <version>9.4.0.0-343</version>
+  </parent>
+
+  <artifactId>doris-stream-loader-impl</artifactId>
+  <name>PDI Doris Stream Loader Impl</name>
+
+  <properties>
+    <slf4j.version>1.7.32</slf4j.version>
+    <log4j.version>2.17.2</log4j.version>
+  </properties>
+
+  <dependencies>
+    <dependency>
+      <groupId>pentaho-kettle</groupId>
+      <artifactId>kettle-core</artifactId>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>pentaho-kettle</groupId>
+      <artifactId>kettle-engine</artifactId>
+      <scope>provided</scope>
+    </dependency>
+
+    <!-- Test dependencies -->
+    <dependency>
+      <groupId>org.mockito</groupId>
+      <artifactId>mockito-all</artifactId>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>pentaho-kettle</groupId>
+      <artifactId>kettle-core</artifactId>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+    <dependency>
+      <groupId>pentaho-kettle</groupId>
+      <artifactId>kettle-engine</artifactId>
+      <classifier>tests</classifier>
+      <scope>test</scope>
+    </dependency>
+
+    <!--log-->
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>slf4j-api</artifactId>
+      <version>${slf4j.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jul-to-slf4j</artifactId>
+      <version>${slf4j.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.slf4j</groupId>
+      <artifactId>jcl-over-slf4j</artifactId>
+      <version>${slf4j.version}</version>
+      <scope>provided</scope>
+    </dependency>
+
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-api</artifactId>
+      <version>${log4j.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-core</artifactId>
+      <version>${log4j.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-slf4j-impl</artifactId>
+      <version>${log4j.version}</version>
+      <scope>provided</scope>
+    </dependency>
+    <dependency>
+      <groupId>log4j</groupId>
+      <artifactId>log4j</artifactId>
+      <version>1.2.17</version>
+    </dependency>
+
+    <dependency>
+      <!-- API bridge between log4j 1 and 2 -->
+      <groupId>org.apache.logging.log4j</groupId>
+      <artifactId>log4j-1.2-api</artifactId>
+      <version>${log4j.version}</version>
+      <scope>provided</scope>
+    </dependency>
+  </dependencies>
+</project>