Skip to content

Commit f410d8d

Browse files
arshadmohammadarshadmohammad
authored
[Feature] Add Seatunnel Ambari MPack Support (#290)
1 parent 2abcb21 commit f410d8d

35 files changed

+2490
-0
lines changed

‎pom.xml

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@
3838
<module>seatunnel-datasource</module>
3939
<module>seatunnel-web-dist</module>
4040
<module>seatunnel-web-it</module>
41+
<module>seatunnel-mpack</module>
4142
</modules>
4243

4344
<properties>

‎seatunnel-mpack/README.md

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
## Seaatunnel Ambari MPack Installation Guide
2+
3+
**Note:** Replace `versions`, `paths`, and `urls` with values applicable to your environment.
4+
### Stop the Ambari server:
5+
```bash
6+
ambari-server stop
7+
```
8+
### Uninstall MPack, if already installed
9+
```bash
10+
ambari-server uninstall-mpack --mpack-name seatunnel-mpack --verbose
11+
```
12+
### Install the Seatunnel MPack on Ambari Server:
13+
Execute the following command to install the Seatunnel MPack on the Ambari server.
14+
15+
```bash
16+
ambari-server install-mpack --mpack=${mpack_url} --verbose
17+
# Example:
18+
ambari-server install-mpack --mpack=http://localhost:8080/myrepo/mpack-mpack-2.3.8.tar.gz --verbose
19+
20+
# Specify the local path to the MPack tarball if it is available on your system:
21+
# Example:
22+
ambari-server install-mpack --mpack=/some/path/mpack-mpack-${mpack_version}.tar.gz --verbose
23+
```
24+
25+
### Install Seatunnel MPack on Ambari agents:
26+
Ambari server does not automatically copy the MPack to all agents. Manually copy the MPack to all agents.
27+
Commands for reference:
28+
```bash
29+
# From Ambari server machine:
30+
stack-version=3.3.0
31+
cp -r /var/lib/ambari-server/resources/stacks/BIGTOP/${stack-version}/services/SEATUNNEL /var/lib/ambari-agent/cache/stacks/BIGTOP/${stack-version}/services/
32+
scp -r /var/lib/ambari-server/resources/stacks/BIGTOP/${stack-version}/services/SEATUNNEL username@agent_host2:/var/lib/ambari-agent/cache/stacks/BIGTOP/${stack-version}/services/
33+
scp -r /var/lib/ambari-server/resources/stacks/BIGTOP/${stack-version}/services/SEATUNNEL username@agent_host3:/var/lib/ambari-agent/cache/stacks/BIGTOP/${stack-version}/services/
34+
```
35+
36+
### Start the Ambari server:
37+
```bash
38+
ambari-server start
39+
```

‎seatunnel-mpack/pom.xml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one or more
4+
contributor license agreements. See the NOTICE file distributed with
5+
this work for additional information regarding copyright ownership.
6+
The ASF licenses this file to You under the Apache License, Version 2.0
7+
(the "License"); you may not use this file except in compliance with
8+
the License. You may obtain a copy of the License at
9+
10+
http://www.apache.org/licenses/LICENSE-2.0
11+
12+
Unless required by applicable law or agreed to in writing, software
13+
distributed under the License is distributed on an "AS IS" BASIS,
14+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15+
See the License for the specific language governing permissions and
16+
limitations under the License.
17+
-->
18+
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
20+
<modelVersion>4.0.0</modelVersion>
21+
<parent>
22+
<groupId>org.apache.seatunnel</groupId>
23+
<artifactId>seatunnel-web</artifactId>
24+
<version>${revision}</version>
25+
</parent>
26+
27+
<artifactId>seatunnel-mpack</artifactId>
28+
<packaging>pom</packaging>
29+
<name>SeaTunnel Ambari MPack</name>
30+
31+
<build>
32+
<plugins>
33+
<plugin>
34+
<artifactId>maven-assembly-plugin</artifactId>
35+
<configuration>
36+
<finalName>seatunnel-mpack-${project.version}</finalName>
37+
<appendAssemblyId>false</appendAssemblyId>
38+
<descriptors>
39+
<descriptor>src/main/assemblies/seatunnel-mpack.xml</descriptor>
40+
</descriptors>
41+
</configuration>
42+
<executions>
43+
<execution>
44+
<id>build-tarball</id>
45+
<goals>
46+
<goal>single</goal>
47+
</goals>
48+
<phase>package</phase>
49+
</execution>
50+
</executions>
51+
</plugin>
52+
</plugins>
53+
</build>
54+
55+
</project>

‎seatunnel-mpack/src/main/assemblies/seatunnel-mpack.xml

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
<!--
2+
Licensed to the Apache Software Foundation (ASF) under one or more
3+
contributor license agreements. See the NOTICE file distributed with
4+
this work for additional information regarding copyright ownership.
5+
The ASF licenses this file to You under the Apache License, Version 2.0
6+
(the "License"); you may not use this file except in compliance with
7+
the License. You may obtain a copy of the License at
8+
9+
http://www.apache.org/licenses/LICENSE-2.0
10+
11+
Unless required by applicable law or agreed to in writing, software
12+
distributed under the License is distributed on an "AS IS" BASIS,
13+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
See the License for the specific language governing permissions and
15+
limitations under the License.
16+
-->
17+
<assembly xmlns="http://maven.apache.org/ASSEMBLY/2.1.0"
18+
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
19+
xsi:schemaLocation="http://maven.apache.org/ASSEMBLY/2.1.0 http://maven.apache.org/xsd/assembly-2.1.0.xsd">
20+
<id>archive</id>
21+
<formats>
22+
<format>tar.gz</format>
23+
</formats>
24+
<includeBaseDirectory>true</includeBaseDirectory>
25+
<fileSets>
26+
<fileSet>
27+
<directory>src/main/resources/addon-services</directory>
28+
<outputDirectory>addon-services</outputDirectory>
29+
<includes>
30+
<include>**</include>
31+
</includes>
32+
</fileSet>
33+
<fileSet>
34+
<directory>src/main/resources/extensions</directory>
35+
<outputDirectory>extensions</outputDirectory>
36+
<includes>
37+
<include>**</include>
38+
</includes>
39+
</fileSet>
40+
<fileSet>
41+
<directory>src/main/resources/extensions</directory>
42+
<outputDirectory>extensions</outputDirectory>
43+
<includes>
44+
<include>**</include>
45+
</includes>
46+
</fileSet>
47+
<fileSet>
48+
<directory>src/main/resources</directory>
49+
<outputDirectory>.</outputDirectory>
50+
<includes>
51+
<include>mpack.json</include>
52+
</includes>
53+
</fileSet>
54+
</fileSets>
55+
</assembly>

‎seatunnel-mpack/src/main/resources/addon-services/SEATUNNEL/2.3.8/configuration/ranger-seatunnel-audit.xml

Lines changed: 133 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,133 @@
1+
<?xml version="1.0"?>
2+
<!--
3+
/**
4+
* Licensed to the Apache Software Foundation (ASF) under one
5+
* or more contributor license agreements. See the NOTICE file
6+
* distributed with this work for additional information
7+
* regarding copyright ownership. The ASF licenses this file
8+
* to you under the Apache License, Version 2.0 (the
9+
* "License"); you may not use this file except in compliance
10+
* with the License. You may obtain a copy of the License at
11+
*
12+
* http://www.apache.org/licenses/LICENSE-2.0
13+
*
14+
* Unless required by applicable law or agreed to in writing, software
15+
* distributed under the License is distributed on an "AS IS" BASIS,
16+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
17+
* See the License for the specific language governing permissions and
18+
* limitations under the License.
19+
*/
20+
-->
21+
<configuration>
22+
<property>
23+
<name>xasecure.audit.is.enabled</name>
24+
<value>true</value>
25+
<description>Is Audit enabled?</description>
26+
<value-attributes>
27+
<type>boolean</type>
28+
</value-attributes>
29+
<on-ambari-upgrade add="false"/>
30+
</property>
31+
<property>
32+
<name>xasecure.audit.destination.hdfs</name>
33+
<value>true</value>
34+
<display-name>Audit to HDFS</display-name>
35+
<description>Is Audit to HDFS enabled?</description>
36+
<value-attributes>
37+
<type>boolean</type>
38+
</value-attributes>
39+
<depends-on>
40+
<property>
41+
<type>ranger-env</type>
42+
<name>xasecure.audit.destination.hdfs</name>
43+
</property>
44+
</depends-on>
45+
<on-ambari-upgrade add="false"/>
46+
</property>
47+
<property>
48+
<name>xasecure.audit.destination.hdfs.dir</name>
49+
<value>hdfs://NAMENODE_HOSTNAME:8020/ranger/audit</value>
50+
<description>HDFS folder to write audit to, make sure the service user has requried permissions</description>
51+
<depends-on>
52+
<property>
53+
<type>ranger-env</type>
54+
<name>xasecure.audit.destination.hdfs.dir</name>
55+
</property>
56+
</depends-on>
57+
<on-ambari-upgrade add="false"/>
58+
</property>
59+
<property>
60+
<name>xasecure.audit.destination.hdfs.batch.filespool.dir</name>
61+
<value>/var/log/seatunnel-web/audit/hdfs/spool</value>
62+
<description>/var/log/seatunnel-web/audit/hdfs/spool</description>
63+
<on-ambari-upgrade add="false"/>
64+
</property>
65+
<property>
66+
<name>xasecure.audit.destination.solr</name>
67+
<value>false</value>
68+
<display-name>Audit to SOLR</display-name>
69+
<description>Is Solr audit enabled?</description>
70+
<value-attributes>
71+
<type>boolean</type>
72+
</value-attributes>
73+
<depends-on>
74+
<property>
75+
<type>ranger-env</type>
76+
<name>xasecure.audit.destination.solr</name>
77+
</property>
78+
</depends-on>
79+
<on-ambari-upgrade add="false"/>
80+
</property>
81+
<property>
82+
<name>xasecure.audit.destination.solr.urls</name>
83+
<value/>
84+
<description>Solr URL</description>
85+
<value-attributes>
86+
<empty-value-valid>true</empty-value-valid>
87+
</value-attributes>
88+
<depends-on>
89+
<property>
90+
<type>ranger-admin-site</type>
91+
<name>ranger.audit.solr.urls</name>
92+
</property>
93+
</depends-on>
94+
<on-ambari-upgrade add="false"/>
95+
</property>
96+
<property>
97+
<name>xasecure.audit.destination.solr.zookeepers</name>
98+
<value>NONE</value>
99+
<description>Solr Zookeeper string</description>
100+
<depends-on>
101+
<property>
102+
<type>ranger-admin-site</type>
103+
<name>ranger.audit.solr.zookeepers</name>
104+
</property>
105+
</depends-on>
106+
<on-ambari-upgrade add="false"/>
107+
</property>
108+
<property>
109+
<name>xasecure.audit.destination.solr.batch.filespool.dir</name>
110+
<value>/var/log/seatunnel-web/audit/solr/spool</value>
111+
<description>/var/log/seatunnel-web/audit/solr/spool</description>
112+
<on-ambari-upgrade add="false"/>
113+
</property>
114+
<property>
115+
<name>xasecure.audit.provider.summary.enabled</name>
116+
<value>false</value>
117+
<display-name>Audit provider summary enabled</display-name>
118+
<description>Enable Summary audit?</description>
119+
<value-attributes>
120+
<type>boolean</type>
121+
</value-attributes>
122+
<on-ambari-upgrade add="false"/>
123+
</property>
124+
<property>
125+
<name>ranger.plugin.seatunnel.ambari.cluster.name</name>
126+
<value>{{cluster_name}}</value>
127+
<description>Capture cluster name from where Ranger seatunnel plugin is enabled.</description>
128+
<value-attributes>
129+
<empty-value-valid>true</empty-value-valid>
130+
</value-attributes>
131+
<on-ambari-upgrade add="false"/>
132+
</property>
133+
</configuration>

0 commit comments

Comments
 (0)