aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorOlaf Flebbe <of@oflebbe.de>2015-05-09 20:43:37 +0200
committerOlaf Flebbe <of@oflebbe.de>2015-05-09 20:43:37 +0200
commit52077c6250925b8f470f0027ea79b077190e8e16 (patch)
tree5d96536242fb4b0af3db66f5c6c6c832356fd6c7
parentc2d39603b8148116e68e3d6a3036857fd5480cd4 (diff)
parent1540c4e28c5df593ec6101135eb288a76165ea04 (diff)
downloadbigtop-odpi-52077c6250925b8f470f0027ea79b077190e8e16.tar.gz
Merge branch 'master' of ssh://diskstation/volume1/git/bigtop
-rw-r--r--bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml4
-rw-r--r--bigtop-deploy/puppet/hieradata/site.yaml1
-rw-r--r--bigtop-deploy/puppet/manifests/cluster.pp8
-rw-r--r--bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml7
-rw-r--r--bigtop-deploy/puppet/modules/hadoop/manifests/init.pp2
-rw-r--r--bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh12
-rw-r--r--bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml17
-rw-r--r--bigtop-deploy/puppet/modules/tez/manifests/init.pp22
-rw-r--r--bigtop-deploy/puppet/modules/tez/tests/init.pp17
-rw-r--r--bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy48
-rw-r--r--bigtop-packages/src/common/hadoop/init-hcfs.json5
-rw-r--r--bigtop-packages/src/rpm/hue/SPECS/hue.spec2
-rw-r--r--bigtop_toolchain/manifests/deps.pp4
-rw-r--r--bigtop_toolchain/manifests/gradle.pp10
-rw-r--r--gradle/wrapper/gradle-wrapper.properties2
-rw-r--r--packages.gradle9
16 files changed, 146 insertions, 24 deletions
diff --git a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
index 1ea64078..44cb1b77 100644
--- a/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
+++ b/bigtop-deploy/puppet/hieradata/bigtop/cluster.yaml
@@ -137,3 +137,7 @@ hue::server::kerberos_realm: "%{hiera('kerberos::site::realm')}"
giraph::client::zookeeper_quorum: "%{hiera('bigtop::hadoop_head_node')}"
hadoop-hive::client::hbase_zookeeper_quorum: "%{hiera('hadoop-hbase::common_config::zookeeper_quorum')}"
+
+# tez
+hadoop::common::tez_conf_dir: "/etc/tez/conf"
+hadoop::common::tez_jars: "/usr/lib/tez"
diff --git a/bigtop-deploy/puppet/hieradata/site.yaml b/bigtop-deploy/puppet/hieradata/site.yaml
index 54060f82..00861c44 100644
--- a/bigtop-deploy/puppet/hieradata/site.yaml
+++ b/bigtop-deploy/puppet/hieradata/site.yaml
@@ -29,6 +29,7 @@ hadoop::hadoop_storage_dirs:
# - spark
# - sqoop
# - tachyon
+# - tez
# - yarn
# - zookeeper
diff --git a/bigtop-deploy/puppet/manifests/cluster.pp b/bigtop-deploy/puppet/manifests/cluster.pp
index 43e090f7..35ef1956 100644
--- a/bigtop-deploy/puppet/manifests/cluster.pp
+++ b/bigtop-deploy/puppet/manifests/cluster.pp
@@ -147,6 +147,11 @@ if ($hadoop_security_authentication == "kerberos") {
include hadoop-zookeeper::server
}
+ if ($all or "tez" in $components) {
+ include tez::client
+ Class['tez::client'] -> Exec<| title == "init hdfs" |>
+ }
+
# class hadoop::rsync_hdfs isn't used anywhere
#Exec<| title == "init hdfs" |> -> Class['Hadoop::Rsync_hdfs']
}
@@ -203,4 +208,7 @@ class hadoop_gateway_node inherits hadoop_cluster_node {
if ($all or "zookeeper" in $components) {
include hadoop-zookeeper::client
}
+ if ($all or "tez" in $components) {
+ include tez::client
+ }
}
diff --git a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
index 5861d971..4e339077 100644
--- a/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop-hive/templates/hive-site.xml
@@ -42,6 +42,13 @@
</property>
<% end %>
+<% if @all or @components.include? "tez" -%>
+<property>
+ <name>hive.execution.engine</name>
+ <value>tez</value>
+</property>
+<% end %>
+
<property>
<name>javax.jdo.option.ConnectionURL</name>
<value>jdbc:derby:;databaseName=/var/lib/hive/metastore/metastore_db;create=true</value>
diff --git a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
index 45168bae..749773d8 100644
--- a/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
+++ b/bigtop-deploy/puppet/modules/hadoop/manifests/init.pp
@@ -294,7 +294,7 @@ class hadoop ($hadoop_security_authentication = "simple",
$mapreduce_jobhistory_host = undef,
$mapreduce_jobhistory_port = "10020",
$mapreduce_jobhistory_webapp_port = "19888",
- $mapreduce_framework_name = "yarn",
+ $mapreduce_framework_name = undef,
$mapred_data_dirs = suffix($hadoop::hadoop_storage_dirs, "/mapred"),
$mapreduce_cluster_temp_dir = "/mapred/system",
$yarn_app_mapreduce_am_staging_dir = "/user",
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
index 318589f5..f959f5bc 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/hadoop-env.sh
@@ -34,10 +34,6 @@
# The java implementation to use. Required.
<%= shell_config("JAVA_HOME", "hadoop_java_home") %>
-# tez environment, needed to enable tez
-<%= shell_config("TEZ_CONF_DIR") %>
-<%= shell_config("TEZ_JARS") %>
-
# Extra Java CLASSPATH elements. Optional.
<%= shell_config("HADOOP_CLASSPATH") %>
@@ -84,6 +80,14 @@
# The scheduling priority for daemon processes. See 'man nice'.
<%= shell_config("HADOOP_NICENESS") %>
+# tez environment, needed to enable tez
+<% if (@all or @components.include? "tez") -%>
+<%= shell_config("TEZ_CONF_DIR") %>
+<%= shell_config("TEZ_JARS") %>
+# Add tez into HADOOP_CLASSPATH
+export HADOOP_CLASSPATH=$HADOOP_CLASSPATH:${TEZ_CONF_DIR}:${TEZ_JARS}/*:${TEZ_JARS}/lib/*
+<% end -%>
+
### WARNING: the following is NOT really optional. It is a shame that stock Hadoop
### hadoop_env.sh doesn't make it clear -- you can NOT turn com.sun.management.jmxremote off
### and have a working Hadoop cluster.
diff --git a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
index 1c8f3fdf..2723e826 100644
--- a/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
+++ b/bigtop-deploy/puppet/modules/hadoop/templates/mapred-site.xml
@@ -86,11 +86,28 @@
</property>
<% end %>
+<% if defined? @mapreduce_framework_name -%>
<property>
<name>mapreduce.framework.name</name>
<value><%= @mapreduce_framework_name %></value>
</property>
+<!-- If not specified, we automatically choose the best framework for users -->
+<% else -%>
+<% if @all or @components.include? "tez" -%>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn-tez</value>
+ </property>
+
+<% else -%>
+ <property>
+ <name>mapreduce.framework.name</name>
+ <value>yarn</value>
+ </property>
+
+<% end -%>
+<% end -%>
<property>
<name>mapreduce.cluster.local.dir</name>
<value><%= @mapred_data_dirs.join(",") %></value>
diff --git a/bigtop-deploy/puppet/modules/tez/manifests/init.pp b/bigtop-deploy/puppet/modules/tez/manifests/init.pp
new file mode 100644
index 00000000..6db2220b
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/tez/manifests/init.pp
@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+class tez {
+ class client {
+ package { "tez":
+ ensure => latest,
+ }
+ }
+}
diff --git a/bigtop-deploy/puppet/modules/tez/tests/init.pp b/bigtop-deploy/puppet/modules/tez/tests/init.pp
new file mode 100644
index 00000000..9ee06368
--- /dev/null
+++ b/bigtop-deploy/puppet/modules/tez/tests/init.pp
@@ -0,0 +1,17 @@
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+include tez
+tez::client { "test-tez": }
diff --git a/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy b/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy
index a224843d..d07ef010 100644
--- a/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy
+++ b/bigtop-packages/src/common/bigtop-utils/init-hcfs.groovy
@@ -139,6 +139,9 @@ def final FileSystem fs = FileSystem.get(conf);
LOG.info("PROVISIONING WITH FILE SYSTEM : " + fs.getClass());
+// Longest back off time to check whether the file system is ready for write
+def final int maxBackOff = 64;
+
/**
* Make a directory. Note when providing input to this functino that if
* nulls are given, the commands will work but behaviour varies depending on
@@ -150,7 +153,19 @@ LOG.info("PROVISIONING WITH FILE SYSTEM : " + fs.getClass());
* @param group can be null,
*/
def mkdir = { FileSystem fsys, Path dname, FsPermission mode, String user, String group ->
- fsys.mkdirs(dname);
+ boolean success = false;
+ for(i = 1; i <= maxBackOff; i*=2) {
+ try {
+ success = fsys.mkdirs(dname)
+ break;
+ } catch(Exception e) {
+ LOG.info("Failed to create directory " + dname + "... Retry after " + i + " second(s)");
+ Thread.sleep(i*1000);
+ }
+ }
+ if (!success) {
+ LOG.info("Can not create directory " + dname + " on " + fsys.getClass());
+ }
if (user != null) {
fsys.setOwner(dname, user, group);
}
@@ -256,8 +271,21 @@ def copyJars = { FileSystem fsys, File input, String jarstr, Path target ->
return validJar && filename.contains(jarstr)
}
}).each({ jar_file ->
- copied++;
- fsys.copyFromLocalFile(new Path(jar_file.getAbsolutePath()), target)
+ boolean success = false;
+ for(i = 1; i <= maxBackOff; i*=2) {
+ try {
+ fsys.copyFromLocalFile(new Path(jar_file.getAbsolutePath()), target)
+ copied++;
+ success = true;
+ break;
+ } catch(Exception e) {
+ LOG.info("Failed to upload " + jar_file.getAbsolutePath() + " to " + target + "... Retry after " + i + " second(s)");
+ Thread.sleep(i*1000);
+ }
+ if (!success) {
+ LOG.info("Can not upload " + jar_file.getAbsolutePath() + " to " + target + " on " + fsys.getClass());
+ }
+ }
});
return copied;
}
@@ -298,4 +326,18 @@ total_jars += copyJars(fs,
new File(PIG_HOME), "",
new Path(OOZIE_SHARE, "pig"))
+LOG.info("Now copying Jars into the DFS for tez ");
+LOG.info("This might take a few seconds...");
+
+def final TEZ_APPS = "/apps";
+def final TEZ_HOME = "/usr/lib/tez/";
+
+total_jars += copyJars(fs,
+ new File(TEZ_HOME, "lib/"), "",
+ new Path(TEZ_APPS, "tez/lib"))
+
+total_jars += copyJars(fs,
+ new File(TEZ_HOME), "",
+ new Path(TEZ_APPS, "tez"))
+
LOG.info("Total jars copied into the DFS : " + total_jars);
diff --git a/bigtop-packages/src/common/hadoop/init-hcfs.json b/bigtop-packages/src/common/hadoop/init-hcfs.json
index 88beb1af..7f9b6cca 100644
--- a/bigtop-packages/src/common/hadoop/init-hcfs.json
+++ b/bigtop-packages/src/common/hadoop/init-hcfs.json
@@ -88,7 +88,10 @@
["/user/oozie/share/lib/distcp", null, null, null],
["/user/oozie/share/lib/pig", null, null, null],
["/var/log/spark/apps","1777","spark","spark"],
- ["/user/hbase","755","hbase",null]
+ ["/user/hbase","755","hbase",null],
+ ["/apps",null,null,null],
+ ["/apps/tez",null,null,null],
+ ["/apps/tez/lib",null,null,null]
],
"user": [
["tom", "0755", null],
diff --git a/bigtop-packages/src/rpm/hue/SPECS/hue.spec b/bigtop-packages/src/rpm/hue/SPECS/hue.spec
index a77ec174..93e36223 100644
--- a/bigtop-packages/src/rpm/hue/SPECS/hue.spec
+++ b/bigtop-packages/src/rpm/hue/SPECS/hue.spec
@@ -181,7 +181,7 @@ Summary: A browser-based desktop interface for Hadoop
BuildRequires: python-devel, python-setuptools, gcc, gcc-c++
BuildRequires: libxml2-devel, libxslt-devel, zlib-devel
BuildRequires: cyrus-sasl-devel
-BuildRequires: openssl
+BuildRequires: openssl-devel
BuildRequires: krb5-devel
BuildRequires: asciidoc
Group: Applications/Engineering
diff --git a/bigtop_toolchain/manifests/deps.pp b/bigtop_toolchain/manifests/deps.pp
index 585cdfd1..2c20a459 100644
--- a/bigtop_toolchain/manifests/deps.pp
+++ b/bigtop_toolchain/manifests/deps.pp
@@ -30,9 +30,9 @@ class bigtop_toolchain::deps {
unless => "/usr/bin/test -f /usr/src/apache-maven-3.0.5-bin.tar.gz",
}
- exec {"/usr/bin/wget http://services.gradle.org/distributions/gradle-2.0-bin.zip":
+ exec {"/usr/bin/wget http://services.gradle.org/distributions/gradle-2.4-bin.zip":
cwd => "/usr/src",
require => Package[$packages::pkgs],
- unless => "/usr/bin/test -f /usr/src/gradle-2.0-bin.zip",
+ unless => "/usr/bin/test -f /usr/src/gradle-2.4-bin.zip",
}
}
diff --git a/bigtop_toolchain/manifests/gradle.pp b/bigtop_toolchain/manifests/gradle.pp
index 16021f80..2c0081f3 100644
--- a/bigtop_toolchain/manifests/gradle.pp
+++ b/bigtop_toolchain/manifests/gradle.pp
@@ -17,16 +17,16 @@ class bigtop_toolchain::gradle {
include bigtop_toolchain::deps
- exec {'/usr/bin/unzip -x -o /usr/src/gradle-2.0-bin.zip':
+ exec {'/usr/bin/unzip -x -o /usr/src/gradle-2.4-bin.zip':
cwd => '/usr/local',
refreshonly => true,
- subscribe => Exec["/usr/bin/wget http://services.gradle.org/distributions/gradle-2.0-bin.zip"],
- require => Exec["/usr/bin/wget http://services.gradle.org/distributions/gradle-2.0-bin.zip"],
+ subscribe => Exec["/usr/bin/wget http://services.gradle.org/distributions/gradle-2.4-bin.zip"],
+ require => Exec["/usr/bin/wget http://services.gradle.org/distributions/gradle-2.4-bin.zip"],
}
file {'/usr/local/gradle':
ensure => link,
- target => '/usr/local/gradle-2.0',
- require => Exec['/usr/bin/unzip -x -o /usr/src/gradle-2.0-bin.zip'],
+ target => '/usr/local/gradle-2.4',
+ require => Exec['/usr/bin/unzip -x -o /usr/src/gradle-2.4-bin.zip'],
}
}
diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties
index 7dcaad82..8b980357 100644
--- a/gradle/wrapper/gradle-wrapper.properties
+++ b/gradle/wrapper/gradle-wrapper.properties
@@ -15,4 +15,4 @@
distributionPath=wrapper/dists
zipStorePath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-2.1-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-2.4-bin.zip
diff --git a/packages.gradle b/packages.gradle
index 7ccf5eb9..412de563 100644
--- a/packages.gradle
+++ b/packages.gradle
@@ -676,12 +676,9 @@ project.afterEvaluate {
group: PACKAGES_GROUP
) << { }
- task clean (overwrite: true, dependsOn: tasks.findAll { alltask -> alltask.name.endsWith("-clean")}*.name,
- description: "Cleaning all components' build and output directories. Overwrites standard clean task",
- group: PACKAGES_GROUP) << {
- }
- task allclean (dependsOn: clean,
- description: "Removing $BUILD_DIR, $OUTPUT_DIR, and $DIST_DIR",
+ task allclean (dependsOn: [clean, tasks.findAll { alltask -> alltask.name.endsWith("-clean")}*.name],
+ description: "Removing $BUILD_DIR, $OUTPUT_DIR, and $DIST_DIR.\n\t\t" +
+ "Cleaning all components' build and output directories.",
group: PACKAGES_GROUP) << {
delete (BUILD_DIR)
delete (OUTPUT_DIR)