一、安装Hive
1.下载并解压Hive
[[email protected] soft]# tar -zxf apache-hive-0.13.1-bin.tar.gz -C /usr/local/ [[email protected] soft]# ln -s /usr/local/apache-hive-0.13.1-bin/ /usr/local/hive [[email protected] soft]#
2.配置hive环境变量
[[email protected] soft]# cat /etc/profile.d/hive.sh HIVE_HOME=/usr/local/hive PATH=$HIVE_HOME/bin:$PATH [[email protected] soft]# . /etc/profile.d/hive.sh [[email protected] soft]#
3.安装mysql数据库
[[email protected] soft]# yum -y install mysql-server [[email protected] soft]#chkconfig mysqld on [[email protected] soft]#service mysqld restart
4.配置mysql权限
mysql> create database hive; Query OK, 1 row affected (0.00 sec) mysql> grant all on *.* to ‘hive‘@‘%‘ identified by ‘123456‘; Query OK, 0 rows affected (0.00 sec) mysql>
5.COPY配置文件
[[email protected] conf]# pwd /usr/local/hive/conf [[email protected] conf]# cp hive-env.sh.template hive-env.sh [[email protected] conf]# cphive-default.xml.template hive-site.xml [[email protected] conf]# cphive-exec-log4j.properties.template hive-exec-log4j.properties [[email protected] conf]# cphive-log4j.properties.template hive-log4j.properties [[email protected] conf]#
6.配置hive-env.sh
[[email protected] conf]# pwd /usr/local/hive/conf [[email protected] conf]# cat hive-env.sh |grep -v ^$ |grep -v ^# HADOOP_HOME=/usr/local/hadoop [[email protected] conf]#
7.配置hive-site.sh
[[email protected] conf]# pwd /usr/local/bigdata/hive/conf [[email protected] conf]# cat hive-site.xml |grep -v ^$|grep -v ^# <?xmlversion="1.0"?> <?xml-stylesheettype="text/xsl" href="configuration.xsl"?> <configuration> <property> <name>javax.jdo.option.ConnectionURL</name> <value>jdbc:mysql://node1:3306/hive?createDatabaseIfNotExist=true</value> <description>JDBC connect stringfor a JDBC metastore</description> </property> <property> <name>javax.jdo.option.ConnectionDriverName</name> <value>com.mysql.jdbc.Driver</value> <description>Driver class name fora JDBC metastore</description> </property> <property> <name>javax.jdo.option.ConnectionUserName</name> <value>hive</value> <description>username to useagainst metastore database</description> </property> <property> <name>javax.jdo.option.ConnectionPassword</name> <value>123456</value> <description>password to useagainst metastore database</description> </property> </configuration> [[email protected] conf]#
8.复制jdbc驱动
[[email protected] soft]# cp mysql-connector-java-5.1.22-bin.jar /usr/local/hive/lib/ [[email protected] soft]#
二、安装shark
1.下载并加压shark
[[email protected] soft]# tar -zxf shark-0.9.1-bin-hadoop2.tgz -C /usr/local/ [[email protected] soft]# ln -s /usr/local/shark-0.9.1-bin-hadoop2/ /usr/local/shark [[email protected] soft]#
2.配置shark环境变量
[[email protected] soft]# cat /etc/profile.d/shark.sh SHARK_HOME=/usr/local/shark PATH=$SHARK_HOME/bin:$PATH [[email protected] soft]# . /etc/profile.d/shark.sh [[email protected] soft]#
3.配置shark-env.sh
[[email protected] conf]# pwd /usr/local/shark/conf [[email protected] conf]# cat shark-env.sh|grep -v ^$|grep -v ^# export JAVA_HOME=/usr/java/latest export SPARK_MEM=128m export SHARK_MASTER_MEM=128m export HADOOP_HOME="/usr/local/hadoop" export HIVE_HOME="/usr/local/hive" export HIVE_CONF_DIR="/usr/local/hive/conf" export SCALA_HOME="/usr/local/scala" export MASTER="spark://192.168.1.1:7077" export SPARK_HOME="/usr/local/spark" SPARK_JAVA_OPTS=" -Dspark.local.dir=/tmp " SPARK_JAVA_OPTS+="-Dspark.kryoserializer.buffer.mb=10" SPARK_JAVA_OPTS+="-verbose:gc -XX:-PrintGCDetails -XX:+PrintGCTimeStamps " export SPARK_JAVA_OPTS [[email protected] conf]#
4.复制hive配置到所有slaves
[[email protected] ~]# scp -r/usr/local/apache-hive-0.13.1-bin/ node2:/usr/local/ [[email protected] ~]# scp -r/usr/local/apache-hive-0.13.1-bin/ node3:/usr/local/ [[email protected] ~]# sshnode2 ln -s /usr/local/apache-hive-0.13.1-bin/ /usr/local/hive [[email protected] ~]# sshnode3 ln -s /usr/local/apache-hive-0.13.1-bin/ /usr/local/hive [[email protected] ~]# scp/etc/profile.d/hive.sh node2:/etc/profile.d/ hive.sh 100% 52 0.1KB/s 00:00 [[email protected] ~]# scp/etc/profile.d/hive.sh node3:/etc/profile.d/ hive.sh 100% 52 0.1KB/s 00:00 [[email protected] ~]#
5.复制Shark目录到所有的slaves
[[email protected] ~]# scp -r/usr/local/shark-0.9.1-bin-hadoop2/ node2:/usr/local/ [ro[email protected] ~]# scp -r/usr/local/shark-0.9.1-bin-hadoop2/ node3:/usr/local/ [[email protected] ~]# sshnode2 ln -s /usr/local/shark-0.9.1-bin-hadoop2/ /usr/local/shark [[email protected] ~]# sshnode3 ln -s /usr/local/shark-0.9.1-bin-hadoop2/ /usr/local/shark [[email protected] ~]# scp/etc/profile.d/shark.sh node2:/etc/profile.d/ shark.sh 100% 55 0.1KB/s 00:00 [[email protected] ~]# scp/etc/profile.d/shark.sh node3:/etc/profile.d/ shark.sh 100% 55 0.1KB/s 00:00 [[email protected] ~]#
6.替换 jar 包 “hive-exec-0.11.0-shark-0.9.1.jar”
cd $SHARK_HOME/lib_managed/jars/edu.berkeley.cs.shark/hive-exec unziphive-exec-0.11.0-shark-0.9.1.jar rm -fcom/google/protobuf/* rm hive-exec-0.11.0-shark-0.9.1.jar zip -rhive-exec-0.11.0-shark-0.9.1.jar * rm -rf comhive-exec-log4j.properties javaewah/ javax/ javolution/ META-INF/ org/
三、Shark整合Tachyon
1.修改shark-env.sh添加如下2行
export TACHYON_MASTER="tachyon://node1:19998" export TACHYON_WAREHOUSE_PATH=/sharktables
2.同步修改的配置到其它节点
[[email protected] conf]# scpshark-env.sh node2:/usr/local/shark/conf/ shark-env.sh 100%2253 2.2KB/s 00:00 [[email protected] conf]# scpshark-env.sh node3:/usr/local/shark/conf/ shark-env.sh 100%2253 2.2KB/s 00:00 [[email protected] conf]#
3.启动shark
[[email protected] conf]# shark shark> show tables; OK ssq Time taken: 0.841 seconds shark>
4. 指定TBLPROPERTIES(“shark.cache” = “tachyon”)将表缓存到tachyon
shark> create table test1 TBLPROPERTIES("shark.cache" = "tachyon") as select * from ssq;
5. 指定表的名称与_tachyon结尾将表缓存到Tachyon
shark> CREATE TABLEtest_tachyon AS SELECT * FROM ssq;
时间： 2024-08-06 11:56:44