2018年5月23日水曜日

VagrantでAmbari(HDP2.6)/HDFS/Sparkをインストールした仮想マシン(CentOS7.4)を構築する

Apache SparkでHadoop上のデータにクエリーを実行する事ができます。

〇Ambariの画面

ブラウザでhttp://192.168.1.117:8080/にアクセスします。ユーザ/パスワードはadmin/adminです。

〇構築方法
以下のVagrantfileを使用して、Ambari/HDFS/Sparkをインストールした仮想マシン(CentOS7.4)を構築する事ができます。

Vagrantfile
VAGRANTFILE_API_VERSION = "2"

Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
  config.vm.box = "bento/centos-7.4"
  config.vm.hostname = "co74spark.vm.internal"
  config.vm.provider :virtualbox do |vbox|
     vbox.name = "co74spark.vm.internal"
     vbox.cpus = 4
     vbox.memory = 10240
     vbox.customize ["modifyvm", :id, "--nicpromisc2","allow-all"]
  end
  # private network
config.vm.network "private_network", ip: "192.168.55.117", :netmask => "255.255.255.0"
  # bridge netwrok
config.vm.network "public_network", ip: "192.168.1.117", :netmask => "255.255.255.0"
  config.vm.provision "shell", inline: <<-SHELL
localectl set-locale LANG=ja_JP.UTF-8
echo "192.168.55.117 co74spark" >> /etc/hosts

cd /root
mkdir ./.ssh
ssh-keygen -f ./.ssh/id_rsa -t rsa -N ''

# copy private key
cp -f ./.ssh/id_rsa /vagrant
cat ./.ssh/id_rsa.pub >> ./.ssh/authorized_keys
chmod 600 ./.ssh/authorized_keys

# install and configure ambari server
cd /etc/yum.repos.d/
wget http://public-repo-1.hortonworks.com/ambari/centos7/2.x/updates/2.6.2.0/ambari.repo
yum -y install ambari-server ambari-agent
ambari-server setup --silent
ambari-server start

# create hive database and hive user
echo "host    all         all         127.0.0.1/32          password" >> /var/lib/pgsql/data/pg_hba.conf
echo "host    all         all         192.168.1.0/24          password" >> /var/lib/pgsql/data/pg_hba.conf
echo "host    all         all         192.168.55.0/24          password" >> /var/lib/pgsql/data/pg_hba.conf
sudo su - postgres << EOF
createdb hive
createdb oozie
psql -c "
create user hive with password 'hive';
grant all privileges on database hive to hive;
create user oozie with password 'oozie';
grant all privileges on database oozie to oozie;
"
EOF
service postgresql restart

yum -y install postgresql-jdbc*
chmod 644 /usr/share/java/postgresql-jdbc.jar

ambari-server setup -s --database=postgres --databasehost=localhost --databaseport=5432 --databasename=hive --databaseusername=hive --databasepassword=hive --jdbc-db=postgres --jdbc-driver=/usr/share/java/postgresql-jdbc.jar
ambari-agent start


cat << EOF > /home/vagrant/cluster_configuration.json
{
  "configurations" : [
    {
      "hive-site": {
        "hive.support.concurrency": "true",
        "hive.txn.manager": "org.apache.hadoop.hive.ql.lockmgr.DbTxnManager",
        "hive.compactor.initiator.on": "true",
        "hive.compactor.worker.threads": "5",
        "javax.jdo.option.ConnectionDriverName": "org.postgresql.Driver",
        "javax.jdo.option.ConnectionPassword": "hive",
        "javax.jdo.option.ConnectionURL": "jdbc:postgresql://localhost/hive",
        "javax.jdo.option.ConnectionUserName": "hive"
      }
    },
    {
      "hive-env": {
        "hive_ambari_database": "PostgreSQL",
        "hive_database": "Existing PostgreSQL Database",
        "hive_database_type": "postgres",
        "hive_database_name": "hive"
      }
    },
    {
      "core-site": {
        "properties" : {
          "hadoop.proxyuser.root.groups" : "*",
          "hadoop.proxyuser.root.hosts" : "*"
        }
      }
    }
  ],
  "host_groups" : [
    {
      "name" : "host_group_1",
      "components" : [
        {
          "name" : "NAMENODE"
        },
        {
          "name" : "SECONDARY_NAMENODE"
        },
        {
          "name" : "DATANODE"
        },
        {
          "name" : "HDFS_CLIENT"
        },
        {
          "name" : "RESOURCEMANAGER"
        },
        {
          "name" : "NODEMANAGER"
        },
        {
          "name" : "YARN_CLIENT"
        },
        {
          "name" : "HISTORYSERVER"
        },
        {
          "name" : "APP_TIMELINE_SERVER"
        },
        {
          "name" : "ZOOKEEPER_SERVER"
        },
        {
          "name" : "ZOOKEEPER_CLIENT"
        },
        {
          "name" : "METRICS_MONITOR"
        },
        {
          "name" : "TEZ_CLIENT"
        },
        {
          "name" : "HIVE_SERVER"
        },
        {
          "name" : "HIVE_METASTORE"
        },
        {
          "name" : "METRICS_COLLECTOR"
        },
        {
          "name" : "WEBHCAT_SERVER"
        },
        {
          "name" : "PIG"
        },
        {
          "name" : "SLIDER"
        },
        {
          "name" : "SPARK2_THRIFTSERVER"
        },
        {
          "name" : "SPARK2_CLIENT"
        },
        {
          "name" : "SPARK2_JOBHISTORYSERVER"
        }
      ],
      "cardinality" : "1"
    }
  ],
  "settings" : [{
     "recovery_settings" : [{
       "recovery_enabled" : "true"
    }]
  }],
  "Blueprints" : {
    "blueprint_name" : "hdp26-minimal-spark",
    "stack_name" : "HDP",
    "stack_version" : "2.6"
  }
}
EOF

curl -H "X-Requested-By: ambari" -X POST -u admin:admin http://localhost:8080/api/v1/blueprints/hdp26-minimal-spark -d @/home/vagrant/cluster_configuration.json

cat << EOF > /home/vagrant/hostmapping.json
{
  "blueprint" : "hdp26-minimal-spark",
  "default_password" : "admin",
  "provision_action" : "INSTALL_AND_START",
  "host_groups" :[
    {
      "name" : "host_group_1",
      "hosts" : [
        {
          "fqdn" : "co74spark.vm.internal"
        }
      ]
    }
  ]
}
EOF

curl -H "X-Requested-By: ambari" -X POST -u admin:admin http://localhost:8080/api/v1/clusters/hdp26-minimal-spark -d @/home/vagrant/hostmapping.json
sleep 30


# wait until the cluster is ready.
ProgressPercent=`curl -s --user admin:admin -X GET http://localhost:8080/api/v1/clusters/hdp26-minimal-spark/requests/1 | grep progress_percent | awk '{print $3}' | cut -d . -f 1`
while [[ `echo $ProgressPercent | grep -v 100` ]]; do
  ProgressPercent=`curl -s --user admin:admin -X GET http://localhost:8080/api/v1/clusters/hdp26-minimal-spark/requests/1 | grep progress_percent | awk '{print $3}' | cut -d . -f 1`
  echo " Progress: $ProgressPercent %"
  sleep 10
done


cat << EOF > /home/vagrant/shutdown_components.sh
#!/bin/bash
#stop all services
curl -u admin:admin -i -H 'X-Requested-By: ambari' -X PUT \
   -d '{"RequestInfo":{"context":"_PARSE_.STOP.ALL_SERVICES","operation_level":{"level":"CLUSTER","cluster_name":"hdp26-minnimal-hive"}},"Body":{"ServiceInfo":{"state":"INSTALLED"}}}' \
http://localhost:8080/api/v1/clusters/hdp26-minimal-spark/services
EOF
chmod +x /home/vagrant/shutdown_components.sh


cat << EOF > /tmp/test.csv
100,15000000
200,20000000
300,18000000
EOF


cat << EOF > /tmp/sample.sql
CREATE EXTERNAL TABLE sample (
  store_id INT,
  sales INT
)
ROW FORMAT SERDE 'org.apache.hadoop.hive.serde2.OpenCSVSerde'
WITH SERDEPROPERTIES (
   "separatorChar" = ",",
   "quoteChar"     = '"'
)
stored as textfile
LOCATION '/user/hive';

LOAD DATA LOCAL INPATH '/tmp/test.csv' OVERWRITE INTO TABLE sample;

select * from sample;
EOF


# upload sample content
sudo -u hive hdfs dfs -put /tmp/sample.sql /user/hive
sudo -u hive hdfs dfs -ls /user/hive

# create table and select
beeline -u 'jdbc:hive2://localhost:10016/' -n hive -f /tmp/sample.sql

echo 'access -> http://192.168.1.117:8080'
echo 'user/password -> admin/admin'
SHELL
end

○関連情報
・Ambariに関する他の記事はこちらを参照してください。

0 件のコメント:

コメントを投稿