Automating Hadoop using Ansible

What is hadoop?

How does Ansible work?

Ansible playbooks

Variables in Our cluster

- namenodeIP: “192.168.43.202”
- port: “9091”

Ansible configuration file

Namenode playbook

- hosts: Name
gather_facts: false
vars_files:
- vars.yml
vars:
- node: "name"
- directory: "namenode"

tasks:
- file:
path: "/{{ directory }}"
state: directory
- debug:
msg: "{{ namenodeIP }}"
# Copy Hadoop and JDK files to a target nodes
- name: "Copy Hadoop file"
copy:
src: "/root/hadoop-1.2.1-1.x86_64.rpm"
dest: "/root/"
- name: "Copy JDK file"
copy:
src: "/root/jdk-8u171-linux-x64.rpm"
dest: "/root/"
# Check whether Hadoop and JDK is present or not
- name: "Check whether Hadoop"
package:
name: "hadoop"
state: present
register: x
# if error occur means Dosnot exist
ignore_errors: yes
- name: "Check whether JDK"
package:
name: "java"
state: present
register: y
ignore_errors: yes
# Install Hadoop and JDK files using Command
- name: "Install Hadoop"
command: "rpm -ivh hadoop-1.2.1-1.x86_64.rpm --force"
when: x.rc == 1
- name: "Install JDK"
command: "rpm -ivh jdk-8u171-linux-x64.rpm"
when: y.rc == 1
# Copy hdfs and core files
- name: "Copy core file"
template:
src: "core-site.xml.j2"
dest: "/etc/hadoop/core-site.xml"
- name: "Copy HDFS file"
template:
src: "hdfs-site.xml.j2"
dest: "/etc/hadoop/hdfs-site.xml"
# Enable port number
- name: "Enabling firewall daimon of {{ port }} port and accesss permamnent"
firewalld:
port: "{{ port }}/tcp"
state: enabled
permanent: yes
immediate: yes
# Format Namenode
- pause:
prompt: "Do you wants to Format MetaData in Namenode (yes/no)"
register: format
- name: "Format Namenode"
command: "echo Y | hadoop namenode -format"
when: format.user_input | bool
- debug:
var: format.user_input
# Start NameNode
- name: "Start Namenode"
command: "hadoop-daemon.sh start namenode"
register: start_namenode
- debug:
var: start_namenode
# JPS
- name: "JPS"
command: "jps"
register: jps_n
- debug:
var: jps_n
ansible-playbook hadoopNamenode.yml

Datanode Playbook

- hosts: Data 
gather_facts: false
vars_files:
- vars.yml
vars:
- node: "data"
- directory: "datanode"

tasks:
- file:
path: "/{{ directory }}"
state: directory
- debug:
msg: "{{ namenodeIP }}"
# Copy Hadoop and JDK files to a target nodes
- name: "Copy Hadoop file"
copy:
src: "/root/hadoop-1.2.1-1.x86_64.rpm"
dest: "/root/"
- name: "Copy JDK file"
copy:
src: "/root/jdk-8u171-linux-x64.rpm"
dest: "/root/"
# Check whether Hadoop and JDK is present or not
- name: "Check Hadoop file exists"
package:
name: "hadoop"
state: present
register: x
# if error occur means Dosnot exist
ignore_errors: yes
- name: "Check JDK file exists"
package:
name: "java"
state: present
register: y
ignore_errors: yes
# Install Hadoop and JDK files using Command
- name: "Install Hadoop"
command: "rpm -ivh hadoop-1.2.1-1.x86_64.rpm --force"
when: x.rc == 1
- name: "Install JDK"
command: "rpm -ivh jdk-8u171-linux-x64.rpm"
when: y.rc == 1
# Copy hdfs and core files
- name: "Configure Core Files"
template:
src: "core-site.xml.j2"
dest: "/etc/hadoop/core-site.xml"
- name: "Configure HDFS Files"
template:
src: "hdfs-site.xml.j2"
dest: "/etc/hadoop/hdfs-site.xml"
# Start DataNode
- name: "Start DataNode"
command: "hadoop-daemon.sh start datanode"
register: start_datanode
- debug:
var: start_datanode
# JPS
- name: "JPS"
command: "jps"
register: jps
- debug:
var: jps
ansible-playbook hadoopDatanode.yml
- hosts:  Client
gather_facts: false
vars_files:
- vars.yml
vars:
- blocks: "1024"
- replicas: "3"

tasks:
- debug:
msg: "{{ namenodeIP }}"
# Copy Hadoop and JDK files to a target nodes
- name: "Copy Hadoop file"
copy:
src: "/root/hadoop-1.2.1-1.x86_64.rpm"
dest: "/root/"
- name: "Copy JDK file"
copy:
src: "/root/jdk-8u171-linux-x64.rpm"
dest: "/root/"
# Check whether Hadoop and JDK is present or not
- name: "Check whether Hadoop exists"
package:
name: "hadoop"
state: present
register: x
# if error occur means Dosnot exist
ignore_errors: yes
- name: "Check whether JDK exists"
package:
name: "java"
state: present
register: y
ignore_errors: yes
# Install Hadoop and JDK files using Command
- name: "Install Hadoop"
command: "rpm -ivh hadoop-1.2.1-1.x86_64.rpm --force"
when: x.rc == 1
- name: "Install JDK"
command: "rpm -ivh jdk-8u171-linux-x64.rpm"
when: y.rc == 1
# Copy hdfs and core files
- name: "Configure HDFS file"
template:
src: "core-site.xml.j2"
dest: "/etc/hadoop/core-site.xml"
- name: "Configure CORE file"
template:
src: "hdfs-site-client.xml.j2"
dest: "/etc/hadoop/hdfs-site.xml"
- name: " Create a file"
file:
path: "/new.txt"
state: touch

- copy:
content: "Hi this is Client file"
dest: "/client.txt"
- name: "Check file exists or not"
command: "hadoop fs -ls /client.txt"
register: fileCheck
ignore_errors: yes
# Share data
- name: "Upload file into Hadoop/hdfs cluster"
command: "hadoop fs -put /client.txt /"
register: data
when: fileCheck.rc > 0
- debug:
var: data
ansible-playbook hadoopClientnode.yml

--

--

Get the Medium app

A button that says 'Download on the App Store', and if clicked it will lead you to the iOS App store
A button that says 'Get it on, Google Play', and if clicked it will lead you to the Google Play store
Vinodha kumara

Vinodha kumara

ARTH-School of technology, BCA graduate