Skip to content

Commit 45fe6b3

Browse files
samerfaridasamerfarida
committed
Added docker swarm setup and mount cephfs scripts.
1 parent 87675b9 commit 45fe6b3

2 files changed

Lines changed: 177 additions & 0 deletions

File tree

‎docker_swarm_setup/docker_swarm_setup.sh‎

Lines changed: 88 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,88 @@
1+
#!/bin/bash
2+
3+
# script description: Script to deploy a cluster of managers and workers on a cluster of servers.
4+
5+
# SSH user
6+
SSH_USER="serveradmin"
7+
8+
# Managers and workers IP addresses
9+
MANAGER_IPS=("192.168.80.248" "192.168.80.136" "192.168.80.123")
10+
WORKER_IPS=("192.168.80.182" "192.168.80.76" "192.168.80.110")
11+
12+
# SSH certificate file
13+
SSH_CERTIFICATE="/root/.ssh/id_rsa"
14+
15+
# Add host keys to known_hosts file
16+
add_host_keys() {
17+
for ip in "${MANAGER_IPS[@]}" "${WORKER_IPS[@]}"; do
18+
echo "Adding host key for $ip to known_hosts file..."
19+
ssh-keyscan -H "$ip" >> ~/.ssh/known_hosts
20+
done
21+
}
22+
23+
# Install Docker on remote servers
24+
install_docker() {
25+
for ip in "${MANAGER_IPS[@]}" "${WORKER_IPS[@]}"; do
26+
echo "Installing Docker on $ip..."
27+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'curl -fsSL https://get.docker.com -o install-docker.sh && sudo sh install-docker.sh'
28+
done
29+
}
30+
31+
# Function to apply worker labels on the first manager
32+
apply_worker_labels() {
33+
local first_manager=${MANAGER_IPS[0]}
34+
local worker_hostnames
35+
36+
echo "Retrieving worker hostnames from the first manager ($first_manager)..."
37+
worker_hostnames=$(ssh -i $SSH_CERTIFICATE $SSH_USER@"$first_manager" 'sudo docker node ls --filter role=worker --format "{{.Hostname}}"')
38+
39+
echo "Applying worker labels on the first manager ($first_manager)..."
40+
while IFS= read -r worker_hostname; do
41+
echo "Applying worker label to $worker_hostname..."
42+
ssh -n -i $SSH_CERTIFICATE $SSH_USER@"$first_manager" "sudo docker node update --label-add worker=true $worker_hostname"
43+
done <<< "$worker_hostnames"
44+
}
45+
46+
# Create Docker Swarm
47+
create_swarm() {
48+
# Initialize Swarm on the first manager
49+
FIRST_MANAGER=${MANAGER_IPS[0]}
50+
echo "Initializing Swarm on $FIRST_MANAGER..."
51+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$FIRST_MANAGER" "sudo docker swarm init --advertise-addr $FIRST_MANAGER"
52+
53+
# Get manager join token
54+
MANAGER_TOKEN=$(ssh -i $SSH_CERTIFICATE $SSH_USER@"$FIRST_MANAGER" 'sudo docker swarm join-token manager -q')
55+
56+
# Join additional managers to swarm
57+
for ip in "${MANAGER_IPS[@]:1}"; do
58+
echo "Joining manager $ip to the swarm..."
59+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" "sudo docker swarm join --token $MANAGER_TOKEN $FIRST_MANAGER:2377"
60+
done
61+
62+
# Get worker join token
63+
WORKER_TOKEN=$(ssh -i $SSH_CERTIFICATE $SSH_USER@"$FIRST_MANAGER" 'sudo docker swarm join-token worker -q')
64+
65+
# Join workers to swarm
66+
for ip in "${WORKER_IPS[@]}"; do
67+
echo "Joining worker $ip to the swarm..."
68+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" "sudo docker swarm join --token $WORKER_TOKEN $FIRST_MANAGER:2377"
69+
done
70+
}
71+
72+
# Display Docker Swarm status
73+
display_swarm_status() {
74+
echo "Docker Swarm Status:"
75+
ssh -i $SSH_CERTIFICATE $SSH_USER@"${MANAGER_IPS[0]}" 'sudo docker node ls'
76+
}
77+
78+
# Main script
79+
echo "Add host keys to known_hosts file"
80+
add_host_keys
81+
echo "Install Docker on remote servers"
82+
install_docker
83+
echo "Create Docker Swarm and join nodes"
84+
create_swarm
85+
echo "Apply worker labels on the first manager"
86+
apply_worker_labels
87+
echo "Display Docker Swarm status"
88+
display_swarm_status

‎docker_swarm_setup/mount_cephfs_docker_swarm_setup.sh‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/bin/bash
2+
3+
# script description: Script to create cephfs cleint auth file on proxmox and and mount cephfs on a remote server.
4+
5+
# Ceph FS variables
6+
CEPHFS_NAME="cephfs"
7+
CEPHFS_SUBDIRECTORY="docker-shared-prod"
8+
MNT_DIR_NAME="$CEPHFS_NAME/$CEPHFS_SUBDIRECTORY"
9+
CEPH_CONF=""
10+
CLIENT_KEYRING=""
11+
CLIENT_KEY=""
12+
SSH_USER="serveradmin"
13+
SERVERS_IPS=("192.168.80.248" "192.168.80.136" "192.168.80.123" "192.168.80.182" "192.168.80.76" "192.168.80.110")
14+
15+
SSH_CERTIFICATE="/root/.ssh/id_rsa"
16+
17+
# Function to add host keys to known_hosts file
18+
add_host_keys() {
19+
if [ ! -f ~/.ssh/known_hosts ]; then
20+
touch ~/.ssh/known_hosts
21+
fi
22+
for ip in "${SERVERS_IPS[@]}"; do
23+
if ! grep -q "$ip" ~/.ssh/known_hosts; then
24+
echo "Adding host key for $ip to known_hosts file..."
25+
ssh-keyscan -H "$ip" >> ~/.ssh/known_hosts
26+
fi
27+
done
28+
}
29+
30+
# Function to install ceph-common on all servers
31+
install_ceph_common() {
32+
for ip in "${SERVERS_IPS[@]}"; do
33+
echo "Installing ceph-common on $ip..."
34+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'sudo apt install -y ceph-common'
35+
done
36+
}
37+
38+
# Function to create directories on all servers
39+
create_directories() {
40+
for ip in "${SERVERS_IPS[@]}"; do
41+
echo "Creating directories on $ip..."
42+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'sudo mkdir -p /etc/ceph && sudo mkdir -p /mnt/'"$MNT_DIR_NAME"
43+
done
44+
}
45+
46+
# Function to generate ceph config file and copy to all servers
47+
generate_ceph_config() {
48+
CEPH_CONF=$(sudo ceph config generate-minimal-conf)
49+
for ip in "${SERVERS_IPS[@]}"; do
50+
echo "Copying ceph config file to $ip..."
51+
echo "$CEPH_CONF" | ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'sudo tee /etc/ceph/ceph.conf > /dev/null'
52+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'sudo chmod 644 /etc/ceph/ceph.conf'
53+
done
54+
}
55+
56+
# Function to authorize client and mount ceph fs on all servers
57+
authorize_and_mount_cephfs() {
58+
for ip in "${SERVERS_IPS[@]}"; do
59+
echo "Authorizing client and mounting ceph fs on $ip..."
60+
SERVERS_HOSTNAME=$(ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" hostname)
61+
# CLIENT_KEYRING=$(sudo ceph fs authorize $CEPHFS_NAME client.$SERVERS_HOSTNAME /$CEPHFS_SUBDIRECTORY rw)
62+
CLIENT_KEYRING=$(sudo ceph fs authorize $CEPHFS_NAME client."$SERVERS_HOSTNAME" / rw)
63+
echo "$CLIENT_KEYRING" | ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'sudo tee /etc/ceph/ceph.client.'"$SERVERS_HOSTNAME"'.keyring > /dev/null'
64+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'sudo chmod 600 /etc/ceph/ceph.client.'"$SERVERS_HOSTNAME"'.keyring'
65+
CLIENT_KEY=$(sudo ceph auth get-key client."$SERVERS_HOSTNAME")
66+
echo "$CLIENT_KEY" | ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'sudo tee /etc/ceph/ceph.client.'"$SERVERS_HOSTNAME"'.key > /dev/null'
67+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" "sudo chmod 600 /etc/ceph/ceph.client.$SERVERS_HOSTNAME.key"
68+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" "sudo mount -t ceph $SERVERS_HOSTNAME@.$CEPHFS_NAME=/$CEPHFS_SUBDIRECTORY /mnt/$MNT_DIR_NAME -o secretfile=/etc/ceph/ceph.client.$SERVERS_HOSTNAME.key"
69+
echo "Setting up persistent mount on $SERVERS_HOSTNAME - $ip..."
70+
echo "$SERVERS_HOSTNAME@.$CEPHFS_NAME=/$CEPHFS_SUBDIRECTORY /mnt/$MNT_DIR_NAME ceph secretfile=/etc/ceph/ceph.client.$SERVERS_HOSTNAME.key,noatime,_netdev 0 0" | ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" 'sudo tee -a /etc/fstab > /dev/null'
71+
echo "Changing ownership /mnt/$MNT_DIR_NAME to root:docker"
72+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" "sudo chown -R root:docker /mnt/$MNT_DIR_NAME"
73+
echo "Initiating server reboot on $SERVERS_HOSTNAME - $ip..."
74+
ssh -i $SSH_CERTIFICATE $SSH_USER@"$ip" "sudo reboot"
75+
done
76+
}
77+
78+
79+
# Main function
80+
main() {
81+
#add_host_keys
82+
install_ceph_common
83+
create_directories
84+
generate_ceph_config
85+
authorize_and_mount_cephfs
86+
}
87+
88+
# Call main function
89+
main

0 commit comments

Comments
 (0)