# setup for new dolphin capacity on hetzner

three nodes provisioned by selecting best available options from hetzner dedicated server auction (https://www.hetzner.com/sb) using criteria: under $50 monthly, 32gb ram, 2tb hdd

- a1.calamari.seabird.systems
- ip: 144.76.220.165
- datacenter fsn1-dc11
- a2.calamari.seabird.systems
- ip: 95.216.2.97
- datacenter hel1-dc2
- a3.calamari.seabird.systems
- ip: 213.239.220.238
- datacenter nbg1-dc1

operating system (ubuntu 22.04.1 lts base) provisioned through hetzner console (https://robot.hetzner.com/server)

dns A records for each node (https://dash.cloudflare.com/114c8066a703b106326275ea24858a09/seabird.systems/dns/records)

set hostname, create *mobula* user on each node

for hostname in a1 a2 a3; do
  fqdn=${hostname}.calamari.seabird.systems
  echo ${fqdn}
  ssh -o ConnectTimeout=3 root@${fqdn} "
    hostnamectl set-hostname ${fqdn};
    getent passwd mobula &> /dev/null || useradd --create-home --shell /bin/bash --comment 'manta ops' --user-group --groups sudo,systemd-journal mobula;
    test -d /home/mobula/.ssh || sudo -H -u mobula mkdir /home/mobula/.ssh;
    chmod 700 /home/mobula/.ssh;
    sudo -H -u mobula bash -c 'echo ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJIBSdR9Wy3S4L+Zdcu8waYe5vW2VzMoi+QafgV3IIFj > /home/mobula/.ssh/authorized_keys';
    chmod 644 /home/mobula/.ssh/authorized_keys;
    echo 'mobula ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/mobula;
    chmod 0440 /etc/sudoers.d/mobula;
  "
done

test mobula user has password-less sudo by updating distro packages

for hostname in a1 a2 a3; do
  fqdn=${hostname}.calamari.seabird.systems
  echo ${fqdn}
  ssh -o ConnectTimeout=3 mobula@${fqdn} "
    sudo apt-get update;
    sudo apt-get upgrade -y;
  "
done

create rubberneck configurations

repo_path=~/git/Manta-Network/rubberneck
for hostname in a1 a2 a3; do
  mkdir -p ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system
  cp ${repo_path}/config/calamari.seabird.systems/c1/etc/systemd/system/manta-indexer.service ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/
  cp ${repo_path}/config/calamari.seabird.systems/c1/etc/systemd/system/calamari.service ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/
  sed -i "s/c1/${hostname}/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
  sed -i '/collator/d' ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
  sed -i '/node-key-file/d' ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
  unit_sha=$(sha256sum ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service | cut -d ' ' -f 1)
  
  mkdir -p ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/nginx/sites-available
  cp ${repo_path}/config/calamari.seabird.systems/c1/etc/nginx/sites-available/substrate.conf ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/nginx/sites-available/
  sed -i "s/c1/${hostname}/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/nginx/sites-available/substrate.conf
  conf_sha=$(sha256sum ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/nginx/sites-available/substrate.conf | cut -d ' ' -f 1)
  
  mkdir -p ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/manta-indexer
  mkdir -p ${repo_path}/config/calamari.seabird.systems/${hostname}/var/lib/manta-indexer/migrations
  cp ${repo_path}/config/calamari.seabird.systems/c1/etc/manta-indexer/config.toml ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/manta-indexer/
  cp ${repo_path}/config/calamari.seabird.systems/c1/etc/manta-indexer/log.yaml ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/manta-indexer/
  cp ${repo_path}/config/calamari.seabird.systems/c1/var/lib/manta-indexer/migrations/0_schema.sql ${repo_path}/config/calamari.seabird.systems/${hostname}/var/lib/manta-indexer/migrations/

  cp ${repo_path}/config/calamari.seabird.systems/c1/config.yml ${repo_path}/config/calamari.seabird.systems/${hostname}/
  sed -i "s/shock-dedicated/hetzner-robot/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml
  sed -i "s/hostname: c1/hostname: ${hostname}/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml
  sed -i "s#/c1/#/${hostname}/#g" ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml
  sed -i "s/feb45e4af866c07fad361cd097c193fe25c47be4d12773bf03049dedcf7588fe/${unit_sha}/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml
  sed -i "s/e5c84554b1a8343a47f68b257c221ea8e2bb209aec8303bb64756707b785b71a/${conf_sha}/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml

  git --git-dir=${repo_path}/.git --work-tree=${repo_path} add ${repo_path}/config/calamari.seabird.systems/${hostname}
done
git --git-dir=${repo_path}/.git --work-tree=${repo_path} commit -m 'dolphin capacity'
git --git-dir=${repo_path}/.git --work-tree=${repo_path} push origin main

the commit/push above resulted in: https://github.com/Manta-Network/rubberneck/commit/2023644

manually created: https://github.com/Manta-Network/rubberneck/commit/b3c8666

extend rubberneck configurations to use statics from above commit

repo_path=~/git/Manta-Network/rubberneck
for hostname in a1 a2 a3; do
  old_unit_sha=$(sha256sum ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service | cut -d ' ' -f 1)
  sed -i "s#calamari/parachain-spec#substrate/calamari-staging#g" ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
  sed -i "s#calamari/relaychain-spec#substrate/kusama-staging#g" ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
  new_unit_sha=$(sha256sum ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service | cut -d ' ' -f 1)
  sed -i "s/${old_unit_sha}/${new_unit_sha}/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml
  git --git-dir=${repo_path}/.git --work-tree=${repo_path} add ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml
done
git --git-dir=${repo_path}/.git --work-tree=${repo_path} commit -m 'use dolphin statics'
git --git-dir=${repo_path}/.git --work-tree=${repo_path} push origin main

the commit/push above resulted in: https://github.com/Manta-Network/rubberneck/commit/fd5d616

extend rubberneck configurations to use statics from above commit

repo_path=~/git/Manta-Network/rubberneck
for hostname in a1 a2 a3; do
  git --git-dir=${repo_path}/.git --work-tree=${repo_path} add ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
done
git --git-dir=${repo_path}/.git --work-tree=${repo_path} commit -m 'use dolphin statics'
git --git-dir=${repo_path}/.git --work-tree=${repo_path} push origin main

the commit/push above resulted in: https://github.com/Manta-Network/rubberneck/commit/d958205

correction to deployed service to use *substrate* rather than *root* user for calamari service

repo_path=~/git/Manta-Network/rubberneck
for hostname in a1 a2 a3; do
  old_unit_sha=$(yq -r '.file[] | select(.target == "/etc/systemd/system/calamari.service") | .sha256' config/calamari.seabird.systems/${hostname}/config.yml)
  sed -i "s#/var/lib/calamari#/var/lib/substrate#" ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
  sed -i "s/root/substrate/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
  git --git-dir=${repo_path}/.git --work-tree=${repo_path} add ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service
  
  new_unit_sha=$(sha256sum ${repo_path}/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service | cut -d ' ' -f 1)
  sed -i "s/${old_unit_sha}/${new_unit_sha}/g" ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml
  git --git-dir=${repo_path}/.git --work-tree=${repo_path} add ${repo_path}/config/calamari.seabird.systems/${hostname}/config.yml
done
git --git-dir=${repo_path}/.git --work-tree=${repo_path} commit -m 'switch service user: root -> substrate'
git --git-dir=${repo_path}/.git --work-tree=${repo_path} push origin main
for hostname in a1 a2 a3; do
  fqdn=${hostname}.calamari.seabird.systems
  ssh mobula@${fqdn} "
    sudo systemctl stop calamari.service;
    sudo curl -Lo /etc/systemd/system/calamari.service https://raw.githubusercontent.com/Manta-Network/rubberneck/main/config/calamari.seabird.systems/${hostname}/etc/systemd/system/calamari.service;
    sudo mv /var/lib/calamari /var/lib/substrate;
    sudo chown -R substrate:substrate /var/lib/substrate;
    sudo systemctl daemon-reload;
    sudo systemctl start calamari.service;
  "
done
for hostname in a1 a2 a3; do
  fqdn=${hostname}.calamari.seabird.systems
  ssh mobula@${fqdn} sudo rm -rf /var/lib/calamari
done

the commit/push above resulted in: https://github.com/Manta-Network/rubberneck/commit/ca27f08