Keyboard shortcuts

Press ← or → to navigate between chapters

Press S or / to search in the book

Press ? to show this help

Press Esc to hide this help

snippets

genpasswd

LC_CTYPE=C.UTF-8 tr -dc 'A-Za-z0-9=_-' < /dev/urandom | head -c 32 | xargs

ssh

ssh proxy in one port

ssh -L $LOCALPORT:$REMOTEADDR:$REMOTEPORT $JUMPHOST

ssh socks proxy

ssh -D 8080 $JUMPHOST

After that use socks-proxy config in browser to localhost:8080

ssh gen pub key

ssh-keygen -f ~/.ssh/id_rsa -y > ~/.ssh/id_rsa.pub

sed remove all comments from file

sed -e '/^[[:blank:]]*[#;]/d;s/#.*//' -e '/^[[:space:]]*$/d' $file

hex to base64

echo '0xE1CB04A0fA36DdD16a06ea828007E35e1a3cBC37' | sed 's?0x??' | tr '[[:upper:]]' '[[:lower:]]' | xxd -ps -r | base64

regex fqdn

^(?!:\/\/)(?=.{1,255}$)((.{1,63}\.){1,127}(?![0-9]*$)[a-z0-9-]+\.?)$

network

get all open ports on server devided with ,

ss -tulpn | awk '{print $5}' | awk -F: '{print $NF}' | grep -v Local | sort | uniq | paste -d, -s

get host's ip addr

hostname -I # all addresses
curl ifconfig.me # outer ip address

nmap to check open ports

nmap -Pn $ADDR -p $PORTS

vim

set mouse-=a

from root

:! echo 'set mouse-=a' >> $VIMRUNTIME/defaults.vim

add lines to file without editor

tee -a ~/.ssh/config << END
Host localhost
  ForwardAgent yes
END

curl

curl write out format

{"http_code": %{http_code}, "time_namelookup": %{time_namelookup}, "time_connect": %{time_connect}, "time_appconnect": %{time_appconnect}, "time_pretransfer": %{time_pretransfer}, "time_redirect": %{time_redirect}, "time_starttransfer": %{time_starttransfer}, "time_total": %{time_total} }\n

or

curl -w @/Users/booger/occamfi/notes/_usefull/curl_write_out_format.txt -s -o/dev/null http://aeza.boogerman.xyz/joshuto.sh

arguments to script from curl

curl http://example.com/script.sh | bash -s -- arg1 arg2

bash

export .env

linux

export $(grep -v '^#' .env | xargs -d '\n')

macos

export $(grep -v '^#' .env | xargs -0)

bash locale problem

localedef -i en_US -f UTF-8 en_US.UTF-8

locale -a | grep UTF-8
locale-gen en_US.UTF-8
update-locale LANG=en_US.UTF-8
echo 'export LC_ALL=en_US.UTF-8' >> ~/.bashrc
echo 'export LANG=en_US.UTF-8' >> ~/.bashrc
source ~/.bashrc

Probably need to add to ssh config

Host your_remote_server
    SendEnv LANG LC_*

for loop

https://www.cyberciti.biz/faq/bash-for-loop-array/

prep.sh

curl https://aeza.boogerman.xyz/prep.sh | bash

swap file

dd if=/dev/zero of=/swapfile bs=1024M count=4 && \
chmod 600 /swapfile && \
mkswap /swapfile && \
swapon /swapfile && \
swapon -s && \
echo '/swapfile swap swap defaults 0 0' | tee -a /etc/fstab

ranger / joshuto

install joshito

RELEASE_VER='v0.9.8' INSTALL_PREFIX="/usr/local/bin" bash <(curl -s https://raw.githubusercontent.com/kamiyaa/joshuto/master/utils/install.sh)

or

curl -L https://github.com/kamiyaa/joshuto/releases/download/v0.9.8/joshuto-v0.9.8-x86_64-unknown-linux-musl.tar.gz | tar zx --strip-components=1  -C /usr/local/bin/

or

curl https://aeza.boogerman.xyz/joshuto.sh | bash

config for joshuto

##TODO: make safe for that

du ncdu dust

curl -L https://github.com/bootandy/dust/releases/download/v1.1.1/du-dust_1.1.1-1_amd64.deb -O && dpkg -i du-dust_1.1.1-1_amd64.deb && rm du-dust_1.1.1-1_amd64.deb

tar zstd

tar c /home | nice -n1 zstd --long --adapt -T0 --auto-threads=logical -c > /srv/home.tar.zst

k8s

get all pod with namespaces

k get pod -A -o go-template='{{ range $depl := .items }}{{ .metadata.namespace }}/{{.metadata.name }};{{end}}' | awk -F';' '{for(i=1; i<=NF; i++) print $i}'

get all depl with namespaces

k get deployments.apps -A -o go-template='{{ range $depl := .items }}{{ .metadata.namespace }}/{{.metadata.name }};{{end}}' | awk -F';' '{for(i=1; i<=NF; i++) print $i}'

get all ingresses with namespaces

k get ingresses -A -o go-template='{{ range $depl := .items }}{{ .metadata.namespace }}/{{.metadata.name }};{{end}}' | awk -F';' '{for(i=1; i<=NF; i++) print $i}'

helm: delete resources which generated via helm chart

helm template rel-Name path/to/chart --namespace ns | kubectl delete -f -

helm template from unified

helm template --debug -n default app oci://registry.fulgur.tech/library/chart

kubectl start debian bash

c
kubectl delete pod handbash

apt update
apt install -y curl dnsutils netcat-openbsd traceroute

kubectl add tls cert

kubectl get secrets --field-selector=type=kubernetes.io/tls
kubectl create secret tls fulgur.io --cert=path/to/tls.crt --key=path/to/tls.key

kubectl move from old cluster

kubectl --context old_context -n namespace get secret some_secret -oyaml | grep -vE 'creationTimestamp|namespace|resourceVersion|uid' | kubectl --context new_context apply -f -

check later

https://github.com/stakater/Reloader

yq

yq install

export VERSION=v4.45.4 BINARY=yq_$( uname | tr '[[:upper:]]' '[[:lower:]]' )_amd64
curl -L https://github.com/mikefarah/yq/releases/download/$VERSION/$BINARY.tar.gz | tar xz && mv $BINARY /usr/bin/yq

yq delete comments

cat sample.yaml | yq eval '... comments=""'

jq

cat docker.images.json | jq -r '.[] | [.ID,.Repository] | @tsv'

jq try to parse

cat $file | jq -R '. as $line | try (fromjson) catch $line'

graphql request

curl -g \
-X POST \
-H "Content-Type: application/json" \
-H "Authorization: Bearer 8c7dbd270cb98e83f9d8d57fb8a2ab7bac9d7501905fb013c69995ebf1b2a719" \
-d '{"query":"query{showCollection {items { title firstEpisodeDate lastEpisodeDate henshinMp4 { url }}}}"}' \
https://graphql.contentful.com/content/v1/spaces/mt0pmhki5db7

make

make help

help: ## Show this help
	@printf "\033[33m%s:\033[0m\n" 'Available commands'
	@awk 'BEGIN {FS = ":.*?## "} /^[[:alpha:][:punct:]]+:.*?## / {printf "  \033[32m%-18s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST)

make list

do not insert it to makefile - cause recursiveness

make -npq : 2> /dev/null | awk -v RS= -F: '$1 ~ /^[^.#%]+$/ { print $1 }' | xargs

rust

rust minimal install

curl https://sh.rustup.rs -sSf | sh -s -- -y --profile minimal --default-toolchain stable

cargo clean cache

cargo install cargo-cache && cargo cache -a

tracing in rust

usefull article

rustfmt.toml

tab_spaces = 4
hard_tabs = true
fn_params_layout = "Tall"
match_block_trailing_comma = true

optimizations

[profile.release]
opt-level = 3          # Highest optimization for speed (default in release, but explicit is fine).
lto = "fat"            # Enables full link-time optimization for better inlining and dead code elimination across crates.
codegen-units = 1      # Reduces parallelism in codegen to allow more aggressive optimizations (increases compile time).
panic = "abort"        # Aborts on panic instead of unwinding, reducing overhead and binary size.
strip = "symbols"      # Strips debug symbols and unused code for a leaner binary.

bash

export .env with spaces

export $(grep -v '^#' .env | xargs -d '\n')

#!/bin/sh
## Usage:
##   . ./export-env.sh ; $COMMAND
##   . ./export-env.sh ; echo ${MINIENTREGA_FECHALIMITE}
unamestr=$(uname)
if [ "$unamestr" = 'Linux' ]; then
  export $(grep -v '^#' .env | xargs -d '\n')
elif [ "$unamestr" = 'FreeBSD' ] || [ "$unamestr" = 'Darwin' ]; then
  export $(grep -v '^#' .env | xargs -0)
fi

date with most universe format

date '+%FT%T.%N%:z'

bash history keep more

# ~/.bashrc
export HISTTIMEFORMAT="%h %d %H:%M:%S "
export HISTSIZE=5000
export HISTFILESIZE=5000
shopt -s histappend
export PROMPT_COMMAND='history -a'

golnag

golang install

export VERSION=1.24.1 && mkdir -p /opt/go/$VERSION | curl -L https://go.dev/dl/go${VERSION}.linux-amd64.tar.gz | tar -xz -C /opt/go/${VERSION} && echo "export PATH=$PATH:/opt/go/$VERSION/go/bin" >> ~/.bashrc && source ~/.bashrc && go version

nginx

nginx basic auth

make htaccess file

	location / {
		auth_basic "Restricted Content";
		auth_basic_user_file /etc/nginx/.htpasswd;
	}

ingress controller logs parser in loki

nginx log regex https://regex101.com/r/eDdwzW/1

^(?P<client_ip>\S+) - (?P<remote_user>\S+) \[(?P<time_local>[^\]]+)\] "(?P<request>(?P<request_method>\S+) (?P<request_uri>\S+) (?P<request_httpv>\S+))?" (?P<status>\d+) (?P<body_bytes_sent>\d+) "(?P<http_referer>[^"]+)" "(?P<http_user_agent>[^"]+)" (?P<request_length>\d+) (?P<request_time>\d+\.\d+) \[(?P<proxy_upstream_name>\S+)?\] \[(?P<proxy_alternative_upstream_name>\S+)?\] (?P<upstream_addr>\S+(\-)?) ((?P<upstream_response_length>\d+)|-) ((?P<upstream_response_time>\d+\.\d+)|\-) ((?P<upstream_status>\d+)|-) (?P<req_id>\S+)

nginx conditional logging

map $status $loggable {
    ~^[23]  0;
    default 1;
}

access_log /path/to/access.log main if=$loggable;

nginx logrotate

/var/log/nginx/*.log {
	hourly
	maxsize 500M
	dateext
	missingok
	rotate 72
	compress
	#delaycompress
	notifempty
	create 0640 root root
	sharedscripts

	compresscmd /usr/bin/zstd
	uncompresscmd /usr/bin/unzstd
	compressoptions -9 --long -T0
	compressext .zst

	postrotate
		/usr/bin/docker exec nginx-balancer nginx -s reload
	endscript
}

openssl htaccess

printf "sbt-indexer:$(openssl passwd -apr1 cBGuVZz3rmGegcfjVPMkSu8F)\n" | tee -a /etc/nginx/.htpasswd

git

git archive

git archive --format tar.gz -o archive.tar.gz HEAD

git tags

git tag --sort=committerdate -l 'v*'
git ls-remote --tags origin

git checkout tag

git fetch --all --tags
git checkout tags/v1.0 -b v1.0-branch

git keep credentials to https

git config --global credential.helper store
git fetch

git latest tag

git tag --sort=committerdate | grep -E '[0-9]' | tail -1

git largest objects in history

git rev-list --objects --all --missing=print |
  git cat-file --batch-check='%(objecttype) %(objectname) %(objectsize) %(rest)' |
  sed -n 's/^blob //p' |
  sort --numeric-sort --key=2 |
  cut -c 1-12,41- |
  $(command -v gnumfmt || echo numfmt) --field=2 --to=iec-i --suffix=B --padding=7 --round=nearest |
  tail -30

.git/info/exclude

The TL;DR answer is that neither .git/info/exclude nor .gitignore have any effect on a tracked file. Using git update-index --skip-worktree (or --assume-unchanged) just make Git stop comparing the index version to the work-tree version.

nice every process for user

/etc/security/limits.conf

user hard priority 1

ipv6

preffer ipv4 addresses

resolve problem when curl or another http client trying to resolve name to ip and get ipv6 that can be not accessable

/etc/gai.conf

label  ::1/128       0
label  ::/0          1
label  2002::/16     2
label ::/96          3
label ::ffff:0:0/96  4
precedence  ::1/128       50
precedence  ::/0          40
precedence  2002::/16     30
precedence ::/96          20
precedence ::ffff:0:0/96  100

docker

restarted runners

docker ps | grep runner | grep Rest | cut -d' ' -f1 | xargs -L1 docker inspect | jq -r '.[]| .Config.Env[]' | grep REPO | tee >(wc -l)

docker compose resources limits

version: "3.8"
services:
  redis:
    image: redis:alpine
    deploy:
      resources:
        limits:
          cpus: '0.01'
          memory: 1000M

docker manifest

check that image exists on registry before build (from)

docker manifest inspect registry.domain.com/repo/image:tag

nodejs

start node js container in app dir

docker run -v "$(pwd)":/app -w /app -it --rm --entrypoint /bin/bash node:22.14
npm install -g pnpm@10.3.0 turbo@2.5.0

nodejs memory issue

export NODE_OPTIONS=--max-old-space-size=8192
# retry

python

python parse http response date

#datetime.datetime.strptime('Wed, 23 Sep 2009 22:15:29 GMT', '%a, %d %b %Y %H:%M:%S GMT')

resp = requests.get(URL)
date_str = resp.headers["Date"]
_dt = datetime.strptime(self.lastBlockTime_str, '%a, %d %b %Y %H:%M:%S GMT')
prom_timestamp = _dt.timestamp() * 1000

ip addresses pool

>>> import ipaddress
>>> [str(ip) for ip in ipaddress.IPv4Network('192.0.2.0/28')]
['192.0.2.0', '192.0.2.1', '192.0.2.2',
'192.0.2.3', '192.0.2.4', '192.0.2.5',
'192.0.2.6', '192.0.2.7', '192.0.2.8',
'192.0.2.9', '192.0.2.10', '192.0.2.11',
'192.0.2.12', '192.0.2.13', '192.0.2.14',
'192.0.2.15']

grafana

loki

count_over_time({cluster="proheku"}[1m])

github

clean workflow dir

      - uses: eviden-actions/clean-self-hosted-runner@v1
        if: ${{ always() }}

aws

ecr

export REGION=eu-central-1
export ACCOUNT=`aws sts get-caller-identity | jq -r .Account`
docker build -t ${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com/${REPO} .

aws ecr get-login-password --region ${REGION} | docker login --username AWS --password-stdin ${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com

docker push ${ACCOUNT}.dkr.ecr.${REGION}.amazonaws.com/${REPO}

s3 find by name in bucket

aws s3api list-objects --bucket $BUCKET_NAME --query "Contents[?contains(Key, '$FILENAME')]"

Find files modified on a given date

aws s3api list-objects --bucket $BUCKET_NAME --query "Contents[?contains(LastModified) < '2023-06-12')]"

Find files modified between given times

aws s3api list-objects --bucket $BUCKET_NAME --query "Contents[?LastModified > '2017-08-03T23' && LastModified < '2017-08-03T23:15']"

debian

noninteractive apt

export DEBIAN_FRONTEND=noninteractive && apt update && apt install -y --no-install-recommends pv file curl vim make jq ripgrep

apt mark do not update

apt-mark hold locales && apt update && apt install -y --no-install-recommends build-essential && apt-mark unhold locales

Force Apt-Get to IPv4 or IPv6 on Ubuntu or Debian

apt-get -o Acquire::ForceIPv4=true update
apt-get -o Acquire::ForceIPv6=true update

Persistent option

sudoedit /etc/apt/apt.conf.d/99force-ipv4
# Put the following contents in it:

Acquire::ForceIPv4 "true";

Vim

vim as default editor

sudo update-alternatives --set editor /usr/bin/vim.basic

some usefull vim config

echo 'syntax on\nfiletype plugin indent on\nset mouse-=a\nautocmd FileType yaml setlocal ts=2 sts=2 sw=2 expandtab' > /etc/vim/vimrc.local

problems with virtio_net

/etc/initramfs-tools/modules

virtio_pci
virtio_blk
virtio_net

then running sudo update-initramfs -u then rebooting

ubuntu

delete snap

snap list
### delete all snaps with
sudo snap remove --purge {}
sudo snap remove --purge bare
sudo snap remove --purge core20
sudo snap remove --purge snapd
sudo apt remove --autoremove snapd

sudo tee -a /etc/apt/preferences.d/nosnap.pref <<EOF
Package: snapd
Pin: release a=*
Pin-Priority: -10
EOF

dmesg from journalctl

journalctl -b

mysql

install on debian

export DEBIAN_FRONTEND=noninteractive \
&& apt update && apt install -y wget gnupg lsb-release && wget https://dev.mysql.com/get/mysql-apt-config_0.8.29-1_all.deb && dpkg -i ./mysql-apt-config_0.8.29-1_all.deb && apt update && apt install -y mysql-client

get size of databases

SELECT table_schema "DB Name",
        ROUND(SUM(data_length + index_length) / 1024 / 1024, 1) "DB Size in MB"
FROM information_schema.tables
GROUP BY table_schema;

show tables

show tables;

select to kill client connections

SELECT
CONCAT('KILL ', id, ';')
FROM INFORMATION_SCHEMA.PROCESSLIST
WHERE `User` = 'some_user'
AND `Host` = '192.168.1.1'
AND `db` = 'my_db';

redis

install redis-cli on debian

sudo apt-get install redis-tools

hetzner

use nat to access ipv4 egress

article

xml format

cat $some_xml_file | xmllint --format -

clickhouse

select size of tables/databases

select
    parts.*,
    columns.compressed_size,
    columns.uncompressed_size,
    columns.ratio
from (
    select database,
        table,
        formatReadableSize(sum(data_uncompressed_bytes))          AS uncompressed_size,
        formatReadableSize(sum(data_compressed_bytes))            AS compressed_size,
        sum(data_compressed_bytes) / sum(data_uncompressed_bytes) AS ratio
    from system.columns
    group by database, table
) columns right join (
    select database,
           table,
           sum(rows)                                            as rows,
           max(modification_time)                               as latest_modification,
           formatReadableSize(sum(bytes))                       as disk_size,
           formatReadableSize(sum(primary_key_bytes_in_memory)) as primary_keys_size,
           any(engine)                                          as engine,
           sum(bytes)                                           as bytes_size
    from system.parts
    where active
    group by database, table
) parts on ( columns.database = parts.database and columns.table = parts.table )
order by parts.bytes_size desc;

fail2ban

$ sudo cat /etc/fail2ban/jail.local
  [sshd]
+ backend=systemd
  enabled = true

mdbook

docs about book - https://rust-lang.github.io/mdBook/index.html

mermaid

flowchart LR
classDef done fill:#2f2,stroke:#111,color:#111,stroke-width:4px;
classDef inwork fill:#882,stroke:#111,color:#111,stroke-width:4px;
classDef conf fill:#55f,stroke:#111,color:#111,stroke-width:4px;

undone -->
conf:::conf -->
serv:::inwork -->
done:::done

ansible

adhoc copy file

ansible all --module-name copy --args "src=/tmp/foo.txt dest=/tmp/foo.txt remote_src=true"

macos

sudo powermetrics | grep "CPU Average frequency as fraction of nominal"

postgres

some output formatting options

postgresql pager off

\pset pager 0

postgres psql print only data

\t

postgres aling off

\a 0

postgresql pg_stat_statements longest query

SELECT query, total_time, calls, rows FROM  pg_stat_statements ORDER BY total_time DESC LIMIT 1;

postgres connections

SELECT pid, datname, usename, application_name,client_hostname, client_port, backend_start, query_start,query, state FROM pg_stat_activity 
-- WHERE state = 'active'
;

install client on debian

export DEBIAN_FRONTEND=noninteractive && apt update && apt -y --no-install-recommends install postgresql-client

systemd

user`s service

# enable user`s systemd units on boot (lingering)
export NONROOTUSER=galactica
loginctl enable-linger "$NONROOTUSER"
# check
loginctl list-users
# force to edit systemd unit file for user` service

su "$NONROOTUSER"
export DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$UID/bus"
export XDG_RUNTIME_DIR=/run/user/$(id -u)
echo 'export DBUS_SESSION_BUS_ADDRESS="unix:path=/run/user/$UID/bus"' >> ~/.bashrc
echo 'export XDG_RUNTIME_DIR=/run/user/$(id -u)' >>  ~/.bashrc
systemctl edit --user --force --full some-service.service
# enter systemd unit file
systemctl --user start some-service.service
systemctl --user enable some-service.service

[Unit]
Description=My Custom Service
After=default.target
Wants=mnt-kingston.automount

[Service]
Type=simple
WorkingDirectory=/mnt/kingston/influxdb-1.8.10-1
ExecStart=/mnt/kingston/influxdb-1.8.10-1/usr/bin/influxd run -config /mnt/kingston/influxdb-1.8.10-1/etc/influxdb/influxdb.conf
Restart=always

[Install]
WantedBy=default.target

systemctl --user start some-service.service

in some cases

unit for github runner under non root user

[Unit]
Description=Github runner for galactica ci tests in repo https://github.com/boogeroccam/galactica/settings/actions/runners
After=default.target

[Service]
Type=simple
WorkingDirectory=/home/devops
ExecStart=/home/devops/run.sh
Restart=always
RestartSec=5

[Install]
WantedBy=default.target

action runner

[Unit]
Description=github runner service
After=default.target

[Service]
Type=simple
Environment="PATH=/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/home/chat/.local/bin"
Environment=XDG_RUNTIME_DIR=/run/user/30001
Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/30001/bus
Environment=HOME=/home/chat
WorkingDirectory= /home/deploy/actions-runner/
ExecStart= /home/chat/actions-runner/run.sh
Restart=always
RestartSec=5

[Install]
WantedBy=default.target

systemd automount

sudo mkfs.ext4 /dev/xvdh
sudo mkdir /data
tee -a ~/mnt-kingston.mount <<EOT
[Unit]
Description=Mount partition

[Mount]
What=/dev/sdb1
Where=/mnt/kingston
Type=ext4
Options=rw,noatime

[Install]
WantedBy=multi-user.target
EOT
sudo mv ~/mnt-kingston.mount /etc/systemd/system/
tee -a ~/mnt-kingston.automount <<EOT
[Unit]
Description=Automount partition
ConditionPathExists=/mnt/kingston

[Automount]
Where=/mnt/kingston
TimeoutIdleSec=10

[Install]
WantedBy=multi-user.target
EOT
sudo mv ~/mnt-kingston.mount /etc/systemd/system/
sudo chown root:root /etc/systemd/system/mnt-kingston.mount /etc/systemd/system/mnt-kingston.automount
sudo chmod 755 /etc/systemd/system/mnt-kingston.mount /etc/systemd/system/mnt-kingston.automount

sudo systemctl daemon-reload
sudo systemctl enable --now data.automount

automount exfat

#!/bin/bash
path=/home/booger/subdisk
disk=/dev/nvme0n1p3
unit_filename=`echo $path | tr '/' '-' | cut -c 2-`
tee -a $unit_filename.mount <<EOT
[Unit]
Description=Mount partition

[Mount]
What=$disk
Where=$path
Type=exfat
Options=defaults,uid=1000,gid=1000,umask=002

[Install]
WantedBy=multi-user.target
EOT
sudo mv $unit_filename.mount /etc/systemd/system/
tee -a $unit_filename.automount <<EOT
[Unit]
Description=Automount partition
ConditionPathExists=$path

[Automount]
Where=$path
TimeoutIdleSec=10

[Install]
WantedBy=multi-user.target
EOT
sudo mv $unit_filename.automount /etc/systemd/system/
sudo chown root:root /etc/systemd/system/$unit_filename.mount /etc/systemd/system/$unit_filename.automount
sudo chmod 755 /etc/systemd/system/$unit_filename.mount /etc/systemd/system/$unit_filename.automount

sudo systemctl daemon-reload
sudo systemctl enable --now $unit_filename.automount

systemd

some adv unit for service

[Unit]
Description=pg-cache-sync service
After=network.target

[Service]
Type=simple
WorkingDirectory=/srv/nodesale-app/packages/pg-cache-sync/
ExecStart=/srv/nodesale-app/packages/pg-cache-sync/pg-cache-sync
Environment=DB_URL="postgresql://nsale:pass@10.24.1.7:5433/nsale"
Environment=HOST=0.0.0.0
Environment=PORT=8080
Environment=LOG_LEVEL=debug
Environment=LOG_FORMAT=json
Environment=SECRET_KEY=key
Environment=NODE_OPTIONS=--max-old-space-size=2048
Restart=always
RestartSec=5
LimitNOFILE=65535
TimeoutStopSec=30
; ProtectSystem=full
; ProtectHome=yes
PrivateTmp=true
NoNewPrivileges=true
SyslogIdentifier=pg-cache-sync
StandardOutput=syslog
StandardError=syslog
; MemoryLimit=500M
; CPUQuota=50%
; ExecStartPre=/usr/bin/bash -c "until nc -z -v -w30 10.24.1.7 5433; do echo 'Waiting for DB...'; sleep 5; done"

[Install]
WantedBy=multi-user.target

SOPS Usage Guide

Setup

Installation

You can install SOPS either by downloading the binary directly or using a package manager.

From Repository or Binary

  1. Visit the SOPS GitHub Repository.
  2. Download the binary (ensure you select the appropriate version for your OS and architecture): 
    curl -LO https://github.com/getsops/sops/releases/download/v3.9.2/sops-v3.9.2.linux.amd64
  3. Move the binary to your PATH: 
    mv sops-v3.9.2.linux.amd64 /usr/local/bin/sops
  4. Make the binary executable: 
    chmod +x /usr/local/bin/sops

tldr:

curl -LO https://github.com/getsops/sops/releases/download/v3.9.2/sops-v3.9.2.linux.amd64 && mv sops-v3.9.2.linux.amd64 /usr/local/bin/sops && chmod +x /usr/local/bin/sops

From a Package Manager (macOS Example)

Install SOPS using Homebrew:

brew install sops

GPG Keys Setup

Check Existing GPG Keys

To list your GPG keys:

  • Private keys: 
    gpg -K
  • Public keys: 
    gpg -k

Create a New GPG Key

Follow the GitHub guide for generating GPG keys. Key generation steps include:

  1. Generate a new key: 
    gpg --default-new-key-algo ed25519 --full-generate-key
  2. List the public key IDs: 
    gpg --list-secret-keys --keyid-format=long
  3. Export your public key: 
    gpg --armor --export ${YOUR_KEY_ID}

Add Your Public Key to GitHub

  1. Copy your GPG public key. It starts with: 
    -----BEGIN PGP PUBLIC KEY BLOCK-----
    And ends with: 
    -----END PGP PUBLIC KEY BLOCK-----
  2. Follow GitHub's guide to add the key.

Working with Encrypted Files

Basic Operations

Decrypting Files

To decrypt a file, you must have a private key listed in the sops metadata of the encrypted file.

Encrypting Files

To encrypt a file, you need all public keys that will be used for encryption. Add your public key to GitHub so others can import it from your profile.

For advanced use cases (e.g., encrypting with AWS KMS or Age), refer to the SOPS repository documentation.

Editing an Encrypted File

  1. Edit an encrypted file with your default editor: 
    sops file.yaml
  2. To use a specific editor: 
    EDITOR='code --wait' sops file.yaml
  3. Commit changes:
    • Add the modified lines to the git index.

Note: If changes are made outside SOPS, the file won't decrypt properly. Use the --ignore-mac option if needed.

Encrypting a File

To encrypt a file with a GPG key:

sops -e -i --pgp "${GPG_KEY_ID}"
# Options:
# -e: Encrypt
# -i: In-place encryption
# --pgp: Specify public GPG keys for encryption
# "${GPG_KEY_ID}" -- Public key ids separated with comma

Decrypting a File

  1. Decrypt to stdout: 
    sops -d file.yaml
  2. Decrypt in place: 
    sops -i -d file

gpg tty on macos

  • Проверить что which pinentry-mac что-то отдаст иначе установить через brew isntall pinentry-mac
  • Добавить в файл конфигурации ~/.gnupg/gpg-agent.conf
default-cache-ttl 34560000
max-cache-ttl 34560000
max-cache-ttl-ssh 34560000
default-cache-ttl-ssh 34560000
pinentry-program /usr/local/bin/pinentry-mac
  • export GPG_TTY=$(tty) и добавить эту строку в ~/.zshrc
  • Перезагрузить gpg-agent gpgconf --kill gpg-agent

Case Setting Up a New Repository for SOPS

Example .sops.yaml

Create a .sops.yaml file in the repository root. This file defines keys and rules for encryption.

# YAML Formatting:
stores:
  yaml:
    indent: 2
# Encryption Rules:
creation_rules:
    - pgp: >- ## List of GPG key IDs for encryption
        14E89E005155B68088D72693D7D61759A1EEEB5D,
        CD10A854ADDD2402BE1EF5AD075D1B59D139D890,
        5EF7079DE682A4BA200A834726EA0666E4FE5472,
        713429BA80BCB22675989D3AD1C9A13C324ECE5E
      ## <devops@occam.fi>,<mikhail@occam.fi>,<ilya@occam.fi>,<yakudgm@gmail.com>
      # encrypted_regex: ^SecretEnv  ## (Optional) Regex for encrypting specific YAML keys inside files
      # path_regex: values.*.yaml$  ## (Optional) Regex to target specific files

Encrupt files with sops

sops -e -i ops/helm-values/dev/app.sops.yaml

Case: decrypt in github actions

  • setup secret variable to get ci gpg key into action
export REPO=owner/repo
export VALUE=$(gpg --export-secret-keys 14E89E005155B68088D72693D7D61759A1EEEB5D | base64)
# with a github cli
gh secret set SOPS_CI_KEY --repo ${REPO} --body "$VALUE"
## gh secret set SOPS_CI_KEY --repo https://github.com/badaitech/bad-connector  --body "$(gpg --export-secret-keys 14E89E005155B68088D72693D7D61759A1EEEB5D | base64)"
# or in settings of the repo
  • make sure that gpg available in github runner
jobs:
   job:
      steps:
         - uses: mdgreenwald/mozilla-sops-action@v1.6.0
         - name: import gpg ci key
           run: |
            echo ${{ secrets.SOPS_CI_KEY }} | base64 -d | gpg --import
         # ... do stuff
         - name: deploy
           run: |
             sops -d config.sops.yaml > config.yaml
         - name: delete ci gpg key
           if: always()
           run: |
               gpg -K --with-colons | awk -F: '/^(fpr)/ {print $10; exit}' | xargs gpg --delete-secret-and-public-keys --yes --batch

Case: replace one key with another

export oldkey=9585F123A7E98549AD7E88FA300C05FE86177A5C
export newkey=96F339BEC4A967D173406DE0E189CDDBCAF01351
rg "fp: $oldkey" -l | xargs -n1 sops -r -i --add-pgp=$newkey --rm-pgp=$oldkey

Advanced case: with ansible

https://docs.ansible.com/ansible/latest/collections/community/sops/load_vars_module.html

- name: Configure PostgreSQL Role and Database
  hosts: psql-prod
  become: true
  vars:
    db_environment: bad_connector_stage
  pre_tasks:
    - name: Include variables of secrets.sops.yaml into the variable
      community.sops.load_vars:
        file: ./../secret.sops.yaml
        name: secret
    - name: Set variables from secret
      set_fact:
        db_user: "{{ secret['db_environment'][db_environment]['db_user'] }}"
        db_password: "{{ secret['db_environment'][db_environment]['db_password'] }}"
        db_database: "{{ secret['db_environment'][db_environment]['db_database']}}"

  tasks:

hetzner rescue

installimage

by default hetzner offers rescue os whem you add server

From start you should check hardware (disks,cpu,etc)

After that cmd installimage

Input

DRIVE1 /dev/sda
DRIVE2 /dev/sdb
FORMATDRIVE1 1
FORMATDRIVE2 1

SWRAID 0
SWRAIDLEVEL 0

PART /boot/efi esp 256M
PART /boot ext3 1024M
PART lvm vg0 all
LV vg0 root / ext4 all

BOOTLOADER grub

HOSTNAME machine
#IPV4_ONLY yes
#IMAGE /root/images/Ubuntu-2204-jammy-amd64-base.tar.gz

Galactica node rescue config

echo <<ECHO | tee -a install-config
DRIVE1 /dev/nvme0n1
DRIVE2 /dev/nvme1n1
DRIVE3 /dev/nvme2n1

FORMATDRIVE1 1
FORMATDRIVE2 1
FORMATDRIVE3 1

SWRAID 0
SWRAIDLEVEL 0

BOOTLOADER grub

HOSTNAME gala-reti-bm-node02

PART /boot/efi esp 256M
PART /boot ext3 512M
PART / ext4 50G
PART lvm vg0 all
LV vg0 root /galactica ext4 all

IMAGE /root/images/Debian-bookworm-latest-amd64-base.tar.gz
ECHO
installimage -c install-config -a

In guest os to add lvm:

pvcreate /dev/nvme1n1
vgextend vg0 /dev/nvme1n1
lvextend -l +100%FREE /dev/vg0/root
resize2fs /dev/vg0/root

or

for disk in nvme0n1 nvme1n1 ; do echo $disk ; pvcreate /dev/$disk && vgextend vg0 /dev/$disk && lvextend -l +100%FREE /dev/vg0/root ; done && resize2fs /dev/vg0/root

in vm

sysctl -w net.ipv6.conf.all.disable_ipv6=1
sysctl -w net.ipv6.conf.default.disable_ipv6=1


ufw allow 22
ufw enable

# apt install fail2ban
# systemctl status fail2ban.service
# vim /etc/fail2ban/jail.local

crowdsec / fail2ban

curl -s https://packagecloud.io/install/repositories/crowdsec/crowdsec/script.deb.sh | sudo bash
sudo apt-get update
sudo apt-get install crowdsec

[INCLUDES]
before = paths-debian.conf
[DEFAULT]
ignorecommand =
bantime  = 10m
findtime  = 10m
maxretry = 5
maxmatches = %(maxretry)s
backend = auto
usedns = warn
logencoding = auto
enabled = false
mode = normal
filter = %(__name__)s[mode=%(mode)s]
destemail = root@localhost
sender = root@<fq-hostname>
mta = sendmail
protocol = tcp
chain = <known/chain>
port = 0:65535
fail2ban_agent = Fail2Ban/%(fail2ban_version)s
banaction = iptables-multiport
banaction_allports = iptables-allports
action_ = %(banaction)s[port="%(port)s", protocol="%(protocol)s", chain="%(chain)s"]
action_mw = %(action_)s
            %(mta)s-whois[sender="%(sender)s", dest="%(destemail)s", protocol="%(protocol)s", chain="%(chain)s"]
action_mwl = %(action_)s
             %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
action_xarf = %(action_)s
             xarf-login-attack[service=%(__name__)s, sender="%(sender)s", logpath="%(logpath)s", port="%(port)s"]
action_cf_mwl = cloudflare[cfuser="%(cfemail)s", cftoken="%(cfapikey)s"]
                %(mta)s-whois-lines[sender="%(sender)s", dest="%(destemail)s", logpath="%(logpath)s", chain="%(chain)s"]
action_blocklist_de  = blocklist_de[email="%(sender)s", service="%(__name__)s", apikey="%(blocklist_de_apikey)s", agent="%(fail2ban_agent)s"]
action_badips = badips.py[category="%(__name__)s", banaction="%(banaction)s", agent="%(fail2ban_agent)s"]
action_badips_report = badips[category="%(__name__)s", agent="%(fail2ban_agent)s"]
action_abuseipdb = abuseipdb
action = %(action_)s
[sshd]
port    = ssh
logpath = %(sshd_log)s
backend = %(sshd_backend)s
[dropbear]
port     = ssh
logpath  = %(dropbear_log)s
backend  = %(dropbear_backend)s
[selinux-ssh]
port     = ssh
logpath  = %(auditd_log)s

systemctl restart fail2ban


DRIVE1 /dev/nvme0n1
DRIVE2 /dev/nvme0n1
FORMATDRIVE1 1
FORMATDRIVE2 1

SWRAID 0
SWRAIDLEVEL 0

PART /boot/efi esp 256M
PART /boot ext3 1024M
PART btrfs.1 btrfs all
SUBVOL btrfs.1 @ /
SUBVOL btrfs.1 @root /root
SUBVOL btrfs.1 @log /var/log

BOOTLOADER grub

HOSTNAME cl-0.infra.badai.io
#IPV4_ONLY yes
IMAGE /root/images/Debian-trixie-latest-amd64-base.tar.gz

nginx

get source ip from cloudflare

The problem - we log only cf addresses if we not configure nginx properly

curl -s https://www.cloudflare.com/ips-v4/ | awk '{print "set_real_ip_from "$1 ";"}'
curl -s https://www.cloudflare.com/ips-v6/ | awk '{print "set_real_ip_from "$1 ";"}'

http {

set_real_ip_from 173.245.48.0/20;
set_real_ip_from 103.21.244.0/22;
set_real_ip_from 103.22.200.0/22;
set_real_ip_from 103.31.4.0/22;
set_real_ip_from 141.101.64.0/18;
set_real_ip_from 108.162.192.0/18;
set_real_ip_from 190.93.240.0/20;
set_real_ip_from 188.114.96.0/20;
set_real_ip_from 197.234.240.0/22;
set_real_ip_from 198.41.128.0/17;
set_real_ip_from 162.158.0.0/15;
set_real_ip_from 104.16.0.0/13;
set_real_ip_from 104.24.0.0/14;
set_real_ip_from 172.64.0.0/13;
set_real_ip_from 131.0.72.0/22;
set_real_ip_from 2400:cb00::/32;
set_real_ip_from 2606:4700::/32;
set_real_ip_from 2803:f800::/32;
set_real_ip_from 2405:b500::/32;
set_real_ip_from 2405:8100::/32;
set_real_ip_from 2a06:98c0::/29;
set_real_ip_from 2c0f:f248::/32;

real_ip_header CF-Connecting-IP;
}

basic-auth

read -p "Enter username: " username && read -sp "Enter password: " password && echo "$username:$(openssl passwd -apr1 $password)" | sudo tee -a /etc/nginx/.htaccess && echo -e "\nCredentials added successfully."

    location / {
        # Basic authentication setup
        auth_basic "Restricted Access";
        auth_basic_user_file /etc/nginx/.htaccess ;
    }

fix upstream sent too big header

 proxy_busy_buffers_size   512k;
 proxy_buffers   4 512k;
 proxy_buffer_size   256k;

nginx ingress cotroller config snippet

https://ellie.wtf/notes/ingress-nginx-risky-annotations

Today I found myself needing to configure ingress-nginx. I needed to write a bit of nginx config to rewrite status codes for certain routes.

Something like

nginx.ingress.kubernetes.io/configuration-snippet: |-
    location /metrics {
	    return 404;
    }

I’ve done this many times in the past, but today I received the following error

Error: UPGRADE FAILED: cannot patch "xyz" with kind Ingress: admission webhook "validate.nginx.ingress.kubernetes.io" denied the request: annotation group ConfigurationSnippet contains risky annotation based on ingress configuration

I already had

allowSnippetAnnotations: true

set, so this was confusing!

It turns out, in a recent release (controller 1.12), annotations are flagged by risk. There’s a table here

You now need to specify

annotations-risk-level: Critical

in the configmap. If you’re using the helm chart, it can be added like so

controller:
  config:
    annotations-risk-level: Critical

Note that this change is a reaction to a security issue. This is mostly an issue if you’re using a multi-tenant cluster.

webdav setup with nginx

install nginx

With webdav module

apt install libxslt-dev --no-install-recommends
make -f nginxbuild.mk install DAV=true

add /etc/nginx/webdav.passwd

read -p "Enter username: " username && read -sp "Enter password: " password && echo "$username:$(openssl passwd -apr1 $password)" | sudo tee -a /etc/nginx/webdav.passwd && echo -e "\nCredentials added successfully."
usermod -aG booger nginx
usermod -aG nginx booger
chmod g+rwx /srv/webdav/

nginx server conf

server {
	#listen 80 ;
	listen 443 ssl ;
	http2 on;
	server_name webdav.aeza.boogerman.xyz ;

	ssl_certificate /home/booger/.acme.sh/aeza.boogerman.xyz_ecc/fullchain.cer;
	ssl_certificate_key /home/booger/.acme.sh/aeza.boogerman.xyz_ecc/aeza.boogerman.xyz.key;

	ssl_session_cache shared:SSL:1m;
	ssl_session_timeout 5m;

	ssl_ciphers HIGH:!aNULL:!MD5;
	ssl_prefer_server_ciphers on;

	root /srv/webdav ;

	location ~ /\. {
		deny all;
		access_log off;
		log_not_found off;
	}

	location / {
		dav_methods PUT DELETE MKCOL COPY MOVE;
		dav_ext_methods PROPFIND OPTIONS;
		dav_access user:rw group:rw;

		client_max_body_size 0;
		create_full_put_path on;
		client_body_temp_path /tmp/;

		# Basic authentication setup
		auth_basic "Restricted Access";
		auth_basic_user_file /etc/nginx/webdav.passwd;

		# Deny all access unless authenticated
		satisfy all;
		allow all; # This allows all authenticated users
		deny all; # This denies all other users

		# Deny all access except zelenika
		#allow 62.4.35.1 ;
		#deny all ;
	}
}

Pulumi

  • setup
  • example new project
  • tips

install & initial setup

brew install pulumi
pulumi login s3://occam-tf-state/pulumi?region=eu-central-1
export PULUMI_CONFIG_PASSPHRASE=""
echo '\nexport PULUMI_CONFIG_PASSPHRASE=""\n' >> ~/.zshrc

python - uv

Потому что в качестве мэнеджера пакетов для питона я использую его

curl -LsSf https://astral.sh/uv/install.sh | sh

Example project

create project

mkdir pulumi-example
cd $_
pulumi new python
# enter name
# enter description
# use uv
# enter stack name ususally main

add modules

google pulumi <something> - goes to module and get module for python

cloudflare

uv add pulumi-cloudflare
# Add config for module, usually api keys to access provider (cloudflare api key with access to account zones read and dns in zone read/write)
pulumi config set --secret cloudflare:apiToken ${TOKEN}

import zone or use its string values

hetzner

uv add pulumi-hcloud
# Add config for module, usually api keys to access provider (hetzner project api key)
pulumi config set hcloud:token --secret ${API_KEY}

import network and import subnet and after that use attach to apply propper ip

main cli operations

preview (plan)

pulumi preview

up (apply)

pulumi up

show resources with urn

pulumi stack --show-urns

get updates

if some resources updated mannually

pulumi state upgrade

delete resources with urn

in cases where import was success but some variables was renamed/replaced

pulumi state delete urn:pulumi:main::pulumi-cloudflare::cloudflare:index/zone:Zone::thearchai --force

destroy without protected objects

pulumi destroy --exclude-protected

tips

hetzner network

import pulumi
import pulumi_hcloud as hcloud

import variables

reticulum_network = hcloud.Network(
    "reticulum_network",
    ip_range="10.67.0.0/16",
    labels={
        "Name": "Galactica-reticulum",
    },
    name="reticulum",
    opts=pulumi.ResourceOptions(protect=True),
)

reticulum_network_subnet = hcloud.NetworkSubnet(
    "reticulum_network_subnet",
    ip_range="10.67.0.0/24",
    network_id=reticulum_network.id,
    network_zone="eu-central",
    type="cloud",
    opts=pulumi.ResourceOptions(protect=True),
)

safe_server = hcloud.Server(
    "safe_server",
    name="gala-reti-safe",
    image="debian-12",
    server_type="cpx31",
    public_nets=[
        {
            "ipv4_enabled": True,
            "ipv6_enabled": True,
        }
    ],
    location="fsn1",
    ssh_keys=variables.common_ssh_keys,
    labels=variables.common_labels,
)

net_attach = hcloud.ServerNetwork(
    "safe-server-netattach",
    server_id=safe_server.id,
    network_id=reticulum_network.id,
    ip="10.67.0.202",
)

Using SOPS with Pulumi

  • safe PULUMI_CONFIG_PASSPHRASE to sops file

    1. encrypt PULUMI_CONFIG_PASSPHRASE with sops into file
    2. sops exec-env encrypted-config-file.json 'pulumi preview'

  • safe config to sops file

    sops exec-file sops.yaml 'pulumi up --config-file {}'

  • load config from file as resource

    https://sarg.org.ru/blog/pulumi-sops/ -- ts implementation

outputs TBD

Table of Contents

  1. Install libvirt
  2. Install Pulumi
  3. Setup Pulumi project and Dev Stack
  4. Create a VM
  5. Create filesystem volume
  6. Fix libvirt permission errors
  7. Attach virtual console to VM
  8. Use cloud-init to setup ubuntu user
  9. Setup network so we can SSH into VM
  10. Add Pulumi outputs
  11. Enable autostart
  12. Use Pulumi provider and config to support multiple stacks

Pulumi is an Infrastructure as Code (IaC) tool that supports using Go, .Net, Python, and TypeScript/JavaScript. Libvirt is a tool for managing virtual machines (VM). Typically, teams use Pulumi with different cloud providers, but we can leverage libvirt to manage virtual machines on bare-metal servers, perfect for a homelab.

We’ll go through the steps of setting up libvirt and Pulumi to run against our local machine to create a VM running Ubuntu 20.04. This post caters to folks that have never used libvirt or Pulumi.

I’m running this on Ubuntu 21.10 x64. We’ll use Pulumi v3.22.1 with the Go SDK and Go v1.17.6.

Note: If you just want to see the code, visit pulumi-libvirt-ubuntu-example.

Install libvirt

First, install libvirt via:

sudo apt install qemu-kvm libvirt-daemon-system

This will create a systemd service and automatically start running libvirt.

By default, our regular user can’t interact with libvirt without running as root. Fortunately, we can add ourselves to the libvirt group.

sudo adduser $USER libvirt

You’ll need to log out and back in for this to take effect.

Then verify the user can interact with libvirt by running:

virsh list

and you should see the following output:

 Id   Name   State
--------------------

This command shows us the list of domains running.

Libvirt calls virtual machines domains. I’ll be using these terms interchangeably throughout this post.

Install Pulumi

Install Pulumi v3.22.1 for Linux x64 by running the following commands, which will download, extract, and move the required binaries to /usr/local/bin/.

cd ~
wget https://get.pulumi.com/releases/sdk/pulumi-v3.22.1-linux-x64.tar.gz
tar \
  --extract \
  --file pulumi-v3.22.1-linux-x64.tar.gz \
  --gzip
sudo mv ~/pulumi/pulumi /usr/local/bin/
sudo mv ~/pulumi/pulumi-language-go /usr/local/bin/

This will move the pulumi and pulumi-language-go binaries to /usr/local/bin to make them available in our $PATH.

pulumi version

should output:

v3.22.1

Since we’re using Go in this post, we’ve only copied the pulumi-language-go binary. For other languages, copy the respective language binaries.

Setup Pulumi project and Dev Stack

Now that we’ve installed libvirt and Pulumi, we can begin creating our Pulumi project.

Create and navigate to a new directory at ~/pulumi-libvirt-ubuntu by running:

mkdir ~/pulumi-libvirt-ubuntu
cd ~/pulumi-libvirt-ubuntu

Before we can begin using Pulumi we need to specify a backend to save our infrastructure state. Pulumi supports multiple backends such as S3 and their own hosted service. For convenience, we’ll use our local filesystem to store our state.

Let’s create a directory to hold our project’s state and then log in using the newly created directory:

mkdir ~/pulumi-libvirt-ubuntu-state
pulumi login file://~/pulumi-libvirt-ubuntu-state

NOTE: pulumi login supports a --local option, defaulting to using ~/ to save state. This causes issues when dealing with multiple Pulumi projects because they’ll start sharing state. So better to create separate directories for each project.

Let’s create a new Pulumi project setup for Go via:

pulumi new go \
  --description "Creates a Ubuntu 20.04 VM via libvirt" \
  --name pulumi-libvirt-ubuntu \
  --stack dev

After running this, Pulumi will prompt asking for a passphrase for our dev stack. Provide one, re-enter it, and enter it again to finish setting up the Pulumi project and our dev stack.

Note: you can think of a stack as an environment. Later, we’ll learn how to create another stack such as prod.

The above command scaffolds out a Pulumi project that looks like this:

.
├── go.mod
├── go.sum
├── main.go
├── Pulumi.dev.yaml
└── Pulumi.yaml
  • go.mod and go.sum are created for us with dependencies needed by Pulumi.
  • main.go contains our actual code for creating our infrastructure, which we’ll be focused on in this post.
  • Pulumi.yaml is info about the Project such as name, description, and the runtime (go in this case).
  • Pulumi.dev.yaml is configuration for our dev stack. You can think of a stack as an environment. So we can eventually deploy our VM to our dev stack, and we could create a new stack named prod.

Create a VM

We’ll need to install the Pulumi libvirt provider before creating VMs.

To install this provider, run:

go get github.com/pulumi/pulumi-libvirt/sdk@v0.2.1

Update main.go to look like:

package main

import (
	"github.com/pulumi/pulumi-libvirt/sdk/go/libvirt"
	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
)

func main() {
	pulumi.Run(func(ctx *pulumi.Context) error {
		// create a provider, this isn't required, but will make it easier to configure
		// a libvirt_uri, which we'll discuss in a bit
		provider, err := libvirt.NewProvider(ctx, "provider", &libvirt.ProviderArgs{})
		if err != nil {
			return err
		}

		// create a VM that has a name starting with ubuntu
		_, err = libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{}, pulumi.Provider(provider))
		if err != nil {
			return err
		}

		return nil
	})
}

Create an environment variable named PULUMI_CONFIG_PASSPHRASE, so the Pulumi CLI can decrypt our stack by running:

export PULUMI_CONFIG_PASSPHRASE=password

where password is the passphrase used when creating the dev stack.

Next, run the following command to create our domain:

pulumi up

Pulumi will install dependencies and then display a preview such as:

Previewing update (dev):
     Type                     Name                       Plan
 +   pulumi:pulumi:Stack      pulumi-libvirt-ubuntu-dev  create
 +   └─ libvirt:index:Domain  ubuntu                     create

Select “yes” to make the domain.

Afterward, Pulumi will output that it created two resources (the stack and the domain).

We can verify the libvirt VM exists by running:

virsh list

which will output

 Id   Name             State
--------------------------------
 1    ubuntu-0b14e16   running

The ubuntu-0b14e16 VM isn’t doing much for now. Let’s work on creating a volume and giving our VM a filesystem.

Create filesystem volume

We can install Ubuntu on our domain by creating a new volume from a Ubuntu ISO, and then creating another volume to act as the actual filesystem for the VM to use based on the Ubuntu volume.

Modify main.go to create a storage pool for our volumes and two volumes:

 import (
 	"github.com/pulumi/pulumi-libvirt/sdk/go/libvirt"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
 )

 func main() {
 	pulumi.Run(func(ctx *pulumi.Context) error {
 		// create a provider, this isn't required, but will make it easier to configure
 		// a libvirt_uri, which we'll discuss in a bit
 		provider, err := libvirt.NewProvider(ctx, "provider", &libvirt.ProviderArgs{})
 		if err != nil {
 			return err
 		}

+		// `pool` is a storage pool that can be used to create volumes
+		// the `dir` type uses a directory to manage files
+		// `Path` maps to a directory on the host filesystem, so we'll be able to
+		// volume contents in `/pool/cluster_storage/`
+		pool, err := libvirt.NewPool(ctx, "cluster", &libvirt.PoolArgs{
+			Type: pulumi.String("dir"),
+			Path: pulumi.String("/pool/cluster_storage"),
+		}, pulumi.Provider(provider))
+		if err != nil {
+			return err
+		}
+
+		// create a volume with the contents being a Ubuntu 20.04 server image
+		ubuntu, err := libvirt.NewVolume(ctx, "ubuntu", &libvirt.VolumeArgs{
+			Pool:   pool.Name,
+			Source: pulumi.String("https://cloud-images.ubuntu.com/releases/focal/release/ubuntu-20.04-server-cloudimg-amd64.img"),
+		}, pulumi.Provider(provider))
+		if err != nil {
+			return err
+		}
+
+		// create a filesystem volume for our VM
+		// This filesystem will be based on the `ubuntu` volume above
+		// we'll use a size of 10GB
+		filesystem, err := libvirt.NewVolume(ctx, "filesystem", &libvirt.VolumeArgs{
+			BaseVolumeId: ubuntu.ID(),
+			Pool:         pool.Name,
+			Size:         pulumi.Int(10000000000),
+		}, pulumi.Provider(provider))
+		if err != nil {
+			return err
+		}
+
		// create a VM that has a name starting with ubuntu
-		_, err = libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{}, pulumi.Provider(provider))
+		_, err = libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{
+			Disks: libvirt.DomainDiskArray{
+				libvirt.DomainDiskArgs{
+					VolumeId: filesystem.ID(),
+				},
+			},
+		}, pulumi.Provider(provider))
 		if err != nil {
 			return err
 		}

 		return nil
 	})
 }

Then run

pulumi up

Select yes after viewing the preview. Pulumi will create a storage pool, download the Ubuntu image, and then hit a Permission denied error when attempting to use the image like:

error: error creating libvirt domain: internal error: process exited while connecting to monitor: 2022-01-12T01:36:34.005110Z qemu-system-x86_64: -blockdev {"driver":"file","filename":"/pool/cluster_storage/ubuntu-552ab14","node-name":"libvirt-2-storage","auto-read-only":true,"discard":"unmap"}: Could not open '/pool/cluster_storage/ubuntu-552ab14': Permission denied

Fix libvirt permission errors

There are a few ways to handle this, but the easiest solution is disabling SELinux.

To disable SELinux, modify /etc/libvirt/qemu.conf:

sudo sed --in-place 's/#security_driver = "selinux"/security_driver = "none"/' /etc/libvirt/qemu.conf

Then restart libvirtd for this config change to take effect.

sudo systemctl restart libvirtd

And finally, re-try running:

pulumi up

It’ll be successful this time. We can then run the following commands to see the impact on libvirt.

virsh list

will output a new domain:

Id   Name             State
--------------------------------
 2    ubuntu-e629e71   running

virsh pool-list

will show our storage pool:

 Name              State    Autostart
---------------------------------------
 cluster-1d3f78e   active   yes

and we can view our volumes by running:

virsh vol-list cluster-1d3f78e

will show our volumes in our pool:

 Name                 Path
----------------------------------------------------------------
 filesystem-103d88a   /pool/cluster_storage/filesystem-103d88a
 ubuntu-552ab14       /pool/cluster_storage/ubuntu-552ab14

So now we have a VM with Ubuntu 20.04 running, but we cannot interact with it just yet.

Attach virtual console to VM

We can attach a virtual console to our VM so we can login from a terminal.

Modify main.go so the domain has a console attached like:

		// create a VM that has a name starting with ubuntu
 		_, err = libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{
+			Consoles: libvirt.DomainConsoleArray{
+				// enables using `virsh console ...`
+				libvirt.DomainConsoleArgs{
+					Type:       pulumi.String("pty"),
+					TargetPort: pulumi.String("0"),
+					TargetType: pulumi.String("serial"),
+				},
+			},
 			Disks: libvirt.DomainDiskArray{
 				libvirt.DomainDiskArgs{
 					VolumeId: filesystem.ID(),
 				},
 			},
-		}, pulumi.Provider(provider))
+			// delete existing VM before creating replacement to avoid two VMs trying to use the same volume
+		}, pulumi.Provider(provider), pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))
 		if err != nil {
 			return err
 		}

Note: pay attention to the ReplaceOnChanges and DeleteBeforeReplace gotcha. Since only a single domain can use our volumes at once, we need to delete the existing domain before creating a new one.

Run pulumi up again to create a new VM.

Get the name of the VM from virsh list, and then we can access the VM by running:

virsh console ubuntu-3c69e6a

Press enter to be access a username and password prompt. Unfortunately, there isn’t a default password for ubuntu, so we’re can’t login, yet.

virsh console ... can be great for debugging issues such as cloud-init, which we’ll do next.

Note: to exit the console, hold CTRL and press ].

Use cloud-init to setup ubuntu user

We can leverage cloud-init to create credentials for the ubuntu user, amongst other things.

Create a new file named cloud_init_user_data.yaml.

touch ~/pulumi-libvirt-ubuntu/cloud_init_user_data.yaml

with the following content:

#cloud-config
ssh_pwauth: True
chpasswd:
  list: |
    ubuntu:ubuntu    
  expire: False

Now update main.go so that we create a cloud-init resource and initialize the VM with the cloud-init mounted.

+		cloud_init_user_data, err := os.ReadFile("./cloud_init_user_data.yaml")
+		if err != nil {
+			return err
+		}
+
+		// create a cloud init disk that will setup the ubuntu credentials
+		cloud_init, err := libvirt.NewCloudInitDisk(ctx, "cloud-init", &libvirt.CloudInitDiskArgs{
+			MetaData: pulumi.String(string(cloud_init_user_data)),
+			Pool:     pool.Name,
+			UserData: pulumi.String(string(cloud_init_user_data)),
+		}, pulumi.Provider(provider))
+		if err != nil {
+			return err
+		}
+
 		// create a VM that has a name starting with ubuntu
 		_, err = libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{
+			Cloudinit: cloud_init.ID(),
 			Consoles: libvirt.DomainConsoleArray{
 				// enables using `virsh console ...`
 				libvirt.DomainConsoleArgs{
 					Type:       pulumi.String("pty"),
 					TargetPort: pulumi.String("0"),
 					TargetType: pulumi.String("serial"),
 				},
 			},
 			Disks: libvirt.DomainDiskArray{
 				libvirt.DomainDiskArgs{
 					VolumeId: filesystem.ID(),
 				},
 			},
 			// delete existing VM before creating replacement to avoid two VMs trying to use the same volume
 		}, pulumi.Provider(provider), pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))
 		if err != nil {
 			return err
 		}

Run pulumi up again.

Now, get the name of the VM again with virsh list and execute virsh console NAME_OF_VM.

Press enter, and you can then log in with the ubuntu username and ubuntu password. You may need to wait a few minutes for cloud-init to complete before these credentials are valid.

This is great, but most of the time we’ll want to SSH instead. Let’s get that working.

Setup network so we can SSH into VM

Currently, our VM doesn’t have an IP address that we can connect to for SSH. We’ll need a libvirt network to attach our VM to and configure our VM to get an IP address from libvirt’s DHCP server automatically.

Create a new file named cloud_init_network_config.yaml

touch ~/pulumi-libvirt-ubuntu/cloud_init_network_config.yaml

with the following content:

version: 2
ethernets:
  ens3:
    dhcp4: true

We’ll add this to our cloud-init, so the VM will attempt to get an IP address assigned at boot up.

Update main.go to add this network config to cloud-init and create a libvirt network.

+		cloud_init_network_config, err := os.ReadFile("./cloud_init_network_config.yaml")
+		if err != nil {
+			return err
+		}
+
-		// create a cloud init disk that will setup the ubuntu credentials
+		// create a cloud init disk that will setup the ubuntu credentials and enable dhcp
 		cloud_init, err := libvirt.NewCloudInitDisk(ctx, "cloud-init", &libvirt.CloudInitDiskArgs{
 			MetaData:      pulumi.String(string(cloud_init_user_data)),
+			NetworkConfig: pulumi.String(string(cloud_init_network_config)),
 			Pool:          pool.Name,
 			UserData:      pulumi.String(string(cloud_init_user_data)),
 		}, pulumi.Provider(provider))
 		if err != nil {
 			return err
 		}

+		// create NAT network using 192.168.10/24 CIDR
+		network, err := libvirt.NewNetwork(ctx, "network", &libvirt.NetworkArgs{
+			Addresses: pulumi.StringArray{pulumi.String("192.168.10.0/24")},
+			Mode:      pulumi.String("nat"),
+		}, pulumi.Provider(provider))
+		if err != nil {
+			return err
+		}
+
 		// create a VM that has a name starting with ubuntu
 		_, err = libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{
 			Cloudinit: cloud_init.ID(),
 			Consoles: libvirt.DomainConsoleArray{
 				// enables using `virsh console ...`
 				libvirt.DomainConsoleArgs{
 					Type:       pulumi.String("pty"),
 					TargetPort: pulumi.String("0"),
 					TargetType: pulumi.String("serial"),
 				},
 			},
 			Disks: libvirt.DomainDiskArray{
 				libvirt.DomainDiskArgs{
 					VolumeId: filesystem.ID(),
 				},
 			},
+			NetworkInterfaces: libvirt.DomainNetworkInterfaceArray{
+				libvirt.DomainNetworkInterfaceArgs{
+					NetworkId:    network.ID(),
+					WaitForLease: pulumi.Bool(true),
+				},
+			},
 			// delete existing VM before creating replacement to avoid two VMs trying to use the same volume
 		}, pulumi.Provider(provider), pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))
 		if err != nil {
 			return err
 		}

Once again, run:

pulumi up

We can see a newly created network by running:

virsh net-list

which will output something similar to:

Name              State    Autostart   Persistent
----------------------------------------------------
 default           active   yes         yes
 network-171e7af   active   no          yes

To find the IP address of the VM, we can look at the leases by running:

virsh net-dhcp-leases network-171e7af

to see

 Expiry Time           MAC address         Protocol   IP address         Hostname         Client ID or DUID
----------------------------------------------------------------------------------------------------------------------------------------------------
 2022-01-11 21:40:37   52:54:00:ca:f2:e4   ipv4       192.168.10.52/24   ubuntu-af93b6f   ff:b5:5e:67:ff:00:02:00:00:ab:11:f3:04:a5:1b:1a:65:18:76

We can finally SSH by running:

ssh ubuntu@192.168.10.52

and logging in with the ubuntu password again.

Note: This network is uses NAT, so it will only be reachable from the host that libvirt is running by default.

Add Pulumi outputs

Throughout this post, we’ve had to use virsh to find the VM name and IP address, but we can actually use Pulumi Outputs. Pulumi will then automatically retrieve these values and display them after provisioning resources.

We can define an IP Address and VM Name output by modifying main.go again:

 		// create a VM that has a name starting with ubuntu
-		_, err = libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{
+		domain, err := libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{
 			Cloudinit: cloud_init.ID(),
 			Consoles: libvirt.DomainConsoleArray{
 				// enables using `virsh console ...`
 				libvirt.DomainConsoleArgs{
 					Type:       pulumi.String("pty"),
 					TargetPort: pulumi.String("0"),
 					TargetType: pulumi.String("serial"),
 				},
 			},
 			Disks: libvirt.DomainDiskArray{
 				libvirt.DomainDiskArgs{
 					VolumeId: filesystem.ID(),
 				},
 			},
 			NetworkInterfaces: libvirt.DomainNetworkInterfaceArray{
 				libvirt.DomainNetworkInterfaceArgs{
 					NetworkId:    network.ID(),
 					WaitForLease: pulumi.Bool(true),
 				},
 			},
 			// delete existing VM before creating replacement to avoid two VMs trying to use the same volume
 		}, pulumi.Provider(provider), pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))
 		if err != nil {
 			return err
 		}

+		ctx.Export("IP Address", domain.NetworkInterfaces.Index(pulumi.Int(0)).Addresses().Index(pulumi.Int(0)))
+		ctx.Export("VM name", domain.Name)

Run pulumi up again, and we’ll now see the following output:

Outputs:
  + IP Address: "192.168.10.52"
  + VM name   : "ubuntu-af93b6f"

We can also retrieve these by running pulumi stack output to list all outputs. To get a specific output, run pulumi stack output "IP Address", which can be usefull in shell scripts.

Enable autostart

If we restart the computer, libvirt will not automatically start our network and domain. Fortunately, there’s an option to handle that.

Modify main.go:

 		// create NAT network using 192.168.10/24 CIDR
 		network, err := libvirt.NewNetwork(ctx, "network", &libvirt.NetworkArgs{
 			Addresses: pulumi.StringArray{pulumi.String("192.168.10.0/24")},
+			Autostart: pulumi.Bool(true),
 			Mode:      pulumi.String("nat"),
 		}, pulumi.Provider(provider))
 		if err != nil {
 			return err
 		}

 		// create a VM that has a name starting with ubuntu
 		domain, err := libvirt.NewDomain(ctx, "ubuntu", &libvirt.DomainArgs{
+			Autostart: pulumi.Bool(true),
 			Cloudinit: cloud_init.ID(),
 			Consoles: libvirt.DomainConsoleArray{
 				// enables using `virsh console ...`
 				libvirt.DomainConsoleArgs{
 					Type:       pulumi.String("pty"),
 					TargetPort: pulumi.String("0"),
 					TargetType: pulumi.String("serial"),
 				},
 			},
 			Disks: libvirt.DomainDiskArray{
 				libvirt.DomainDiskArgs{
 					VolumeId: filesystem.ID(),
 				},
 			},
 			NetworkInterfaces: libvirt.DomainNetworkInterfaceArray{
 				libvirt.DomainNetworkInterfaceArgs{
 					NetworkId:    network.ID(),
 					WaitForLease: pulumi.Bool(true),
 				},
 			},
 			// delete existing VM before creating replacement to avoid two VMs trying to use the same volume
 		}, pulumi.Provider(provider), pulumi.ReplaceOnChanges([]string{"*"}), pulumi.DeleteBeforeReplace(true))
 		if err != nil {
 			return err
 		}

Run pulumi up again to create a new network and domain that will autostart on reboot.

Use Pulumi provider and config to support multiple stacks

So far, our main.go works great to create a VM on the same machine we’re running Pulumi. This is nice for a dev environment, but what about a staging or production environment where we want to run Pulumi against a remote libvirt instance?

By default, the libvirt provider uses a libvirt URI found in the LIBVIRT_DEFAULT_URI environment variable. If that isn’t defined, then it assumes qemu:///system for the libvirt URI.

Try it out by running virsh --connect qemu:///system list to see the same output as virsh list.

We could know to specify a LIBVIRT_DEFAULT_URI each time we run Pulumi, or we could leverage configuring a libvirt provider, so each environment will provide its own libvirt URI.

Install the Pulumi config package by running:

go get github.com/pulumi/pulumi/sdk/v3/go/pulumi/config@v3.19.0

and add as an import in main.go:

 import (
 	"os"

 	"github.com/pulumi/pulumi-libvirt/sdk/go/libvirt"
 	"github.com/pulumi/pulumi/sdk/v3/go/pulumi"
+	"github.com/pulumi/pulumi/sdk/v3/go/pulumi/config"
 )

Modify main.go to require libvirt_uri to be defined by the stack and pass the value to the provider:

+		conf := config.New(ctx, "")
+
+		// require each stack to specify a libvirt_uri
+		libvirt_uri := conf.Require("libvirt_uri")
+
 		// create a provider, this isn't required, but will make it easier to configure
 		// a libvirt_uri, which we'll discuss in a bit
-		provider, err := libvirt.NewProvider(ctx, "provider", &libvirt.ProviderArgs{})
+		provider, err := libvirt.NewProvider(ctx, "provider", &libvirt.ProviderArgs{
+			Uri: pulumi.String(libvirt_uri),
+		})
 		if err != nil {
 			return err
 		}

Now run pulumi up and we’ll see the following error message:

panic: fatal: A failure has occurred: missing required configuration variable 'pulumi-libvirt-ubuntu:libvirt_uri'; run `pulumi config` to set

To define libvirt_uri for our dev stack, run:

pulumi config set libvirt_uri qemu:///system

Note: this will also update Pulumi.dev.yaml.

and now pulumi up will run successfully.

If we wanted to create a production stack, for example, we could run the following commands:

pulumi stack init prod
pulumi config set libvirt_uri qemu://PROD_IP_ADDRESS/system
pulumi up

Note: you can switch to another stack by running pulumi stack select dev.

And then, we can use the same code for different environments!

smartctl exporter

obtain metrics from disks

install and check

export DEBIAN_FRONTEND=noninteractive && apt update && apt install smartmontools
curl -L https://github.com/prometheus-community/smartctl_exporter/releases/download/v0.14.0/smartctl_exporter-0.14.0.linux-amd64.tar.gz | tar xz -C /tmp/
timeout 5 --preserve-status -s SIGTERM /tmp/smartctl_exporter-0.14.0.linux-amd64/smartctl_exporter --web.listen-address=localhost:9633 --smartctl.rescan=60m --smartctl.interval=15s ; mv /tmp/smartctl_exporter-0.14.0.linux-amd64/smartctl_exporter /usr/local/bin/smartctl_exporter

systemd unit file

#!/bin/bash
if [ "$EUID" -ne 0 ]; then
  echo "Please run as root"
  exit
fi

tee -a /etc/systemd/system/smartctl_exporter.service <<ECHO
[Unit]
Description=Smartctl Exporter Service
After=network.target

[Service]
ExecStart=/usr/local/bin/smartctl_exporter --web.listen-address=localhost:9633 --smartctl.rescan=60m --smartctl.interval=15s
Restart=on-failure

[Install]
WantedBy=multi-user.target
ECHO
systemctl daemon-reload && systemctl enable --now smartctl_exporter.service

On reload/edit sysd-unit:

systemctl daemon-reload && systemctl restart smartctl_exporter.service

vector.dev config

vim /etc/vector/vector.yaml

sources:
  smartctl:
    type: prometheus_scrape
    scrape_interval_secs: 15
    honor_labels: true
    endpoints:
      - http://localhost:9633/metrics

prometheus metrics to check

sum by (instance) (smartctl_device_critical_warning > 0)

Answer as briefly as possible, without explanations and unnecessary text. Only the necessary code or commands.

Gathered from phi4 llm To achieve your goal of creating a Git bare repository on a Linux server with a post-receive hook that triggers a CI build, follow these steps:

Step 1: Create the Bare Repository

  1. SSH into your Linux server.

  2. Create the directory for the bare repository:

mkdir -p /path/to/repo.git
cd /path/to/repo.git
  1. Initialize the bare repository:
git init --bare

Step 2: Create the post-receive Hook

  1. Navigate to the hooks directory:
cd hooks
  1. Create a new post-receive hook script:
touch post-receive
chmod +x post-receive

  1. Edit the post-receive file with your desired script:

    Here's an example script that reads the new_ref, exports a variable for the CI archive, and executes make ci with new_ref as an argument:

#!/bin/bash
prj=basename $(pwd) | sed 's?.git??'
while read oldrev newrev refname; do
      # Extract branch name from refname (e.g., refs/heads/main -> main)
      branch=$(echo $refname | sed 's|refs/heads/||')

      # Define the filename for CI archive based on the branch and commit hash
      ci_filename="$prj-$branch-${newrev:0:7}.tar.gz"

      echo "Branch '$branch' updated to ${newrev:0:7}."

      # Export the filename variable (for use in other scripts orcommands)
      export CI_FILENAME=$ci_filename

      # Execute 'make ci' with the new ref as an argument
      echo "Running make ci for branch '$branch' at commit '${newrev:0:7}'..."
      if ! make ci REF=$refname; then
         echo "CI build failed for branch '$branch'."
         exit 1
      fi

      echo "CI build succeeded for branch '$branch'. Archive: $ci_filename"
done

Step 3: Configure Developer Repositories to Use the CI Remote

Each developer needs to add a new remote called ci pointing to your bare repository. This can be done with the following steps:

  1. On each developer's machine, navigate to their local Git repository:
cd /path/to/local/repo
  1. Add the CI remote:
git remote add ci ssh://username@server_ip:/path/to/repo.git

Replace username, server_ip, and /path/to/repo.git with your server's SSH username, IP address, and the path to the bare repository.

  1. Push changes to trigger CI: When a developer wants to push their changes and trigger the CI process, they can use:
git push ci main  # or replace 'main' with any branch name as needed

Notes

  • Ensure that make and any other dependencies required by your make ci command are installed on the server.
  • The script assumes you have a Makefile with a target named ci which accepts a REF argument. Adjust this part if necessary.
  • Consider adding error handling or logging to better track CI builds in production environments.

By following these steps, you should be able to set up your Git bare repository on the server and configure it to trigger CI builds using a post-receive hook script.

#!/bin/bash
prj=$(basename $(pwd) | sed 's?.git??')
bare_dir="$(pwd)"

function ci() {
	make ci REF=$refname CI_FILENAME=$1
}

while read oldrev newrev refname; do
	# Extract branch name from refname (e.g., refs/heads/main -> main)
	branch=$(echo $refname | sed 's|refs/heads/||')
	echo "Branch '$branch' updated to ${newrev:0:7}."

	# Define the filename for CI archive based on the branch and commit hash
	ci_filename="$prj-$branch-${newrev:0:7}.tar.gz"

	# Define directory for the worktree
	worktree_dir="$HOME/$prj-$branch-worktree"

	# Check if worktree exists
	if git --git-dir="$bare_dir" worktree list | grep -q "$worktree_dir"; then
		echo "Worktree for '$branch' already exists at $worktree_dir"
	else
		echo "Creating new worktree for '$branch'"
		git --git-dir="$bare_dir" worktree add -f "$worktree_dir" "$branch" || {
			echo "Failed to create worktree for branch '$branch'."
			exit 1
		}
	fi

	# Change to the worktree directory
	cd "$worktree_dir" || {
		echo "Failed to change to worktree directory."
		exit 1
	}

	# Update the worktree directly to the pushed commit
	echo "Updating worktree to latest commit ${newrev:0:7}"

	# Stash any local changes
	echo "Stashing local changes if any exist"
	git --git-dir="$bare_dir" --work-tree="$worktree_dir" stash push -m "Auto-stashed before updating to ${newrev:0:7}" || {
		echo "Note: No local changes to stash or stash failed."
		# Continue even if stash fails (could be no changes)
	}

	# Skip checkout and directly reset to the new commit
	# Using -f to force the update even if the branch is already checked out
	echo "Resetting to commit ${newrev:0:7}"
	git --git-dir="$bare_dir" --work-tree="$worktree_dir" reset --hard "$newrev" || {
		echo "Failed to reset to commit ${newrev:0:7}."
		exit 1
	}

	# Pop the stash if there are any stashed changes
	if git --git-dir="$bare_dir" --work-tree="$worktree_dir" stash list | grep -q "Auto-stashed before updating to ${newrev:0:7}"; then
		echo "Restoring local changes from stash"
		git --git-dir="$bare_dir" --work-tree="$worktree_dir" stash pop || {
			echo "Warning: Stash pop had conflicts. Please check the worktree."
			# Continue execution even if there are conflicts
		}
	fi

	# Export the filename variable (for use in other scripts or commands)
	export CI_FILENAME=$ci_filename

	if ! ci "$CI_FILENAME"; then
		echo "CI build failed for branch '$branch'."
		cd - >/dev/null || cd "$HOME"
		exit 1
	else
		echo "make ci was successful for branch '$branch'."
	fi

	# Return to the original directory
	cd "$bare_dir" || cd "$HOME"
done