Jonathon

Jonathon

The problem was yum cache. If I did yum update right after yum update xcp-ng-release-linstor it would still fail. To get it working right away did the following

yum update xcp-ng-release-linstor
yum clean all
yum update

Jonathon

OK I figured it out! I made an init container that gets a manually created node label for the node the pod is running on. This value is the bare metal host for that k8s node. The init contianer then takes that value and makes a script wrapper and then calls linstor-csi with the correct values. After making these changes all the linstor csi containers are running with no errors.

Current problem comes from deploying and using storage class. Started with a basic one that failed, and noticed I did not know what the correct storage_pool_name name was, so went to http://IP:3370/v1/nodes/NODE/storage-pools and http://IP:3370/v1/nodes/NODE to get information.

Still troubleshooting, but wanted to provide info.

Jonathon

@andrewperry I myself migrated our rancher management cluster from the original rke to a new rke2 cluster using this plan not too long ago, so you should not have much trouble. Feel free to ask questions

Jonathon

@nathanael-h Nice

If you have any questions let me know, I have been using this for all our on prem clusters for a while now.

Jonathon

I do not have any asks ATM, but I thought I would just share my plan that I use to create k8s clusters that we have been using for a while now.

It has grown over time and may be a bit messy, but figured better then nothing. We use this for rke2 rancher k8s clusters deployed onto out xcp-ng cluster. We use xostor for drives, and the vlan5 network is for piraeus operator to use for pv. We also use IPVS. We are using a rocky linux 9 vm template.

If these are useful to anyone and they have questions I will do my best to answer.

variable "pool" {
  default = "OVBH-PROD-XENPOOL04"
}

variable "network0" {
  default = "Native vRack"
}
variable "network1" {
  default = "VLAN80"
}
variable "network2" {
  default = "VLAN5"
}

variable "cluster_name" {
  default = "Production K8s Cluster"
}

variable "enrollment_command" {
  default = "curl -fL https://rancher.<redacted>.net/system-agent-install.sh | sudo  sh -s - --server https://rancher.<redacted>.net --label 'cattle.io/os=linux' --token <redacted>"
}


variable "node_type" {
  description = "Node type flag"
  default = {
    "1" = "--etcd --controlplane",
    "2" = "--etcd --controlplane",
    "3" = "--etcd --controlplane",
    "4" = "--worker",
    "5" = "--worker",
    "6" = "--worker",
    "7" = "--worker --taints smtp=true:NoSchedule",
    "8" = "--worker --taints smtp=true:NoSchedule",
    "9" = "--worker --taints smtp=true:NoSchedule"
  }
}
variable "node_networks" {
  description = "Node network flag"
  default = {
    "1" = "--internal-address 10.1.8.100 --address <redacted>",
    "2" = "--internal-address 10.1.8.101 --address <redacted>",
    "3" = "--internal-address 10.1.8.102 --address <redacted>",
    "4" = "--internal-address 10.1.8.103 --address <redacted>",
    "5" = "--internal-address 10.1.8.104 --address <redacted>",
    "6" = "--internal-address 10.1.8.105 --address <redacted>",
    "7" = "--internal-address 10.1.8.106 --address <redacted>",
    "8" = "--internal-address 10.1.8.107 --address <redacted>",
    "9" = "--internal-address 10.1.8.108 --address <redacted>"
  }
}


variable "vm_name" {
  description = "Node type flag"
  default = {
    "1" = "OVBH-VPROD-K8S01-MASTER01",
    "2" = "OVBH-VPROD-K8S01-MASTER02",
    "3" = "OVBH-VPROD-K8S01-MASTER03",
    "4" = "OVBH-VPROD-K8S01-WORKER01",
    "5" = "OVBH-VPROD-K8S01-WORKER02",
    "6" = "OVBH-VPROD-K8S01-WORKER03",
    "7" = "OVBH-VPROD-K8S01-WORKER04",
    "8" = "OVBH-VPROD-K8S01-WORKER05",
    "9" = "OVBH-VPROD-K8S01-WORKER06"
  }
}

variable "preferred_host" {
  default = {
    "1" = "85838113-e4b8-4520-9f6d-8f3cf554c8f1",
    "2" = "783c27ac-2dcb-4798-9ca8-27f5f30791f6",
    "3" = "c03e1a45-4c4c-46f5-a2a1-d8de2e22a866",
    "4" = "85838113-e4b8-4520-9f6d-8f3cf554c8f1",
    "5" = "783c27ac-2dcb-4798-9ca8-27f5f30791f6",
    "6" = "c03e1a45-4c4c-46f5-a2a1-d8de2e22a866",
    "7" = "85838113-e4b8-4520-9f6d-8f3cf554c8f1",
    "8" = "783c27ac-2dcb-4798-9ca8-27f5f30791f6",
    "9" = "c03e1a45-4c4c-46f5-a2a1-d8de2e22a866"
  }
}

variable "xoa_admin_password" {
}

variable "host_count" {
  description = "All drives go to xostor"
  default = {
    "1" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "2" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "3" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "4" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "5" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "6" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "7" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "8" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "9" = "479ca676-20a1-4051-7189-a4a9ca47e00d"
  }
}

variable "network1_ip_mapping" {
  description = "Mapping for network1 ips, vlan80"
  default = {
    "1" = "10.1.8.100",
    "2" = "10.1.8.101",
    "3" = "10.1.8.102",
    "4" = "10.1.8.103",
    "5" = "10.1.8.104",
    "6" = "10.1.8.105",
    "7" = "10.1.8.106",
    "8" = "10.1.8.107",
    "9" = "10.1.8.108"
  }
}

variable "network1_gateway" {
  description = "Mapping for public ip gateways, from hosts"
  default     = "10.1.8.1"
}

variable "network1_prefix" {
  description = "Prefix for the network used"
  default     = "22"
}

variable "network2_ip_mapping" {
  description = "Mapping for network2 ips, VLAN5"
  default = {
    "1" = "10.2.5.30",
    "2" = "10.2.5.31",
    "3" = "10.2.5.32",
    "4" = "10.2.5.33",
    "5" = "10.2.5.34",
    "6" = "10.2.5.35",
    "7" = "10.2.5.36",
    "8" = "10.2.5.37",
    "9" = "10.2.5.38"
  }
}


variable "network2_prefix" {
  description = "Prefix for the network used"
  default     = "22"
}

variable "network0_ip_mapping" {
  description = "Mapping for network0 ips, public"
  default = {
<redacted>
  }
}

variable "network0_gateway" {
  description = "Mapping for public ip gateways, from hosts"
  default = {
<redacted>
  }
}

variable "network0_prefix" {
  description = "Prefix for the network used"
  default = {
<redacted>
  }
}

# Instruct terraform to download the provider on `terraform init`
terraform {
  required_providers {
    xenorchestra = {
      source  = "vatesfr/xenorchestra"
      version = "~> 0.29.0"
    }
  }
}

# Configure the XenServer Provider
provider "xenorchestra" {
  # Must be ws or wss
  url      = "ws://10.2.0.5"        # Or set XOA_URL environment variable
  username = "admin@admin.net"      # Or set XOA_USER environment variable
  password = var.xoa_admin_password # Or set XOA_PASSWORD environment variable
}

data "xenorchestra_pool" "pool" {
  name_label = var.pool
}

data "xenorchestra_template" "template" {
  name_label = "Rocky Linux 9 Template"
  pool_id    = data.xenorchestra_pool.pool.id
}

data "xenorchestra_network" "net1" {
  name_label = var.network1
  pool_id    = data.xenorchestra_pool.pool.id
}
data "xenorchestra_network" "net2" {
  name_label = var.network2
  pool_id    = data.xenorchestra_pool.pool.id
}
data "xenorchestra_network" "net0" {
  name_label = var.network0
  pool_id    = data.xenorchestra_pool.pool.id
}

resource "xenorchestra_cloud_config" "node" {
  count    = 9
  name     = "${lower(lookup(var.vm_name, count.index + 1))}_cloud_config"
  template = <<EOF

#cloud-config
ssh_authorized_keys:
  - ssh-rsa <redacted>

write_files:
  - path: /etc/NetworkManager/conf.d/rke2-canal.conf
    permissions: '0755'
    owner: root
    content: |
      [keyfile]
      unmanaged-devices=interface-name:cali*;interface-name:flannel*
  - path: /tmp/selinux_kmod_drbd.log
    permissions: '0640'
    owner: root
    content: |
      type=AVC msg=audit(1661803314.183:778): avc:  denied  { module_load } for  pid=148256 comm="insmod" path="/tmp/ko/drbd.ko" dev="overlay" ino=101839829 scontext=system_u:system_r:unconfined_service_t:s0 tcontext=system_u:object_r:var_lib_t:s0 tclass=system permissive=0
      type=AVC msg=audit(1661803314.185:779): avc:  denied  { module_load } for  pid=148257 comm="insmod" path="/tmp/ko/drbd_transport_tcp.ko" dev="overlay" ino=101839831 scontext=system_u:system_r:unconfined_service_t:s0 tcontext=system_u:object_r:var_lib_t:s0 tclass=system permissive=0
  - path: /etc/sysconfig/modules/ipvs.modules
    permissions: 0755
    owner: root
    content: |
      #!/bin/bash
      modprobe -- ip_vs
      modprobe -- ip_vs_rr
      modprobe -- ip_vs_wrr
      modprobe -- ip_vs_sh
      modprobe -- nf_conntrack
  - path: /etc/modules-load.d/ipvs.conf
    permissions: 0755
    owner: root
    content: |
      ip_vs
      ip_vs_rr
      ip_vs_wrr
      ip_vs_sh
      nf_conntrack

#cloud-init
runcmd:
  - sudo hostnamectl set-hostname --static ${lower(lookup(var.vm_name, count.index + 1))}.<redacted>.com
  - sudo hostnamectl set-hostname ${lower(lookup(var.vm_name, count.index + 1))}.<redacted>.com
  - nmcli -t -f NAME con show | xargs -d '\n' -I {} nmcli con delete "{}"
  - nmcli con add type ethernet con-name public ifname enX0
  - nmcli con mod public ipv4.address '${lookup(var.network0_ip_mapping, count.index + 1)}/${lookup(var.network0_prefix, count.index + 1)}'
  - nmcli con mod public ipv4.method manual
  - nmcli con mod public ipv4.ignore-auto-dns yes
  - nmcli con mod public ipv4.gateway '${lookup(var.network0_gateway, count.index + 1)}'
  - nmcli con mod public ipv4.dns "8.8.8.8 8.8.4.4"
  - nmcli con mod public connection.autoconnect true
  - nmcli con up public
  - nmcli con add type ethernet con-name vlan80 ifname enX1
  - nmcli con mod vlan80 ipv4.address '${lookup(var.network1_ip_mapping, count.index + 1)}/${var.network1_prefix}'
  - nmcli con mod vlan80 ipv4.method manual
  - nmcli con mod vlan80 ipv4.ignore-auto-dns yes
  - nmcli con mod vlan80 ipv4.ignore-auto-routes yes
  - nmcli con mod vlan80 ipv4.gateway '${var.network1_gateway}'
  - nmcli con mod vlan80 ipv4.dns "${var.network1_gateway}"
  - nmcli con mod vlan80 connection.autoconnect true
  - nmcli con mod vlan80 ipv4.never-default true
  - nmcli con mod vlan80 ipv6.never-default true
  - nmcli con mod vlan80 ipv4.routes "10.0.0.0/8 ${var.network1_gateway}"
  - nmcli con up vlan80
  - nmcli con add type ethernet con-name vlan5 ifname enX2
  - nmcli con mod vlan5 ipv4.address '${lookup(var.network2_ip_mapping, count.index + 1)}/${var.network2_prefix}'
  - nmcli con mod vlan5 ipv4.method manual
  - nmcli con mod vlan5 ipv4.ignore-auto-dns yes
  - nmcli con mod vlan5 ipv4.ignore-auto-routes yes
  - nmcli con mod vlan5 connection.autoconnect true
  - nmcli con mod vlan5 ipv4.never-default true
  - nmcli con mod vlan5 ipv6.never-default true
  - nmcli con up vlan5
  - systemctl restart NetworkManager
  - dnf upgrade -y
  - dnf install ipset ipvsadm -y
  - bash /etc/sysconfig/modules/ipvs.modules
  - dnf install chrony -y
  - sudo systemctl enable --now chronyd
  - yum install kernel-devel kernel-headers -y
  - yum install elfutils-libelf-devel -y
  - swapoff -a
  - modprobe -- ip_tables
  - systemctl disable --now firewalld.service
  - systemctl disable --now rngd
  - dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
  - dnf install containerd.io tar -y
  - dnf install policycoreutils-python-utils -y
  - cat /tmp/selinux_kmod_drbd.log | sudo audit2allow -M insmoddrbd
  - sudo semodule -i insmoddrbd.pp
  - ${var.enrollment_command} ${lookup(var.node_type, count.index + 1)} ${lookup(var.node_networks, count.index + 1)}

bootcmd:
  - swapoff -a
  - modprobe -- ip_tables
EOF
}

resource "xenorchestra_vm" "master" {
  count            = 3
  cpus             = 4
  memory_max       = 8589934592
  cloud_config     = xenorchestra_cloud_config.node[count.index].template
  name_label       = lookup(var.vm_name, count.index + 1)
  name_description = "${var.cluster_name} master"
  template         = data.xenorchestra_template.template.id
  auto_poweron     = true
  affinity_host    = lookup(var.preferred_host, count.index + 1)

  network {
    network_id = data.xenorchestra_network.net0.id
  }
  network {
    network_id = data.xenorchestra_network.net1.id
  }
  network {
    network_id = data.xenorchestra_network.net2.id
  }
  disk {
    sr_id      = lookup(var.host_count, count.index + 1)
    name_label = "Terraform_disk_imavo"
    size       = 107374182400
  }
}


resource "xenorchestra_vm" "worker" {
  count            = 3
  cpus             = 32
  memory_max       = 68719476736
  cloud_config     = xenorchestra_cloud_config.node[count.index + 3].template
  name_label       = lookup(var.vm_name, count.index + 3 + 1)
  name_description = "${var.cluster_name} worker"
  template         = data.xenorchestra_template.template.id
  auto_poweron     = true
  affinity_host    = lookup(var.preferred_host, count.index + 3 + 1)
  
  network {
    network_id = data.xenorchestra_network.net0.id
  }
  network {
    network_id = data.xenorchestra_network.net1.id
  }
  network {
    network_id = data.xenorchestra_network.net2.id
  }
  disk {
    sr_id      = lookup(var.host_count, count.index + 3 + 1)
    name_label = "Terraform_disk_imavo"
    size       = 322122547200
  }
}

resource "xenorchestra_vm" "smtp" {
  count            = 3
  cpus             = 4
  memory_max       = 8589934592
  cloud_config     = xenorchestra_cloud_config.node[count.index + 6].template
  name_label       = lookup(var.vm_name, count.index + 6 + 1)
  name_description = "${var.cluster_name} smtp worker"
  template         = data.xenorchestra_template.template.id
  auto_poweron     = true
  affinity_host    = lookup(var.preferred_host, count.index + 6 + 1)
  
  network {
    network_id = data.xenorchestra_network.net0.id
  }
  network {
    network_id = data.xenorchestra_network.net1.id
  }
  network {
    network_id = data.xenorchestra_network.net2.id
  }
  disk {
    sr_id      = lookup(var.host_count, count.index + 6 + 1)
    name_label = "Terraform_disk_imavo"
    size       = 53687091200
  }
}

Jonathon

@Danp Sorry, due to the host being added and me continuing on with my other tasks, it is not. I could have made that more clear.

Jonathon

It does not matter now, the host has been added. I had just found the error to be interesting that thought someone would want to know about it.

Jonathon

I ran the upgrade yesterday. And this is the state that it gave me

  installing : node-v22.15.1
       mkdir : /usr/local/n/versions/node/22.15.1
       fetch : https://nodejs.org/dist/v22.15.1/node-v22.15.1-linux-x64.tar.xz
   installed : v22.15.1 (with npm 10.9.2)

Stopping xo-server...
Checking for Yarn package...
Checking for Yarn update...
Reading package lists...
...
yarn install v1.22.22
[1/5] Validating package.json...
[2/5] Resolving packages...
[3/5] Fetching packages...
[4/5] Linking dependencies...
[5/5] Building fresh packages...
$ husky install
husky - Git hooks installed
Done in 46.08s.
yarn run v1.22.22
$ TURBO_TELEMETRY_DISABLED=1 turbo run build --filter xo-server --filter xo-server-'*' --filter xo-web
• Packages in scope: xo-server, xo-server-audit, xo-server-auth-github, xo-server-auth-google, xo-server-auth-ldap, xo-server-auth-oidc, xo-server-auth-saml, xo-server-backup-reports, xo-server-load-balancer, xo-server-netbox, xo-server-perf-alert, xo-server-sdn-controller, xo-server-test, xo-server-test-plugin, xo-server-transport-email, xo-server-transport-icinga2, xo-server-transport-nagios, xo-server-transport-slack, xo-server-transport-xmpp, xo-server-usage-report, xo-server-web-hooks, xo-web
• Running build in 22 packages
• Remote caching disabled

 Tasks:    29 successful, 29 total
Cached:    0 cached, 29 total
  Time:    1m18.85s 

Done in 79.13s.
Updated version 5.177.0 / 5.173.2
Updated commit a49b27bff7d325f704957b8aac3055ad0407bd40 2025-05-20 16:42:02 +0200
Checking plugins...
Ignoring xo-server-test plugin
Cleanup plugins...
Restarting xo-server...

Jonathon

I got the following when trying to add a new host to an existing pool

pool.mergeInto
{
  "sources": [
    "0af9d764-44e6-1393-4eea-6cb59b934f2a"
  ],
  "target": "38aea760-cf23-927c-ccf5-90969681e04b",
  "force": true
}
{
  "message": "app.getLicenses is not a function",
  "name": "TypeError",
  "stack": "TypeError: app.getLicenses is not a function
    at enforceHostsHaveLicense (file:///opt/xen-orchestra/packages/xo-server/src/xo-mixins/pool.mjs:15:30)
    at Pools.apply (file:///opt/xen-orchestra/packages/xo-server/src/xo-mixins/pool.mjs:80:13)
    at Pools.mergeInto (/opt/xen-orchestra/node_modules/golike-defer/src/index.js:85:19)
    at Xo.mergeInto (file:///opt/xen-orchestra/packages/xo-server/src/api/pool.mjs:311:15)
    at Task.runInside (/opt/xen-orchestra/@vates/task/index.js:175:22)
    at Task.run (/opt/xen-orchestra/@vates/task/index.js:159:20)
    at Api.#callApiMethod (file:///opt/xen-orchestra/packages/xo-server/src/xo-mixins/api.mjs:469:18)"
}

Had no idea what was up with this error, so updated the xoa vm and installed updates. But I still got the same error

I then tried adding it from the server itself, that also failed
f411a9a5-ae28-4d43-8134-c730266669b5-Screenshot from 2025-05-20 15-26-59.png /home/jonathon/Pictures/Screenshots/Screenshot from 2025-05-20 15-26-59.png

Just to see if it would work, I then tried with the latest xcp-ng center, and that worked.

So I do not need anything now, but thought it was interesting and worth posting.

Jonathon

@andrewperry I myself migrated our rancher management cluster from the original rke to a new rke2 cluster using this plan not too long ago, so you should not have much trouble. Feel free to ask questions

Jonathon

@nathanael-h Nice

If you have any questions let me know, I have been using this for all our on prem clusters for a while now.

Jonathon

I do not have any asks ATM, but I thought I would just share my plan that I use to create k8s clusters that we have been using for a while now.

It has grown over time and may be a bit messy, but figured better then nothing. We use this for rke2 rancher k8s clusters deployed onto out xcp-ng cluster. We use xostor for drives, and the vlan5 network is for piraeus operator to use for pv. We also use IPVS. We are using a rocky linux 9 vm template.

If these are useful to anyone and they have questions I will do my best to answer.

variable "pool" {
  default = "OVBH-PROD-XENPOOL04"
}

variable "network0" {
  default = "Native vRack"
}
variable "network1" {
  default = "VLAN80"
}
variable "network2" {
  default = "VLAN5"
}

variable "cluster_name" {
  default = "Production K8s Cluster"
}

variable "enrollment_command" {
  default = "curl -fL https://rancher.<redacted>.net/system-agent-install.sh | sudo  sh -s - --server https://rancher.<redacted>.net --label 'cattle.io/os=linux' --token <redacted>"
}


variable "node_type" {
  description = "Node type flag"
  default = {
    "1" = "--etcd --controlplane",
    "2" = "--etcd --controlplane",
    "3" = "--etcd --controlplane",
    "4" = "--worker",
    "5" = "--worker",
    "6" = "--worker",
    "7" = "--worker --taints smtp=true:NoSchedule",
    "8" = "--worker --taints smtp=true:NoSchedule",
    "9" = "--worker --taints smtp=true:NoSchedule"
  }
}
variable "node_networks" {
  description = "Node network flag"
  default = {
    "1" = "--internal-address 10.1.8.100 --address <redacted>",
    "2" = "--internal-address 10.1.8.101 --address <redacted>",
    "3" = "--internal-address 10.1.8.102 --address <redacted>",
    "4" = "--internal-address 10.1.8.103 --address <redacted>",
    "5" = "--internal-address 10.1.8.104 --address <redacted>",
    "6" = "--internal-address 10.1.8.105 --address <redacted>",
    "7" = "--internal-address 10.1.8.106 --address <redacted>",
    "8" = "--internal-address 10.1.8.107 --address <redacted>",
    "9" = "--internal-address 10.1.8.108 --address <redacted>"
  }
}


variable "vm_name" {
  description = "Node type flag"
  default = {
    "1" = "OVBH-VPROD-K8S01-MASTER01",
    "2" = "OVBH-VPROD-K8S01-MASTER02",
    "3" = "OVBH-VPROD-K8S01-MASTER03",
    "4" = "OVBH-VPROD-K8S01-WORKER01",
    "5" = "OVBH-VPROD-K8S01-WORKER02",
    "6" = "OVBH-VPROD-K8S01-WORKER03",
    "7" = "OVBH-VPROD-K8S01-WORKER04",
    "8" = "OVBH-VPROD-K8S01-WORKER05",
    "9" = "OVBH-VPROD-K8S01-WORKER06"
  }
}

variable "preferred_host" {
  default = {
    "1" = "85838113-e4b8-4520-9f6d-8f3cf554c8f1",
    "2" = "783c27ac-2dcb-4798-9ca8-27f5f30791f6",
    "3" = "c03e1a45-4c4c-46f5-a2a1-d8de2e22a866",
    "4" = "85838113-e4b8-4520-9f6d-8f3cf554c8f1",
    "5" = "783c27ac-2dcb-4798-9ca8-27f5f30791f6",
    "6" = "c03e1a45-4c4c-46f5-a2a1-d8de2e22a866",
    "7" = "85838113-e4b8-4520-9f6d-8f3cf554c8f1",
    "8" = "783c27ac-2dcb-4798-9ca8-27f5f30791f6",
    "9" = "c03e1a45-4c4c-46f5-a2a1-d8de2e22a866"
  }
}

variable "xoa_admin_password" {
}

variable "host_count" {
  description = "All drives go to xostor"
  default = {
    "1" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "2" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "3" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "4" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "5" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "6" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "7" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "8" = "479ca676-20a1-4051-7189-a4a9ca47e00d",
    "9" = "479ca676-20a1-4051-7189-a4a9ca47e00d"
  }
}

variable "network1_ip_mapping" {
  description = "Mapping for network1 ips, vlan80"
  default = {
    "1" = "10.1.8.100",
    "2" = "10.1.8.101",
    "3" = "10.1.8.102",
    "4" = "10.1.8.103",
    "5" = "10.1.8.104",
    "6" = "10.1.8.105",
    "7" = "10.1.8.106",
    "8" = "10.1.8.107",
    "9" = "10.1.8.108"
  }
}

variable "network1_gateway" {
  description = "Mapping for public ip gateways, from hosts"
  default     = "10.1.8.1"
}

variable "network1_prefix" {
  description = "Prefix for the network used"
  default     = "22"
}

variable "network2_ip_mapping" {
  description = "Mapping for network2 ips, VLAN5"
  default = {
    "1" = "10.2.5.30",
    "2" = "10.2.5.31",
    "3" = "10.2.5.32",
    "4" = "10.2.5.33",
    "5" = "10.2.5.34",
    "6" = "10.2.5.35",
    "7" = "10.2.5.36",
    "8" = "10.2.5.37",
    "9" = "10.2.5.38"
  }
}


variable "network2_prefix" {
  description = "Prefix for the network used"
  default     = "22"
}

variable "network0_ip_mapping" {
  description = "Mapping for network0 ips, public"
  default = {
<redacted>
  }
}

variable "network0_gateway" {
  description = "Mapping for public ip gateways, from hosts"
  default = {
<redacted>
  }
}

variable "network0_prefix" {
  description = "Prefix for the network used"
  default = {
<redacted>
  }
}

# Instruct terraform to download the provider on `terraform init`
terraform {
  required_providers {
    xenorchestra = {
      source  = "vatesfr/xenorchestra"
      version = "~> 0.29.0"
    }
  }
}

# Configure the XenServer Provider
provider "xenorchestra" {
  # Must be ws or wss
  url      = "ws://10.2.0.5"        # Or set XOA_URL environment variable
  username = "admin@admin.net"      # Or set XOA_USER environment variable
  password = var.xoa_admin_password # Or set XOA_PASSWORD environment variable
}

data "xenorchestra_pool" "pool" {
  name_label = var.pool
}

data "xenorchestra_template" "template" {
  name_label = "Rocky Linux 9 Template"
  pool_id    = data.xenorchestra_pool.pool.id
}

data "xenorchestra_network" "net1" {
  name_label = var.network1
  pool_id    = data.xenorchestra_pool.pool.id
}
data "xenorchestra_network" "net2" {
  name_label = var.network2
  pool_id    = data.xenorchestra_pool.pool.id
}
data "xenorchestra_network" "net0" {
  name_label = var.network0
  pool_id    = data.xenorchestra_pool.pool.id
}

resource "xenorchestra_cloud_config" "node" {
  count    = 9
  name     = "${lower(lookup(var.vm_name, count.index + 1))}_cloud_config"
  template = <<EOF

#cloud-config
ssh_authorized_keys:
  - ssh-rsa <redacted>

write_files:
  - path: /etc/NetworkManager/conf.d/rke2-canal.conf
    permissions: '0755'
    owner: root
    content: |
      [keyfile]
      unmanaged-devices=interface-name:cali*;interface-name:flannel*
  - path: /tmp/selinux_kmod_drbd.log
    permissions: '0640'
    owner: root
    content: |
      type=AVC msg=audit(1661803314.183:778): avc:  denied  { module_load } for  pid=148256 comm="insmod" path="/tmp/ko/drbd.ko" dev="overlay" ino=101839829 scontext=system_u:system_r:unconfined_service_t:s0 tcontext=system_u:object_r:var_lib_t:s0 tclass=system permissive=0
      type=AVC msg=audit(1661803314.185:779): avc:  denied  { module_load } for  pid=148257 comm="insmod" path="/tmp/ko/drbd_transport_tcp.ko" dev="overlay" ino=101839831 scontext=system_u:system_r:unconfined_service_t:s0 tcontext=system_u:object_r:var_lib_t:s0 tclass=system permissive=0
  - path: /etc/sysconfig/modules/ipvs.modules
    permissions: 0755
    owner: root
    content: |
      #!/bin/bash
      modprobe -- ip_vs
      modprobe -- ip_vs_rr
      modprobe -- ip_vs_wrr
      modprobe -- ip_vs_sh
      modprobe -- nf_conntrack
  - path: /etc/modules-load.d/ipvs.conf
    permissions: 0755
    owner: root
    content: |
      ip_vs
      ip_vs_rr
      ip_vs_wrr
      ip_vs_sh
      nf_conntrack

#cloud-init
runcmd:
  - sudo hostnamectl set-hostname --static ${lower(lookup(var.vm_name, count.index + 1))}.<redacted>.com
  - sudo hostnamectl set-hostname ${lower(lookup(var.vm_name, count.index + 1))}.<redacted>.com
  - nmcli -t -f NAME con show | xargs -d '\n' -I {} nmcli con delete "{}"
  - nmcli con add type ethernet con-name public ifname enX0
  - nmcli con mod public ipv4.address '${lookup(var.network0_ip_mapping, count.index + 1)}/${lookup(var.network0_prefix, count.index + 1)}'
  - nmcli con mod public ipv4.method manual
  - nmcli con mod public ipv4.ignore-auto-dns yes
  - nmcli con mod public ipv4.gateway '${lookup(var.network0_gateway, count.index + 1)}'
  - nmcli con mod public ipv4.dns "8.8.8.8 8.8.4.4"
  - nmcli con mod public connection.autoconnect true
  - nmcli con up public
  - nmcli con add type ethernet con-name vlan80 ifname enX1
  - nmcli con mod vlan80 ipv4.address '${lookup(var.network1_ip_mapping, count.index + 1)}/${var.network1_prefix}'
  - nmcli con mod vlan80 ipv4.method manual
  - nmcli con mod vlan80 ipv4.ignore-auto-dns yes
  - nmcli con mod vlan80 ipv4.ignore-auto-routes yes
  - nmcli con mod vlan80 ipv4.gateway '${var.network1_gateway}'
  - nmcli con mod vlan80 ipv4.dns "${var.network1_gateway}"
  - nmcli con mod vlan80 connection.autoconnect true
  - nmcli con mod vlan80 ipv4.never-default true
  - nmcli con mod vlan80 ipv6.never-default true
  - nmcli con mod vlan80 ipv4.routes "10.0.0.0/8 ${var.network1_gateway}"
  - nmcli con up vlan80
  - nmcli con add type ethernet con-name vlan5 ifname enX2
  - nmcli con mod vlan5 ipv4.address '${lookup(var.network2_ip_mapping, count.index + 1)}/${var.network2_prefix}'
  - nmcli con mod vlan5 ipv4.method manual
  - nmcli con mod vlan5 ipv4.ignore-auto-dns yes
  - nmcli con mod vlan5 ipv4.ignore-auto-routes yes
  - nmcli con mod vlan5 connection.autoconnect true
  - nmcli con mod vlan5 ipv4.never-default true
  - nmcli con mod vlan5 ipv6.never-default true
  - nmcli con up vlan5
  - systemctl restart NetworkManager
  - dnf upgrade -y
  - dnf install ipset ipvsadm -y
  - bash /etc/sysconfig/modules/ipvs.modules
  - dnf install chrony -y
  - sudo systemctl enable --now chronyd
  - yum install kernel-devel kernel-headers -y
  - yum install elfutils-libelf-devel -y
  - swapoff -a
  - modprobe -- ip_tables
  - systemctl disable --now firewalld.service
  - systemctl disable --now rngd
  - dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
  - dnf install containerd.io tar -y
  - dnf install policycoreutils-python-utils -y
  - cat /tmp/selinux_kmod_drbd.log | sudo audit2allow -M insmoddrbd
  - sudo semodule -i insmoddrbd.pp
  - ${var.enrollment_command} ${lookup(var.node_type, count.index + 1)} ${lookup(var.node_networks, count.index + 1)}

bootcmd:
  - swapoff -a
  - modprobe -- ip_tables
EOF
}

resource "xenorchestra_vm" "master" {
  count            = 3
  cpus             = 4
  memory_max       = 8589934592
  cloud_config     = xenorchestra_cloud_config.node[count.index].template
  name_label       = lookup(var.vm_name, count.index + 1)
  name_description = "${var.cluster_name} master"
  template         = data.xenorchestra_template.template.id
  auto_poweron     = true
  affinity_host    = lookup(var.preferred_host, count.index + 1)

  network {
    network_id = data.xenorchestra_network.net0.id
  }
  network {
    network_id = data.xenorchestra_network.net1.id
  }
  network {
    network_id = data.xenorchestra_network.net2.id
  }
  disk {
    sr_id      = lookup(var.host_count, count.index + 1)
    name_label = "Terraform_disk_imavo"
    size       = 107374182400
  }
}


resource "xenorchestra_vm" "worker" {
  count            = 3
  cpus             = 32
  memory_max       = 68719476736
  cloud_config     = xenorchestra_cloud_config.node[count.index + 3].template
  name_label       = lookup(var.vm_name, count.index + 3 + 1)
  name_description = "${var.cluster_name} worker"
  template         = data.xenorchestra_template.template.id
  auto_poweron     = true
  affinity_host    = lookup(var.preferred_host, count.index + 3 + 1)
  
  network {
    network_id = data.xenorchestra_network.net0.id
  }
  network {
    network_id = data.xenorchestra_network.net1.id
  }
  network {
    network_id = data.xenorchestra_network.net2.id
  }
  disk {
    sr_id      = lookup(var.host_count, count.index + 3 + 1)
    name_label = "Terraform_disk_imavo"
    size       = 322122547200
  }
}

resource "xenorchestra_vm" "smtp" {
  count            = 3
  cpus             = 4
  memory_max       = 8589934592
  cloud_config     = xenorchestra_cloud_config.node[count.index + 6].template
  name_label       = lookup(var.vm_name, count.index + 6 + 1)
  name_description = "${var.cluster_name} smtp worker"
  template         = data.xenorchestra_template.template.id
  auto_poweron     = true
  affinity_host    = lookup(var.preferred_host, count.index + 6 + 1)
  
  network {
    network_id = data.xenorchestra_network.net0.id
  }
  network {
    network_id = data.xenorchestra_network.net1.id
  }
  network {
    network_id = data.xenorchestra_network.net2.id
  }
  disk {
    sr_id      = lookup(var.host_count, count.index + 6 + 1)
    name_label = "Terraform_disk_imavo"
    size       = 53687091200
  }
}

Jonathon

OK we have debugged and improved this process, so including it here if it helps anyone else.

How to migrate resources between XOSTOR (linstor) clusters. This also works with piraeus-operator, which we use for k8s.

Manually moving listor resource with thin_send_recv

Migration of data

Commands

# PV: pvc-6408a214-6def-44c4-8d9a-bebb67be5510
# S: pgdata-snapshot
# s: 10741612544B

#get size
lvs --noheadings --units B -o lv_size linstor_group/pvc-6408a214-6def-44c4-8d9a-bebb67be5510_00000

#prep
lvcreate -V 10741612544B --thinpool linstor_group/thin_device -n pvc-6408a214-6def-44c4-8d9a-bebb67be5510_00000 linstor_group

#create snapshot
linstor --controller original-xostor-server s create pvc-6408a214-6def-44c4-8d9a-bebb67be5510  pgdata-snapshot

#send
thin_send linstor_group/pvc-6408a214-6def-44c4-8d9a-bebb67be5510_00000_pgdata-snapshot 2>/dev/null | ssh root@new-xostor-server-01 thin_recv linstor_group/pvc-6408a214-6def-44c4-8d9a-bebb67be5510_00000 2>/dev/null

Walk-through

Prep migration

[13:29 original-xostor-server ~]# lvs --noheadings --units B -o lv_size linstor_group/pvc-12aca72c-d94a-4c09-8102-0a6646906f8d_00000
  26851934208B


[13:53 new-xostor-server-01 ~]# lvcreate -V 26851934208B --thinpool linstor_group/thin_device -n pvc-12aca72c-d94a-4c09-8102-0a6646906f8d_00000 linstor_group
  Logical volume "pvc-12aca72c-d94a-4c09-8102-0a6646906f8d_00000" created.

Create snapshot

15:35:03] jonathon@jonathon-framework:~$ linstor --controller original-xostor-server s create pvc-12aca72c-d94a-4c09-8102-0a6646906f8d s_test
SUCCESS:
Description:
    New snapshot 's_test' of resource 'pvc-12aca72c-d94a-4c09-8102-0a6646906f8d' registered.
Details:
    Snapshot 's_test' of resource 'pvc-12aca72c-d94a-4c09-8102-0a6646906f8d' UUID is: 3a07d2fd-6dc3-4994-b13f-8c3a2bb206b8
SUCCESS:
    Suspended IO of '[pvc-12aca72c-d94a-4c09-8102-0a6646906f8d]' on 'ovbh-vprod-k8s04-worker02' for snapshot
SUCCESS:
    Suspended IO of '[pvc-12aca72c-d94a-4c09-8102-0a6646906f8d]' on 'original-xostor-server' for snapshot
SUCCESS:
    Took snapshot of '[pvc-12aca72c-d94a-4c09-8102-0a6646906f8d]' on 'ovbh-vprod-k8s04-worker02'
SUCCESS:
    Took snapshot of '[pvc-12aca72c-d94a-4c09-8102-0a6646906f8d]' on 'original-xostor-server'
SUCCESS:
    Resumed IO of '[pvc-12aca72c-d94a-4c09-8102-0a6646906f8d]' on 'ovbh-vprod-k8s04-worker02' after snapshot
SUCCESS:
    Resumed IO of '[pvc-12aca72c-d94a-4c09-8102-0a6646906f8d]' on 'original-xostor-server' after snapshot

Migration

[13:53 original-xostor-server ~]# thin_send /dev/linstor_group/pvc-12aca72c-d94a-4c09-8102-0a6646906f8d_00000_s_test 2>/dev/null | ssh root@new-xostor-server-01 thin_recv linstor_group/pvc-12aca72c-d94a-4c09-8102-0a6646906f8d_00000 2>/dev/null

Need to yeet errors on both ends of command or it will fail.

This is the same setup process for replica-1 or replica-3. For replica-3 can target new-xostor-server-01 each time, for replica-1 be sure to spread them out right.

Replica-3 Setup

Explanation

thin_send to new-xostor-server-01, will need to run commands to force sync of data to replicas.

Commands

# PV: pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
# snapshot: snipeit-snapshot
# size: 21483225088B

#get size
lvs --noheadings --units B -o lv_size linstor_group/pvc-96cbebbe-f827-4a47-ae95-38b078e0d584_00000

#prep
lvcreate -V 21483225088B --thinpool linstor_group/thin_device -n pvc-96cbebbe-f827-4a47-ae95-38b078e0d584_00000 linstor_group

#create snapshot
linstor --controller original-xostor-server s create pvc-96cbebbe-f827-4a47-ae95-38b078e0d584 snipeit-snapshot
linstor --controller original-xostor-server s l | grep -e 'snipeit-snapshot'

#send
thin_send linstor_group/pvc-96cbebbe-f827-4a47-ae95-38b078e0d584_00000_snipeit-snapshot 2>/dev/null | ssh root@new-xostor-server-01 thin_recv linstor_group/pvc-96cbebbe-f827-4a47-ae95-38b078e0d584_00000 2>/dev/null

#linstor setup
linstor --controller new-xostor-server-01 resource-definition create pvc-96cbebbe-f827-4a47-ae95-38b078e0d584 --resource-group sc-74e1434b-b435-587e-9dea-fa067deec898
linstor --controller new-xostor-server-01 volume-definition create pvc-96cbebbe-f827-4a47-ae95-38b078e0d584 21483225088B --storage-pool xcp-sr-linstor_group_thin_device

linstor --controller new-xostor-server-01 resource create --storage-pool xcp-sr-linstor_group_thin_device --providers LVM_THIN new-xostor-server-01 pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
linstor --controller new-xostor-server-01 resource create --auto-place +1 pvc-96cbebbe-f827-4a47-ae95-38b078e0d584

#Run the following on the node with the data. This is the prefered command
drbdadm invalidate-remote pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
#Run the following on the node without the data. This is just for reference
drbdadm invalidate pvc-96cbebbe-f827-4a47-ae95-38b078e0d584

linstor --controller new-xostor-server-01 r l | grep -e 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584'

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
  annotations:
    pv.kubernetes.io/provisioned-by: linstor.csi.linbit.com
  finalizers:
    - external-provisioner.volume.kubernetes.io/finalizer
    - kubernetes.io/pv-protection
    - external-attacher/linstor-csi-linbit-com
spec:
  accessModes:
    - ReadWriteOnce
  capacity:
    storage: 20Gi # Ensure this matches the actual size of the LINSTOR volume
  persistentVolumeReclaimPolicy: Retain
  storageClassName: linstor-replica-three # Adjust to the storage class you want to use
  volumeMode: Filesystem
  csi:
    driver: linstor.csi.linbit.com
    fsType: ext4
    volumeHandle: pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
    volumeAttributes:
      linstor.csi.linbit.com/mount-options: ''
      linstor.csi.linbit.com/post-mount-xfs-opts: ''
      linstor.csi.linbit.com/uses-volume-context: 'true'
      linstor.csi.linbit.com/remote-access-policy: 'true'
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    pv.kubernetes.io/bind-completed: 'yes'
    pv.kubernetes.io/bound-by-controller: 'yes'
    volume.beta.kubernetes.io/storage-provisioner: linstor.csi.linbit.com
    volume.kubernetes.io/storage-provisioner: linstor.csi.linbit.com
  finalizers:
    - kubernetes.io/pvc-protection
  name: pp-snipeit-pvc
  namespace: snipe-it
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 20Gi
  storageClassName: linstor-replica-three
  volumeMode: Filesystem
  volumeName: pvc-96cbebbe-f827-4a47-ae95-38b078e0d584

Walk-through

jonathon@jonathon-framework:~$ linstor --controller new-xostor-server-01 resource-definition create pvc-96cbebbe-f827-4a47-ae95-38b078e0d584 --resource-group sc-74e1434b-b435-587e-9dea-fa067deec898
SUCCESS:
Description:
    New resource definition 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' created.
Details:
    Resource definition 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' UUID is: 772692e2-3fca-4069-92e9-2bef22c68a6f
jonathon@jonathon-framework:~$ linstor --controller new-xostor-server-01 volume-definition create pvc-96cbebbe-f827-4a47-ae95-38b078e0d584 21483225088B --storage-pool xcp-sr-linstor_group_thin_device
SUCCESS:
    Successfully set property key(s): StorPoolName
SUCCESS:
    New volume definition with number '0' of resource definition 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' created.
jonathon@jonathon-framework:~$ linstor --controller new-xostor-server-01 resource create --storage-pool xcp-sr-linstor_group_thin_device --providers LVM_THIN new-xostor-server-01 pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
SUCCESS:
    Successfully set property key(s): StorPoolName
INFO:
    Updated pvc-96cbebbe-f827-4a47-ae95-38b078e0d584 DRBD auto verify algorithm to 'crct10dif-pclmul'
SUCCESS:
Description:
    New resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on node 'new-xostor-server-01' registered.
Details:
    Resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on node 'new-xostor-server-01' UUID is: 3072aaae-4a34-453e-bdc6-facb47809b3d
SUCCESS:
Description:
    Volume with number '0' on resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on node 'new-xostor-server-01' successfully registered
Details:
    Volume UUID is: 52b11ef6-ec50-42fb-8710-1d3f8c15c657
SUCCESS:
    Created resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on 'new-xostor-server-01'
jonathon@jonathon-framework:~$ linstor --controller new-xostor-server-01 resource create --auto-place +1 pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
SUCCESS:
    Successfully set property key(s): StorPoolName
SUCCESS:
    Successfully set property key(s): StorPoolName
SUCCESS:
Description:
    Resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' successfully autoplaced on 2 nodes
Details:
    Used nodes (storage pool name): 'new-xostor-server-02 (xcp-sr-linstor_group_thin_device)', 'new-xostor-server-03 (xcp-sr-linstor_group_thin_device)'
INFO:
    Resource-definition property 'DrbdOptions/Resource/quorum' updated from 'off' to 'majority' by auto-quorum
INFO:
    Resource-definition property 'DrbdOptions/Resource/on-no-quorum' updated from 'off' to 'suspend-io' by auto-quorum
SUCCESS:
    Added peer(s) 'new-xostor-server-02', 'new-xostor-server-03' to resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on 'new-xostor-server-01'
SUCCESS:
    Created resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on 'new-xostor-server-02'
SUCCESS:
    Created resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on 'new-xostor-server-03'
SUCCESS:
Description:
    Resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on 'new-xostor-server-03' ready
Details:
    Auto-placing resource: pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
SUCCESS:
Description:
    Resource 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584' on 'new-xostor-server-02' ready
Details:
    Auto-placing resource: pvc-96cbebbe-f827-4a47-ae95-38b078e0d584

At this point

jonathon@jonathon-framework:~$ linstor --controller new-xostor-server-01 v l | grep -e 'pvc-96cbebbe-f827-4a47-ae95-38b078e0d584'
| new-xostor-server-01                         | pvc-96cbebbe-f827-4a47-ae95-38b078e0d584        | xcp-sr-linstor_group_thin_device |     0 |    1032 | /dev/drbd1032 |   9.20 GiB | Unused | UpToDate |
| new-xostor-server-02                         | pvc-96cbebbe-f827-4a47-ae95-38b078e0d584        | xcp-sr-linstor_group_thin_device |     0 |    1032 | /dev/drbd1032 | 112.73 MiB | Unused | UpToDate |
| new-xostor-server-03                         | pvc-96cbebbe-f827-4a47-ae95-38b078e0d584        | xcp-sr-linstor_group_thin_device |     0 |    1032 | /dev/drbd1032 | 112.73 MiB | Unused | UpToDate |

To force the sync, run the following command on the node with the data

drbdadm invalidate-remote pvc-96cbebbe-f827-4a47-ae95-38b078e0d584

This will kick it to get the data re-synced.

[14:51 new-xostor-server-01 ~]# drbdadm invalidate-remote pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
[14:51 new-xostor-server-01 ~]# drbdadm status pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
pvc-96cbebbe-f827-4a47-ae95-38b078e0d584 role:Secondary
  disk:UpToDate
  new-xostor-server-02 role:Secondary
    replication:SyncSource peer-disk:Inconsistent done:1.14
  new-xostor-server-03 role:Secondary
    replication:SyncSource peer-disk:Inconsistent done:1.18

[14:51 new-xostor-server-01 ~]# drbdadm status pvc-96cbebbe-f827-4a47-ae95-38b078e0d584
pvc-96cbebbe-f827-4a47-ae95-38b078e0d584 role:Secondary
  disk:UpToDate
  new-xostor-server-02 role:Secondary
    peer-disk:UpToDate
  new-xostor-server-03 role:Secondary
    peer-disk:UpToDate

See: https://github.com/LINBIT/linstor-server/issues/389

Replica-1setup

# PV: pvc-6408a214-6def-44c4-8d9a-bebb67be5510
# S: pgdata-snapshot
# s: 10741612544B

#get size
lvs --noheadings --units B -o lv_size linstor_group/pvc-6408a214-6def-44c4-8d9a-bebb67be5510_00000

#prep
lvcreate -V 10741612544B --thinpool linstor_group/thin_device -n pvc-6408a214-6def-44c4-8d9a-bebb67be5510_00000 linstor_group

#create snapshot
linstor --controller original-xostor-server s create pvc-6408a214-6def-44c4-8d9a-bebb67be5510  pgdata-snapshot

#send
thin_send linstor_group/pvc-6408a214-6def-44c4-8d9a-bebb67be5510_00000_pgdata-snapshot 2>/dev/null | ssh root@new-xostor-server-01 thin_recv linstor_group/pvc-6408a214-6def-44c4-8d9a-bebb67be5510_00000 2>/dev/null

# 1
linstor --controller new-xostor-server-01 resource-definition create pvc-6408a214-6def-44c4-8d9a-bebb67be5510  --resource-group sc-b066e430-6206-5588-a490-cc91ecef53d6
linstor --controller new-xostor-server-01 volume-definition create pvc-6408a214-6def-44c4-8d9a-bebb67be5510  10741612544B --storage-pool xcp-sr-linstor_group_thin_device
linstor --controller new-xostor-server-01 resource create new-xostor-server-01 pvc-6408a214-6def-44c4-8d9a-bebb67be5510

---
apiVersion: v1
kind: PersistentVolume
metadata:
  name: pvc-6408a214-6def-44c4-8d9a-bebb67be5510
  annotations:
    pv.kubernetes.io/provisioned-by: linstor.csi.linbit.com
  finalizers:
    - external-provisioner.volume.kubernetes.io/finalizer
    - kubernetes.io/pv-protection
    - external-attacher/linstor-csi-linbit-com
spec:
  accessModes:
    - ReadWriteOnce
  capacity:
    storage: 10Gi # Ensure this matches the actual size of the LINSTOR volume
  persistentVolumeReclaimPolicy: Retain
  storageClassName: linstor-replica-one-local # Adjust to the storage class you want to use
  volumeMode: Filesystem
  csi:
    driver: linstor.csi.linbit.com
    fsType: ext4
    volumeHandle: pvc-6408a214-6def-44c4-8d9a-bebb67be5510
    volumeAttributes:
      linstor.csi.linbit.com/mount-options: ''
      linstor.csi.linbit.com/post-mount-xfs-opts: ''
      linstor.csi.linbit.com/uses-volume-context: 'true'
      linstor.csi.linbit.com/remote-access-policy: |
        - fromSame:
            - xcp-ng/node
  nodeAffinity:
    required:
      nodeSelectorTerms:
        - matchExpressions:
            - key: xcp-ng/node
              operator: In
              values:
                - new-xostor-server-01
---
apiVersion: v1
kind: PersistentVolumeClaim
metadata:
  annotations:
    pv.kubernetes.io/bind-completed: 'yes'
    pv.kubernetes.io/bound-by-controller: 'yes'
    volume.beta.kubernetes.io/storage-provisioner: linstor.csi.linbit.com
    volume.kubernetes.io/selected-node: ovbh-vtest-k8s01-worker01
    volume.kubernetes.io/storage-provisioner: linstor.csi.linbit.com
  finalizers:
    - kubernetes.io/pvc-protection
  name: acid-merch-2
  namespace: default
spec:
  accessModes:
    - ReadWriteOnce
  resources:
    requests:
      storage: 10Gi
  storageClassName: linstor-replica-one-local
  volumeMode: Filesystem
  volumeName: pvc-6408a214-6def-44c4-8d9a-bebb67be5510

0sorkon created this issue in LINBIT/linstor-server

open Incorrect replica size after resource-definition auto-place #389

Jonathon

The reason that it may be socat, is because the commands fail when I try using it, as instructed by https://github.com/LINBIT/thin-send-recv

[13:03 ovbh-pprod-xen11 ~]# thin_send linstor_group/pvc-12aca72c-d94a-4c09-8102-0a6646906f8d_00000_s_test 2>/dev/null | zstd | socat STDIN TCP:10.2.0.10:4321
2024/10/28 13:04:59 socat[25701] E write(5, 0x55da36101da0, 8192): Broken pipe
...
[13:03 ovbh-pprod-xen01 ~]# socat TCP-LISTEN:4321 STDOUT | zstd -d | thin_recv linstor_group/pvc-12aca72c-d94a-4c09-8102-0a6646906f8d_00000 2>/dev/null
2024/10/28 13:04:59 socat[27039] E read(1, 0x560ef6ff4350, 8192): Bad file descriptor

And the same thing happens if I exclude zstd from both commands.

Jonathon

So, did more testing. Looks like thin_send_recv is not the problem, but maybe socat.
I am able to manually migrate resource between XOSTOR (linstor) cluster using thin_send_recv. I have encluded all steps below so that it can be replicated.

And we know socat is used, cause it complains if it is not there.

jonathon@jonathon-framework:~$ linstor --controller 10.2.0.19 backup ship newCluster pvc-086a5817-d813-41fe-86d8-3fac2ae2028f pvc-086a5817-d813-41fe-86d8-3fac2ae2028f
INFO:
    Cannot use node 'ovbh-pprod-xen10' as it does not support the tool(s): SOCAT
INFO:
    Cannot use node 'ovbh-pprod-xen12' as it does not support the tool(s): SOCAT
INFO:
    Cannot use node 'ovbh-pprod-xen13' as it does not support the tool(s): SOCAT
ERROR:
    Backup shipping of resource 'pvc-086a5817-d813-41fe-86d8-3fac2ae2028f' cannot be started since there is no node available that supports backup shipping.

Using 1.0.1 thin_send_recv.

[16:16 ovbh-pprod-xen11 ~]# thin_send --version
1.0.1
[16:16 ovbh-pprod-xen01 ~]# thin_recv --version
1.0.1

Versions of socat match.

[16:16 ovbh-pprod-xen11 ~]# socat -V
socat by Gerhard Rieger and contributors - see www.dest-unreach.org
socat version 1.7.3.2 on Aug  4 2017 04:57:10
   running on Linux version #1 SMP Tue Jan 23 14:12:55 CET 2024, release 4.19.0+1, machine x86_64
features:
  #define WITH_STDIO 1
  #define WITH_FDNUM 1
  #define WITH_FILE 1
  #define WITH_CREAT 1
  #define WITH_GOPEN 1
  #define WITH_TERMIOS 1
  #define WITH_PIPE 1
  #define WITH_UNIX 1
  #define WITH_ABSTRACT_UNIXSOCKET 1
  #define WITH_IP4 1
  #define WITH_IP6 1
  #define WITH_RAWIP 1
  #define WITH_GENERICSOCKET 1
  #define WITH_INTERFACE 1
  #define WITH_TCP 1
  #define WITH_UDP 1
  #define WITH_SCTP 1
  #define WITH_LISTEN 1
  #define WITH_SOCKS4 1
  #define WITH_SOCKS4A 1
  #define WITH_PROXY 1
  #define WITH_SYSTEM 1
  #define WITH_EXEC 1
  #define WITH_READLINE 1
  #define WITH_TUN 1
  #define WITH_PTY 1
  #define WITH_OPENSSL 1
  #undef WITH_FIPS
  #define WITH_LIBWRAP 1
  #define WITH_SYCLS 1
  #define WITH_FILAN 1
  #define WITH_RETRY 1
  #define WITH_MSGLEVEL 0 /*debug*/
...
[16:17 ovbh-pprod-xen01 ~]# socat -V
socat by Gerhard Rieger and contributors - see www.dest-unreach.org
socat version 1.7.3.2 on Aug  4 2017 04:57:10
   running on Linux version #1 SMP Tue Jan 23 14:12:55 CET 2024, release 4.19.0+1, machine x86_64
features:
  #define WITH_STDIO 1
  #define WITH_FDNUM 1
  #define WITH_FILE 1
  #define WITH_CREAT 1
  #define WITH_GOPEN 1
  #define WITH_TERMIOS 1
  #define WITH_PIPE 1
  #define WITH_UNIX 1
  #define WITH_ABSTRACT_UNIXSOCKET 1
  #define WITH_IP4 1
  #define WITH_IP6 1
  #define WITH_RAWIP 1
  #define WITH_GENERICSOCKET 1
  #define WITH_INTERFACE 1
  #define WITH_TCP 1
  #define WITH_UDP 1
  #define WITH_SCTP 1
  #define WITH_LISTEN 1
  #define WITH_SOCKS4 1
  #define WITH_SOCKS4A 1
  #define WITH_PROXY 1
  #define WITH_SYSTEM 1
  #define WITH_EXEC 1
  #define WITH_READLINE 1
  #define WITH_TUN 1
  #define WITH_PTY 1
  #define WITH_OPENSSL 1
  #undef WITH_FIPS
  #define WITH_LIBWRAP 1
  #define WITH_SYCLS 1
  #define WITH_FILAN 1
  #define WITH_RETRY 1
  #define WITH_MSGLEVEL 0 /*debug*/

Migrating using only thin_send_recv works.

Jonathon

@Jonathon

Best posts made by Jonathon

Latest posts made by Jonathon

Manually moving listor resource with thin_send_recv

Migration of data

Commands

Walk-through

Replica-3 Setup

Explanation

Commands

Walk-through

Replica-1setup

open Incorrect replica size after resource-definition auto-place #389