../../images/logo.svg

Openshift IPSEC N/S

Note
In this POC, we will cover how to configure an IPSEC communication from OCP to a RHEL node

Configuration RHEL node side

Requirements

butane libreswan

dns install -y butane libreswan

Create CA and certs

Note
For simplicty, we’ell use the same certificate with SAN matching both side
mkdir ca certs private
openssl genrsa -out ca/ca.key.pem 2048
openssl req -x509 -new -nodes -key ca/ca.key.pem -sha256 -days 3650 -out ca/ca.crt.pem -subj "/CN=IPsec Test CA/O=MyOrg/OU=MyUnit/L=MyCity/ST=MyState/C=US"
openssl genrsa -out private/hosts.key.pem 2048

cat > openssl.cnf <<EOF
[ req ]
distinguished_name = req_distinguished_name
req_extensions = v3_req
prompt = no

[ req_distinguished_name ]
C = US
ST = MyState
L = MyCity
O = MyOrg
OU = MyUnit
CN = worker-n.example.com

[ v3_req ]
basicConstraints = CA:FALSE
keyUsage = nonRepudiation, digitalSignature, keyEncipherment
extendedKeyUsage = serverAuth, clientAuth
subjectAltName = @alt_names

[ alt_names ]
DNS.1 = worker-n.example.com #CHANGEME
DNS.2 = rhel-remote.example.com #CHANGEME
EOF

openssl req -new -key private/hosts.key.pem -out certs/hosts.csr.pem -config openssl.cnf

openssl x509 -req -in certs/hosts.csr.pem -CA ca/ca.crt.pem -CAkey ca/ca.key.pem -CAcreateserial -out certs/hosts.crt.pem -days 365 -sha256 -extfile <(printf "subjectAltName=DNS:openshift-host.example.com,DNS:rhel-host.example.com")


openssl pkcs12 -export -out certs/hosts.p12 -inkey private/hosts.key.pem -in certs/hosts.crt.pem -certfile ca/ca.crt.pem -name "allnodes"

Import cert on RHEL node

ipsec initnss
ipsec import certs/hosts.p12

# Verify import is successfull with the name provided ; here "allnodes"
certutil -L -d sql:/var/lib/ipsec/nss/
Certificate Nickname                                         Trust Attributes
                                                             SSL,S/MIME,JAR/XPI

allnodes                                                     u,u,u
IPsec Test CA - MyOrg                                        CT,

IPSEC conf

cat > /etc/ipsec.d/poc.conf << EOF
conn my-host-to-host-vpn
    left=10.8.109.144 #  CAHNGEME This is the RHEL node IP
    leftid=%fromcert
    leftcert="allnodes"    
    right=10.8.51.222 # CHANGEME This is the OCP node IP
    rightid=%fromcert
    authby=rsasig              
    ikev2=yes
    ike=aes256-sha2      
    ikelifetime=8h                     
    esp=aes_gcm256
    auto=start                  
    type=transport 
EOF

Start ipsec.service

systemctl start ipsec.service

Configuration OCP side

Requirements

Note
Install nmstate operator

NNCP

cat > ipsec_nncp.yaml << EOF
apiVersion: nmstate.io/v1
kind: NodeNetworkConfigurationPolicy
metadata:
  name: ipsec-config
spec:
  nodeSelector:
    kubernetes.io/hostname: mmayeras-tqhkk-worker-0-b4ktk
  desiredState:
    interfaces:
    - name: test-ipsec
      type: ipsec
      libreswan:
        left: worker-n.example.com # CHANGEME
        leftid: '%fromcert'
        leftrsasigkey: '%cert'
        leftcert: left_server
        leftmodecfgclient: false
        right: rhel-host.example.com # CHANGEME
        rightid: '%fromcert'
        rightrsasigkey: '%cert'
        rightsubnet: 10.x.x.x/xx # CHANGEME
        ikev2: insist
        type: transport
        esp: aes_gcm256
        ike: aes256-sha2;dh20
EOF

oc apply -f ipsec_nncp.yaml 

Machine config for certificate import

cat > 99-ipsec-worker-endpoint-config.bu << EOF
  variant: openshift
  version: 4.18.0
  metadata:
    name: 99-worker-import-certs
    labels:
      machineconfiguration.openshift.io/role: worker
  systemd:
    units:
    - name: ipsec-import.service
      enabled: true
      contents: |
        [Unit]
        Description=Import external certs into ipsec NSS
        Before=ipsec.service

        [Service]
        Type=oneshot
        ExecStart=/usr/local/bin/ipsec-addcert.sh
        RemainAfterExit=false
        StandardOutput=journal

        [Install]
        WantedBy=multi-user.target
  storage:
    files:
    - path: /etc/pki/certs/ca.pem
      mode: 0400
      overwrite: true
      contents:
        local: ca/ca.crt.pem
    - path: /etc/pki/certs/hosts.p12
      mode: 0400
      overwrite: true
      contents:
        local: certs/hosts.p12
    - path: /usr/local/bin/ipsec-addcert.sh
      mode: 0740
      overwrite: true
      contents:
        inline: |
          #!/bin/bash -e
          echo "importing cert to NSS"
          certutil -A -n "CA" -t "CT,C,C" -d /var/lib/ipsec/nss/ -i /etc/pki/certs/ca.pem
          pk12util -W "" -i /etc/pki/certs/hosts.p12 -d /var/lib/ipsec/nss/
          certutil -M -n "allnodes" -t "u,u,u" -d /var/lib/ipsec/nss/
EOF

butane -d . 99-ipsec-worker-endpoint-config.bu -o ./99-ipsec-worker-endpoint-config.yaml

oc apply -f 
Warning
Ensure thne name used in the certutil command matches the name used in the openssl pkcs12 -export command used previously
Note
Waint until end of machine config rollout

Check config

Ensure tunnel is active

After rollout OCP side, the tunnel should be up

Maintenance script

Warning
Use at your own risk (dangerous steps commented)
#!/bin/bash

set -e

USE_SSH=false
DRAINMODE=""

while getopts ":sd:" opt; do
  case ${opt} in
    s )
      USE_SSH=true
      echo "SSH mode enabled."
      ;;
    d )
      DRAINMODE=$OPTARG
      echo "Drain mode set to: $DRAINMODE"
      ;;
    \? )
      echo "Invalid option: $OPTARG" 1>&2
      echo "Usage: $0 [-s] [-d DRAINMODE]"
      echo "  -s    Use SSH to reboot nodes instead of oc debug"
      echo "  -d    Set drain mode options (e.g., --ignore-daemonsets --delete-emptydir-data --force --disable-eviction)"
      exit 1
      ;;
  esac
done

drain_node() {
    local node=$1
    echo "Draining node: $node with options: $DRAINMODE"
    #oc adm drain $node $DRAINMODE
}

reboot_node() {
    echo $USE_SSH
    local node=$1
    if $USE_SSH; then
        echo "Rebooting node using SSH: $node"
        #ssh core@$node sudo reboot
    else
        echo "Rebooting node using oc debug: $node"
        #oc debug node/$node -- chroot /host reboot
    fi
}

is_node_ready() {
    local node=$1
    oc get node $node --no-headers | awk '{print $2}'
}

nodes=$(oc get nodes -o jsonpath='{.items[*].metadata.name}')

for node in $nodes; do
    echo "Processing node: $node"

    drain_node $node

    reboot_node $node

    echo "Waiting for node $node to be ready..."
    while [[ $(is_node_ready $node) != "Ready" ]]; do
        sleep 30
    done

    echo "Node $node is ready. Proceeding with the next node..."
done

echo "All nodes have been successfully restarted."

Single Node Openshift running on OCP-V disconnected

LAB overview

In this LAB, I’m going to deploy a single node OpenShift Cluster in Openshift Virtualization using a private network without internet access.

1. Requirements

Control allowed registries with Gatekeeper policies

Gatekeeper Operator Overview

Install operator

Create NS

apiVersion: v1
kind: Namespace
metadata:  
  name: openshift-gatekeeper-system 

Create Subscription

apiVersion: operators.coreos.com/v1alpha1
kind: Subscription
metadata: 
  name: gatekeeper-operator-product
  namespace: openshift-operators 
spec:
  channel: stable
  installPlanApproval: Automatic
  name: gatekeeper-operator-product
  source: redhat-operators
  sourceNamespace: openshift-marketplace
  startingCSV: gatekeeper-operator-product.v3.17.0

Create OPA Gatekeeper instance

kind: Gatekeeper
apiVersion: operator.gatekeeper.sh/v1alpha1
metadata:
  name: gatekeeper
spec:
  validatingWebhook: Enabled

Create tests resources

Mirror a test image

In this scenario I already mirrored an UBI9 image in my private repository, the image is avaialble at `quay.io/rh_ee_mmayeras/ubi9:latest``

Ansible Execution Environment Build Factory

Components overview

Ansible Automation Platform

Red Hat® Ansible® Automation Platform is an end-to-end automation platform to configure systems, deploy software, and orchestrate advanced workflows. It includes resources to create, manage, and scale across the entire enterprise.

Execution Environements

These are container images which include the operating system kernel (Red Hat Enterprise Linux® Universal Base Image), automation engine (ansible-core), programming language (Python), as well as all necessary dependencies. Together, they create an isolated execution environment that can interact with—and run on—almost any IT platform.

OCP etcd backup

How to schedule etcd backups using cronjob

What you will need :

  • Namespace
  • Service Account
  • Cluster Role
  • Cluster Role Binding
  • Extend SA priviliges
  • Cronjob

Namespace

---
apiVersion: project.openshift.io/v1
kind: Project
metadata:
  annotations:
    openshift.io/description: Openshift Backup Automation Tool
    openshift.io/display-name: Backup ETCD Automation  
  name: ocp-etcd-backup
  finalizers:
  - kubernetes

Service Account

---
kind: ServiceAccount
apiVersion: v1
metadata:
  name: openshift-backup
  namespace: ocp-etcd-backup
  labels:
    app: openshift-backup

Cluster role

---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
  name: cluster-etcd-backup
rules:
- apiGroups: [""]
  resources:
     - "namespaces"
  verbs: ["get", "list", "create"]

- apiGroups: [""]
  resources:
     - "nodes"
  verbs: ["get", "list"]
- apiGroups: [""]
  resources:
     - "pods"
     - "pods/log"
  verbs: ["get", "list", "create", "delete", "watch"]

Cluster Role Binding

---
kind: ClusterRoleBinding
apiVersion: rbac.authorization.k8s.io/v1
metadata:
  name: openshift-backup
  labels:
    app: openshift-backup
subjects:
  - kind: ServiceAccount
    name: openshift-backup
    namespace: ocp-etcd-backup
roleRef:
  apiGroup: rbac.authorization.k8s.io
  kind: ClusterRole
  name: cluster-etcd-backup

Cronjob

---
kind: CronJob
apiVersion: batch/v1
metadata:
  name: openshift-backup
  namespace: ocp-etcd-backup
  labels:
    app: openshift-backup
spec:
  schedule: "56 23 * * *"
  concurrencyPolicy: Forbid
  successfulJobsHistoryLimit: 5
  failedJobsHistoryLimit: 5
  jobTemplate:
    metadata:
      labels:
        app: openshift-backup
    spec:
      backoffLimit: 0
      template:
        metadata:
          labels:
            app: openshift-backup
        spec:
          containers:
            - name: backup
              image: "registry.redhat.io/openshift4/ose-cli"
              command:
                - "/bin/bash"
                - "-c"
                - oc get no -l node-role.kubernetes.io/master --no-headers -o name | xargs -I {} --  oc debug {} -- bash -c 'chroot /host sudo -E /usr/local/bin/cluster-backup.sh /home/core/backup/ && chroot /host sudo -E find /home/core/backup/ -type f -mmin +"1" -delete'
          restartPolicy: "Never"
          terminationGracePeriodSeconds: 30
          activeDeadlineSeconds: 500
          dnsPolicy: "ClusterFirst"
          serviceAccountName: "openshift-backup"
          serviceAccount: "openshift-backup"
Info

If you wish to backup to a NFS share, change the command into the backup container to :