From ab8be47b7b8c178c1e3c9ee8da2d51782b1d55db Mon Sep 17 00:00:00 2001 From: ange Date: Fri, 16 Feb 2024 21:23:32 +0100 Subject: [PATCH] feat: init cluster --- .gitignore | 2 +- 00-init.yaml | 95 + 10-init-k8s.yaml | 21 + get_kube_conf.sh | 7 +- group_vars/all | 10 + hosts | 3 +- manifests/ceph/cluster.yaml | 327 + manifests/ceph/common.yaml | 1252 ++ manifests/ceph/crds.yaml | 13064 +++++++++++++++++++ manifests/ceph/operator.yaml | 671 + manifests/cm-ingress-nginx-controller.yaml | 10 + manifests/ingress-nginx-controller.yaml | 34 + manifests/letsencrypt-prod.yaml | 15 + manifests/letsencrypt-staging.yaml | 15 + manifests/nfs/csi-driver-nfs.yaml | 353 + manifests/nfs/nfs-server.yaml | 81 + playbook.yaml | 120 - rootfs/etc/haproxy/haproxy.cfg | 45 + rootfs/etc/iptables/rules.v4 | 12 + rootfs/etc/iptables/rules.v6 | 8 + rootfs/etc/ssh/sshd_config | 122 + 21 files changed, 16143 insertions(+), 124 deletions(-) create mode 100644 00-init.yaml create mode 100644 10-init-k8s.yaml create mode 100644 group_vars/all create mode 100644 manifests/ceph/cluster.yaml create mode 100644 manifests/ceph/common.yaml create mode 100644 manifests/ceph/crds.yaml create mode 100644 manifests/ceph/operator.yaml create mode 100644 manifests/cm-ingress-nginx-controller.yaml create mode 100644 manifests/ingress-nginx-controller.yaml create mode 100644 manifests/letsencrypt-prod.yaml create mode 100644 manifests/letsencrypt-staging.yaml create mode 100644 manifests/nfs/csi-driver-nfs.yaml create mode 100644 manifests/nfs/nfs-server.yaml delete mode 100644 playbook.yaml create mode 100644 rootfs/etc/haproxy/haproxy.cfg create mode 100644 rootfs/etc/iptables/rules.v4 create mode 100644 rootfs/etc/iptables/rules.v6 create mode 100644 rootfs/etc/ssh/sshd_config diff --git a/.gitignore b/.gitignore index 49d3add..4cbfcb0 100644 --- a/.gitignore +++ b/.gitignore @@ -3,6 +3,6 @@ ### Ansible ### *.retry -test/hosts # End of https://www.toptal.com/developers/gitignore/api/ansible +kubeconfig diff --git a/00-init.yaml b/00-init.yaml new file mode 100644 index 0000000..526e9e9 --- /dev/null +++ b/00-init.yaml @@ -0,0 +1,95 @@ +--- +- name: Init + become: true + hosts: all + + tasks: + - name: Copy conf + copy: + src: rootfs/ + dest: / + + - name: Restart sshd + service: + name: sshd.service + state: restarted + + - name: SSH port 40022 + set_fact: + ansible_port: 40022 + + - name: Add {{ codename }} repo + deb822_repository: + name: debian + types: deb + uris: http://deb.debian.org/debian + suites: "{{ codename }} {{ codename }}-updates {{ codename }}-backports" + components: + - main + - non-free-firmware + + - name: Add {{ codename }}-security repo + deb822_repository: + name: debian-security + types: deb + uris: http://security.debian.org/debian-security/ + suites: "{{ codename }}-security" + components: + - main + - non-free-firmware + + - name: Add libcontainers repo (CRI-O) + deb822_repository: + name: libcontainers + types: deb + uris: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ os }}/" + suites: / + signed_by: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ os }}/Release.key" + + - name: Add libcontainers-crio repo (CRI-O) + deb822_repository: + name: libcontainers-crio + types: deb + uris: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ k8s_version }}/{{ os }}/" + suites: / + signed_by: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ k8s_version }}/{{ os }}/Release.key" + + - name: Add k8s repo + deb822_repository: + name: k8s + types: deb + uris: "https://pkgs.k8s.io/core:/stable:/v{{ k8s_version }}/deb/" + suites: / + signed_by: "https://pkgs.k8s.io/core:/stable:/v{{ k8s_version }}/deb/Release.key" + + - name: Upgrade + apt: + update_cache: true + upgrade: dist + autoremove: true + purge: true + + - name: Install pkgs + apt: + install_recommends: false + name: + - ceph + - ceph-common + - cri-o + - cri-o-runc + - cri-tools + - cron + - iptables-persistent + - kubeadm + - kubectl + - kubectx + - kubelet + - vim + + - name: Enable crio service + service: + name: crio.service + enabled: true + + - name: Reboot + reboot: diff --git a/10-init-k8s.yaml b/10-init-k8s.yaml new file mode 100644 index 0000000..bdb9b5e --- /dev/null +++ b/10-init-k8s.yaml @@ -0,0 +1,21 @@ +--- +- name: Init + become: true + hosts: all + + environment: + KUBECONFIG: /etc/kubernetes/admin.conf + + tasks: + - name: Init k8s + shell: kubeadm init --pod-network-cidr=10.244.0.0/16 --control-plane-endpoint="{{ endpoint }}" + - name: Remove master node taint + shell: kubectl taint node --all node-role.kubernetes.io/control-plane:NoSchedule- + - name: Apply flannel + shell: kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml + - name: Apply nginx ingress + shell: kubectl apply -f "https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v{{ nginx_version }}/deploy/static/provider/baremetal/deploy.yaml" + - name: Apply cert-manager + shell: kubectl apply -f "https://github.com/cert-manager/cert-manager/releases/download/v{{ certmanager_version }}/cert-manager.yaml" + - name: Reboot + reboot: diff --git a/get_kube_conf.sh b/get_kube_conf.sh index 1a3be09..d21882b 100755 --- a/get_kube_conf.sh +++ b/get_kube_conf.sh @@ -1 +1,6 @@ -ssh debian@k8s.gmoker.com sudo cat /etc/kubernetes/admin.conf > kubeconfig +#!/bin/bash -e + +endpoint="$(awk -F '[ \t]*:[ \t]*' '$1 == "endpoint" {print $NF}' group_vars/all)" + +ssh "$endpoint" sudo cat /etc/kubernetes/admin.conf > kubeconfig +chmod 600 kubeconfig diff --git a/group_vars/all b/group_vars/all new file mode 100644 index 0000000..361eca4 --- /dev/null +++ b/group_vars/all @@ -0,0 +1,10 @@ +endpoint: k8s.gmoker.com +letsencrypt_email: ange@yw5n.com + +arch: amd64 +os: Debian_12 +codename: bookworm +k8s_version: 1.27 # https://kubernetes.io/releases/patch-releases/#detailed-release-history-for-active-branches +nginx_version: 1.9.6 # https://github.com/kubernetes/ingress-nginx/releases +certmanager_version: 1.14.2 # https://github.com/cert-manager/cert-manager/releases/ +rook_version: 1.13.4 # https://github.com/rook/rook/releases diff --git a/hosts b/hosts index 901938d..0e63f7a 100644 --- a/hosts +++ b/hosts @@ -1,2 +1 @@ -#debian@vps-gra-k8s10.gmoker.com -debian@vps-sbg-10.gmoker.com +gmk-gra-k8s10 diff --git a/manifests/ceph/cluster.yaml b/manifests/ceph/cluster.yaml new file mode 100644 index 0000000..57fc257 --- /dev/null +++ b/manifests/ceph/cluster.yaml @@ -0,0 +1,327 @@ +################################################################################################################# +# Define the settings for the rook-ceph cluster with common settings for a production cluster. +# All nodes with available raw devices will be used for the Ceph cluster. At least three nodes are required +# in this example. See the documentation for more details on storage settings available. + +# For example, to create the cluster: +# kubectl create -f crds.yaml -f common.yaml -f operator.yaml +# kubectl create -f cluster.yaml +################################################################################################################# + +apiVersion: ceph.rook.io/v1 +kind: CephCluster +metadata: + name: rook-ceph + namespace: rook-ceph # namespace:cluster +spec: + cephVersion: + # The container image used to launch the Ceph daemon pods (mon, mgr, osd, mds, rgw). + # v17 is Quincy, v18 is Reef. + # RECOMMENDATION: In production, use a specific version tag instead of the general v17 flag, which pulls the latest release and could result in different + # versions running within the cluster. See tags available at https://hub.docker.com/r/ceph/ceph/tags/. + # If you want to be more precise, you can always use a timestamp tag such as quay.io/ceph/ceph:v18.2.1-20240103 + # This tag might not contain a new Ceph version, just security fixes from the underlying operating system, which will reduce vulnerabilities + image: quay.io/ceph/ceph:v18.2.1 + # Whether to allow unsupported versions of Ceph. Currently `quincy` and `reef` are supported. + # Future versions such as `squid` (v19) would require this to be set to `true`. + # Do not set to true in production. + allowUnsupported: false + # The path on the host where configuration files will be persisted. Must be specified. + # Important: if you reinstall the cluster, make sure you delete this directory from each host or else the mons will fail to start on the new cluster. + # In Minikube, the '/data' directory is configured to persist across reboots. Use "/data/rook" in Minikube environment. + dataDirHostPath: /var/lib/rook + # Whether or not upgrade should continue even if a check fails + # This means Ceph's status could be degraded and we don't recommend upgrading but you might decide otherwise + # Use at your OWN risk + # To understand Rook's upgrade process of Ceph, read https://rook.io/docs/rook/latest/ceph-upgrade.html#ceph-version-upgrades + skipUpgradeChecks: false + # Whether or not continue if PGs are not clean during an upgrade + continueUpgradeAfterChecksEvenIfNotHealthy: false + # WaitTimeoutForHealthyOSDInMinutes defines the time (in minutes) the operator would wait before an OSD can be stopped for upgrade or restart. + # If the timeout exceeds and OSD is not ok to stop, then the operator would skip upgrade for the current OSD and proceed with the next one + # if `continueUpgradeAfterChecksEvenIfNotHealthy` is `false`. If `continueUpgradeAfterChecksEvenIfNotHealthy` is `true`, then operator would + # continue with the upgrade of an OSD even if its not ok to stop after the timeout. This timeout won't be applied if `skipUpgradeChecks` is `true`. + # The default wait timeout is 10 minutes. + waitTimeoutForHealthyOSDInMinutes: 10 + mon: + # Set the number of mons to be started. Generally recommended to be 3. + # For highest availability, an odd number of mons should be specified. + count: 3 + # The mons should be on unique nodes. For production, at least 3 nodes are recommended for this reason. + # Mons should only be allowed on the same node for test environments where data loss is acceptable. + allowMultiplePerNode: false + mgr: + # When higher availability of the mgr is needed, increase the count to 2. + # In that case, one mgr will be active and one in standby. When Ceph updates which + # mgr is active, Rook will update the mgr services to match the active mgr. + count: 2 + allowMultiplePerNode: false + modules: + # List of modules to optionally enable or disable. + # Note the "dashboard" and "monitoring" modules are already configured by other settings in the cluster CR. + # - name: rook + # enabled: true + # enable the ceph dashboard for viewing cluster status + dashboard: + enabled: true + # serve the dashboard under a subpath (useful when you are accessing the dashboard via a reverse proxy) + # urlPrefix: /ceph-dashboard + # serve the dashboard at the given port. + # port: 8443 + # serve the dashboard using SSL + ssl: true + # The url of the Prometheus instance + # prometheusEndpoint: ://: + # Whether SSL should be verified if the Prometheus server is using https + # prometheusEndpointSSLVerify: false + # enable prometheus alerting for cluster + monitoring: + # requires Prometheus to be pre-installed + enabled: false + # Whether to disable the metrics reported by Ceph. If false, the prometheus mgr module and Ceph exporter are enabled. + # If true, the prometheus mgr module and Ceph exporter are both disabled. Default is false. + metricsDisabled: false + network: + connections: + # Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the network. + # The default is false. When encryption is enabled, all communication between clients and Ceph daemons, or between Ceph daemons will be encrypted. + # When encryption is not enabled, clients still establish a strong initial authentication and data integrity is still validated with a crc check. + # IMPORTANT: Encryption requires the 5.11 kernel for the latest nbd and cephfs drivers. Alternatively for testing only, + # you can set the "mounter: rbd-nbd" in the rbd storage class, or "mounter: fuse" in the cephfs storage class. + # The nbd and fuse drivers are *not* recommended in production since restarting the csi driver pod will disconnect the volumes. + encryption: + enabled: false + # Whether to compress the data in transit across the wire. The default is false. + # See the kernel requirements above for encryption. + compression: + enabled: false + # Whether to require communication over msgr2. If true, the msgr v1 port (6789) will be disabled + # and clients will be required to connect to the Ceph cluster with the v2 port (3300). + # Requires a kernel that supports msgr v2 (kernel 5.11 or CentOS 8.4 or newer). + requireMsgr2: false + # enable host networking + #provider: host + # enable the Multus network provider + #provider: multus + #selectors: + # The selector keys are required to be `public` and `cluster`. + # Based on the configuration, the operator will do the following: + # 1. if only the `public` selector key is specified both public_network and cluster_network Ceph settings will listen on that interface + # 2. if both `public` and `cluster` selector keys are specified the first one will point to 'public_network' flag and the second one to 'cluster_network' + # + # In order to work, each selector value must match a NetworkAttachmentDefinition object in Multus + # + # public: public-conf --> NetworkAttachmentDefinition object name in Multus + # cluster: cluster-conf --> NetworkAttachmentDefinition object name in Multus + # Provide internet protocol version. IPv6, IPv4 or empty string are valid options. Empty string would mean IPv4 + #ipFamily: "IPv6" + # Ceph daemons to listen on both IPv4 and Ipv6 networks + #dualStack: false + # Enable multiClusterService to export the mon and OSD services to peer cluster. + # This is useful to support RBD mirroring between two clusters having overlapping CIDRs. + # Ensure that peer clusters are connected using an MCS API compatible application, like Globalnet Submariner. + #multiClusterService: + # enabled: false + + # enable the crash collector for ceph daemon crash collection + crashCollector: + disable: false + # Uncomment daysToRetain to prune ceph crash entries older than the + # specified number of days. + #daysToRetain: 30 + # enable log collector, daemons will log on files and rotate + logCollector: + enabled: true + periodicity: daily # one of: hourly, daily, weekly, monthly + maxLogSize: 500M # SUFFIX may be 'M' or 'G'. Must be at least 1M. + # automate [data cleanup process](https://github.com/rook/rook/blob/master/Documentation/Storage-Configuration/ceph-teardown.md#delete-the-data-on-hosts) in cluster destruction. + cleanupPolicy: + # Since cluster cleanup is destructive to data, confirmation is required. + # To destroy all Rook data on hosts during uninstall, confirmation must be set to "yes-really-destroy-data". + # This value should only be set when the cluster is about to be deleted. After the confirmation is set, + # Rook will immediately stop configuring the cluster and only wait for the delete command. + # If the empty string is set, Rook will not destroy any data on hosts during uninstall. + confirmation: "" + # sanitizeDisks represents settings for sanitizing OSD disks on cluster deletion + sanitizeDisks: + # method indicates if the entire disk should be sanitized or simply ceph's metadata + # in both case, re-install is possible + # possible choices are 'complete' or 'quick' (default) + method: quick + # dataSource indicate where to get random bytes from to write on the disk + # possible choices are 'zero' (default) or 'random' + # using random sources will consume entropy from the system and will take much more time then the zero source + dataSource: zero + # iteration overwrite N times instead of the default (1) + # takes an integer value + iteration: 1 + # allowUninstallWithVolumes defines how the uninstall should be performed + # If set to true, cephCluster deletion does not wait for the PVs to be deleted. + allowUninstallWithVolumes: false + # To control where various services will be scheduled by kubernetes, use the placement configuration sections below. + # The example under 'all' would have all services scheduled on kubernetes nodes labeled with 'role=storage-node' and + # tolerate taints with a key of 'storage-node'. + # placement: + # all: + # nodeAffinity: + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: role + # operator: In + # values: + # - storage-node + # podAffinity: + # podAntiAffinity: + # topologySpreadConstraints: + # tolerations: + # - key: storage-node + # operator: Exists + # The above placement information can also be specified for mon, osd, and mgr components + # mon: + # Monitor deployments may contain an anti-affinity rule for avoiding monitor + # collocation on the same node. This is a required rule when host network is used + # or when AllowMultiplePerNode is false. Otherwise this anti-affinity rule is a + # preferred rule with weight: 50. + # osd: + # prepareosd: + # mgr: + # cleanup: + annotations: + # all: + # mon: + # osd: + # cleanup: + # prepareosd: + # clusterMetadata annotations will be applied to only `rook-ceph-mon-endpoints` configmap and the `rook-ceph-mon` and `rook-ceph-admin-keyring` secrets. + # And clusterMetadata annotations will not be merged with `all` annotations. + # clusterMetadata: + # kubed.appscode.com/sync: "true" + # If no mgr annotations are set, prometheus scrape annotations will be set by default. + # mgr: + labels: + # all: + # mon: + # osd: + # cleanup: + # mgr: + # prepareosd: + # monitoring is a list of key-value pairs. It is injected into all the monitoring resources created by operator. + # These labels can be passed as LabelSelector to Prometheus + # monitoring: + # crashcollector: + resources: + #The requests and limits set here, allow the mgr pod to use half of one CPU core and 1 gigabyte of memory + # mgr: + # limits: + # memory: "1024Mi" + # requests: + # cpu: "500m" + # memory: "1024Mi" + # The above example requests/limits can also be added to the other components + # mon: + # osd: + # For OSD it also is a possible to specify requests/limits based on device class + # osd-hdd: + # osd-ssd: + # osd-nvme: + # prepareosd: + # mgr-sidecar: + # crashcollector: + # logcollector: + # cleanup: + # exporter: + # The option to automatically remove OSDs that are out and are safe to destroy. + removeOSDsIfOutAndSafeToRemove: false + priorityClassNames: + #all: rook-ceph-default-priority-class + mon: system-node-critical + osd: system-node-critical + mgr: system-cluster-critical + #crashcollector: rook-ceph-crashcollector-priority-class + storage: # cluster level storage configuration and selection + useAllNodes: true + useAllDevices: true + #deviceFilter: + config: + # crushRoot: "custom-root" # specify a non-default root label for the CRUSH map + # metadataDevice: "md0" # specify a non-rotational storage so ceph-volume will use it as block db device of bluestore. + # databaseSizeMB: "1024" # uncomment if the disks are smaller than 100 GB + # osdsPerDevice: "1" # this value can be overridden at the node or device level + # encryptedDevice: "true" # the default value for this option is "false" + # Individual nodes and their config can be specified as well, but 'useAllNodes' above must be set to false. Then, only the named + # nodes below will be used as storage resources. Each node's 'name' field should match their 'kubernetes.io/hostname' label. + # nodes: + # - name: "172.17.4.201" + # devices: # specific devices to use for storage can be specified for each node + # - name: "sdb" + # - name: "nvme01" # multiple osds can be created on high performance devices + # config: + # osdsPerDevice: "5" + # - name: "/dev/disk/by-id/ata-ST4000DM004-XXXX" # devices can be specified using full udev paths + # config: # configuration can be specified at the node level which overrides the cluster level config + # - name: "172.17.4.301" + # deviceFilter: "^sd." + # when onlyApplyOSDPlacement is false, will merge both placement.All() and placement.osd + onlyApplyOSDPlacement: false + # Time for which an OSD pod will sleep before restarting, if it stopped due to flapping + # flappingRestartIntervalHours: 24 + # The section for configuring management of daemon disruptions during upgrade or fencing. + disruptionManagement: + # If true, the operator will create and manage PodDisruptionBudgets for OSD, Mon, RGW, and MDS daemons. OSD PDBs are managed dynamically + # via the strategy outlined in the [design](https://github.com/rook/rook/blob/master/design/ceph/ceph-managed-disruptionbudgets.md). The operator will + # block eviction of OSDs by default and unblock them safely when drains are detected. + managePodBudgets: true + # A duration in minutes that determines how long an entire failureDomain like `region/zone/host` will be held in `noout` (in addition to the + # default DOWN/OUT interval) when it is draining. This is only relevant when `managePodBudgets` is `true`. The default value is `30` minutes. + osdMaintenanceTimeout: 30 + # A duration in minutes that the operator will wait for the placement groups to become healthy (active+clean) after a drain was completed and OSDs came back up. + # Operator will continue with the next drain if the timeout exceeds. It only works if `managePodBudgets` is `true`. + # No values or 0 means that the operator will wait until the placement groups are healthy before unblocking the next drain. + pgHealthCheckTimeout: 0 + + # csi defines CSI Driver settings applied per cluster. + csi: + readAffinity: + # Enable read affinity to enable clients to optimize reads from an OSD in the same topology. + # Enabling the read affinity may cause the OSDs to consume some extra memory. + # For more details see this doc: + # https://rook.io/docs/rook/latest/Storage-Configuration/Ceph-CSI/ceph-csi-drivers/#enable-read-affinity-for-rbd-volumes + enabled: false + + # cephfs driver specific settings. + cephfs: + # Set CephFS Kernel mount options to use https://docs.ceph.com/en/latest/man/8/mount.ceph/#options. + # kernelMountOptions: "" + # Set CephFS Fuse mount options to use https://docs.ceph.com/en/quincy/man/8/ceph-fuse/#options. + # fuseMountOptions: "" + + # healthChecks + # Valid values for daemons are 'mon', 'osd', 'status' + healthCheck: + daemonHealth: + mon: + disabled: false + interval: 45s + osd: + disabled: false + interval: 60s + status: + disabled: false + interval: 60s + # Change pod liveness probe timing or threshold values. Works for all mon,mgr,osd daemons. + livenessProbe: + mon: + disabled: false + mgr: + disabled: false + osd: + disabled: false + # Change pod startup probe timing or threshold values. Works for all mon,mgr,osd daemons. + startupProbe: + mon: + disabled: false + mgr: + disabled: false + osd: + disabled: false diff --git a/manifests/ceph/common.yaml b/manifests/ceph/common.yaml new file mode 100644 index 0000000..c344860 --- /dev/null +++ b/manifests/ceph/common.yaml @@ -0,0 +1,1252 @@ +#################################################################################################### +# Create the common resources that are necessary to start the operator and the ceph cluster. +# These resources *must* be created before the operator.yaml and cluster.yaml or their variants. +# The samples all assume that a single operator will manage a single cluster crd in the same +# "rook-ceph" namespace. +#################################################################################################### + +# Namespace where the operator and other rook resources are created +apiVersion: v1 +kind: Namespace +metadata: + name: rook-ceph # namespace:cluster +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin +rules: + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: objectstorage-provisioner-role + labels: + app.kubernetes.io/part-of: container-object-storage-interface + app.kubernetes.io/component: driver-ceph + app.kubernetes.io/name: cosi-driver-ceph +rules: + - apiGroups: ["objectstorage.k8s.io"] + resources: ["buckets", "bucketaccesses", "bucketclaims", "bucketaccessclasses", "buckets/status", "bucketaccesses/status", "bucketclaims/status", "bucketaccessclasses/status"] + verbs: ["get", "list", "watch", "update", "create", "delete"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: [""] + resources: ["secrets", "events"] + verbs: ["get", "delete", "update", "create"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get"] +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-runner +rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "update", "delete", "patch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments"] + verbs: ["get", "list", "watch", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["volumeattachments/status"] + verbs: ["patch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["persistentvolumeclaims/status"] + verbs: ["patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshots"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "patch", "update"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["update", "patch"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts"] + verbs: ["get"] + - apiGroups: [""] + resources: ["serviceaccounts/token"] + verbs: ["create"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] +--- +# The cluster role for managing all the cluster-specific resources in a namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: rook-ceph-cluster-mgmt + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + - apiGroups: + - "" + - apps + - extensions + resources: + - secrets + - pods + - pods/log + - services + - configmaps + - deployments + - daemonsets + verbs: + - get + - list + - watch + - patch + - create + - update + - delete +--- +# The cluster role for managing the Rook CRDs +apiVersion: rbac.authorization.k8s.io/v1 +# Rook watches for its CRDs in all namespaces, so this should be a cluster-scoped role unless the +# operator config `ROOK_CURRENT_NAMESPACE_ONLY=true`. +kind: ClusterRole +metadata: + name: rook-ceph-global + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + - apiGroups: + - "" + resources: + # Pod access is needed for fencing + - pods + # Node access is needed for determining nodes where mons should run + - nodes + - nodes/proxy + # Rook watches secrets which it uses to configure access to external resources. + # e.g., external Ceph cluster or object store + - secrets + # Rook watches for changes to the rook-operator-config configmap + - configmaps + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + # Rook creates events for its custom resources + - events + # Rook creates PVs and PVCs for OSDs managed by the Rook provisioner + - persistentvolumes + - persistentvolumeclaims + # Rook creates endpoints for mgr and object store access + - endpoints + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch + - apiGroups: + - batch + resources: + - jobs + - cronjobs + verbs: + - get + - list + - watch + - create + - update + - delete + - deletecollection + # The Rook operator must be able to watch all ceph.rook.io resources to reconcile them. + - apiGroups: ["ceph.rook.io"] + resources: + - cephclients + - cephclusters + - cephblockpools + - cephfilesystems + - cephnfses + - cephobjectstores + - cephobjectstoreusers + - cephobjectrealms + - cephobjectzonegroups + - cephobjectzones + - cephbuckettopics + - cephbucketnotifications + - cephrbdmirrors + - cephfilesystemmirrors + - cephfilesystemsubvolumegroups + - cephblockpoolradosnamespaces + - cephcosidrivers + verbs: + - get + - list + - watch + # Ideally the update permission is not required, but Rook needs it to add finalizers to resources. + - update + # Rook must have update access to status subresources for its custom resources. + - apiGroups: ["ceph.rook.io"] + resources: + - cephclients/status + - cephclusters/status + - cephblockpools/status + - cephfilesystems/status + - cephnfses/status + - cephobjectstores/status + - cephobjectstoreusers/status + - cephobjectrealms/status + - cephobjectzonegroups/status + - cephobjectzones/status + - cephbuckettopics/status + - cephbucketnotifications/status + - cephrbdmirrors/status + - cephfilesystemmirrors/status + - cephfilesystemsubvolumegroups/status + - cephblockpoolradosnamespaces/status + verbs: ["update"] + # The "*/finalizers" permission may need to be strictly given for K8s clusters where + # OwnerReferencesPermissionEnforcement is enabled so that Rook can set blockOwnerDeletion on + # resources owned by Rook CRs (e.g., a Secret owned by an OSD Deployment). See more: + # https://kubernetes.io/docs/reference/access-authn-authz/_print/#ownerreferencespermissionenforcement + - apiGroups: ["ceph.rook.io"] + resources: + - cephclients/finalizers + - cephclusters/finalizers + - cephblockpools/finalizers + - cephfilesystems/finalizers + - cephnfses/finalizers + - cephobjectstores/finalizers + - cephobjectstoreusers/finalizers + - cephobjectrealms/finalizers + - cephobjectzonegroups/finalizers + - cephobjectzones/finalizers + - cephbuckettopics/finalizers + - cephbucketnotifications/finalizers + - cephrbdmirrors/finalizers + - cephfilesystemmirrors/finalizers + - cephfilesystemsubvolumegroups/finalizers + - cephblockpoolradosnamespaces/finalizers + verbs: ["update"] + - apiGroups: + - policy + - apps + - extensions + resources: + # This is for the clusterdisruption controller + - poddisruptionbudgets + # This is for both clusterdisruption and nodedrain controllers + - deployments + - replicasets + verbs: + - get + - list + - watch + - create + - update + - delete + - deletecollection + - apiGroups: + - apps + resources: + # This is to add osd deployment owner ref on key rotation + # cron jobs. + - deployments/finalizers + verbs: + - update + - apiGroups: + - healthchecking.openshift.io + resources: + - machinedisruptionbudgets + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - machine.openshift.io + resources: + - machines + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - storage.k8s.io + resources: + - csidrivers + verbs: + - create + - delete + - get + - update + - apiGroups: + - k8s.cni.cncf.io + resources: + - network-attachment-definitions + verbs: + - get +--- +# Aspects of ceph-mgr that require cluster-wide access +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-mgr-cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + - apiGroups: + - "" + resources: + - configmaps + - nodes + - nodes/proxy + - persistentvolumes + verbs: + - get + - list + - watch + - apiGroups: + - "" + resources: + - events + verbs: + - create + - patch + - list + - get + - watch + - apiGroups: + - storage.k8s.io + resources: + - storageclasses + verbs: + - get + - list + - watch +--- +# Aspects of ceph-mgr that require access to the system namespace +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-mgr-system +rules: + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get + - list + - watch +--- +# Used for provisioning ObjectBuckets (OBs) in response to ObjectBucketClaims (OBCs). +# Note: Rook runs a copy of the lib-bucket-provisioner's OBC controller. +# OBCs can be created in any Kubernetes namespace, so this must be a cluster-scoped role. +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-object-bucket + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + - apiGroups: [""] + resources: ["secrets", "configmaps"] + verbs: + # OBC controller creates secrets and configmaps containing information for users about how to + # connect to object buckets. It deletes them when an OBC is deleted. + - get + - create + - update + - delete + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: + # OBC controller gets parameters from the OBC's storageclass + # Rook gets additional parameters from the OBC's storageclass + - get + - apiGroups: ["objectbucket.io"] + resources: ["objectbucketclaims"] + verbs: + # OBC controller needs to list/watch OBCs and get latest version of a reconciled OBC + - list + - watch + - get + # Ideally, update should not be needed, but the OBC controller updates the OBC with bucket + # information outside of the status subresource + - update + # OBC controller does not delete OBCs; users do this + - apiGroups: ["objectbucket.io"] + resources: ["objectbuckets"] + verbs: + # OBC controller needs to list/watch OBs and get latest version of a reconciled OB + - list + - watch + - get + # OBC controller creates an OB when an OBC's bucket has been provisioned by Ceph, updates them + # when an OBC is updated, and deletes them when the OBC is de-provisioned. + - create + - update + - delete + - apiGroups: ["objectbucket.io"] + resources: ["objectbucketclaims/status", "objectbuckets/status"] + verbs: + # OBC controller updates OBC and OB statuses + - update + - apiGroups: ["objectbucket.io"] + # This does not strictly allow the OBC/OB controllers to update finalizers. That is handled by + # the direct "update" permissions above. Instead, this allows Rook's controller to create + # resources which are owned by OBs/OBCs and where blockOwnerDeletion is set. + resources: ["objectbucketclaims/finalizers", "objectbuckets/finalizers"] + verbs: + - update +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-osd +rules: + - apiGroups: + - "" + resources: + - nodes + verbs: + - get + - list +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-system + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + # Most resources are represented by a string representation of their name, such as "pods", just as it appears in the URL for the relevant API endpoint. + # However, some Kubernetes APIs involve a "subresource", such as the logs for a pod. [...] + # To represent this in an RBAC role, use a slash to delimit the resource and subresource. + # https://kubernetes.io/docs/reference/access-authn-authz/rbac/#referring-to-resources + - apiGroups: [""] + resources: ["pods", "pods/log"] + verbs: ["get", "list"] + - apiGroups: [""] + resources: ["pods/exec"] + verbs: ["create"] + - apiGroups: ["csiaddons.openshift.io"] + resources: ["networkfences"] + verbs: ["create", "get", "update", "delete", "watch", "list", "deletecollection"] + - apiGroups: ["apiextensions.k8s.io"] + resources: ["customresourcedefinitions"] + verbs: ["get"] +--- +# This is required by operator-sdk to map the cluster/clusterrolebindings with SA +# otherwise operator-sdk will create a individual file for these. +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-nodeplugin-role +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph # namespace:operator +roleRef: + kind: ClusterRole + name: cephfs-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph # namespace:operator +roleRef: + kind: ClusterRole + name: cephfs-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +# RBAC for ceph cosi driver service account +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: objectstorage-provisioner-role-binding + labels: + app.kubernetes.io/part-of: container-object-storage-interface + app.kubernetes.io/component: driver-ceph + app.kubernetes.io/name: cosi-driver-ceph +subjects: + - kind: ServiceAccount + name: objectstorage-provisioner + namespace: rook-ceph # namespace:operator +roleRef: + kind: ClusterRole + name: objectstorage-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph # namespace:operator +roleRef: + kind: ClusterRole + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph # namespace:operator +roleRef: + kind: ClusterRole + name: rbd-external-provisioner-runner + apiGroup: rbac.authorization.k8s.io +--- +# Grant the rook system daemons cluster-wide access to manage the Rook CRDs, PVCs, and storage classes +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-global + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-global +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator +--- +# Allow the ceph mgr to access cluster-wide resources necessary for the mgr modules +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-mgr-cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-mgr-cluster +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +--- +kind: ClusterRoleBinding +# Give Rook-Ceph Operator permissions to provision ObjectBuckets in response to ObjectBucketClaims. +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-object-bucket +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-object-bucket +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator +--- +# Allow the ceph osd to access cluster-wide resources necessary for determining their topology location +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-osd +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-system + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-system +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-external-provisioner-cfg + namespace: rook-ceph # namespace:operator +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: ["csiaddons.openshift.io"] + resources: ["csiaddonsnodes"] + verbs: ["create"] +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin + namespace: rook-ceph # namespace:operator +rules: + - apiGroups: ["csiaddons.openshift.io"] + resources: ["csiaddonsnodes"] + verbs: ["create"] +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-external-provisioner-cfg + namespace: rook-ceph # namespace:operator +rules: + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "watch", "list", "delete", "update", "create"] + - apiGroups: ["csiaddons.openshift.io"] + resources: ["csiaddonsnodes"] + verbs: ["create"] +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster +rules: + - apiGroups: + - "" + resources: + - pods + - configmaps + verbs: + - get + - list + - watch + - create + - update + - delete +--- +# Aspects of ceph-mgr that operate within the cluster's namespace +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +rules: + - apiGroups: + - "" + resources: + - pods + - services + - pods/log + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - batch + resources: + - jobs + verbs: + - get + - list + - watch + - create + - update + - delete + - apiGroups: + - ceph.rook.io + resources: + - cephclients + - cephclusters + - cephblockpools + - cephfilesystems + - cephnfses + - cephobjectstores + - cephobjectstoreusers + - cephobjectrealms + - cephobjectzonegroups + - cephobjectzones + - cephbuckettopics + - cephbucketnotifications + - cephrbdmirrors + - cephfilesystemmirrors + - cephfilesystemsubvolumegroups + - cephblockpoolradosnamespaces + - cephcosidrivers + verbs: + - get + - list + - watch + - create + - update + - delete + - patch + - apiGroups: + - apps + resources: + - deployments/scale + - deployments + verbs: + - patch + - delete + - apiGroups: + - '' + resources: + - persistentvolumeclaims + verbs: + - delete +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster +rules: + # this is needed for rook's "key-management" CLI to fetch the vault token from the secret when + # validating the connection details and for key rotation operations. + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "update"] + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get", "list", "watch", "create", "update", "delete"] + - apiGroups: ["ceph.rook.io"] + resources: ["cephclusters", "cephclusters/finalizers"] + verbs: ["get", "list", "create", "update", "delete"] +--- +# Aspects of ceph osd purge job that require access to the cluster namespace +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:cluster +rules: + - apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + - apiGroups: ["apps"] + resources: ["deployments"] + verbs: ["get", "delete"] + - apiGroups: ["batch"] + resources: ["jobs"] + verbs: ["get", "list", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "update", "delete", "list"] +--- +kind: Role +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-rgw + namespace: rook-ceph # namespace:cluster +rules: + # Placeholder role so the rgw service account will + # be generated in the csv. Remove this role and role binding + # when fixing https://github.com/rook/rook/issues/10141. + - apiGroups: + - "" + resources: + - configmaps + verbs: + - get +--- +# Allow the operator to manage resources in its own namespace +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: rook-ceph-system + namespace: rook-ceph # namespace:operator + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +rules: + - apiGroups: + - "" + resources: + - pods + - configmaps + - services + verbs: + - get + - list + - watch + - patch + - create + - update + - delete + - apiGroups: + - apps + - extensions + resources: + - daemonsets + - statefulsets + - deployments + verbs: + - get + - list + - watch + - create + - update + - delete + - deletecollection + - apiGroups: + - batch + resources: + - cronjobs + verbs: + - delete + - apiGroups: + - cert-manager.io + resources: + - certificates + - issuers + verbs: + - get + - create + - delete + - apiGroups: + - multicluster.x-k8s.io + resources: + - serviceexports + verbs: + - get + - create +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: cephfs-csi-provisioner-role-cfg + namespace: rook-ceph # namespace:operator +subjects: + - kind: ServiceAccount + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph # namespace:operator +roleRef: + kind: Role + name: cephfs-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-nodeplugin-role-cfg + namespace: rook-ceph # namespace:operator +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph # namespace:operator +roleRef: + kind: Role + name: rbd-csi-nodeplugin + apiGroup: rbac.authorization.k8s.io +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rbd-csi-provisioner-role-cfg + namespace: rook-ceph # namespace:operator +subjects: + - kind: ServiceAccount + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph # namespace:operator +roleRef: + kind: Role + name: rbd-external-provisioner-cfg + apiGroup: rbac.authorization.k8s.io +--- +# Allow the operator to create resources in this cluster's namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-cluster-mgmt + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-cluster-mgmt +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator +--- +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-cmd-reporter +subjects: + - kind: ServiceAccount + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster +--- +# Allow the ceph mgr to access resources scoped to the CephCluster namespace necessary for mgr modules +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-mgr +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +--- +# Allow the ceph mgr to access resources in the Rook operator namespace necessary for mgr modules +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-mgr-system + namespace: rook-ceph # namespace:operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: rook-ceph-mgr-system +subjects: + - kind: ServiceAccount + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster +--- +# Allow the osd pods in this namespace to work with configmaps +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster +--- +# Allow the osd purge job to run in this namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-purge-osd +subjects: + - kind: ServiceAccount + name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:cluster +--- +# Allow the rgw pods in this namespace to work with configmaps +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-rgw + namespace: rook-ceph # namespace:cluster +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-rgw +subjects: + - kind: ServiceAccount + name: rook-ceph-rgw + namespace: rook-ceph # namespace:cluster +--- +# Grant the operator, agent, and discovery agents access to resources in the rook-ceph-system namespace +kind: RoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: rook-ceph-system + namespace: rook-ceph # namespace:operator + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: rook-ceph-system +subjects: + - kind: ServiceAccount + name: rook-ceph-system + namespace: rook-ceph # namespace:operator +--- +# Service account for Ceph COSI driver +apiVersion: v1 +kind: ServiceAccount +metadata: + name: objectstorage-provisioner + namespace: rook-ceph # namespace:operator + labels: + app.kubernetes.io/part-of: container-object-storage-interface + app.kubernetes.io/component: driver-ceph + app.kubernetes.io/name: cosi-driver-ceph +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the job that reports the Ceph version in an image +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-cmd-reporter + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for Ceph mgrs +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-mgr + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for Ceph OSDs +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-osd + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for job that purges OSDs from a Rook-Ceph cluster +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-purge-osd + namespace: rook-ceph # namespace:cluster +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for RGW server +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-rgw + namespace: rook-ceph # namespace:cluster + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the Rook-Ceph operator +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-ceph-system + namespace: rook-ceph # namespace:operator + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/part-of: rook-ceph-operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the CephFS CSI driver +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-plugin-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the CephFS CSI provisioner +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-cephfs-provisioner-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the RBD CSI driver +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-plugin-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret +--- +# Service account for the RBD CSI provisioner +apiVersion: v1 +kind: ServiceAccount +metadata: + name: rook-csi-rbd-provisioner-sa + namespace: rook-ceph # namespace:operator +# imagePullSecrets: +# - name: my-registry-secret diff --git a/manifests/ceph/crds.yaml b/manifests/ceph/crds.yaml new file mode 100644 index 0000000..bf84413 --- /dev/null +++ b/manifests/ceph/crds.yaml @@ -0,0 +1,13064 @@ +############################################################################## +# Create the CRDs that are necessary before creating your Rook cluster. +# These resources *must* be created before the cluster.yaml or their variants. +############################################################################## +--- +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: cephblockpoolradosnamespaces.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBlockPoolRadosNamespace + listKind: CephBlockPoolRadosNamespaceList + plural: cephblockpoolradosnamespaces + singular: cephblockpoolradosnamespace + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephBlockPoolRadosNamespace represents a Ceph BlockPool Rados Namespace + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + type: object + spec: + description: Spec represents the specification of a Ceph BlockPool Rados Namespace + properties: + blockPoolName: + description: BlockPoolName is the name of Ceph BlockPool. Typically it's the name of the CephBlockPool CR. + type: string + x-kubernetes-validations: + - message: blockPoolName is immutable + rule: self == oldSelf + name: + description: The name of the CephBlockPoolRadosNamespaceSpec namespace. + type: string + x-kubernetes-validations: + - message: name is immutable + rule: self == oldSelf + required: + - blockPoolName + type: object + status: + description: Status represents the status of a CephBlockPool Rados Namespace + properties: + info: + additionalProperties: + type: string + nullable: true + type: object + phase: + description: ConditionType represent a resource's status + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: cephblockpools.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBlockPool + listKind: CephBlockPoolList + plural: cephblockpools + singular: cephblockpool + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + name: v1 + schema: + openAPIV3Schema: + description: CephBlockPool represents a Ceph Storage Pool + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + type: object + spec: + description: NamedBlockPoolSpec allows a block pool to be created with a non-default name. + properties: + compressionMode: + description: 'DEPRECATED: use Parameters instead, e.g.' + enum: + - none + - passive + - aggressive + - force + - "" + nullable: true + type: string + crushRoot: + description: The root of the crush hierarchy utilized by the pool + nullable: true + type: string + deviceClass: + description: The device class the OSD should set to for use in the pool + nullable: true + type: string + enableRBDStats: + description: EnableRBDStats is used to enable gathering of statistics for all RBD images in the pool + type: boolean + erasureCoded: + description: The erasure code settings + properties: + algorithm: + description: The algorithm for erasure coding + type: string + codingChunks: + description: Number of coding chunks per object in an erasure coded storage pool (required for erasure-coded pool + minimum: 0 + type: integer + dataChunks: + description: Number of data chunks per object in an erasure coded storage pool (required for erasure-coded pool t + minimum: 0 + type: integer + required: + - codingChunks + - dataChunks + type: object + failureDomain: + description: 'The failure domain: osd/host/(region or zone if available) - technically also any type in the crush ' + type: string + mirroring: + description: The mirroring settings + properties: + enabled: + description: Enabled whether this pool is mirrored or not + type: boolean + mode: + description: 'Mode is the mirroring mode: either pool or image' + type: string + peers: + description: Peers represents the peers spec + nullable: true + properties: + secretNames: + description: SecretNames represents the Kubernetes Secret names to add rbd-mirror or cephfs-mirror peers + items: + type: string + type: array + type: object + snapshotSchedules: + description: SnapshotSchedules is the scheduling of snapshot for mirrored images/pools + items: + description: SnapshotScheduleSpec represents the snapshot scheduling settings of a mirrored pool + properties: + interval: + description: Interval represent the periodicity of the snapshot. + type: string + path: + description: Path is the path to snapshot, only valid for CephFS + type: string + startTime: + description: StartTime indicates when to start the snapshot + type: string + type: object + type: array + type: object + name: + description: The desired name of the pool if different from the CephBlockPool CR name. + enum: + - device_health_metrics + - .nfs + - .mgr + type: string + parameters: + additionalProperties: + type: string + description: Parameters is a list of properties to enable on a given pool + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + quotas: + description: The quota settings + nullable: true + properties: + maxBytes: + description: MaxBytes represents the quota in bytes Deprecated in favor of MaxSize + format: int64 + type: integer + maxObjects: + description: MaxObjects represents the quota in objects + format: int64 + type: integer + maxSize: + description: MaxSize represents the quota in bytes as a string + pattern: ^[0-9]+[\.]?[0-9]*([KMGTPE]i|[kMGTPE])?$ + type: string + type: object + replicated: + description: The replication settings + properties: + hybridStorage: + description: HybridStorage represents hybrid storage tier settings + nullable: true + properties: + primaryDeviceClass: + description: PrimaryDeviceClass represents high performance tier (for example SSD or NVME) for Primary OSD + minLength: 1 + type: string + secondaryDeviceClass: + description: SecondaryDeviceClass represents low performance tier (for example HDDs) for remaining OSDs + minLength: 1 + type: string + required: + - primaryDeviceClass + - secondaryDeviceClass + type: object + replicasPerFailureDomain: + description: ReplicasPerFailureDomain the number of replica in the specified failure domain + minimum: 1 + type: integer + requireSafeReplicaSize: + description: RequireSafeReplicaSize if false allows you to set replica 1 + type: boolean + size: + description: Size - Number of copies per object in a replicated storage pool, including the object itself (requir + minimum: 0 + type: integer + subFailureDomain: + description: SubFailureDomain the name of the sub-failure domain + type: string + targetSizeRatio: + description: TargetSizeRatio gives a hint (%) to Ceph in terms of expected consumption of the total cluster capac + type: number + required: + - size + type: object + statusCheck: + description: The mirroring statusCheck + properties: + mirror: + description: HealthCheckSpec represents the health check of an object store bucket + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + status: + description: CephBlockPoolStatus represents the mirroring status of Ceph Storage Pool + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + info: + additionalProperties: + type: string + nullable: true + type: object + mirroringInfo: + description: MirroringInfoSpec is the status of the pool mirroring + properties: + details: + type: string + lastChanged: + type: string + lastChecked: + type: string + mode: + description: Mode is the mirroring mode + type: string + peers: + description: Peers are the list of peer sites connected to that cluster + items: + description: PeersSpec contains peer details + properties: + client_name: + description: ClientName is the CephX user used to connect to the peer + type: string + direction: + description: Direction is the peer mirroring direction + type: string + mirror_uuid: + description: MirrorUUID is the mirror UUID + type: string + site_name: + description: SiteName is the current site name + type: string + uuid: + description: UUID is the peer UUID + type: string + type: object + type: array + site_name: + description: SiteName is the current site name + type: string + type: object + mirroringStatus: + description: MirroringStatusSpec is the status of the pool mirroring + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last changed + type: string + lastChecked: + description: LastChecked is the last time time the status was checked + type: string + summary: + description: Summary is the mirroring status summary + properties: + daemon_health: + description: DaemonHealth is the health of the mirroring daemon + type: string + health: + description: Health is the mirroring health + type: string + image_health: + description: ImageHealth is the health of the mirrored image + type: string + states: + description: States is the various state for all mirrored images + nullable: true + properties: + error: + description: Error is when the mirroring state is errored + type: integer + replaying: + description: Replaying is when the replay of the mirroring journal is on-going + type: integer + starting_replay: + description: StartingReplay is when the replay of the mirroring journal starts + type: integer + stopped: + description: Stopped is when the mirroring state is stopped + type: integer + stopping_replay: + description: StopReplaying is when the replay of the mirroring journal stops + type: integer + syncing: + description: Syncing is when the image is syncing + type: integer + unknown: + description: Unknown is when the mirroring state is unknown + type: integer + type: object + type: object + type: object + observedGeneration: + description: ObservedGeneration is the latest generation observed by the controller. + format: int64 + type: integer + phase: + description: ConditionType represent a resource's status + type: string + snapshotScheduleStatus: + description: SnapshotScheduleStatusSpec is the status of the snapshot schedule + properties: + details: + description: Details contains potential status errors + type: string + lastChanged: + description: LastChanged is the last time time the status last changed + type: string + lastChecked: + description: LastChecked is the last time time the status was checked + type: string + snapshotSchedules: + description: SnapshotSchedules is the list of snapshots scheduled + items: + description: SnapshotSchedulesSpec is the list of snapshot scheduled for images in a pool + properties: + image: + description: Image is the mirrored image + type: string + items: + description: Items is the list schedules times for a given snapshot + items: + description: SnapshotSchedule is a schedule + properties: + interval: + description: Interval is the interval in which snapshots will be taken + type: string + start_time: + description: StartTime is the snapshot starting time + type: string + type: object + type: array + namespace: + description: Namespace is the RADOS namespace the image is part of + type: string + pool: + description: Pool is the pool name + type: string + type: object + nullable: true + type: array + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: cephbucketnotifications.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBucketNotification + listKind: CephBucketNotificationList + plural: cephbucketnotifications + singular: cephbucketnotification + scope: Namespaced + versions: + - name: v1 + schema: + openAPIV3Schema: + description: CephBucketNotification represents a Bucket Notifications + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + type: object + spec: + description: BucketNotificationSpec represent the spec of a Bucket Notification + properties: + events: + description: List of events that should trigger the notification + items: + description: BucketNotificationSpec represent the event type of the bucket notification + enum: + - s3:ObjectCreated:* + - s3:ObjectCreated:Put + - s3:ObjectCreated:Post + - s3:ObjectCreated:Copy + - s3:ObjectCreated:CompleteMultipartUpload + - s3:ObjectRemoved:* + - s3:ObjectRemoved:Delete + - s3:ObjectRemoved:DeleteMarkerCreated + type: string + type: array + filter: + description: Spec of notification filter + properties: + keyFilters: + description: Filters based on the object's key + items: + description: NotificationKeyFilterRule represent a single key rule in the Notification Filter spec + properties: + name: + description: Name of the filter - prefix/suffix/regex + enum: + - prefix + - suffix + - regex + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + metadataFilters: + description: Filters based on the object's metadata + items: + description: NotificationFilterRule represent a single rule in the Notification Filter spec + properties: + name: + description: Name of the metadata or tag + minLength: 1 + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + tagFilters: + description: Filters based on the object's tags + items: + description: NotificationFilterRule represent a single rule in the Notification Filter spec + properties: + name: + description: Name of the metadata or tag + minLength: 1 + type: string + value: + description: Value to filter on + type: string + required: + - name + - value + type: object + type: array + type: object + topic: + description: The name of the topic associated with this notification + minLength: 1 + type: string + required: + - topic + type: object + status: + description: Status represents the status of an object + properties: + conditions: + items: + description: Condition represents a status condition on any Rook-Ceph Custom Resource. + properties: + lastHeartbeatTime: + format: date-time + type: string + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + description: ConditionReason is a reason for a condition + type: string + status: + type: string + type: + description: ConditionType represent a resource's status + type: string + type: object + type: array + observedGeneration: + description: ObservedGeneration is the latest generation observed by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: cephbuckettopics.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephBucketTopic + listKind: CephBucketTopicList + plural: cephbuckettopics + singular: cephbuckettopic + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + name: v1 + schema: + openAPIV3Schema: + description: CephBucketTopic represents a Ceph Object Topic for Bucket Notifications + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + type: object + spec: + description: BucketTopicSpec represent the spec of a Bucket Topic + properties: + endpoint: + description: Contains the endpoint spec of the topic + properties: + amqp: + description: Spec of AMQP endpoint + properties: + ackLevel: + default: broker + description: The ack level required for this topic (none/broker/routeable) + enum: + - none + - broker + - routeable + type: string + disableVerifySSL: + description: Indicate whether the server certificate is validated by the client or not + type: boolean + exchange: + description: Name of the exchange that is used to route messages based on topics + minLength: 1 + type: string + uri: + description: The URI of the AMQP endpoint to push notification to + minLength: 1 + type: string + required: + - exchange + - uri + type: object + http: + description: Spec of HTTP endpoint + properties: + disableVerifySSL: + description: Indicate whether the server certificate is validated by the client or not + type: boolean + sendCloudEvents: + description: 'Send the notifications with the CloudEvents header: https://github.' + type: boolean + uri: + description: The URI of the HTTP endpoint to push notification to + minLength: 1 + type: string + required: + - uri + type: object + kafka: + description: Spec of Kafka endpoint + properties: + ackLevel: + default: broker + description: The ack level required for this topic (none/broker) + enum: + - none + - broker + type: string + disableVerifySSL: + description: Indicate whether the server certificate is validated by the client or not + type: boolean + uri: + description: The URI of the Kafka endpoint to push notification to + minLength: 1 + type: string + useSSL: + description: Indicate whether to use SSL when communicating with the broker + type: boolean + required: + - uri + type: object + type: object + objectStoreName: + description: The name of the object store on which to define the topic + minLength: 1 + type: string + objectStoreNamespace: + description: The namespace of the object store on which to define the topic + minLength: 1 + type: string + opaqueData: + description: Data which is sent in each event + type: string + persistent: + description: Indication whether notifications to this endpoint are persistent or not + type: boolean + required: + - endpoint + - objectStoreName + - objectStoreNamespace + type: object + status: + description: BucketTopicStatus represents the Status of a CephBucketTopic + properties: + ARN: + description: The ARN of the topic generated by the RGW + nullable: true + type: string + observedGeneration: + description: ObservedGeneration is the latest generation observed by the controller. + format: int64 + type: integer + phase: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: cephclients.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephClient + listKind: CephClientList + plural: cephclients + singular: cephclient + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .status.phase + name: Phase + type: string + name: v1 + schema: + openAPIV3Schema: + description: CephClient represents a Ceph Client + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + type: object + spec: + description: Spec represents the specification of a Ceph Client + properties: + caps: + additionalProperties: + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + name: + type: string + required: + - caps + type: object + status: + description: Status represents the status of a Ceph Client + properties: + info: + additionalProperties: + type: string + nullable: true + type: object + observedGeneration: + description: ObservedGeneration is the latest generation observed by the controller. + format: int64 + type: integer + phase: + description: ConditionType represent a resource's status + type: string + type: object + x-kubernetes-preserve-unknown-fields: true + required: + - metadata + - spec + type: object + served: true + storage: true + subresources: + status: {} +--- +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.11.3 + creationTimestamp: null + name: cephclusters.ceph.rook.io +spec: + group: ceph.rook.io + names: + kind: CephCluster + listKind: CephClusterList + plural: cephclusters + singular: cephcluster + scope: Namespaced + versions: + - additionalPrinterColumns: + - description: Directory used on the K8s nodes + jsonPath: .spec.dataDirHostPath + name: DataDirHostPath + type: string + - description: Number of MONs + jsonPath: .spec.mon.count + name: MonCount + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.phase + name: Phase + type: string + - description: Message + jsonPath: .status.message + name: Message + type: string + - description: Ceph Health + jsonPath: .status.ceph.health + name: Health + type: string + - jsonPath: .spec.external.enable + name: External + type: boolean + - description: Ceph FSID + jsonPath: .status.ceph.fsid + name: FSID + type: string + name: v1 + schema: + openAPIV3Schema: + description: CephCluster is a Ceph storage cluster + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + type: object + spec: + description: ClusterSpec represents the specification of Ceph Cluster + properties: + annotations: + additionalProperties: + additionalProperties: + type: string + description: Annotations are annotations + type: object + description: The annotations-related configuration to add/set on each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + cephConfig: + additionalProperties: + additionalProperties: + type: string + type: object + description: Ceph Config options + nullable: true + type: object + cephVersion: + description: The version information that instructs Rook to orchestrate a particular version of Ceph. + nullable: true + properties: + allowUnsupported: + description: Whether to allow unsupported versions (do not set to true in production) + type: boolean + image: + description: Image is the container image used to launch the ceph daemons, such as quay. + type: string + imagePullPolicy: + description: ImagePullPolicy describes a policy for if/when to pull a container image One of Always, Never, IfNot + enum: + - IfNotPresent + - Always + - Never + - "" + type: string + type: object + cleanupPolicy: + description: Indicates user intent when deleting a cluster; blocks orchestration and should not be set if cluster + nullable: true + properties: + allowUninstallWithVolumes: + description: AllowUninstallWithVolumes defines whether we can proceed with the uninstall if they are RBD images s + type: boolean + confirmation: + description: Confirmation represents the cleanup confirmation + nullable: true + pattern: ^$|^yes-really-destroy-data$ + type: string + sanitizeDisks: + description: SanitizeDisks represents way we sanitize disks + nullable: true + properties: + dataSource: + description: DataSource is the data source to use to sanitize the disk with + enum: + - zero + - random + type: string + iteration: + description: Iteration is the number of pass to apply the sanitizing + format: int32 + type: integer + method: + description: Method is the method we use to sanitize disks + enum: + - complete + - quick + type: string + type: object + type: object + continueUpgradeAfterChecksEvenIfNotHealthy: + description: ContinueUpgradeAfterChecksEvenIfNotHealthy defines if an upgrade should continue even if PGs are not + type: boolean + crashCollector: + description: A spec for the crash controller + nullable: true + properties: + daysToRetain: + description: DaysToRetain represents the number of days to retain crash until they get pruned + type: integer + disable: + description: Disable determines whether we should enable the crash collector + type: boolean + type: object + csi: + description: CSI Driver Options applied per cluster. + properties: + cephfs: + description: CephFS defines CSI Driver settings for CephFS driver. + properties: + fuseMountOptions: + description: FuseMountOptions defines the mount options for ceph fuse mounter. + type: string + kernelMountOptions: + description: KernelMountOptions defines the mount options for kernel mounter. + type: string + type: object + readAffinity: + description: ReadAffinity defines the read affinity settings for CSI driver. + properties: + crushLocationLabels: + description: CrushLocationLabels defines which node labels to use as CRUSH location. + items: + type: string + type: array + enabled: + description: Enables read affinity for CSI driver. + type: boolean + type: object + type: object + dashboard: + description: Dashboard settings + nullable: true + properties: + enabled: + description: Enabled determines whether to enable the dashboard + type: boolean + port: + description: Port is the dashboard webserver port + maximum: 65535 + minimum: 0 + type: integer + prometheusEndpoint: + description: Endpoint for the Prometheus host + type: string + prometheusEndpointSSLVerify: + description: Whether to verify the ssl endpoint for prometheus. Set to false for a self-signed cert. + type: boolean + ssl: + description: SSL determines whether SSL should be used + type: boolean + urlPrefix: + description: URLPrefix is a prefix for all URLs to use the dashboard with a reverse proxy + type: string + type: object + dataDirHostPath: + description: The path on the host where config and data can be persisted + pattern: ^/(\S+) + type: string + x-kubernetes-validations: + - message: DataDirHostPath is immutable + rule: self == oldSelf + disruptionManagement: + description: A spec for configuring disruption management. + nullable: true + properties: + machineDisruptionBudgetNamespace: + description: Deprecated. Namespace to look for MDBs by the machineDisruptionBudgetController + type: string + manageMachineDisruptionBudgets: + description: Deprecated. This enables management of machinedisruptionbudgets. + type: boolean + managePodBudgets: + description: This enables management of poddisruptionbudgets + type: boolean + osdMaintenanceTimeout: + description: 'OSDMaintenanceTimeout sets how many additional minutes the DOWN/OUT interval is for drained failure ' + format: int64 + type: integer + pgHealthCheckTimeout: + description: PGHealthCheckTimeout is the time (in minutes) that the operator will wait for the placement groups t + format: int64 + type: integer + pgHealthyRegex: + description: PgHealthyRegex is the regular expression that is used to determine which PG states should be conside + type: string + type: object + external: + description: Whether the Ceph Cluster is running external to this Kubernetes cluster mon, mgr, osd, mds, and disc + nullable: true + properties: + enable: + description: Enable determines whether external mode is enabled or not + type: boolean + type: object + x-kubernetes-preserve-unknown-fields: true + healthCheck: + description: Internal daemon healthchecks and liveness probe + nullable: true + properties: + daemonHealth: + description: DaemonHealth is the health check for a given daemon + nullable: true + properties: + mon: + description: Monitor represents the health check settings for the Ceph monitor + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + osd: + description: ObjectStorageDaemon represents the health check settings for the Ceph OSDs + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + status: + description: Status represents the health check settings for the Ceph health + nullable: true + properties: + disabled: + type: boolean + interval: + description: Interval is the internal in second or minute for the health check to run like 60s for 60 seconds + type: string + timeout: + type: string + type: object + type: object + livenessProbe: + additionalProperties: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: 'Probe describes a health check to be performed against a container to determine whether it is alive ' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: 'Command is the command line to execute inside the container, the working directory for the command ' + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: Number of seconds after the container has started before liveness probes are initiated. + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + format: int64 + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. + format: int32 + type: integer + type: object + type: object + description: LivenessProbe allows changing the livenessProbe configuration for a given daemon + type: object + startupProbe: + additionalProperties: + description: ProbeSpec is a wrapper around Probe so it can be enabled or disabled for a Ceph daemon + properties: + disabled: + description: Disabled determines whether probe is disable or not + type: boolean + probe: + description: 'Probe describes a health check to be performed against a container to determine whether it is alive ' + properties: + exec: + description: Exec specifies the action to take. + properties: + command: + description: 'Command is the command line to execute inside the container, the working directory for the command ' + items: + type: string + type: array + type: object + failureThreshold: + description: Minimum consecutive failures for the probe to be considered failed after having succeeded. + format: int32 + type: integer + grpc: + description: GRPC specifies an action involving a GRPC port. + properties: + port: + description: Port number of the gRPC service. Number must be in the range 1 to 65535. + format: int32 + type: integer + service: + description: Service is the name of the service to place in the gRPC HealthCheckRequest (see https://github. + type: string + required: + - port + type: object + httpGet: + description: HTTPGet specifies the http request to perform. + properties: + host: + description: Host name to connect to, defaults to the pod IP. + type: string + httpHeaders: + description: Custom headers to set in the request. HTTP allows repeated headers. + items: + description: HTTPHeader describes a custom header to be used in HTTP probes + properties: + name: + description: The header field name. + type: string + value: + description: The header field value + type: string + required: + - name + - value + type: object + type: array + path: + description: Path to access on the HTTP server. + type: string + port: + anyOf: + - type: integer + - type: string + description: Name or number of the port to access on the container. Number must be in the range 1 to 65535. + x-kubernetes-int-or-string: true + scheme: + description: Scheme to use for connecting to the host. Defaults to HTTP. + type: string + required: + - port + type: object + initialDelaySeconds: + description: Number of seconds after the container has started before liveness probes are initiated. + format: int32 + type: integer + periodSeconds: + description: How often (in seconds) to perform the probe. Default to 10 seconds. Minimum value is 1. + format: int32 + type: integer + successThreshold: + description: Minimum consecutive successes for the probe to be considered successful after having failed. + format: int32 + type: integer + tcpSocket: + description: TCPSocket specifies an action involving a TCP port. + properties: + host: + description: 'Optional: Host name to connect to, defaults to the pod IP.' + type: string + port: + anyOf: + - type: integer + - type: string + description: Number or name of the port to access on the container. Number must be in the range 1 to 65535. + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + description: Optional duration in seconds the pod needs to terminate gracefully upon probe failure. + format: int64 + type: integer + timeoutSeconds: + description: Number of seconds after which the probe times out. Defaults to 1 second. Minimum value is 1. + format: int32 + type: integer + type: object + type: object + description: StartupProbe allows changing the startupProbe configuration for a given daemon + type: object + type: object + labels: + additionalProperties: + additionalProperties: + type: string + description: Labels are label for a given daemons + type: object + description: The labels-related configuration to add/set on each Pod related object. + nullable: true + type: object + x-kubernetes-preserve-unknown-fields: true + logCollector: + description: Logging represents loggings settings + nullable: true + properties: + enabled: + description: Enabled represents whether the log collector is enabled + type: boolean + maxLogSize: + anyOf: + - type: integer + - type: string + description: MaxLogSize is the maximum size of the log per ceph daemons. Must be at least 1M. + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + periodicity: + description: Periodicity is the periodicity of the log rotation. + pattern: ^$|^(hourly|daily|weekly|monthly|1h|24h|1d)$ + type: string + type: object + mgr: + description: A spec for mgr related options + nullable: true + properties: + allowMultiplePerNode: + description: AllowMultiplePerNode allows to run multiple managers on the same node (not recommended) + type: boolean + count: + description: Count is the number of manager daemons to run + maximum: 5 + minimum: 0 + type: integer + modules: + description: Modules is the list of ceph manager modules to enable/disable + items: + description: Module represents mgr modules that the user wants to enable or disable + properties: + enabled: + description: Enabled determines whether a module should be enabled or not + type: boolean + name: + description: Name is the name of the ceph manager module + type: string + type: object + nullable: true + type: array + type: object + mon: + description: A spec for mon related options + nullable: true + properties: + allowMultiplePerNode: + description: AllowMultiplePerNode determines if we can run multiple monitors on the same node (not recommended) + type: boolean + count: + description: Count is the number of Ceph monitors + maximum: 9 + minimum: 0 + type: integer + failureDomainLabel: + type: string + stretchCluster: + description: StretchCluster is the stretch cluster specification + properties: + failureDomainLabel: + description: 'FailureDomainLabel the failure domain name (e,g: zone)' + type: string + subFailureDomain: + description: SubFailureDomain is the failure domain within a zone + type: string + zones: + description: Zones is the list of zones + items: + description: MonZoneSpec represents the specification of a zone in a Ceph Cluster + properties: + arbiter: + description: Arbiter determines if the zone contains the arbiter used for stretch cluster mode + type: boolean + name: + description: Name is the name of the zone + type: string + volumeClaimTemplate: + description: VolumeClaimTemplate is the PVC template + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.' + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: spec defines the desired characteristics of a volume requested by a pod author. + properties: + accessModes: + description: 'accessModes contains the desired access modes the volume should have. More info: https://kubernetes.' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify either: * An existing VolumeSnapshot object (snapshot.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: dataSourceRef specifies the object from which to populate the volume with data, if a non-empty volum + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being referenced Note that when a namespace is specified, a g + type: string + required: + - kind + - name + type: object + resources: + description: resources represents the minimum resources the volume should have. + properties: + claims: + description: Claims lists the names of resources, defined in spec. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum amount of compute resources required. + type: object + type: object + selector: + description: selector is a label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: storageClassName is the name of the StorageClass required by the claim. + type: string + volumeMode: + description: volumeMode defines what type of volume is required by the claim. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + status: + description: status represents the current information/status of a persistent volume claim. Read-only. + properties: + accessModes: + description: accessModes contains the actual access modes the volume backing the PVC has. + items: + type: string + type: array + allocatedResourceStatuses: + additionalProperties: + description: When a controller receives persistentvolume claim update with ClaimResourceStatus for a resource tha + type: string + description: allocatedResourceStatuses stores status of resource being resized for the given PVC. + type: object + x-kubernetes-map-type: granular + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: allocatedResources tracks the resources allocated to a PVC including its capacity. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: capacity represents the actual resources of the underlying volume. + type: object + conditions: + description: conditions is the current Condition of persistent volume claim. + items: + description: PersistentVolumeClaimCondition contains details about state of pvc + properties: + lastProbeTime: + description: lastProbeTime is the time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: lastTransitionTime is the time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: message is the human-readable message indicating details about last transition. + type: string + reason: + description: 'reason is a unique, this should be a short, machine understandable string that gives the reason for ' + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + nullable: true + type: array + type: object + volumeClaimTemplate: + description: VolumeClaimTemplate is the PVC definition + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.' + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: spec defines the desired characteristics of a volume requested by a pod author. + properties: + accessModes: + description: 'accessModes contains the desired access modes the volume should have. More info: https://kubernetes.' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify either: * An existing VolumeSnapshot object (snapshot.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: dataSourceRef specifies the object from which to populate the volume with data, if a non-empty volum + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being referenced Note that when a namespace is specified, a g + type: string + required: + - kind + - name + type: object + resources: + description: resources represents the minimum resources the volume should have. + properties: + claims: + description: Claims lists the names of resources, defined in spec. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum amount of compute resources required. + type: object + type: object + selector: + description: selector is a label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: storageClassName is the name of the StorageClass required by the claim. + type: string + volumeMode: + description: volumeMode defines what type of volume is required by the claim. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + status: + description: status represents the current information/status of a persistent volume claim. Read-only. + properties: + accessModes: + description: accessModes contains the actual access modes the volume backing the PVC has. + items: + type: string + type: array + allocatedResourceStatuses: + additionalProperties: + description: When a controller receives persistentvolume claim update with ClaimResourceStatus for a resource tha + type: string + description: allocatedResourceStatuses stores status of resource being resized for the given PVC. + type: object + x-kubernetes-map-type: granular + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: allocatedResources tracks the resources allocated to a PVC including its capacity. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: capacity represents the actual resources of the underlying volume. + type: object + conditions: + description: conditions is the current Condition of persistent volume claim. + items: + description: PersistentVolumeClaimCondition contains details about state of pvc + properties: + lastProbeTime: + description: lastProbeTime is the time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: lastTransitionTime is the time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: message is the human-readable message indicating details about last transition. + type: string + reason: + description: 'reason is a unique, this should be a short, machine understandable string that gives the reason for ' + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + zones: + description: Zones are specified when we want to provide zonal awareness to mons + items: + description: MonZoneSpec represents the specification of a zone in a Ceph Cluster + properties: + arbiter: + description: Arbiter determines if the zone contains the arbiter used for stretch cluster mode + type: boolean + name: + description: Name is the name of the zone + type: string + volumeClaimTemplate: + description: VolumeClaimTemplate is the PVC template + properties: + apiVersion: + description: APIVersion defines the versioned schema of this representation of an object. + type: string + kind: + description: Kind is a string value representing the REST resource this object represents. + type: string + metadata: + description: 'Standard object''s metadata. More info: https://git.k8s.' + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + name: + type: string + namespace: + type: string + type: object + spec: + description: spec defines the desired characteristics of a volume requested by a pod author. + properties: + accessModes: + description: 'accessModes contains the desired access modes the volume should have. More info: https://kubernetes.' + items: + type: string + type: array + dataSource: + description: 'dataSource field can be used to specify either: * An existing VolumeSnapshot object (snapshot.' + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + required: + - kind + - name + type: object + x-kubernetes-map-type: atomic + dataSourceRef: + description: dataSourceRef specifies the object from which to populate the volume with data, if a non-empty volum + properties: + apiGroup: + description: APIGroup is the group for the resource being referenced. + type: string + kind: + description: Kind is the type of resource being referenced + type: string + name: + description: Name is the name of resource being referenced + type: string + namespace: + description: Namespace is the namespace of resource being referenced Note that when a namespace is specified, a g + type: string + required: + - kind + - name + type: object + resources: + description: resources represents the minimum resources the volume should have. + properties: + claims: + description: Claims lists the names of resources, defined in spec. + items: + description: ResourceClaim references one entry in PodSpec.ResourceClaims. + properties: + name: + description: Name must match the name of one entry in pod.spec. + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute resources allowed. More info: https://kubernetes.' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: Requests describes the minimum amount of compute resources required. + type: object + type: object + selector: + description: selector is a label query over volumes to consider for binding. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + storageClassName: + description: storageClassName is the name of the StorageClass required by the claim. + type: string + volumeMode: + description: volumeMode defines what type of volume is required by the claim. + type: string + volumeName: + description: volumeName is the binding reference to the PersistentVolume backing this claim. + type: string + type: object + status: + description: status represents the current information/status of a persistent volume claim. Read-only. + properties: + accessModes: + description: accessModes contains the actual access modes the volume backing the PVC has. + items: + type: string + type: array + allocatedResourceStatuses: + additionalProperties: + description: When a controller receives persistentvolume claim update with ClaimResourceStatus for a resource tha + type: string + description: allocatedResourceStatuses stores status of resource being resized for the given PVC. + type: object + x-kubernetes-map-type: granular + allocatedResources: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: allocatedResources tracks the resources allocated to a PVC including its capacity. + type: object + capacity: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: capacity represents the actual resources of the underlying volume. + type: object + conditions: + description: conditions is the current Condition of persistent volume claim. + items: + description: PersistentVolumeClaimCondition contains details about state of pvc + properties: + lastProbeTime: + description: lastProbeTime is the time we probed the condition. + format: date-time + type: string + lastTransitionTime: + description: lastTransitionTime is the time the condition transitioned from one status to another. + format: date-time + type: string + message: + description: message is the human-readable message indicating details about last transition. + type: string + reason: + description: 'reason is a unique, this should be a short, machine understandable string that gives the reason for ' + type: string + status: + type: string + type: + description: PersistentVolumeClaimConditionType is a valid value of PersistentVolumeClaimCondition.Type + type: string + required: + - status + - type + type: object + type: array + phase: + description: phase represents the current phase of PersistentVolumeClaim. + type: string + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + type: object + type: array + type: object + x-kubernetes-validations: + - message: zones must be less than or equal to count + rule: '!has(self.zones) || (has(self.zones) && (size(self.zones) <= self.count))' + - message: stretchCluster zones must be equal to 3 + rule: '!has(self.stretchCluster) || (has(self.stretchCluster) && (size(self.stretchCluster.zones) > 0) && (size(self.stretchCluster.zones) == 3))' + monitoring: + description: Prometheus based Monitoring settings + nullable: true + properties: + enabled: + description: Enabled determines whether to create the prometheus rules for the ceph cluster. + type: boolean + externalMgrEndpoints: + description: ExternalMgrEndpoints points to an existing Ceph prometheus exporter endpoint + items: + description: EndpointAddress is a tuple that describes single IP address. + properties: + hostname: + description: The Hostname of this endpoint + type: string + ip: + description: The IP of this endpoint. May not be loopback (127.0.0.0/8 or ::1), link-local (169.254.0. + type: string + nodeName: + description: 'Optional: Node hosting this endpoint. This can be used to determine endpoints local to a node.' + type: string + targetRef: + description: Reference to object providing the endpoint. + properties: + apiVersion: + description: API version of the referent. + type: string + fieldPath: + description: If referring to a piece of an object instead of an entire object, this string should contain a valid + type: string + kind: + description: 'Kind of the referent. More info: https://git.k8s.' + type: string + name: + description: 'Name of the referent. More info: https://kubernetes.' + type: string + namespace: + description: 'Namespace of the referent. More info: https://kubernetes.' + type: string + resourceVersion: + description: 'Specific resourceVersion to which this reference is made, if any. More info: https://git.k8s.' + type: string + uid: + description: 'UID of the referent. More info: https://kubernetes.' + type: string + type: object + x-kubernetes-map-type: atomic + required: + - ip + type: object + x-kubernetes-map-type: atomic + nullable: true + type: array + externalMgrPrometheusPort: + description: ExternalMgrPrometheusPort Prometheus exporter port + maximum: 65535 + minimum: 0 + type: integer + interval: + description: Interval determines prometheus scrape interval + type: string + metricsDisabled: + description: Whether to disable the metrics reported by Ceph. + type: boolean + port: + description: Port is the prometheus server port + maximum: 65535 + minimum: 0 + type: integer + type: object + network: + description: Network related configuration + nullable: true + properties: + addressRanges: + description: AddressRanges specify a list of CIDRs that Rook will apply to Ceph's 'public_network' and/or 'cluste + nullable: true + properties: + cluster: + description: Cluster defines a list of CIDRs to use for Ceph cluster network communication. + items: + description: An IPv4 or IPv6 network CIDR. + pattern: ^[0-9a-fA-F:.]{2,}\/[0-9]{1,3}$ + type: string + type: array + public: + description: Public defines a list of CIDRs to use for Ceph public network communication. + items: + description: An IPv4 or IPv6 network CIDR. + pattern: ^[0-9a-fA-F:.]{2,}\/[0-9]{1,3}$ + type: string + type: array + type: object + connections: + description: Settings for network connections such as compression and encryption across the wire. + nullable: true + properties: + compression: + description: Compression settings for the network connections. + nullable: true + properties: + enabled: + description: Whether to compress the data in transit across the wire. The default is not set. + type: boolean + type: object + encryption: + description: Encryption settings for the network connections. + nullable: true + properties: + enabled: + description: Whether to encrypt the data in transit across the wire to prevent eavesdropping the data on the netw + type: boolean + type: object + requireMsgr2: + description: Whether to require msgr2 (port 3300) even if compression or encryption are not enabled. + type: boolean + type: object + dualStack: + description: DualStack determines whether Ceph daemons should listen on both IPv4 and IPv6 + type: boolean + hostNetwork: + description: HostNetwork to enable host network. + type: boolean + ipFamily: + description: IPFamily is the single stack IPv6 or IPv4 protocol + enum: + - IPv4 + - IPv6 + nullable: true + type: string + multiClusterService: + description: Enable multiClusterService to export the Services between peer clusters + properties: + clusterID: + description: ClusterID uniquely identifies a cluster. It is used as a prefix to nslookup exported services. + type: string + enabled: + description: Enable multiClusterService to export the mon and OSD services to peer cluster. + type: boolean + type: object + provider: + description: Provider is what provides network connectivity to the cluster e.g. "host" or "multus". + enum: + - "" + - host + - multus + nullable: true + type: string + x-kubernetes-validations: + - message: network provider must be disabled (reverted to empty string) before a new provider is enabled + rule: self == '' || self == oldSelf + selectors: + additionalProperties: + type: string + description: Selectors define NetworkAttachmentDefinitions to be used for Ceph public and/or cluster networks whe + nullable: true + type: object + type: object + x-kubernetes-preserve-unknown-fields: true + x-kubernetes-validations: + - message: at least one network selector must be specified when using multus + rule: '!has(self.provider) || (self.provider != ''multus'' || (self.provider == ''multus'' && size(self.selectors) > 0))' + - message: the legacy hostNetwork setting can only be set if the network.provider is set to the empty string + rule: '!has(self.hostNetwork) || self.hostNetwork == false || !has(self.provider) || self.provider == ""' + placement: + additionalProperties: + description: Placement is the placement for an object + properties: + nodeAffinity: + description: NodeAffinity is a group of node affinity scheduling rules + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified ' + items: + description: An empty preferred scheduling term matches all objects with implicit weight 0 (i.e. it's a no-op). + properties: + preference: + description: A node selector term, associated with the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: 'A node selector requirement is a selector that contains values, a key, and an operator that relates ' + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: 'A node selector requirement is a selector that contains values, a key, and an operator that relates ' + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + weight: + description: Weight associated with matching the corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will no + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. The terms are ORed. + items: + description: A null or empty node selector term matches no objects. The requirements of them are ANDed. + properties: + matchExpressions: + description: A list of node selector requirements by node's labels. + items: + description: 'A node selector requirement is a selector that contains values, a key, and an operator that relates ' + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements by node's fields. + items: + description: 'A node selector requirement is a selector that contains values, a key, and an operator that relates ' + properties: + key: + description: The label key that the selector applies to. + type: string + operator: + description: Represents a key's relationship to a set of values. + type: string + values: + description: An array of string values. If the operator is In or NotIn, the values array must be non-empty. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + x-kubernetes-map-type: atomic + type: array + required: + - nodeSelectorTerms + type: object + x-kubernetes-map-type: atomic + type: object + podAffinity: + description: PodAffinity is a group of inter pod affinity scheduling rules + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: 'The scheduler will prefer to schedule pods to nodes that satisfy the affinity expressions specified ' + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching th + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by this field are not met at scheduling time, the pod will no + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) t + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching th + type: string + required: + - topologyKey + type: object + type: array + type: object + podAntiAffinity: + description: PodAntiAffinity is a group of inter pod anti affinity scheduling rules + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods to nodes that satisfy the anti-affinity expressions speci + items: + description: The weights of all of the matched WeightedPodAffinityTerm fields are added per-node to find the most + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching th + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the anti-affinity requirements specified by this field are not met at scheduling time, the pod wi + items: + description: Defines a set of pods (namely those matching the labelSelector relative to the given namespace(s)) t + properties: + labelSelector: + description: A label query over a set of resources, in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + namespaceSelector: + description: A label query over the set of namespaces that the term applies to. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: A label selector requirement is a selector that contains values, a key, and an operator that relates + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: operator represents a key's relationship to a set of values. + type: string + values: + description: values is an array of string values. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} pairs. + type: object + type: object + x-kubernetes-map-type: atomic + namespaces: + description: namespaces specifies a static list of namespace names that the term applies to. + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) or not co-located (anti-affinity) with the pods matching th + type: string + required: + - topologyKey + type: object + type: array + type: object + tolerations: + description: The pod this Toleration is attached to tolerates any taint that matches the triple = 1. If this variable is not set or is an invalid value, it's default to 15. + ROOK_CEPH_COMMANDS_TIMEOUT_SECONDS: "15" + # Enable the csi addons sidecar. + CSI_ENABLE_CSIADDONS: "false" + # Enable watch for faster recovery from rbd rwo node loss + ROOK_WATCH_FOR_NODE_FAILURE: "true" + # ROOK_CSIADDONS_IMAGE: "quay.io/csiaddons/k8s-sidecar:v0.8.0" + # The CSI GRPC timeout value (in seconds). It should be >= 120. If this variable is not set or is an invalid value, it's default to 150. + CSI_GRPC_TIMEOUT_SECONDS: "150" + + # Enable topology based provisioning. + CSI_ENABLE_TOPOLOGY: "false" + # Domain labels define which node labels to use as domains + # for CSI nodeplugins to advertise their domains + # NOTE: the value here serves as an example and needs to be + # updated with node labels that define domains of interest + # CSI_TOPOLOGY_DOMAIN_LABELS: "kubernetes.io/hostname,topology.kubernetes.io/zone,topology.rook.io/rack" + + # Enable read affinity for RBD volumes. Recommended to + # set to true if running kernel 5.8 or newer. + CSI_ENABLE_READ_AFFINITY: "false" + # CRUSH location labels define which node labels to use + # as CRUSH location. This should correspond to the values set in + # the CRUSH map. + # Defaults to all the labels mentioned in + # https://rook.io/docs/rook/latest/CRDs/Cluster/ceph-cluster-crd/#osd-topology + # CSI_CRUSH_LOCATION_LABELS: "kubernetes.io/hostname,topology.kubernetes.io/zone,topology.rook.io/rack" + + # Whether to skip any attach operation altogether for CephCSI PVCs. + # See more details [here](https://kubernetes-csi.github.io/docs/skip-attach.html#skip-attach-with-csi-driver-object). + # If set to false it skips the volume attachments and makes the creation of pods using the CephCSI PVC fast. + # **WARNING** It's highly discouraged to use this for RWO volumes. for RBD PVC it can cause data corruption, + # csi-addons operations like Reclaimspace and PVC Keyrotation will also not be supported if set to false + # since we'll have no VolumeAttachments to determine which node the PVC is mounted on. + # Refer to this [issue](https://github.com/kubernetes/kubernetes/issues/103305) for more details. + CSI_CEPHFS_ATTACH_REQUIRED: "true" + CSI_RBD_ATTACH_REQUIRED: "true" + CSI_NFS_ATTACH_REQUIRED: "true" + # Rook Discover toleration. Will tolerate all taints with all keys. + # (Optional) Rook Discover tolerations list. Put here list of taints you want to tolerate in YAML format. + # DISCOVER_TOLERATIONS: | + # - effect: NoSchedule + # key: node-role.kubernetes.io/control-plane + # operator: Exists + # - effect: NoExecute + # key: node-role.kubernetes.io/etcd + # operator: Exists + # (Optional) Rook Discover priority class name to set on the pod(s) + # DISCOVER_PRIORITY_CLASS_NAME: "" + # (Optional) Discover Agent NodeAffinity. + # DISCOVER_AGENT_NODE_AFFINITY: | + # requiredDuringSchedulingIgnoredDuringExecution: + # nodeSelectorTerms: + # - matchExpressions: + # - key: myKey + # operator: DoesNotExist + # (Optional) Discover Agent Pod Labels. + # DISCOVER_AGENT_POD_LABELS: "key1=value1,key2=value2" + # Disable automatic orchestration when new devices are discovered + ROOK_DISABLE_DEVICE_HOTPLUG: "false" + # The duration between discovering devices in the rook-discover daemonset. + ROOK_DISCOVER_DEVICES_INTERVAL: "60m" + # DISCOVER_DAEMON_RESOURCES: | + # - name: DISCOVER_DAEMON_RESOURCES + # resources: + # limits: + # memory: 512Mi + # requests: + # cpu: 100m + # memory: 128Mi +--- +# OLM: BEGIN OPERATOR DEPLOYMENT +apiVersion: apps/v1 +kind: Deployment +metadata: + name: rook-ceph-operator + namespace: rook-ceph # namespace:operator + labels: + operator: rook + storage-backend: ceph + app.kubernetes.io/name: rook-ceph + app.kubernetes.io/instance: rook-ceph + app.kubernetes.io/component: rook-ceph-operator + app.kubernetes.io/part-of: rook-ceph-operator +spec: + selector: + matchLabels: + app: rook-ceph-operator + strategy: + type: Recreate + replicas: 1 + template: + metadata: + labels: + app: rook-ceph-operator + spec: + tolerations: + - effect: NoExecute + key: node.kubernetes.io/unreachable + operator: Exists + tolerationSeconds: 5 + serviceAccountName: rook-ceph-system + containers: + - name: rook-ceph-operator + image: rook/ceph:v1.13.4 + args: ["ceph", "operator"] + securityContext: + runAsNonRoot: true + runAsUser: 2016 + runAsGroup: 2016 + capabilities: + drop: ["ALL"] + volumeMounts: + - mountPath: /var/lib/rook + name: rook-config + - mountPath: /etc/ceph + name: default-config-dir + env: + # If the operator should only watch for cluster CRDs in the same namespace, set this to "true". + # If this is not set to true, the operator will watch for cluster CRDs in all namespaces. + - name: ROOK_CURRENT_NAMESPACE_ONLY + value: "false" + + # Whether to start pods as privileged that mount a host path, which includes the Ceph mon and osd pods. + # Set this to true if SELinux is enabled (e.g. OpenShift) to workaround the anyuid issues. + # For more details see https://github.com/rook/rook/issues/1314#issuecomment-355799641 + - name: ROOK_HOSTPATH_REQUIRES_PRIVILEGED + value: "false" + # Provide customised regex as the values using comma. For eg. regex for rbd based volume, value will be like "(?i)rbd[0-9]+". + # In case of more than one regex, use comma to separate between them. + # Default regex will be "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" + # Add regex expression after putting a comma to blacklist a disk + # If value is empty, the default regex will be used. + - name: DISCOVER_DAEMON_UDEV_BLACKLIST + value: "(?i)dm-[0-9]+,(?i)rbd[0-9]+,(?i)nbd[0-9]+" + + # Time to wait until the node controller will move Rook pods to other + # nodes after detecting an unreachable node. + # Pods affected by this setting are: + # mgr, rbd, mds, rgw, nfs, PVC based mons and osds, and ceph toolbox + # The value used in this variable replaces the default value of 300 secs + # added automatically by k8s as Toleration for + # + # The total amount of time to reschedule Rook pods in healthy nodes + # before detecting a condition will be the sum of: + # --> node-monitor-grace-period: 40 seconds (k8s kube-controller-manager flag) + # --> ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS: 5 seconds + - name: ROOK_UNREACHABLE_NODE_TOLERATION_SECONDS + value: "5" + + # The name of the node to pass with the downward API + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + # The pod name to pass with the downward API + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + # The pod namespace to pass with the downward API + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + # Recommended resource requests and limits, if desired + #resources: + # limits: + # memory: 512Mi + # requests: + # cpu: 200m + # memory: 128Mi + + # Uncomment it to run lib bucket provisioner in multithreaded mode + #- name: LIB_BUCKET_PROVISIONER_THREADS + # value: "5" + + # Uncomment it to run rook operator on the host network + #hostNetwork: true + volumes: + - name: rook-config + emptyDir: {} + - name: default-config-dir + emptyDir: {} +# OLM: END OPERATOR DEPLOYMENT diff --git a/manifests/cm-ingress-nginx-controller.yaml b/manifests/cm-ingress-nginx-controller.yaml new file mode 100644 index 0000000..daa3083 --- /dev/null +++ b/manifests/cm-ingress-nginx-controller.yaml @@ -0,0 +1,10 @@ +apiVersion: v1 +data: + use-proxy-protocol: "true" +kind: ConfigMap +metadata: + name: ingress-nginx-controller + namespace: ingress-nginx + labels: + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx diff --git a/manifests/ingress-nginx-controller.yaml b/manifests/ingress-nginx-controller.yaml new file mode 100644 index 0000000..d1b5e2e --- /dev/null +++ b/manifests/ingress-nginx-controller.yaml @@ -0,0 +1,34 @@ +--- +apiVersion: v1 +kind: Service +metadata: + labels: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + app.kubernetes.io/part-of: ingress-nginx + app.kubernetes.io/version: 1.9.6 + name: ingress-nginx-controller + namespace: ingress-nginx +spec: + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + ports: + - appProtocol: http + name: http + nodePort: 30080 + port: 80 + protocol: TCP + targetPort: http + - appProtocol: https + name: https + nodePort: 30443 + port: 443 + protocol: TCP + targetPort: https + selector: + app.kubernetes.io/component: controller + app.kubernetes.io/instance: ingress-nginx + app.kubernetes.io/name: ingress-nginx + type: NodePort diff --git a/manifests/letsencrypt-prod.yaml b/manifests/letsencrypt-prod.yaml new file mode 100644 index 0000000..a622a48 --- /dev/null +++ b/manifests/letsencrypt-prod.yaml @@ -0,0 +1,15 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + annotations: + name: letsencrypt-prod +spec: + acme: + email: ange@yw5n.com + privateKeySecretRef: + name: letsencrypt-prod + server: https://acme-v02.api.letsencrypt.org/directory + solvers: + - http01: + ingress: + ingressClassName: nginx diff --git a/manifests/letsencrypt-staging.yaml b/manifests/letsencrypt-staging.yaml new file mode 100644 index 0000000..434a939 --- /dev/null +++ b/manifests/letsencrypt-staging.yaml @@ -0,0 +1,15 @@ +apiVersion: cert-manager.io/v1 +kind: ClusterIssuer +metadata: + annotations: + name: letsencrypt-staging +spec: + acme: + email: ange@yw5n.com + privateKeySecretRef: + name: letsencrypt-staging + server: https://acme-staging-v02.api.letsencrypt.org/directory + solvers: + - http01: + ingress: + ingressClassName: nginx diff --git a/manifests/nfs/csi-driver-nfs.yaml b/manifests/nfs/csi-driver-nfs.yaml new file mode 100644 index 0000000..259a866 --- /dev/null +++ b/manifests/nfs/csi-driver-nfs.yaml @@ -0,0 +1,353 @@ +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nfs-controller-sa + namespace: kube-system +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: csi-nfs-node-sa + namespace: kube-system +--- +kind: ClusterRole +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-external-provisioner-role +rules: + - apiGroups: [""] + resources: ["persistentvolumes"] + verbs: ["get", "list", "watch", "create", "delete"] + - apiGroups: [""] + resources: ["persistentvolumeclaims"] + verbs: ["get", "list", "watch", "update"] + - apiGroups: ["storage.k8s.io"] + resources: ["storageclasses"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotclasses", "volumesnapshots"] + verbs: ["get", "list", "watch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents"] + verbs: ["get", "list", "watch", "update", "patch"] + - apiGroups: ["snapshot.storage.k8s.io"] + resources: ["volumesnapshotcontents/status"] + verbs: ["get", "update", "patch"] + - apiGroups: [""] + resources: ["events"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: ["storage.k8s.io"] + resources: ["csinodes"] + verbs: ["get", "list", "watch"] + - apiGroups: [""] + resources: ["nodes"] + verbs: ["get", "list", "watch"] + - apiGroups: ["coordination.k8s.io"] + resources: ["leases"] + verbs: ["get", "list", "watch", "create", "update", "patch"] + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get"] +--- +kind: ClusterRoleBinding +apiVersion: rbac.authorization.k8s.io/v1 +metadata: + name: nfs-csi-provisioner-binding +subjects: + - kind: ServiceAccount + name: csi-nfs-controller-sa + namespace: kube-system +roleRef: + kind: ClusterRole + name: nfs-external-provisioner-role + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: storage.k8s.io/v1 +kind: CSIDriver +metadata: + name: nfs.csi.k8s.io +spec: + attachRequired: false + volumeLifecycleModes: + - Persistent + fsGroupPolicy: File +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + name: csi-nfs-controller + namespace: kube-system +spec: + replicas: 1 + selector: + matchLabels: + app: csi-nfs-controller + template: + metadata: + labels: + app: csi-nfs-controller + spec: + hostNetwork: true # controller also needs to mount nfs to create dir + dnsPolicy: ClusterFirstWithHostNet # available values: Default, ClusterFirstWithHostNet, ClusterFirst + serviceAccountName: csi-nfs-controller-sa + nodeSelector: + kubernetes.io/os: linux # add "kubernetes.io/role: master" to run controller on master node + priorityClassName: system-cluster-critical + securityContext: + seccompProfile: + type: RuntimeDefault + tolerations: + - key: "node-role.kubernetes.io/master" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/controlplane" + operator: "Exists" + effect: "NoSchedule" + - key: "node-role.kubernetes.io/control-plane" + operator: "Exists" + effect: "NoSchedule" + containers: + - name: csi-provisioner + image: registry.k8s.io/sig-storage/csi-provisioner:v4.0.0 + args: + - "-v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election" + - "--leader-election-namespace=kube-system" + - "--extra-create-metadata=true" + - "--timeout=1200s" + env: + - name: ADDRESS + value: /csi/csi.sock + volumeMounts: + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 400Mi + requests: + cpu: 10m + memory: 20Mi + - name: csi-snapshotter + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.3 + args: + - "--v=2" + - "--csi-address=$(ADDRESS)" + - "--leader-election-namespace=kube-system" + - "--leader-election" + - "--timeout=1200s" + env: + - name: ADDRESS + value: /csi/csi.sock + imagePullPolicy: IfNotPresent + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29652 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: nfs + image: registry.k8s.io/sig-storage/nfsplugin:v4.6.0 + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + imagePullPolicy: IfNotPresent + args: + - "-v=5" + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + ports: + - containerPort: 29652 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + volumeMounts: + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + - mountPath: /csi + name: socket-dir + resources: + limits: + memory: 200Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - name: socket-dir + emptyDir: {} +--- +kind: DaemonSet +apiVersion: apps/v1 +metadata: + name: csi-nfs-node + namespace: kube-system +spec: + updateStrategy: + rollingUpdate: + maxUnavailable: 1 + type: RollingUpdate + selector: + matchLabels: + app: csi-nfs-node + template: + metadata: + labels: + app: csi-nfs-node + spec: + hostNetwork: true # original nfs connection would be broken without hostNetwork setting + dnsPolicy: ClusterFirstWithHostNet # available values: Default, ClusterFirstWithHostNet, ClusterFirst + serviceAccountName: csi-nfs-node-sa + priorityClassName: system-node-critical + securityContext: + seccompProfile: + type: RuntimeDefault + nodeSelector: + kubernetes.io/os: linux + tolerations: + - operator: "Exists" + containers: + - name: liveness-probe + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + args: + - --csi-address=/csi/csi.sock + - --probe-timeout=3s + - --health-port=29653 + - --v=2 + volumeMounts: + - name: socket-dir + mountPath: /csi + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: node-driver-registrar + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.10.0 + args: + - --v=2 + - --csi-address=/csi/csi.sock + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + livenessProbe: + exec: + command: + - /csi-node-driver-registrar + - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH) + - --mode=kubelet-registration-probe + initialDelaySeconds: 30 + timeoutSeconds: 15 + env: + - name: DRIVER_REG_SOCK_PATH + value: /var/lib/kubelet/plugins/csi-nfsplugin/csi.sock + - name: KUBE_NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: registration-dir + mountPath: /registration + resources: + limits: + memory: 100Mi + requests: + cpu: 10m + memory: 20Mi + - name: nfs + securityContext: + privileged: true + capabilities: + add: ["SYS_ADMIN"] + allowPrivilegeEscalation: true + image: registry.k8s.io/sig-storage/nfsplugin:v4.6.0 + args: + - "-v=5" + - "--nodeid=$(NODE_ID)" + - "--endpoint=$(CSI_ENDPOINT)" + env: + - name: NODE_ID + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: CSI_ENDPOINT + value: unix:///csi/csi.sock + ports: + - containerPort: 29653 + name: healthz + protocol: TCP + livenessProbe: + failureThreshold: 5 + httpGet: + path: /healthz + port: healthz + initialDelaySeconds: 30 + timeoutSeconds: 10 + periodSeconds: 30 + imagePullPolicy: "IfNotPresent" + volumeMounts: + - name: socket-dir + mountPath: /csi + - name: pods-mount-dir + mountPath: /var/lib/kubelet/pods + mountPropagation: "Bidirectional" + resources: + limits: + memory: 300Mi + requests: + cpu: 10m + memory: 20Mi + volumes: + - name: socket-dir + hostPath: + path: /var/lib/kubelet/plugins/csi-nfsplugin + type: DirectoryOrCreate + - name: pods-mount-dir + hostPath: + path: /var/lib/kubelet/pods + type: Directory + - hostPath: + path: /var/lib/kubelet/plugins_registry + type: Directory + name: registration-dir diff --git a/manifests/nfs/nfs-server.yaml b/manifests/nfs/nfs-server.yaml new file mode 100644 index 0000000..119fdf3 --- /dev/null +++ b/manifests/nfs/nfs-server.yaml @@ -0,0 +1,81 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + labels: + kubernetes.io/metadata.name: nfs + name: nfs +--- +apiVersion: storage.k8s.io/v1 +kind: StorageClass +metadata: + name: nfs-csi +provisioner: nfs.csi.k8s.io +parameters: + server: nfs-server.nfs.svc.cluster.local + share: / +reclaimPolicy: Delete +volumeBindingMode: Immediate +mountOptions: + - nfsvers=4.2 +--- +kind: Service +apiVersion: v1 +metadata: + namespace: nfs + name: nfs-server + labels: + app: nfs-server +spec: + type: ClusterIP + selector: + app: nfs-server + ports: + - name: tcp-2049 + port: 2049 + protocol: TCP + - name: udp-111 + port: 111 + protocol: UDP +--- +kind: Deployment +apiVersion: apps/v1 +metadata: + namespace: nfs + name: nfs-server +spec: + replicas: 1 + selector: + matchLabels: + app: nfs-server + template: + metadata: + name: nfs-server + labels: + app: nfs-server + spec: + nodeSelector: + "kubernetes.io/os": linux + containers: + - name: nfs-server + image: itsthenetwork/nfs-server-alpine:latest + env: + - name: SHARED_DIRECTORY + value: "/exports" + volumeMounts: + - mountPath: /exports + name: nfs-vol + securityContext: + privileged: true + ports: + - name: tcp-2049 + containerPort: 2049 + protocol: TCP + - name: udp-111 + containerPort: 111 + protocol: UDP + volumes: + - name: nfs-vol + hostPath: + path: /var/lib/volumes/ + type: Directory diff --git a/playbook.yaml b/playbook.yaml deleted file mode 100644 index 1ad500f..0000000 --- a/playbook.yaml +++ /dev/null @@ -1,120 +0,0 @@ ---- -- name: Install k8s - become: true - hosts: all - - vars: - arch: amd64 - os: Debian_12 - codename: bookworm - k8s_version: 1.26 - - environment: - KUBECONFIG: /etc/kubernetes/admin.conf - - tasks: - - name: Fix hostname - copy: - dest: /etc/hostname - content: | - {{ inventory_hostname | regex_replace('.*@', '') | regex_replace('\..*$', '') }} - - - name: Copy conf - copy: - src: rootfs/ - dest: / - - - name: Add {{ codename }} repo - copy: - dest: /etc/apt/sources.list - content: | - deb http://deb.debian.org/debian {{ codename }} main - deb-src http://deb.debian.org/debian {{ codename }} main - deb http://deb.debian.org/debian-security/ {{ codename }}-security main - deb-src http://deb.debian.org/debian-security/ {{ codename }}-security main - deb http://deb.debian.org/debian {{ codename }}-updates main - deb-src http://deb.debian.org/debian {{ codename }}-updates main - deb http://deb.debian.org/debian {{ codename }}-backports main - deb-src http://deb.debian.org/debian {{ codename }}-backports main - - - name: Upgrade - apt: - update_cache: true - cache_valid_time: 86400 # 60s * 60m * 24h - upgrade: dist - autoremove: true - purge: true - - - name: Install deps - apt: - name: - - gpg - - kubectx - - vim - - - name: Add libcontainers public key (CRI-O) - apt_key: - url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ os }}/Release.key" - keyring: /etc/apt/keyrings/libcontainers.gpg - - name: Add libcontainers-crio public key (CRI-O) - apt_key: - url: "https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ k8s_version }}/{{ os }}/Release.key" - keyring: /etc/apt/keyrings/libcontainers-crio.gpg - - name: Add k8s public key - apt_key: - url: https://packages.cloud.google.com/apt/doc/apt-key.gpg - keyring: /etc/apt/keyrings/kubernetes.gpg - - - name: Add libcontainers repo (CRI-O) - apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/libcontainers.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/{{ os }}/ /" - filename: libcontainers - - - name: Add libcontainers-crio repo (CRI-O) - apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/libcontainers-crio.gpg] https://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable:/cri-o:/{{ k8s_version }}/{{ os }}/ /" - filename: libcontainers-crio - - - name: Add k8s repo - apt_repository: - repo: "deb [signed-by=/etc/apt/keyrings/kubernetes.gpg] https://apt.kubernetes.io/ kubernetes-xenial main" - filename: kubernetes - - - name: Install k8s - apt: - name: - - cri-o - - cri-o-runc - - kubeadm={{ k8s_version}}* - - kubectl={{ k8s_version}}* - - kubelet={{ k8s_version}}* - update_cache: true - allow_change_held_packages: true - - name: Hold k8s pkg - shell: apt-mark hold cri-o cri-o-runc kubeadm kubectl kubelet - - name: Install crictl - unarchive: - remote_src: yes - src: "https://github.com/kubernetes-sigs/cri-tools/releases/download/v{{ k8s_version }}.0/crictl-v{{ k8s_version }}.0-linux-{{ arch }}.tar.gz" - dest: /usr/local/bin/ - mode: 755 - - - name: Enable crio service - service: - name: crio.service - enabled: true - - - name: Reboot - reboot: - - #- name: Init k8s - # shell: kubeadm init --pod-network-cidr=10.244.0.0/16 --control-plane-endpoint=k8s.gmoker.com - #- name: Remove master node taint - # shell: kubectl taint node --all node-role.kubernetes.io/control-plane:NoSchedule- - #- name: Apply flannel - # shell: kubectl apply -f https://github.com/flannel-io/flannel/releases/latest/download/kube-flannel.yml - #- name: Apply nginx ingress - # shell: kubectl apply -f https://raw.githubusercontent.com/kubernetes/ingress-nginx/controller-v1.8.1/deploy/static/provider/cloud/deploy.yaml - - #- name: Reboot - # reboot: diff --git a/rootfs/etc/haproxy/haproxy.cfg b/rootfs/etc/haproxy/haproxy.cfg new file mode 100644 index 0000000..5d83946 --- /dev/null +++ b/rootfs/etc/haproxy/haproxy.cfg @@ -0,0 +1,45 @@ +global + log /dev/log local0 + log /dev/log local1 notice + chroot /var/lib/haproxy + stats socket /run/haproxy/admin.sock mode 660 level admin + stats timeout 30s + user haproxy + group haproxy + daemon + +defaults + log global + mode http + option httplog + option dontlognull + timeout connect 5000 + timeout client 50000 + timeout server 50000 + errorfile 400 /etc/haproxy/errors/400.http + errorfile 403 /etc/haproxy/errors/403.http + errorfile 408 /etc/haproxy/errors/408.http + errorfile 500 /etc/haproxy/errors/500.http + errorfile 502 /etc/haproxy/errors/502.http + errorfile 503 /etc/haproxy/errors/503.http + errorfile 504 /etc/haproxy/errors/504.http + +frontend http + mode tcp + bind *:80 + use_backend http + +frontend https + mode tcp + bind *:443 + use_backend https + +backend http + mode tcp + balance leastconn + server s1 149.202.74.193:30080 check send-proxy + +backend https + mode tcp + balance leastconn + server s1 149.202.74.193:30443 check send-proxy diff --git a/rootfs/etc/iptables/rules.v4 b/rootfs/etc/iptables/rules.v4 new file mode 100644 index 0000000..b2d7f5d --- /dev/null +++ b/rootfs/etc/iptables/rules.v4 @@ -0,0 +1,12 @@ +*filter +:INPUT DROP [0:0] +:FORWARD DROP [0:0] +:OUTPUT ACCEPT [0:0] +-A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A INPUT -i lo -j ACCEPT +-A INPUT -p icmp -j ACCEPT +-A INPUT -p tcp --dport 80 -j ACCEPT +-A INPUT -p tcp --dport 443 -j ACCEPT +-A INPUT -p tcp --dport 6443 -j ACCEPT +-A INPUT -p tcp --dport 40022 -j ACCEPT +COMMIT diff --git a/rootfs/etc/iptables/rules.v6 b/rootfs/etc/iptables/rules.v6 new file mode 100644 index 0000000..d841a7f --- /dev/null +++ b/rootfs/etc/iptables/rules.v6 @@ -0,0 +1,8 @@ +*filter +:INPUT DROP [0:0] +:FORWARD DROP [0:0] +:OUTPUT ACCEPT [0:0] +-A INPUT -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT +-A INPUT -i lo -j ACCEPT +-A INPUT -p icmp -j ACCEPT +COMMIT diff --git a/rootfs/etc/ssh/sshd_config b/rootfs/etc/ssh/sshd_config new file mode 100644 index 0000000..2f04193 --- /dev/null +++ b/rootfs/etc/ssh/sshd_config @@ -0,0 +1,122 @@ +# This is the sshd server system-wide configuration file. See +# sshd_config(5) for more information. + +# This sshd was compiled with PATH=/usr/local/bin:/usr/bin:/bin:/usr/games + +# The strategy used for options in the default sshd_config shipped with +# OpenSSH is to specify options with their default value where +# possible, but leave them commented. Uncommented options override the +# default value. + +Include /etc/ssh/sshd_config.d/*.conf + +Port 40022 +#AddressFamily any +#ListenAddress 0.0.0.0 +#ListenAddress :: + +#HostKey /etc/ssh/ssh_host_rsa_key +#HostKey /etc/ssh/ssh_host_ecdsa_key +#HostKey /etc/ssh/ssh_host_ed25519_key + +# Ciphers and keying +#RekeyLimit default none + +# Logging +#SyslogFacility AUTH +#LogLevel INFO + +# Authentication: + +#LoginGraceTime 2m +PermitRootLogin no +#StrictModes yes +#MaxAuthTries 6 +#MaxSessions 10 + +#PubkeyAuthentication yes + +# Expect .ssh/authorized_keys2 to be disregarded by default in future. +#AuthorizedKeysFile .ssh/authorized_keys .ssh/authorized_keys2 + +#AuthorizedPrincipalsFile none + +#AuthorizedKeysCommand none +#AuthorizedKeysCommandUser nobody + +# For this to work you will also need host keys in /etc/ssh/ssh_known_hosts +#HostbasedAuthentication no +# Change to yes if you don't trust ~/.ssh/known_hosts for +# HostbasedAuthentication +#IgnoreUserKnownHosts no +# Don't read the user's ~/.rhosts and ~/.shosts files +#IgnoreRhosts yes + +# To disable tunneled clear text passwords, change to no here! +PasswordAuthentication no +#PermitEmptyPasswords no + +# Change to yes to enable challenge-response passwords (beware issues with +# some PAM modules and threads) +KbdInteractiveAuthentication no + +# Kerberos options +#KerberosAuthentication no +#KerberosOrLocalPasswd yes +#KerberosTicketCleanup yes +#KerberosGetAFSToken no + +# GSSAPI options +#GSSAPIAuthentication no +#GSSAPICleanupCredentials yes +#GSSAPIStrictAcceptorCheck yes +#GSSAPIKeyExchange no + +# Set this to 'yes' to enable PAM authentication, account processing, +# and session processing. If this is enabled, PAM authentication will +# be allowed through the KbdInteractiveAuthentication and +# PasswordAuthentication. Depending on your PAM configuration, +# PAM authentication via KbdInteractiveAuthentication may bypass +# the setting of "PermitRootLogin prohibit-password". +# If you just want the PAM account and session checks to run without +# PAM authentication, then enable this but set PasswordAuthentication +# and KbdInteractiveAuthentication to 'no'. +UsePAM yes + +#AllowAgentForwarding yes +#AllowTcpForwarding yes +#GatewayPorts no +#X11Forwarding no +#X11DisplayOffset 10 +#X11UseLocalhost yes +#PermitTTY yes +PrintMotd no +#PrintLastLog yes +#TCPKeepAlive yes +#PermitUserEnvironment no +#Compression delayed +#ClientAliveInterval 0 +#ClientAliveCountMax 3 +#UseDNS no +#PidFile /run/sshd.pid +#MaxStartups 10:30:100 +#PermitTunnel no +#ChrootDirectory none +#VersionAddendum none + +# no default banner path +#Banner none + +# Allow client to pass locale environment variables +AcceptEnv LANG LC_* + +# override default of no subsystems +Subsystem sftp /usr/lib/openssh/sftp-server + +# Example of overriding settings on a per-user basis +#Match User anoncvs +# X11Forwarding no +# AllowTcpForwarding no +# PermitTTY no +# ForceCommand cvs server +ClientAliveInterval 120