{"type":"doc","content":[{"type":"extension","attrs":{"layout":"default","extensionType":"com.atlassian.confluence.macro.core","extensionKey":"toc","parameters":{"macroParams":{"minLevel":{"value":"1"},"maxLevel":{"value":"2"},"outline":{"value":"false"},"type":{"value":"list"},"printable":{"value":"false"}},"macroMetadata":{"macroId":{"value":"e3d6d77f-9b1e-4577-b9c0-74465875e110"},"schemaVersion":{"value":"1"},"title":"Table of Contents"}},"localId":"3fe77454-2d4d-4827-b89c-8a599085542a"}},{"type":"heading","attrs":{"level":1},"content":[{"text":"OpenStack Magnum: Container-as-a-Service","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Deprecated","type":"text"}]},{"type":"paragraph","content":[{"text":"Magnum is deprecated and will be replaced in the future with Cluster API.","type":"text"}]}]},{"type":"paragraph"},{"type":"paragraph","content":[{"text":"Magnum is the Container-as-a-Service for OpenStack and can be used to create and launch clusters.","type":"text"}]},{"type":"paragraph","content":[{"text":"The clusters Magnum supports are:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Kubernetes","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Swarm","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Mesos","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Magnum uses the following OpenStack Components:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Keystone:","type":"text","marks":[{"type":"strong"}]},{"text":" for multi tenancy","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Nova compute:","type":"text","marks":[{"type":"strong"}]},{"text":" computing service","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Heat:","type":"text","marks":[{"type":"strong"}]},{"text":" virtual application deployment service","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Neutron:","type":"text","marks":[{"type":"strong"}]},{"text":" networking","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Glance:","type":"text","marks":[{"type":"strong"}]},{"text":" virtual machine image","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Cinder:","type":"text","marks":[{"type":"strong"}]},{"text":" volume service","type":"text"}]}]}]},{"type":"paragraph","content":[{"text":"Magnum uses Cluster Templates to define the desired cluster, and passes the template for the cluster to Heat to create the user’s cluster.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"python-magnumclient","type":"text"}]},{"type":"paragraph","content":[{"text":"To use commands for creating or managing clusters using Magnum in the command line, you will need to install the Magnum CLI. This can be done using pip:","type":"text"}]},{"type":"codeBlock","content":[{"text":"pip install python-magnumclient","type":"text"}]},{"type":"paragraph","content":[{"text":"To test that we can run OpenStack commands for container orchestration engines (coe), we can run the following command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster list #list clusters in the project","type":"text"}]},{"type":"paragraph","content":[{"text":"This should return an empty line if there are no clusters in the project, or a table similar to the following:","type":"text"}]},{"type":"codeBlock","content":[{"text":"+----------+---------------------------+----------------+------------+--------------+-----------------+---------------+\n| uuid | name | keypair | node_count | master_count | status | health_status |\n+----------+---------------------------+----------------+------------+--------------+-----------------+---------------+\n| UUID_1 | test-1 | mykeypair | 1 | 1 | CREATE_COMPLETE | UNKNOWN |\n| UUID_2 | kubernetes-cluster-test-2 | mykeypair | 1 | 1 | CREATE_COMPLETE | UNKNOWN |\n+----------+---------------------------+----------------+------------+--------------+-----------------+---------------+","type":"text"}]},{"type":"paragraph","content":[{"text":"Other commands available from ","type":"text"},{"text":"python-magumclient","type":"text","marks":[{"type":"code"}]},{"text":" include:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe \n\ncoe cluster config # download cluster config to current directory\ncoe cluster create # create a cluster\ncoe cluster delete # delete a cluster\ncoe cluster list # list clusters\ncoe cluster resize # resize cluster\ncoe cluster show # view details of cluster\ncoe cluster template create # create a cluster template\ncoe cluster template delete # delete a cluster template - can only be deleted if there are no clusters using the template\ncoe cluster template list # list templates\ncoe cluster template show # view details of cluster template\ncoe cluster template update # update a cluster template\ncoe cluster update # update a cluster e.g node count\ncoe cluster upgrade # upgrade a cluster e.g upgrade Kubernetes version in cluster\ncoe nodegroup create # create a nodegroup\ncoe nodegroup delete # delete a nodegroup\ncoe nodegroup list # list nodegroups in cluster\ncoe nodegroup show # view details of a nodegroup in a cluster\ncoe nodegroup update # update a nodegroup\n","type":"text"}]},{"type":"paragraph","content":[{"text":"To view details of a cluster:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster view \n\n#This should return a table similar to the table below:\n\n+----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+\n| Field | Value |\n+----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+\n| status | CREATE_COMPLETE |\n| health_status | UNKNOWN |\n| cluster_template_id | e186c6e2-dd47-4df0-ac3f-3eb46e64cb3d |\n| node_addresses | ['10.0.0.163'] |\n| uuid | 27cdcad8-375f-4d4f-a186-8fa99b80c5c5 |\n| stack_id | e881d058-db91-4de6-9527-193eecebd05d |\n| status_reason | None |\n| created_at | 2020-09-07T15:39:32+00:00 |\n| updated_at | 2020-09-07T15:52:54+00:00 |\n| coe_version | v1.14.3 |\n| labels | {'auto_healing': 'true', 'kube_tag': 'v1.14.3', 'heat_container_agent_tag': 'train-stable-3', 'kube_dashboard_enabled': '1', 'ingress_controller': 'traefik'} |\n| labels_overridden | |\n| labels_skipped | |\n| labels_added | |\n| fixed_network | None |\n| fixed_subnet | None |\n| floating_ip_enabled | False |\n| faults | |\n| keypair | mykeypair |\n| api_address | https://10.0.0.212:6443 |\n| master_addresses | ['10.0.0.212'] |\n| master_lb_enabled | |\n| create_timeout | 60 |\n| node_count | 1 |\n| discovery_url | https://discovery.etcd.io/31c1d9cf44cf4fda5710946d57980bb1 |\n| master_count | 1 |\n| container_version | 1.12.6 |\n| name | kubernetes-cluster-test-1 |\n| master_flavor_id | c1.medium |\n| flavor_id | c1.medium |\n| health_status_reason | {'api': 'The cluster kubernetes-cluster-test-1 is not accessible.'} |\n| project_id | PROJECT_ID |\n+----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+","type":"text"}]},{"type":"paragraph","content":[{"text":"To view the list of cluster templates:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster template list\n\n# This should return a list of cluster templates in the project\n\n+--------------------------------------+------------------------------+\n| uuid | name |\n+--------------------------------------+------------------------------+\n| e186c6e2-dd47-4df0-ac3f-3eb46e64cb3d | kubernetes-v1_14_3 |\n| 0bd1232d-06f2-42ca-b6d5-c27e57f26c3c | kubernetes-ha-master-v1_14_3 |\n| a07903d0-aecf-4f15-a35f-f4fd74060e2f | coreos-kubernetes-v1_14_3 |\n+--------------------------------------+------------------------------+","type":"text"}]},{"type":"paragraph","content":[{"text":"To view the details of a specific template:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster template show \n\n#This will return a table similar to:\n\n+-----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+\n| Field | Value |\n+-----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+\n| insecure_registry | - |\n| labels | {'kube_tag': 'v1.14.3', 'kube_dashboard_enabled': '1', 'heat_container_agent_tag': 'train-stable-3', 'auto_healing': 'true', 'ingress_controller': 'traefik'} |\n| updated_at | - |\n| floating_ip_enabled | False |\n| fixed_subnet | - |\n| master_flavor_id | c1.medium |\n| uuid | e186c6e2-dd47-4df0-ac3f-3eb46e64cb3d |\n| no_proxy | - |\n| https_proxy | - |\n| tls_disabled | False |\n| keypair_id | - |\n| public | True |\n| http_proxy | - |\n| docker_volume_size | 3 |\n| server_type | vm |\n| external_network_id | External |\n| cluster_distro | fedora-atomic |\n| image_id | cf37f7d0-1d6b-4aab-a23b-df58542c59cb |\n| volume_driver | - |\n| registry_enabled | False |\n| docker_storage_driver | devicemapper |\n| apiserver_port | - |\n| name | kubernetes-v1_14_3 |\n| created_at | 2020-09-07T07:17:13+00:00 |\n| network_driver | flannel |\n| fixed_network | - |\n| coe | kubernetes |\n| flavor_id | c1.medium |\n| master_lb_enabled | False |\n| dns_nameserver | 8.8.8.8 |\n| hidden | False |\n+-----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+","type":"text"}]},{"type":"paragraph","content":[{"text":"To delete a cluster or cluster template:","type":"text"}]},{"type":"blockquote","content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" Cluster Templates can only be deleted if there are no clusters using the template.","type":"text"}]}]},{"type":"codeBlock","content":[{"text":"# To delete a template\nopenstack coe cluster template delete \n\n# To delete a cluster\nopenstack coe cluster delete ","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Creating Clusters","type":"text"}]},{"type":"paragraph","content":[{"text":"Clusters can be created using:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"OpenStack CLI","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Horizon Web UI","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Heat Templates: using the resources ","type":"text"},{"text":"OS::Magnum::ClusterTemplate","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"OS::Magnum::Cluster","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"The documentation ","type":"text"},{"text":"Create A Kubernetes Cluster","type":"text","marks":[{"type":"strong"}]},{"text":" has examples for handling cluster templates and creating a Kubernetes cluster in the command line.","type":"text"}]}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Create a Cluster using OpenStack CLI","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Create A Cluster Template","type":"text"}]},{"type":"paragraph","content":[{"text":"To create a cluster template, we can use the following command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster template create [-h] [-f {json,shell,table,value,yaml}] [-c COLUMN] [--noindent] [--prefix PREFIX]\n [--max-width ] [--fit-width] [--print-empty] --coe --image \n --external-network [--keypair ]\n [--fixed-network ] [--fixed-subnet ]\n [--network-driver ] [--volume-driver ]\n [--dns-nameserver ] [--flavor ]\n [--master-flavor ] [--docker-volume-size ]\n [--docker-storage-driver ] [--http-proxy ]\n [--https-proxy ] [--no-proxy ]\n [--labels ] [--tls-disabled] [--public]\n [--registry-enabled] [--server-type ] [--master-lb-enabled]\n [--floating-ip-enabled] [--floating-ip-disabled] [--hidden] [--visible]\n \n","type":"text"}]},{"type":"paragraph","content":[{"text":": Name of the ClusterTemplate to create. The name does not have to be unique but the template UUID should be used to select a ClusterTemplate if more than one template has the same name.","type":"text"}]},{"type":"paragraph","content":[{"text":": Container Orchestration Engine to use. Supported drivers are: ","type":"text"},{"text":"kubernetes, swarm, mesos","type":"text","marks":[{"type":"strong"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":": Name or UUID of the base image to boot servers for the clusters.","type":"text"}]},{"type":"paragraph","content":[{"text":"Images which OpenStack Magnum supports:","type":"text","marks":[{"type":"strong"}]}]},{"type":"table","attrs":{"layout":"default","localId":"c7c33f2b-3d5d-43a1-9dd4-26ea459b17bb"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"COE","type":"text"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"os_distro","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Kubernetes","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"fedora-atomic, coreos","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Swarm","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"fedora-atomic","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Mesos","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"ubuntu","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"","type":"text","marks":[{"type":"code"}]},{"text":": SSH keypair to configure in servers for ssh access. The login name is specific to the cluster driver.","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"fedora-atomic: ","type":"text"},{"text":"ssh -i fedora@","type":"text","marks":[{"type":"code"}]}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"coreos: ","type":"text"},{"text":"ssh -i core@","type":"text","marks":[{"type":"code"}]}]}]}]},{"type":"paragraph","content":[{"text":"external-network ","type":"text","marks":[{"type":"code"}]},{"text":": name or ID of a Neutron network to provide connectivity to the external internet.","type":"text"}]},{"type":"paragraph","content":[{"text":"--public","type":"text","marks":[{"type":"code"}]},{"text":" Access to a ClusterTemplate is, by default, limited to admin, owner or users within the same tenant as the owners. Using this flag makes the template accessible by other users. Default is not public","type":"text"}]},{"type":"paragraph","content":[{"text":"server-type ","type":"text","marks":[{"type":"code"}]},{"text":": Servers can be VM or bare metal (bm). The default is vm.","type":"text"}]},{"type":"paragraph","content":[{"text":"network-driver ","type":"text","marks":[{"type":"code"}]},{"text":" Name of a network driver for providing networks for the containers - this is different and separate from the Neutron network for the cluster. Drivers that Magnum supports:","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"88c428b8-bb85-4569-bc16-23d9b4ef81a7"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"COE","type":"text"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Network Driver","type":"text"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Default","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Kubernetes","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"flannel, calico","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"flannel","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Swarm","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"docker, flannel","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"flannel","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Mesos","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"docker","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"docker","type":"text"}]}]}]}]},{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" For Kubernetes clusters, we are using the ","type":"text"},{"text":"flannel","type":"text","marks":[{"type":"code"}]},{"text":" network driver.","type":"text"}]},{"type":"paragraph","content":[{"text":"dns-nameserver ","type":"text","marks":[{"type":"code"}]},{"text":": The DNS nameserver for the servers and containers in the cluster to use. The default is 8.8.8.8.","type":"text"}]},{"type":"paragraph","content":[{"text":"flavor ","type":"text","marks":[{"type":"code"}]},{"text":": flavor to use for worker nodes. The default is m1.small. Can be overridden at cluster creation.","type":"text"}]},{"type":"paragraph","content":[{"text":"master-flavor ","type":"text","marks":[{"type":"code"}]},{"text":": flavor for master nodes. Default is m1.small. Can be overridden at cluster creation.","type":"text"}]},{"type":"paragraph","content":[{"text":"http-proxy ","type":"text","marks":[{"type":"code"}]},{"text":": The IP address for a proxy to use when direct http access from the servers to sites on the external internet is blocked. The format is a URL including a port number. The default is None.","type":"text"}]},{"type":"paragraph","content":[{"text":"https-proxy ","type":"text","marks":[{"type":"code"}]},{"text":": The IP address for a proxy to use when direct https access from the servers to sites on the external internet is blocked. The format is a URL including a port number. The default is None.","type":"text"}]},{"type":"paragraph","content":[{"text":"no-proxy ","type":"text","marks":[{"type":"code"}]},{"text":": When a proxy server is used, some sites should not go through the proxy and should be accessed normally. In this case, you can specify these sites as a comma separated list of IPs. The default is None.","type":"text"}]},{"type":"paragraph","content":[{"text":"docker-volume-size ","type":"text","marks":[{"type":"code"}]},{"text":": If specified, container images will be stored in a cinder volume of the specified size in GB. Each cluster node will have a volume attached of the above size. If not specified, images will be stored in the compute instances local disk. For the devicemapper storage driver, must specify volume and the minimum value is 3GB. For the overlay and overlay2 storage driver, the minimum value is 1GB or None(no volume). This value can be overridden at cluster creation.","type":"text"}]},{"type":"paragraph","content":[{"text":"docker-storage-driver ","type":"text","marks":[{"type":"code"}]},{"text":": The name of a driver to manage the storage for the images and the containers writable layer. The default is devicemapper.","type":"text"}]},{"type":"paragraph","content":[{"text":"labels ","type":"text","marks":[{"type":"code"}]},{"text":": Arbitrary labels in the form of key=value pairs. The accepted keys and valid values are defined in the cluster drivers. They are used as a way to pass additional parameters that are specific to a cluster driver. The value can be overridden at cluster creation.","type":"text"}]},{"type":"paragraph","content":[{"text":"--tls-disabled","type":"text","marks":[{"type":"code"}]},{"text":" Transport Layer Security (TLS) is normally enabled to secure the cluster. The default is TLS enabled.","type":"text"}]},{"type":"paragraph","content":[{"text":"--registry-enabled","type":"text","marks":[{"type":"code"}]},{"text":" Docker images by default are pulled from the public Docker registry, but in some cases, users may want to use a private registry. This option provides an alternative registry based on the Registry V2: Magnum will create a local registry in the cluster backed by swift to host the images. Refer to Docker Registry 2.0 for more details. The default is to use the public registry.","type":"text"}]},{"type":"paragraph","content":[{"text":"--master-lb-enabled","type":"text","marks":[{"type":"code"}]},{"text":" Since multiple masters may exist in a cluster, a load balancer is created to provide the API endpoint for the cluster and to direct requests to the masters. As we have Octavia enabled, Octavia would create these load balancers. The default is master load balancers are created.","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Create a Cluster","type":"text"}]},{"type":"paragraph","content":[{"text":"We can create clusters using a cluster template from our template list. To create a cluster, we use the command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster create --cluster-template \n --discovery-url \n --master-count \n --node-count \n --timeout \n --merge-labels\n --master-lb-enabled\n #The following options can be used to overwrite the same options in the cluster template\n --docker-volume-size \n --labels \n --keypair \n --master-flavor \n --flavor \n --fixed-network \n --fixed-subnet \n --floating-ip-enabled\n --floating-ip-disabled\n # To add labels to use with the template labels, we can use:\n --merge-labels\n \n","type":"text"}]},{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" It is recommended that to have master load balancers enabled, to use the kubernetes-ha-master-v1_14_3 template, or create a new cluster template and include the flag ","type":"text"},{"text":"--master-lb-enabled","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Labels","type":"text"}]},{"type":"paragraph","content":[{"text":"Labels are used by OpenStack Magnum to define a range of parameters such as the Kubernetes version, enable autoscaling, enable autohealing, version of draino to use etc. Any labels included at cluster creation overwrite the labels in the cluster template. A table containing all of the labels which Magnum uses can be found here:","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://docs.openstack.org/magnum/train/user/"}},{"text":" ","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" For OpenStack Train release, Magnum only offers labels for installing Helm 2 and Tiller. However, Helm 3 can be installed onto the master node after the cluster has been created.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Horizon Web Interface","type":"text"}]},{"type":"paragraph","content":[{"text":"Clusters can also be created using the Horizon Web Interface. Clusters and their templates can be found under the ","type":"text"},{"text":"Container Infra","type":"text","marks":[{"type":"code"}]},{"text":" section.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"There are a few differences between the parameters which can be defined when creating a cluster using the CLI or Horizon Web UI. If you are using the Horizon web UI to create clusters, the ","type":"text"},{"text":"fixed network, fixed subnet, and floating ip enabled","type":"text","marks":[{"type":"strong"}]},{"text":" can only be defined in the cluster template.","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Heat Templates","type":"text"}]},{"type":"paragraph","content":[{"text":"Clusters can also be created using a Heat template using the resources ","type":"text"},{"text":"OS::Magnum::CluterTemplate","type":"text","marks":[{"type":"code"}]},{"text":" and ","type":"text"},{"text":"OS::Magnum::Cluster","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"This will instruct Heat to pass the resources to Magnum, which will pass a stack template to Heat to create a cluster - so two stacks are built in total.","type":"text"}]}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"OS::Magnum::ClusterTemplate","type":"text"}]},{"type":"codeBlock","content":[{"text":"resources:\n cluster_template:\n type: OS::Magnum::ClusterTemplate\n properties:\n #required\n coe: String # Container Orchestration Engine: kubernetes, swarm, mesos\n external_network: String # External neutron network or UUID to attach the cluster\n image: String # The image name/UUID to use as a base image for the cluster\n # optional\n dns_nameserver: String # DNS nameserver address, must be of type ip_addr\n docker_storage_driver: String # Docker storage driver: devicemapper, overlay\n docker_volume_size: Integer # Size in GB of docker volume, must be at least 1\n fixed_network: String # The fixed neutron network name or UUID to attach the Cluster\n fixed_subnet: String # The fixed neutron subnet name or UUID to attach the Cluster\n flavor: String # Flavor name or UUID to use when launching a cluster\n floating_ip_enabled: Boolean # True by default, determines whether a cluster should have floating IPs\n http_proxy: String # http_proxy address to use for nodes in cluster\n https_proxy: String # https_proxy address to use for nodes in cluster\n keypair: String # SSH keypair to load into cluster nodes\n labels: {...} # labels in form of key=value pairs to associate with cluster\n master_flavor: String # flavor name or UUID to associate with the master node\n master_lb_enabled: Boolean # Defaults to true. Determines whether there should be a load balancer for master nodes\n name: String # Template name\n network_driver: String # Name of driver to use for instantiating container networks. Magnum uses pre-configured driver for specific COE by default\n no_proxy: String # A comma separated list of addresses for which proxies should not be used in the cluster\n public: Boolean # Defaults to false. True makes the cluster template public. Must have the permissions to publish templates in Magnum\n registry_enabled: Boolean # Defaults to false. Enable registry in the cluster\n server_type: String # Define server type to use. Defaults to vm. Allowed: vm, bm\n tls_disabled: Boolean # Disable TLS in the Cluster. Defaults to false\n volume_driver: String # Volume driver name for instantiating container volume. Allowed: cinder, rexray\n\noutputs:\n\n detailed-information:\n description: Detailed Information about the resource\n value: {get_attr: [cluster_template, show]}","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"OS::Magnum::Cluster","type":"text"}]},{"type":"codeBlock","content":[{"text":"resources:\n cluster:\n type: OS::Magnum::Cluster\n properties:\n # required\n cluster_template: String # Name or ID of cluster template\n # optional\n create_timeout: Integer # Timeout for creating cluster in minutes. Defaults to 60 minutes\n discovery_url: String # Specifies a custom discovery url for node discovery\n keypair: String # name of keypair. Uses keypair in template if keypair is not defined here\n master_count: Integer # Number of master nodes, defaults to 1. Must be at least 1\n name: String # name of cluster\n node_count: Integer # Number of worker nodes, defaults to 1. Must be at least 1\n\noutputs:\n\n api_address:\n description: Endpoint URL of COE API exposed to end-users\n value: {get_attr: [cluster, api_address]}\n cluster_template_id:\n description: UUID of cluster template\n value: {get_attr: [cluster, cluster_template_id]}\n coe_version:\n description: Version information of container engine in chosen COE in cluster\n value: {get_attr: [cluster, coe_version]}\n create_timeout:\n description: Timeout in minutes for cluster creation\n value: {get_attr: [cluster, create_timeout]}\n discovery_url:\n description: Custom discovery url for node discovery\n value: {get_attr: [cluster, discovery_url]}\n keypair:\n description: Name of keypair\n value: {get_attr: [cluster, keypair]}\n master_addresses:\n description: List of IPs for all master nodes\n value: {get_attrL [cluster, master_addresses]}\n master_count:\n descripition: Number of servers that serve as master for the cluster\n value: {get_attr: [cluster, master_count]}\n name:\n description: Name of the resource\n value: {get_attr:[cluster, name]}\n node_addresses: IP addresses of all worker nodes in the cluster\n description:\n value: {get_attr: [cluster, node_addresses]}\n node_count:\n description: Number of servers that will serve as node for the cluster\n value: {get_attr: [cluster, node_count]}\n show:\n description: Show detailed information about the cluster\n value: {get_attr: [cluster, show]}\n stack_id:\n description: UUID of orchestration stack for this COE cluster\n value: {get_attr: [cluster, stack_id]}\n status:\n description: Status of this COE cluster\n value: {get_attr: cluster, status}\n status_reason:\n description: The reason for the cluster current status\n value: {get_attr: cluster, status_reason}\n","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Example Template","type":"text"}]},{"type":"paragraph","content":[{"text":"For example, we could have the template ","type":"text"},{"text":"example.yaml","type":"text","marks":[{"type":"code"}]},{"text":" which outlines the template for a Kubernetes cluster and instructs heat to create a cluster using this template:","type":"text"}]},{"type":"codeBlock","content":[{"text":"heat_template_version: 2018-08-31 #Train release\n\ndescription: This is an example template to create a Kubernetes cluster.\n\nparameters:\n keypair:\n type: string\n default: mykeypair\n\n image:\n type: string\n default: \n description: fedora-atomic\n\nresources:\n stack_cluster_template:\n type: OS::Magnum::ClusterTemplate\n properties:\n coe: kubernetes\n dns_nameserver: 8.8.8.8\n docker_storage_driver: devicemapper\n docker_volume_size: 10\n external_network: External\n flavor: c1.medium\n floating_ip_enabled: false\n image: {get_param: image}\n labels: {'kube_tag': 'v1.14.3', 'kube_dashboard_enabled': '1', 'heat_container_agent_tag': 'train-stable-3', 'auto_healing': 'true', 'ingress_controller': 'traefik'}\n master_flavor: c1.medium\n name: my-cluster-template\n network_driver: flannel\n registry_enabled: false\n server_type: vm\n volume_driver: cinder\n master_lb_enabled: false\n\n test_cluster:\n type: OS::Magnum::Cluster\n properties:\n cluster_template: {get_resource: stack_cluster_template}\n create_timeout: 60\n keypair: {get_param: keypair}\n name: test-cluster\n node_count: 1\n master_count: 1\n","type":"text"}]},{"type":"paragraph","content":[{"text":"Then we can launch this stack using:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack stack create -t example.yaml test-cluster-stack","type":"text"}]},{"type":"paragraph","content":[{"text":"To delete a cluster created using example.yaml, delete the stack which was built by example.yaml:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack stack delete test-cluster-stack","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Accessing the Cluster","type":"text"}]},{"type":"paragraph","content":[{"text":"To access the cluster, add a floating IP to the master node and ssh using:","type":"text"}]},{"type":"codeBlock","content":[{"text":"#For Fedora Atomic\nssh -i fedora@\n\n#For coreOS\nssh -i core@","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Upgrading Clusters","type":"text"}]},{"type":"paragraph","content":[{"text":"Rolling upgrades can be applied to Kubernetes Clusters using the command ","type":"text"},{"text":"openstack coe cluster upgrade

","type":"text","marks":[{"type":"code"}]},{"text":". This command can be used for upgrading the Kubernetes version or for upgrading the node operating system version.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" Downgrading is not supported","type":"text"}]}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"openstack coe cluster upgrade","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster upgrade --help\nusage: openstack coe cluster upgrade [-h] [--max-batch-size ] [--nodegroup ] cluster_template\n\nUpgrade a Cluster\n\npositional arguments:\n The name or UUID of cluster to update\n cluster_template The new cluster template ID will be upgraded to.\n\noptional arguments:\n -h, --help show this help message and exit\n --max-batch-size \n The max batch size for upgrading each time.\n --nodegroup \n The name or UUID of the nodegroup of current cluster.","type":"text"}]},{"type":"heading","attrs":{"level":4},"content":[{"text":"Example","type":"text"}]},{"type":"paragraph","content":[{"text":"This example will go through how to upgrade an existing cluster to use Kubernetes v1.15.7.","type":"text"}]},{"type":"paragraph","content":[{"text":"The cluster we will update has the following features:","type":"text"}]},{"type":"codeBlock","content":[{"text":"+----------------------+------------------------------------------------------------------------------------------------------------+\n| Field | Value |\n+----------------------+------------------------------------------------------------------------------------------------------------+\n| status | UPDATE_COMPLETE |\n| health_status | UNKNOWN |\n| cluster_template_id | e186c6e2-dd47-4df0-ac3f-3eb46e64cb3d |\n| node_addresses | ['10.0.0.131', '10.0.0.8'] |\n| uuid | 686f9fa1-eb56-4c23-9afd-67a79c283736 |\n| stack_id | 80b6af23-8a14-4a44-bc62-b77d9eb6736b |\n| status_reason | None |\n| created_at | 2020-11-16T12:46:28+00:00 |\n| updated_at | 2020-11-27T11:47:32+00:00 |\n| coe_version | v1.15.7 |\n| labels | {'auto_healing_controller': 'draino', 'max_node_count': '4', 'kube_tag': 'v1.14.3', 'min_node_count': '1', |\n| | 'ingress_controller': 'traefik', 'auto_healing_enabled': 'true', 'heat_container_agent_tag': 'train- |\n| | stable-3', 'auto_scaling_enabled': 'true'} |\n| labels_overridden | |\n| labels_skipped | |\n| labels_added | |\n| fixed_network | None |\n| fixed_subnet | None |\n| floating_ip_enabled | False |\n| faults | |\n| keypair | mykeypair |\n| api_address | https://10.0.0.117:6443 |\n| master_addresses | ['10.0.0.201'] |\n| master_lb_enabled | |\n| create_timeout | 60 |\n| node_count | 2 |\n| discovery_url | https://discovery.etcd.io/6b47ff194fc4dcefb3a7d430d69e761c |\n| master_count | 1 |\n| container_version | 1.12.6 |\n| name | k8s-cluster |\n| master_flavor_id | c1.medium |\n| flavor_id | c1.medium |\n| health_status_reason | {'api': 'The cluster k8s-cluster is not accessible.'} |\n| project_id | PROJECT_ID |\n+----------------------+------------------------------------------------------------------------------------------------------------+","type":"text"}]},{"type":"paragraph","content":[{"text":"To upgrade the Kubernetes version for our cluster, we create a new template where we change the value of the label ","type":"text"},{"text":"kube_tag","type":"text","marks":[{"type":"code"}]},{"text":" from v1.14.3 to v1.15.7","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster template create --coe kubernetes \\\n --image cf37f7d0-1d6b-4aab-a23b-df58542c59cb \\\n --external-network External \\\n --network-driver flannel \\\n --volume-driver cinder \\\n --dns-nameserver 8.8.8.8 \\\n --flavor c1.medium \\\n --master-flavor c1.medium \\\n --docker-volume-size 10 \\\n --docker-storage-driver devicemapper \\\n --labels auto_healing_controller=draino,auto_healing_enabled=true,heat_container_agent_tag=train-stable-3,ingress_controller=traefik \\\n --labels auto_scaling_enabled=true,min_node_count=1,max_node_count=4,kube_tag=1.15.7\n --server_type vm\n update-template","type":"text"}]},{"type":"paragraph","content":[{"text":"Then we apply the cluster upgrade to this cluster:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster upgrade 686f9fa1-eb56-4c23-9afd-67a79c283736 \n# If the command is successful, the following message should be returned:\n\nRequest to upgrade cluster 686f9fa1-eb56-4c23-9afd-67a79c283736 has been accepted.","type":"text"}]},{"type":"paragraph","content":[{"text":"The cluster will then move into ","type":"text"},{"text":"UPDATE_IN_PROGRESS","type":"text","marks":[{"type":"code"}]},{"text":" state while the cluster updates the Kubernetes version. The cluster will move to ","type":"text"},{"text":"UPDATE_COMPLETE","type":"text","marks":[{"type":"code"}]},{"text":" status when the upgrade is complete. We can verify that our cluster is using a different version of Kubernetes by using SSH to connect to the master node and running the following command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ kubectl version\n\nClient Version: version.Info{Major:\"1\", Minor:\"15\", GitVersion:\"v1.15.7\", GitCommit:\"6c143d35bb11d74970e7bc0b6c45b6bfdffc0bd4\", GitTreeState:\"clean\", BuildDate:\"2019-12-11T12:42:56Z\", GoVersion:\"go1.12.12\", Compiler:\"gc\", Platform:\"linux/amd64\"}\nServer Version: version.Info{Major:\"1\", Minor:\"15\", GitVersion:\"v1.15.7\", GitCommit:\"6c143d35bb11d74970e7bc0b6c45b6bfdffc0bd4\", GitTreeState:\"clean\", BuildDate:\"2019-12-11T12:34:17Z\", GoVersion:\"go1.12.12\", Compiler:\"gc\", Platform:\"linux/amd64\"}\n\n$ sudo docker version\n\nClient:\n Version: 1.13.1\n API version: 1.26\n Package version: docker-1.13.1-68.git47e2230.fc29.x86_64\n Go version: go1.11.12\n Git commit: 47e2230/1.13.1\n Built: Sat Aug 17 20:18:33 2019\n OS/Arch: linux/amd64\n\nServer:\n Version: 1.13.1\n API version: 1.26 (minimum version 1.12)\n Package version: docker-1.13.1-68.git47e2230.fc29.x86_64\n Go version: go1.11.12\n Git commit: 47e2230/1.13.1\n Built: Sat Aug 17 20:18:33 2019\n OS/Arch: linux/amd64\n Experimental: false","type":"text"}]},{"type":"paragraph","content":[{"text":"We can see that the Kubernetes and Docker version have been upgraded for our cluster.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Updating Clusters","type":"text"}]},{"type":"paragraph","content":[{"text":"Clusters can be modified using the command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster update [-h] [--rollback] [ ...]\n\nUpdate a Cluster\n\npositional arguments:\n The name or UUID of cluster to update\n Operations: one of 'add', 'replace' or 'remove'\n Attributes to add/replace or remove (only PATH is necessary on remove)\n\noptional arguments:\n -h, --help show this help message and exit\n --rollback Rollback cluster on update failure.","type":"text"}]},{"type":"paragraph","content":[{"text":"The following table summarizes the possible changes that can be applied to the cluster.","type":"text"}]},{"type":"table","attrs":{"layout":"default","localId":"e0b10032-2bc1-4ad9-b8c3-962d7b59498e"},"content":[{"type":"tableRow","content":[{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"Attribute","type":"text"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"add","type":"text"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"replace","type":"text"}]}]},{"type":"tableHeader","attrs":{"colspan":1,"rowspan":1},"content":[{"type":"paragraph","content":[{"text":"remove","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"node_count","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"add/remove nodes","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"reset to default of 1","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"master_count","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"name","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"#f3f6f6","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]}]},{"type":"tableRow","content":[{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"discovery_url","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]},{"type":"tableCell","attrs":{"colspan":1,"background":"transparent","rowspan":1},"content":[{"type":"paragraph","content":[{"text":"no","type":"text"}]}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Resize a Cluster","type":"text"}]},{"type":"paragraph","content":[{"text":"The size of a cluster can be changed by using the following command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster resize [-h] [--nodes-to-remove ] [--nodegroup ] node_count\n\nResize a Cluster\n\npositional arguments:\n The name or UUID of cluster to update\n node_count Desired node count of the cluser.\n\noptional arguments:\n -h, --help show this help message and exit\n --nodes-to-remove \n Server ID of the nodes to be removed. Repeat to addmore server ID\n --nodegroup \n The name or UUID of the nodegroup of current cluster.","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Create a Kubernetes Cluster","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Deprecated","type":"text"}]},{"type":"paragraph","content":[{"text":"Magnum is deprecated and will be replaced in the future with Cluster API.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Clusters are groups of resources (nova instances, neutron networks, security groups etc.) combined to function as one system. To do this, Magnum uses Heat to orchestrate and create a stack which contains the cluster.","type":"text"}]},{"type":"paragraph","content":[{"text":"This documentation will focus on how to create Kubernetes clusters using OpenStack Magnum.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Magnum CLI","type":"text"}]},{"type":"paragraph","content":[{"text":"Any commands for creating clusters using OpenStack Magnum begin with:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe ","type":"text"}]},{"type":"paragraph","content":[{"text":"In order to have the openstack commands for Magnum available to use through the CLI, you will need to install the python client for Magnum. This can be done using pip:","type":"text"}]},{"type":"codeBlock","content":[{"text":"pip install python-magnumclient","type":"text"}]},{"type":"paragraph","content":[{"text":"Now the commands relating to the container orchestration engine, clusters, cluster templates are available on the command line.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Cluster Templates","type":"text"}]},{"type":"paragraph","content":[{"text":"Clusters can be created from templates which are passed through Heat. To view the list of cluster templates which are in your project, you can use the following command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster template list\n\n# This should return either an empty line if there are no templates, or\n# a table similar to the one below:\n+--------------------------------------+------------------------------+\n| uuid | name |\n+--------------------------------------+------------------------------+\n| e186c6e2-dd47-4df0-ac3f-3eb46e64cb3d | kubernetes-v1_14_3 |\n| 0bd1232d-06f2-42ca-b6d5-c27e57f26c3c | kubernetes-ha-master-v1_14_3 |\n| a07903d0-aecf-4f15-a35f-f4fd74060e2f | coreos-kubernetes-v1_14_3 |\n+--------------------------------------+------------------------------+","type":"text"}]},{"type":"paragraph","content":[{"text":"Templates can be created using the following command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster template create [-h] [-f {json,shell,table,value,yaml}] [-c COLUMN] [--noindent] [--prefix PREFIX]\n [--max-width ] [--fit-width] [--print-empty] --coe --image \n --external-network [--keypair ]\n [--fixed-network ] [--fixed-subnet ]\n [--network-driver ] [--volume-driver ]\n [--dns-nameserver ] [--flavor ]\n [--master-flavor ] [--docker-volume-size ]\n [--docker-storage-driver ] [--http-proxy ]\n [--https-proxy ] [--no-proxy ]\n [--labels ] [--tls-disabled] [--public]\n [--registry-enabled] [--server-type ] [--master-lb-enabled]\n [--floating-ip-enabled] [--floating-ip-disabled] [--hidden] [--visible]\n ","type":"text"}]},{"type":"paragraph","content":[{"text":"Kubernetes Cluster Template:","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ openstack coe cluster template show e186c6e2-dd47-4df0-ac3f-3eb46e64cb3d\n+-----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+\n| Field | Value |\n+-----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+\n| insecure_registry | - |\n| labels | {'kube_tag': 'v1.14.3', 'kube_dashboard_enabled': '1', 'heat_container_agent_tag': 'train-stable-3', 'auto_healing': 'true', 'ingress_controller': 'traefik'} |\n| updated_at | - |\n| floating_ip_enabled | False |\n| fixed_subnet | - |\n| master_flavor_id | c1.medium |\n| uuid | e186c6e2-dd47-4df0-ac3f-3eb46e64cb3d |\n| no_proxy | - |\n| https_proxy | - |\n| tls_disabled | False |\n| keypair_id | - |\n| public | True |\n| http_proxy | - |\n| docker_volume_size | 3 |\n| server_type | vm |\n| external_network_id | External |\n| cluster_distro | fedora-atomic |\n| image_id | cf37f7d0-1d6b-4aab-a23b-df58542c59cb |\n| volume_driver | - |\n| registry_enabled | False |\n| docker_storage_driver | devicemapper |\n| apiserver_port | - |\n| name | kubernetes-v1_14_3 |\n| created_at | 2020-09-07T07:17:13+00:00 |\n| network_driver | flannel |\n| fixed_network | - |\n| coe | kubernetes |\n| flavor_id | c1.medium |\n| master_lb_enabled | False |\n| dns_nameserver | 8.8.8.8 |\n| hidden | False |\n+-----------------------+---------------------------------------------------------------------------------------------------------------------------------------------------------------+","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Create a Kubernetes Cluster","type":"text"}]},{"type":"paragraph","content":[{"text":"We can create a Kubernetes cluster using one of the cluster templates that are available. To create a cluster, we use the command:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster create --cluster-template --discovery-url --master-count --node-count \n --timeout --merge-labels\n #The following options can be used to overwrite the same options in the cluster template\n --docker-volume-size \n --labels # --merge-labels flag will add labels to the ones provided by the template\n --keypair \n --master-flavor \n --flavor \n --fixed-network \n --fixed-subnet \n --floating-ip-enabled\n --floating-ip-disabled\n --master-lb-enabled\n ","type":"text"}]},{"type":"paragraph","content":[{"text":"For example, consider a user that wants to create a cluster using the Kubernetes cluster template. They want the cluster to have:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"one master node","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"one worker node","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"their keypair mykeypair","type":"text"}]}]}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster create --cluster-template e186c6e2-dd47-4df0-ac3f-3eb46e64cb3d \\\n --keypair mykeypair \\\n --master-count 1 \\\n --node-count 1 \\\n kubernetes-cluster-test-1\n\n#This should return an output similar to this one\nRequest to create cluster 27cdcad8-375f-4d4f-a186-8fa99b80c5c5 accepted\n#This indicates that the command was successful and the cluster is being built","type":"text"}]},{"type":"paragraph","content":[{"text":"A cluster containing one master node and one worker node takes approximately 14 minutes to build. By default, cluster creation times out at 60 minutes.","type":"text"}]},{"type":"paragraph","content":[{"text":"After the cluster has been created, you can associate a floating IP to the master node and SSH into the node using:","type":"text"}]},{"type":"paragraph","content":[{"text":"` ssh -i fedora@ `","type":"text","marks":[{"type":"code"}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Submitting jobs to a Kubernetes Cluster","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Deprecated","type":"text"}]},{"type":"paragraph","content":[{"text":"Magnum is deprecated and will be replaced in the future with Cluster API.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"A Kubernetes job creates one or more pods on a cluster and have the added benefit of being retried until a specified number of pods successfully terminate. Jobs are described by YAML and can be executed using kubectl.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Submitting jobs","type":"text"}]},{"type":"paragraph","content":[{"text":"Jobs are defined by a YAML config with a kind parameter of Job. Below is an example job config for computing π to 2000 places.","type":"text"}]},{"type":"paragraph","content":[{"text":"job.yaml","type":"text","marks":[{"type":"em"}]}]},{"type":"codeBlock","content":[{"text":"apiVersion: batch/v1\nkind: Job\nmetadata:\n name: pi\n spec:\n template:\n spec:\n containers:\n - name: pi\n #Docker image\n image: perl\n #Compute pi to 2000 places by running this command in the container\n command: [\"perl\", \"-Mbignum=bpi\", \"-wle\", \"print bpi(2000)\"]\n restartPolicy: Never\n #Number of retries to attempt before stopping (default is 6)\n backoffLimit: 4\n","type":"text"}]},{"type":"paragraph","content":[{"text":"To run this job use:","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl apply -f ./job.yaml","type":"text"}]},{"type":"paragraph","content":[{"text":"This will result in the creation of a pod with a single container named pi that is based on the perl image. The specified command, equivalent to ","type":"text"},{"text":"perl -Mbignum=bpi -wle \"print bpi(2000)\"","type":"text","marks":[{"type":"code"}]},{"text":", will then be executed in the container. You can check on the status of the job using","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl describe jobs/pi","type":"text"}]},{"type":"paragraph","content":[{"text":"Then the output from the container can be obtained through","type":"text"}]},{"type":"codeBlock","content":[{"text":"pods=$(kubectl get pods --selector=job-name=pi --output=jsonpath='{.items[*].metadata.name}')\nkubectl logs $pods","type":"text"}]},{"type":"paragraph","content":[{"text":"Important","type":"text","marks":[{"type":"textColor","attrs":{"color":"#ffffff"}}]}]},{"type":"panel","attrs":{"panelIconId":"atlassian-info","panelIcon":":info:","panelColor":"#57D9A3","panelType":"custom"},"content":[{"type":"paragraph","content":[{"text":"Important","type":"text","marks":[{"type":"strong"}]}]},{"type":"paragraph","content":[{"text":"The job and pods it creates are usually kept after execution to allow the logs to be seen, these can be deleted using ","type":"text"},{"text":"kubectl delete jobs/pi","type":"text","marks":[{"type":"code"}]},{"text":" or ","type":"text"},{"text":"kubectl delete -f ./job.yaml","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"One way to avoid a build up of undeleted jobs and pods is to use ttlSecondsAfterFinished which was delcared stable as of Kubernetes v1.23. This defines the time after which a job in a Complete/Failed state should be deleted automatically. For example the following job will be deleted 100 seconds after finishing.","type":"text"}]},{"type":"paragraph","content":[{"text":"apiVersion: batch/v1","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"kind: Job","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"metadata:","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"name: pi","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"spec:","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":" Delete the job 100 seconds after successful completion","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":" ttlSecondsAfterFinished: 100","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":" ","type":"text"},{"text":" template:","type":"text","marks":[{"type":"code"}]},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":" ","type":"text"},{"text":" spec:","type":"text","marks":[{"type":"code"}]},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":" containers:","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":" - name: pi","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":" image: perl","type":"text","marks":[{"type":"code"}]},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":" command: [\"perl\", \"-Mbignum=bpi\", \"-wle\", \"print bpi(2000)\"]","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":" restartPolicy: Never","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Parallel execution","type":"text"}]},{"type":"paragraph","content":[{"text":"The above example runs a single pod until completion or 4 successive failures. It is also possible to execute multiple instances of pods in parallel. For a simple example we can require a fixed completion count by assigning ","type":"text"},{"text":".spec.completions","type":"text","marks":[{"type":"code"}]},{"text":" in the YAML file to require more than one successful execution of a pod is required before the job is considered complete. We can then also specify ","type":"text"},{"text":".spec.parallelism","type":"text","marks":[{"type":"code"}]},{"text":" to increase the number of pods that can be running at any one time. For example, the below will run up to 2 pods in parallel until 8 of them finish successfully.","type":"text"}]},{"type":"codeBlock","content":[{"text":"apiVersion: batch/v1\nkind: Job\nmetadata:\nname: pi\nspec:\n ttlSecondsAfterFinished: 100\n #Require 8 successful completions before the job is considered finished\n completions: 8\n #Allow up to 2 pods to be running in parallel\n parallelism: 2\n template:\n spec:\n containers:\n - name: pi\n image: perl\n command: [\"perl\", \"-Mbignum=bpi\", \"-wle\", \"print bpi(2000)\"]\n restartPolicy: Never","type":"text"}]},{"type":"paragraph","content":[{"text":"If one pod fails a new pod will be created to take its place and the job will continue.","type":"text"}]},{"type":"paragraph","content":[{"text":"You can also use a work queue for parallel jobs by not specifying ","type":"text"},{"text":".spec.completions","type":"text","marks":[{"type":"code"}]},{"text":" at all. In this case the pods should coordinate amongst themselves or via an external service to determine when they have finished as when any one of them successfully exits the job will be considered complete. Therefore each should exit only there is no more work for any of the pods.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Scheduling jobs","type":"text"}]},{"type":"paragraph","content":[{"text":"To run jobs on a schedule you can use CronJobs. These are also described using a YAML file, for example:","type":"text"}]},{"type":"paragraph","content":[{"text":"cronjob.yaml","type":"text","marks":[{"type":"em"}]}]},{"type":"codeBlock","content":[{"text":"apiVersion: batch/v1\nkind: CronJob\nmetadata:\n name: hello\nspec:\n #This described when the job described below should be executed\n schedule: \"* * * * *\"\n jobTemplate:\n spec:\n template:\n spec:\n containers:\n - name: hello\n image: busybox:1.28\n imagePullPolicy: IfNotPresent\n command: [\"/bin/sh\", \"-c\", \"date; echo Hello from the Kubernetes cluster\"]\n restartPolicy: OnFailure","type":"text"}]},{"type":"paragraph","content":[{"text":"This will run a job every minute that prints “Hello from the Kubernetes cluster”. You can create the CronJob using:","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl create -f ./cronjob.yaml","type":"text"}]},{"type":"paragraph","content":[{"text":"You may check on the status of the CronJob using ","type":"text"},{"text":"kubectl get cronjob hello","type":"text","marks":[{"type":"code"}]},{"text":" and watch the jobs it creates in real time using ","type":"text"},{"text":"kubectl get jobs --watch","type":"text","marks":[{"type":"code"}]},{"text":". From the latter you will see that the job names appear as hello- followed by some numbers e.g. hello-27474266. This can be used to view the output of the job using","type":"text"}]},{"type":"codeBlock","content":[{"text":"pods=$(kubectl get pods --selector=job-name=hello-27474266 --output=jsonpath='{.items[*].metadata.name}')\nkubectl logs $pods","type":"text"}]},{"type":"paragraph","content":[{"text":"The schedule parameter, which in this case causes the job to run every minute, is in the following format.","type":"text"}]},{"type":"codeBlock","content":[{"text":"minute (0 - 59), hour (0 - 23), day of the month(1 - 31), month (1 - 12), day of the week (0 - 6)","type":"text"}]},{"type":"paragraph","content":[{"text":"Where the day of the week is 0 for Sunday and 6 for Saturday. In the example the asterisk is used to indicate any. You may find tools such as ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://crontab.guru/"}},{"text":" helpful in writing these. For example ","type":"text"},{"text":"5 4 * * 2","type":"text","marks":[{"type":"code"}]},{"text":" will run at 04:05 on Tuesdays. The time specified is the local time of the machine.","type":"text"}]},{"type":"paragraph","content":[{"text":"You may delete the CronJob, along with any of its existing jobs and pods using","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl delete cronjob hello","type":"text"}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":1},"content":[{"text":"JupyterHub on Kubernetes","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Deprecated","type":"text"}]},{"type":"paragraph","content":[{"text":"Magnum is deprecated and will be replaced in the future with Cluster API.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"This documentation assumes that you will be installing JupyterHub on a Kubernetes cluster that has been created using OpenStack Magnum.","type":"text"}]},{"type":"paragraph","content":[{"text":"In this tutorial we will break the installation down into the following:","type":"text"}]},{"type":"bulletList","content":[{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Create a cluster template and launch a Kubernetes cluster using OpenStack Magnum","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Install Helm v3 and define persistent volume for the cluster","type":"text"}]}]},{"type":"listItem","content":[{"type":"paragraph","content":[{"text":"Install JupyterHub","type":"text"}]}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Creating a Kubernetes Cluster","type":"text"}]},{"type":"paragraph","content":[{"text":"The template for the cluster:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster template create --coe kubernetes \\\n --image cf37f7d0-1d6b-4aab-a23b-df58542c59cb \\\n --external-network External \\\n --network-driver flannel \\\n --volume-driver cinder \\\n --dns-nameserver 8.8.8.8 \\\n --flavor c1.medium \\\n --master-flavor c1.medium \\\n --docker-volume-size 10 \\\n --docker-storage-driver devicemapper \\\n --labels kube_tag=v1.14.3,kube_dashboard_enabled=1,heat_container_agent_tag=train-stable-3,auto_healing=true,ingress_controller=traefik\n --server_type vm\n test-template","type":"text"}]},{"type":"paragraph","content":[{"text":"Create a cluster:","type":"text"}]},{"type":"codeBlock","content":[{"text":"openstack coe cluster create --cluster-template test-template \\\n --keypair mykeypair \\\n --docker-volume-size 10 \\\n --master-count 1 \\\n --node-count 1 \\\n test-cluster\n\n #This should return an output similar to this one\n Request to create cluster 27cdcad8-375f-4d4f-a186-8fa99b80c5c5 accepted\n #This indicates that the command was successful and the cluster is being built","type":"text"}]},{"type":"paragraph","content":[{"text":"Once the cluster has been created successfully, we can associate a floating IP to the master node VM and then SSH into the cluster:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ssh -i mykeypair.key fedroa@FLOATING_IP\n\n#This should return something similar to:\n\nLast login: Fri Sep 18 13:17:02 2020 from 130.XXX.XXX.XXX\n\n[fedora@test-template-vbo5u2doyiao-master-0 ~]$\n\n#You have now successfully connected to the master node","type":"text"}]},{"type":"heading","attrs":{"level":3},"content":[{"text":"Configure Storage","type":"text"}]},{"type":"paragraph","content":[{"text":"Magnum does not automatically configure cinder storage for clusters.","type":"text"}]},{"type":"paragraph","content":[{"text":"The storage class can be defined using a YAML file. For example we could define the storage class to be:","type":"text"}]},{"type":"paragraph","content":[{"text":"YAML File from: ","type":"text","marks":[{"type":"strong"}]},{"text":"https://github.com/zonca/jupyterhub-deploy-kubernetes-jetstream/blob/master/kubernetes_magnum/storageclass.yaml","type":"text","marks":[{"type":"link","attrs":{"href":"https://github.com/zonca/jupyterhub-deploy-kubernetes-jetstream/blob/master/kubernetes_magnum/storageclass.yaml"}}]}]},{"type":"codeBlock","content":[{"text":"apiVersion: storage.k8s.io/v1\n\nkind: StorageClass\n\nmetadata:\n\n name: standard\n\n annotations:\n\n storageclass.beta.kubernetes.io/is-default-class: \"true\"\n\n labels:\n\n kubernetes.io/cluster-service: \"true\"\n\n addonmanager.kubernetes.io/mode: EnsureExists\n\nprovisioner: kubernetes.io/cinder","type":"text"}]},{"type":"paragraph","content":[{"text":"Then we create the storage class:","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl create -f storageclass.yaml","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Helm v3","type":"text"}]},{"type":"paragraph","content":[{"text":"The Train release supports Helm v2 charts being installed and supports labels for installing Tiller.","type":"text"}]},{"type":"paragraph","content":[{"text":"However, it is possible to install and run charts for Helm v3.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"In the Ussuri release onwards, Magnum supports the use of a label to install Helm v3 client. This label can be added to a template or at cluster creation time.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" Helm v2 reaches end of support in November 2020","type":"text"}]},{"type":"paragraph","content":[{"text":"To install Helm 3:","type":"text"}]},{"type":"codeBlock","content":[{"text":"curl https://raw.githubusercontent.com/helm/helm/master/scripts/get-helm-3 | bash","type":"text"}]},{"type":"paragraph","content":[{"text":"Other methods for installing Helm v3 can be found here: ","type":"text"},{"type":"inlineCard","attrs":{"url":"https://helm.sh/docs/intro/install/"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"Now Helm v3 has been installed, we can install JupyterHub.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"JupyterHub","type":"text"}]},{"type":"paragraph","content":[{"text":"The following is the tutorial from the ","type":"text"},{"text":"_Zero to JupyterHub with Kubernetes_","type":"text","marks":[{"type":"strong"}]},{"text":" installation documentation.","type":"text"}]},{"type":"codeBlock","content":[{"text":"# Generate a random hex string\nopenssl rand -hex 32 #copy the output","type":"text"}]},{"type":"paragraph","content":[{"text":"Then create a file called config.yaml and write the following:","type":"text"}]},{"type":"codeBlock","content":[{"text":"vi config.yaml # fedora doesn't use nano","type":"text"}]},{"type":"codeBlock","content":[{"text":"proxy:\n secretToken: \"\" #this is the random string which you have copied","type":"text"}]},{"type":"paragraph","content":[{"text":"Next is to add the JupyterHub Helm chart to your chart repository and install it.","type":"text"}]},{"type":"codeBlock","content":[{"text":"helm repo add jupyterhub https://jupyterhub.github.io/helm-chart/\nhelm repo update\n\nRELEASE=jhub\nNAMESPACE=jhub\n\nhelm upgrade --cleanup-on-fail \\\n --install $RELEASE jupyterhub/jupyterhub \\\n --namespace $NAMESPACE \\\n --create-namespace \\\n --version=0.9.0 \\\n --values config.yaml \\\n --timeout 30m0s #This is to stop the installation from timing out","type":"text"}]},{"type":"paragraph","content":[{"text":"When installation is complete it should return a message similar to the following:","type":"text"}]},{"type":"codeBlock","content":[{"text":"NAME: jhub\nLAST DEPLOYED: Tue Oct 13 11:01:15 2020\nNAMESPACE: jhub\nSTATUS: deployed\nREVISION: 1\nTEST SUITE: None\nNOTES:\nThank you for installing JupyterHub!\n\nYour release is named jhub and installed into the namespace jhub.\n\nYou can find if the hub and proxy is ready by doing:\n\n kubectl --namespace=jhub get pod\n\nand watching for both those pods to be in status 'Running'.\n\nYou can find the public IP of the JupyterHub by doing:\n\n kubectl --namespace=jhub get svc proxy-public\n\nIt might take a few minutes for it to appear!\n\nNote that this is still an alpha release! If you have questions, feel free to\n 1. Read the guide at https://z2jh.jupyter.org\n 2. Chat with us at https://gitter.im/jupyterhub/jupyterhub\n 3. File issues at https://github.com/jupyterhub/zero-to-jupyterhub-k8s/issues\n","type":"text"}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"Autoscaling Clusters","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Deprecated","type":"text"}]},{"type":"paragraph","content":[{"text":"Magnum is deprecated and will be replaced in the future with Cluster API.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"The Cluster Autoscaler (CA) is a feature in OpenStack Magnum that can be enabled in order for the cluster to scale up or down the worker nodegroup. The default version which the Train release uses is v1.0. The version of CA to use can be changed at cluster creation by using the label ","type":"text"},{"text":"autoscaler_tag","type":"text","marks":[{"type":"code"}]}]},{"type":"paragraph","content":[{"text":"This feature can be enabled by using the label ","type":"text"},{"text":"auto_scaling_enabled=true","type":"text","marks":[{"type":"code"}]},{"text":" in a cluster template or at cluster creation.","type":"text"}]},{"type":"panel","attrs":{"panelType":"info"},"content":[{"type":"paragraph","content":[{"text":"Note:","type":"text","marks":[{"type":"strong"}]},{"text":" The CA will start attempting to scale down nodes as long as the machine IDs for each node matches their VM ID.","type":"text"}]}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Machine IDs","type":"text"}]},{"type":"paragraph","content":[{"text":"On nodes in a Kubernetes cluster, the system UUID matches the ID of the VM hosting that node. However, the Cluster Autoscaler uses the machine ID to refer to the node when the cluster needs to be scaled down and a node removed. Kubernetes reads the ID for the VMs from the file ","type":"text"},{"text":"/etc/machine-id","type":"text","marks":[{"type":"code"}]},{"text":" on each VM in the cluster. However, these IDs may not match the IDs of the VMs. If the machine ID and system UUID (VM ID) on a node do not match, then the following errors may be present in the CA pod’s log:","type":"text"}]},{"type":"codeBlock","content":[{"text":"E1215 12:59:58.084833 1 scale_down.go:932] Problem with empty node deletion: failed to delete cluster-node-5: manager error deleting nodes: could not find stack indices for nodes to be deleted: 1 nodes could not be resolved to stack indices\nE1215 12:59:58.084958 1 static_autoscaler.go:430] Failed to scale down: failed to delete at least one empty node: failed to delete cluster-node-5: manager error deleting nodes: could not find stack indices for nodes to be deleted: 1 nodes could not be resolved to stack indices\nI1215 13:10:09.265757 1 scale_down.go:882] Scale-down: removing empty node cluster-node-5\nI1215 13:10:18.362187 1 magnum_manager_heat.go:347] Could not resolve node {Name:cluster-node-5 MachineID:d6580d63b98346daacd54c644f76bbd6 ProviderID:openstack:///d07e9c8f-e7dd-4342-9ba6-f5c912afc04e IPs:[10.0.0.8]} to a stack index","type":"text"}]},{"type":"paragraph","content":[{"text":"To update the machine ID to match the VM ID, the file can be edited directly using:","type":"text"}]},{"type":"codeBlock","content":[{"text":"vi /etc/machine-id\n\n#Then replace the machine ID with the VM's ID","type":"text"}]},{"type":"paragraph","content":[{"text":"After a few minutes Kubernetes will have updated the IDs for the nodes on those VMs. The system UUID and the machine ID can be seen using ","type":"text"},{"text":"kubectl describe node ","type":"text","marks":[{"type":"code"}]},{"text":".","type":"text"}]},{"type":"paragraph","content":[{"text":"For example:","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ kubectl describe node cluster-node-3\nName: cluster-node-3\nRoles: \nLabels: beta.kubernetes.io/arch=amd64\n beta.kubernetes.io/instance-type=8\n beta.kubernetes.io/os=linux\n draino-enabled=true\n failure-domain.beta.kubernetes.io/region=RegionOne\n failure-domain.beta.kubernetes.io/zone=ceph\n kubernetes.io/arch=amd64\n kubernetes.io/hostname=cluster-node-3\n kubernetes.io/os=linux\n magnum.openstack.org/nodegroup=default-worker\n magnum.openstack.org/role=worker\nAnnotations: flannel.alpha.coreos.com/backend-data: {\"VtepMAC\":\"7e:d2:52:53:d3:8c\"}\n flannel.alpha.coreos.com/backend-type: vxlan\n flannel.alpha.coreos.com/kube-subnet-manager: true\n flannel.alpha.coreos.com/public-ip: 10.0.0.131\n node.alpha.kubernetes.io/ttl: 0\n volumes.kubernetes.io/controller-managed-attach-detach: true\nCreationTimestamp: Mon, 16 Nov 2020 15:45:32 +0000\nTaints: \nUnschedulable: false\nConditions:\n Type Status LastHeartbeatTime LastTransitionTime Reason Message\n ---- ------ ----------------- ------------------ ------ -------\n KernelDeadlock False Tue, 15 Dec 2020 13:06:50 +0000 Fri, 27 Nov 2020 11:45:15 +0000 KernelHasNoDeadlock kernel has no deadlock\n ReadonlyFilesystem False Tue, 15 Dec 2020 13:06:50 +0000 Fri, 27 Nov 2020 11:45:15 +0000 FilesystemIsNotReadOnly Filesystem is not read-only\n MemoryPressure False Tue, 15 Dec 2020 13:07:32 +0000 Mon, 16 Nov 2020 15:45:32 +0000 KubeletHasSufficientMemory kubelet has sufficient memory available\n DiskPressure False Tue, 15 Dec 2020 13:07:32 +0000 Mon, 16 Nov 2020 15:45:32 +0000 KubeletHasNoDiskPressure kubelet has no disk pressure\n PIDPressure False Tue, 15 Dec 2020 13:07:32 +0000 Mon, 16 Nov 2020 15:45:32 +0000 KubeletHasSufficientPID kubelet has sufficient PID available\n Ready True Tue, 15 Dec 2020 13:07:32 +0000 Fri, 27 Nov 2020 11:44:41 +0000 KubeletReady kubelet is posting ready status\nAddresses:\n InternalIP: 10.0.0.131\nCapacity:\n cpu: 2\n ephemeral-storage: 39922Mi\n hugepages-1Gi: 0\n hugepages-2Mi: 0\n memory: 4030348Ki\n pods: 110\nAllocatable:\n cpu: 2\n ephemeral-storage: 37675125903\n hugepages-1Gi: 0\n hugepages-2Mi: 0\n memory: 3927948Ki\n pods: 110\nSystem Info:\n Machine ID: 54d6093e-0b15-4c92-80f7-5f3126e06083\n System UUID: 54d6093e-0b15-4c92-80f7-5f3126e06083\n Boot ID: dddb9b89-559c-4f3f-8b1f-6b6f0d5a62dd\n ..................","type":"text"}]},{"type":"paragraph","content":[{"text":"This shows that this node had the machine ID updated so that it now matches the System UUID and will refer to the VM by the correct ID if the Cluster AutoScaler attempts to remove the node when scaling the cluster.","type":"text"}]},{"type":"paragraph","content":[{"text":"The Cluster Autoscaler will begin to successfully scale down nodes once machine IDs match VM IDs. To prevent a node being scaled down, the following annotation needs to be added to the node:","type":"text"}]},{"type":"codeBlock","content":[{"text":"kubectl annotate node cluster-autoscaler.kubernetes.io/scale-down-disabled=true","type":"text"}]},{"type":"paragraph","content":[{"text":"This will indicate to CA that this node cannot be removed from the cluster when scaling down.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Cluster Autoscaler Deployment","type":"text"}]},{"type":"paragraph","content":[{"text":"The deployment of the CA on the cluster will be similar to the following:","type":"text"}]},{"type":"codeBlock","content":[{"text":"$ kubectl describe deployment cluster-autoscaler -n kube-system\nName: cluster-autoscaler\nNamespace: kube-system\nCreationTimestamp: Mon, 16 Nov 2020 12:55:53 +0000\nLabels: app=cluster-autoscaler\nAnnotations: deployment.kubernetes.io/revision: 1\n kubectl.kubernetes.io/last-applied-configuration:\n {\"apiVersion\":\"apps/v1\",\"kind\":\"Deployment\",\"metadata\":{\"annotations\":{},\"labels\":{\"app\":\"cluster-autoscaler\"},\"name\":\"cluster-autoscaler\"...\nSelector: app=cluster-autoscaler\nReplicas: 1 desired | 1 updated | 1 total | 1 available | 0 unavailable\nStrategyType: RollingUpdate\nMinReadySeconds: 0\nRollingUpdateStrategy: 25% max unavailable, 25% max surge\nPod Template:\n Labels: app=cluster-autoscaler\n Service Account: cluster-autoscaler-account\n Containers:\n cluster-autoscaler:\n Image: docker.io/openstackmagnum/cluster-autoscaler:v1.0\n Port: \n Host Port: \n Command:\n ./cluster-autoscaler\n --alsologtostderr\n --cloud-provider=magnum\n --cluster-name=686f9fa1-eb56-4c23-9afd-67a79c283736\n --cloud-config=/config/cloud-config\n --nodes=1:4:default-worker\n --scale-down-unneeded-time=10m\n --scale-down-delay-after-failure=3m\n --scale-down-delay-after-add=10m\n Environment: \n Mounts:\n /config from cloud-config (ro)\n /etc/kubernetes from ca-bundle (ro)\n Volumes:\n ca-bundle:\n Type: Secret (a volume populated by a Secret)\n SecretName: ca-bundle\n Optional: false\n cloud-config:\n Type: Secret (a volume populated by a Secret)\n SecretName: cluster-autoscaler-cloud-config\n Optional: false\nConditions:\n Type Status Reason\n ---- ------ ------\n Progressing True NewReplicaSetAvailable\n Available True MinimumReplicasAvailable\nOldReplicaSets: cluster-autoscaler-8669c48d54 (1/1 replicas created)\nNewReplicaSet: \nEvents: \n","type":"text"}]},{"type":"paragraph","content":[{"text":"We can see in the ","type":"text"},{"text":"Command","type":"text","marks":[{"type":"code"}]},{"text":" can change the time the autoscaler waits before determining that a node is unneeded and should be scaled down. We can also change the delay time between adding nodes during scaling up and the amount of time to wait after scaling down fails.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"Example: A Cluster Scaling Up","type":"text"}]},{"type":"paragraph","content":[{"text":"Let’s have a cluster that has CA enabled and consists of one master node and one node. If the worker node is cordoned and nginx pods still need to be scheduled, the CA will send an OpenStack request to resize the cluster and increase the node count from 1 to 2 in order to have a node available to schedule a node. This can be seen in the container or pod logs for the CA:","type":"text"}]},{"type":"codeBlock","content":[{"text":"2020-11-16T11:00:13.721916753Z I1116 11:00:13.721164 1 scale_up.go:689] Scale-up: setting group default-worker size to 2\n2020-11-16T11:00:21.441786855Z I1116 11:00:21.441504 1 magnum_nodegroup.go:101] Increasing size by 1, 1->2\n2020-11-16T11:00:59.763966729Z I1116 11:00:59.763422 1 magnum_nodegroup.go:67] Waited for cluster UPDATE_IN_PROGRESS status","type":"text"}]},{"type":"paragraph","content":[{"text":"You should see the stack for the cluster being updated on OpenStack and see the node visible in the cluster:","type":"text"}]},{"type":"codeBlock","content":[{"text":"ssh -i fedora@\n\nkubectl get nodes\n\nNAME STATUS ROLES AGE VERSION\ncluster-test-master-0 Ready master 2d20h v1.14.3\ncluster-test-node-1 Ready,SchedulingDisabled 2d20h v1.14.3\ncluster-test-node-2 NotReady 0s v1.14.3\n\n#Here we can see that the new node has been spun up and it being set up\n\n#After the node has been configured it reports that it is ready\n\nkubectl get nodes\n\nNAME STATUS ROLES AGE VERSION\ncluster-test-master-0 Ready master 2d20h v1.14.3\ncluster-test-node-1 Ready,SchedulingDisabled 2d20h v1.14.3\ncluster-test-node-2 Ready 0s v1.14.3","type":"text"}]},{"type":"paragraph"},{"type":"heading","attrs":{"level":1},"content":[{"text":"OpenStack Magnum Users","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"Deprecated","type":"text"}]},{"type":"paragraph","content":[{"text":"Magnum is deprecated and will be replaced in the future with Cluster API.","type":"text"}]}]},{"type":"paragraph","content":[{"text":"When a cluster is created, Magnum creates unique credentials for each cluster. This allows the cluster to make changes to its structure (e.g. create load balancers for specific services, create and attach cinder volumes, update the stack, etc.) without exposing the user’s cloud credentials.","type":"text"}]},{"type":"heading","attrs":{"level":2},"content":[{"text":"How to find the Magnum User Credentials","type":"text"}]},{"type":"paragraph","content":[{"text":"We can obtain the cluster credentials directly from the VM which the master node is on. First, SSH into the master node’s VM and then:","type":"text"}]},{"type":"codeBlock","content":[{"text":"[fedora@cluster-master-0 ~]$ cd /etc\n[fedora@cluster-master-0 /etc]$ cd kubernetes/\n[fedora@cluster-master-0 kubernetes]$ ls\n#This will return the list of items similar to:\n\napiserver cloud-config controller-manager kube_openstack_config manifests scheduler\nca-bundle.crt cloud-config-occm get_require_kubeconfig.sh kubelet proxy\ncerts config keystone_webhook_config.yaml kubelet-config.yaml proxy-kubeconfig.yaml\n\n#The cluster's credentials can be found in the file 'cloud-config'\n#Print the config to the terminal\n[fedora@cluster-master-0 kubernetes]$ cat cloud-config","type":"text"}]},{"type":"paragraph","content":[{"text":"This will return the cloud-config file containing the cluster’s credentials similar to:","type":"text"}]},{"type":"codeBlock","content":[{"text":"[Global]\nauth-url=https://AUTH-URL\nuser-id=CLUSTER_USER_ID\npassword=PASSWORD\ntrust-id=TRUST-ID\nca-file=/etc/kubernetes/ca-bundle.crt\nregion=RegionOne\n[LoadBalancer] #with the octavia ingress controller enabled\nuse-octavia=True\nsubnet-id=SUBNET_ID\nfloating-network-id=FLOATING_NETWORK_ID\ncreate-monitor=yes\nmonitor-delay=1m\nmonitor-timeout=30s\nmonitor-max-retries=3\n[BlockStorage]\nbs-version=v2","type":"text"}]},{"type":"panel","attrs":{"panelType":"warning"},"content":[{"type":"paragraph","content":[{"text":"These global variables should be used when setting up configmaps such as magnum-auto-healer configmap. ","type":"text"},{"text":"Never","type":"text","marks":[{"type":"strong"}]},{"text":" use your own cloud credentials in Kubernetes configmaps. These would be visible to anyone who has access to the master node in the cluster.","type":"text"}]}]},{"type":"heading","attrs":{"level":1},"content":[{"text":"References:","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://docs.openstack.org/magnum/train/user/"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://docs.openstack.org/heat/train/template_guide/openstack.html"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://www.openstack.org/videos/summits/austin-2016/intro-to-openstack-magnum-with-kubernetes"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"text":"https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-assets-prod/presentation-media/openstack-magnum-hands-on.pdf","type":"text","marks":[{"type":"link","attrs":{"href":"https://object-storage-ca-ymq-1.vexxhost.net/swift/v1/6e4619c416ff4bd19e1c087f27a43eea/www-assets-prod/presentation-media/openstack-magnum-hands-on.pdf"}}]}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://www.openstack.org/videos/summits/denver-2019/container-use-cases-and-developments-at-the-cern-cloud"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://clouddocs.web.cern.ch/containers/quickstart.html"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://kubernetes.io/docs/concepts/workloads/controllers/job/"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/"}}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://github.com/zonca/jupyterhub-deploy-kubernetes-jetstream"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://www.zonca.dev/posts/2020-05-21-jetstream_kubernetes_magnum.html"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://zero-to-jupyterhub.readthedocs.io/en/latest/"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://helm.sh/docs/intro/install/"}},{"text":" ","type":"text"}]},{"type":"paragraph","content":[{"type":"inlineCard","attrs":{"url":"https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/magnum"}},{"text":" ","type":"text"}]},{"type":"paragraph"}],"version":1}

Browser not supported