Container Service for Kubernetes:Containerize and deploy Slurm on an ACK cluster

Deploy the LDAP backend

1. Create a file named ldap.yaml and copy the following content to the file. This creates a basic LDAP instance that stores and manages user information. The ldap.yaml file defines an LDAP backend pod and its associated Service. The pod contains an LDAP container, and the Service exposes the LDAP service within the network.

   View the LDAP backend pod and its associated Service

      ---
      apiVersion: apps/v1
      kind: Deployment
      metadata:
        namespace: default
        name: ldap
        labels:
          app: ldap
      spec:
        selector:
          matchLabels:
            app: ldap
        revisionHistoryLimit: 10
        template:
          metadata:
            labels:
              app: ldap
          spec:
            securityContext:
              seLinuxOptions: {}
            imagePullSecrets: []
            restartPolicy: Always
            initContainers: []
            containers:
              - image: 'osixia/openldap:1.4.0'
                imagePullPolicy: IfNotPresent
                name: ldap
                volumeMounts:
                  - name: openldap-data
                    mountPath: /var/lib/ldap
                    subPath: data
                  - name: openldap-data
                    mountPath: /etc/ldap/slapd.d
                    subPath: config
                  - name: openldap-data
                    mountPath: /container/service/slapd/assets/certs
                    subPath: certs
                  - name: secret-volume
                    mountPath: /container/environment/01-custom
                  - name: container-run
                    mountPath: /container/run
                args:
                  - '--copy-service'
                resources:
                  limits:
                  requests:
                env: []
                readinessProbe:
                  tcpSocket:
                    port: openldap
                  initialDelaySeconds: 20
                  timeoutSeconds: 1
                  periodSeconds: 10
                  successThreshold: 1
                  failureThreshold: 10
                livenessProbe:
                  tcpSocket:
                    port: openldap
                  initialDelaySeconds: 20
                  timeoutSeconds: 1
                  periodSeconds: 10
                  successThreshold: 1
                  failureThreshold: 10
                lifecycle: {}
                ports:
                  - name: openldap
                    containerPort: 389
                    protocol: TCP
                  - name: ssl-ldap-port
                    containerPort: 636
                    protocol: TCP
            volumes:
              - name: openldap-data
                emptyDir: {}
              - name: secret-volume
                secret:
                  secretName: ldap-secret
                  defaultMode: 420
                  items: []
              - name: container-run
                emptyDir: {}
            dnsPolicy: ClusterFirst
            dnsConfig: {}
            terminationGracePeriodSeconds: 30
        progressDeadlineSeconds: 600
        strategy:
          type: RollingUpdate
          rollingUpdate:
            maxUnavailable: 25%
            maxSurge: 25%
        replicas: 1
      ---
      apiVersion: v1
      kind: Service
      metadata:
        annotations: {}
        labels:
          app: ldap
        name: ldap-service
        namespace: default
      spec:
        ports:
          - name: openldap
            port: 389
            protocol: TCP
            targetPort: openldap
          - name: ssl-ldap-port
            port: 636
            protocol: TCP
            targetPort: ssl-ldap-port
        selector:
          app: ldap
        sessionAffinity: None
        type: ClusterIP
      ---
      metadata:
        name: ldap-secret
        namespace: default
        annotations: {}
      data:
        env.startup.yaml: >-
          IyBUaGlzIGlzIHRoZSBkZWZhdWx0IGltYWdlIHN0YXJ0dXAgY29uZmlndXJhdGlvbiBmaWxlCiMgdGhpcyBmaWxlIGRlZmluZSBlbnZpcm9ubWVudCB2YXJpYWJsZXMgdXNlZCBkdXJpbmcgdGhlIGNvbnRhaW5lciAqKmZpcnN0IHN0YXJ0KiogaW4gKipzdGFydHVwIGZpbGVzKiouCgojIFRoaXMgZmlsZSBpcyBkZWxldGVkIHJpZ2h0IGFmdGVyIHN0YXJ0dXAgZmlsZXMgYXJlIHByb2Nlc3NlZCBmb3IgdGhlIGZpcnN0IHRpbWUsCiMgYWZ0ZXIgdGhhdCBhbGwgdGhlc2UgdmFsdWVzIHdpbGwgbm90IGJlIGF2YWlsYWJsZSBpbiB0aGUgY29udGFpbmVyIGVudmlyb25tZW50LgojIFRoaXMgaGVscHMgdG8ga2VlcCB5b3VyIGNvbnRhaW5lciBjb25maWd1cmF0aW9uIHNlY3JldC4KIyBtb3JlIGluZm9ybWF0aW9uIDogaHR0cHM6Ly9naXRodWIuY29tL29zaXhpYS9kb2NrZXItbGlnaHQtYmFzZWltYWdlCgojIFJlcXVpcmVkIGFuZCB1c2VkIGZvciBuZXcgbGRhcCBzZXJ2ZXIgb25seQpMREFQX09SR0FOSVNBVElPTjogRXhhbXBsZSBJbmMuCkxEQVBfRE9NQUlOOiBleGFtcGxlLm9yZwpMREFQX0JBU0VfRE46ICNpZiBlbXB0eSBhdXRvbWF0aWNhbGx5IHNldCBmcm9tIExEQVBfRE9NQUlOCgpMREFQX0FETUlOX1BBU1NXT1JEOiBhZG1pbgpMREFQX0NPTkZJR19QQVNTV09SRDogY29uZmlnCgpMREFQX1JFQURPTkxZX1VTRVI6IGZhbHNlCkxEQVBfUkVBRE9OTFlfVVNFUl9VU0VSTkFNRTogcmVhZG9ubHkKTERBUF9SRUFET05MWV9VU0VSX1BBU1NXT1JEOiByZWFkb25seQoKIyBCYWNrZW5kCkxEQVBfQkFDS0VORDogaGRiCgojIFRscwpMREFQX1RMUzogdHJ1ZQpMREFQX1RMU19DUlRfRklMRU5BTUU6IGxkYXAuY3J0CkxEQVBfVExTX0tFWV9GSUxFTkFNRTogbGRhcC5rZXkKTERBUF9UTFNfQ0FfQ1JUX0ZJTEVOQU1FOiBjYS5jcnQKCkxEQVBfVExTX0VORk9SQ0U6IGZhbHNlCkxEQVBfVExTX0NJUEhFUl9TVUlURTogU0VDVVJFMjU2Oi1WRVJTLVNTTDMuMApMREFQX1RMU19QUk9UT0NPTF9NSU46IDMuMQpMREFQX1RMU19WRVJJRllfQ0xJRU5UOiBkZW1hbmQKCiMgUmVwbGljYXRpb24KTERBUF9SRVBMSUNBVElPTjogZmFsc2UKIyB2YXJpYWJsZXMgJExEQVBfQkFTRV9ETiwgJExEQVBfQURNSU5fUEFTU1dPUkQsICRMREFQX0NPTkZJR19QQVNTV09SRAojIGFyZSBhdXRvbWF0aWNhbHkgcmVwbGFjZWQgYXQgcnVuIHRpbWUKCiMgaWYgeW91IHdhbnQgdG8gYWRkIHJlcGxpY2F0aW9uIHRvIGFuIGV4aXN0aW5nIGxkYXAKIyBhZGFwdCBMREFQX1JFUExJQ0FUSU9OX0NPTkZJR19TWU5DUFJPViBhbmQgTERBUF9SRVBMSUNBVElPTl9EQl9TWU5DUFJPViB0byB5b3VyIGNvbmZpZ3VyYXRpb24KIyBhdm9pZCB1c2luZyAkTERBUF9CQVNFX0ROLCAkTERBUF9BRE1JTl9QQVNTV09SRCBhbmQgJExEQVBfQ09ORklHX1BBU1NXT1JEIHZhcmlhYmxlcwpMREFQX1JFUExJQ0FUSU9OX0NPTkZJR19TWU5DUFJPVjogYmluZGRuPSJjbj1hZG1pbixjbj1jb25maWciIGJpbmRtZXRob2Q9c2ltcGxlIGNyZWRlbnRpYWxzPSRMREFQX0NPTkZJR19QQVNTV09SRCBzZWFyY2hiYXNlPSJjbj1jb25maWciIHR5cGU9cmVmcmVzaEFuZFBlcnNpc3QgcmV0cnk9IjYwICsiIHRpbWVvdXQ9MSBzdGFydHRscz1jcml0aWNhbApMREFQX1JFUExJQ0FUSU9OX0RCX1NZTkNQUk9WOiBiaW5kZG49ImNuPWFkbWluLCRMREFQX0JBU0VfRE4iIGJpbmRtZXRob2Q9c2ltcGxlIGNyZWRlbnRpYWxzPSRMREFQX0FETUlOX1BBU1NXT1JEIHNlYXJjaGJhc2U9IiRMREFQX0JBU0VfRE4iIHR5cGU9cmVmcmVzaEFuZFBlcnNpc3QgaW50ZXJ2YWw9MDA6MDA6MDA6MTAgcmV0cnk9IjYwICsiIHRpbWVvdXQ9MSBzdGFydHRscz1jcml0aWNhbApMREFQX1JFUExJQ0FUSU9OX0hPU1RTOgogIC0gbGRhcDovL2xkYXAuZXhhbXBsZS5vcmcgIyBUaGUgb3JkZXIgbXVzdCBiZSB0aGUgc2FtZSBvbiBhbGwgbGRhcCBzZXJ2ZXJzCiAgLSBsZGFwOi8vbGRhcDIuZXhhbXBsZS5vcmcKCgojIFJlbW92ZSBjb25maWcgYWZ0ZXIgc2V0dXAKTERBUF9SRU1PVkVfQ09ORklHX0FGVEVSX1NFVFVQOiB0cnVlCgojIGNmc3NsIGVudmlyb25tZW50IHZhcmlhYmxlcyBwcmVmaXgKTERBUF9DRlNTTF9QUkVGSVg6IGxkYXAgIyBjZnNzbC1oZWxwZXIgZmlyc3Qgc2VhcmNoIGNvbmZpZyBmcm9tIExEQVBfQ0ZTU0xfKiB2YXJpYWJsZXMsIGJlZm9yZSBDRlNTTF8qIHZhcmlhYmxlcy4K
        env.yaml: >-
          IyBUaGlzIGlzIHRoZSBkZWZhdWx0IGltYWdlIGNvbmZpZ3VyYXRpb24gZmlsZQojIFRoZXNlIHZhbHVlcyB3aWxsIHBlcnNpc3RzIGluIGNvbnRhaW5lciBlbnZpcm9ubWVudC4KCiPCoEFsbCBlbnZpcm9ubWVudCB2YXJpYWJsZXMgdXNlZCBhZnRlciB0aGUgY29udGFpbmVyIGZpcnN0IHN0YXJ0CiMgbXVzdCBiZSBkZWZpbmVkIGhlcmUuCiMgbW9yZSBpbmZvcm1hdGlvbiA6IGh0dHBzOi8vZ2l0aHViLmNvbS9vc2l4aWEvZG9ja2VyLWxpZ2h0LWJhc2VpbWFnZQoKIyBHZW5lcmFsIGNvbnRhaW5lciBjb25maWd1cmF0aW9uCiMgc2VlIHRhYmxlIDUuMSBpbiBodHRwOi8vd3d3Lm9wZW5sZGFwLm9yZy9kb2MvYWRtaW4yNC9zbGFwZGNvbmYyLmh0bWwgZm9yIHRoZSBhdmFpbGFibGUgbG9nIGxldmVscy4KTERBUF9MT0dfTEVWRUw6IDI1Ngo=
      type: Opaque
      kind: Secret
      apiVersion: v1

2. Run the following command to deploy the LDAP backend Service: Expected output:

      kubectl apply -f ldap.yaml

      deployment.apps/ldap created
      service/ldap-service created
      secret/ldap-secret created

(Optional) Deploy the LDAP frontend

1. Create a file named phpldapadmin.yaml and copy the following content to the file. This deploys an LDAP frontend pod and its associated Service for improved management efficiency through a web interface.

   View the LDAP frontend pod and its associated Service

      ---
      apiVersion: apps/v1
      kind: Deployment
      metadata:
        namespace: default
        name: phpldapadmin
        labels:
          io.kompose.service: phpldapadmin
      spec:
        selector:
          matchLabels:
            io.kompose.service: phpldapadmin
        revisionHistoryLimit: 10
        template:
          metadata:
            labels:
              io.kompose.service: phpldapadmin
          spec:
            securityContext:
              seLinuxOptions: {}
            imagePullSecrets: []
            restartPolicy: Always
            initContainers: []
            containers:
              - image: 'osixia/phpldapadmin:0.9.0'
                imagePullPolicy: Always
                name: phpldapadmin
                volumeMounts: []
                resources:
                  limits:
                  requests:
                env:
                  - name: PHPLDAPADMIN_HTTPS
                    value: 'false'
                  - name: PHPLDAPADMIN_LDAP_HOSTS
                    value: ldap-service
                lifecycle: {}
                ports:
                  - containerPort: 80
                    protocol: TCP
            volumes: []
            dnsPolicy: ClusterFirst
            dnsConfig: {}
            terminationGracePeriodSeconds: 30
        progressDeadlineSeconds: 600
        strategy:
          type: RollingUpdate
          rollingUpdate:
            maxUnavailable: 25%
            maxSurge: 25%
        replicas: 1
      ---
      apiVersion: v1
      kind: Service
      metadata:
        namespace: default
        name: phpldapadmin
        annotations:
          k8s.kuboard.cn/workload: phpldapadmin
        labels:
          io.kompose.service: phpldapadmin
      spec:
        selector:
          io.kompose.service: phpldapadmin
        type: ClusterIP
        ports:
          - port: 8080
            targetPort: 80
            protocol: TCP
            name: '8080'
            nodePort: 0
        sessionAffinity: None

   Run the following command to deploy the LDAP frontend Service:

      kubectl apply -f phpldapadmin.yaml

Configure the LDAP client

1. Log on to a pod in the Slurm-managed cluster as described in Step 3, and run the following commands to install the LDAP client package:

      apt update
      apt install libnss-ldapd

2. After the libnss-ldapd package is installed, configure the network authentication service for the Slurm-managed cluster in the pod.

   1. Modify the following parameters in the /etc/nslcd.conf file to define the connection to the LDAP server:

      apt update
      apt install vim

      ...
      BASE	dc=example,dc=org # Replace the value with the distinguished name of the root node in the LDAP directory structure.
      URI	ldap://ldap-service # Replace the value with the uniform resource identifier (URI) of your LDAP server.
      ...

      ...
      uri ldap://ldap-service # Replace the value with the URI of your LDAP server.
      base dc=example,dc=org # Specify this parameter based on your LDAP directory structure.
      ...
      tls_cacertfile /etc/ssl/certs/ca-certificates.crt # Specify the path to the certificate authority (CA) certificate file used to verify the LDAP server certificate.
      ...

Auto scaling for Slurm clusters based on on-cloud nodes

Slurm on ACK supports two types of nodes:

• Local nodes: Physical compute nodes that are directly connected to slurmctld.

• On-cloud nodes: Logical nodes backed by VM instances that can be created and destroyed on demand by cloud service providers.

Auto scaling for Slurm on ACK

Procedure

1. Configure auto scaling permissions. If Helm is installed, auto scaling permissions are automatically configured for the slurmctld pod and you can skip this step. The head pod requires permissions to access and update the SlurmCluster CR for auto scaling. We recommend that you use RBAC to grant the required permissions. Follow these steps: First, create the ServiceAccount, Role, and RoleBinding for the slurmctld pod. In the following example, the Slurm-managed cluster name is slurm-job-demo and the namespace is default. Create a file named rbac.yaml and copy the following content to the file: Run kubectl apply -f rbac.yaml to submit the resource list. Next, grant permissions to the slurmctld pod. Run kubectl edit slurmcluster slurm-job-demo to modify the Slurm-managed cluster. Set Spec.Slurmctld.Template.Spec.ServiceAccountName to the ServiceAccount you created: To apply the changes, rebuild the StatefulSet that manages slurmctld. Run kubectl get sts slurm-job-demo to find the StatefulSet, then run kubectl delete sts slurm-job-demo to delete it. The Slurm Operator rebuilds the StatefulSet and applies the new configurations.

      apiVersion: v1
      kind: ServiceAccount
      metadata:
        name: slurm-job-demo
      ---
      apiVersion: rbac.authorization.k8s.io/v1
      kind: Role
      metadata:
        name: slurm-job-demo
      rules:
      - apiGroups: ["kai.alibabacloud.com"]
        resources: ["slurmclusters"]
        verbs: ["get", "watch", "list", "update", "patch"]
        resourceNames: ["slurm-job-demo"]
      ---
      apiVersion: rbac.authorization.k8s.io/v1
      kind: RoleBinding
      metadata:
        name: slurm-job-demo
      subjects:
      - kind: ServiceAccount
        name: slurm-job-demo
      roleRef:
        kind: Role
        name: slurm-job-demo
        apiGroup: rbac.authorization.k8s.io

      apiVersion: kai.alibabacloud.com/v1
      kind: SlurmCluster
      ...
      spec:
        slurmctld:
          template:
            spec:
              serviceAccountName: slurm-job-demo
      ...

2. Configure the auto scaling parameters in /etc/slurm/slurm.conf.

   Method A: Manage ConfigMaps by using a shared volume

      # The following parameters are required if you use on-cloud nodes.
      # The SuspendProgram and ResumeProgram features are developed by Alibaba Cloud.
      SuspendTimeout=600
      ResumeTimeout=600
      # The interval at which the node is automatically suspended when no job runs on the node.
      SuspendTime=600
      # Set the number of nodes that can be scaled per minute.
      ResumeRate=1
      SuspendRate=1
      # You must set the value of the NodeName parameter in the ${cluster_name}-worker-${group_name}- format. You must specify the amount of resources for the node in this line. Otherwise, the slurmctld pod
      # considers that the node has only one vCPU. Make sure that the resources that you specified on the on-cloud nodes are the same as those declared in the workerGroupSpecs parameter. Otherwise, resources may be wasted.
      NodeName=slurm-job-demo-worker-cpu-[0-10] Feature=cloud State=CLOUD
      # The following configurations are fixed. Keep them unchanged.
      CommunicationParameters=NoAddrCache
      ReconfigFlags=KeepPowerSaveSettings
      SuspendProgram="/slurm-suspend.sh"
      ResumeProgram="/slurm-resume.sh"

   Method B: Manually manage ConfigMaps

   If slurm.conf is stored in the ConfigMap named slurm-config, run kubectl edit slurm-config to add the following configurations:

      slurm.conf:
      ...
        # The following parameters are required if you use on-cloud nodes.
        # The SuspendProgram and ResumeProgram features are developed by Alibaba Cloud.
        SuspendTimeout=600
        ResumeTimeout=600
        # The interval at which the node is automatically suspended when no job runs on the node.
        SuspendTime=600
        # Set the number of nodes that can be scaled per minute.
        ResumeRate=1
        SuspendRate=1
        # You must set the value of the NodeName parameter in the ${cluster_name}-worker-${group_name}- format. You must specify the amount of resources for the node in this line. Otherwise, the slurmctld pod
        # considers that the node has only one vCPU. Make sure that the resources that you specified on the on-cloud nodes are the same as those declared in the workerGroupSpecs parameter. Otherwise, resources may be wasted.
        NodeName=slurm-job-demo-worker-cpu-[0-10] Feature=cloud State=CLOUD
        # The following configurations are fixed. Keep them unchanged.
        CommunicationParameters=NoAddrCache
        ReconfigFlags=KeepPowerSaveSettings
        SuspendProgram="/slurm-suspend.sh"
        ResumeProgram="/slurm-resume.sh"

   Method C: Use Helm to manage ConfigMaps

   1. Run the helm upgrade command to update the Slurm configuration.

      slurm.conf:
      ...
        # The following parameters are required if you use on-cloud nodes.
        # The SuspendProgram and ResumeProgram features are developed by Alibaba Cloud.
        SuspendTimeout=600
        ResumeTimeout=600
        # The interval at which the node is automatically suspended when no job runs on the node.
        SuspendTime=600
        # Set the number of nodes that can be scaled per minute.
        ResumeRate=1
        SuspendRate=1
        # You must set the value of the NodeName parameter in the ${cluster_name}-worker-${group_name}- format. You must specify the amount of resources for the node in this line. Otherwise, the slurmctld pod
        # considers that the node has only one vCPU. Make sure that the resources that you specified on the on-cloud nodes are the same as those declared in the workerGroupSpecs parameter. Otherwise, resources may be wasted.
        NodeName=slurm-job-demo-worker-cpu-[0-10] Feature=cloud State=CLOUD
        # The following configurations are fixed. Keep them unchanged.
        CommunicationParameters=NoAddrCache
        ReconfigFlags=KeepPowerSaveSettings
        SuspendProgram="/slurm-suspend.sh"
        ResumeProgram="/slurm-resume.sh"

3. Apply the new configuration. If the name of the Slurm-managed cluster is slurm-job-demo, run kubectl delete sts slurm-job-demo to apply the new configuration for the slurmctld pod.

4. Set the number of worker node replicas to 0 in the slurmcluster.yaml file so that you can observe node scaling activities in subsequent steps.

   Manual management

   Run kubectl edit slurmcluster slurm-job-demo and change the value of workerCount to 10 in the Slurm-managed cluster. This sets the number of worker node replicas to 0.

   Manage by using Helm

   In the values.yaml file, change .Values.workerGroup[].workerCount to 0. Then run helm upgrade slurm-job-demo . to update the current Helm chart. This sets the number of worker node replicas to 0.

5. Submit a job by using the sbatch command. Enter the following content after the command prompt: Expected output: This output confirms that the script content is correct. Expected output: This output indicates that the job is submitted and assigned a job ID.

   1. Run the following command to submit the script to the Slurm-managed cluster for processing:

      cat << EOF > cloudnodedemo.sh

      > #!/bin/bash
      > srun hostname
      > EOF

      cat cloudnodedemo.sh

      #!/bin/bash
        srun hostname

      sbatch cloudnodedemo.sh

      Submitted batch job 1

6. View the cluster scaling results. Expected output: This output indicates that the Slurm-managed cluster automatically added one compute node to execute the submitted job. Expected output: This output shows that the slurm-demo-worker-cpu-0 pod was added to the cluster, confirming that the cluster scaled out when the job was submitted. Expected output: This output shows that slurm-demo-worker-cpu-0 is the newly started node and another 10 on-cloud nodes are available for scale-out. Expected output: In the output, NodeList=slurm-demo-worker-cpu-0 indicates that the job was executed on the newly added node. Expected output: This output shows that the number of nodes available for scale-out has increased to 11, confirming that the automatic scale-in is complete.

   1. After a period of time, run the following command to view the scale-in results:

      cat /var/log/slurm-resume.log

      namespace: default cluster: slurm-demo
        resume called, args [slurm-demo-worker-cpu-0]
        slurm cluster metadata: default slurm-demo
        get SlurmCluster CR slurm-demo succeed
        hostlists: [slurm-demo-worker-cpu-0]
        resume node slurm-demo-worker-cpu-0
        resume worker -cpu-0
        resume node -cpu-0 end

      kubectl get pod

      NAME                                          READY   STATUS    RESTARTS        AGE
      slurm-demo-head-9hn67                         1/1     Running   0               21m
      slurm-demo-worker-cpu-0                       1/1     Running   0               43s

      sinfo

      PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
      debug*       up   infinite      10  idle~ slurm-job-demo-worker-cpu-[2-10]
      debug*       up   infinite      1   idle slurm-job-demo-worker-cpu-[0-1]

      scontrol show job 1

      JobId=1 JobName=cloudnodedemo.sh
         UserId=root(0) GroupId=root(0) MCS_label=N/A
         Priority=4294901757 Nice=0 Account=(null) QOS=(null)
         JobState=COMPLETED Reason=None Dependency=(null)
         Requeue=1 Restarts=0 BatchFlag=1 Reboot=0 ExitCode=0:0
         RunTime=00:00:00 TimeLimit=UNLIMITED TimeMin=N/A
         SubmitTime=2024-05-28T11:37:36 EligibleTime=2024-05-28T11:37:36
         AccrueTime=2024-05-28T11:37:36
         StartTime=2024-05-28T11:37:36 EndTime=2024-05-28T11:37:36 Deadline=N/A
         SuspendTime=None SecsPreSuspend=0 LastSchedEval=2024-05-28T11:37:36 Scheduler=Main
         Partition=debug AllocNode:Sid=slurm-job-demo:93
         ReqNodeList=(null) ExcNodeList=(null)
         NodeList=slurm-job-demo-worker-cpu-0
         BatchHost=slurm-job-demo-worker-cpu-0
         NumNodes=1 NumCPUs=1 NumTasks=1 CPUs/Task=1 ReqB:S:C:T=0:0:*:*
         ReqTRES=cpu=1,mem=1M,node=1,billing=1
         AllocTRES=cpu=1,mem=1M,node=1,billing=1
         Socks/Node=* NtasksPerN:B:S:C=0:0:*:* CoreSpec=*
         MinCPUsNode=1 MinMemoryNode=0 MinTmpDiskNode=0
         Features=(null) DelayBoot=00:00:00
         OverSubscribe=OK Contiguous=0 Licenses=(null) Network=(null)
         Command=//cloudnodedemo.sh
         WorkDir=/
         StdErr=//slurm-1.out
         StdIn=/dev/null
         StdOut=//slurm-1.out
         Power=

      sinfo

      PARTITION AVAIL  TIMELIMIT  NODES  STATE NODELIST
      debug*       up   infinite     11  idle~ slurm-demo-worker-cpu-[0-10]