Platform For AI：建立訓練任務

使用公用資源提交任務

建立並提交任務的具體調用代碼如下所示。

#!/usr/bin/env python3

from __future__ import print_function

import json
import time

from alibabacloud_tea_openapi.models import Config
from alibabacloud_credentials.client import Client as CredClient
from alibabacloud_pai_dlc20201203.client import Client as DLCClient
from alibabacloud_pai_dlc20201203.models import (
    ListJobsRequest,
    ListEcsSpecsRequest,
    CreateJobRequest,
    GetJobRequest,
)

from alibabacloud_aiworkspace20210204.client import Client as AIWorkspaceClient
from alibabacloud_aiworkspace20210204.models import (
    ListWorkspacesRequest,
    CreateDatasetRequest,
    ListDatasetsRequest,
    ListImagesRequest,
    ListCodeSourcesRequest
)


def create_nas_dataset(client, region, workspace_id, name,
                       nas_id, nas_path, mount_path):
    '''建立NAS的資料集。
    '''
    response = client.create_dataset(CreateDatasetRequest(
        workspace_id=workspace_id,
        name=name,
        data_type='COMMON',
        data_source_type='NAS',
        property='DIRECTORY',
        uri=f'nas://{nas_id}.{region}{nas_path}',
        accessibility='PRIVATE',
        source_type='USER',
        options=json.dumps({
            'mountPath': mount_path
        })
    ))
    return response.body.dataset_id


def create_oss_dataset(client, region, workspace_id, name,
                       oss_bucket, oss_endpoint, oss_path, mount_path):
    '''建立OSS資料集。
    '''
    response = client.create_dataset(CreateDatasetRequest(
        workspace_id=workspace_id,
        name=name,
        data_type='COMMON',
        data_source_type='OSS',
        property='DIRECTORY',
        uri=f'oss://{oss_bucket}.{oss_endpoint}{oss_path}',
        accessibility='PRIVATE',
        source_type='USER',
        options=json.dumps({
            'mountPath': mount_path
        })
    ))
    return response.body.dataset_id



def wait_for_job_to_terminate(client, job_id):
    while True:
        job = client.get_job(job_id, GetJobRequest()).body
        print('job({}) is {}'.format(job_id, job.status))
        if job.status in ('Succeeded', 'Failed', 'Stopped'):
            return job.status
        time.sleep(5)
    return None


def main():

    # 請確認您的主帳號已授權DLC，且擁有足夠的許可權。
    region_id = 'cn-hangzhou'
    # 阿里雲帳號AccessKey擁有所有API的存取權限，建議您使用RAM使用者進行API訪問或日常營運。
    # 強烈建議不要把AccessKey ID和AccessKey Secret儲存到工程代碼裡，否則可能導致AccessKey泄露，威脅您帳號下所有資源的安全。
    # 本樣本通過Credentials SDK預設從環境變數中讀取AccessKey，來實現身分識別驗證為例。
    cred = CredClient()

    # 1. create client;
    workspace_client = AIWorkspaceClient(
        config=Config(
            credential=cred,
            region_id=region_id,
            endpoint="aiworkspace.{}.aliyuncs.com".format(region_id),
        )
    )

    dlc_client = DLCClient(
         config=Config(
            credential=cred,
            region_id=region_id,
            endpoint='pai-dlc.{}.aliyuncs.com'.format(region_id),
         )
    )

    print('------- Workspaces -----------')
    # 擷取工作空間列表。您也可以在參數workspace_name中填入您建立的工作空間名。
    workspaces = workspace_client.list_workspaces(ListWorkspacesRequest(
        page_number=1, page_size=1, workspace_name='',
        module_list='PAI'
    ))
    for workspace in workspaces.body.workspaces:
        print(workspace.workspace_name, workspace.workspace_id,
              workspace.status, workspace.creator)

    if len(workspaces.body.workspaces) == 0:
        raise RuntimeError('found no workspaces')

    workspace_id = workspaces.body.workspaces[0].workspace_id

    print('------- Images ------------')
    # 擷取鏡像列表。
    images = workspace_client.list_images(ListImagesRequest(
        labels=','.join(['system.supported.dlc=true',
                         'system.framework=Tensorflow 1.15',
                         'system.pythonVersion=3.6',
                         'system.chipType=CPU'])))
    for image in images.body.images:
        print(json.dumps(image.to_map(), indent=2))

    image_uri = images.body.images[0].image_uri

    print('------- Datasets ----------')
    # 擷取資料集。
    datasets = workspace_client.list_datasets(ListDatasetsRequest(
        workspace_id=workspace_id,
        name='example-nas-data', properties='DIRECTORY'))
    for dataset in datasets.body.datasets:
        print(dataset.name, dataset.dataset_id, dataset.uri, dataset.options)

    if len(datasets.body.datasets) == 0:
        # 當前資料集不存在時，建立資料集。
        dataset_id = create_nas_dataset(
            client=workspace_client,
            region=region_id,
            workspace_id=workspace_id,
            name='example-nas-data',
            # Nas檔案系統ID。
            # 通用型NAS：31a8e4****。
            # 極速型NAS：必須以extreme-開頭，例如extreme-0015****。
            # CPFS：必須以cpfs-開頭，例如cpfs-125487****。
            nas_id='***',
            nas_path='/',
            mount_path='/mnt/data/nas')
        print('create dataset with id: {}'.format(dataset_id))
    else:
        dataset_id = datasets.body.datasets[0].dataset_id

    print('------- Code Sources ----------')
    # 擷取代碼集列表。
    code_sources = workspace_client.list_code_sources(ListCodeSourcesRequest(
        workspace_id=workspace_id))
    for code_source in code_sources.body.code_sources:
        print(code_source.display_name, code_source.code_source_id, code_source.code_repo)

    print('-------- ECS SPECS ----------')
    # 擷取DLC的節點規格列表。
    ecs_specs = dlc_client.list_ecs_specs(ListEcsSpecsRequest(page_size=100, sort_by='Memory', order='asc'))
    for spec in ecs_specs.body.ecs_specs:
        print(spec.instance_type, spec.cpu, spec.memory, spec.memory, spec.gpu_type)

    print('-------- Create Job ----------')
    # 建立DLC作業。
    create_job_resp = dlc_client.create_job(CreateJobRequest().from_map({
        'WorkspaceId': workspace_id,
        'DisplayName': 'sample-dlc-job',
        'JobType': 'TFJob',
        'JobSpecs': [
            {
                "Type": "Worker",
                "Image": image_uri,
                "PodCount": 1,
                "EcsSpec": ecs_specs.body.ecs_specs[0].instance_type,
            },
        ],
        "UserCommand": "echo 'Hello World' && ls -R /mnt/data/ && sleep 30 && echo 'DONE'",
        'DataSources': [
            {
                "DataSourceId": dataset_id,
            },
        ],
    }))
    job_id = create_job_resp.body.job_id

    wait_for_job_to_terminate(dlc_client, job_id)

    print('-------- List Jobs ----------')
    # 擷取DLC的作業列表。
    jobs = dlc_client.list_jobs(ListJobsRequest(
        workspace_id=workspace_id,
        page_number=1,
        page_size=10,
    ))
    for job in jobs.body.jobs:
        print(job.display_name, job.job_id, job.workspace_name,
              job.status, job.job_type)
    pass


if __name__ == '__main__':
    main()

使用預付費資源配額提交任務

1. 登入PAI控制台。

2. 按照下圖操作指引，在工作空間列表頁面查看您所在的工作空間ID。

3. 按照下圖操作指引，查看您的專有資源群組的資源配額ID。

4. 使用以下代碼建立並提交任務。可使用的公用鏡像列表，詳情請參見步驟二：準備鏡像。

   from alibabacloud_pai_dlc20201203.client import Client
   from alibabacloud_credentials.client import Client as CredClient
   from alibabacloud_tea_openapi.models import Config
   from alibabacloud_pai_dlc20201203.models import (
       CreateJobRequest,
       JobSpec,
       ResourceConfig, GetJobRequest
   )
   
   # 初始化一個Client，用來訪問DLC的API。
   region = 'cn-hangzhou'
   # 阿里雲帳號AccessKey擁有所有API的存取權限，建議您使用RAM使用者進行API訪問或日常營運。
   # 強烈建議不要把AccessKey ID和AccessKey Secret儲存到工程代碼裡，否則可能導致AccessKey泄露，威脅您帳號下所有資源的安全。
   # 本樣本通過Credentials SDK預設從環境變數中讀取AccessKey，來實現身分識別驗證為例。
   cred = CredClient()
   client = Client(
       config=Config(
           credential=cred,
           region_id=region,
           endpoint=f'pai-dlc.{region}.aliyuncs.com',
       )
   )
   
   # 聲明任務的資源配置，關於鏡像選擇可以參考文檔中公用鏡像列表，也可以傳入自己的鏡像地址。
   spec = JobSpec(
       type='Worker',
       image=f'registry-vpc.cn-hangzhou.aliyuncs.com/pai-dlc/tensorflow-training:1.15-cpu-py36-ubuntu18.04',
       pod_count=1,
       resource_config=ResourceConfig(cpu='1', memory='2Gi')
   )
   
   # 聲明任務的執行內容。
   req = CreateJobRequest(
           resource_id='<替換成您自己的資源配額ID>',
           workspace_id='<替換成您自己的WorkspaceID>',
           display_name='sample-dlc-job',
           job_type='TFJob',
           job_specs=[spec],
           user_command='echo "Hello World"',
   )
   
   # 提交任務。
   response = client.create_job(req)
   # 擷取任務ID。
   job_id = response.body.job_id
   
   # 查詢任務狀態。
   job = client.get_job(job_id, GetJobRequest()).body
   print('job status:', job.status)
   
   # 查看任務執行的命令。
   job.user_command

使用競價資源提交任務

• SpotDiscountLimit（Spot折扣）

  #!/usr/bin/env python3
  
  from alibabacloud_tea_openapi.models import Config
  from alibabacloud_credentials.client import Client as CredClient
  
  from alibabacloud_pai_dlc20201203.client import Client as DLCClient
  from alibabacloud_pai_dlc20201203.models import CreateJobRequest
  
  region_id = '<region-id>'  # DLC任務所在地區ID，例如華東1（杭州）為cn-hangzhou。
  cred = CredClient()
  workspace_id = '12****'  # DLC任務所在工作空間ID。
  
  dlc_client = DLCClient(
      Config(credential=cred,
             region_id=region_id,
             endpoint='pai-dlc.{}.aliyuncs.com'.format(region_id),
             protocol='http'))
  
  create_job_resp = dlc_client.create_job(CreateJobRequest().from_map({
      'WorkspaceId': workspace_id,
      'DisplayName': 'sample-spot-job',
      'JobType': 'PyTorchJob',
      'JobSpecs': [
          {
              "Type": "Worker",
              "Image": "dsw-registry-vpc.<region-id>.cr.aliyuncs.com/pai/pytorch-training:1.12-cpu-py39-ubuntu20.04",
              "PodCount": 1,
              "EcsSpec": 'ecs.g7.xlarge',
              "SpotSpec": {
                  "SpotStrategy": "SpotWithPriceLimit",
                  "SpotDiscountLimit": 0.4,
              }
          },
      ],
      'UserVpc': {
          "VpcId": "vpc-0jlq8l7qech3m2ta2****",
          "SwitchId": "vsw-0jlc46eg4k3pivwpz8****",
          "SecurityGroupId": "sg-0jl4bd9wwh5auei9****",
      },
      "UserCommand": "echo 'Hello World' && ls -R /mnt/data/ && sleep 30 && echo 'DONE'",
  }))
  job_id = create_job_resp.body.job_id
  print(f'jobId is {job_id}')
  

• SpotPriceLimit（Spot價格）

  #!/usr/bin/env python3
  
  from alibabacloud_tea_openapi.models import Config
  from alibabacloud_credentials.client import Client as CredClient
  
  from alibabacloud_pai_dlc20201203.client import Client as DLCClient
  from alibabacloud_pai_dlc20201203.models import CreateJobRequest
  
  region_id = '<region-id>'
  cred = CredClient()
  workspace_id = '12****'
  
  dlc_client = DLCClient(
      Config(credential=cred,
             region_id=region_id,
             endpoint='pai-dlc.{}.aliyuncs.com'.format(region_id),
             protocol='http'))
  
  create_job_resp = dlc_client.create_job(CreateJobRequest().from_map({
      'WorkspaceId': workspace_id,
      'DisplayName': 'sample-spot-job',
      'JobType': 'PyTorchJob',
      'JobSpecs': [
          {
              "Type": "Worker",
              "Image": "dsw-registry-vpc.<region-id>.cr.aliyuncs.com/pai/pytorch-training:1.12-cpu-py39-ubuntu20.04",
              "PodCount": 1,
              "EcsSpec": 'ecs.g7.xlarge',
              "SpotSpec": {
                  "SpotStrategy": "SpotWithPriceLimit",
                  "SpotPriceLimit": 0.011,
              }
          },
      ],
      'UserVpc': {
          "VpcId": "vpc-0jlq8l7qech3m2ta2****",
          "SwitchId": "vsw-0jlc46eg4k3pivwpz8****",
          "SecurityGroupId": "sg-0jl4bd9wwh5auei9****",
      },
      "UserCommand": "echo 'Hello World' && ls -R /mnt/data/ && sleep 30 && echo 'DONE'",
  }))
  job_id = create_job_resp.body.job_id
  print(f'jobId is {job_id}')
  

其中關鍵配置說明如下：