ApsaraVideo Live:Integrate Queen SDK for web

Install the Queen SDK engine

npm install aliyun-queen-engine

Import the default edition

The default edition (Advanced Edition) of Queen SDK supports basic face retouching, advanced face retouching, makeup, face shaping, filters, and stickers.

import QueenEngine, {kQueenBeautyType, kQueenBeautyParams, kQueenBeautyMakeupType, kQueenBeautyFaceShapeType, kQueenBeautyBlend} from "aliyun-queen-engine"
queenEngine = new QueenEngine();

Import Basic Edition

Queen SDK Basic Edition supports only basic face retouching.

import {QueenEngineLite, kQueenBeautyType, kQueenBeautyParams} from "aliyun-queen-engine"
queenEngine = new QueenEngineLite();

Import Pro Edition

Queen SDK Pro Edition supports basic face retouching, advanced face retouching, makeup, face shaping, body shaping, filters, stickers, AR writing, and chroma key.

import {QueenEnginePro, kQueenBeautyType, kQueenBeautyParams, kQueenBeautyMakeupType, kQueenBeautyFaceShapeType, kQueenBeautyBodyShapeType, kQueenBeautyBlend} from "aliyun-queen-engine"
queenEngine = new QueenEnginePro();

Import Full Edition

Queen SDK Full Edition supports basic face retouching, advanced face retouching, makeup, face shaping, body shaping, filters, stickers, AR writing, chroma key, and Animojis.

import {QueenEngineFull, kQueenBeautyType, kQueenBeautyParams, kQueenBeautyMakeupType, kQueenBeautyFaceShapeType, kQueenBeautyBodyShapeType, kQueenBeautyBlend} from "aliyun-queen-engine"
queenEngine = new QueenEngineFull();

Import a worker

When you initialize the worker, you can specify Basic Edition, Advanced Edition, Pro Edition, or Full Edition by passing in the kQueenVersion parameter.

import {QueenEngineWorker, kQueenBeautyType, kQueenBeautyParams, kQueenBeautyMakeupType, kQueenBeautyFaceShapeType, kQueenBeautyBodyShapeType, kQueenBeautyBlend, kQueenVersion} from "aliyun-queen-engine"
queenEngine = new QueenEngineWorker(kQueenVersion.Pro);

Initialize QueenEngine

1. Initialize queenEngine.

   const sdkLicenseKey = ""; // The license key of Queen SDK.
   const queenEngine = new QueenEngine();
   //const queenEngine = new QueenEnginePro(); // Initialize Queen SDK Pro Edition.
   queenEngine.init(sdkLicenseKey, function(){
    // The initialization is complete.
   }, function(progress){
    // The progress bar is loaded.
   });

2. Initialize a specified canvas.

   For the canvas, set the canvasElement type parameter to webgl2 or leave this parameter empty.

   const canvasElement = document.getElementById("canvas");
   const queenEngine = new QueenEngine();
   queenEngine.init(sdkLicenseKey, function(){
    // The initialization is complete.
   }, function(progress){
    // The progress bar is loaded.
   }, canvasElement);

3. Initialize specified inference models.

   const queenEngine = new QueenEnginePro();
   queenEngine.init(sdkLicenseKey, function(){
    // The initialization is complete.
   }, function(progress){
    // The progress bar is loaded.
   }, canvasElement,{
    "segment": kQueenModelShapeType.None,// Do not load the chroma key model during initialization.
    "pose": kQueenModelShapeType.None, // Do not load the body shaping model during initialization.
    "backend": kBackendType.Auto // Automatically select the inference backend.
   });

   Note

   • kQueenModelShapeType.None: By default, the chroma key or body shaping model is not loaded. When you want to use the model, manually load it as needed.

   • kQueenModelShapeType.Horizontal: The horizontal chroma key or body shaping model is loaded during initialization.

   • kQueenModelShapeType.Vertical: The vertical chroma key or body shaping model is loaded during initialization.

   • kQueenModelShapeType.Both: The horizontal and vertical chroma key or body shaping models are both loaded during initialization.

   • For a horizontal model, the width of the input source is greater than the height of the input source, which is indicated by width > height.

   • For a vertical model, the height of the input source is greater than the width of the input source, which is indicated by height > width.

   • kBackendType.WebGL: The WebGL inference backend is used.

   • kBackendType.WebGPU: The WebGPU inference backend is used.

   • Model loading affects the speed of page loading.

4. (Optional) Initialize the worker.

   import {QueenEngineWorker, kQueenBeautyType, kQueenBeautyParams, kQueenBeautyMakeupType, kQueenBeautyFaceShapeType, kQueenBeautyBlend} from "aliyun-queen-engine"
   const canvasElement = document.getElementById("canvas");
   queenEngine = new QueenEngineWorker(kQueenVersion.Pro);
   queenEngine.init(sdkLicenseKey, function(){
    // The initialization is complete.
   }, function(progress){
    // The progress bar is loaded.
   }, canvasElement);

Configure retouching parameters

1. Configure basic face retouching.

   // Enable basic retouching.
   queenEngine.setQueenBeautyType(kQueenBeautyType.SkinBuffing, true);
   // Set the level of skin smoothing.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinBuffing, 0.7);
   // Set the level of image sharpening.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.Sharpen, 0.5);
   // Enable skin whitening.
   queenEngine.setQueenBeautyType(kQueenBeautyType.SkinWhiting, true);
   // Set the level of skin whitening.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.Whitening, 0.6);
   // Set the level of rosy cheeks.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinRed, 0.2);

2. Configure advanced face retouching.

   // Enable advanced face retouching.
   queenEngine.setQueenBeautyType(kQueenBeautyType.FaceBuffing, true);
   // Set the level of eye-bag removal.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.Pouch, 0.9);
   // Set the level of nasolabial fold removal.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.NasolabialFolds, 0.9);
   // Set the level of teeth whitening.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.WhiteTeeth, 0.9);
   // Set the level of lipstick.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.Lipstick, 0.2);
   // Set the level of blush.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.Blush, 0.1);
   // Set the color of lipstick.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickColorParam, 0.1);
   // Set the saturation of lipstick.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickGlossParam, 0.1);
   // Set the brightness of lipstick.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.LipstickBrightnessParam, 0.1);
   // Set the level of eye brightening.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.BrightenEye, 0.9);
   // Set the level of rosy cheeks.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.SkinRed, 0.2);
   // Set the level of wrinkle removal.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.Wrinkles, 0.9);
   // Set the level of skin brightening.
   queenEngine.setQueenBeautyParams(kQueenBeautyParams.BrightenFace, 0.3);

3. Configure face shaping.

   // Enable face shaping. The face shaping parameters are indicated by kQueenBeautyFaceShapeType.
   queenEngine.setQueenBeautyType(kQueenBeautyType.FaceShape, true);
   // Set the cheekbones. Valid values: [0,1]. Default value: 0.
   queenEngine.setFaceShape(kQueenBeautyFaceShapeType.CutCheek, 0.6);
   // Set cheekbone narrowing. Valid values: [0,1]. Default value: 0.
   queenEngine.setFaceShape(kQueenBeautyFaceShapeType.CutFace, 0.7);
   // Set face slimming. Valid values: [0,1]. Default value: 0.
   queenEngine.setFaceShape(kQueenBeautyFaceShapeType.ThinFace, 0.8);
   // Set the face length. Valid values: [0,1]. Default value: 0.
   queenEngine.setFaceShape(kQueenBeautyFaceShapeType.LowerJaw, 0.8);
   // Set chin lengthening. Valid values: [0,1]. Default value: 0.
   queenEngine.setFaceShape(kQueenBeautyFaceShapeType.HigherJaw, 0.6); 

4. Configure makeup.

   • Use the built-in makeup resources of the SDK.

     When you use the built-in resources, the SDK downloads retouching materials from Alibaba Cloud CDN.

     // Enable makeup.
     queenEngine.setQueenBeautyType(kQueenBeautyType.Makeup, true);
     // Set the eyebrow effect.
     await queenEngine.setMakeupEyeBrow(Assets.kResMakeupEyeBrow.BiaoZhunMei, 0.6);
     // Set the eyelash effect.
     await queenEngine.setMackupEyeLash(Assets.kResMakeupEyeLash.ChenJing, 0.6);
     // Set the eye shadow effect.
     await queenEngine.setMakeupEyeShadow(Assets.kResMakeupEyeShadow.DaDiSe, 0.5);
     // Set the eyeliner effect.
     await queenEngine.setMakeupEyeLiner(Assets.kResMakeupEyeLiner.DaYan, 0.4);
     // Set the colored eye contacts effect.
     await queenEngine.setMakeupEyeBall(Assets.kResMakeupEyeBall.BiMuYu, 0.5);
     // Set the lipstick effect.
     await queenEngine.setMakeupMouth(Assets.kResMakeupMouth.AnYeZi, 0.3);
     // Set the blush effect.
     await queenEngine.setMakeupBlush(Assets.kResMakeupBlush.BlushWuGu, 0.2);
     // Set the highlight effect.
     await queenEngine.setMakeupHighlight(Assets.kResMakeupHighLight.Highlight, 0.1);
     // Remove the colored eye contacts effect.
     queenEngine.removeMakeupWithType(kQueenBeautyMakeupType.Eyeball);
     // For more information, see the QueenEngin.d.ts file.

   • Use your local makeup resources.

     const makeupPackage = "./mouth.zip"
     const makeupName = "1.2.3.png";
     const band = kQueenBeautyBlend.LabMix;
     // Enable makeup. The makeup parameters are indicated by kQueenBeautyMakeupType.
     queenEngine.setQueenBeautyType(kQueenBeautyType.Makeup, true);
     // Set the transparency of the lipstick effect.
     queenEngine.setMakeupAlphaWithType(kQueenBeautyMakeupType.Mouth, true, 0.6);
     // Set the lipstick effect.
     queenEngine.setMakeupWithPackage(kQueenBeautyMakeupType.Mouth, makeupPackage, makeupName, band).then(() => {
     
     });

5. Configure filters.

   • Use the built-in filters.

     await queenEngine.setLutByType(Assets.kResLut.M1, 0.5);

   • Use custom filters.

     const lutImageUrl = "./lut.png";
     queenEngine.setLutImageUrl(lutImageUrl).then(function () {
       queenEngine.setQueenBeautyType(kQueenBeautyType.LUT, true);
       queenEngine.setQueenBeautyParams(kQueenBeautyParams.LUT, 0.5);
     });

6. Configure stickers.

   1. Use the built-in stickers.

      queenEngine.addMaterialWithType(Assets.kResSticker.ILoveChina);
      // Specify a single sticker.
      queenEngine.addMaterialWithIndex(0);
      // Or, specify multiple stickers.
      queenEngine.addMaterialWithIndexs([0,1]);

   2. Use custom stickers.

      const stickerZip = "./sticker.zip";
      queenEngine.addMaterialWithUrl(stickerZip).then(() => {
       });

7. Configure background replacement.

   const backgroundUrl = "./bg.png";
   queenEngine.setSegmentBackgroundUrl(backgroundUrl).then(() => {
    });

8. Configure green-screen or blue-screen chroma key.

   const backgroundUrl = "./bg.png";
   const isBlue = false;// Specify whether to use a blue screen.
   queenEngine.setGreenScreenWithUrl(isBlue, backgroundUrl).then(() => {
    });

9. Configure background processing.

   // Blur the background.
   queenEngine.enableBokehBackground(true);
   // Make the background transparent.
   queenEngine.enableTransparentBackground(true);

Perform rendering

1. Render the camera stream.

    queenEngine.openCameraAndRender().then((stream=>{
    const video = document.querySelector('video');
    video.srcObject = stream;
    video.play();
    })) 

2. Apply the rendered media to the canvas.

   The canvas is the canvas that was specified when the engine was initialized.

    const sourceVideo = document.querySelector('video');
    queenEngine.renderMediaObjectToRenderCanvas(sourceVideo, sourceVideo.clientWidth, sourceVideo.clientHeight);

3. Render the video stream.

    navigator.mediaDevices.getUserMedia(constraints)
    .then(mediaStream => {
    let renderMediaStream = queenEngine.renderMediaStream(mediaStream);
    const video = document.querySelector('video');
    video.srcObject = renderMediaStream;
    video.play();
    });

4. Render an image.

   fetch(Image URL)
   .then(buffer => buffer.blob())
   .then(createImageBitmap)
   .then(img => {
    queenEngine.renderWithMediaObject(img, img.width, img.height, function(imageBufferData, imageWidth, imageHeight){
    const canvas = document.getElementById('playCanvas'); // The canvas. 
    const ctx = canvas.getContext("2d");
    const imageBuffer = new Uint8ClampedArray(imageBufferData);
    const imageData = new ImageData(imageBuffer, imageWidth, imageHeight);
    ctx.clearRect(0, 0, imageWidth, imageHeight);
    ctx.putImageData(imageData, 0, 0);
   });
   });

5. Render the texture.

   const canvas = document.getElementById("sourceCanvas");
   let outTexture = queenEngine.renderMediaObjectToTexture(canvas, canvas.width, canvas.height);
   queenEngine.drawOutTexture(outTexture);// Draw the texture in the canvas that was specified during initialization.

6. Render the texture input.

   let inputTexture = queenEngine.generateTextureId();
   queenEngine.updateTexture(inputTexture, imageData);
   let outTexture = queenEngine.renderTextureId(inputTexture, width, height);
   queenEngine.drawOutTexture(outTexture)// Draw the texture in the canvas that was specified during initialization.

7. Render the pipeline stream.

   const videoTrack = stream.getVideoTracks()[0];
   const processor = new MediaStreamTrackProcessor({ track: videoTrack });
   const readFrameStream = processor.readable;
   const generator = new MediaStreamTrackGenerator({ kind: 'video' });
   let writeFrameStream = generator.writable;
   readFrameStream.pipeThrough(queenEngine.getTransformStream()).pipeTo(writeFrameStream);