在Agora Video SDK之上运行AI姿势检测

在 Agora 的 Video SDK 之上运行 AI

先决条件：

• 节点
• 火力基地功能
• Agora Web SDK 3.1.0 或以上
• Firebase 托管

第一步：注册 Agora 账号

Agora 每月提供 10,000 分钟的免费 sdk 使用时间，可以完美地用于测试应用程序。要注册一个帐户，请访问https://sso.agora.io/v2/signup ，您可以从那里开始。

注册 Agora

 

创建帐户后，只需创建一个 API 密钥。

Agora 仪表板生成临时令牌

 

。。

。

。

id="output" >
id="image" />

。

imagectx.clearRect(0, 0, $('#output').width(), $('#output').height());

    imagectx.save();
    imagectx.scale(-1, 1);
    imagectx.translate(-$('#output').width(), 0);
    imagectx.drawImage(video, 0, 0, $('#output').width(), $('#output').height());
    imagectx.restore();

第 4 步：在画布上运行 tfjs

现在我们有了一个可以运行 AI 推理的元素，我们终于可以使用 TFJS。在撰写本文时，我们正在通过 TF 2.0 进行

></script>


script>

我们现在可以在输出画布上运行推理

async function poseDetectionFrame() {    
    if(!JSON.parse(getMeta("useai")))
    {      
      return;
    }
    let poses = [];
    let minPoseConfidence;
    let minPartConfidence;

    imagectx.clearRect(0, 0, $('#output').width(), $('#output').height());

    imagectx.save();
    imagectx.scale(-1, 1);
    imagectx.translate(-$('#output').width(), 0);
    imagectx.drawImage(video, 0, 0, $('#output').width(), $('#output').height());
    imagectx.restore();

      const pose = await net.estimatePoses(image, {
            flipHorizontal: false,
            decodingMethod: 'single-person'
          });

    ctx.clearRect(0, 0, $('#output').width(), $('#output').height());

    ctx.save();
    ctx.scale(-1, 1);
    ctx.translate(-$('#output').width(), 0);
    ctx.drawImage(video, 0, 0, $('#output').width(), $('#output').height());
    ctx.restore();

      poses = poses.concat(pose);
      minPoseConfidence = + 0.15;
      minPartConfidence = + 0.1;
          // For each pose (i.e. person) detected in an image, loop through the poses
      // and draw the resulting skeleton and keypoints if over certain confidence
      // Step 5

      requestAnimationFrame(poseDetectionFrame);
    

  }
  poseDetectionFrame();
  setTimeout(function () {
    $owlCarouselNew.trigger('refresh.owl.carousel');
  }, 1500);
}

第 5 步：获取 AR 简笔画

AR其实很简单，一旦你掌握了关键点，我们就可以画出代表实时推理的简笔画。现在，我们在 AI 数据之上添加了 AR，以便让用户更好地展示他们的姿势。

       poses.forEach(({score, keypoints}) => {
        
        if (score >= minPoseConfidence) {
          drawKeypoints(keypoints, minPartConfidence, ctx);
          drawSkeleton(keypoints, minPartConfidence, ctx);
          /*
          if (guiState.output.showBoundingBox) {
            drawBoundingBox(keypoints, ctx);
          }*/
        }
      });

通过以下功能

function drawPoint(ctx, y, x, r, color) {
  ctx.beginPath();
  ctx.arc(x, y, r, 0, 2 * Math.PI);
  ctx.fillStyle = color;
  ctx.fill();
}

/**
 * Draw the bounding box of a pose. For example, for a whole person standing
 * in an image, the bounding box will begin at the nose and extend to one of
 * ankles
 */
function drawBoundingBox(keypoints, ctx) {
  const boundingBox = posenet.getBoundingBox(keypoints);

  ctx.rect(
      boundingBox.minX, boundingBox.minY, boundingBox.maxX - boundingBox.minX,
      boundingBox.maxY - boundingBox.minY);

  ctx.strokeStyle = boundingBoxColor;
  ctx.stroke();
}

/**
 * Draws a line on a canvas, i.e. a joint
 */
function drawSegment([ay, ax], [by, bx], color, scale, ctx) {
  ctx.beginPath();
  ctx.moveTo(ax * scale, ay * scale);
  ctx.lineTo(bx * scale, by * scale);
  ctx.lineWidth = lineWidth;
  ctx.strokeStyle = color;
  ctx.stroke();
}

/**
 * Draws a pose skeleton by looking up all adjacent keypoints/joints
 */
function drawSkeleton(keypoints, minConfidence, ctx, scale = 1) {
  const adjacentKeyPoints =function drawPoint(ctx, y, x, r, color) {
  ctx.beginPath();
  ctx.arc(x, y, r, 0, 2 * Math.PI);
  ctx.fillStyle = color;
  ctx.fill();
}

/**
 * Draw the bounding box of a pose. For example, for a whole person standing
 * in an image, the bounding box will begin at the nose and extend to one of
 * ankles
 */
function drawBoundingBox(keypoints, ctx) {
  const boundingBox = posenet.getBoundingBox(keypoints);

  ctx.rect(
      boundingBox.minX, boundingBox.minY, boundingBox.maxX - boundingBox.minX,
      boundingBox.maxY - boundingBox.minY);

  ctx.strokeStyle = boundingBoxColor;
  ctx.stroke();
}

/**
 * Draws a line on a canvas, i.e. a joint
 */
function drawSegment([ay, ax], [by, bx], color, scale, ctx) {
  ctx.beginPath();
  ctx.moveTo(ax * scale, ay * scale);
  ctx.lineTo(bx * scale, by * scale);
  ctx.lineWidth = lineWidth;
  ctx.strokeStyle = color;
  ctx.stroke();
}

/**
 * Draws a pose skeleton by looking up all adjacent keypoints/joints
 */
function drawSkeleton(keypoints, minConfidence, ctx, scale = 1) {
  const adjacentKeyPoints =
      posenet.getAdjacentKeyPoints(keypoints, minConfidence);

  adjacentKeyPoints.forEach((keypoints) => {
    drawSegment(
        toTuple(keypoints[0].position), toTuple(keypoints[1].position), color,
        scale, ctx);
  });
}

/**
 * Draw pose keypoints onto a canvas
 */
function drawKeypoints(keypoints, minConfidence, ctx, scale = 1) {
  for (let i = 0; i < keypoints.length; i++) {
    const keypoint = keypoints[i];

    if (keypoint.score < minConfidence) {
      continue;
    }

    const {y, x} = keypoint.position;
    drawPoint(ctx, y * scale, x * scale, 3, color);
  }
}

function toTuple({y, x}) {
  return [y, x];
}
      posenet.getAdjacentKeyPoints(keypoints, minConfidence);

  adjacentKeyPoints.forEach((keypoints) => {
    drawSegment(
        toTuple(keypoints[0].position), toTuple(keypoints[1].position), color,
        scale, ctx);
  });
}

/**
 * Draw pose keypoints onto a canvas
 */
function drawKeypoints(keypoints, minConfidence, ctx, scale = 1) {
  for (let i = 0; i < keypoints.length; i++) {
    const keypoint = keypoints[i];

    if (keypoint.score < minConfidence) {
      continue;
    }

    const {y, x} = keypoint.position;
    drawPoint(ctx, y * scale, x * scale, 3, color);
  }
}

function toTuple({y, x}) {
  return [y, x];
}

最后一步：现场测试并将其集成到生产中

完成所有工作后，测试通过，然后您可以将其集成到您自己的类似类型的应用程序中

要在瑜伽课上现场体验，请访问https://mixpose.com

https://mixpose.com