Android¶

开发环境搭建¶

推荐配置如下：

Android Studio 3.X及以上
JDK 1.8及以上
NDK 19.2.5345600
您的应用应满足以下条件
- minSdkVersion 23
- targetSdkVersion 29
- compileSdkVersion 29
- buildToolsVersion 30.0.0
- Gradle 6.1.1

接入说明¶

您需要按照如下流程完成应用的开发工作。

License申请

使用算法SDK前需要申请对应的License，否则无法正常使用，具体的申请流程可参考授权方式章节。

集成基础SDK

基础SDK主要分为Astra SDK和Orbbec SDK，具体需要根据实际选用的模组来选择基础SDK进行集成，AstraSDK和OrbbecSDK 可通过耀世注册3D视觉开发者社区下载最新版本，详细的集成步骤可参考基础SDK 部分。

集成算法SDK

Orbbec Pose SDK Android版本可通过耀世注册3D视觉开发者社区下载最新版本，详细集成步骤可参考算法SDK 部分。

添加权限

开发时需要在AndroidManifest.xml中给应用申请对应的权限：

<uses-permission android:name="android.permission.READ_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.WRITE_EXTERNAL_STORAGE" />
<uses-permission android:name="android.permission.CAMERA" />

基础SDK¶

Astra SDK¶

要使用Astra SDK，首先需要导入Astra SDK的aar包到您的应用工程中的lib目录下，导入后记得点击Sync Project with Gradle Files进行更新，具体如图所示：

SDK使用示例¶

1.初始化SDK并打开设备

// 创建Astra SDK Context并设置设备状态回调
mAstraContext = new AstraAndroidContext(mContext, new AstraDeviceManagerListener() {
        @Override
        public void onOpenAllDevicesCompleted(Iterable<UsbDevice> iterable) {
        }

        @Override
        public void onOpenDeviceCompleted(UsbDevice usbDevice, boolean b) {
        }

        @Override
        public void onNoDevice() {
        }

        @Override
        public void onPermissionDenied(UsbDevice usbDevice) {
        }
    });
// 初始化SDK
mAstraContext.initialize();
// 打开设备
mAstraContext.openAllDevices();

2.当设备打开成功后（即onOpenAllDevicesCompleted方法回调后）再设置开流配置

ImageStreamMode newMode = null;
// 开启默认Sensor
mStreamSet = StreamSet.open();

// 通过StreamSet创建StreamReader
mStreamReader = mStreamSet.createReader();

// 通过StreamReader获取ColorStream
mColorStream = ColorStream.get(mStreamReader);

// 获取彩色流支持的所有模式
Iterable<ImageStreamMode> colorVideoModes = mColorStream.getAvailableModes();
for (ImageStreamMode mode : colorVideoModes) {
    int X = mode.getWidth();
    int Y = mode.getHeight();
    int fps = mode.getFramesPerSecond();

    Log.d(TAG, " support resolution: " + X + " x " + Y + " fps: " + fps + ", (" + mode.getFormat() + ")");
    // 根据分辨率、格式及帧率进行过滤，得到想要开启的流的模式
    if (X == RESOLUTION_W && Y == RESOLUTION_H && mode.getFormat() == PixelFormat.RGB888 && fps == FPS) {
        newMode = mode;
    }
}
if (newMode != null) {
    Log.d(TAG, "color stream set video mode: " + newMode.toString());
    // 配置过滤得到的彩色开流模式
    mColorStream.setMode(newMode);
}

// 通过StreamReader获取DepthStream
mDepthStream = DepthStream.get(mStreamReader);

// 获取深度流支持的所有模式
Iterable<ImageStreamMode> depthVideoModes = mDepthStream.getAvailableModes();
for (ImageStreamMode mode : depthVideoModes) {
    int X = mode.getWidth();
    int Y = mode.getHeight();
    int fps = mode.getFramesPerSecond();

    Log.d(TAG, " support resolution: " + X + " x " + Y + " fps: " + fps + ", (" + mode.getFormat() + ")");
    // 根据分辨率、格式及帧率进行过滤，得到想要开启的流的模式
    if (X == RESOLUTION_W && Y == RESOLUTION_H && mode.getFormat() == PixelFormat.DEPTH_MM && fps == FPS) {
        newMode = mode;
    }
}
if (newMode != null) {
    Log.d(TAG, "depth stream set video mode: " + newMode.toString());
    // 配置过滤得到的深度流开流模式
    mDepthStream.setMode(newMode);
}

3.通过StreamSet获取相机内参

if (null != mStreamSet) {
    // 获取相机内参
    mCameraParam = mStreamSet.createDeviceController().GetOrbbecCameraParams();
}

4.开启彩色流

mColorStream.start();

5.开启D2C，并打开深度流

// 开启D2C
mDepthStream.setRegistration(true);

// 开启深度流
mDepthStream.start();

6.创建并启动取流线程

if (null == mStreamThread) {
    // 创建取流线程
    mStreamThread = new Thread(mStreamRunnable);
    mStreamThread.setName("StreamThread");
}
// 启动取流线程
mStreamStart = true;
mStreamThread.start();

7.取流

while (mStreamStart) {
    // 刷新流
    Astra.update();
    // 判断是否有新的Frame
    if (mStreamReader.hasNewFrame()) {
        // 通过StreamReader获取ReaderFrame
        ReaderFrame readerFrame = mStreamReader.openFrame();

        // 通过ReaderFrame获取ColorFrame
        ColorFrame colorFrame = ColorFrame.get(readerFrame);

        // 通过ReaderFrame获取DepthFrame
        DepthFrame depthFrame = DepthFrame.get(readerFrame);

        // 释放readerFrame资源
        readerFrame.close();
    }
}

8.停流

// 停止取流线程
mStreamStart = false;
if (null != mStreamThread) {
    try {
        mStreamThread.join();
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
    mStreamThread = null;
}

// 停止彩色流
if (mColorStream != null) {
    mColorStream.stop();
}

// 停止深度流
if (mDepthStream != null) {
    mDepthStream.stop();
}

9.资源释放

try {
    // 释放StreamReader
    if (null != mStreamReader) {
        mStreamReader.destroy();
        mStreamReader = null;
    }

    // 释放StreamSet
    if (null != mStreamSet) {
        mStreamSet.close();
    }
} catch (Exception e) {
    Log.w(TAG, "deInitCamera: " + e.getMessage());
}

// 释放SDK资源
if (null != mAstraContext) {
    mAstraContext.terminate();
    mAstraContext = null;
}

Orbbec SDK¶

要使用Orbbec SDK，首先需要导入Orbbec SDK的aar包到您的应用工程中的lib目录下，导入后记得点击Sync Project with Gradle Files进行更新，具体如图所示：

SDK使用示例¶

1.初始化SDK并设置设备监听

// 初始化SDK
mOBContext = new OBContext(mContext, new DeviceChangedCallback() {
        @Override
        public void onDeviceAttach(DeviceList deviceList) {
            // 设备连接回调
            try {
                // 释放deviceList资源
                deviceList.close();
            } catch (Exception e) {
                Log.e(TAG, "onDeviceAttach: " + e.getMessage());
            }
        }

        @Override
        public void onDeviceDetach(DeviceList deviceList) {
            // 设备断开连接回调
            try {
                // 重要提示，如果断开连接的设备是当前创建的Device，
                // 务必要在此处释放Device和Pipeline资源，以及与之相关的任何需要释放的资源

                // 释放deviceList资源
                deviceList.close();
            } catch (Exception e) {
                Log.e(TAG, "onDeviceDetach: " + e.getMessage());
            }
        }
    });

2.创建Device并通过Device创建Pipeline

if (null == mPipeline) {
    // 通过回调上来的deviceList创建Device
    mDevice = deviceList.getDevice(0);

    // 通过Device创建Pipeline
    mPipeline = new Pipeline(mDevice);
}

3.配置开流相关信息

// 创建Config
mConfig = new Config();

// 配置开启硬件D2C
mConfig.setAlignMode(AlignMode.ALIGN_D2C_HW_ENABLE);


StreamProfile colorStreamProfile = null;
StreamProfile depthStreamProfile = null;
// 获取彩色流配置列表
StreamProfileList colorProfileList = mPipeline.getStreamProfileList(SensorType.COLOR);

// 获取深度流配置列表
StreamProfileList depthProfileList = mPipeline.getStreamProfileList(SensorType.DEPTH);

if (null != colorProfileList) {
    // 根据分辨率、格式、帧率获取指定的彩色流配置
    colorStreamProfile = colorProfileList.getVideoStreamProfile(colorW, colorH, Format.RGB888, FPS);
    colorProfileList.close();
}
if (null != colorStreamProfile) {
    // 使能指定的彩色流配置
    mConfig.enableStream(colorStreamProfile);
    Log.i(TAG, "initCamera: " + printStreamProfile(colorStreamProfile.as(StreamType.VIDEO)));
    colorStreamProfile.close();
}

if (null != depthProfileList) {
    // 根据分辨率、格式、帧率获取指定的深度流配置
    depthStreamProfile = depthProfileList.getVideoStreamProfile(RESOLUTION_W, RESOLUTION_H, Format.Y16, FPS);
    depthProfileList.close();
}
if (null != depthStreamProfile) {
    // 使能指定的深度流配置
    mConfig.enableStream(depthStreamProfile);
    Log.i(TAG, "initCamera: " + printStreamProfile(depthStreamProfile.as(StreamType.VIDEO)));
    depthStreamProfile.close();
}
if (null != mDevice) {
    // 关闭自动曝光优先以提高帧率
    if (mDevice.isPropertySupported(DeviceProperty.OB_PROP_COLOR_AUTO_EXPOSURE_PRIORITY_INT, OB_PERMISSION_WRITE)) {
        mDevice.setPropertyValueI(DeviceProperty.OB_PROP_COLOR_AUTO_EXPOSURE_PRIORITY_INT, 0);
    }
}

// 开启帧同步
try {
    mPipeline.enableFrameSync();
} catch (Exception e) {
    Log.w(TAG, "initCamera: " + e.getMessage());
}

4.创建FrameSet回调并通过Pipeline开流

// 创建FrameSet回调，在该回调中尽量不要处理较多耗时操作，否则可能导致SDK内部出现丢帧情况
private FrameSetCallback mFrameSetCallback = (frameSet) -> {
    // frameSet资源释放
    frameSet.close();
};

// 通过Config的配置进行开流
mPipeline.start(mConfig, mFrameSetCallback);

5.获取相机内参

if (null != mPipeline) {
    mCameraParam = mPipeline.getCameraParam();
}

6.停流

if (null != mPipeline) {
    mPipeline.stop();
}

7.资源释放

// 释放Config资源
if (null != mConfig) {
    mConfig.close();
    mConfig = null;
}
// 释放Pipeline资源
if (null != mPipeline) {
    mPipeline.close();
    mPipeline = null;
}
// 释放Device资源
if (null != mDevice) {
    mDevice.close();
    mDevice = null;
}
// 释放SDK资源
if (null != mOBContext) {
    mOBContext.close();
    mOBContext = null;
}

算法SDK¶

要使用Orbbec Pose SDK，首先需要导入Orbbec Pose SDK的aar包到您的应用工程中的lib目录下，导入后记得点击Sync Project with Gradle Files进行更新，具体如图所示：

SDK使用示例¶

1.设置License并初始化SDK

try {
    // 将申请的License中的appKey， appSecret， authCode设置进来
    Obt.setLicense(appKey, appSecret, authCode);

    // 创建SDK并进行初始化，如果License激活失败则抛异常
    mObtContext = new ObtAndroidContext(context);
    mObtContext.initialize();
} catch (Exception e) {
    Log.w(TAG, "init: " + e.getMessage());
}

2.创建Tracker

mTracker = Tracker.create();

3.通过获取到的相机内参构建Calibration并进行设置

// 通过AstraSDK获取到的相机内参构建Calibration的方式
private Calibration loadCalibration(OrbbecCameraParams cameraParam, boolean isRotate) {
    if (null == cameraParam) {
        Log.w(TAG, "loadCalibration: CameraParam is null, use default calibration!");
        return (isRotate ? GlobalDef.CALIBRATION_ROTATE : GlobalDef.CALIBRATION);
    }
    Calibration calibration = new Calibration();
    float[] l_intr_p = cameraParam.l_intr_p;
    calibration.colorWidth = isRotate ? RESOLUTION_H : RESOLUTION_W;
    calibration.colorHeight = isRotate ? RESOLUTION_W : RESOLUTION_H;
    calibration.depthWidth = isRotate ? RESOLUTION_H : RESOLUTION_W;
    calibration.depthHeight = isRotate ? RESOLUTION_W : RESOLUTION_H;
    calibration.depthUnit = DepthUnit.DEPTH_1_MM.toNative();

    calibration.fx = isRotate ? l_intr_p[1] : l_intr_p[0];
    calibration.fy = isRotate ? l_intr_p[0] : l_intr_p[1];
    calibration.cx = isRotate ? l_intr_p[3] : l_intr_p[2];
    calibration.cy = isRotate ? l_intr_p[2] : l_intr_p[3];

    if (calibration.fx <= 0 || calibration.fy <= 0 || calibration.cx <= 0 || calibration.cy <= 0 ||
            Float.isNaN(calibration.fx) || Float.isNaN(calibration.fy) || Float.isNaN(calibration.cx) || Float.isNaN(calibration.cy)) {
        return (isRotate ? GlobalDef.CALIBRATION_ROTATE : GlobalDef.CALIBRATION);
    }

    Log.d(TAG, "loadCalibration: fx:" + calibration.fx + " fy:" + calibration.fy +
            " cx:" + calibration.cx + " cy:" + calibration.cy +
            " colorWidth:" + calibration.colorWidth + " colorHeight:" + calibration.colorHeight +
            " depthWidth:" + calibration.depthWidth + " depthHeight:" + calibration.depthHeight +
            " depthUnit:" + calibration.depthUnit);
    return calibration;
}

// 通过OrbbecSDK获取到的相机内参构建Calibration的方式
private Calibration loadCalibration(CameraParam cameraParam, boolean isRotate) {
    if (null == cameraParam) {
        Log.w(TAG, "loadCalibration: CameraParam is null, use default calibration!");
        return (isRotate ? GlobalDef.CALIBRATION_ROTATE : GlobalDef.CALIBRATION);
    }
    // 获取彩色相机内参
    CameraIntrinsic colorIntrinsic = cameraParam.getColorIntrinsic();
    // 获取深度相机内参
    CameraIntrinsic depthIntrinsic = cameraParam.getDepthIntrinsic();
    Calibration calibration = new Calibration();
    calibration.colorWidth = isRotate ? colorIntrinsic.getHeight() : colorIntrinsic.getWidth();
    calibration.colorHeight = isRotate ? colorIntrinsic.getWidth() : colorIntrinsic.getHeight();
    calibration.depthWidth = isRotate ? depthIntrinsic.getHeight() : depthIntrinsic.getWidth();
    calibration.depthHeight = isRotate ? depthIntrinsic.getWidth() : depthIntrinsic.getHeight();
    calibration.depthUnit = DepthUnit.DEPTH_1_MM.toNative();

    calibration.fx = isRotate ? depthIntrinsic.getFy() : depthIntrinsic.getFx();
    calibration.fy = isRotate ? depthIntrinsic.getFx() : depthIntrinsic.getFy();
    calibration.cx = isRotate ? depthIntrinsic.getCy() : depthIntrinsic.getCx();
    calibration.cy = isRotate ? depthIntrinsic.getCx() : depthIntrinsic.getCy();

    if (calibration.fx <= 0 || calibration.fy <= 0 || calibration.cx <= 0 || calibration.cy <= 0 ||
            Float.isNaN(calibration.fx) || Float.isNaN(calibration.fy) || Float.isNaN(calibration.cx) || Float.isNaN(calibration.cy)) {
        return (isRotate ? GlobalDef.CALIBRATION_ROTATE : GlobalDef.CALIBRATION);
    }

    Log.d(TAG, "loadCalibration: fx:" + calibration.fx + " fy:" + calibration.fy +
            " cx:" + calibration.cx + " cy:" + calibration.cy +
            " colorWidth:" + calibration.colorWidth + " colorHeight:" + calibration.colorHeight +
            " depthWidth:" + calibration.depthWidth + " depthHeight:" + calibration.depthHeight +
            " depthUnit:" + calibration.depthUnit);
    return calibration;
}

// 设置Calibration到算法模块
mTracker.setCalibration(mCalibration);

4.设置跟踪模式

mTracker.setMode(mTrackMode);

5.创建并启动跟踪线程

mPoseTrackerThread = new Thread(mPoseTrackerRunnable);
mPoseTrackerThread.setName("PoseTrackerThread");
mIsTracking = true;
mPoseTrackerThread.start();

6.算法数据封装

// 彩色Image封装
Image colorImage = Image.create(ImageFormat.RGB888, w, h, stride, data);
// 深度Image封装
Image depthImage = Image.create(ImageFormat.DEPTH16, w, h, stride, data);

//使用完后记得资源释放
colorImage.release();
depthImage.release();

7.算法处理

if (mSkeletonMode == GlobalDef.SKELETON_3D) {
    // 3D骨架模式
    frame = mTracker.process(colorImage, depthImage);
} else {
    // 2D骨架模式
    frame = mTracker.process(colorImage);
}
// frame结果使用完后记得资源释放
frame.release();

8.设置算法滤波参数

mTracker.setSmoothingFactor(smoothingFactor);

9.资源释放

// 释放Tracker资源
if (null != mTracker) {
    mTracker.release();
    mTracker = null;
}

// 释放SDK资源
if (null != mObtContext) {
    mObtContext.terminate();
    mObtContext = null;
}