Camera#使用的camera2 api，在so层实现，写完之后写Yolo推理的时候才发现，有个大佬写好的项目和我的需求简直完美契合，先贴一下我写的ndk camera代码吧。

1#include 2#include 3#include 4

5#include 6

7#include 8#include 9

10#include 11

12#define LOG_TAG "Native_Camera"13#define LOGI(...) __android_log_print(ANDROID_LOG_INFO,LOG_TAG,__VA_ARGS__)14#define LOGE(...) __android_log_print(ANDROID_LOG_ERROR,LOG_TAG,__VA_ARGS__)15

16static ANativeWindow *theNativeWindow;17static ACameraDevice *cameraDevice;18static ACaptureRequest *captureRequest;19static ACameraOutputTarget *cameraOutputTarget;20static ACaptureSessionOutput *sessionOutput;21static ACaptureSessionOutputContainer *captureSessionOutputContainer;22static ACameraCaptureSession *captureSession;23

24static ACameraDevice_StateCallbacks deviceStateCallbacks;25static ACameraCaptureSession_stateCallbacks captureSessionStateCallbacks;26

27static void camera_device_on_disconnected(void *context, ACameraDevice *device) {28 LOGI("Camera(id: %s) is diconnected.\n", ACameraDevice_getId(device));29}30

31static void camera_device_on_error(void *context, ACameraDevice *device, int error) {32 LOGE("Error(code: %d) on Camera(id: %s).\n", error, ACameraDevice_getId(device));33}34

35static void capture_session_on_ready(void *context, ACameraCaptureSession *session) {36 LOGI("Session is ready. %p\n", session);37}38

39static void capture_session_on_active(void *context, ACameraCaptureSession *session) {40 LOGI("Session is activated. %p\n", session);41}42

43static void capture_session_on_closed(void *context, ACameraCaptureSession *session) {44 LOGI("Session is closed. %p\n", session);45}46

47

48static void openCamera(ACameraDevice_request_template templateId)49{50 ACameraIdList *cameraIdList = NULL;51 ACameraMetadata *cameraMetadata = NULL;52

53 const char *selectedCameraId = NULL;54 camera_status_t camera_status = ACAMERA_OK;55 ACameraManager *cameraManager = ACameraManager_create();56

57 camera_status = ACameraManager_getCameraIdList(cameraManager, &cameraIdList);58 if (camera_status != ACAMERA_OK) {59 LOGE("Failed to get camera id list (reason: %d)\n", camera_status);60 return;61 }62

63 if (cameraIdList->numCameras < 1) {64 LOGE("No camera device detected.\n");65 return;66 }67 //实测三星S22 numCameras = 468 selectedCameraId = cameraIdList->cameraIds[0];69

70 LOGI("Trying to open Camera2 (id: %s, num of camera : %d)\n", selectedCameraId,71 cameraIdList->numCameras);72

73 camera_status = ACameraManager_getCameraCharacteristics(cameraManager, selectedCameraId,74 &cameraMetadata);75

76 if (camera_status != ACAMERA_OK) {77 LOGE("Failed to get camera meta data of ID:%s\n", selectedCameraId);78 }79

80 deviceStateCallbacks.onDisconnected = camera_device_on_disconnected;81 deviceStateCallbacks.onError = camera_device_on_error;82

83 camera_status = ACameraManager_openCamera(cameraManager, selectedCameraId,84 &deviceStateCallbacks, &cameraDevice);85

86 if (camera_status != ACAMERA_OK) {87 LOGE("Failed to open camera device (id: %s)\n", selectedCameraId);88 }89

90 camera_status = ACameraDevice_createCaptureRequest(cameraDevice, templateId,91 &captureRequest);92

93 if (camera_status != ACAMERA_OK) {94 LOGE("Failed to create preview capture request (id: %s)\n", selectedCameraId);95 }96

97 ACaptureSessionOutputContainer_create(&captureSessionOutputContainer);98

99 captureSessionStateCallbacks.onReady = capture_session_on_ready;100 captureSessionStateCallbacks.onActive = capture_session_on_active;101 captureSessionStateCallbacks.onClosed = capture_session_on_closed;102

103 ACameraMetadata_free(cameraMetadata);104 ACameraManager_deleteCameraIdList(cameraIdList);105 ACameraManager_delete(cameraManager);106}107

108static void closeCamera(void)109{110 camera_status_t camera_status = ACAMERA_OK;111

112 if (captureRequest != NULL) {113 ACaptureRequest_free(captureRequest);114 captureRequest = NULL;115 }116

117 if (cameraOutputTarget != NULL) {118 ACameraOutputTarget_free(cameraOutputTarget);119 cameraOutputTarget = NULL;120 }121

122 if (cameraDevice != NULL) {123 camera_status = ACameraDevice_close(cameraDevice);124

125 if (camera_status != ACAMERA_OK) {126 LOGE("Failed to close CameraDevice.\n");127 }128 cameraDevice = NULL;129 }130

131 if (sessionOutput != NULL) {132 ACaptureSessionOutput_free(sessionOutput);133 sessionOutput = NULL;134 }135

136 if (captureSessionOutputContainer != NULL) {137 ACaptureSessionOutputContainer_free(captureSessionOutputContainer);138 captureSessionOutputContainer = NULL;139 }140

141 LOGI("Close Camera\n");142}143

144static ANativeWindow *extraViewWindow;145static ACaptureRequest *extraViewCaptureRequest;146static ACameraOutputTarget *extraViewOutputTarget;147static ACaptureSessionOutput *extraViewSessionOutput;148

149

150extern "C" JNIEXPORT void JNICALL Java_com_yuuki_Native_1Camera_Utils_Camera2Helper_startPreview(JNIEnv *env,151 jclass clazz,152 jobject surface) {153 theNativeWindow = ANativeWindow_fromSurface(env, surface);154

155 openCamera(TEMPLATE_PREVIEW);156

157 LOGI("Surface is prepared in %p.\n", surface);158

159 ACameraOutputTarget_create(theNativeWindow, &cameraOutputTarget);160 ACaptureRequest_addTarget(captureRequest, cameraOutputTarget);161

162 ACaptureSessionOutput_create(theNativeWindow, &sessionOutput);163 ACaptureSessionOutputContainer_add(captureSessionOutputContainer, sessionOutput);164

165 ACameraDevice_createCaptureSession(cameraDevice, captureSessionOutputContainer,166 &captureSessionStateCallbacks, &captureSession);167

168 ACameraCaptureSession_setRepeatingRequest(captureSession, NULL, 1, &captureRequest, NULL);169

170}171

172extern "C" JNIEXPORT void JNICALL Java_com_yuuki_Native_1Camera_Utils_Camera2Helper_stopPreview(JNIEnv *env,173 jclass clazz) {174 closeCamera();175 if (theNativeWindow != NULL) {176 ANativeWindow_release(theNativeWindow);177 theNativeWindow = NULL;178 }179}180

181extern "C" JNIEXPORT void JNICALL Java_com_yuuki_Native_1Camera_Utils_Camera2Helper_startExtraView(JNIEnv *env,182 jclass clazz,183 jobject surface) {184

185 /* Assuming that camera preview has already been started */186 extraViewWindow = ANativeWindow_fromSurface(env, surface);187

188 LOGI("Extra view surface is prepared in %p.\n", surface);189 ACameraCaptureSession_stopRepeating(captureSession);190

191 ACameraDevice_createCaptureRequest(cameraDevice, TEMPLATE_STILL_CAPTURE,192 &extraViewCaptureRequest);193

194 ACameraOutputTarget_create(extraViewWindow, &extraViewOutputTarget);195 ACaptureRequest_addTarget(extraViewCaptureRequest, extraViewOutputTarget);196

197 ACaptureSessionOutput_create(extraViewWindow, &extraViewSessionOutput);198 ACaptureSessionOutputContainer_add(captureSessionOutputContainer,199 extraViewSessionOutput);200

201 /* Not sure why the session should be recreated.202 * Otherwise, the session state goes to ready */203 ACameraCaptureSession_close(captureSession);204 ACameraDevice_createCaptureSession(cameraDevice, captureSessionOutputContainer,205 &captureSessionStateCallbacks, &captureSession);206

207 ACaptureRequest *requests[2];208 requests[0] = captureRequest;209 requests[1] = extraViewCaptureRequest;210

211 ACameraCaptureSession_setRepeatingRequest(captureSession, NULL, 2, requests, NULL);212}213

214extern "C" JNIEXPORT void JNICALL Java_com_yuuki_Native_1Camera_Utils_Camera2Helper_stopExtraView(JNIEnv *env,215 jclass clazz) {216

217 ACameraCaptureSession_stopRepeating(captureSession);218

219 ACaptureSessionOutputContainer_remove(captureSessionOutputContainer, extraViewSessionOutput);220

221 ACaptureRequest_removeTarget(extraViewCaptureRequest, extraViewOutputTarget);222

223 ACameraOutputTarget_free(extraViewOutputTarget);224 ACaptureSessionOutput_free(extraViewSessionOutput);225 ACaptureRequest_free(extraViewCaptureRequest);226

227 ACameraCaptureSession_setRepeatingRequest(captureSession, NULL, 1, &captureRequest, NULL);228

229 if (extraViewWindow != NULL) {230 ANativeWindow_release(extraViewWindow);231 extraViewWindow = NULL;232 LOGI("Extra view surface is released\n");233

234 }235}1package com.yuuki.Native_Camera.Utils;2

3import android.view.Surface;4

5public class Camera2Helper {6 static {7 System.loadLibrary("Native_Camera");8 }9

10 private native void startPreview(Surface surface);11 private native void stopPreview();12 private native void startExtraView();13 private native void stopExtraView();14

15 public void start(Surface surface) {16 startPreview(surface);17 }18

19 public void stop() {20 stopPreview();21 }22

23}1package com.yuuki.Native_Camera;2

3import androidx.appcompat.app.AppCompatActivity;4

5import android.os.Bundle;6import android.util.Log;7import android.view.SurfaceHolder;8import android.view.SurfaceView;9

10import com.yuuki.Native_Camera.Utils.Camera2Helper;11import com.yuuki.Native_Camera.databinding.ActivityMainBinding;12

13

14public class MainActivity extends AppCompatActivity {15

16 static final String TAG = "JAVA_Camera";17 private ActivityMainBinding binding;18

19 SurfaceView surfaceView;20 SurfaceHolder surfaceHolder;21

22 @Override23 protected void onCreate(Bundle savedInstanceState) {24 super.onCreate(savedInstanceState);25

26 binding = ActivityMainBinding.inflate(getLayoutInflater());27 setContentView(binding.getRoot());28 surfaceView = binding.surfaceview;29 surfaceHolder = surfaceView.getHolder();30

31 Camera2Helper camera = new Camera2Helper();32 surfaceHolder.addCallback(new SurfaceHolder.Callback() {33 @Override34 public void surfaceCreated(SurfaceHolder holder) {35

36 Log.v(TAG, "surface created.");37 camera.start(holder.getSurface());38 }39

40 @Override41 public void surfaceDestroyed(SurfaceHolder holder) {42 camera.stop();43 }44

45 @Override46 public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {47 Log.v(TAG, "format=" + format + " w/h : (" + width + ", " + height + ")");48 }49 });50

51 }52 @Override53 protected void onDestroy() {54 Camera2Helper camera = new Camera2Helper();55 camera.stop();56 super.onDestroy();57 }58

59}为什么自己写呢，因为不想用opencv库，但是后来想起来要在每一帧上面画东西，想想还是得用库哈哈哈哈，一阵白忙活。后来逛Github发现FeiGeChuanShu大佬有个项目简直太满足我的使用场景了，于是干脆直接抄了哈哈哈。

这里贴一下大佬的NdkCamera部分代码

1#include "ndkcamera.h"2#include 3#include 4#include 5#include "mat.h"6

7static void onDisconnected(void* context, ACameraDevice* device)8{9 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onDisconnected %p", device);10}11

12static void onError(void* context, ACameraDevice* device, int error)13{14 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onError %p %d", device, error);15}16

17static void onImageAvailable(void* context, AImageReader* reader)18{19// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onImageAvailable %p", reader);20

21 AImage* image = 0;22 media_status_t status = AImageReader_acquireLatestImage(reader, &image);23

24 if (status != AMEDIA_OK)25 {26 // error27 return;28 }29

30 int32_t format;31 AImage_getFormat(image, &format);32

33 // assert format == AIMAGE_FORMAT_YUV_420_88834

35 int32_t width = 0;36 int32_t height = 0;37 AImage_getWidth(image, &width);38 AImage_getHeight(image, &height);39

40 int32_t y_pixelStride = 0;41 int32_t u_pixelStride = 0;42 int32_t v_pixelStride = 0;43 AImage_getPlanePixelStride(image, 0, &y_pixelStride);44 AImage_getPlanePixelStride(image, 1, &u_pixelStride);45 AImage_getPlanePixelStride(image, 2, &v_pixelStride);46

47 int32_t y_rowStride = 0;48 int32_t u_rowStride = 0;49 int32_t v_rowStride = 0;50 AImage_getPlaneRowStride(image, 0, &y_rowStride);51 AImage_getPlaneRowStride(image, 1, &u_rowStride);52 AImage_getPlaneRowStride(image, 2, &v_rowStride);53

54 uint8_t* y_data = 0;55 uint8_t* u_data = 0;56 uint8_t* v_data = 0;57 int y_len = 0;58 int u_len = 0;59 int v_len = 0;60 AImage_getPlaneData(image, 0, &y_data, &y_len);61 AImage_getPlaneData(image, 1, &u_data, &u_len);62 AImage_getPlaneData(image, 2, &v_data, &v_len);63

64 if (u_data == v_data + 1 && v_data == y_data + width * height && y_pixelStride == 1 && u_pixelStride == 2 && v_pixelStride == 2 && y_rowStride == width && u_rowStride == width && v_rowStride == width)65 {66 // already nv21 :)67 ((NdkCamera*)context)->on_image((unsigned char*)y_data, (int)width, (int)height);68 }69 else70 {71 // construct nv2172 unsigned char* nv21 = new unsigned char[width * height + width * height / 2];73 {74 // Y75 unsigned char* yptr = nv21;76 for (int y=0; y

87 // UV88 unsigned char* uvptr = nv21 + width * height;89 for (int y=0; y

104 ((NdkCamera*)context)->on_image((unsigned char*)nv21, (int)width, (int)height);105

106 delete[] nv21;107 }108

109 AImage_delete(image);110}111

112static void onSessionActive(void* context, ACameraCaptureSession *session)113{114 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionActive %p", session);115}116

117static void onSessionReady(void* context, ACameraCaptureSession *session)118{119 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionReady %p", session);120}121

122static void onSessionClosed(void* context, ACameraCaptureSession *session)123{124 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onSessionClosed %p", session);125}126

127void onCaptureFailed(void* context, ACameraCaptureSession* session, ACaptureRequest* request, ACameraCaptureFailure* failure)128{129 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureFailed %p %p %p", session, request, failure);130}131

132void onCaptureSequenceCompleted(void* context, ACameraCaptureSession* session, int sequenceId, int64_t frameNumber)133{134 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceCompleted %p %d %ld", session, sequenceId, frameNumber);135}136

137void onCaptureSequenceAborted(void* context, ACameraCaptureSession* session, int sequenceId)138{139 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureSequenceAborted %p %d", session, sequenceId);140}141

142void onCaptureCompleted(void* context, ACameraCaptureSession* session, ACaptureRequest* request, const ACameraMetadata* result)143{144// __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "onCaptureCompleted %p %p %p", session, request, result);145}146

147NdkCamera::NdkCamera()148{149 camera_facing = 0;150 camera_orientation = 0;151

152 camera_manager = 0;153 camera_device = 0;154 image_reader = 0;155 image_reader_surface = 0;156 image_reader_target = 0;157 capture_request = 0;158 capture_session_output_container = 0;159 capture_session_output = 0;160 capture_session = 0;161

162

163 // setup imagereader and its surface164 {165 AImageReader_new(640, 480, AIMAGE_FORMAT_YUV_420_888, /*maxImages*/2, &image_reader);166

167 AImageReader_ImageListener listener;168 listener.context = this;169 listener.onImageAvailable = onImageAvailable;170

171 AImageReader_setImageListener(image_reader, &listener);172

173 AImageReader_getWindow(image_reader, &image_reader_surface);174

175 ANativeWindow_acquire(image_reader_surface);176 }177}178

179NdkCamera::~NdkCamera()180{181 close();182

183 if (image_reader)184 {185 AImageReader_delete(image_reader);186 image_reader = 0;187 }188

189 if (image_reader_surface)190 {191 ANativeWindow_release(image_reader_surface);192 image_reader_surface = 0;193 }194}195

196int NdkCamera::open(int _camera_facing)197{198 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open");199

200 camera_facing = _camera_facing;201

202 camera_manager = ACameraManager_create();203

204 // find front camera205 std::string camera_id;206 {207 ACameraIdList* camera_id_list = 0;208 ACameraManager_getCameraIdList(camera_manager, &camera_id_list);209

210 for (int i = 0; i < camera_id_list->numCameras; ++i)211 {212 const char* id = camera_id_list->cameraIds[i];213 ACameraMetadata* camera_metadata = 0;214 ACameraManager_getCameraCharacteristics(camera_manager, id, &camera_metadata);215

216 // query faceing217 acamera_metadata_enum_android_lens_facing_t facing = ACAMERA_LENS_FACING_FRONT;218 {219 ACameraMetadata_const_entry e = { 0 };220 ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_LENS_FACING, &e);221 facing = (acamera_metadata_enum_android_lens_facing_t)e.data.u8[0];222 }223

224 if (camera_facing == 0 && facing != ACAMERA_LENS_FACING_FRONT)225 {226 ACameraMetadata_free(camera_metadata);227 continue;228 }229

230 if (camera_facing == 1 && facing != ACAMERA_LENS_FACING_BACK)231 {232 ACameraMetadata_free(camera_metadata);233 continue;234 }235

236 camera_id = id;237

238 // query orientation239 int orientation = 0;240 {241 ACameraMetadata_const_entry e = { 0 };242 ACameraMetadata_getConstEntry(camera_metadata, ACAMERA_SENSOR_ORIENTATION, &e);243

244 orientation = (int)e.data.i32[0];245 }246

247 camera_orientation = orientation;248

249 ACameraMetadata_free(camera_metadata);250

251 break;252 }253

254 ACameraManager_deleteCameraIdList(camera_id_list);255 }256

257 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "open %s %d", camera_id.c_str(), camera_orientation);258

259 // open camera260 {261 ACameraDevice_StateCallbacks camera_device_state_callbacks;262 camera_device_state_callbacks.context = this;263 camera_device_state_callbacks.onDisconnected = onDisconnected;264 camera_device_state_callbacks.onError = onError;265

266 ACameraManager_openCamera(camera_manager, camera_id.c_str(), &camera_device_state_callbacks, &camera_device);267 }268

269 // capture request270 {271 ACameraDevice_createCaptureRequest(camera_device, TEMPLATE_PREVIEW, &capture_request);272

273 ACameraOutputTarget_create(image_reader_surface, &image_reader_target);274 ACaptureRequest_addTarget(capture_request, image_reader_target);275 }276

277 // capture session278 {279 ACameraCaptureSession_stateCallbacks camera_capture_session_state_callbacks;280 camera_capture_session_state_callbacks.context = this;281 camera_capture_session_state_callbacks.onActive = onSessionActive;282 camera_capture_session_state_callbacks.onReady = onSessionReady;283 camera_capture_session_state_callbacks.onClosed = onSessionClosed;284

285 ACaptureSessionOutputContainer_create(&capture_session_output_container);286

287 ACaptureSessionOutput_create(image_reader_surface, &capture_session_output);288

289 ACaptureSessionOutputContainer_add(capture_session_output_container, capture_session_output);290

291 ACameraDevice_createCaptureSession(camera_device, capture_session_output_container, &camera_capture_session_state_callbacks, &capture_session);292

293 ACameraCaptureSession_captureCallbacks camera_capture_session_capture_callbacks;294 camera_capture_session_capture_callbacks.context = this;295 camera_capture_session_capture_callbacks.onCaptureStarted = 0;296 camera_capture_session_capture_callbacks.onCaptureProgressed = 0;297 camera_capture_session_capture_callbacks.onCaptureCompleted = onCaptureCompleted;298 camera_capture_session_capture_callbacks.onCaptureFailed = onCaptureFailed;299 camera_capture_session_capture_callbacks.onCaptureSequenceCompleted = onCaptureSequenceCompleted;300 camera_capture_session_capture_callbacks.onCaptureSequenceAborted = onCaptureSequenceAborted;301 camera_capture_session_capture_callbacks.onCaptureBufferLost = 0;302

303 ACameraCaptureSession_setRepeatingRequest(capture_session, &camera_capture_session_capture_callbacks, 1, &capture_request, nullptr);304 }305

306 return 0;307}308

309void NdkCamera::close()310{311 __android_log_print(ANDROID_LOG_WARN, "NdkCamera", "close");312

313 if (capture_session)314 {315 ACameraCaptureSession_stopRepeating(capture_session);316 ACameraCaptureSession_close(capture_session);317 capture_session = 0;318 }319

320 if (camera_device)321 {322 ACameraDevice_close(camera_device);323 camera_device = 0;324 }325

326 if (capture_session_output_container)327 {328 ACaptureSessionOutputContainer_free(capture_session_output_container);329 capture_session_output_container = 0;330 }331

332 if (capture_session_output)333 {334 ACaptureSessionOutput_free(capture_session_output);335 capture_session_output = 0;336 }337

338 if (capture_request)339 {340 ACaptureRequest_free(capture_request);341 capture_request = 0;342 }343

344 if (image_reader_target)345 {346 ACameraOutputTarget_free(image_reader_target);347 image_reader_target = 0;348 }349

350 if (camera_manager)351 {352 ACameraManager_delete(camera_manager);353 camera_manager = 0;354 }355}356

357void NdkCamera::on_image(const cv::Mat& rgb) const358{359}360

361void NdkCamera::on_image(const unsigned char* nv21, int nv21_width, int nv21_height) const362{363 // rotate nv21364 int w = 0;365 int h = 0;366 int rotate_type = 0;367 {368 if (camera_orientation == 0)369 {370 w = nv21_width;371 h = nv21_height;372 rotate_type = camera_facing == 0 ? 2 : 1;373 }374 if (camera_orientation == 90)375 {376 w = nv21_height;377 h = nv21_width;378 rotate_type = camera_facing == 0 ? 5 : 6;379 }380 if (camera_orientation == 180)381 {382 w = nv21_width;383 h = nv21_height;384 rotate_type = camera_facing == 0 ? 4 : 3;385 }386 if (camera_orientation == 270)387 {388 w = nv21_height;389 h = nv21_width;390 rotate_type = camera_facing == 0 ? 7 : 8;391 }392 }393

394 cv::Mat nv21_rotated(h + h / 2, w, CV_8UC1);395 ncnn::kanna_rotate_yuv420sp(nv21, nv21_width, nv21_height, nv21_rotated.data, w, h, rotate_type);396

397 // nv21_rotated to rgb398 cv::Mat rgb(h, w, CV_8UC3);399 ncnn::yuv420sp2rgb(nv21_rotated.data, w, h, rgb.data);400

401 on_image(rgb);402}403

404static const int NDKCAMERAWINDOW_ID = 233;405

406NdkCameraWindow::NdkCameraWindow() : NdkCamera()407{408 sensor_manager = 0;409 sensor_event_queue = 0;410 accelerometer_sensor = 0;411 win = 0;412

413 accelerometer_orientation = 0;414

415 // sensor416 sensor_manager = ASensorManager_getInstance();417

418 accelerometer_sensor = ASensorManager_getDefaultSensor(sensor_manager, ASENSOR_TYPE_ACCELEROMETER);419}420

421NdkCameraWindow::~NdkCameraWindow()422{423 if (accelerometer_sensor)424 {425 ASensorEventQueue_disableSensor(sensor_event_queue, accelerometer_sensor);426 accelerometer_sensor = 0;427 }428

429 if (sensor_event_queue)430 {431 ASensorManager_destroyEventQueue(sensor_manager, sensor_event_queue);432 sensor_event_queue = 0;433 }434

435 if (win)436 {437 ANativeWindow_release(win);438 }439}440

441void NdkCameraWindow::set_window(ANativeWindow* _win)442{443 if (win)444 {445 ANativeWindow_release(win);446 }447

448 win = _win;449 ANativeWindow_acquire(win);450}451

452void NdkCameraWindow::on_image_render(cv::Mat& rgb) const453{454}455

456void NdkCameraWindow::on_image(const unsigned char* nv21, int nv21_width, int nv21_height) const457{458 // resolve orientation from camera_orientation and accelerometer_sensor459 {460 if (!sensor_event_queue)461 {462 sensor_event_queue = ASensorManager_createEventQueue(sensor_manager, ALooper_prepare(ALOOPER_PREPARE_ALLOW_NON_CALLBACKS), NDKCAMERAWINDOW_ID, 0, 0);463

464 ASensorEventQueue_enableSensor(sensor_event_queue, accelerometer_sensor);465 }466

467 int id = ALooper_pollAll(0, 0, 0, 0);468 if (id == NDKCAMERAWINDOW_ID)469 {470 ASensorEvent e[8];471 ssize_t num_event = 0;472 while (ASensorEventQueue_hasEvents(sensor_event_queue) == 1)473 {474 num_event = ASensorEventQueue_getEvents(sensor_event_queue, e, 8);475 if (num_event < 0)476 break;477 }478

479 if (num_event > 0)480 {481 float acceleration_x = e[num_event - 1].acceleration.x;482 float acceleration_y = e[num_event - 1].acceleration.y;483 float acceleration_z = e[num_event - 1].acceleration.z;484// __android_log_print(ANDROID_LOG_WARN, "NdkCameraWindow", "x = %f, y = %f, z = %f", x, y, z);485

486 if (acceleration_y > 7)487 {488 accelerometer_orientation = 0;489 }490 if (acceleration_x < -7)491 {492 accelerometer_orientation = 90;493 }494 if (acceleration_y < -7)495 {496 accelerometer_orientation = 180;497 }498 if (acceleration_x > 7)499 {500 accelerometer_orientation = 270;501 }502 }503 }504 }505

506 // roi crop and rotate nv21507 int nv21_roi_x = 0;508 int nv21_roi_y = 0;509 int nv21_roi_w = 0;510 int nv21_roi_h = 0;511 int roi_x = 0;512 int roi_y = 0;513 int roi_w = 0;514 int roi_h = 0;515 int rotate_type = 0;516 int render_w = 0;517 int render_h = 0;518 int render_rotate_type = 0;519 {520 int win_w = ANativeWindow_getWidth(win);521 int win_h = ANativeWindow_getHeight(win);522

523 if (accelerometer_orientation == 90 || accelerometer_orientation == 270)524 {525 std::swap(win_w, win_h);526 }527

528 const int final_orientation = (camera_orientation + accelerometer_orientation) % 360;529

530 if (final_orientation == 0 || final_orientation == 180)531 {532 if (win_w * nv21_height > win_h * nv21_width)533 {534 roi_w = nv21_width;535 roi_h = (nv21_width * win_h / win_w) / 2 * 2;536 roi_x = 0;537 roi_y = ((nv21_height - roi_h) / 2) / 2 * 2;538 }539 else540 {541 roi_h = nv21_height;542 roi_w = (nv21_height * win_w / win_h) / 2 * 2;543 roi_x = ((nv21_width - roi_w) / 2) / 2 * 2;544 roi_y = 0;545 }546

547 nv21_roi_x = roi_x;548 nv21_roi_y = roi_y;549 nv21_roi_w = roi_w;550 nv21_roi_h = roi_h;551 }552 if (final_orientation == 90 || final_orientation == 270)553 {554 if (win_w * nv21_width > win_h * nv21_height)555 {556 roi_w = nv21_height;557 roi_h = (nv21_height * win_h / win_w) / 2 * 2;558 roi_x = 0;559 roi_y = ((nv21_width - roi_h) / 2) / 2 * 2;560 }561 else562 {563 roi_h = nv21_width;564 roi_w = (nv21_width * win_w / win_h) / 2 * 2;565 roi_x = ((nv21_height - roi_w) / 2) / 2 * 2;566 roi_y = 0;567 }568

569 nv21_roi_x = roi_y;570 nv21_roi_y = roi_x;571 nv21_roi_w = roi_h;572 nv21_roi_h = roi_w;573 }574

575 if (camera_facing == 0)576 {577 if (camera_orientation == 0 && accelerometer_orientation == 0)578 {579 rotate_type = 2;580 }581 if (camera_orientation == 0 && accelerometer_orientation == 90)582 {583 rotate_type = 7;584 }585 if (camera_orientation == 0 && accelerometer_orientation == 180)586 {587 rotate_type = 4;588 }589 if (camera_orientation == 0 && accelerometer_orientation == 270)590 {591 rotate_type = 5;592 }593 if (camera_orientation == 90 && accelerometer_orientation == 0)594 {595 rotate_type = 5;596 }597 if (camera_orientation == 90 && accelerometer_orientation == 90)598 {599 rotate_type = 2;600 }601 if (camera_orientation == 90 && accelerometer_orientation == 180)602 {603 rotate_type = 7;604 }605 if (camera_orientation == 90 && accelerometer_orientation == 270)606 {607 rotate_type = 4;608 }609 if (camera_orientation == 180 && accelerometer_orientation == 0)610 {611 rotate_type = 4;612 }613 if (camera_orientation == 180 && accelerometer_orientation == 90)614 {615 rotate_type = 5;616 }617 if (camera_orientation == 180 && accelerometer_orientation == 180)618 {619 rotate_type = 2;620 }621 if (camera_orientation == 180 && accelerometer_orientation == 270)622 {623 rotate_type = 7;624 }625 if (camera_orientation == 270 && accelerometer_orientation == 0)626 {627 rotate_type = 7;628 }629 if (camera_orientation == 270 && accelerometer_orientation == 90)630 {631 rotate_type = 4;632 }633 if (camera_orientation == 270 && accelerometer_orientation == 180)634 {635 rotate_type = 5;636 }637 if (camera_orientation == 270 && accelerometer_orientation == 270)638 {639 rotate_type = 2;640 }641 }642 else643 {644 if (final_orientation == 0)645 {646 rotate_type = 1;647 }648 if (final_orientation == 90)649 {650 rotate_type = 6;651 }652 if (final_orientation == 180)653 {654 rotate_type = 3;655 }656 if (final_orientation == 270)657 {658 rotate_type = 8;659 }660 }661

662 if (accelerometer_orientation == 0)663 {664 render_w = roi_w;665 render_h = roi_h;666 render_rotate_type = 1;667 }668 if (accelerometer_orientation == 90)669 {670 render_w = roi_h;671 render_h = roi_w;672 render_rotate_type = 8;673 }674 if (accelerometer_orientation == 180)675 {676 render_w = roi_w;677 render_h = roi_h;678 render_rotate_type = 3;679 }680 if (accelerometer_orientation == 270)681 {682 render_w = roi_h;683 render_h = roi_w;684 render_rotate_type = 6;685 }686 }687

688 // crop and rotate nv21689 cv::Mat nv21_croprotated(roi_h + roi_h / 2, roi_w, CV_8UC1);690 {691 const unsigned char* srcY = nv21 + nv21_roi_y * nv21_width + nv21_roi_x;692 unsigned char* dstY = nv21_croprotated.data;693 ncnn::kanna_rotate_c1(srcY, nv21_roi_w, nv21_roi_h, nv21_width, dstY, roi_w, roi_h, roi_w, rotate_type);694

695 const unsigned char* srcUV = nv21 + nv21_width * nv21_height + nv21_roi_y * nv21_width / 2 + nv21_roi_x;696 unsigned char* dstUV = nv21_croprotated.data + roi_w * roi_h;697 ncnn::kanna_rotate_c2(srcUV, nv21_roi_w / 2, nv21_roi_h / 2, nv21_width, dstUV, roi_w / 2, roi_h / 2, roi_w, rotate_type);698 }699

700 // nv21_croprotated to rgb701 cv::Mat rgb(roi_h, roi_w, CV_8UC3);702 ncnn::yuv420sp2rgb(nv21_croprotated.data, roi_w, roi_h, rgb.data);703

704 on_image_render(rgb);705

706 // rotate to native window orientation707 cv::Mat rgb_render(render_h, render_w, CV_8UC3);708 ncnn::kanna_rotate_c3(rgb.data, roi_w, roi_h, rgb_render.data, render_w, render_h, render_rotate_type);709

710 ANativeWindow_setBuffersGeometry(win, render_w, render_h, AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM);711

712 ANativeWindow_Buffer buf;713 ANativeWindow_lock(win, &buf, NULL);714

715 // scale to target size716 if (buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8A8_UNORM || buf.format == AHARDWAREBUFFER_FORMAT_R8G8B8X8_UNORM)717 {718 for (int y = 0; y < render_h; y++)719 {720 const unsigned char* ptr = rgb_render.ptr(y);721 unsigned char* outptr = (unsigned char*)buf.bits + buf.stride * 4 * y;722

723 int x = 0;724#if __ARM_NEON725 for (; x + 7 < render_w; x += 8)726 {727 uint8x8x3_t _rgb = vld3_u8(ptr);728 uint8x8x4_t _rgba;729 _rgba.val[0] = _rgb.val[0];730 _rgba.val[1] = _rgb.val[1];731 _rgba.val[2] = _rgb.val[2];732 _rgba.val[3] = vdup_n_u8(255);733 vst4_u8(outptr, _rgba);734

735 ptr += 24;736 outptr += 32;737 }738#endif // __ARM_NEON739 for (; x < render_w; x++)740 {741 outptr[0] = ptr[0];742 outptr[1] = ptr[1];743 outptr[2] = ptr[2];744 outptr[3] = 255;745

746 ptr += 3;747 outptr += 4;748 }749 }750 }751

752 ANativeWindow_unlockAndPost(win);753}就不过多解读了，很多代码都在做格式的转换。只有两个函数是我们需要关注的，分别是onImageAvailable和on_image（on_image_render也可以，但它在另一个类里被重写了），因为我们需要把每一帧都发送给客户端，所以我们的发送操作需要在这两个函数里进行（这两个函数用于服务端的画面渲染，在这里进行很方便，但是会降低帧率，也可以选择提升帧率的方式，但是太麻烦了，而且会产生延迟）。

这里我选择在on_image里进行帧的发送操作，分析代码可以得知在on_image_render(rgb);这个操作之后，图片就只剩下格式转换的操作了，我们直接把此时的Mat压缩成jpeg就可以发送了(直接发也可以的，但是需要分片，操你妈的烦死了这里，我本来直接发RGB数据，但是客户端解析老是出现花屏，画面分割等一堆问题，无奈出此下策)，这里贴一下我的代码吧

1// 自定义哈希函数2struct sockaddr_in_hash {3 size_t operator()(const sockaddr_in& addr) const {4 return std::hash()(addr.sin_addr.s_addr) ^ std::hash()(addr.sin_port);5 }6};7

8// 比较函数9struct sockaddr_in_equal {10 bool operator()(const sockaddr_in& lhs, const sockaddr_in& rhs) const {11 return lhs.sin_family == rhs.sin_family &&12 lhs.sin_addr.s_addr == rhs.sin_addr.s_addr &&13 lhs.sin_port == rhs.sin_port;14 }15};16

17class UdpServer {18public:19 UdpServer(int port) : port(port) {20 sock = socket(AF_INET, SOCK_DGRAM, 0);21 if (sock < 0) {22 __android_log_print(ANDROID_LOG_ERROR, "SOCKET", "Error: Could not create socket - %s", strerror(errno));23 return;24 }25

26 server_addr.sin_family = AF_INET;27 server_addr.sin_addr.s_addr = INADDR_ANY;28 server_addr.sin_port = htons(port);29

30 if (bind(sock, (struct sockaddr*)&server_addr, sizeof(server_addr)) < 0) {31 __android_log_print(ANDROID_LOG_ERROR, "SOCKET", "Error: Could not bind socket - %s", strerror(errno));32 close(sock);33 sock = -1;34 }35

36 // 设置接收超时37 struct timeval timeout;38 timeout.tv_sec = 5; // 5秒超时39 timeout.tv_usec = 0;40 if (setsockopt(sock, SOL_SOCKET, SO_RCVTIMEO, &timeout, sizeof(timeout)) < 0) {41 __android_log_print(ANDROID_LOG_ERROR, "SOCKET", "Error: Could not set socket options - %s", strerror(errno));42 }43

44 listen_thread = std::thread(&UdpServer::listen_for_clients, this);45 }46

47 ~UdpServer() {48 if (sock >= 0) {49 close(sock);50 }51 if (listen_thread.joinable()) {52 listen_thread.join();53 }54 }55

56 void send(const cv::Mat& img) {57 std::lock_guard lock(client_mutex);58 if (sock < 0 || client_addresses.empty() || !is_client) {59 __android_log_print(ANDROID_LOG_ERROR, "SOCKET", "Error: Socket is not initialized or no clients are connected");60 return;61 }62

63 cv::Mat img_rgb;64 cv::cvtColor(img, img_rgb, cv::COLOR_BGR2RGB); // 转换为RGB色彩空间65 std::vector buf;66 std::vector params = {cv::IMWRITE_JPEG_QUALITY, 30}; // 极低的JPEG质量参数 max = 6067 if (!cv::imencode(".jpg", img_rgb, buf, params)) {68 __android_log_print(ANDROID_LOG_ERROR, "ENCODE", "Error: Could not encode image");69 return;70 }71

72 if (buf.size() > 65507) {73 __android_log_print(ANDROID_LOG_ERROR, "ENCODE", "Error: Encoded image size exceeds UDP packet limit");74 return;75 }76

77 for (const auto& client_addr : client_addresses) {78 ssize_t sent_bytes = sendto(sock, buf.data(), buf.size(), 0, (struct sockaddr*)&client_addr, sizeof(client_addr));79 if (sent_bytes < 0) {80 __android_log_print(ANDROID_LOG_ERROR, "SEND", "Error: Could not send data - %s", strerror(errno));81 }82 }83 }84

85private:86 void listen_for_clients() {87 while (true) {88 socklen_t len = sizeof(sockaddr_in);89 sockaddr_in client_addr;90 char buffer[1024];91 ssize_t received_bytes = recvfrom(sock, buffer, sizeof(buffer) - 1, 0, (struct sockaddr*)&client_addr, &len);92 if (received_bytes < 0) {93 if (errno == EWOULDBLOCK || errno == EAGAIN) {94 // 超时95 __android_log_print(ANDROID_LOG_WARN, "RECEIVE", "Timeout: No data received from client");96 is_client = false;97 } else {98 __android_log_print(ANDROID_LOG_ERROR, "RECEIVE", "Error: Could not receive data - %s", strerror(errno));99 }100 } else {101 buffer[received_bytes] = '\0';102 __android_log_print(ANDROID_LOG_INFO, "RECEIVE", "Received message from client: %s", buffer);103 __android_log_print(ANDROID_LOG_INFO, "RECEIVE", "Client IP: %s, Port: %d", inet_ntoa(client_addr.sin_addr), ntohs(client_addr.sin_port));104 is_client = true;105 std::lock_guard lock(client_mutex);106 client_addresses.insert(client_addr);107 }108 }109 }110

111 int sock = -1;112 bool is_client = false;113 sockaddr_in server_addr;114 std::unordered_set client_addresses;115 std::thread listen_thread;116 std::mutex client_mutex;117 int port;118};119

120// 实例化 UdpServer121static UdpServer udp_server(8888);细节没的说，考虑的还是比较周到了。剩下的代码基本没改，模型我也直接用的(没有数据集训练集贸)，就不贴了。客户端用的C#写的，也是写的十分简陋，安卓端的客户端以后再写吧。贴下代码。

1using System;2using System.Net;3using System.Net.Sockets;4using System.Text;5using System.Threading;6using System.Windows;7using System.Windows.Media.Imaging;8using System.IO;9

10namespace Motinor11{12 public partial class MainWindow : Window13 {14 private UdpClient udpClient;15 private IPEndPoint serverEndPoint;16 private Thread receiveThread;17 private System.Timers.Timer heartbeatTimer;18

19 public MainWindow()20 {21 InitializeComponent();22 serverEndPoint = new IPEndPoint(IPAddress.Parse("其实内网ip漏了也没事"), 8888); // 替换为服务器的IP和端口23 udpClient = new UdpClient();24 SendInitialMessage();25 StartHeartbeat();26 receiveThread = new Thread(ReceiveImages);27 receiveThread.Start();28 }29

30 private void SendInitialMessage()31 {32 byte[] message = Encoding.UTF8.GetBytes("Hello, server");33 udpClient.Send(message, message.Length, serverEndPoint);34 }35

36 private void StartHeartbeat()37 {38 heartbeatTimer = new System.Timers.Timer(4000); // 每4秒发送一次心跳消息39 heartbeatTimer.Elapsed += (sender, e) => SendHeartbeat();40 heartbeatTimer.AutoReset = true;41 heartbeatTimer.Enabled = true;42 }43

44 private void SendHeartbeat()45 {46 try47 {48 byte[] message = Encoding.UTF8.GetBytes("Heartbeat");49 udpClient.Send(message, message.Length, serverEndPoint);50 }51 catch (Exception ex)52 {53 Console.WriteLine($"Error sending heartbeat: {ex.Message}");54 }55 }56

57 private void ReceiveImages()58 {59 while (true)60 {61 try62 {63 byte[] receivedData = udpClient.Receive(ref serverEndPoint);64 Application.Current.Dispatcher.Invoke(() => UpdateImage(receivedData));65 }66 catch (Exception ex)67 {68 Console.WriteLine($"Error receiving data: {ex.Message}");69 }70 }71 }72

73 private void UpdateImage(byte[] imageData)74 {75 try76 {77 using (MemoryStream ms = new MemoryStream(imageData))78 {79 BitmapImage bitmap = new BitmapImage();80 bitmap.BeginInit();81 bitmap.StreamSource = ms;82 bitmap.CacheOption = BitmapCacheOption.OnLoad;83 bitmap.EndInit();84 ImageControl.Source = bitmap;85 }86 }87 catch (Exception ex)88 {89 Console.WriteLine($"Error updating image: {ex.Message}");90 }91 }92

93 private void Window_Closed(object sender, EventArgs e)94 {95 heartbeatTimer.Stop();96 receiveThread.Abort();97 udpClient.Close();98 }99 }100}接下来就是内网穿透，把内网ip映射到公网，这一块我踏马一点都不懂。

经历了两天的玩耍，也是没有找到提供Udp协议的内网穿透免费服务，由于我是个穷逼，买不起公网ip，就不搞了，知道有这么回事就行。