FFmpeg
android_camera.c
Go to the documentation of this file.
1 /*
2  * Android camera input device
3  *
4  * Copyright (C) 2017 Felix Matouschek
5  *
6  * This file is part of FFmpeg.
7  *
8  * FFmpeg is free software; you can redistribute it and/or
9  * modify it under the terms of the GNU Lesser General Public
10  * License as published by the Free Software Foundation; either
11  * version 2.1 of the License, or (at your option) any later version.
12  *
13  * FFmpeg is distributed in the hope that it will be useful,
14  * but WITHOUT ANY WARRANTY; without even the implied warranty of
15  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
16  * Lesser General Public License for more details.
17  *
18  * You should have received a copy of the GNU Lesser General Public
19  * License along with FFmpeg; if not, write to the Free Software
20  * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA
21  */
22 
23 #include <errno.h>
24 #include <pthread.h>
25 #include <stdatomic.h>
26 #include <stdbool.h>
27 #include <stdint.h>
28 
29 #include <camera/NdkCameraDevice.h>
30 #include <camera/NdkCameraManager.h>
31 #include <media/NdkImage.h>
32 #include <media/NdkImageReader.h>
33 
34 #include "libavformat/avformat.h"
35 #include "libavformat/internal.h"
36 #include "libavutil/avstring.h"
37 #include "libavutil/display.h"
38 #include "libavutil/imgutils.h"
39 #include "libavutil/log.h"
40 #include "libavutil/opt.h"
41 #include "libavutil/parseutils.h"
42 #include "libavutil/pixfmt.h"
44 #include "libavutil/time.h"
45 
46 #include "version.h"
47 
48 /* This image format is available on all Android devices
49  * supporting the Camera2 API */
50 #define IMAGE_FORMAT_ANDROID AIMAGE_FORMAT_YUV_420_888
51 
52 #define MAX_BUF_COUNT 2
53 #define VIDEO_STREAM_INDEX 0
54 #define VIDEO_TIMEBASE_ANDROID 1000000000
55 
56 #define RETURN_CASE(x) case x: return AV_STRINGIFY(x);
57 #define RETURN_DEFAULT(x) default: return AV_STRINGIFY(x);
58 
59 typedef struct AndroidCameraCtx {
60  const AVClass *class;
61 
67 
70  int width;
71  int height;
74 
75  ACameraManager *camera_mgr;
76  char *camera_id;
77  ACameraMetadata *camera_metadata;
78  ACameraDevice *camera_dev;
79  ACameraDevice_StateCallbacks camera_state_callbacks;
80  AImageReader *image_reader;
81  AImageReader_ImageListener image_listener;
82  ANativeWindow *image_reader_window;
83  ACaptureSessionOutputContainer *capture_session_output_container;
84  ACaptureSessionOutput *capture_session_output;
85  ACameraOutputTarget *camera_output_target;
86  ACaptureRequest *capture_request;
87  ACameraCaptureSession_stateCallbacks capture_session_state_callbacks;
88  ACameraCaptureSession *capture_session;
89 
94 
95 static const char *camera_status_string(camera_status_t val)
96 {
97  switch(val) {
98  RETURN_CASE(ACAMERA_OK)
99  RETURN_CASE(ACAMERA_ERROR_UNKNOWN)
100  RETURN_CASE(ACAMERA_ERROR_INVALID_PARAMETER)
101  RETURN_CASE(ACAMERA_ERROR_CAMERA_DISCONNECTED)
102  RETURN_CASE(ACAMERA_ERROR_NOT_ENOUGH_MEMORY)
103  RETURN_CASE(ACAMERA_ERROR_METADATA_NOT_FOUND)
104  RETURN_CASE(ACAMERA_ERROR_CAMERA_DEVICE)
105  RETURN_CASE(ACAMERA_ERROR_CAMERA_SERVICE)
106  RETURN_CASE(ACAMERA_ERROR_SESSION_CLOSED)
107  RETURN_CASE(ACAMERA_ERROR_INVALID_OPERATION)
108  RETURN_CASE(ACAMERA_ERROR_STREAM_CONFIGURE_FAIL)
109  RETURN_CASE(ACAMERA_ERROR_CAMERA_IN_USE)
110  RETURN_CASE(ACAMERA_ERROR_MAX_CAMERA_IN_USE)
111  RETURN_CASE(ACAMERA_ERROR_CAMERA_DISABLED)
112  RETURN_CASE(ACAMERA_ERROR_PERMISSION_DENIED)
113  RETURN_DEFAULT(ACAMERA_ERROR_UNKNOWN)
114  }
115 }
116 
117 static const char *media_status_string(media_status_t val)
118 {
119  switch(val) {
120  RETURN_CASE(AMEDIA_OK)
121  RETURN_CASE(AMEDIA_ERROR_UNKNOWN)
122  RETURN_CASE(AMEDIA_ERROR_MALFORMED)
123  RETURN_CASE(AMEDIA_ERROR_UNSUPPORTED)
124  RETURN_CASE(AMEDIA_ERROR_INVALID_OBJECT)
125  RETURN_CASE(AMEDIA_ERROR_INVALID_PARAMETER)
126  RETURN_CASE(AMEDIA_ERROR_INVALID_OPERATION)
127  RETURN_CASE(AMEDIA_DRM_NOT_PROVISIONED)
128  RETURN_CASE(AMEDIA_DRM_RESOURCE_BUSY)
129  RETURN_CASE(AMEDIA_DRM_DEVICE_REVOKED)
130  RETURN_CASE(AMEDIA_DRM_SHORT_BUFFER)
131  RETURN_CASE(AMEDIA_DRM_SESSION_NOT_OPENED)
132  RETURN_CASE(AMEDIA_DRM_TAMPER_DETECTED)
133  RETURN_CASE(AMEDIA_DRM_VERIFY_FAILED)
134  RETURN_CASE(AMEDIA_DRM_NEED_KEY)
135  RETURN_CASE(AMEDIA_DRM_LICENSE_EXPIRED)
136  RETURN_CASE(AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE)
137  RETURN_CASE(AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED)
138  RETURN_CASE(AMEDIA_IMGREADER_CANNOT_LOCK_IMAGE)
139  RETURN_CASE(AMEDIA_IMGREADER_CANNOT_UNLOCK_IMAGE)
140  RETURN_CASE(AMEDIA_IMGREADER_IMAGE_NOT_LOCKED)
141  RETURN_DEFAULT(AMEDIA_ERROR_UNKNOWN)
142  }
143 }
144 
145 static const char *error_state_callback_string(int val)
146 {
147  switch(val) {
148  RETURN_CASE(ERROR_CAMERA_IN_USE)
149  RETURN_CASE(ERROR_MAX_CAMERAS_IN_USE)
150  RETURN_CASE(ERROR_CAMERA_DISABLED)
151  RETURN_CASE(ERROR_CAMERA_DEVICE)
152  RETURN_CASE(ERROR_CAMERA_SERVICE)
153  default:
154  return "ERROR_CAMERA_UNKNOWN";
155  }
156 }
157 
158 static void camera_dev_disconnected(void *context, ACameraDevice *device)
159 {
160  AVFormatContext *avctx = context;
161  AndroidCameraCtx *ctx = avctx->priv_data;
162  atomic_store(&ctx->exit, 1);
163  av_log(avctx, AV_LOG_ERROR, "Camera with id %s disconnected.\n",
164  ACameraDevice_getId(device));
165 }
166 
167 static void camera_dev_error(void *context, ACameraDevice *device, int error)
168 {
169  AVFormatContext *avctx = context;
170  AndroidCameraCtx *ctx = avctx->priv_data;
171  atomic_store(&ctx->exit, 1);
172  av_log(avctx, AV_LOG_ERROR, "Error %s on camera with id %s.\n",
173  error_state_callback_string(error), ACameraDevice_getId(device));
174 }
175 
176 static int open_camera(AVFormatContext *avctx)
177 {
178  AndroidCameraCtx *ctx = avctx->priv_data;
179  camera_status_t ret;
180  ACameraIdList *camera_ids;
181 
182  ret = ACameraManager_getCameraIdList(ctx->camera_mgr, &camera_ids);
183  if (ret != ACAMERA_OK) {
184  av_log(avctx, AV_LOG_ERROR, "Failed to get camera id list, error: %s.\n",
185  camera_status_string(ret));
186  return AVERROR_EXTERNAL;
187  }
188 
189  if (ctx->camera_index < camera_ids->numCameras) {
190  ctx->camera_id = av_strdup(camera_ids->cameraIds[ctx->camera_index]);
191  if (!ctx->camera_id) {
192  av_log(avctx, AV_LOG_ERROR, "Failed to allocate memory for camera_id.\n");
193  return AVERROR(ENOMEM);
194  }
195  } else {
196  av_log(avctx, AV_LOG_ERROR, "No camera with index %d available.\n",
197  ctx->camera_index);
198  return AVERROR(ENXIO);
199  }
200 
201  ACameraManager_deleteCameraIdList(camera_ids);
202 
203  ret = ACameraManager_getCameraCharacteristics(ctx->camera_mgr,
204  ctx->camera_id, &ctx->camera_metadata);
205  if (ret != ACAMERA_OK) {
206  av_log(avctx, AV_LOG_ERROR, "Failed to get metadata for camera with id %s, error: %s.\n",
207  ctx->camera_id, camera_status_string(ret));
208  return AVERROR_EXTERNAL;
209  }
210 
211  ctx->camera_state_callbacks.context = avctx;
212  ctx->camera_state_callbacks.onDisconnected = camera_dev_disconnected;
214 
215  ret = ACameraManager_openCamera(ctx->camera_mgr, ctx->camera_id,
216  &ctx->camera_state_callbacks, &ctx->camera_dev);
217  if (ret != ACAMERA_OK) {
218  av_log(avctx, AV_LOG_ERROR, "Failed to open camera with id %s, error: %s.\n",
219  ctx->camera_id, camera_status_string(ret));
220  return AVERROR_EXTERNAL;
221  }
222 
223  return 0;
224 }
225 
227 {
228  AndroidCameraCtx *ctx = avctx->priv_data;
229  ACameraMetadata_const_entry lens_facing;
230  ACameraMetadata_const_entry sensor_orientation;
231 
232  ACameraMetadata_getConstEntry(ctx->camera_metadata,
233  ACAMERA_LENS_FACING, &lens_facing);
234  ACameraMetadata_getConstEntry(ctx->camera_metadata,
235  ACAMERA_SENSOR_ORIENTATION, &sensor_orientation);
236 
237  ctx->lens_facing = lens_facing.data.u8[0];
238  ctx->sensor_orientation = sensor_orientation.data.i32[0];
239 }
240 
242 {
243  AndroidCameraCtx *ctx = avctx->priv_data;
244  ACameraMetadata_const_entry available_configs;
245  int found = 0;
246 
247  ACameraMetadata_getConstEntry(ctx->camera_metadata,
248  ACAMERA_SCALER_AVAILABLE_STREAM_CONFIGURATIONS,
249  &available_configs);
250 
251  for (int i = 0; i < available_configs.count; i++) {
252  int32_t input = available_configs.data.i32[i * 4 + 3];
253  int32_t format = available_configs.data.i32[i * 4 + 0];
254 
255  if (input) {
256  continue;
257  }
258 
259  if (format == IMAGE_FORMAT_ANDROID) {
260  int32_t width = available_configs.data.i32[i * 4 + 1];
261  int32_t height = available_configs.data.i32[i * 4 + 2];
262 
263  //Same ratio
264  if ((ctx->requested_width == width && ctx->requested_height == height) ||
265  (ctx->requested_width == height && ctx->requested_height == width)) {
266  ctx->width = width;
267  ctx->height = height;
268  found = 1;
269  break;
270  }
271  }
272  }
273 
274  if (!found || ctx->width == 0 || ctx->height == 0) {
275  ctx->width = available_configs.data.i32[1];
276  ctx->height = available_configs.data.i32[2];
277 
278  av_log(avctx, AV_LOG_WARNING,
279  "Requested video_size %dx%d not available, falling back to %dx%d\n",
280  ctx->requested_width, ctx->requested_height, ctx->width, ctx->height);
281  }
282 
283  return;
284 }
285 
286 static void match_framerate(AVFormatContext *avctx)
287 {
288  AndroidCameraCtx *ctx = avctx->priv_data;
289  ACameraMetadata_const_entry available_framerates;
290  int found = 0;
291  int current_best_match = -1;
292  int requested_framerate = av_q2d(ctx->framerate);
293 
294  ACameraMetadata_getConstEntry(ctx->camera_metadata,
295  ACAMERA_CONTROL_AE_AVAILABLE_TARGET_FPS_RANGES,
296  &available_framerates);
297 
298  for (int i = 0; i < available_framerates.count; i++) {
299  int32_t min = available_framerates.data.i32[i * 2 + 0];
300  int32_t max = available_framerates.data.i32[i * 2 + 1];
301 
302  if (requested_framerate == max) {
303  if (min == max) {
304  ctx->framerate_range[0] = min;
305  ctx->framerate_range[1] = max;
306  found = 1;
307  break;
308  } else if (current_best_match >= 0) {
309  int32_t current_best_match_min = available_framerates.data.i32[current_best_match * 2 + 0];
310  if (min > current_best_match_min) {
311  current_best_match = i;
312  }
313  } else {
314  current_best_match = i;
315  }
316  }
317  }
318 
319  if (!found) {
320  if (current_best_match >= 0) {
321  ctx->framerate_range[0] = available_framerates.data.i32[current_best_match * 2 + 0];
322  ctx->framerate_range[1] = available_framerates.data.i32[current_best_match * 2 + 1];
323 
324  } else {
325  ctx->framerate_range[0] = available_framerates.data.i32[0];
326  ctx->framerate_range[1] = available_framerates.data.i32[1];
327  }
328 
329  av_log(avctx, AV_LOG_WARNING,
330  "Requested framerate %d not available, falling back to min: %d and max: %d fps\n",
331  requested_framerate, ctx->framerate_range[0], ctx->framerate_range[1]);
332  }
333 
334  return;
335 }
336 
337 static int get_image_format(AVFormatContext *avctx, AImage *image)
338 {
339  AndroidCameraCtx *ctx = avctx->priv_data;
340  int32_t image_pixelstrides[2];
341  uint8_t *image_plane_data[2];
342  int plane_data_length[2];
343 
344  for (int i = 0; i < 2; i++) {
345  AImage_getPlanePixelStride(image, i + 1, &image_pixelstrides[i]);
346  AImage_getPlaneData(image, i + 1, &image_plane_data[i], &plane_data_length[i]);
347  }
348 
349  if (image_pixelstrides[0] != image_pixelstrides[1]) {
350  av_log(avctx, AV_LOG_ERROR,
351  "Pixel strides of U and V plane should have been the same.\n");
352  return AVERROR_EXTERNAL;
353  }
354 
355  switch (image_pixelstrides[0]) {
356  case 1:
358  break;
359  case 2:
360  if (image_plane_data[0] < image_plane_data[1]) {
362  } else {
364  }
365  break;
366  default:
367  av_log(avctx, AV_LOG_ERROR,
368  "Unknown pixel stride %d of U and V plane, cannot determine camera image format.\n",
369  image_pixelstrides[0]);
370  return AVERROR(ENOSYS);
371  }
372 
373  return 0;
374 }
375 
376 static void image_available(void *context, AImageReader *reader)
377 {
378  AVFormatContext *avctx = context;
379  AndroidCameraCtx *ctx = avctx->priv_data;
380  media_status_t media_status;
381  int ret = 0;
382 
383  AImage *image;
384  int64_t image_timestamp;
385  int32_t image_linestrides[4];
386  uint8_t *image_plane_data[4];
387  int plane_data_length[4];
388 
389  AVPacket pkt;
390  int pkt_buffer_size = 0;
391 
392  media_status = AImageReader_acquireLatestImage(reader, &image);
393  if (media_status != AMEDIA_OK) {
394  if (media_status == AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE) {
395  av_log(avctx, AV_LOG_WARNING,
396  "An image reader frame was discarded");
397  } else {
398  av_log(avctx, AV_LOG_ERROR,
399  "Failed to acquire latest image from image reader, error: %s.\n",
400  media_status_string(media_status));
401  ret = AVERROR_EXTERNAL;
402  }
403  goto error;
404  }
405 
406  // Silently drop frames when exit is set
407  if (atomic_load(&ctx->exit)) {
408  goto error;
409  }
410 
411  // Determine actual image format
412  if (!atomic_load(&ctx->got_image_format)) {
413  ret = get_image_format(avctx, image);
414  if (ret < 0) {
415  av_log(avctx, AV_LOG_ERROR,
416  "Could not get image format of camera.\n");
417  goto error;
418  } else {
419  atomic_store(&ctx->got_image_format, 1);
420  }
421  }
422 
423  pkt_buffer_size = av_image_get_buffer_size(ctx->image_format, ctx->width, ctx->height, 32);
424  AImage_getTimestamp(image, &image_timestamp);
425 
426  AImage_getPlaneRowStride(image, 0, &image_linestrides[0]);
427  AImage_getPlaneData(image, 0, &image_plane_data[0], &plane_data_length[0]);
428 
429  switch (ctx->image_format) {
430  case AV_PIX_FMT_YUV420P:
431  AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
432  AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
433  AImage_getPlaneRowStride(image, 2, &image_linestrides[2]);
434  AImage_getPlaneData(image, 2, &image_plane_data[2], &plane_data_length[2]);
435  break;
436  case AV_PIX_FMT_NV12:
437  AImage_getPlaneRowStride(image, 1, &image_linestrides[1]);
438  AImage_getPlaneData(image, 1, &image_plane_data[1], &plane_data_length[1]);
439  break;
440  case AV_PIX_FMT_NV21:
441  AImage_getPlaneRowStride(image, 2, &image_linestrides[1]);
442  AImage_getPlaneData(image, 2, &image_plane_data[1], &plane_data_length[1]);
443  break;
444  default:
445  av_log(avctx, AV_LOG_ERROR, "Unsupported camera image format.\n");
446  ret = AVERROR(ENOSYS);
447  goto error;
448  }
449 
450  ret = av_new_packet(&pkt, pkt_buffer_size);
451  if (ret < 0) {
452  av_log(avctx, AV_LOG_ERROR,
453  "Failed to create new av packet, error: %s.\n", av_err2str(ret));
454  goto error;
455  }
456 
458  pkt.pts = image_timestamp;
459  av_image_copy_to_buffer(pkt.data, pkt_buffer_size,
460  (const uint8_t * const *) image_plane_data,
461  image_linestrides, ctx->image_format,
462  ctx->width, ctx->height, 32);
463 
465 
466 error:
467  if (ret < 0) {
468  if (ret != AVERROR(EAGAIN)) {
469  av_log(avctx, AV_LOG_ERROR,
470  "Error while processing new image, error: %s.\n", av_err2str(ret));
472  atomic_store(&ctx->exit, 1);
473  } else {
474  av_log(avctx, AV_LOG_WARNING,
475  "Input queue was full, dropping frame, consider raising the input_queue_size option (current value: %d)\n",
476  ctx->input_queue_size);
477  }
478  if (pkt_buffer_size) {
479  av_packet_unref(&pkt);
480  }
481  }
482 
483  AImage_delete(image);
484 
485  return;
486 }
487 
489 {
490  AndroidCameraCtx *ctx = avctx->priv_data;
491  media_status_t ret;
492 
493  ret = AImageReader_new(ctx->width, ctx->height, IMAGE_FORMAT_ANDROID,
494  MAX_BUF_COUNT, &ctx->image_reader);
495  if (ret != AMEDIA_OK) {
496  av_log(avctx, AV_LOG_ERROR,
497  "Failed to create image reader, error: %s.\n", media_status_string(ret));
498  return AVERROR_EXTERNAL;
499  }
500 
501  ctx->image_listener.context = avctx;
502  ctx->image_listener.onImageAvailable = image_available;
503 
504  ret = AImageReader_setImageListener(ctx->image_reader, &ctx->image_listener);
505  if (ret != AMEDIA_OK) {
506  av_log(avctx, AV_LOG_ERROR,
507  "Failed to set image listener on image reader, error: %s.\n",
508  media_status_string(ret));
509  return AVERROR_EXTERNAL;
510  }
511 
512  ret = AImageReader_getWindow(ctx->image_reader, &ctx->image_reader_window);
513  if (ret != AMEDIA_OK) {
514  av_log(avctx, AV_LOG_ERROR,
515  "Could not get image reader window, error: %s.\n",
516  media_status_string(ret));
517  return AVERROR_EXTERNAL;
518  }
519 
520  return 0;
521 }
522 
523 static void capture_session_closed(void *context, ACameraCaptureSession *session)
524 {
525  av_log(context, AV_LOG_INFO, "Android camera capture session was closed.\n");
526 }
527 
528 static void capture_session_ready(void *context, ACameraCaptureSession *session)
529 {
530  av_log(context, AV_LOG_INFO, "Android camera capture session is ready.\n");
531 }
532 
533 static void capture_session_active(void *context, ACameraCaptureSession *session)
534 {
535  av_log(context, AV_LOG_INFO, "Android camera capture session is active.\n");
536 }
537 
539 {
540  AndroidCameraCtx *ctx = avctx->priv_data;
541  camera_status_t ret;
542 
543  ret = ACaptureSessionOutputContainer_create(&ctx->capture_session_output_container);
544  if (ret != ACAMERA_OK) {
545  av_log(avctx, AV_LOG_ERROR,
546  "Failed to create capture session output container, error: %s.\n",
547  camera_status_string(ret));
548  return AVERROR_EXTERNAL;
549  }
550 
551  ANativeWindow_acquire(ctx->image_reader_window);
552 
553  ret = ACaptureSessionOutput_create(ctx->image_reader_window, &ctx->capture_session_output);
554  if (ret != ACAMERA_OK) {
555  av_log(avctx, AV_LOG_ERROR,
556  "Failed to create capture session container, error: %s.\n",
557  camera_status_string(ret));
558  return AVERROR_EXTERNAL;
559  }
560 
561  ret = ACaptureSessionOutputContainer_add(ctx->capture_session_output_container,
563  if (ret != ACAMERA_OK) {
564  av_log(avctx, AV_LOG_ERROR,
565  "Failed to add output to output container, error: %s.\n",
566  camera_status_string(ret));
567  return AVERROR_EXTERNAL;
568  }
569 
570  ret = ACameraOutputTarget_create(ctx->image_reader_window, &ctx->camera_output_target);
571  if (ret != ACAMERA_OK) {
572  av_log(avctx, AV_LOG_ERROR,
573  "Failed to create camera output target, error: %s.\n",
574  camera_status_string(ret));
575  return AVERROR_EXTERNAL;
576  }
577 
578  ret = ACameraDevice_createCaptureRequest(ctx->camera_dev, TEMPLATE_RECORD, &ctx->capture_request);
579  if (ret != ACAMERA_OK) {
580  av_log(avctx, AV_LOG_ERROR,
581  "Failed to create capture request, error: %s.\n",
582  camera_status_string(ret));
583  return AVERROR_EXTERNAL;
584  }
585 
586  ret = ACaptureRequest_setEntry_i32(ctx->capture_request, ACAMERA_CONTROL_AE_TARGET_FPS_RANGE,
587  2, ctx->framerate_range);
588  if (ret != ACAMERA_OK) {
589  av_log(avctx, AV_LOG_ERROR,
590  "Failed to set target fps range in capture request, error: %s.\n",
591  camera_status_string(ret));
592  return AVERROR_EXTERNAL;
593  }
594 
595  ret = ACaptureRequest_addTarget(ctx->capture_request, ctx->camera_output_target);
596  if (ret != ACAMERA_OK) {
597  av_log(avctx, AV_LOG_ERROR,
598  "Failed to add capture request capture request, error: %s.\n",
599  camera_status_string(ret));
600  return AVERROR_EXTERNAL;
601  }
602 
603  ctx->capture_session_state_callbacks.context = avctx;
607 
608  ret = ACameraDevice_createCaptureSession(ctx->camera_dev, ctx->capture_session_output_container,
610  if (ret != ACAMERA_OK) {
611  av_log(avctx, AV_LOG_ERROR,
612  "Failed to create capture session, error: %s.\n",
613  camera_status_string(ret));
614  return AVERROR_EXTERNAL;
615  }
616 
617  ret = ACameraCaptureSession_setRepeatingRequest(ctx->capture_session, NULL, 1, &ctx->capture_request, NULL);
618  if (ret != ACAMERA_OK) {
619  av_log(avctx, AV_LOG_ERROR,
620  "Failed to set repeating request on capture session, error: %s.\n",
621  camera_status_string(ret));
622  return AVERROR_EXTERNAL;
623  }
624 
625  return 0;
626 }
627 
629 {
630  AndroidCameraCtx *ctx = avctx->priv_data;
631 
632  while (!atomic_load(&ctx->got_image_format) && !atomic_load(&ctx->exit)) {
633  //Wait until first frame arrived and actual image format was determined
634  usleep(1000);
635  }
636 
637  return atomic_load(&ctx->got_image_format);
638 }
639 
641 {
642  AndroidCameraCtx *ctx = avctx->priv_data;
643  uint8_t *side_data;
644  int32_t display_matrix[9];
645 
646  av_display_rotation_set(display_matrix, ctx->sensor_orientation);
647 
648  if (ctx->lens_facing == ACAMERA_LENS_FACING_FRONT) {
649  av_display_matrix_flip(display_matrix, 1, 0);
650  }
651 
652  side_data = av_stream_new_side_data(st,
653  AV_PKT_DATA_DISPLAYMATRIX, sizeof(display_matrix));
654 
655  if (!side_data) {
656  return AVERROR(ENOMEM);
657  }
658 
659  memcpy(side_data, display_matrix, sizeof(display_matrix));
660 
661  return 0;
662 }
663 
665 {
666  AndroidCameraCtx *ctx = avctx->priv_data;
667  AVStream *st;
668  AVCodecParameters *codecpar;
669 
670  st = avformat_new_stream(avctx, NULL);
671  if (!st) {
672  return AVERROR(ENOMEM);
673  }
674 
675  st->id = VIDEO_STREAM_INDEX;
676  st->avg_frame_rate = (AVRational) { ctx->framerate_range[1], 1 };
677  st->r_frame_rate = (AVRational) { ctx->framerate_range[1], 1 };
678 
679  if (!wait_for_image_format(avctx)) {
680  return AVERROR_EXTERNAL;
681  }
682 
683  codecpar = st->codecpar;
684  codecpar->codec_type = AVMEDIA_TYPE_VIDEO;
685  codecpar->codec_id = AV_CODEC_ID_RAWVIDEO;
686  codecpar->format = ctx->image_format;
687  codecpar->width = ctx->width;
688  codecpar->height = ctx->height;
689 
691 
692  return add_display_matrix(avctx, st);
693 }
694 
696 {
697  AndroidCameraCtx *ctx = avctx->priv_data;
698 
699  atomic_store(&ctx->exit, 1);
700 
701  if (ctx->capture_session) {
702  ACameraCaptureSession_stopRepeating(ctx->capture_session);
703  // Following warning is emitted, after capture session closed callback is received:
704  // ACameraCaptureSession: Device is closed but session 0 is not notified
705  // Seems to be a bug in Android, we can ignore this
706  ACameraCaptureSession_close(ctx->capture_session);
707  ctx->capture_session = NULL;
708  }
709 
710  if (ctx->capture_request) {
711  ACaptureRequest_removeTarget(ctx->capture_request, ctx->camera_output_target);
712  ACaptureRequest_free(ctx->capture_request);
713  ctx->capture_request = NULL;
714  }
715 
716  if (ctx->camera_output_target) {
717  ACameraOutputTarget_free(ctx->camera_output_target);
718  ctx->camera_output_target = NULL;
719  }
720 
721  if (ctx->capture_session_output) {
722  ACaptureSessionOutputContainer_remove(ctx->capture_session_output_container,
724  ACaptureSessionOutput_free(ctx->capture_session_output);
726  }
727 
728  if (ctx->image_reader_window) {
729  ANativeWindow_release(ctx->image_reader_window);
730  ctx->image_reader_window = NULL;
731  }
732 
734  ACaptureSessionOutputContainer_free(ctx->capture_session_output_container);
736  }
737 
738  if (ctx->camera_dev) {
739  ACameraDevice_close(ctx->camera_dev);
740  ctx->camera_dev = NULL;
741  }
742 
743  if (ctx->image_reader) {
744  AImageReader_delete(ctx->image_reader);
745  ctx->image_reader = NULL;
746  }
747 
748  if (ctx->camera_metadata) {
749  ACameraMetadata_free(ctx->camera_metadata);
750  ctx->camera_metadata = NULL;
751  }
752 
753  av_freep(&ctx->camera_id);
754 
755  if (ctx->camera_mgr) {
756  ACameraManager_delete(ctx->camera_mgr);
757  ctx->camera_mgr = NULL;
758  }
759 
760  if (ctx->input_queue) {
761  AVPacket pkt;
764  av_packet_unref(&pkt);
765  }
767  }
768 
769  return 0;
770 }
771 
773 {
774  AndroidCameraCtx *ctx = avctx->priv_data;
775  int ret;
776 
777  atomic_init(&ctx->got_image_format, 0);
778  atomic_init(&ctx->exit, 0);
779 
781  if (ret < 0) {
782  av_log(avctx, AV_LOG_ERROR,
783  "Failed to allocate input queue, error: %s.\n", av_err2str(ret));
784  goto error;
785  }
786 
787  ctx->camera_mgr = ACameraManager_create();
788  if (!ctx->camera_mgr) {
789  av_log(avctx, AV_LOG_ERROR, "Failed to create Android camera manager.\n");
790  ret = AVERROR_EXTERNAL;
791  goto error;
792  }
793 
794  ret = open_camera(avctx);
795  if (ret < 0) {
796  av_log(avctx, AV_LOG_ERROR, "Failed to open camera.\n");
797  goto error;
798  }
799 
800  get_sensor_orientation(avctx);
801  match_video_size(avctx);
802  match_framerate(avctx);
803 
804  ret = create_image_reader(avctx);
805  if (ret < 0) {
806  goto error;
807  }
808 
809  ret = create_capture_session(avctx);
810  if (ret < 0) {
811  goto error;
812  }
813 
814  ret = add_video_stream(avctx);
815 
816 error:
817  if (ret < 0) {
819  av_log(avctx, AV_LOG_ERROR, "Failed to open android_camera.\n");
820  }
821 
822  return ret;
823 }
824 
826 {
827  AndroidCameraCtx *ctx = avctx->priv_data;
828  int ret;
829 
830  if (!atomic_load(&ctx->exit)) {
833  } else {
834  ret = AVERROR_EOF;
835  }
836 
837  if (ret < 0) {
838  return ret;
839  } else {
840  return pkt->size;
841  }
842 }
843 
844 #define OFFSET(x) offsetof(AndroidCameraCtx, x)
845 #define DEC AV_OPT_FLAG_DECODING_PARAM
846 static const AVOption options[] = {
847  { "video_size", "set video size given as a string such as 640x480 or hd720", OFFSET(requested_width), AV_OPT_TYPE_IMAGE_SIZE, {.str = NULL}, 0, 0, DEC },
848  { "framerate", "set video frame rate", OFFSET(framerate), AV_OPT_TYPE_VIDEO_RATE, {.str = "30"}, 0, INT_MAX, DEC },
849  { "camera_index", "set index of camera to use", OFFSET(camera_index), AV_OPT_TYPE_INT, {.i64 = 0}, 0, INT_MAX, DEC },
850  { "input_queue_size", "set maximum number of frames to buffer", OFFSET(input_queue_size), AV_OPT_TYPE_INT, {.i64 = 5}, 0, INT_MAX, DEC },
851  { NULL },
852 };
853 
855  .class_name = "android_camera indev",
856  .item_name = av_default_item_name,
857  .option = options,
858  .version = LIBAVUTIL_VERSION_INT,
860 };
861 
863  .name = "android_camera",
864  .long_name = NULL_IF_CONFIG_SMALL("Android camera input device"),
865  .priv_data_size = sizeof(AndroidCameraCtx),
869  .flags = AVFMT_NOFILE,
870  .priv_class = &android_camera_class,
871 };
#define NULL
Definition: coverity.c:32
const char const char void * val
Definition: avisynth_c.h:863
static void capture_session_ready(void *context, ACameraCaptureSession *session)
#define DEC
static void capture_session_active(void *context, ACameraCaptureSession *session)
#define atomic_store(object, desired)
Definition: stdatomic.h:85
AVOption.
Definition: opt.h:246
void av_thread_message_queue_set_err_recv(AVThreadMessageQueue *mq, int err)
Set the receiving error code.
int av_image_copy_to_buffer(uint8_t *dst, int dst_size, const uint8_t *const src_data[4], const int src_linesize[4], enum AVPixelFormat pix_fmt, int width, int height, int align)
Copy image data from an image into a buffer.
Definition: imgutils.c:453
ACaptureSessionOutput * capture_session_output
misc image utilities
#define AV_LOG_WARNING
Something somehow does not look correct.
Definition: log.h:182
#define LIBAVUTIL_VERSION_INT
Definition: version.h:85
void avpriv_set_pts_info(AVStream *s, int pts_wrap_bits, unsigned int pts_num, unsigned int pts_den)
Set the time base and wrapping info for a given stream.
Definition: utils.c:4926
#define IMAGE_FORMAT_ANDROID
enum AVCodecID codec_id
Specific type of the encoded data (the codec used).
Definition: avcodec.h:3968
int size
Definition: avcodec.h:1481
const char * av_default_item_name(void *ptr)
Return the context name.
Definition: log.c:191
static const char * media_status_string(media_status_t val)
#define VIDEO_STREAM_INDEX
static AVPacket pkt
ANativeWindow * image_reader_window
intptr_t atomic_int
Definition: stdatomic.h:55
static int wait_for_image_format(AVFormatContext *avctx)
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample format(the sample packing is implied by the sample format) and sample rate.The lists are not just lists
This struct describes the properties of an encoded stream.
Definition: avcodec.h:3960
void av_display_matrix_flip(int32_t matrix[9], int hflip, int vflip)
Flip the input matrix horizontally and/or vertically.
Definition: display.c:65
void av_display_rotation_set(int32_t matrix[9], double angle)
Initialize a transformation matrix describing a pure counterclockwise rotation by the specified angle...
Definition: display.c:50
Format I/O context.
Definition: avformat.h:1358
const char * class_name
The name of the class; usually it is the same name as the context structure type to which the AVClass...
Definition: log.h:72
#define AVFMT_FLAG_NONBLOCK
Do not block when reading packets from input.
Definition: avformat.h:1492
static int add_video_stream(AVFormatContext *avctx)
uint8_t
int av_thread_message_queue_recv(AVThreadMessageQueue *mq, void *msg, unsigned flags)
Receive a message from the queue.
int width
Video only.
Definition: avcodec.h:4034
static const AVClass android_camera_class
AVOptions.
static int android_camera_read_close(AVFormatContext *avctx)
ACaptureSessionOutputContainer * capture_session_output_container
This side data contains a 3x3 transformation matrix describing an affine transformation that needs to...
Definition: avcodec.h:1255
int av_thread_message_queue_send(AVThreadMessageQueue *mq, void *msg, unsigned flags)
Send a message on the queue.
int id
Format-specific stream ID.
Definition: avformat.h:888
static const char * camera_status_string(camera_status_t val)
#define VIDEO_TIMEBASE_ANDROID
AVStream * avformat_new_stream(AVFormatContext *s, const AVCodec *c)
Add a new stream to a media file.
Definition: utils.c:4499
#define MAX_BUF_COUNT
ACaptureRequest * capture_request
int flags
Flags modifying the (de)muxer behaviour.
Definition: avformat.h:1489
uint8_t * data
Definition: avcodec.h:1480
static double av_q2d(AVRational a)
Convert an AVRational to a double.
Definition: rational.h:104
#define AVERROR_EOF
End of file.
Definition: error.h:55
int32_t framerate_range[2]
static av_cold int read_close(AVFormatContext *ctx)
Definition: libcdio.c:145
#define RETURN_DEFAULT(x)
#define max(a, b)
Definition: cuda_runtime.h:33
static void camera_dev_error(void *context, ACameraDevice *device, int error)
#define av_log(a,...)
static int android_camera_read_header(AVFormatContext *avctx)
int av_new_packet(AVPacket *pkt, int size)
Allocate the payload of a packet and initialize its fields with default values.
Definition: avpacket.c:86
AVThreadMessageQueue * input_queue
int av_image_get_buffer_size(enum AVPixelFormat pix_fmt, int width, int height, int align)
Return the size in bytes of the amount of data required to store an image with the given parameters...
Definition: imgutils.c:431
#define i(width, name, range_min, range_max)
Definition: cbs_h2645.c:259
#define AV_LOG_ERROR
Something went wrong and cannot losslessly be recovered.
Definition: log.h:176
#define atomic_load(object)
Definition: stdatomic.h:93
static void capture_session_closed(void *context, ACameraCaptureSession *session)
AVRational framerate
#define NULL_IF_CONFIG_SMALL(x)
Return NULL if CONFIG_SMALL is true, otherwise the argument without modification. ...
Definition: internal.h:186
Display matrix.
enum AVMediaType codec_type
General type of the encoded data.
Definition: avcodec.h:3964
planar YUV 4:2:0, 12bpp, 1 plane for Y and 1 plane for the UV components, which are interleaved (firs...
Definition: pixfmt.h:89
AVInputFormat ff_android_camera_demuxer
AVRational avg_frame_rate
Average framerate.
Definition: avformat.h:954
static int create_image_reader(AVFormatContext *avctx)
ACameraCaptureSession * capture_session
as above, but U and V bytes are swapped
Definition: pixfmt.h:90
ACameraDevice_StateCallbacks camera_state_callbacks
static void get_sensor_orientation(AVFormatContext *avctx)
#define av_err2str(errnum)
Convenience macro, the return value should be used only directly in function arguments but never stan...
Definition: error.h:119
int32_t
AVFormatContext * ctx
Definition: movenc.c:48
ACameraManager * camera_mgr
int32_t sensor_orientation
uint8_t * av_stream_new_side_data(AVStream *stream, enum AVPacketSideDataType type, int size)
Allocate new information from stream.
Definition: utils.c:5538
static void error(const char *err)
static int read_header(FFV1Context *f)
Definition: ffv1dec.c:530
#define RETURN_CASE(x)
#define OFFSET(x)
Stream structure.
Definition: avformat.h:881
static int read_packet(void *opaque, uint8_t *buf, int buf_size)
Definition: avio_reading.c:42
ACameraMetadata * camera_metadata
#define AV_LOG_INFO
Standard information.
Definition: log.h:187
char * av_strdup(const char *s)
Duplicate a string.
Definition: mem.c:251
static int open_camera(AVFormatContext *avctx)
void av_thread_message_queue_set_err_send(AVThreadMessageQueue *mq, int err)
Set the sending error code.
static void match_video_size(AVFormatContext *avctx)
void av_packet_unref(AVPacket *pkt)
Wipe the packet.
Definition: avpacket.c:599
Perform non-blocking operation.
Definition: threadmessage.h:31
static int add_display_matrix(AVFormatContext *avctx, AVStream *st)
Describe the class of an AVClass context structure.
Definition: log.h:67
static const AVOption options[]
static int create_capture_session(AVFormatContext *avctx)
Rational number (pair of numerator and denominator).
Definition: rational.h:58
int av_thread_message_queue_alloc(AVThreadMessageQueue **mq, unsigned nelem, unsigned elsize)
Allocate a new message queue.
Definition: threadmessage.c:40
offset must point to AVRational
Definition: opt.h:236
and forward the test the status of outputs and forward it to the corresponding return FFERROR_NOT_READY If the filters stores internally one or a few frame for some input
ACameraOutputTarget * camera_output_target
AImageReader_ImageListener image_listener
offset must point to two consecutive integers
Definition: opt.h:233
misc parsing utilities
Libavdevice version macros.
#define flags(name, subs,...)
Definition: cbs_av1.c:561
void av_thread_message_queue_free(AVThreadMessageQueue **mq)
Free a message queue.
Definition: threadmessage.c:91
ACameraCaptureSession_stateCallbacks capture_session_state_callbacks
uint8_t * data[AV_NUM_DATA_POINTERS]
pointer to the picture/channel planes.
Definition: frame.h:309
static void match_framerate(AVFormatContext *avctx)
static const char * error_state_callback_string(int val)
static int android_camera_read_packet(AVFormatContext *avctx, AVPacket *pkt)
Main libavformat public API header.
ACameraDevice * camera_dev
static void image_available(void *context, AImageReader *reader)
planar YUV 4:2:0, 12bpp, (1 Cr & Cb sample per 2x2 Y samples)
Definition: pixfmt.h:66
#define AVFMT_NOFILE
Demuxer will use avio_open, no opened file should be provided by the caller.
Definition: avformat.h:463
static void camera_dev_disconnected(void *context, ACameraDevice *device)
pixel format definitions
void * priv_data
Format private data.
Definition: avformat.h:1386
atomic_int got_image_format
#define atomic_init(obj, value)
Definition: stdatomic.h:33
#define av_freep(p)
const char * name
A comma separated list of short names for the format.
Definition: avformat.h:654
AImageReader * image_reader
AVCodecParameters * codecpar
Codec parameters associated with this stream.
Definition: avformat.h:1028
int stream_index
Definition: avcodec.h:1482
Filter the word “frame” indicates either a video frame or a group of audio as stored in an AVFrame structure Format for each input and each output the list of supported formats For video that means pixel format For audio that means channel sample they are references to shared objects When the negotiation mechanism computes the intersection of the formats supported at each end of a all references to both lists are replaced with a reference to the intersection And when a single format is eventually chosen for a link amongst the remaining all references to the list are updated That means that if a filter requires that its input and output have the same format amongst a supported all it has to do is use a reference to the same list of formats query_formats can leave some formats unset and return AVERROR(EAGAIN) to cause the negotiation mechanism toagain later.That can be used by filters with complex requirements to use the format negotiated on one link to set the formats supported on another.Frame references ownership and permissions
it s the only field you need to keep assuming you have a context There is some magic you don t need to care about around this just let it vf default minimum maximum flags name is the option keep it simple and lowercase description are in without and describe what they for example set the foo of the bar offset is the offset of the field in your local context
AVRational r_frame_rate
Real base framerate of the stream.
Definition: avformat.h:1005
float min
#define AVERROR_EXTERNAL
Generic error in an external library.
Definition: error.h:57
This structure stores compressed data.
Definition: avcodec.h:1457
static int get_image_format(AVFormatContext *avctx, AImage *image)
int64_t pts
Presentation timestamp in AVStream->time_base units; the time at which the decompressed packet will b...
Definition: avcodec.h:1473