support autofocus / autogain in libv4l2

support autofocus / autogain in libv4l2

[Mauro Carvalho Chehab]

Hi Pavel,

Em Mon, 24 Apr 2017 11:30:59 +0200
Pavel Machek <pavel@ucw.cz> escreveu:

> Hi!
> 
> For focus to be useful, we need autofocus implmented
> somewhere. Unfortunately, v4l framework does not seem to provide good
> place where to put autofocus. I believe, long-term, we'll need some
> kind of "video server" providing this kind of services.
> 
> Anyway, we probably don't want autofocus in kernel (even through some
> cameras do it in hardware), and we probably don't want autofocus in
> each and every user application.
> 
> So what remains is libv4l2. 

IMO, the best place for autofocus is at libv4l2. Putting it on a
separate "video server" application looks really weird for me.

Btw, libv4l2 already has some autotools for auto gain and auto
white balance. See the implementation under:
	lib/libv4lconvert/processing

The libv4l internal controls can be seen at:
	lib/libv4lconvert/control/libv4lcontrol.h

The ones implemented by the processing part of the library are:

$ git grep v4lcontrol_get_ctrl lib/libv4lconvert/processing/
lib/libv4lconvert/processing/autogain.c:        autogain = v4lcontrol_get_ctrl(data->control, V4LCONTROL_AUTOGAIN);
lib/libv4lconvert/processing/autogain.c:        target = v4lcontrol_get_ctrl(data->control, V4LCONTROL_AUTOGAIN_TARGET);
lib/libv4lconvert/processing/gamma.c:   int gamma = v4lcontrol_get_ctrl(data->control, V4LCONTROL_GAMMA);
lib/libv4lconvert/processing/gamma.c:   gamma = v4lcontrol_get_ctrl(data->control, V4LCONTROL_GAMMA);
lib/libv4lconvert/processing/whitebalance.c:    wb = v4lcontrol_get_ctrl(data->control, V4LCONTROL_WHITEBALANCE);

I guess it shouldn't be hard to add an extra processing module
there for auto-focus.

> Now, this is in no way clean or complete,
> and functionality provided by sdl.c and asciicam.c probably _should_
> be in application, but... I'd like to get the code out there.
> 
> Oh and yes, I've canibalized decode_tm6000.c application instead of
> introducing my own. Autotools scare me, sorry.

Why replace decode_tm6000.c by something else? If you want to add another
test application, just place it on a new file.

I added a few notes together with the code, pointing the main things
I think it require changes, in order for me to do a better review
at the code. I didn't test nor tried to check the algorithms inside,
as the code, on its current state, requires rework and code cleanup.

> 
> Regards,
> 							Pavel
> 
> diff --git a/lib/libv4l2/asciicam.c b/lib/libv4l2/asciicam.c
> new file mode 100644
> index 0000000..5388967
> --- /dev/null
> +++ b/lib/libv4l2/asciicam.c
> @@ -0,0 +1,63 @@
> +/* gcc asciicam.c /usr/lib/i386-linux-gnu/libv4l2.so.0.0.0 -o asciicam
> +   gcc asciicam.c /usr/lib/arm-linux-gnueabi/libv4l2.so.0 -o asciicam
> +
> +gcc -std=gnu99 -DHAVE_CONFIG_H -I. -I../.. -fvisibility=hidden -I../../lib/include -Wall -Wpointer-arith -D_GNU_SOURCE -I../../include -g -O2 asciicam.c libv4l2.c /usr/lib/arm-linux-gnueabi/libv4lconvert.so.0 log.c v4l2convert.c v4l2-plugin.c -o asciicam
> + */
> +#include <stdio.h>
> +#include <sys/types.h>
> +#include <sys/stat.h>
> +#include <fcntl.h>
> +
> +#include <linux/videodev2.h>
> +
> +#define SIZE 10*1024*1024
> +
> +char buf[SIZE];
> +
> +void main(void)

Please don't add a new application under lib/. It is fine if you want
some testing application, if the ones there aren't enough, but please
place it under contrib/test/.

You should likely take a look at v4l2grab first, as it could have
almost everything you would need.

> +{
> +  int fd = v4l2_open("/dev/video2", O_RDWR);

Hardcoding /dev/video2 doesn't seem a good idea.

> +  int i;
> +  static struct v4l2_format fmt;
> +
> +#if 1
> +  fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
> +  fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB24;
> +  fmt.fmt.pix.field = V4L2_FIELD_INTERLACED;
> +  fmt.fmt.pix.width = 640;
> +  fmt.fmt.pix.height = 480;
> +  if (fmt.fmt.pix.pixelformat != 'RGB3') /* v4l2_fourcc('R', 'G', 'B', '3'); */
> +    printf("hmm. strange format?\n");
> +  
> +  printf("ioctl = %d\n", v4l2_ioctl(fd, VIDIOC_S_FMT, &fmt));
> +#endif
> +
> +  for (i=0; i<500; i++) {
> +    int num = v4l2_read(fd, buf, SIZE);
> +    int y,x;
> +    
> +    printf("%d\n", num);
> +#if 0
> +    for (y = 0; y < 25; y++) {
> +      for (x = 0; x < 80; x++) {
> +	int y1 = y * 480/25;
> +	int x1 = x * 640/80;
> +	int c = buf[fmt.fmt.pix.width*3*y1 + 3*x1] +
> +	  buf[fmt.fmt.pix.width*3*y1 + 3*x1 + 1] +
> +	  buf[fmt.fmt.pix.width*3*y1 + 3*x1 + 2];
> +
> +	if (c < 30) c = ' ';
> +	else if (c < 60) c = '.';
> +	else if (c < 120) c = '_';
> +	else if (c < 180) c = 'o';
> +	else if (c < 300) c = 'x';
> +	else if (c < 400) c = 'X';
> +	else c = '#';
> +	  
> +	printf("%c", c);
> +      }
> +      printf("\n");
> +    }

IMHO, it would be better to use aalib. Btw, xawtv3 has a code example
using it, under:
	console/ttv.c

As it already uses libv4l, prhaps you could use it, instead of adding
a new ascii app.

> +#endif    
> +  }
> +}
> diff --git a/lib/libv4l2/libv4l2-priv.h b/lib/libv4l2/libv4l2-priv.h
> index 343db5e..af740a7 100644
> --- a/lib/libv4l2/libv4l2-priv.h
> +++ b/lib/libv4l2/libv4l2-priv.h
> @@ -1,3 +1,4 @@
> +/* -*- c-file-style: "linux" -*- */
>  /*
>  #             (C) 2008 Hans de Goede <hdegoede@redhat.com>
>  
> @@ -70,6 +71,14 @@
>  	} while (0)
>  
>  #define MIN(a, b) (((a) < (b)) ? (a) : (b))
> +#define V4L2_MAX_SUBDEVS 16
> +
> +#define V4L2_MAX_FOCUS 10
> +struct v4l2_focus_step {
> +	int num;
> +	int sharpness[V4L2_MAX_FOCUS];
> +	int position[V4L2_MAX_FOCUS];
> +};
>  
>  struct v4l2_dev_info {
>  	int fd;
> @@ -104,6 +113,14 @@ struct v4l2_dev_info {
>  	void *plugin_library;
>  	void *dev_ops_priv;
>  	const struct libv4l_dev_ops *dev_ops;
> +        int subdev_fds[V4L2_MAX_SUBDEVS];
> +  	int exposure; 
> +  	int frame;
> +  	int focus;
> +
> +	/* Autofocus parameters */
> +	int focus_frame, focus_exposure, focus_step, sh_prev;
> +	struct v4l2_focus_step focus_step_params;
>  };
>  
>  /* From v4l2-plugin.c */
> diff --git a/lib/libv4l2/libv4l2.c b/lib/libv4l2/libv4l2.c
> index 0ba0a88..3b84437 100644
> --- a/lib/libv4l2/libv4l2.c
> +++ b/lib/libv4l2/libv4l2.c
> @@ -1,3 +1,4 @@
> +/* -*- c-file-style: "linux" -*- */

Don't add editor-specific macros inside the code. Not everybody uses emacs.

>  /*
>  #             (C) 2008 Hans de Goede <hdegoede@redhat.com>
>  
> @@ -100,6 +101,13 @@ static struct v4l2_dev_info devices[V4L2_MAX_DEVICES] = {
>  };
>  static int devices_used;
>  
> +#include "sdl.c"
> +
> +static struct sdl sdl;
> +
> +int v4l2_get_index(int fd);
> +void my_main(void);
> +

The above looks really odd. Why do you want to make libv4l2 dependent
on sdl?

>  static int v4l2_ensure_convert_mmap_buf(int index)
>  {
>  	if (devices[index].convert_mmap_buf != MAP_FAILED) {
> @@ -311,7 +319,409 @@ static int v4l2_queue_read_buffer(int index, int buffer_index)
>  	return 0;
>  }
>  
> -static int v4l2_dequeue_and_convert(int index, struct v4l2_buffer *buf,
> +static void v4l2_paint(char *buf, int x1, int y1, const struct v4l2_format *fmt)
> +{
> +  int y, x;
> +  int width = fmt->fmt.pix.width;
> +  printf("width = %d\n", width);
> +    
> +    for (y = 0; y < 25; y++) {
> +      for (x = 0; x < 80; x++) {
> +	int c = buf[width*4*y1 + 4*x1] +
> +	  buf[width*4*y1 + 4*x1 + 1] +
> +	  buf[width*4*y1 + 4*x1 + 2];
> +
> +	if (c < 30) c = ' ';
> +	else if (c < 60) c = '.';
> +	else if (c < 120) c = '_';
> +	else if (c < 180) c = 'o';
> +	else if (c < 300) c = 'x';
> +	else if (c < 400) c = 'X';
> +	else c = '#';
> +	  
> +	printf("%c", c);
> +      }
> +      printf("\n");
> +    }
> +}

Please don't add anything like that inside the library.

> +
> +#define SX 80
> +#define SY 25
> +#define BUCKETS 20
> +
> +static void v4l2_histogram(unsigned char *buf, int cdf[], struct v4l2_format *fmt)
> +{
> +    for (int y = 0; y < fmt->fmt.pix.height; y+=19)
> +      for (int x = 0; x < fmt->fmt.pix.width; x+=19) {
> +	pixel p = buf_pixel(fmt, buf, x, y);
> +	
> +	int b;
> +	/* HACK: we divide green by 2 to have nice picture, undo it here. */
> +	b = p.r + 2*p.g + p.b;
> +	b = (b * BUCKETS)/(256);
> +	cdf[b]++;
> +      }
> +}
> +
> +static long v4l2_sharpness(unsigned char *buf, struct v4l2_format *fmt)
> +{
> +  int h = fmt->fmt.pix.height;
> +  int w = fmt->fmt.pix.width;
> +  long r = 0;
> +
> +    for (int y = h/3; y < h-h/3; y+=h/9)
> +      for (int x = w/3; x < w-w/3; x++) {
> +	pixel p1 = buf_pixel(fmt, buf, x, y);
> +	pixel p2 = buf_pixel(fmt, buf, x+2, y);
> +	
> +	int b1, b2;
> +	/* HACK: we divide green by 2 to have nice picture, undo it here. */
> +	b1 = p1.r + 2*p1.g + p1.b;
> +	b2 = p2.r + 2*p2.g + p2.b;
> +
> +	int v;
> +	v = (b1-b2)*(b1-b2);
> +	if (v > 36)
> +		r+=v;
> +      }
> +
> +    return r;
> +}

IMO, the above belongs to a separate processing module under
	lib/libv4lconvert/processing/

> +
> +int v4l2_set_exposure(int fd, int exposure)
> +{
> +	int index = v4l2_get_index(fd);
> +
> +	if (index == -1 || devices[index].convert == NULL) {
> +		V4L2_LOG_ERR("v4l2_set_exposure called with invalid fd: %d\n", fd);
> +		errno = EBADF;
> +		return -1;
> +	}
> +
> +	struct v4l2_control ctrl;
> +	ctrl.id = V4L2_CID_EXPOSURE;
> +	ctrl.value = exposure;
> +	if (ioctl(devices[index].subdev_fds[0], VIDIOC_S_CTRL, &ctrl) < 0) {
> +	  printf("Could not set exposure\n");
> +	}
> +	return 0;
> +}

Shouldn't it be together with lib/libv4lconvert/processing/autogain.c,
perhaps as an alternative implementation, if what's there is not enough?

> +
> +int v4l2_set_gain(int fd, int gain)
> +{
> +	int index = v4l2_get_index(fd);
> +
> +	if (index == -1 || devices[index].convert == NULL) {
> +		V4L2_LOG_ERR("v4l2_set_exposure called with invalid fd: %d\n", fd);
> +		errno = EBADF;
> +		return -1;
> +	}
> +	
> +	struct v4l2_control ctrl;
> +	ctrl.id = 0x00980913;

Don't hardcode control numbers here.

> +	ctrl.value = gain;
> +	if (ioctl(devices[index].subdev_fds[0], VIDIOC_S_CTRL, &ctrl) < 0) {
> +	  printf("Could not set exposure\n");
> +	}
> +	return 0;
> +}
> +
> +int v4l2_set_focus(int fd, int diopt)
> +{
> +	int index = v4l2_get_index(fd);
> +
> +	if (index == -1 || devices[index].convert == NULL) {
> +		V4L2_LOG_ERR("v4l2_set_focus called with invalid fd: %d\n", fd);
> +		errno = EBADF;
> +		return -1;
> +	}
> +
> +	struct v4l2_control ctrl;
> +	ctrl.id = V4L2_CID_FOCUS_ABSOLUTE;
> +	ctrl.value = diopt;
> +	if (ioctl(devices[index].subdev_fds[1], VIDIOC_S_CTRL, &ctrl) < 0) {
> +		printf("Could not set focus\n");
> +	}
> +	return 0;
> +}
> +
> +#define LESS 1
> +#define MID 0
> +#define MORE 2
> +
> +static void v4l2_start_focus_step(struct v4l2_focus_step *fs, int focus, int step)
> +{
> +	int i;
> +
> +	fs->num = 3;
> +	for (i=0; i<V4L2_MAX_FOCUS; i++) {
> +		fs->sharpness[i] = -1;
> +		fs->position[i] = -1;
> +	}
> +
> +	fs->position[LESS] = focus - step;
> +	fs->position[MID] = focus;
> +	fs->position[MORE] = focus + step;
> +}
> +
> +static int v4l2_focus_get_best(struct v4l2_focus_step *fs)
> +{
> +	int i, max = -1, maxi = -1;
> +	
> +	for (i=0; i<fs->num; i++)
> +		if (max < fs->sharpness[i]) {
> +			max = fs->sharpness[i];
> +			maxi = i;
> +		}
> +
> +	return maxi;
> +}
> +
> +static void v4l2_auto_focus_continuous(struct v4l2_dev_info *m, int sh)
> +{
> +	int f = m->frame - m->focus_frame;
> +	int step;
> +	const int max_step = 300, min_step = 20;
> +	struct v4l2_focus_step *fs = &m->focus_step_params;
> +
> +	if (m->focus_step == 0 || m->focus_step > max_step) {
> +		printf("step reset -- max\n");
> +		m->focus_step = max_step;
> +	}
> +	if (m->focus_step < min_step) {
> +		printf("step reset -- 10 (%d)\n", m->focus_step);
> +		m->focus_step = min_step;
> +		/* It takes cca 5.7 seconds to achieve the focus:
> +		   0.76user 0.30system 5.66 (0m5.661s) elapsed 18.72%CPU
> +		 */

The above note seems hardware dependent. 

> +		printf("Focused at %d\n", m->focus);
> +		exit(0);
> +	}
> +
> +	step = m->focus_step;
> +
> +	if (m->exposure != m->focus_exposure) {
> +		m->focus_frame = m->frame;
> +		m->focus_exposure = m->exposure;
> +		v4l2_start_focus_step(fs, m->focus, m->focus_step);
> +		return;
> +	}
> +	if (m->focus < step) {
> +		m->focus = step;
> +	}
> +
> +	const int every = 3;
> +	if (f%every)
> +		return;
> +
> +	{
> +		int i = f/every;
> +
> +		if (i == 0) {
> +			printf("Can not happen?\n");
> +			return;
> +		}
> +		i--;
> +		if (i < fs->num)
> +			v4l2_set_focus(m->fd, fs->position[i]);
> +		if (i > 0)
> +			fs->sharpness[i-1] = sh;
> +		if (i < fs->num)
> +			return;
> +	}
> +	int i;
> +	for (i=0; i<fs->num; i++) {
> +		printf("%d: %d | ", fs->position[i], fs->sharpness[i]);
> +	}

You should probably print something only if log is enabled. Take a look
at lib/libv4l2/log.c. Same applies to similar printf() calls.

> +	int best = v4l2_focus_get_best(fs);
> +	if ((fs->sharpness[best] < m->sh_prev) && (m->focus_step < max_step)) {
> +		m->focus_step *=2;
> +		m->sh_prev = m->sh_prev * 0.9;
> +		printf("step up %d\n", m->focus_step);
> +	} else if (best == LESS) {
> +		printf("less ");
> +		m->focus -= step;
> +	} else if (best == MORE) {
> +		printf("more ");
> +		m->focus += step;
> +	} else {
> +		m->sh_prev = fs->sharpness[MID];
> +		m->focus_step = m->focus_step / 2;
> +		printf("step %d ", m->focus_step);
> +	}
> +	m->focus_frame = m->frame;
> +	v4l2_start_focus_step(fs, m->focus, m->focus_step);
> +	printf("Focus now %d\n", m->focus);
> +}
> +
> +static void v4l2_start_focus_sweep(struct v4l2_focus_step *fs, int focus, int step)
> +{
> +	int i;
> +
> +	fs->num = V4L2_MAX_FOCUS;
> +	for (i=0; i<V4L2_MAX_FOCUS; i++) {
> +		fs->sharpness[i] = -1;
> +		fs->position[i] = -1;
> +	}
> +
> +	int f = focus;
> +	for (i=0; i<V4L2_MAX_FOCUS; i++) {
> +		fs->position[i] = f;
> +		f += step;
> +	}
> +}
> +
> +static void v4l2_auto_focus_single(struct v4l2_dev_info *m, int sh)
> +{
> +	int f = m->frame - m->focus_frame;
> +	int step;
> +	struct v4l2_focus_step *fs = &m->focus_step_params;
> +
> +	if (m->focus_step == 0) {
> +		printf("step reset -- max\n");
> +		m->focus_step = 1;
> +	}
> +
> +	if (m->exposure != m->focus_exposure) {
> +		m->focus_frame = m->frame;
> +		m->focus_exposure = m->exposure;
> +		v4l2_start_focus_sweep(fs, 0, 100);
> +		return;
> +	}
> +
> +	const int every = 3;
> +	if (f%every)
> +		return;
> +
> +	{
> +		int i = f/every;
> +
> +		if (i == 0) {
> +			printf("Can not happen?\n");
> +			return;
> +		}
> +		i--;
> +		if (i < fs->num)
> +			v4l2_set_focus(m->fd, fs->position[i]);
> +		if (i > 0)
> +			fs->sharpness[i-1] = sh;
> +		if (i < fs->num)
> +			return;
> +	}
> +#if 0
> +	int i;
> +	for (i=0; i<fs->num; i++) {
> +		printf("%d: %d | ", fs->position[i], fs->sharpness[i]);
> +	}
> +#endif
> +	int best = v4l2_focus_get_best(fs);
> +	m->focus_frame = m->frame;
> +	switch (m->focus_step) {
> +	case 1:
> +		printf("Best now %d / %d\n", fs->position[best], fs->sharpness[best]);
> +		v4l2_start_focus_sweep(fs, fs->position[best] - 50, 10);
> +		m->focus_step = 2;
> +		break;
> +	case 2:
> +		printf("Best now %d / %d\n", fs->position[best], fs->sharpness[best]);
> +		v4l2_start_focus_sweep(fs, fs->position[best] - 10, 2);
> +		m->focus_step = 3;
> +		break;
> +	case 3:
> +		printf("Best now %d / %d\n", fs->position[best], fs->sharpness[best]);
> +		printf("done.\n");
> +		exit(0);
> +	}
> +}
> +
> +
> +static void v4l2_auto_exposure(int index, struct v4l2_buffer *buf)
> +{
> +	struct v4l2_format *fmt;
> +	int cdf[BUCKETS] = { 0, };
> +	int i;
> +
> +	fmt = &devices[index].src_fmt;
> +
> +	v4l2_histogram(devices[index].frame_pointers[buf->index], cdf, fmt);
> +
> +#if 0
> +	printf("hist: ");
> +	for (i = 0; i<BUCKETS; i++)
> +		printf("%d ", cdf[i]);
> +	printf("\n");
> +#endif
> +	for (i=1; i<BUCKETS; i++)
> +		cdf[i] += cdf[i-1];
> +
> +	int b = BUCKETS;
> +	int brightPixels = cdf[b-1] - cdf[b-8];
> +	int targetBrightPixels = cdf[b-1]/50;
> +	int maxSaturatedPixels = cdf[b-1]/200;
> +	int saturatedPixels = cdf[b-1] - cdf[b-2];
> +	// how much should I change brightness by
> +	float adjustment = 1.0f;
> +#if 0
> +	printf( "AutoExposure: totalPixels: %d,"
> +		"brightPixels: %d, targetBrightPixels: %d,"
> +		"saturatedPixels: %d, maxSaturatedPixels: %d\n",
> +		cdf[b-1], brightPixels, targetBrightPixels,
> +		saturatedPixels, maxSaturatedPixels);
> +#endif
> +	  
> +	if (saturatedPixels > maxSaturatedPixels) {
> +		// first don't let things saturate too much
> +		adjustment = 1.0f - ((float)(saturatedPixels - maxSaturatedPixels))/cdf[b-1];
> +	} else if (brightPixels < (targetBrightPixels - (saturatedPixels * 4))) {
> +		// increase brightness to try and hit the desired number of well exposed pixels
> +		int l = b-6;
> +		while (brightPixels < targetBrightPixels && l > 0) {
> +			brightPixels += cdf[l];
> +			brightPixels -= cdf[l-1];
> +			l--;
> +		}
> +
> +		// that level is supposed to be at b-11;
> +		adjustment = ((float) (b-6+1))/(l+1);
> +	} else {
> +		// we're not oversaturated, and we have enough bright pixels. Do nothing.
> +	}
> +
> +	{
> +		float limit = 4;
> +		if (adjustment > limit) { adjustment = limit; }
> +		if (adjustment < 1/limit) { adjustment = 1/limit; }
> +	}
> +	  
> +	if (!devices[index].exposure)
> +		devices[index].exposure = 1;
> +	devices[index].exposure *= adjustment;
> +	if (adjustment != 1.)
> +		printf( "AutoExposure: adjustment: %f exposure %d\n", adjustment, devices[index].exposure);
> +
> +	v4l2_set_exposure(devices[index].fd, devices[index].exposure);
> +}
> +
> +static void v4l2_statistics(int index, struct v4l2_buffer *buf)
> +{
> +	unsigned char *b;
> +	struct v4l2_format *fmt;
> +
> +	fmt = &devices[index].src_fmt;
> +	b = devices[index].frame_pointers[buf->index];
> +
> +	if (!(devices[index].frame%3))
> +		v4l2_auto_exposure(index, buf);
> +	
> +	int sh = v4l2_sharpness(b, fmt);
> +	v4l2_auto_focus_single(&devices[index], sh);
> +	
> +	devices[index].frame++;
> +	if (!(devices[index].frame%4))
> +		sdl_render(&sdl, b, fmt);
> +}
> +
> +int v4l2_dequeue_and_convert(int index, struct v4l2_buffer *buf,
>  		unsigned char *dest, int dest_size)
>  {
>  	const int max_tries = V4L2_IGNORE_FIRST_FRAME_ERRORS + 1;
> @@ -345,6 +755,13 @@ static int v4l2_dequeue_and_convert(int index, struct v4l2_buffer *buf,
>  			errno = -EINVAL;
>  			return -1;
>  		}
> +		
> +#if 1
> +		v4l2_statistics(index, buf);
> +#endif
> +#if 0
> +		/* This is rather major eater of CPU time. CPU time goes from 80% to 4% 
> +		   when conversion is disabled. */
>  
>  		result = v4lconvert_convert(devices[index].convert,
>  				&devices[index].src_fmt, &devices[index].dest_fmt,
> @@ -352,7 +769,7 @@ static int v4l2_dequeue_and_convert(int index, struct v4l2_buffer *buf,
>  				buf->bytesused, dest ? dest : (devices[index].convert_mmap_buf +
>  					buf->index * devices[index].convert_mmap_frame_size),
>  				dest_size);
> -
> +#endif
>  		if (devices[index].first_frame) {
>  			/* Always treat convert errors as EAGAIN during the first few frames, as
>  			   some cams produce bad frames at the start of the stream
> @@ -789,18 +1206,24 @@ no_capture:
>  
>  	/* Note we always tell v4lconvert to optimize src fmt selection for
>  	   our default fps, the only exception is the app explicitly selecting
> -	   a fram erate using the S_PARM ioctl after a S_FMT */
> +	   a frame rate using the S_PARM ioctl after a S_FMT */
>  	if (devices[index].convert)
>  		v4lconvert_set_fps(devices[index].convert, V4L2_DEFAULT_FPS);
>  	v4l2_update_fps(index, &parm);
>  
> +	devices[index].subdev_fds[0] = SYS_OPEN("/dev/video_sensor", O_RDWR, 0);
> +	devices[index].subdev_fds[1] = SYS_OPEN("/dev/video_focus", O_RDWR, 0);
> +
> +	printf("Sensor: %d, focus: %d\n", devices[index].subdev_fds[0], 
> +	       devices[index].subdev_fds[1]);
> +
>  	V4L2_LOG("open: %d\n", fd);
>  
>  	return fd;
>  }
>  
>  /* Is this an fd for which we are emulating v4l1 ? */
> -static int v4l2_get_index(int fd)
> +int v4l2_get_index(int fd)
>  {
>  	int index;
>  
> @@ -823,6 +1246,10 @@ int v4l2_close(int fd)
>  {
>  	int index, result;
>  
> +	if (fd == -2) {
> +	  my_main();
> +	}
> +

That looks a hack!

>  	index = v4l2_get_index(fd);
>  	if (index == -1)
>  		return SYS_CLOSE(fd);
> @@ -1782,3 +2209,65 @@ int v4l2_get_control(int fd, int cid)
>  			(qctrl.maximum - qctrl.minimum) / 2) /
>  		(qctrl.maximum - qctrl.minimum);
>  }
> +
> +void v4l2_debug(void)
> +{
> +	printf("debug\n");
> +}
> +
> +/* ------------------------------------------------------------------ */
> +
> +#define SIZE 10*1024*1024
> +
> +char buf[SIZE];
> +
> +void my_main(void)
> +{
> +	int fd = v4l2_open("/dev/video2", O_RDWR);
> +	int i;
> +	static struct v4l2_format fmt;
> +
> +	fmt.type = V4L2_BUF_TYPE_VIDEO_CAPTURE;
> +	fmt.fmt.pix.pixelformat = V4L2_PIX_FMT_RGB24;
> +	fmt.fmt.pix.field = V4L2_FIELD_INTERLACED;
> +	fmt.fmt.pix.width = 640;
> +	fmt.fmt.pix.height = 480;
> +
> +	v4l2_set_gain(fd, 300);
> +	v4l2_set_exposure(fd, 10000);
> +	v4l2_set_focus(fd, 0);
> +
> +
> +	printf("ioctl = %d\n", v4l2_ioctl(fd, VIDIOC_S_FMT, &fmt));
> +
> +	printf("capture is %d, %d\n", fmt.fmt.pix.width, fmt.fmt.pix.height);
> +	/* factor == 2 still fits window, but very slow. factor == 6 .. */
> +	/* factor needs to be odd, otherwise ... fun with BA10 format. */
> +	sdl_init(&sdl, fmt.fmt.pix.width, fmt.fmt.pix.height, 5);
> +  
> +	v4l2_debug();
> +
> +	/* In 800x600 "raw" mode, this should take cca 17.8 seconds (without
> +	   sdl output. CPU usage should be cca 5% without conversion). That's 28 fps.
> +	   (benchmark with i<500)
> +	*/
> +	for (i=0; i<50000; i++) {
> +		int num = v4l2_read(fd, buf, SIZE);
> +
> +		if (i==490) {
> +			printf("Focus to closest.... -------------------\n");
> +			v4l2_set_focus(fd, 99999);
> +		}
> +    
> +#if 0
> +		v4l2_paint(buf, 640/80, 480/25, &fmt);
> +#endif
> +		/* Over USB connection, rendering every single frame slows
> +		   execution down from 23 seconds to 36 seconds. */
> +#if 0
> +		if (!(i%4))
> +			sdl_render(&sdl, buf, &fmt);
> +#endif
> +	}
> +  
> +}
> diff --git a/lib/libv4l2/sdl.c b/lib/libv4l2/sdl.c
> new file mode 100644
> index 0000000..17a8d24
> --- /dev/null
> +++ b/lib/libv4l2/sdl.c
> @@ -0,0 +1,530 @@
> +/* -*- c-file-style: "linux" -*- */
> +/* SDL support.
> +
> +   Copyright 2017 Pavel Machek, LGPL
> +*/
> +
> +#include <SDL2/SDL.h>
> +#include <SDL2/SDL_image.h>

If you're adding a SDL-specific application, you'll need to add the 
needed autoconf bits to detect if SDL devel package is installed,
auto-disabling it if not.

Yet, I don't think that SDL should be part of the library, but,
instead, part of some application.

> +
> +struct sdl {
> +	SDL_Window *window;
> +	SDL_Surface *liveview, *screen;
> +
> +	int wx, wy;
> +	int sx, sy;
> +	int bx, by;
> +	int factor;
> +	float focus, gain, exposure, do_focus, do_exposure; 
> +};
> +
> +#if 0
> +void loop(void) {
> +	int done;
> +	SDL_Event event;
> +
> +	while(!done){ //While program isn't done                                   
> +		while(SDL_PollEvent(&event)){ //Poll events                        
> +			switch(event.type){ //Check event type                     
> +			case SDL_QUIT: //User hit the X (or equivelent)            
> +				done = true; //Make the loop end                   
> +				break; //We handled the event                      
> +			} //Finished with current event                            
> +		} //Done with all events for now                                   
> +	} //Program done, exited                                                   
> +}
> +#endif
> +
> +typedef struct {
> +	Uint8 r;
> +	Uint8 g;
> +	Uint8 b;
> +	Uint8 alpha;
> +} pixel;
> +
> +#define d_raw 1
> +
> +void sfc_put_pixel(SDL_Surface* liveview, int x, int y, pixel *p)
> +{
> +	Uint32* p_liveview = (Uint32*)liveview->pixels;
> +	p_liveview += y*liveview->w+x;
> +	*p_liveview = SDL_MapRGBA(liveview->format,p->r,p->g,p->b,p->alpha);
> +}
> +
> +#if 0
> +int pix_exposure(float x)
> +{
> +	return sy*x / 199410.0;
> +}
> +
> +int pix_gain(float x)
> +{
> +	return sy*x / 16.0;
> +}
> +
> +int render_statistics(SDL_Surface* liveview)
> +{
> +	pixel white;
> +	white.r = (Uint8)0xff;
> +	white.g = (Uint8)0xff;
> +	white.b = (Uint8)0xff;
> +	white.alpha = (Uint8)128;
> +
> +	//printf("Stats: focus %d, gain %d, exposure %d\n", focus, gain, exposure);
> +
> +	for (int x=0; x<sx && x<sx*focus; x++)
> +		put_pixel(liveview, x, 0, &white);
> +
> +	for (int y=0; y<sy && y<pix_gain(gain); y++)
> +		put_pixel(liveview, 0, y, &white);
> +
> +	for (int y=0; y<sy && y<pix_exposure(exposure); y++)
> +		put_pixel(liveview, sx-1, y, &white);
> +
> +	for (int x=0; x<sx; x++)
> +		put_pixel(liveview, x, sy-1, &white);
> +
> +	return 0;
> +}
> +#endif
> +
> +void sdl_begin_paint(struct sdl *m) {
> +	//Fill the surface white                                                   
> +	SDL_FillRect(m->liveview, NULL, SDL_MapRGB( m->liveview->format, 0, 0, 0 ));
> +
> +	SDL_LockSurface(m->liveview);
> +}
> +
> +void sdl_finish_paint(struct sdl *m) {
> +	SDL_UnlockSurface(m->liveview);
> +	SDL_Rect rcDest = { m->bx, m->by, m->sx, m->sy };
> +
> +	SDL_BlitSurface(m->liveview, NULL, m->screen, &rcDest);
> +	//Update the surface                                                       
> +	SDL_UpdateWindowSurfaceRects(m->window, &rcDest, 1);
> +}
> +  
> +void sdl_paint_image(struct sdl *m, char **xpm, int x, int y) {
> +	SDL_Surface *image = IMG_ReadXPMFromArray(xpm);
> +	if (!image) {
> +		printf("IMG_Load: %s\n", IMG_GetError());
> +		exit(1);
> +	}
> +
> +	int x_pos = x - image->w/2, y_pos = y - image->h/2;
> +
> +	SDL_Rect rcDest = { x_pos, y_pos, image->w, image->h };
> +	int r = SDL_BlitSurface ( image, NULL, m->screen, &rcDest );
> +
> +	if (r) {
> +		printf("Error blitting: %s\n", SDL_GetError());
> +		exit(1);
> +	}
> +	SDL_FreeSurface ( image );
> +}
> +
> +void sdl_paint_ui(struct sdl *m) {
> +	static char *wait_xpm[] = {
> +		"16 9 2 1",
> +		"# c #ffffff",
> +		". c #000000",
> +		"....########....",
> +		".....#....#.....",
> +		".....#....#.....",
> +		"......#..#......",
> +		".......##.......",
> +		"......#..#......",
> +		".....#....#.....",
> +		".....#....#.....",
> +		"....########....",
> +	};
> +
> +	static char *ok_xpm[] = {
> +		"16 9 2 1",
> +		"# c #ffffff",
> +		". c #000000",
> +		"...............#",
> +		"............###.",
> +		"..........##....",
> +		"#.......##......",
> +		".#.....#........",
> +		"..#...#.........",
> +		"..#..#..........",
> +		"...##...........",
> +		"...#............",
> +	};
> +
> +	static char *exit_xpm[] = {
> +		"16 9 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		"....x......x....",
> +		".....x....x.....",
> +		"......x..x......",
> +		".......xx.......",
> +		".......xx.......",
> +		"......x..x......",
> +		".....x....x.....",
> +		"....x......x....",
> +		"................",
> +	};
> +
> +	static char *f1m_xpm[] = {
> +		"16 9 2 1",
> +		"# c #ffffff",
> +		". c #000000",
> +		"....##..........",
> +		"...#.#..........",
> +		"..#..#..........",
> +		".....#...#.#.##.",
> +		".....#...##.#..#",
> +		".....#...#..#..#",
> +		".....#...#..#..#",
> +		".....#...#..#..#",
> +		"................",
> +	};
> +
> +	static char *f25cm_xpm[] = {
> +		"16 9 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		".xxx..xxxx......",
> +		"x...x.x.........",
> +		"...x..xxx.......",
> +		"..x......x..xx.x",
> +		".x.......x.x.xxx",
> +		"xxxxx.xxx...xxxx",
> +		"................",
> +		"................",
> +		"................",
> +	};
> +
> +	static char *iso400_xpm[] = {
> +		"16 12 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		"x..x.xxxx.xxxx..",
> +		"x..x.x..x.x..x..",
> +		"xxxx.x..x.x..x..",
> +		"...x.x..x.x..x..",
> +		"...x.xxxx.xxxx..",
> +		"................",
> +		".x..xx..x.......",
> +		".x.x...x.x......",
> +		".x..x..x.x......",
> +		".x...x.x.x......",
> +		".x.xx...x.......",
> +		"................",
> +	};
> +
> +	static char *time_1_100_xpm[] = {
> +		"16 12 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		".x....x.........",
> +		".x...x..........",
> +		".x..x...........",
> +		".x.x............",
> +		"................",
> +		"..x.xxxx.xxxx...",
> +		"..x.x..x.x..x...",
> +		"..x.x..x.x..x...",
> +		"..x.x..x.x..x...",
> +		"..x.x..x.x..x...",
> +		"..x.xxxx.xxxx...",
> +		"................",
> +	};
> +
> +	static char *time_1_10_xpm[] = {
> +		"16 12 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		".x....x.........",
> +		".x...x..........",
> +		".x..x...........",
> +		".x.x............",
> +		"................",
> +		"..x.xxxx........",
> +		"..x.x..x........",
> +		"..x.x..x........",
> +		"..x.x..x........",
> +		"..x.x..x........",
> +		"..x.xxxx........",
> +		"................",
> +	};
> +
> +	static char *af_xpm[] = {
> +		"16 12 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		".....xxxxxxx....",
> +		".....x..........",
> +		".....x..........",
> +		".x...xxxxx......",
> +		"x.x..x..........",
> +		"xxx..x..........",
> +		"x.x..x..........",
> +		"x.x..x..........",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +	};
> +
> +	static char *ae_xpm[] = {
> +		"16 12 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		".....xxxxxxx....",
> +		".....x..........",
> +		".....x..........",
> +		".x...xxxxx......",
> +		"x.x..x..........",
> +		"xxx..x..........",
> +		"x.x..x..........",
> +		"x.x..xxxxxxx....",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +	};
> +    
> +	static char *not_xpm[] = {
> +		"16 12 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		"......xxxxx.....",
> +		"....xx.....xx...",
> +		"...x.........x..",
> +		"..x........xx.x.",
> +		"..x......xx...x.",
> +		".x.....xx......x",
> +		".x...xx........x",
> +		"..xxx.........x.",
> +		"..x...........x.",
> +		"...x.........x..",
> +		"....xx.....xx...",
> +		"......xxxxx.....",
> +	};
> +
> +	static char *empty_xpm[] = {
> +		"16 12 2 1",
> +		"x c #ffffff",
> +		". c #000000",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +		"................",
> +	};
> +
> +	SDL_FillRect(m->screen, NULL, SDL_MapRGB( m->liveview->format, 0, 0, 0 ));
> +	sdl_paint_image(m, wait_xpm,  m->wx/2,     m->wy/2);
> +	sdl_paint_image(m, ok_xpm,    m->wx-m->bx/2,  m->wy-m->by/2);
> +	sdl_paint_image(m, exit_xpm,  m->bx/2,     m->wy-m->by/2);
> +	sdl_paint_image(m, f1m_xpm,   m->bx+m->sx/20, m->by/2);
> +	sdl_paint_image(m, f25cm_xpm, m->bx+m->sx/5,  m->by/2);
> +
> +	sdl_paint_image(m, af_xpm,    m->bx/2,     m->by/2);
> +	if (!m->do_focus) {
> +		sdl_paint_image(m, not_xpm,  16+m->bx/2,     m->by/2);      
> +	}
> +	sdl_paint_image(m, ae_xpm,    m->wx-m->bx/2,  m->by/2);
> +	if (!m->do_exposure) {
> +		sdl_paint_image(m, not_xpm,  16+m->bx/2,     m->by/2);      
> +	}
> +
> +#if 0
> +	sdl_paint_image(m, time_1_100_xpm, m->wx-m->bx/2, m->by+pix_exposure(10000));
> +	sdl_paint_image(m, time_1_10_xpm,  m->wx-m->bx/2, m->by+pix_exposure(100000));
> +	sdl_paint_image(m, iso400_xpm,     m->bx/2,    m->by+pix_gain(4.0));
> +#endif
> +		
> +	SDL_UpdateWindowSurface(m->window);
> +}
> +
> +void fmt_print(struct v4l2_format *fmt)
> +{
> +	int f;
> +	printf("Format: %dx%d. ", fmt->fmt.pix.width, fmt->fmt.pix.height);
> +	printf("%x ", fmt->fmt.pix.pixelformat);
> +	f = fmt->fmt.pix.pixelformat;
> +	for (int i=0; i<4; i++) {
> +		printf("%c", f & 0xff);
> +		f >>= 8;
> +	}
> +	printf("\n");
> +}
> +
> +pixel buf_pixel(struct v4l2_format *fmt, unsigned char *buf, int x, int y)
> +{
> +	pixel p = { 0, 0, 0, 0 };
> +	int pos = x + y*fmt->fmt.pix.width;
> +	int b;
> +
> +	p.alpha = 128;
> +
> +	switch (fmt->fmt.pix.pixelformat) {
> +	case '01AB': /* BA10, aka GRBG10, 
> +			https://www.linuxtv.org/downloads/v4l-dvb-apis-new/uapi/v4l/pixfmt-srggb10.html?highlight=ba10
> +		     */
> +		b = buf[pos*2];
> +		b += buf[pos*2+1] << 8;
> +		b /= 4;
> +		if ((y % 2) == 0) {
> +			switch (x % 2) {
> +			case 0: p.g = b/2; break;
> +			case 1: p.r = b; break;
> +			}
> +		} else {
> +			switch (x % 2) {
> +			case 0: p.b = b; break;
> +			case 1: p.g = b/2; break;
> +			}
> +		}
> +		break;
> +
> +	case V4L2_PIX_FMT_RGB24:
> +		pos *= 4;
> +		p.r = buf[pos];
> +		p.g = buf[pos+1];
> +		p.b = buf[pos+2];
> +		break;
> +
> +	default:
> +		printf("Wrong pixel format!\n");
> +		fmt_print(fmt);
> +		exit(1);
> +	}
> +
> +	return p;
> +}
> +
> +void sdl_render(struct sdl *m, unsigned char *buf, struct v4l2_format *fmt)
> +{
> +	if (!m->window) 
> +		return;
> +	sdl_begin_paint(m);    
> +	/* do your rendering here */
> +
> +	for (int y = 0; y < m->sy; y++)
> +		for (int x = 0; x < m->sx; x++) {
> +			pixel p = buf_pixel(fmt, buf, x*m->factor, y*m->factor);
> +			p.alpha = 128;
> +			sfc_put_pixel(m->liveview, x, y, &p);
> +		}
> +
> +	//render_statistics(liveview);
> +
> +	sdl_finish_paint(m);
> +}
> +
> +#if 0
> +void imageStatistics(FCam::Image img)
> +{
> +	int sx, sy;
> +
> +	sx = img.size().width;                                                          
> +	sy = img.size().height;                                                         
> +	printf("Image is %d x %d, ", sx, sy);                                           
> +
> +	printf("(%d) ", img.brightness(sx/2, sy/2));
> +
> +	int dark = 0;                                                                   
> +	int bright = 0;                                                                 
> +	int total = 0;                                                                  
> +#define RATIO 10
> +	for (int y = sy/(2*RATIO); y < sy; y += sy/RATIO)                               
> +		for (int x = sx/(2*RATIO); x < sx; x += sx/RATIO) {                    
> +			int br = img.brightness(x, y);                               
> +
> +			if (!d_raw) {
> +				/* It seems real range is 60 to 218 */
> +				if (br > 200)
> +					bright++;                                               
> +				if (br < 80)
> +					dark++;
> +			} else {
> +				/* there's a lot of noise, it seems black is commonly 65..71,
> +				   bright is cca 1023 */
> +				if (br > 1000)
> +					bright++;
> +				if (br < 75)
> +					dark++;
> +			}
> +			total++;                                                      
> +		}
> +
> +	printf("%d dark %d bri,", dark, bright);                     
> +
> +	long sharp = 0;
> +	for (int y = sy/3; y < 2*(sy/3); y+=sy/12) {
> +		int b = -1;
> +		for (int x = sx/3; x < 2*(sx/3); x++) {
> +			int b2;                                                                
> +			b2 = img.brightness(x, y/2);                                          
> +			if (b != -1)
> +				sharp += (b-b2) * (b-b2);                                              
> +			b = b2;                                                                
> +		}
> +	}
> +	printf("sh %d\n", sharp);
> +}
> +#endif
> +
> +
> +void sdl_init(struct sdl *m, int _sx, int _sy, int _factor)
> +{
> +	printf("Initing SDL\n");
> +
> +	if (SDL_Init(SDL_INIT_VIDEO) < 0) {
> +		printf("Could not init SDL\n");
> +		exit(1);
> +	}
> +
> +	atexit(SDL_Quit);
> +
> +	m->wx = 800;
> +	m->wy = 400;
> +
> +	m->window = SDL_CreateWindow( "Camera", SDL_WINDOWPOS_UNDEFINED,
> +				      SDL_WINDOWPOS_UNDEFINED, m->wx, m->wy,
> +				      SDL_WINDOW_SHOWN );
> +	if (m->window == NULL) {
> +		printf( "Window could not be created! SDL_Error: %s\n", SDL_GetError() );
> +	}
> +
> +	m->screen = SDL_GetWindowSurface(m->window);
> +	if (!m->screen) {
> +		printf("Couldn't create liveview\n");
> +		exit(1);
> +	}
> +
> +	m->sx = _sx;
> +	m->sy = _sy;
> +	m->factor = _factor;
> +
> +	m->sx /= m->factor;
> +	m->sy /= m->factor;
> +    
> +	m->focus = 0;
> +	m->gain = 0;
> +	m->exposure = 0;
> +
> +	m->bx = (m->wx-m->sx)/2;
> +	m->by = (m->wy-m->sy)/2;
> +
> +	m->liveview = SDL_CreateRGBSurface(0,m->sx,m->sy,32,0,0,0,0);
> +	if (!m->liveview) {
> +		printf("Couldn't create liveview\n");
> +		exit(1);
> +	}
> +	sdl_paint_ui(m);
> +}
> diff --git a/lib/libv4lconvert/libv4lconvert.c b/lib/libv4lconvert/libv4lconvert.c
> index d3d8936..7521ec8 100644
> --- a/lib/libv4lconvert/libv4lconvert.c
> +++ b/lib/libv4lconvert/libv4lconvert.c
> @@ -1205,6 +1205,8 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
>  			v4lconvert_swap_uv(src, dest, fmt);
>  			break;
>  		}
> +		default:
> +		  /* This is bad, fixme? */
>  		break;
>  
>  	case V4L2_PIX_FMT_YVU420:
> @@ -1228,6 +1230,8 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
>  		case V4L2_PIX_FMT_YVU420:
>  			memcpy(dest, src, width * height * 3 / 2);
>  			break;
> +		default:
> +		  /* This is bad, fixme? */
>  		}
>  		break;
>  
> diff --git a/utils/decode_tm6000/Makefile.am b/utils/decode_tm6000/Makefile.am
> index ac4e85e..f059e3c 100644
> --- a/utils/decode_tm6000/Makefile.am
> +++ b/utils/decode_tm6000/Makefile.am
> @@ -1,4 +1,4 @@
>  bin_PROGRAMS = decode_tm6000
>  decode_tm6000_SOURCES = decode_tm6000.c
> -decode_tm6000_LDADD = ../libv4l2util/libv4l2util.la
> +decode_tm6000_LDADD = ../libv4l2/libv4l2.la
>  decode_tm6000_LDFLAGS = $(ARGP_LIBS)
> diff --git a/utils/decode_tm6000/decode_tm6000.c b/utils/decode_tm6000/decode_tm6000.c
> index 4bffbdd..fda7e3b 100644
> --- a/utils/decode_tm6000/decode_tm6000.c
> +++ b/utils/decode_tm6000/decode_tm6000.c

Everything below it is completely wrong!

> @@ -1,365 +1,16 @@
> -/*
> -   decode_tm6000.c - decode multiplexed format from TM5600/TM6000 USB
> +/* gcc asciicam.c /usr/lib/i386-linux-gnu/libv4l2.so.0.0.0 -o asciicam
> +   gcc asciicam.c /usr/lib/arm-linux-gnueabi/libv4l2.so.0 -o asciicam
>  
> -   Copyright (C) 2007 Mauro Carvalho Chehab <mchehab@infradead.org>
> -
> -   This program is free software; you can redistribute it and/or modify
> -   it under the terms of the GNU General Public License as published by
> -   the Free Software Foundation version 2.
> -
> -   This program is distributed in the hope that it will be useful,
> -   but WITHOUT ANY WARRANTY; without even the implied warranty of
> -   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
> -   GNU General Public License for more details.
> -
> -   You should have received a copy of the GNU General Public License
> -   along with this program; if not, write to the Free Software
> -   Foundation, Inc., 51 Franklin Street, Suite 500, Boston, MA 02110-1335 USA.
> +gcc -std=gnu99 -DHAVE_CONFIG_H -I. -I../.. -fvisibility=hidden -I../../lib/include -Wall -Wpointer-arith -D_GNU_SOURCE -I../../include -g -O2 asciicam.c libv4l2.c /usr/lib/arm-linux-gnueabi/libv4lconvert.so.0 log.c v4l2convert.c v4l2-plugin.c -o asciicam
>   */
> -#include "../libv4l2util/v4l2_driver.h"
>  #include <stdio.h>
> -#include <string.h>
> -#include <argp.h>
> -#include <unistd.h>
>  #include <sys/types.h>
>  #include <sys/stat.h>
>  #include <fcntl.h>
> -#include <stdlib.h>
> -#include <errno.h>
> -
> -const char *argp_program_version="decode_tm6000 version 0.0.1";
> -const char *argp_program_bug_address="Mauro Carvalho Chehab <mchehab@infradead.org>";
> -const char doc[]="Decodes tm6000 proprietary format streams";
> -const struct argp_option options[] = {
> -	{"verbose",	'v',	0,	0,	"enables debug messages", 0},
> -	{"device",	'd',	"DEV",	0,	"uses device for reading", 0},
> -	{"output",	'o',	"FILE",	0,	"outputs raw stream to a file", 0},
> -	{"input",	'i',	"FILE",	0,	"parses a file, instead of a device", 0},
> -	{"freq",	'f',	"Freq",	0,	"station frequency, in MHz (default is 193.25)", 0},
> -	{"nbufs",	'n',	"quant",0,	"number of video buffers", 0},
> -	{"audio",	'a',	0,	0,	"outputs audio on stdout", 0},
> -	{"read",	'r',	0,	0,	"use read() instead of mmap method", 0},
> -	{ 0, 0, 0, 0, 0, 0 }
> -};
> -
> -static char outbuf[692224];
> -static int debug=0, audio=0, use_mmap=1, nbufs=4;
> -static float freq_mhz=193.25;
> -static char *devicename="/dev/video0";
> -static char *filename=NULL;
> -static enum {
> -	NORMAL,
> -	INPUT,
> -	OUTPUT
> -} mode = NORMAL;
> -
> -static FILE *fout;
> -
> -//const char args_doc[]="ARG1 ARG2";
> -
> -static error_t parse_opt (int key, char *arg, struct argp_state *state)
> -{
> -	switch (key) {
> -	case 'a':
> -		audio++;
> -		break;
> -	case 'r':
> -		use_mmap=0;
> -		break;
> -	case 'v':
> -		debug++;
> -		break;
> -	case 'd':
> -		devicename=arg;
> -		break;
> -	case 'i':
> -	case 'o':
> -		if (mode!=NORMAL) {
> -			argp_error(state,"You can't use input/output options simultaneously.\n");
> -			break;
> -		}
> -		if (key=='i')
> -			mode=INPUT;
> -		else
> -			mode=OUTPUT;
> -
> -		filename=arg;
> -		break;
> -	case 'f':
> -		freq_mhz=atof(arg);
> -		break;
> -	case 'n':
> -		nbufs=atoi(arg);
> -		if  (nbufs<2)
> -			nbufs=2;
> -		break;
> -	default:
> -		return ARGP_ERR_UNKNOWN;
> -	}
> -	return 0;
> -}
> -
> -static struct argp argp = {
> -	.options=options,
> -	.parser=parse_opt,
> -	.args_doc=NULL,
> -	.doc=doc,
> -};
>  
> -#define TM6000_URB_MSG_LEN 180
> -enum {
> -	TM6000_URB_MSG_VIDEO=1,
> -	TM6000_URB_MSG_AUDIO,
> -	TM6000_URB_MSG_VBI,
> -	TM6000_URB_MSG_PTS,
> -	TM6000_URB_MSG_ERR,
> -};
> -
> -const char *tm6000_msg_type[]= {
> -	"unknown(0)",	//0
> -	"video",	//1
> -	"audio",	//2
> -	"vbi",		//3
> -	"pts",		//4
> -	"err",		//5
> -	"unknown(6)",	//6
> -	"unknown(7)",	//7
> -};
> -
> -#define dprintf(fmt,arg...) \
> -	if (debug) fprintf(stderr, fmt, ##arg)
> -
> -static int recebe_buffer (struct v4l2_buffer *v4l2_buf, struct v4l2_t_buf *buf)
> -{
> -	dprintf("Received %zd bytes\n", buf->length);
> -fflush(stdout);
> -	memcpy (outbuf,buf->start,buf->length);
> -	return buf->length;
> -}
> -
> -
> -static int prepare_read (struct v4l2_driver *drv)
> -{
> -	struct v4l2_format fmt;
> -	double freq;
> -	int rc;
> -
> -	memset (drv,0,sizeof(*drv));
> -
> -	if (v4l2_open (devicename, 1,drv)<0) {
> -		perror ("Error opening dev");
> -		return -1;
> -	}
> -
> -	memset (&fmt,0,sizeof(fmt));
> -
> -	uint32_t pixelformat=V4L2_PIX_FMT_TM6000;
> -
> -	if (v4l2_gettryset_fmt_cap (drv,V4L2_SET,&fmt, 720, 480,
> -				    pixelformat,V4L2_FIELD_ANY)) {
> -		perror("set_input to tm6000 raw format");
> -		return -1;
> -	}
> -
> -	if (freq_mhz) {
> -		freq=freq_mhz * 1000 * 1000;
> -		rc=v4l2_getset_freq (drv,V4L2_SET, &freq);
> -		if (rc<0)
> -			printf ("Cannot set freq to %.3f MHz\n",freq_mhz);
> -	}
> -
> -	if (use_mmap) {
> -		printf("Preparing for receiving frames on %d buffers...\n",nbufs);
> -		fflush (stdout);
> -
> -		rc=v4l2_mmap_bufs(drv, nbufs);
> -		if (rc<0) {
> -			printf ("Cannot mmap %d buffers\n",nbufs);
> -			return -1;
> -		}
> -
> -//		v4l2_stop_streaming(&drv);
> -		rc=v4l2_start_streaming(drv);
> -		if (rc<0) {
> -			printf ("Cannot start streaming\n");
> -			return -1;
> -		}
> -	}
> -	printf("Waiting for frames...\n");
> -
> -	return 0;
> -}
> +#include <linux/videodev2.h>
>  
> -static int read_stream (struct v4l2_driver *drv, int fd)
> +void main(void)
>  {
> -	if (use_mmap) {
> -		fd_set fds;
> -		struct timeval tv;
> -		int r;
> -
> -		FD_ZERO (&fds);
> -		FD_SET (fd, &fds);
> -
> -		/* Timeout. */
> -		tv.tv_sec = 2;
> -		tv.tv_usec = 0;
> -
> -		r = select (fd + 1, &fds, NULL, NULL, &tv);
> -		if (-1 == r) {
> -			if (EINTR == errno) {
> -				perror ("select");
> -				return -errno;
> -			}
> -		}
> -
> -		if (0 == r) {
> -			fprintf (stderr, "select timeout\n");
> -			return -errno;
> -		}
> -
> -		return v4l2_rcvbuf(drv, recebe_buffer);
> -	} else {
> -		int size=read(fd, outbuf, sizeof(outbuf));
> -		return size;
> -	}
> -
> -	return 0;
> -}
> -
> -static int read_char (struct v4l2_driver *drv, int fd)
> -{
> -	static int sizebuf=0;
> -	static unsigned char *p=NULL;
> -	unsigned char c;
> -
> -	if (sizebuf<=0) {
> -		sizebuf=read_stream(drv,fd);
> -		if (sizebuf<=0)
> -			return -1;
> -		p=(unsigned char *)outbuf;
> -	}
> -	c=*p;
> -	p++;
> -	sizebuf--;
> -
> -	return c;
> -}
> -
> -
> -int main (int argc, char*argv[])
> -{
> -	int fd;
> -	unsigned int i;
> -	unsigned char buf[TM6000_URB_MSG_LEN], img[720*2*480];
> -	unsigned int  cmd, size, field, block, line, pos=0;
> -	unsigned long header=0;
> -	int           linesize=720*2,skip=0;
> -	struct v4l2_driver drv;
> -
> -	argp_parse (&argp, argc, argv, 0, 0, 0);
> -
> -	if (mode!=INPUT) {
> -		if (prepare_read (&drv)<0)
> -			return -1;
> -		fd=drv.fd;
> -	} else {
> -		/*mode == INPUT */
> -
> -		fd=open(filename,O_RDONLY);
> -		if (fd<0) {
> -			perror ("error opening a file for parsing");
> -			return -1;
> -		}
> -		dprintf("file %s opened for parsing\n",filename);
> -		use_mmap=0;
> -	}
> -
> -	if (mode==OUTPUT) {
> -		fout=fopen(filename,"w");
> -		if (!fout) {
> -			perror ("error opening a file to write");
> -			return -1;
> -		}
> -		dprintf("file %s opened for output\n",filename);
> -
> -		do {
> -			size=read_stream (&drv,fd);
> -
> -			if (size<=0) {
> -				close (fd);
> -				return -1;
> -			}
> -			dprintf("writing %d bytes\n",size);
> -			fwrite(outbuf,1, size,fout);
> -//			fflush(fout);
> -		} while (1);
> -	}
> -
> -
> -	while (1) {
> -		skip=0;
> -		header=0;
> -		do {
> -			int c;
> -			c=read_char (&drv,fd);
> -			if (c<0) {
> -				perror("read");
> -				return -1;
> -			}
> -
> -			header=(header>>8)&0xffffff;  
> -			header=header|(c<<24);
> -			skip++;
> -		} while ( (((header>>24)&0xff) != 0x47) );
> -
> -		/* split the header fields */
> -		size  = (((header & 0x7e)<<1) -1) * 4;
> -		block = (header>>7) & 0xf;
> -		field = (header>>11) & 0x1;
> -		line  = (header>>12) & 0x1ff;
> -		cmd   = (header>>21) & 0x7;
> -
> -		/* Read the remaining buffer */
> -		for (i=0;i<sizeof(buf);i++) {
> -			int c;
> -			c=read_char (&drv,fd);
> -			if (c<0) {
> -				perror("read");
> -				return -1;
> -			}
> -			buf[i]=c;
> -		}
> -
> -		/* FIXME: Mounts the image as field0+field1
> -			* It should, instead, check if the user selected
> -			* entrelaced or non-entrelaced mode
> -			*/
> -		pos=((line<<1)+field)*linesize+
> -					block*TM6000_URB_MSG_LEN;
> -
> -			/* Prints debug info */
> -		dprintf("0x%08x (skip %d), %s size=%d, line=%d, field=%d, block=%d\n",
> -				(unsigned int)header, skip,
> -				 tm6000_msg_type[cmd],
> -				 size, line, field, block);
> -
> -		/* Don't allow to write out of the buffer */
> -		if (pos+sizeof(buf) > sizeof(img))
> -			cmd = TM6000_URB_MSG_ERR;
> -
> -		/* handles each different URB message */
> -		switch(cmd) {
> -		case TM6000_URB_MSG_VIDEO:
> -			/* Fills video buffer */
> -			memcpy(buf,&img[pos],sizeof(buf));
> -		case TM6000_URB_MSG_AUDIO:
> -			if (audio)
> -				fwrite(buf,sizeof(buf),1,stdout);
> -//		case TM6000_URB_MSG_VBI:
> -//		case TM6000_URB_MSG_PTS:
> -		break;
> -		}
> -	}
> -	close(fd);
> -	return 0;
> +  v4l2_close(-2);
>  }
> 



Thanks,
Mauro

support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!

> > For focus to be useful, we need autofocus implmented
> > somewhere. Unfortunately, v4l framework does not seem to provide good
> > place where to put autofocus. I believe, long-term, we'll need some
> > kind of "video server" providing this kind of services.
> > 
> > Anyway, we probably don't want autofocus in kernel (even through some
> > cameras do it in hardware), and we probably don't want autofocus in
> > each and every user application.
> > 
> > So what remains is libv4l2. 
> 
> IMO, the best place for autofocus is at libv4l2. Putting it on a
> separate "video server" application looks really weird for me.

Well... let me see. libraries are quite limited -- it is hard to open
files, or use threads/have custom main loop. It may be useful to
switch resolutions -- do autofocus/autogain at lower resolution, then
switch to high one for taking picture. It would be good to have that
in "system" code, but I'm not at all sure libv4l2 design will allow
that.

It would be good if application could say "render live camera into
this window" and only care about user interface, then say "give me a
high resolution jpeg". But that would require main loop in the
library...

It would be nice if more than one application could be accessing the
camera at the same time... (I.e. something graphical running preview
then using command line tool to grab a picture.) This one is
definitely not solveable inside a library...

> Btw, libv4l2 already has some autotools for auto gain and auto
> white balance. See the implementation under:
> 	lib/libv4lconvert/processing
> 
> The libv4l internal controls can be seen at:
> 	lib/libv4lconvert/control/libv4lcontrol.h
> 
> The ones implemented by the processing part of the library are:

Thanks for pointer, will take a look.

> > Now, this is in no way clean or complete,
> > and functionality provided by sdl.c and asciicam.c probably _should_
> > be in application, but... I'd like to get the code out there.
> > 
> > Oh and yes, I've canibalized decode_tm6000.c application instead of
> > introducing my own. Autotools scare me, sorry.
> 
> Why replace decode_tm6000.c by something else? If you want to add another
> test application, just place it on a new file.

Scary scary scary autotools ;-). Yes, I did rather lot of hacks, as
you noted below. I do development on n900, so not everything is easy.

> I added a few notes together with the code, pointing the main things
> I think it require changes, in order for me to do a better review
> at the code. I didn't test nor tried to check the algorithms inside,
> as the code, on its current state, requires rework and code cleanup.

Thanks, I'll take a look.

> Please don't add a new application under lib/. It is fine if you want
> some testing application, if the ones there aren't enough, but please
> place it under contrib/test/.
> 
> You should likely take a look at v4l2grab first, as it could have
> almost everything you would need.

Will take a look, thanks for pointer.

> IMHO, it would be better to use aalib. Btw, xawtv3 has a code example
> using it, under:
> 	console/ttv.c
> 
> As it already uses libv4l, prhaps you could use it, instead of adding
> a new ascii app.

No need to duplicate it, then. I was trying to quickly test video
works, this was before SDL.

> > +#include "sdl.c"
> > +
> > +static struct sdl sdl;
> > +
> > +int v4l2_get_index(int fd);
> > +void my_main(void);
> > +
> 
> The above looks really odd. Why do you want to make libv4l2 dependent
> on sdl?

I don't, but I had some nasty problems with linker; this should really
go into application but it refused to link. Scary libtool.

> > +static void v4l2_histogram(unsigned char *buf, int cdf[], struct v4l2_format *fmt)
> > +{
> > +    for (int y = 0; y < fmt->fmt.pix.height; y+=19)
> > +      for (int x = 0; x < fmt->fmt.pix.width; x+=19) {
> > +	pixel p = buf_pixel(fmt, buf, x, y);
> > +	
> > +	int b;
> > +	/* HACK: we divide green by 2 to have nice picture, undo it here. */
> > +	b = p.r + 2*p.g + p.b;
> > +	b = (b * BUCKETS)/(256);
> > +	cdf[b]++;
> > +      }
> > +}
> > +
> > +static long v4l2_sharpness(unsigned char *buf, struct v4l2_format *fmt)
> > +{
> > +  int h = fmt->fmt.pix.height;
> > +  int w = fmt->fmt.pix.width;
> > +  long r = 0;
> > +
> > +    for (int y = h/3; y < h-h/3; y+=h/9)
> > +      for (int x = w/3; x < w-w/3; x++) {
> > +	pixel p1 = buf_pixel(fmt, buf, x, y);
> > +	pixel p2 = buf_pixel(fmt, buf, x+2, y);
> > +	
> > +	int b1, b2;
> > +	/* HACK: we divide green by 2 to have nice picture, undo it here. */
> > +	b1 = p1.r + 2*p1.g + p1.b;
> > +	b2 = p2.r + 2*p2.g + p2.b;
> > +
> > +	int v;
> > +	v = (b1-b2)*(b1-b2);
> > +	if (v > 36)
> > +		r+=v;
> > +      }
> > +
> > +    return r;
> > +}
> 
> IMO, the above belongs to a separate processing module under
> 	lib/libv4lconvert/processing/

I guess so.

> > +
> > +int v4l2_set_exposure(int fd, int exposure)
> > +{
> > +	int index = v4l2_get_index(fd);
> > +
> > +	if (index == -1 || devices[index].convert == NULL) {
> > +		V4L2_LOG_ERR("v4l2_set_exposure called with invalid fd: %d\n", fd);
> > +		errno = EBADF;
> > +		return -1;
> > +	}
> > +
> > +	struct v4l2_control ctrl;
> > +	ctrl.id = V4L2_CID_EXPOSURE;
> > +	ctrl.value = exposure;
> > +	if (ioctl(devices[index].subdev_fds[0], VIDIOC_S_CTRL, &ctrl) < 0) {
> > +	  printf("Could not set exposure\n");
> > +	}
> > +	return 0;
> > +}
> 
> Shouldn't it be together with lib/libv4lconvert/processing/autogain.c,
> perhaps as an alternative implementation, if what's there is not
> enough?

I'll take a look, thanks.

> > @@ -823,6 +1246,10 @@ int v4l2_close(int fd)
> >  {
> >  	int index, result;
> >  
> > +	if (fd == -2) {
> > +	  my_main();
> > +	}
> > +
> 
> That looks a hack!

That is _the_ hack ;-). Yes, agreed, need to look at
processing/. .. when I get time.


> > +#include <SDL2/SDL.h>
> > +#include <SDL2/SDL_image.h>
> 
> If you're adding a SDL-specific application, you'll need to add the 
> needed autoconf bits to detect if SDL devel package is installed,
> auto-disabling it if not.
> 
> Yet, I don't think that SDL should be part of the library, but,
> instead, part of some application.

Agreed. libtool prevented me from doing the right thing.

> > index 4bffbdd..fda7e3b 100644
> > --- a/utils/decode_tm6000/decode_tm6000.c
> > +++ b/utils/decode_tm6000/decode_tm6000.c
> 
> Everything below it is completely wrong!

And most of the stuff above is, too :-). I wanted to get the code out
in case I won't have time...

Thanks,
									Pavel

-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170424/420fafa1/attachment.sig>

support autofocus / autogain in libv4l2

[Mauro Carvalho Chehab]

Em Mon, 24 Apr 2017 23:29:14 +0200
Pavel Machek <pavel@ucw.cz> escreveu:

> Hi!
> 
> > > For focus to be useful, we need autofocus implmented
> > > somewhere. Unfortunately, v4l framework does not seem to provide good
> > > place where to put autofocus. I believe, long-term, we'll need some
> > > kind of "video server" providing this kind of services.
> > > 
> > > Anyway, we probably don't want autofocus in kernel (even through some
> > > cameras do it in hardware), and we probably don't want autofocus in
> > > each and every user application.
> > > 
> > > So what remains is libv4l2.   
> > 
> > IMO, the best place for autofocus is at libv4l2. Putting it on a
> > separate "video server" application looks really weird for me.  
> 
> Well... let me see. libraries are quite limited -- it is hard to open
> files, or use threads/have custom main loop. It may be useful to
> switch resolutions -- do autofocus/autogain at lower resolution, then
> switch to high one for taking picture. It would be good to have that
> in "system" code, but I'm not at all sure libv4l2 design will allow
> that.

I don't see why it would be hard to open files or have threads inside
a library. There are several libraries that do that already, specially
the ones designed to be used on multimidia apps.

Resolution switch can indeed be a problem on devices that use MC
and subdev API, as a plugin would be required to teach the library
about N9 specifics (or the Kernel API should be improved to let
a generic application to better detect the hardware capabilities).

> It would be good if application could say "render live camera into
> this window" and only care about user interface, then say "give me a
> high resolution jpeg". But that would require main loop in the
> library...

Nothing prevents writing an upper layer on the top of libv4l in
order to provide such kind of functions.

> It would be nice if more than one application could be accessing the
> camera at the same time... (I.e. something graphical running preview
> then using command line tool to grab a picture.) This one is
> definitely not solveable inside a library...

Someone once suggested to have something like pulseaudio for V4L.
For such usage, a server would be interesting. Yet, I would code it
in a way that applications using libv4l will talk with such daemon
in a transparent way.

> > The above looks really odd. Why do you want to make libv4l2 dependent
> > on sdl?  
> 
> I don't, but I had some nasty problems with linker; this should really
> go into application but it refused to link. Scary libtool.

That's weird. 


> > If you're adding a SDL-specific application, you'll need to add the 
> > needed autoconf bits to detect if SDL devel package is installed,
> > auto-disabling it if not.
> > 
> > Yet, I don't think that SDL should be part of the library, but,
> > instead, part of some application.  
> 
> Agreed. libtool prevented me from doing the right thing.

if you add libSDL detection at configure.ac, you likely won't need to
deal with libtool.

On a quick look at web, it seems that there's a m4 module that does
the right thing, according with:
	https://wiki.libsdl.org/FAQLinux


Thanks,
Mauro

support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!

> > > > For focus to be useful, we need autofocus implmented
> > > > somewhere. Unfortunately, v4l framework does not seem to provide good
> > > > place where to put autofocus. I believe, long-term, we'll need some
> > > > kind of "video server" providing this kind of services.
> > > > 
> > > > Anyway, we probably don't want autofocus in kernel (even through some
> > > > cameras do it in hardware), and we probably don't want autofocus in
> > > > each and every user application.
> > > > 
> > > > So what remains is libv4l2.   
> > > 
> > > IMO, the best place for autofocus is at libv4l2. Putting it on a
> > > separate "video server" application looks really weird for me.  
> > 
> > Well... let me see. libraries are quite limited -- it is hard to open
> > files, or use threads/have custom main loop. It may be useful to
> > switch resolutions -- do autofocus/autogain at lower resolution, then
> > switch to high one for taking picture. It would be good to have that
> > in "system" code, but I'm not at all sure libv4l2 design will allow
> > that.
> 
> I don't see why it would be hard to open files or have threads inside
> a library. There are several libraries that do that already, specially
> the ones designed to be used on multimidia apps.

Well, fd's are hard, because application can do fork() and now
interesting stuff happens. Threads are tricky, because now you have
locking etc.

libv4l2 is designed to be LD_PRELOADED. That is not really feasible
with "complex" library.

> > It would be good if application could say "render live camera into
> > this window" and only care about user interface, then say "give me a
> > high resolution jpeg". But that would require main loop in the
> > library...
> 
> Nothing prevents writing an upper layer on the top of libv4l in
> order to provide such kind of functions.

Agreed.

> > It would be nice if more than one application could be accessing the
> > camera at the same time... (I.e. something graphical running preview
> > then using command line tool to grab a picture.) This one is
> > definitely not solveable inside a library...
> 
> Someone once suggested to have something like pulseaudio for V4L.
> For such usage, a server would be interesting. Yet, I would code it
> in a way that applications using libv4l will talk with such daemon
> in a transparent way.

Yes, we need something like pulseaudio for V4L. And yes, we should
make it transparent for applications using libv4l.

Regards,
									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/576eccb4/attachment.sig>

support autofocus / autogain in libv4l2

[Pali Rohár]

On Tuesday 25 April 2017 10:05:38 Pavel Machek wrote:
> > > It would be nice if more than one application could be accessing the
> > > camera at the same time... (I.e. something graphical running preview
> > > then using command line tool to grab a picture.) This one is
> > > definitely not solveable inside a library...
> > 
> > Someone once suggested to have something like pulseaudio for V4L.
> > For such usage, a server would be interesting. Yet, I would code it
> > in a way that applications using libv4l will talk with such daemon
> > in a transparent way.
> 
> Yes, we need something like pulseaudio for V4L. And yes, we should
> make it transparent for applications using libv4l.

IIRC there is already some effort in writing such "video" server which
would support accessing more application into webcam video, like
pulseaudio server for accessing more applications to microphone input.

-- 
Pali Roh?r
pali.rohar at gmail.com

support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!
On Tue 2017-04-25 10:08:15, Pali Roh?r wrote:
> On Tuesday 25 April 2017 10:05:38 Pavel Machek wrote:
> > > > It would be nice if more than one application could be accessing the
> > > > camera at the same time... (I.e. something graphical running preview
> > > > then using command line tool to grab a picture.) This one is
> > > > definitely not solveable inside a library...
> > > 
> > > Someone once suggested to have something like pulseaudio for V4L.
> > > For such usage, a server would be interesting. Yet, I would code it
> > > in a way that applications using libv4l will talk with such daemon
> > > in a transparent way.
> > 
> > Yes, we need something like pulseaudio for V4L. And yes, we should
> > make it transparent for applications using libv4l.
> 
> IIRC there is already some effort in writing such "video" server which
> would support accessing more application into webcam video, like
> pulseaudio server for accessing more applications to microphone input.

Do you have project name / url / something?

Thanks,
									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/b6562bbd/attachment.sig>

support autofocus / autogain in libv4l2

[Pali Rohár]

On Tuesday 25 April 2017 13:23:30 Pavel Machek wrote:
> Hi!
> On Tue 2017-04-25 10:08:15, Pali Roh?r wrote:
> > On Tuesday 25 April 2017 10:05:38 Pavel Machek wrote:
> > > > > It would be nice if more than one application could be accessing the
> > > > > camera at the same time... (I.e. something graphical running preview
> > > > > then using command line tool to grab a picture.) This one is
> > > > > definitely not solveable inside a library...
> > > > 
> > > > Someone once suggested to have something like pulseaudio for V4L.
> > > > For such usage, a server would be interesting. Yet, I would code it
> > > > in a way that applications using libv4l will talk with such daemon
> > > > in a transparent way.
> > > 
> > > Yes, we need something like pulseaudio for V4L. And yes, we should
> > > make it transparent for applications using libv4l.
> > 
> > IIRC there is already some effort in writing such "video" server which
> > would support accessing more application into webcam video, like
> > pulseaudio server for accessing more applications to microphone input.
> 
> Do you have project name / url / something?

Pinos (renamed from PulseVideo)

https://blogs.gnome.org/uraeus/2015/06/30/introducing-pulse-video/
https://cgit.freedesktop.org/~wtay/pinos/

But from git history it looks like it is probably dead now...

-- 
Pali Roh?r
pali.rohar at gmail.com

support autofocus / autogain in libv4l2

[Pavel Machek]

On Tue 2017-04-25 13:30:09, Pali Roh?r wrote:
> On Tuesday 25 April 2017 13:23:30 Pavel Machek wrote:
> > Hi!
> > On Tue 2017-04-25 10:08:15, Pali Roh?r wrote:
> > > On Tuesday 25 April 2017 10:05:38 Pavel Machek wrote:
> > > > > > It would be nice if more than one application could be accessing the
> > > > > > camera at the same time... (I.e. something graphical running preview
> > > > > > then using command line tool to grab a picture.) This one is
> > > > > > definitely not solveable inside a library...
> > > > > 
> > > > > Someone once suggested to have something like pulseaudio for V4L.
> > > > > For such usage, a server would be interesting. Yet, I would code it
> > > > > in a way that applications using libv4l will talk with such daemon
> > > > > in a transparent way.
> > > > 
> > > > Yes, we need something like pulseaudio for V4L. And yes, we should
> > > > make it transparent for applications using libv4l.
> > > 
> > > IIRC there is already some effort in writing such "video" server which
> > > would support accessing more application into webcam video, like
> > > pulseaudio server for accessing more applications to microphone input.
> > 
> > Do you have project name / url / something?
> 
> Pinos (renamed from PulseVideo)
> 
> https://blogs.gnome.org/uraeus/2015/06/30/introducing-pulse-video/
> https://cgit.freedesktop.org/~wtay/pinos/
> 
> But from git history it looks like it is probably dead now...

Actually, last commit is an hour ago on "work" branch. Seems alive to
me ;-).

Thanks for pointer...
									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/cdcc4df0/attachment.sig>

support autofocus / autogain in libv4l2

[Pali Rohár]

On Tuesday 25 April 2017 14:28:20 Pavel Machek wrote:
> On Tue 2017-04-25 13:30:09, Pali Roh?r wrote:
> > On Tuesday 25 April 2017 13:23:30 Pavel Machek wrote:
> > > Hi!
> > > On Tue 2017-04-25 10:08:15, Pali Roh?r wrote:
> > > > On Tuesday 25 April 2017 10:05:38 Pavel Machek wrote:
> > > > > > > It would be nice if more than one application could be accessing the
> > > > > > > camera at the same time... (I.e. something graphical running preview
> > > > > > > then using command line tool to grab a picture.) This one is
> > > > > > > definitely not solveable inside a library...
> > > > > > 
> > > > > > Someone once suggested to have something like pulseaudio for V4L.
> > > > > > For such usage, a server would be interesting. Yet, I would code it
> > > > > > in a way that applications using libv4l will talk with such daemon
> > > > > > in a transparent way.
> > > > > 
> > > > > Yes, we need something like pulseaudio for V4L. And yes, we should
> > > > > make it transparent for applications using libv4l.
> > > > 
> > > > IIRC there is already some effort in writing such "video" server which
> > > > would support accessing more application into webcam video, like
> > > > pulseaudio server for accessing more applications to microphone input.
> > > 
> > > Do you have project name / url / something?
> > 
> > Pinos (renamed from PulseVideo)
> > 
> > https://blogs.gnome.org/uraeus/2015/06/30/introducing-pulse-video/
> > https://cgit.freedesktop.org/~wtay/pinos/
> > 
> > But from git history it looks like it is probably dead now...
> 
> Actually, last commit is an hour ago on "work" branch. Seems alive to
> me ;-).

Great! I just (blindly) looked at master branch and it is old...

> Thanks for pointer...
> 									Pavel

-- 
Pali Roh?r
pali.rohar at gmail.com

support autofocus / autogain in libv4l2

[Nicolas Dufresne]

Le mardi 25 avril 2017 ? 13:30 +0200, Pali Roh?r a ?crit?:
> Pinos (renamed from PulseVideo)
> 
> https://blogs.gnome.org/uraeus/2015/06/30/introducing-pulse-video/
> https://cgit.freedesktop.org/~wtay/pinos/
> 
> But from git history it looks like it is probably dead now...

This is also incorrect. See "work" branch. It is still a one man show,
code being aggressively re-factored. I suspect this will be the case
until the "form" is considered acceptable.

Nicolas
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: This is a digitally signed message part
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/7a9df95e/attachment.sig>

support autofocus / autogain in libv4l2

[Nicolas Dufresne]

Le mardi 25 avril 2017 ? 10:08 +0200, Pali Roh?r a ?crit?:
> On Tuesday 25 April 2017 10:05:38 Pavel Machek wrote:
> > > > It would be nice if more than one application could be
> > > > accessing the
> > > > camera at the same time... (I.e. something graphical running
> > > > preview
> > > > then using command line tool to grab a picture.) This one is
> > > > definitely not solveable inside a library...
> > > 
> > > Someone once suggested to have something like pulseaudio for V4L.
> > > For such usage, a server would be interesting. Yet, I would code
> > > it
> > > in a way that applications using libv4l will talk with such
> > > daemon
> > > in a transparent way.
> > 
> > Yes, we need something like pulseaudio for V4L. And yes, we should
> > make it transparent for applications using libv4l.
> 
> IIRC there is already some effort in writing such "video" server
> which
> would support accessing more application into webcam video, like
> pulseaudio server for accessing more applications to microphone
> input.
> 

Because references are nice:

https://blogs.gnome.org/uraeus/2015/06/30/introducing-pulse-video/
https://gstconf.ubicast.tv/videos/camera-sharing-and-sandboxing-with-pinos/

And why the internals are not going to be implemented using GStreamer in the end:
https://gstconf.ubicast.tv/videos/keep-calm-and-refactor-about-the-essence-of-gstreamer/

regards,
Nicolas
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: This is a digitally signed message part
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/c6b3f95c/attachment-0001.sig>

support autofocus / autogain in libv4l2

[Nicolas Dufresne]

Le mardi 25 avril 2017 ? 10:05 +0200, Pavel Machek a ?crit?:
> Well, fd's are hard, because application can do fork() and now
> interesting stuff happens. Threads are tricky, because now you have
> locking etc.
> 
> libv4l2 is designed to be LD_PRELOADED. That is not really feasible
> with "complex" library.

That is incorrect. The library propose an API where you simply replace
certain low level calls, like ioctl -> v4l2_ioctl, open -> v4l2_open().
You have to do that explicitly in your existing code. It does not
abstract the API itself unlike libdrm.

Nicolas
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: This is a digitally signed message part
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/82d4ed25/attachment.sig>

support autofocus / autogain in libv4l2

[Pavel Machek]

On Tue 2017-04-25 12:53:27, Nicolas Dufresne wrote:
> Le mardi 25 avril 2017 ? 10:05 +0200, Pavel Machek a ?crit?:
> > Well, fd's are hard, because application can do fork() and now
> > interesting stuff happens. Threads are tricky, because now you have
> > locking etc.
> > 
> > libv4l2 is designed to be LD_PRELOADED. That is not really feasible
> > with "complex" library.
> 
> That is incorrect. The library propose an API where you simply replace
> certain low level calls, like ioctl -> v4l2_ioctl, open -> v4l2_open().
> You have to do that explicitly in your existing code. It does not
> abstract the API itself unlike libdrm.

You are right, no LD_PRELOAD. But same API as kernel, which is really
limiting -- see my other mail.

									Pavel

-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170426/022f73cb/attachment.sig>

[patch] propagating controls in libv4l2 was Re: support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!

> > > IMO, the best place for autofocus is at libv4l2. Putting it on a
> > > separate "video server" application looks really weird for me.  
> > 
> > Well... let me see. libraries are quite limited -- it is hard to open
> > files, or use threads/have custom main loop. It may be useful to
> > switch resolutions -- do autofocus/autogain at lower resolution, then
> > switch to high one for taking picture. It would be good to have that
> > in "system" code, but I'm not at all sure libv4l2 design will allow
> > that.
> 
> I don't see why it would be hard to open files or have threads inside
> a library. There are several libraries that do that already, specially
> the ones designed to be used on multimidia apps.

Well, This is what the libv4l2 says:

   This file implements libv4l2, which offers v4l2_ prefixed versions
   of
      open/close/etc. The API is 100% the same as directly opening
   /dev/videoX
      using regular open/close/etc, the big difference is that format
   conversion
   
but if I open additional files in v4l2_open(), API is no longer the
same, as unix open() is defined to open just one file descriptor.

Now. There is autogain support in libv4lconvert, but it expects to use
same fd for camera and for the gain... which does not work with
subdevs.

Of course, opening subdevs by name like this is not really
acceptable. But can you suggest a method that is?

Thanks,
								Pavel

commit 4cf9d10ead014c0db25452e4bb9cd144632407c3
Author: Pavel <pavel@ucw.cz>
Date:   Wed Apr 26 11:38:04 2017 +0200

    Add subdevices.

diff --git a/lib/libv4l2/libv4l2-priv.h b/lib/libv4l2/libv4l2-priv.h
index 343db5e..a6bc48e 100644
--- a/lib/libv4l2/libv4l2-priv.h
+++ b/lib/libv4l2/libv4l2-priv.h
@@ -26,6 +26,7 @@
 #include "../libv4lconvert/libv4lsyscall-priv.h"
 
 #define V4L2_MAX_DEVICES 16
+#define V4L2_MAX_SUBDEVS 8
 /* Warning when making this larger the frame_queued and frame_mapped members of
    the v4l2_dev_info struct can no longer be a bitfield, so the code needs to
    be adjusted! */
@@ -104,6 +105,7 @@ struct v4l2_dev_info {
 	void *plugin_library;
 	void *dev_ops_priv;
 	const struct libv4l_dev_ops *dev_ops;
+        int subdev_fds[V4L2_MAX_SUBDEVS];
 };
 
 /* From v4l2-plugin.c */
diff --git a/lib/libv4l2/libv4l2.c b/lib/libv4l2/libv4l2.c
index 0ba0a88..edc9642 100644
--- a/lib/libv4l2/libv4l2.c
+++ b/lib/libv4l2/libv4l2.c
@@ -1,3 +1,4 @@
+/* -*- c-file-style: "linux" -*- */
 /*
 #             (C) 2008 Hans de Goede <hdegoede@redhat.com>
 
@@ -789,18 +790,25 @@ no_capture:
 
 	/* Note we always tell v4lconvert to optimize src fmt selection for
 	   our default fps, the only exception is the app explicitly selecting
-	   a fram erate using the S_PARM ioctl after a S_FMT */
+	   a frame rate using the S_PARM ioctl after a S_FMT */
 	if (devices[index].convert)
 		v4lconvert_set_fps(devices[index].convert, V4L2_DEFAULT_FPS);
 	v4l2_update_fps(index, &parm);
 
+	devices[index].subdev_fds[0] = SYS_OPEN("/dev/video_sensor", O_RDWR, 0);
+	devices[index].subdev_fds[1] = SYS_OPEN("/dev/video_focus", O_RDWR, 0);
+	devices[index].subdev_fds[2] = -1;
+
+	printf("Sensor: %d, focus: %d\n", devices[index].subdev_fds[0], 
+	       devices[index].subdev_fds[1]);
+
 	V4L2_LOG("open: %d\n", fd);
 
 	return fd;
 }
 
 /* Is this an fd for which we are emulating v4l1 ? */
-static int v4l2_get_index(int fd)
+int v4l2_get_index(int fd)
 {
 	int index;
 

commit 1d6a9ce121f53e8f2e38549eed597a3c3dea5233
Author: Pavel <pavel@ucw.cz>
Date:   Wed Apr 26 12:34:04 2017 +0200

    Enable ioctl propagation.

diff --git a/lib/libv4l2/libv4l2.c b/lib/libv4l2/libv4l2.c
index edc9642..6dab661 100644
--- a/lib/libv4l2/libv4l2.c
+++ b/lib/libv4l2/libv4l2.c
@@ -1064,6 +1064,23 @@ static int v4l2_s_fmt(int index, struct v4l2_format *dest_fmt)
 	return 0;
 }
 
+static int v4l2_propagate_ioctl(int index, unsigned long request, void *arg)
+{
+	int i = 0;
+	int result;
+	while (1) {
+		if (devices[index].subdev_fds[i] == -1)
+			return -1;
+		printf("g_ctrl failed, trying...\n");
+		result = SYS_IOCTL(devices[index].subdev_fds[i], request, arg);
+		printf("subdev %d result %d\n", i, result);
+		if (result == 0)
+			return 0;
+		i++;
+	}
+	return -1;
+}
+
 int v4l2_ioctl(int fd, unsigned long int request, ...)
 {
 	void *arg;
@@ -1193,14 +1210,20 @@ no_capture_request:
 	switch (request) {
 	case VIDIOC_QUERYCTRL:
 		result = v4lconvert_vidioc_queryctrl(devices[index].convert, arg);
+		if (result == -1)
+			result = v4l2_propagate_ioctl(index, request, arg);
 		break;
 
 	case VIDIOC_G_CTRL:
 		result = v4lconvert_vidioc_g_ctrl(devices[index].convert, arg);
+		if (result == -1)
+			result = v4l2_propagate_ioctl(index, request, arg);
 		break;
 
 	case VIDIOC_S_CTRL:
 		result = v4lconvert_vidioc_s_ctrl(devices[index].convert, arg);
+		if (result == -1)
+			result = v4l2_propagate_ioctl(index, request, arg);
 		break;
 
 	case VIDIOC_G_EXT_CTRLS:


-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170426/d174a739/attachment-0001.sig>

[patch] propagating controls in libv4l2 was Re: support autofocus / autogain in libv4l2

[Mauro Carvalho Chehab]

Hi Pavel,

Em Wed, 26 Apr 2017 12:53:00 +0200
Pavel Machek <pavel@ucw.cz> escreveu:

> Hi!
> 
> > > > IMO, the best place for autofocus is at libv4l2. Putting it on a
> > > > separate "video server" application looks really weird for me.    
> > > 
> > > Well... let me see. libraries are quite limited -- it is hard to open
> > > files, or use threads/have custom main loop. It may be useful to
> > > switch resolutions -- do autofocus/autogain at lower resolution, then
> > > switch to high one for taking picture. It would be good to have that
> > > in "system" code, but I'm not at all sure libv4l2 design will allow
> > > that.  
> > 
> > I don't see why it would be hard to open files or have threads inside
> > a library. There are several libraries that do that already, specially
> > the ones designed to be used on multimidia apps.  
> 
> Well, This is what the libv4l2 says:
> 
>    This file implements libv4l2, which offers v4l2_ prefixed versions
>    of
>       open/close/etc. The API is 100% the same as directly opening
>    /dev/videoX
>       using regular open/close/etc, the big difference is that format
>    conversion
>    
> but if I open additional files in v4l2_open(), API is no longer the
> same, as unix open() is defined to open just one file descriptor.
> 
> Now. There is autogain support in libv4lconvert, but it expects to use
> same fd for camera and for the gain... which does not work with
> subdevs.
> 
> Of course, opening subdevs by name like this is not really
> acceptable. But can you suggest a method that is?

There are two separate things here:

1) Autofoucs for a device that doesn't use subdev API
2) libv4l2 support for devices that require MC and subdev API

for (1), it should use the /dev/videoX device that was opened with
v4l2_open().

For (2), libv4l2 should be aware of MC and subdev APIs. Sakari
once tried to write a libv4l2 plugin for OMAP3, but never finished it.
A more recent trial were to add a libv4l2 plugin for Exynos.
Unfortunately, none of those code got merged. Last time I checked,
the Exynos plugin was almost ready to be merged, but Sakari asked
some changes on it. The developer that was working on it got job on
some other company. Last time I heard from him, he was still interested
on finishing his work, but in the need to setup a test environment
using his own devices.

So, currently, there's no code at all adding MC/subdev API
support merged at libv4l2.

-

IMHO, the right thing to do with regards to autofocus is to
implement it via a processing module, assuming that just one
video device is opened.

Then, add a N900 plugin to make libv4l2 aware of OMAP3 specifics.

After that, rework at the processing module to let it use a 
different file descriptor if such plugin is in usage.

-

The hole idea is that a libv4l2 client, running on a N900 device
would just open a fake /dev/video0. Internally, libv4l2 will
open whatever video nodes it needs to control the device, exporting
all hardware capabilities (video formats, controls, resolutions,
etc) as if it was a normal V4L2 camera, hiding all dirty details
about MC and subdev APIs from userspace application.

This way, a normal application, like xawtv, tvtime, camorama,
zbar, mplayer, vlc, ... will work without any changes.


> 
> Thanks,
> 								Pavel
> 
> commit 4cf9d10ead014c0db25452e4bb9cd144632407c3
> Author: Pavel <pavel@ucw.cz>
> Date:   Wed Apr 26 11:38:04 2017 +0200
> 
>     Add subdevices.
> 
> diff --git a/lib/libv4l2/libv4l2-priv.h b/lib/libv4l2/libv4l2-priv.h
> index 343db5e..a6bc48e 100644
> --- a/lib/libv4l2/libv4l2-priv.h
> +++ b/lib/libv4l2/libv4l2-priv.h
> @@ -26,6 +26,7 @@
>  #include "../libv4lconvert/libv4lsyscall-priv.h"
>  
>  #define V4L2_MAX_DEVICES 16
> +#define V4L2_MAX_SUBDEVS 8
>  /* Warning when making this larger the frame_queued and frame_mapped members of
>     the v4l2_dev_info struct can no longer be a bitfield, so the code needs to
>     be adjusted! */
> @@ -104,6 +105,7 @@ struct v4l2_dev_info {
>  	void *plugin_library;
>  	void *dev_ops_priv;
>  	const struct libv4l_dev_ops *dev_ops;
> +        int subdev_fds[V4L2_MAX_SUBDEVS];
>  };
>  
>  /* From v4l2-plugin.c */
> diff --git a/lib/libv4l2/libv4l2.c b/lib/libv4l2/libv4l2.c
> index 0ba0a88..edc9642 100644
> --- a/lib/libv4l2/libv4l2.c
> +++ b/lib/libv4l2/libv4l2.c
> @@ -1,3 +1,4 @@
> +/* -*- c-file-style: "linux" -*- */
>  /*
>  #             (C) 2008 Hans de Goede <hdegoede@redhat.com>
>  
> @@ -789,18 +790,25 @@ no_capture:
>  
>  	/* Note we always tell v4lconvert to optimize src fmt selection for
>  	   our default fps, the only exception is the app explicitly selecting
> -	   a fram erate using the S_PARM ioctl after a S_FMT */
> +	   a frame rate using the S_PARM ioctl after a S_FMT */
>  	if (devices[index].convert)
>  		v4lconvert_set_fps(devices[index].convert, V4L2_DEFAULT_FPS);
>  	v4l2_update_fps(index, &parm);
>  
> +	devices[index].subdev_fds[0] = SYS_OPEN("/dev/video_sensor", O_RDWR, 0);
> +	devices[index].subdev_fds[1] = SYS_OPEN("/dev/video_focus", O_RDWR, 0);
> +	devices[index].subdev_fds[2] = -1;
> +
> +	printf("Sensor: %d, focus: %d\n", devices[index].subdev_fds[0], 
> +	       devices[index].subdev_fds[1]);
> +
>  	V4L2_LOG("open: %d\n", fd);
>  
>  	return fd;
>  }
>  
>  /* Is this an fd for which we are emulating v4l1 ? */
> -static int v4l2_get_index(int fd)
> +int v4l2_get_index(int fd)
>  {
>  	int index;
>  
> 
> commit 1d6a9ce121f53e8f2e38549eed597a3c3dea5233
> Author: Pavel <pavel@ucw.cz>
> Date:   Wed Apr 26 12:34:04 2017 +0200
> 
>     Enable ioctl propagation.
> 
> diff --git a/lib/libv4l2/libv4l2.c b/lib/libv4l2/libv4l2.c
> index edc9642..6dab661 100644
> --- a/lib/libv4l2/libv4l2.c
> +++ b/lib/libv4l2/libv4l2.c
> @@ -1064,6 +1064,23 @@ static int v4l2_s_fmt(int index, struct v4l2_format *dest_fmt)
>  	return 0;
>  }
>  
> +static int v4l2_propagate_ioctl(int index, unsigned long request, void *arg)
> +{
> +	int i = 0;
> +	int result;
> +	while (1) {
> +		if (devices[index].subdev_fds[i] == -1)
> +			return -1;
> +		printf("g_ctrl failed, trying...\n");
> +		result = SYS_IOCTL(devices[index].subdev_fds[i], request, arg);
> +		printf("subdev %d result %d\n", i, result);
> +		if (result == 0)
> +			return 0;
> +		i++;
> +	}
> +	return -1;
> +}
> +
>  int v4l2_ioctl(int fd, unsigned long int request, ...)
>  {
>  	void *arg;
> @@ -1193,14 +1210,20 @@ no_capture_request:
>  	switch (request) {
>  	case VIDIOC_QUERYCTRL:
>  		result = v4lconvert_vidioc_queryctrl(devices[index].convert, arg);
> +		if (result == -1)
> +			result = v4l2_propagate_ioctl(index, request, arg);
>  		break;
>  
>  	case VIDIOC_G_CTRL:
>  		result = v4lconvert_vidioc_g_ctrl(devices[index].convert, arg);
> +		if (result == -1)
> +			result = v4l2_propagate_ioctl(index, request, arg);
>  		break;
>  
>  	case VIDIOC_S_CTRL:
>  		result = v4lconvert_vidioc_s_ctrl(devices[index].convert, arg);
> +		if (result == -1)
> +			result = v4l2_propagate_ioctl(index, request, arg);
>  		break;
>  
>  	case VIDIOC_G_EXT_CTRLS:
> 
> 



Thanks,
Mauro

[patch] autogain support for bayer10 format (was Re: [patch] propagating controls in libv4l2)

[Pavel Machek]

Hi!

> > > I don't see why it would be hard to open files or have threads inside
> > > a library. There are several libraries that do that already, specially
> > > the ones designed to be used on multimidia apps.  
> > 
> > Well, This is what the libv4l2 says:
> > 
> >    This file implements libv4l2, which offers v4l2_ prefixed versions
> >    of
> >       open/close/etc. The API is 100% the same as directly opening
> >    /dev/videoX
> >       using regular open/close/etc, the big difference is that format
> >    conversion
> >    
> > but if I open additional files in v4l2_open(), API is no longer the
> > same, as unix open() is defined to open just one file descriptor.
> > 
> > Now. There is autogain support in libv4lconvert, but it expects to use
> > same fd for camera and for the gain... which does not work with
> > subdevs.
> > 
> > Of course, opening subdevs by name like this is not really
> > acceptable. But can you suggest a method that is?
> 
> There are two separate things here:
> 
> 1) Autofoucs for a device that doesn't use subdev API
> 2) libv4l2 support for devices that require MC and subdev API

Actually there are three: 0) autogain. Unfortunately, I need autogain
first before autofocus has a chance...

And that means... bayer10 support for autogain.

Plus, I changed avg_lum to long long. Quick calculation tells me int
could overflow with few megapixel sensor.

Oh, btw http://ytse.tricolour.net/docs/LowLightOptimization.html no
longer works.

Regards,
								Pavel

diff --git a/lib/libv4lconvert/processing/autogain.c b/lib/libv4lconvert/processing/autogain.c
index c6866d6..0b52d0f 100644
--- a/lib/libv4lconvert/processing/autogain.c
+++ b/lib/libv4lconvert/processing/autogain.c
@@ -68,6 +71,41 @@ static void autogain_adjust(struct v4l2_queryctrl *ctrl, int *value,
 	}
 }
 
+static int get_luminosity_bayer10(uint16_t *buf, const struct v4l2_format *fmt)
+{
+	long long avg_lum = 0;
+	int x, y;
+	
+	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
+		fmt->fmt.pix.width / 4;
+
+	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
+		for (x = 0; x < fmt->fmt.pix.width / 2; x++)
+			avg_lum += *buf++;
+		buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
+	}
+	avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
+	avg_lum /= 4;
+	return avg_lum;
+}
+
+static int get_luminosity_bayer8(unsigned char *buf, const struct v4l2_format *fmt)
+{
+	long long avg_lum = 0;
+	int x, y;
+	
+	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
+		fmt->fmt.pix.width / 4;
+
+	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
+		for (x = 0; x < fmt->fmt.pix.width / 2; x++)
+			avg_lum += *buf++;
+		buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
+	}
+	avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
+	return avg_lum;
+}
+
 /* auto gain and exposure algorithm based on the knee algorithm described here:
 http://ytse.tricolour.net/docs/LowLightOptimization.html */
 static int autogain_calculate_lookup_tables(
@@ -100,17 +142,16 @@ static int autogain_calculate_lookup_tables(
 	switch (fmt->fmt.pix.pixelformat) {
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SBGGR10:
+	case V4L2_PIX_FMT_SRGGB10:
+		avg_lum = get_luminosity_bayer10((void *) buf, fmt);
+		break;
+
 	case V4L2_PIX_FMT_SGBRG8:
 	case V4L2_PIX_FMT_SGRBG8:
 	case V4L2_PIX_FMT_SBGGR8:
 	case V4L2_PIX_FMT_SRGGB8:
-		buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
-			fmt->fmt.pix.width / 4;
-
-		for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
-			for (x = 0; x < fmt->fmt.pix.width / 2; x++)
-				avg_lum += *buf++;
-			buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
-		}
-		avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
+		avg_lum = get_luminosity_bayer8(buf, fmt);
 		break;
 
 	case V4L2_PIX_FMT_RGB24:
diff --git a/lib/libv4lconvert/processing/libv4lprocessing.c b/lib/libv4lconvert/processing/libv4lprocessing.c
index b061f50..b98d024 100644
--- a/lib/libv4lconvert/processing/libv4lprocessing.c
+++ b/lib/libv4lconvert/processing/libv4lprocessing.c
@@ -164,6 +165,10 @@ void v4lprocessing_processing(struct v4lprocessing_data *data,
 	case V4L2_PIX_FMT_SGRBG8:
 	case V4L2_PIX_FMT_SBGGR8:
 	case V4L2_PIX_FMT_SRGGB8:
+	case V4L2_PIX_FMT_SGBRG10:
+	case V4L2_PIX_FMT_SGRBG10:
+	case V4L2_PIX_FMT_SBGGR10:
+	case V4L2_PIX_FMT_SRGGB10:
 	case V4L2_PIX_FMT_RGB24:
 	case V4L2_PIX_FMT_BGR24:
 		break;



-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170426/7ba6278a/attachment-0001.sig>

[patch] autogain support for bayer10 format (was Re: [patch] propagating controls in libv4l2)

[Ivaylo Dimitrov]

Hi,

On 26.04.2017 16:23, Pavel Machek wrote:
> Hi!
>
>>>> I don't see why it would be hard to open files or have threads inside
>>>> a library. There are several libraries that do that already, specially
>>>> the ones designed to be used on multimidia apps.
>>>
>>> Well, This is what the libv4l2 says:
>>>
>>>    This file implements libv4l2, which offers v4l2_ prefixed versions
>>>    of
>>>       open/close/etc. The API is 100% the same as directly opening
>>>    /dev/videoX
>>>       using regular open/close/etc, the big difference is that format
>>>    conversion
>>>
>>> but if I open additional files in v4l2_open(), API is no longer the
>>> same, as unix open() is defined to open just one file descriptor.
>>>
>>> Now. There is autogain support in libv4lconvert, but it expects to use
>>> same fd for camera and for the gain... which does not work with
>>> subdevs.
>>>
>>> Of course, opening subdevs by name like this is not really
>>> acceptable. But can you suggest a method that is?
>>
>> There are two separate things here:
>>
>> 1) Autofoucs for a device that doesn't use subdev API
>> 2) libv4l2 support for devices that require MC and subdev API
>
> Actually there are three: 0) autogain. Unfortunately, I need autogain
> first before autofocus has a chance...
>
> And that means... bayer10 support for autogain.
>
> Plus, I changed avg_lum to long long. Quick calculation tells me int
> could overflow with few megapixel sensor.
>
> Oh, btw http://ytse.tricolour.net/docs/LowLightOptimization.html no
> longer works.
>
> Regards,
> 								Pavel
>
> diff --git a/lib/libv4lconvert/processing/autogain.c b/lib/libv4lconvert/processing/autogain.c
> index c6866d6..0b52d0f 100644
> --- a/lib/libv4lconvert/processing/autogain.c
> +++ b/lib/libv4lconvert/processing/autogain.c
> @@ -68,6 +71,41 @@ static void autogain_adjust(struct v4l2_queryctrl *ctrl, int *value,
>  	}
>  }
>
> +static int get_luminosity_bayer10(uint16_t *buf, const struct v4l2_format *fmt)
> +{
> +	long long avg_lum = 0;
> +	int x, y;
> +	
> +	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> +		fmt->fmt.pix.width / 4;
> +
> +	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> +		for (x = 0; x < fmt->fmt.pix.width / 2; x++)

That would take some time :). AIUI, we have NEON support in ARM kernels 
(CONFIG_KERNEL_MODE_NEON), I wonder if it makes sense (me) to convert 
the above loop to NEON-optimized when it comes to it? Are there any 
drawbacks in using NEON code in kernel?

> +			avg_lum += *buf++;
> +		buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
> +	}
> +	avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
> +	avg_lum /= 4;
> +	return avg_lum;
> +}
> +
> +static int get_luminosity_bayer8(unsigned char *buf, const struct v4l2_format *fmt)
> +{
> +	long long avg_lum = 0;
> +	int x, y;
> +	
> +	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> +		fmt->fmt.pix.width / 4;
> +
> +	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> +		for (x = 0; x < fmt->fmt.pix.width / 2; x++)

ditto.

> +			avg_lum += *buf++;
> +		buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
> +	}
> +	avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
> +	return avg_lum;
> +}
> +
>  /* auto gain and exposure algorithm based on the knee algorithm described here:
>  http://ytse.tricolour.net/docs/LowLightOptimization.html */
>  static int autogain_calculate_lookup_tables(
> @@ -100,17 +142,16 @@ static int autogain_calculate_lookup_tables(
>  	switch (fmt->fmt.pix.pixelformat) {
> +	case V4L2_PIX_FMT_SGBRG10:
> +	case V4L2_PIX_FMT_SGRBG10:
> +	case V4L2_PIX_FMT_SBGGR10:
> +	case V4L2_PIX_FMT_SRGGB10:
> +		avg_lum = get_luminosity_bayer10((void *) buf, fmt);
> +		break;
> +
>  	case V4L2_PIX_FMT_SGBRG8:
>  	case V4L2_PIX_FMT_SGRBG8:
>  	case V4L2_PIX_FMT_SBGGR8:
>  	case V4L2_PIX_FMT_SRGGB8:
> -		buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> -			fmt->fmt.pix.width / 4;
> -
> -		for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> -			for (x = 0; x < fmt->fmt.pix.width / 2; x++)
> -				avg_lum += *buf++;
> -			buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
> -		}
> -		avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
> +		avg_lum = get_luminosity_bayer8(buf, fmt);
>  		break;
>
>  	case V4L2_PIX_FMT_RGB24:
> diff --git a/lib/libv4lconvert/processing/libv4lprocessing.c b/lib/libv4lconvert/processing/libv4lprocessing.c
> index b061f50..b98d024 100644
> --- a/lib/libv4lconvert/processing/libv4lprocessing.c
> +++ b/lib/libv4lconvert/processing/libv4lprocessing.c
> @@ -164,6 +165,10 @@ void v4lprocessing_processing(struct v4lprocessing_data *data,
>  	case V4L2_PIX_FMT_SGRBG8:
>  	case V4L2_PIX_FMT_SBGGR8:
>  	case V4L2_PIX_FMT_SRGGB8:
> +	case V4L2_PIX_FMT_SGBRG10:
> +	case V4L2_PIX_FMT_SGRBG10:
> +	case V4L2_PIX_FMT_SBGGR10:
> +	case V4L2_PIX_FMT_SRGGB10:
>  	case V4L2_PIX_FMT_RGB24:
>  	case V4L2_PIX_FMT_BGR24:
>  		break;
>
>
>

[patch] autogain support for bayer10 format (was Re: [patch] propagating controls in libv4l2)

[Pavel Machek]

Hi!

> >>There are two separate things here:
> >>
> >>1) Autofoucs for a device that doesn't use subdev API
> >>2) libv4l2 support for devices that require MC and subdev API
> >
> >Actually there are three: 0) autogain. Unfortunately, I need autogain
> >first before autofocus has a chance...
> >
> >And that means... bayer10 support for autogain.
> >
> >Plus, I changed avg_lum to long long. Quick calculation tells me int
> >could overflow with few megapixel sensor.
> >
> >Oh, btw http://ytse.tricolour.net/docs/LowLightOptimization.html no
> >longer works.
> >
> >Regards,
> >								Pavel
> >
> >diff --git a/lib/libv4lconvert/processing/autogain.c b/lib/libv4lconvert/processing/autogain.c
> >index c6866d6..0b52d0f 100644
> >--- a/lib/libv4lconvert/processing/autogain.c
> >+++ b/lib/libv4lconvert/processing/autogain.c
> >@@ -68,6 +71,41 @@ static void autogain_adjust(struct v4l2_queryctrl *ctrl, int *value,
> > 	}
> > }
> >
> >+static int get_luminosity_bayer10(uint16_t *buf, const struct v4l2_format *fmt)
> >+{
> >+	long long avg_lum = 0;
> >+	int x, y;
> >+	
> >+	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> >+		fmt->fmt.pix.width / 4;
> >+
> >+	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> >+		for (x = 0; x < fmt->fmt.pix.width / 2; x++)
> 
> That would take some time :). AIUI, we have NEON support in ARM kernels
> (CONFIG_KERNEL_MODE_NEON), I wonder if it makes sense (me) to convert the
> above loop to NEON-optimized when it comes to it? Are there any drawbacks in
> using NEON code in kernel?

Well, thanks for offer. This is actualy libv4l2.

But I'd say NEON conversion is not neccessary anytime soon. First,
this is just trying to get average luminosity. We can easily skip
quite a lot of pixels, and still get reasonable answer.

Second, omap3isp actually has a hardware block computing statistics
for us. We just don't use it for simplicity.

(But if you want to play with camera, I'll get you patches; there's
ton of work to be done, both kernel and userspace :-).

Regards,
								Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170427/664aa13a/attachment.sig>

[patch] autogain support for bayer10 format (was Re: [patch] propagating controls in libv4l2)

[Ivaylo Dimitrov]

On 27.04.2017 01:51, Pavel Machek wrote:
> Hi!
>
>>>> There are two separate things here:
>>>>
>>>> 1) Autofoucs for a device that doesn't use subdev API
>>>> 2) libv4l2 support for devices that require MC and subdev API
>>>
>>> Actually there are three: 0) autogain. Unfortunately, I need autogain
>>> first before autofocus has a chance...
>>>
>>> And that means... bayer10 support for autogain.
>>>
>>> Plus, I changed avg_lum to long long. Quick calculation tells me int
>>> could overflow with few megapixel sensor.
>>>
>>> Oh, btw http://ytse.tricolour.net/docs/LowLightOptimization.html no
>>> longer works.
>>>
>>> Regards,
>>> 								Pavel
>>>
>>> diff --git a/lib/libv4lconvert/processing/autogain.c b/lib/libv4lconvert/processing/autogain.c
>>> index c6866d6..0b52d0f 100644
>>> --- a/lib/libv4lconvert/processing/autogain.c
>>> +++ b/lib/libv4lconvert/processing/autogain.c
>>> @@ -68,6 +71,41 @@ static void autogain_adjust(struct v4l2_queryctrl *ctrl, int *value,
>>> 	}
>>> }
>>>
>>> +static int get_luminosity_bayer10(uint16_t *buf, const struct v4l2_format *fmt)
>>> +{
>>> +	long long avg_lum = 0;
>>> +	int x, y;
>>> +	
>>> +	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
>>> +		fmt->fmt.pix.width / 4;
>>> +
>>> +	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
>>> +		for (x = 0; x < fmt->fmt.pix.width / 2; x++)
>>
>> That would take some time :). AIUI, we have NEON support in ARM kernels
>> (CONFIG_KERNEL_MODE_NEON), I wonder if it makes sense (me) to convert the
>> above loop to NEON-optimized when it comes to it? Are there any drawbacks in
>> using NEON code in kernel?
>
> Well, thanks for offer. This is actualy libv4l2.
>

Oh, somehow I got confused that this is kernel code :)

> But I'd say NEON conversion is not neccessary anytime soon. First,
> this is just trying to get average luminosity. We can easily skip
> quite a lot of pixels, and still get reasonable answer.
>
> Second, omap3isp actually has a hardware block computing statistics
> for us. We just don't use it for simplicity.
>

Right, I forgot about that.

> (But if you want to play with camera, I'll get you patches; there's
> ton of work to be done, both kernel and userspace :-).

Well, I saw a low hanging fruit I thought I can convert to NEON in a day 
or two, while having some rest from the huge "project" I am devoting all 
my spare time recently (rebasing hildon/maemo 5 on top of devuan 
Jessie). Still, if there is something relatively small to be done, just 
email me and I'll have a look.

Regards,
Ivo

[patch] autogain support for bayer10 format (was Re: [patch] propagating controls in libv4l2)

[Pavel Machek]

Hi!

> Oh, somehow I got confused that this is kernel code :)
> 
> >But I'd say NEON conversion is not neccessary anytime soon. First,
> >this is just trying to get average luminosity. We can easily skip
> >quite a lot of pixels, and still get reasonable answer.
> >
> >Second, omap3isp actually has a hardware block computing statistics
> >for us. We just don't use it for simplicity.
> >
> 
> Right, I forgot about that.
> 
> >(But if you want to play with camera, I'll get you patches; there's
> >ton of work to be done, both kernel and userspace :-).
> 
> Well, I saw a low hanging fruit I thought I can convert to NEON in a day or
> two, while having some rest from the huge "project" I am devoting all my
> spare time recently (rebasing hildon/maemo 5 on top of devuan Jessie).
> Still, if there is something relatively small to be done, just email me and
> I'll have a look.

Well, there's a ton of work on camera, and some work on
libcmtspeechdata. The later is rather self-contained.
									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170713/01ea4d01/attachment.sig>

[patch] autogain support for bayer10 format (was Re: [patch] propagating controls in libv4l2)

[Russell King - ARM Linux]

On Wed, Apr 26, 2017 at 06:43:54PM +0300, Ivaylo Dimitrov wrote:
> >+static int get_luminosity_bayer10(uint16_t *buf, const struct v4l2_format *fmt)
> >+{
> >+	long long avg_lum = 0;
> >+	int x, y;
> >+	
> >+	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> >+		fmt->fmt.pix.width / 4;
> >+
> >+	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> >+		for (x = 0; x < fmt->fmt.pix.width / 2; x++)
> 
> That would take some time :). AIUI, we have NEON support in ARM kernels
> (CONFIG_KERNEL_MODE_NEON), I wonder if it makes sense (me) to convert the
> above loop to NEON-optimized when it comes to it? Are there any drawbacks in
> using NEON code in kernel?

Using neon without the VFP state saved and restored corrupts userspace's
FP state.  So, you have to save the entire VFP state to use neon in kernel
mode.  There are helper functions for this: kernel_neon_begin() and
kernel_neon_end().

You can't build C code with the compiler believing that neon is available
as the compiler could emit neon instructions in unprotected kernel code.

Note that kernel_neon_begin() is only allowed to be called outside
interrupt context and with preemption disabled.

Given that, do we really want to be walking over multi-megabytes of image
data in the kernel with preemption disabled - it sounds like a recipe for
a very sluggish system.  I think this should (and can only sensibly be
done) in userspace.

-- 
RMK's Patch system: http://www.armlinux.org.uk/developer/patches/
FTTC broadband for 0.8mile line: currently at 9.6Mbps down 400kbps up
according to speedtest.net.

[patch] autogain support for bayer10 format (was Re: [patch] propagating controls in libv4l2)

[Pavel Machek]

On Wed 2017-05-03 20:05:56, Russell King - ARM Linux wrote:
> On Wed, Apr 26, 2017 at 06:43:54PM +0300, Ivaylo Dimitrov wrote:
> > >+static int get_luminosity_bayer10(uint16_t *buf, const struct v4l2_format *fmt)
> > >+{
> > >+	long long avg_lum = 0;
> > >+	int x, y;
> > >+	
> > >+	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> > >+		fmt->fmt.pix.width / 4;
> > >+
> > >+	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> > >+		for (x = 0; x < fmt->fmt.pix.width / 2; x++)
> > 
> > That would take some time :). AIUI, we have NEON support in ARM kernels
> > (CONFIG_KERNEL_MODE_NEON), I wonder if it makes sense (me) to convert the
> > above loop to NEON-optimized when it comes to it? Are there any drawbacks in
> > using NEON code in kernel?
> 
> Using neon without the VFP state saved and restored corrupts userspace's
> FP state.  So, you have to save the entire VFP state to use neon in kernel
> mode.  There are helper functions for this: kernel_neon_begin() and
> kernel_neon_end().
...
> Given that, do we really want to be walking over multi-megabytes of image
> data in the kernel with preemption disabled - it sounds like a recipe for
> a very sluggish system.  I think this should (and can only sensibly be
> done) in userspace.

The patch was for libv4l2. (And I explained why we don't need to
overoptimize this.)
									Pavel

-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170503/e1091649/attachment.sig>

[patch] autogain support for bayer10 format (was Re: [patch] propagating controls in libv4l2)

[Pavel Machek]

On Wed 2017-04-26 15:23:37, Pavel Machek wrote:
> Hi!
> 
> > > > I don't see why it would be hard to open files or have threads inside
> > > > a library. There are several libraries that do that already, specially
> > > > the ones designed to be used on multimidia apps.  
> > > 
> > > Well, This is what the libv4l2 says:
> > > 
> > >    This file implements libv4l2, which offers v4l2_ prefixed versions
> > >    of
> > >       open/close/etc. The API is 100% the same as directly opening
> > >    /dev/videoX
> > >       using regular open/close/etc, the big difference is that format
> > >    conversion
> > >    
> > > but if I open additional files in v4l2_open(), API is no longer the
> > > same, as unix open() is defined to open just one file descriptor.
> > > 
> > > Now. There is autogain support in libv4lconvert, but it expects to use
> > > same fd for camera and for the gain... which does not work with
> > > subdevs.
> > > 
> > > Of course, opening subdevs by name like this is not really
> > > acceptable. But can you suggest a method that is?
> > 
> > There are two separate things here:
> > 
> > 1) Autofoucs for a device that doesn't use subdev API
> > 2) libv4l2 support for devices that require MC and subdev API
> 
> Actually there are three: 0) autogain. Unfortunately, I need autogain
> first before autofocus has a chance...
> 
> And that means... bayer10 support for autogain.
> 
> Plus, I changed avg_lum to long long. Quick calculation tells me int
> could overflow with few megapixel sensor.
> 
> Oh, btw http://ytse.tricolour.net/docs/LowLightOptimization.html no
> longer works.

Can I get some comments here? Patch will need fixup (constants need
adjusting), but is style/design acceptable?

Thanks,
								Pavel

> diff --git a/lib/libv4lconvert/processing/autogain.c b/lib/libv4lconvert/processing/autogain.c
> index c6866d6..0b52d0f 100644
> --- a/lib/libv4lconvert/processing/autogain.c
> +++ b/lib/libv4lconvert/processing/autogain.c
> @@ -68,6 +71,41 @@ static void autogain_adjust(struct v4l2_queryctrl *ctrl, int *value,
>  	}
>  }
>  
> +static int get_luminosity_bayer10(uint16_t *buf, const struct v4l2_format *fmt)
> +{
> +	long long avg_lum = 0;
> +	int x, y;
> +	
> +	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> +		fmt->fmt.pix.width / 4;
> +
> +	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> +		for (x = 0; x < fmt->fmt.pix.width / 2; x++)
> +			avg_lum += *buf++;
> +		buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
> +	}
> +	avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
> +	avg_lum /= 4;
> +	return avg_lum;
> +}
> +
> +static int get_luminosity_bayer8(unsigned char *buf, const struct v4l2_format *fmt)
> +{
> +	long long avg_lum = 0;
> +	int x, y;
> +	
> +	buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> +		fmt->fmt.pix.width / 4;
> +
> +	for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> +		for (x = 0; x < fmt->fmt.pix.width / 2; x++)
> +			avg_lum += *buf++;
> +		buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
> +	}
> +	avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
> +	return avg_lum;
> +}
> +
>  /* auto gain and exposure algorithm based on the knee algorithm described here:
>  http://ytse.tricolour.net/docs/LowLightOptimization.html */
>  static int autogain_calculate_lookup_tables(
> @@ -100,17 +142,16 @@ static int autogain_calculate_lookup_tables(
>  	switch (fmt->fmt.pix.pixelformat) {
> +	case V4L2_PIX_FMT_SGBRG10:
> +	case V4L2_PIX_FMT_SGRBG10:
> +	case V4L2_PIX_FMT_SBGGR10:
> +	case V4L2_PIX_FMT_SRGGB10:
> +		avg_lum = get_luminosity_bayer10((void *) buf, fmt);
> +		break;
> +
>  	case V4L2_PIX_FMT_SGBRG8:
>  	case V4L2_PIX_FMT_SGRBG8:
>  	case V4L2_PIX_FMT_SBGGR8:
>  	case V4L2_PIX_FMT_SRGGB8:
> -		buf += fmt->fmt.pix.height * fmt->fmt.pix.bytesperline / 4 +
> -			fmt->fmt.pix.width / 4;
> -
> -		for (y = 0; y < fmt->fmt.pix.height / 2; y++) {
> -			for (x = 0; x < fmt->fmt.pix.width / 2; x++)
> -				avg_lum += *buf++;
> -			buf += fmt->fmt.pix.bytesperline - fmt->fmt.pix.width / 2;
> -		}
> -		avg_lum /= fmt->fmt.pix.height * fmt->fmt.pix.width / 4;
> +		avg_lum = get_luminosity_bayer8(buf, fmt);
>  		break;
>  
>  	case V4L2_PIX_FMT_RGB24:
> diff --git a/lib/libv4lconvert/processing/libv4lprocessing.c b/lib/libv4lconvert/processing/libv4lprocessing.c
> index b061f50..b98d024 100644
> --- a/lib/libv4lconvert/processing/libv4lprocessing.c
> +++ b/lib/libv4lconvert/processing/libv4lprocessing.c
> @@ -164,6 +165,10 @@ void v4lprocessing_processing(struct v4lprocessing_data *data,
>  	case V4L2_PIX_FMT_SGRBG8:
>  	case V4L2_PIX_FMT_SBGGR8:
>  	case V4L2_PIX_FMT_SRGGB8:
> +	case V4L2_PIX_FMT_SGBRG10:
> +	case V4L2_PIX_FMT_SGRBG10:
> +	case V4L2_PIX_FMT_SBGGR10:
> +	case V4L2_PIX_FMT_SRGGB10:
>  	case V4L2_PIX_FMT_RGB24:
>  	case V4L2_PIX_FMT_BGR24:
>  		break;
> 
> 
> 



-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170501/f729b3a9/attachment-0001.sig>

[patch] propagating controls in libv4l2 was Re: support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!

> > Now. There is autogain support in libv4lconvert, but it expects to use
> > same fd for camera and for the gain... which does not work with
> > subdevs.
> > 
> > Of course, opening subdevs by name like this is not really
> > acceptable. But can you suggest a method that is?
> 
> There are two separate things here:
> 
> 1) Autofoucs for a device that doesn't use subdev API
> 2) libv4l2 support for devices that require MC and subdev API
> 
> for (1), it should use the /dev/videoX device that was opened with
> v4l2_open().
> 
> For (2), libv4l2 should be aware of MC and subdev APIs. Sakari
> once tried to write a libv4l2 plugin for OMAP3, but never finished it.
> A more recent trial were to add a libv4l2 plugin for Exynos.
> Unfortunately, none of those code got merged. Last time I checked,
> the Exynos plugin was almost ready to be merged, but Sakari asked
> some changes on it. The developer that was working on it got job on
> some other company. Last time I heard from him, he was still interested
> on finishing his work, but in the need to setup a test environment
> using his own devices.

First... we should not have hardware-specific code in the
userspace. Hardware abstraction should be kernel's job.

Second... we really should solve "ioctl propagation" in
libv4l2. Because otherwise existing userspace is unusable in MC/subdev
drivers.

> IMHO, the right thing to do with regards to autofocus is to
> implement it via a processing module, assuming that just one
> video device is opened.

Ok, I have done that.

> The hole idea is that a libv4l2 client, running on a N900 device
> would just open a fake /dev/video0. Internally, libv4l2 will
> open whatever video nodes it needs to control the device, exporting
> all hardware capabilities (video formats, controls, resolutions,
> etc) as if it was a normal V4L2 camera, hiding all dirty details
> about MC and subdev APIs from userspace application.
> 
> This way, a normal application, like xawtv, tvtime, camorama,
> zbar, mplayer, vlc, ... will work without any changes.

Well, yes, we'd like to get there eventually. But we are not there at
the moment, and ioctl() propagation is one of the steps.

(I do have support specific for N900, but currently it is in python;
this is enough to make the camera usable.)

Regards,
								Pavel

								
> > diff --git a/lib/libv4l2/libv4l2-priv.h b/lib/libv4l2/libv4l2-priv.h
> > index 343db5e..a6bc48e 100644
> > --- a/lib/libv4l2/libv4l2-priv.h
> > +++ b/lib/libv4l2/libv4l2-priv.h
> > @@ -26,6 +26,7 @@
> >  #include "../libv4lconvert/libv4lsyscall-priv.h"
> >  
> >  #define V4L2_MAX_DEVICES 16
> > +#define V4L2_MAX_SUBDEVS 8
> >  /* Warning when making this larger the frame_queued and frame_mapped members of
> >     the v4l2_dev_info struct can no longer be a bitfield, so the code needs to
> >     be adjusted! */
> > @@ -104,6 +105,7 @@ struct v4l2_dev_info {
> >  	void *plugin_library;
> >  	void *dev_ops_priv;
> >  	const struct libv4l_dev_ops *dev_ops;
> > +        int subdev_fds[V4L2_MAX_SUBDEVS];
> >  };
> >  
> >  /* From v4l2-plugin.c */
> > diff --git a/lib/libv4l2/libv4l2.c b/lib/libv4l2/libv4l2.c
> > index 0ba0a88..edc9642 100644
> > --- a/lib/libv4l2/libv4l2.c
> > +++ b/lib/libv4l2/libv4l2.c
> > @@ -1,3 +1,4 @@
> > +/* -*- c-file-style: "linux" -*- */
> >  /*
> >  #             (C) 2008 Hans de Goede <hdegoede@redhat.com>
> >  
> > @@ -789,18 +790,25 @@ no_capture:
> >  
> >  	/* Note we always tell v4lconvert to optimize src fmt selection for
> >  	   our default fps, the only exception is the app explicitly selecting
> > -	   a fram erate using the S_PARM ioctl after a S_FMT */
> > +	   a frame rate using the S_PARM ioctl after a S_FMT */
> >  	if (devices[index].convert)
> >  		v4lconvert_set_fps(devices[index].convert, V4L2_DEFAULT_FPS);
> >  	v4l2_update_fps(index, &parm);
> >  
> > +	devices[index].subdev_fds[0] = SYS_OPEN("/dev/video_sensor", O_RDWR, 0);
> > +	devices[index].subdev_fds[1] = SYS_OPEN("/dev/video_focus", O_RDWR, 0);
> > +	devices[index].subdev_fds[2] = -1;
> > +
> > +	printf("Sensor: %d, focus: %d\n", devices[index].subdev_fds[0], 
> > +	       devices[index].subdev_fds[1]);
> > +
> >  	V4L2_LOG("open: %d\n", fd);
> >  
> >  	return fd;
> >  }
> >  
> >  /* Is this an fd for which we are emulating v4l1 ? */
> > -static int v4l2_get_index(int fd)
> > +int v4l2_get_index(int fd)
> >  {
> >  	int index;
> >  
> > 
> > commit 1d6a9ce121f53e8f2e38549eed597a3c3dea5233
> > Author: Pavel <pavel@ucw.cz>
> > Date:   Wed Apr 26 12:34:04 2017 +0200
> > 
> >     Enable ioctl propagation.
> > 
> > diff --git a/lib/libv4l2/libv4l2.c b/lib/libv4l2/libv4l2.c
> > index edc9642..6dab661 100644
> > --- a/lib/libv4l2/libv4l2.c
> > +++ b/lib/libv4l2/libv4l2.c
> > @@ -1064,6 +1064,23 @@ static int v4l2_s_fmt(int index, struct v4l2_format *dest_fmt)
> >  	return 0;
> >  }
> >  
> > +static int v4l2_propagate_ioctl(int index, unsigned long request, void *arg)
> > +{
> > +	int i = 0;
> > +	int result;
> > +	while (1) {
> > +		if (devices[index].subdev_fds[i] == -1)
> > +			return -1;
> > +		printf("g_ctrl failed, trying...\n");
> > +		result = SYS_IOCTL(devices[index].subdev_fds[i], request, arg);
> > +		printf("subdev %d result %d\n", i, result);
> > +		if (result == 0)
> > +			return 0;
> > +		i++;
> > +	}
> > +	return -1;
> > +}
> > +
> >  int v4l2_ioctl(int fd, unsigned long int request, ...)
> >  {
> >  	void *arg;
> > @@ -1193,14 +1210,20 @@ no_capture_request:
> >  	switch (request) {
> >  	case VIDIOC_QUERYCTRL:
> >  		result = v4lconvert_vidioc_queryctrl(devices[index].convert, arg);
> > +		if (result == -1)
> > +			result = v4l2_propagate_ioctl(index, request, arg);
> >  		break;
> >  
> >  	case VIDIOC_G_CTRL:
> >  		result = v4lconvert_vidioc_g_ctrl(devices[index].convert, arg);
> > +		if (result == -1)
> > +			result = v4l2_propagate_ioctl(index, request, arg);
> >  		break;
> >  
> >  	case VIDIOC_S_CTRL:
> >  		result = v4lconvert_vidioc_s_ctrl(devices[index].convert, arg);
> > +		if (result == -1)
> > +			result = v4l2_propagate_ioctl(index, request, arg);
> >  		break;
> >  
> >  	case VIDIOC_G_EXT_CTRLS:
> > 
> > 
> 
> 
> 
> Thanks,
> Mauro

-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170713/72bee76f/attachment.sig>

[patch] propagating controls in libv4l2 was Re: support autofocus / autogain in libv4l2

[Mauro Carvalho Chehab]

Em Wed, 26 Apr 2017 12:53:00 +0200
Pavel Machek <pavel@ucw.cz> escreveu:

> Hi!
> 
> > > > IMO, the best place for autofocus is at libv4l2. Putting it on a
> > > > separate "video server" application looks really weird for me.    
> > > 
> > > Well... let me see. libraries are quite limited -- it is hard to open
> > > files, or use threads/have custom main loop. It may be useful to
> > > switch resolutions -- do autofocus/autogain at lower resolution, then
> > > switch to high one for taking picture. It would be good to have that
> > > in "system" code, but I'm not at all sure libv4l2 design will allow
> > > that.  
> > 
> > I don't see why it would be hard to open files or have threads inside
> > a library. There are several libraries that do that already, specially
> > the ones designed to be used on multimidia apps.  
> 
> Well, This is what the libv4l2 says:
> 
>    This file implements libv4l2, which offers v4l2_ prefixed versions
>    of
>       open/close/etc. The API is 100% the same as directly opening
>    /dev/videoX
>       using regular open/close/etc, the big difference is that format
>    conversion
>    
> but if I open additional files in v4l2_open(), API is no longer the
> same, as unix open() is defined to open just one file descriptor.
> 
> Now. There is autogain support in libv4lconvert, but it expects to use
> same fd for camera and for the gain... which does not work with
> subdevs.
> 
> Of course, opening subdevs by name like this is not really
> acceptable. But can you suggest a method that is?
> 
> Thanks,
> 								Pavel
> 
> commit 4cf9d10ead014c0db25452e4bb9cd144632407c3
> Author: Pavel <pavel@ucw.cz>
> Date:   Wed Apr 26 11:38:04 2017 +0200
> 
>     Add subdevices.
> 
> diff --git a/lib/libv4l2/libv4l2-priv.h b/lib/libv4l2/libv4l2-priv.h
> index 343db5e..a6bc48e 100644
> --- a/lib/libv4l2/libv4l2-priv.h
> +++ b/lib/libv4l2/libv4l2-priv.h
> @@ -26,6 +26,7 @@
>  #include "../libv4lconvert/libv4lsyscall-priv.h"
>  
>  #define V4L2_MAX_DEVICES 16
> +#define V4L2_MAX_SUBDEVS 8

Isn't it a short number?

>  /* Warning when making this larger the frame_queued and frame_mapped members of
>     the v4l2_dev_info struct can no longer be a bitfield, so the code needs to
>     be adjusted! */
> @@ -104,6 +105,7 @@ struct v4l2_dev_info {
>  	void *plugin_library;
>  	void *dev_ops_priv;
>  	const struct libv4l_dev_ops *dev_ops;
> +        int subdev_fds[V4L2_MAX_SUBDEVS];
>  };
>  
>  /* From v4l2-plugin.c */
> diff --git a/lib/libv4l2/libv4l2.c b/lib/libv4l2/libv4l2.c
> index 0ba0a88..edc9642 100644
> --- a/lib/libv4l2/libv4l2.c
> +++ b/lib/libv4l2/libv4l2.c
> @@ -1,3 +1,4 @@
> +/* -*- c-file-style: "linux" -*- */

No emacs comments, please.

>  /*
>  #             (C) 2008 Hans de Goede <hdegoede@redhat.com>
>  
> @@ -789,18 +790,25 @@ no_capture:
>  
>  	/* Note we always tell v4lconvert to optimize src fmt selection for
>  	   our default fps, the only exception is the app explicitly selecting
> -	   a fram erate using the S_PARM ioctl after a S_FMT */
> +	   a frame rate using the S_PARM ioctl after a S_FMT */
>  	if (devices[index].convert)
>  		v4lconvert_set_fps(devices[index].convert, V4L2_DEFAULT_FPS);
>  	v4l2_update_fps(index, &parm);
>  
> +	devices[index].subdev_fds[0] = SYS_OPEN("/dev/video_sensor", O_RDWR, 0);
> +	devices[index].subdev_fds[1] = SYS_OPEN("/dev/video_focus", O_RDWR, 0);
> +	devices[index].subdev_fds[2] = -1;

Hardcoding names here is not a good idea. Ideally, it should open
the MC, using the newgen API, and parse the media graph.

The problem is that, even with newgen API, without the properties API
you likely won't be able to write a generic parser. So, we need a
plugin specific for OMAP3 (or at least some database that would teach
a generic plugin about OMAP3 specifics).

I guess that the approach that Jacek was taken were very close to what
a generic plugin would need:
	https://lwn.net/Articles/619449/

The last version of his patch set is here:
	https://patchwork.linuxtv.org/patch/37496/

I didn't review his patchset, but from what I saw, Sakari is the one
that found some issues on v7.1 patchset.

Sakari,

Could you shed us a light about why this patchset was not merged?

Are there anything really bad at the code, or just minor issues that
could be fixed later?

If it is the last case, perhaps we could merge the code, if this
would make easier for Pavel to work on a N9 solution using the
same approach.


Thanks,
Mauro

[patch] propagating controls in libv4l2 was Re: support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!

> > +	devices[index].subdev_fds[0] = SYS_OPEN("/dev/video_sensor", O_RDWR, 0);
> > +	devices[index].subdev_fds[1] = SYS_OPEN("/dev/video_focus", O_RDWR, 0);
> > +	devices[index].subdev_fds[2] = -1;
> 
> Hardcoding names here is not a good idea. Ideally, it should open
> the MC, using the newgen API, and parse the media graph.
> 
> The problem is that, even with newgen API, without the properties API
> you likely won't be able to write a generic parser. So, we need a
> plugin specific for OMAP3 (or at least some database that would teach
> a generic plugin about OMAP3 specifics).
> 
> I guess that the approach that Jacek was taken were very close to what
> a generic plugin would need:
> 	https://lwn.net/Articles/619449/
> 
> The last version of his patch set is here:
> 	https://patchwork.linuxtv.org/patch/37496/
> 
> I didn't review his patchset, but from what I saw, Sakari is the one
> that found some issues on v7.1 patchset.
> 
> Sakari,
> 
> Could you shed us a light about why this patchset was not merged?
> 
> Are there anything really bad at the code, or just minor issues that
> could be fixed later?
> 
> If it is the last case, perhaps we could merge the code, if this
> would make easier for Pavel to work on a N9 solution using the
> same approach.

It would be nice to get some solution here. Camera without libv4l
support is pretty much useless :-(.

									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170429/5a448afb/attachment.sig>

Camera support, Prague next week, sdlcam

[Hans Verkuil]

On 22/10/17 00:00, Pavel Machek wrote:
> Hi!
> 
> I'd still like to get some reasonable support for cellphone camera in
> Linux.
> 
> IMO first reasonable step is to merge sdlcam, then we can implement
> autofocus, improve autogain... and rest of the boring stuff. Ouch and
> media graph support would be nice. Currently, _nothing_ works with
> media graph device, such as N900.

Can you post your latest rebased patch for sdlcam for v4l-utils?

I'll do a review and will likely merge it for you. Yes, I've changed my
mind on that.

> 
> I'll talk about the issues at ELCE in few days:
> 
> https://osseu17.sched.com/event/ByYH/cheap-complex-cameras-pavel-machek-denx-software-engineering-gmbh
> 
> Will someone else be there? Is there some place where v4l people meet?

Why don't we discuss this Tuesday morning at 9am? I have no interest in the
keynotes on that day, so those who are interested can get together.

I'll be at your presentation tomorrow and we can discuss a bit during
the following coffee break if time permits.

Regards,

	Hans

Camera support, Prague next week, sdlcam

[Pavel Machek]

Hi!

> > I'd still like to get some reasonable support for cellphone camera in
> > Linux.
> > 
> > IMO first reasonable step is to merge sdlcam, then we can implement
> > autofocus, improve autogain... and rest of the boring stuff. Ouch and
> > media graph support would be nice. Currently, _nothing_ works with
> > media graph device, such as N900.
> 
> Can you post your latest rebased patch for sdlcam for v4l-utils?
> 
> I'll do a review and will likely merge it for you. Yes, I've changed my
> mind on that.

Ok, will do, thanks!

> > I'll talk about the issues at ELCE in few days:
> > 
> > https://osseu17.sched.com/event/ByYH/cheap-complex-cameras-pavel-machek-denx-software-engineering-gmbh
> > 
> > Will someone else be there? Is there some place where v4l people meet?
> 
> Why don't we discuss this Tuesday morning at 9am? I have no interest in the
> keynotes on that day, so those who are interested can get together.
> 
> I'll be at your presentation tomorrow and we can discuss a bit during
> the following coffee break if time permits.

Ok, sounds like a plan. Lets confirm Tuesday morning tommorow.

Thanks,
									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20171022/955955a1/attachment.sig>

Camera support, Prague next week, sdlcam

[Pavel Machek]

Hi!

> > I'll talk about the issues at ELCE in few days:
> > 
> > https://osseu17.sched.com/event/ByYH/cheap-complex-cameras-pavel-machek-denx-software-engineering-gmbh
> > 
> > Will someone else be there? Is there some place where v4l people meet?
> 
> Why don't we discuss this Tuesday morning at 9am? I have no interest in the
> keynotes on that day, so those who are interested can get together.

Ok, what about 9:30am? The place near the elevators where we were
talking last time?

And.... sorry for late reply.
									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20171023/00ab004f/attachment.sig>

Camera support, Prague next week, sdlcam

[Hans Verkuil]

Sounds good. That's the elevators on level LL by the way.

Regards,

Hans

On October 23, 2017 8:54:49 PM GMT+02:00, Pavel Machek <pavel@ucw.cz> wrote:
>Hi!
>
>> > I'll talk about the issues at ELCE in few days:
>> > 
>> >
>https://osseu17.sched.com/event/ByYH/cheap-complex-cameras-pavel-machek-denx-software-engineering-gmbh
>> > 
>> > Will someone else be there? Is there some place where v4l people
>meet?
>> 
>> Why don't we discuss this Tuesday morning at 9am? I have no interest
>in the
>> keynotes on that day, so those who are interested can get together.
>
>Ok, what about 9:30am? The place near the elevators where we were
>talking last time?
>
>And.... sorry for late reply.
>									Pavel

-- 
Sent from my Android device with K-9 Mail. Please excuse my brevity.

Camera support, Prague next week, sdlcam

[Sakari Ailus]

On Mon, Oct 23, 2017 at 09:24:24PM +0200, Hans Verkuil wrote:
> Sounds good. That's the elevators on level LL by the way.

I'll be there, too!

-- 
Sakari Ailus
e-mail: sakari.ailus at iki.fi

Camera support, Prague next week, sdlcam

[Mauro Carvalho Chehab]

Em Mon, 23 Oct 2017 23:15:57 +0300
Sakari Ailus <sakari.ailus@iki.fi> escreveu:

> On Mon, Oct 23, 2017 at 09:24:24PM +0200, Hans Verkuil wrote:
> > Sounds good. That's the elevators on level LL by the way.  
> 
> I'll be there, too!
> 

I'll try to join you there too.


Cheers,
Mauro

Nokia N9: fun with camera

[Pavel Machek]

Hi!

> Sakari, I am actually playing with N9 camera, not N950. That comes
> next.
> 
> And the clock error I mentioned ... seems to be
> -EPROBE_DEFER. So... not an issue.

Hmm, and with similar config, I got N950 to work. ... which should
give me enough clues to get N9 to work. I guess I forgot to reset the
pipeline between the tries, or something.

For the record, this got me some data on n950:

 m.media_ctl( [ '-f', '"OMAP3 ISP CSI2a":0 [fmt:%s/%dx%d]' % (m.fmt, m.cap_x, m.cap_y) ] )
 m.media_ctl( [ '-l', '"OMAP3 ISP CSI2a":1 -> "OMAP3 ISP CSI2a output":0[1]' ] )

 # WORKS!!!!
 # pavel at n900:~/g/tui/camera$ sudo /my/tui/yavta/yavta
 # --capture=8 --skip 0 --format SGRBG10 --size 4272x3016 /dev/video1 --file=/tmp/delme#

...ouch. It only worked twice :-(. Either driver gets confused by my
attempts, or it relied on some other initialization code. Strange.

Best regards,
									Pavel

-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20171101/6e31e864/attachment.sig>

Nokia N9: fun with camera

[Pavel Machek]

Hi!

> > Sakari, I am actually playing with N9 camera, not N950. That comes
> > next.
> > 
> > And the clock error I mentioned ... seems to be
> > -EPROBE_DEFER. So... not an issue.
> 
> Hmm, and with similar config, I got N950 to work. ... which should
> give me enough clues to get N9 to work. I guess I forgot to reset the
> pipeline between the tries, or something.
> 
> For the record, this got me some data on n950:
> 
>  m.media_ctl( [ '-f', '"OMAP3 ISP CSI2a":0 [fmt:%s/%dx%d]' % (m.fmt, m.cap_x, m.cap_y) ] )
>  m.media_ctl( [ '-l', '"OMAP3 ISP CSI2a":1 -> "OMAP3 ISP CSI2a output":0[1]' ] )
> 
>  # WORKS!!!!
>  # pavel at n900:~/g/tui/camera$ sudo /my/tui/yavta/yavta
>  # --capture=8 --skip 0 --format SGRBG10 --size 4272x3016 /dev/video1 --file=/tmp/delme#
> 
> ...ouch. It only worked twice :-(. Either driver gets confused by my
> attempts, or it relied on some other initialization code. Strange.

Hmm, so it works "reliably" after boot. But it also locks up machine
with high probability. If you have N950, it might still be handy...

Messages are:

['-r']
['-f', '"OMAP3 ISP CSI2a":0 [fmt:SGRBG10/4272x3016]']
Warning: the -f option is deprecated and has been replaced by -V.
['-l', '"OMAP3 ISP CSI2a":1 -> "OMAP3 ISP CSI2a output":0[1]']
Testing: is raw
Testing:  /my/tui/yavta/yavta --capture=8 --skip 0 --format SGRBG10
--size 4272x3016 /dev/video_sensor --file=/tmp/delme#
Error opening device /dev/video_sensor: Permission denied (13).
Raw mode is  1
Raw mode is  1
Testing: is raw
Testing:  /my/tui/yavta/yavta --capture=8 --skip 0 --format SGRBG10
--size 4272x3016 /dev/video_sensor --file=/tmp/delme#
Device /dev/video_sensor opened.
Device `OMAP3 ISP CSI2a output' on `media' is a video capture (without
mplanes) device.
Video format set: SGRBG10 (30314142) 4272x3016 (stride 8544) field
none buffer size 25768704
Video format: SGRBG10 (30314142) 4272x3016 (stride 8544) field none
buffer size 25768704
1 buffers requested.
length: 25768704 offset: 0 timestamp type/source: mono/EoF
Buffer 0/0 mapped at address 0xb54e9000.
0 (0) [-] none 0 25768704 B 691.756907 691.758403 6.212 fps ts
mono/EoF
1 (0) [-] none 1 25768704 B 694.821025 694.821422 0.326 fps ts
mono/EoF
2 (0) [E] none 2 25768704 B 698.530833 698.534739 0.270 fps ts
mono/EoF
3 (0) [-] none 3 25768704 B 700.788127 700.790996 0.443 fps ts
mono/EoF
(and here it locked up :-(. Sometimes it captures more.)

Thanks,

								Pavel




-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20171101/18dc9d80/attachment.sig>

support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!

> Please don't add a new application under lib/. It is fine if you want
> some testing application, if the ones there aren't enough, but please
> place it under contrib/test/.
> 
> You should likely take a look at v4l2grab first, as it could have
> almost everything you would need.

I really need some kind of video output. v4l2grab is not useful
there. v4l2gl might be, but I don't think I have enough dependencies.

Umm, and it looks like libv4l can not automatically convert from
GRBG10.. and if it could, going through RGB24 would probably be too
slow on this device :-(.

> IMO, the above belongs to a separate processing module under
> 	lib/libv4lconvert/processing/

Is there an example using autogain/autowhitebalance from
libv4lconvert?

Best regards,
									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/c41537cc/attachment.sig>

support autofocus / autogain in libv4l2

[Mauro Carvalho Chehab]

Em Tue, 25 Apr 2017 00:07:01 +0200
Pavel Machek <pavel@ucw.cz> escreveu:

> Hi!
> 
> > Please don't add a new application under lib/. It is fine if you want
> > some testing application, if the ones there aren't enough, but please
> > place it under contrib/test/.
> > 
> > You should likely take a look at v4l2grab first, as it could have
> > almost everything you would need.  
> 
> I really need some kind of video output. v4l2grab is not useful
> there. v4l2gl might be, but I don't think I have enough dependencies.

Well, you could use some app to show the snaps that v4l2grab takes.

Yeah, compiling v4l2gl on N9 can indeed be complex. I suspect that it 
shouldn't hard to compile xawtv there (probably disabling some optional
features).

> Umm, and it looks like libv4l can not automatically convert from
> GRBG10.. and if it could, going through RGB24 would probably be too
> slow on this device :-(.

I suspect it shouldn't be hard to add support for GRBG10. It already
supports 8 and 16 bits Bayer formats, at lib/libv4lconvert/bayer.c
(to both RGB and YUV formats).

How it would preform is another question ;)

> > IMO, the above belongs to a separate processing module under
> > 	lib/libv4lconvert/processing/  
> 
> Is there an example using autogain/autowhitebalance from
> libv4lconvert?

Well, if you plug a USB camera without those controls, it should
automatically expose controls for it, as if the device had such
controls.

Thanks,
Mauro

support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!

> > > Please don't add a new application under lib/. It is fine if you want
> > > some testing application, if the ones there aren't enough, but please
> > > place it under contrib/test/.
> > > 
> > > You should likely take a look at v4l2grab first, as it could have
> > > almost everything you would need.  
> > 
> > I really need some kind of video output. v4l2grab is not useful
> > there. v4l2gl might be, but I don't think I have enough dependencies.
> 
> Well, you could use some app to show the snaps that v4l2grab takes.

That would be too slow :-(.

> Yeah, compiling v4l2gl on N9 can indeed be complex. I suspect that it 
> shouldn't hard to compile xawtv there (probably disabling some optional
> features).

I do have mplayer working, but that one is not linked against libv4l2
:-(.

> > Umm, and it looks like libv4l can not automatically convert from
> > GRBG10.. and if it could, going through RGB24 would probably be too
> > slow on this device :-(.
> 
> I suspect it shouldn't be hard to add support for GRBG10. It already
> supports 8 and 16 bits Bayer formats, at lib/libv4lconvert/bayer.c
> (to both RGB and YUV formats).

Is 16bit bayer a recent development? I can't see it in

commit 374806e868f5a7a48ecffde4c6a1abfcfa5ccd65
Author: Hans Verkuil <hans.verkuil@cisco.com>
Date:   Fri Apr 22 09:31:57 2016 +0200

> > Is there an example using autogain/autowhitebalance from
> > libv4lconvert?
> 
> Well, if you plug a USB camera without those controls, it should
> automatically expose controls for it, as if the device had such
> controls.

And settings are persistent, so I can enable autogain, then lauch
something like xawtv, and it will automatically get autogain? Ok,
good.

Regards,
									Pavel
-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/23ad62a6/attachment.sig>

support autofocus / autogain in libv4l2

[Pavel Machek]

Hi!

> > Umm, and it looks like libv4l can not automatically convert from
> > GRBG10.. and if it could, going through RGB24 would probably be too
> > slow on this device :-(.
> 
> I suspect it shouldn't be hard to add support for GRBG10. It already
> supports 8 and 16 bits Bayer formats, at lib/libv4lconvert/bayer.c
> (to both RGB and YUV formats).

Proper format for 16 bit bayer would be tricky, AFAICT. Anyway, does
this look reasonable? It does not work too well here, since omap3isp
driver does not seem to support ENUM_FMT. (And I get just half of the
vertical image, strange. Interlacing?)

Best regards,
								Pavel

diff --git a/lib/libv4lconvert/libv4lconvert.c b/lib/libv4lconvert/libv4lconvert.c
index d3d8936..2a469b2 100644
--- a/lib/libv4lconvert/libv4lconvert.c
+++ b/lib/libv4lconvert/libv4lconvert.c
@@ -123,6 +126,8 @@ static const struct v4lconvert_pixfmt supported_src_pixfmts[] = {
 	{ V4L2_PIX_FMT_SGRBG8,		 8,	 8,	 8,	1 },
 	{ V4L2_PIX_FMT_SRGGB8,		 8,	 8,	 8,	1 },
 	{ V4L2_PIX_FMT_STV0680,		 8,	 8,	 8,	1 },
+
+	{ V4L2_PIX_FMT_SGRBG10,		16,	 8,	 8,	1 },
 	/* compressed bayer */
 	{ V4L2_PIX_FMT_SPCA561,		 0,	 9,	 9,	1 },
 	{ V4L2_PIX_FMT_SN9C10X,		 0,	 9,	 9,	1 },
@@ -668,6 +680,7 @@ static int v4lconvert_processing_needs_double_conversion(
 	case V4L2_PIX_FMT_SGRBG8:
 	case V4L2_PIX_FMT_SRGGB8:
 	case V4L2_PIX_FMT_STV0680:
+	case V4L2_PIX_FMT_SGRBG10:
 		return 0;
 	}
 	switch (dest_pix_fmt) {
@@ -694,6 +707,17 @@ unsigned char *v4lconvert_alloc_buffer(int needed,
 	return *buf;
 }
 
+static void v4lconvert_10to8(void *_src, unsigned char *dst, int width, int height)
+{
+	int i;
+	uint16_t *src = _src;
+	
+	printf("sizes %d x %d\n", width, height);
+	for (i=0; i<width*height; i++) {
+		dst[i] = src[i] >> 2;
+	}
+}
+
 int v4lconvert_oom_error(struct v4lconvert_data *data)
 {
 	V4LCONVERT_ERR("could not allocate memory\n");
@@ -867,7 +893,8 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 #endif
 	case V4L2_PIX_FMT_SN9C2028:
 	case V4L2_PIX_FMT_SQ905C:
-	case V4L2_PIX_FMT_STV0680: { /* Not compressed but needs some shuffling */
+	case V4L2_PIX_FMT_STV0680:
+	case V4L2_PIX_FMT_SGRBG10: { /* Not compressed but needs some shuffling */
 		unsigned char *tmpbuf;
 		struct v4l2_format tmpfmt = *fmt;
 
@@ -877,6 +904,11 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 			return v4lconvert_oom_error(data);
 
 		switch (src_pix_fmt) {
+		case V4L2_PIX_FMT_SGRBG10:
+			v4lconvert_10to8(src, tmpbuf, width, height);
+
+			tmpfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_SGRBG8;
+			break;
 		case V4L2_PIX_FMT_SPCA561:
 			v4lconvert_decode_spca561(src, tmpbuf, width, height);
 			tmpfmt.fmt.pix.pixelformat = V4L2_PIX_FMT_SGBRG8;
@@ -949,6 +981,7 @@ static int v4lconvert_convert_pixfmt(struct v4lconvert_data *data,
 			V4LCONVERT_ERR("short raw bayer data frame\n");
 			errno = EPIPE;
 			result = -1;
+			/* FIXME: but then we proceed anyway?! */
 		}
 		switch (dest_pix_fmt) {
 		case V4L2_PIX_FMT_RGB24:




-- 
(english) http://www.livejournal.com/~pavelmachek
(cesky, pictures) http://atrey.karlin.mff.cuni.cz/~pavel/picture/horses/blog.html
-------------- next part --------------
A non-text attachment was scrubbed...
Name: signature.asc
Type: application/pgp-signature
Size: 181 bytes
Desc: Digital signature
URL: <http://lists.infradead.org/pipermail/linux-arm-kernel/attachments/20170425/a03b799f/attachment-0001.sig>