backend-vnc: add VNC support using Neat VNC library

This adds basic VNC protocol support using the Neat VNC library
(https://github.com/any1/neatvnc). Neat VNC depends on the AML main
loop library. The backend makes use of AML's integrated epoll backend
and connects AML via file descriptor with the Wayland event loop.

This implementation does not support authentication and hardcodes the
pixel format currently.

Co-authored-by: Philipp Zabel <p.zabel@pengutronix.de>
Co-authored-by: Rouven Czerwinski <r.czerwinski@pengutronix.de>
Signed-off-by: Stefan Agner <stefan@agner.ch>
[r.czerwinski@pengutronix.de:
 - use new (as of 0.5.0) Neat VNC buffer API, with a buffer pool]
Signed-off-by: Rouven Czerwinski <r.czerwinski@pengutronix.de>
[p.zabel@pengutronix.de:
 - transform repaint damage to output coordinates
 - transform pointer coordinates into global space
 - check that outputs and heads are in fact ours, see aab722bb1785..060ef82d9360
 - track damage across multiple frame buffers
 - choose pixel format by drm_fourcc, see 8b6c3fe0ad
 - enable ctrl and alt modifiers
 - fix frame timing to achieve a constant repaint rate
 - pass initial size explicitly, see f4559b0760
 - use resize_output with pixman-renderer, see 55d08f9634e8..84b5d0eb4bee
 - allow configuring the refresh rate]
Signed-off-by: Philipp Zabel <p.zabel@pengutronix.de>

compositor/main.c

@@ -61,6 +61,7 @@
 #include <libweston/backend-drm.h>
 #include <libweston/backend-headless.h>
 #include <libweston/backend-rdp.h>
+#include <libweston/backend-vnc.h>
 #include <libweston/backend-x11.h>
 #include <libweston/backend-wayland.h>
 #include <libweston/windowed-output-api.h>
@@ -653,6 +654,9 @@ usage(int error_code)
 #if defined(BUILD_RDP_COMPOSITOR)
 			"\t\t\t\trdp-backend.so\n"
 #endif
+#if defined(BUILD_VNC_COMPOSITOR)
+			"\t\t\t\tvnc-backend.so\n"
+#endif
 #if defined(BUILD_WAYLAND_COMPOSITOR)
 			"\t\t\t\twayland-backend.so\n"
 #endif
@@ -719,6 +723,15 @@ usage(int error_code)
 		"\n");
 #endif
 
+#if defined(BUILD_VNC_COMPOSITOR)
+	fprintf(out,
+		"Options for vnc-backend.so:\n\n"
+		"  --width=WIDTH\t\tWidth of desktop\n"
+		"  --height=HEIGHT\tHeight of desktop\n"
+		"  --port=PORT\t\tThe port to listen on\n"
+		"\n");
+#endif
+
 #if defined(BUILD_WAYLAND_COMPOSITOR)
 	fprintf(out,
 		"Options for wayland-backend.so:\n\n"
@@ -3098,6 +3111,90 @@ load_rdp_backend(struct weston_compositor *c,
 	return ret;
 }
 
+static int
+vnc_backend_output_configure(struct weston_output *output)
+{
+	struct wet_compositor *compositor = to_wet_compositor(output->compositor);
+	struct wet_output_config *parsed_options = compositor->parsed_options;
+	const struct weston_vnc_output_api *api = weston_vnc_output_get_api(output->compositor);
+	int width = 640;
+	int height = 480;
+
+	assert(parsed_options);
+
+	if (!api) {
+		weston_log("Cannot use weston_vnc_output_api.\n");
+		return -1;
+	}
+
+	if (parsed_options->width)
+		width = parsed_options->width;
+
+	if (parsed_options->height)
+		height = parsed_options->height;
+
+	weston_output_set_scale(output, 1);
+	weston_output_set_transform(output, WL_OUTPUT_TRANSFORM_NORMAL);
+
+	if (api->output_set_size(output, width, height) < 0) {
+		weston_log("Cannot configure output \"%s\" using weston_vnc_output_api.\n",
+			   output->name);
+		return -1;
+	}
+	weston_log("vnc_backend_output_configure.. Done\n");
+
+	return 0;
+}
+
+
+static void
+weston_vnc_backend_config_init(struct weston_vnc_backend_config *config)
+{
+	config->base.struct_version = WESTON_VNC_BACKEND_CONFIG_VERSION;
+	config->base.struct_size = sizeof(struct weston_vnc_backend_config);
+
+	config->bind_address = NULL;
+	config->port = 5900;
+	config->refresh_rate = VNC_DEFAULT_FREQ;
+}
+
+static int
+load_vnc_backend(struct weston_compositor *c,
+		int *argc, char *argv[], struct weston_config *wc)
+{
+	struct weston_vnc_backend_config config  = {{ 0, }};
+	struct weston_config_section *section;
+	int ret = 0;
+
+	struct wet_output_config *parsed_options = wet_init_parsed_options(c);
+	if (!parsed_options)
+		return -1;
+
+	weston_vnc_backend_config_init(&config);
+
+	const struct weston_option vnc_options[] = {
+		{ WESTON_OPTION_INTEGER, "width", 0, &parsed_options->width },
+		{ WESTON_OPTION_INTEGER, "height", 0, &parsed_options->height },
+		{ WESTON_OPTION_STRING,  "address", 0, &config.bind_address },
+		{ WESTON_OPTION_INTEGER, "port", 0, &config.port },
+	};
+
+	parse_options(vnc_options, ARRAY_LENGTH(vnc_options), argc, argv);
+
+	wet_set_simple_head_configurator(c, vnc_backend_output_configure);
+	section = weston_config_get_section(wc, "vnc", NULL, NULL);
+	weston_config_section_get_int(section, "refresh-rate",
+				      &config.refresh_rate,
+				      VNC_DEFAULT_FREQ);
+
+	ret = weston_compositor_load_backend(c, WESTON_BACKEND_VNC,
+					     &config.base);
+
+	free(config.bind_address);
+
+	return ret;
+}
+
 static int
 x11_backend_output_configure(struct weston_output *output)
 {
@@ -3343,6 +3440,8 @@ load_backend(struct weston_compositor *compositor, const char *backend,
 		return load_headless_backend(compositor, argc, argv, config);
 	else if (strstr(backend, "rdp-backend.so"))
 		return load_rdp_backend(compositor, argc, argv, config);
+	else if (strstr(backend, "vnc-backend.so"))
+		return load_vnc_backend(compositor, argc, argv, config);
 	else if (strstr(backend, "drm-backend.so"))
 		return load_drm_backend(compositor, argc, argv, config);
 	else if (strstr(backend, "x11-backend.so"))

include/libweston/backend-vnc.h

@@ -0,0 +1,71 @@
+/*
+ * Copyright © 2019 Stefan Agner <stefan@agner.ch>
+ *
+ * Permission is hereby granted, free of charge, to any person obtaining
+ * a copy of this software and associated documentation files (the
+ * "Software"), to deal in the Software without restriction, including
+ * without limitation the rights to use, copy, modify, merge, publish,
+ * distribute, sublicense, and/or sell copies of the Software, and to
+ * permit persons to whom the Software is furnished to do so, subject to
+ * the following conditions:
+ *
+ * The above copyright notice and this permission notice (including the
+ * next paragraph) shall be included in all copies or substantial
+ * portions of the Software.
+ *
+ * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+ * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+ * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+ * NONINFRINGEMENT.  IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
+ * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
+ * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
+ * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+ * SOFTWARE.
+ */
+
+#ifndef WESTON_COMPOSITOR_VNC_H
+#define WESTON_COMPOSITOR_VNC_H
+
+#ifdef  __cplusplus
+extern "C" {
+#endif
+
+#include <libweston/libweston.h>
+#include <libweston/plugin-registry.h>
+
+#define WESTON_VNC_OUTPUT_API_NAME "weston_vnc_output_api_v1"
+#define VNC_DEFAULT_FREQ 60
+
+struct weston_vnc_output_api {
+	/** Initialize a VNC output with specified width and height.
+	 *
+	 * Returns 0 on success, -1 on failure.
+	 */
+	int (*output_set_size)(struct weston_output *output,
+			       int width, int height);
+};
+
+static inline const struct weston_vnc_output_api *
+weston_vnc_output_get_api(struct weston_compositor *compositor)
+{
+	const void *api;
+	api = weston_plugin_api_get(compositor, WESTON_VNC_OUTPUT_API_NAME,
+				    sizeof(struct weston_vnc_output_api));
+
+	return (const struct weston_vnc_output_api *)api;
+}
+
+#define WESTON_VNC_BACKEND_CONFIG_VERSION 1
+
+struct weston_vnc_backend_config {
+	struct weston_backend_config base;
+	char *bind_address;
+	int port;
+	int refresh_rate;
+};
+
+#ifdef  __cplusplus
+}
+#endif
+
+#endif /* WESTON_COMPOSITOR_VNC_H */

include/libweston/libweston.h

@@ -2042,6 +2042,7 @@ enum weston_compositor_backend {
 	WESTON_BACKEND_DRM,
 	WESTON_BACKEND_HEADLESS,
 	WESTON_BACKEND_RDP,
+	WESTON_BACKEND_VNC,
 	WESTON_BACKEND_WAYLAND,
 	WESTON_BACKEND_X11,
 };

include/libweston/meson.build

@@ -15,6 +15,7 @@ install_headers(
 backend_drm_h =      files('backend-drm.h')
 backend_headless_h = files('backend-headless.h')
 backend_rdp_h =      files('backend-rdp.h')
+backend_vnc_h =      files('backend-vnc.h')
 backend_wayland_h =  files('backend-wayland.h')
 backend_x11_h =      files('backend-x11.h')
 

libweston/backend-vnc/meson.build

@@ -0,0 +1,32 @@
+if not get_option('backend-vnc')
+	subdir_done()
+endif
+
+config_h.set('BUILD_VNC_COMPOSITOR', '1')
+dep_neatvnc = dependency('neatvnc', version: ['>= 0.5.0', '< 0.6.0'], required: false)
+if not dep_neatvnc.found()
+	error('VNC backend requires neatvnc which was not found. Or, you can use \'-Dbackend-vnc=false\'.')
+endif
+
+dep_aml = dependency('aml', version: ['>= 0.1.0', '< 0.3.0'], required: false)
+if not dep_aml.found()
+	error('VNC backend requires libaml which was not found. Or, you can use \'-Dbackend-vnc=false\'.')
+endif
+
+deps_vnc = [
+	dep_libweston_private,
+	dep_neatvnc,
+	dep_aml,
+	dep_libdrm_headers,
+]
+plugin_vnc = shared_library(
+	'vnc-backend',
+	[ 'vnc.c' ],
+	include_directories: common_inc,
+	dependencies: deps_vnc,
+	name_prefix: '',
+	install: true,
+	install_dir: dir_module_libweston
+)
+env_modmap += 'vnc-backend.so=@0@;'.format(plugin_vnc.full_path())
+install_headers(backend_vnc_h, subdir: dir_include_libweston_install)

libweston/compositor.c

@@ -8687,6 +8687,7 @@ static const char * const backend_map[] = {
 	[WESTON_BACKEND_DRM] =		"drm-backend.so",
 	[WESTON_BACKEND_HEADLESS] =	"headless-backend.so",
 	[WESTON_BACKEND_RDP] =		"rdp-backend.so",
+	[WESTON_BACKEND_VNC] =		"vnc-backend.so",
 	[WESTON_BACKEND_WAYLAND] =	"wayland-backend.so",
 	[WESTON_BACKEND_X11] =		"x11-backend.so",
 };

libweston/meson.build

@@ -231,5 +231,6 @@ subdir('renderer-gl')
 subdir('backend-drm')
 subdir('backend-headless')
 subdir('backend-rdp')
+subdir('backend-vnc')
 subdir('backend-wayland')
 subdir('backend-x11')

meson_options.txt

@@ -32,6 +32,12 @@ option(
 	value: true,
 	description: 'Compositor: RDP screen-sharing support'
 )
+option(
+	'backend-vnc',
+	type: 'boolean',
+	value: true,
+	description: 'Weston backend: VNC remote screensharing'
+)
 option(
 	'backend-wayland',
 	type: 'boolean',