config/output: Pick among all 8-bit and 10-bit formats

The usual 8-bit and 10-bit formats have a lot of variations:

        XRGB, XBGR, RGBX, BGRX, ARGB, ABGR, RGBA, BGRA

A particular display is likely to support multiple of these, and as we
need to allocate a buffer of a particular type to test it, we'd like to
avoid doing too many unnecessary tests.

If we go off on the assumption that a display device generally supports
a subset of format variations but otherwise support each variation
equally well - that is, if both RGB and BGR is supported, they will
either both succeed or both fail at any given time - then instead of
trying all formats, we can simply pick one supported format within each
format group to try. For example, XBGR2101010 and XRGB8888.

By being conservative in how many formats we try, we can aggressively
expand the list of candidate formats to handle odd hardware.

We can get rid of the the conservative aspect if KMS ever grows the
ability to commit-test a buffer format without allocating the buffer
first, allowing us to quickly test every format under the sun.

sway/config/output.c

@@ -410,10 +410,6 @@ static enum render_bit_depth bit_depth_from_format(uint32_t render_format) {
 	return RENDER_BIT_DEPTH_DEFAULT;
 }
 
-static bool render_format_is_bgr(uint32_t fmt) {
-	return fmt == DRM_FORMAT_XBGR2101010 || fmt == DRM_FORMAT_XBGR8888;
-}
-
 static bool output_config_is_disabling(struct output_config *oc) {
 	return oc && (!oc->enabled || oc->power == 0);
 }
@@ -785,44 +781,68 @@ static bool search_mode(struct search_context *ctx, size_t output_idx) {
 	return false;
 }
 
-static bool search_render_format(struct search_context *ctx, size_t output_idx) {
-	struct matched_output_config *cfg = &ctx->configs[output_idx];
-	struct wlr_backend_output_state *backend_state = &ctx->states[output_idx];
-	struct wlr_output_state *state = &backend_state->base;
-	struct wlr_output *wlr_output = backend_state->output;
+struct fmtmap_entry {
+	enum render_bit_depth bit_depth;
+	const uint32_t *fmts;
+};
 
-	uint32_t fmts[] = {
+static const struct fmtmap_entry map[] = {
+	{RENDER_BIT_DEPTH_10, (uint32_t[]){
 		DRM_FORMAT_XRGB2101010,
 		DRM_FORMAT_XBGR2101010,
+		DRM_FORMAT_RGBX1010102,
+		DRM_FORMAT_BGRX1010102,
+		DRM_FORMAT_ARGB2101010,
+		DRM_FORMAT_ABGR2101010,
+		DRM_FORMAT_RGBA1010102,
+		DRM_FORMAT_BGRA1010102,
+		DRM_FORMAT_INVALID,
+	}},
+	{RENDER_BIT_DEPTH_8, (uint32_t[]){
 		DRM_FORMAT_XRGB8888,
+		DRM_FORMAT_XBGR8888,
+		DRM_FORMAT_RGBX8888,
+		DRM_FORMAT_BGRX8888,
 		DRM_FORMAT_ARGB8888,
+		DRM_FORMAT_ABGR8888,
+		DRM_FORMAT_RGBA8888,
+		DRM_FORMAT_BGRA8888,
+		DRM_FORMAT_INVALID,
+	}},
+	{RENDER_BIT_DEPTH_6, (uint32_t[]){
 		DRM_FORMAT_RGB565,
 		DRM_FORMAT_INVALID,
-	};
-	if (render_format_is_bgr(wlr_output->render_format)) {
-		// Start with BGR in the unlikely event that we previously required it.
-		fmts[0] = DRM_FORMAT_XBGR2101010;
-		fmts[1] = DRM_FORMAT_XRGB2101010;
-	}
+	}},
+};
+
+static bool search_render_format(struct search_context *ctx, size_t output_idx) {
+	struct matched_output_config *cfg = &ctx->configs[output_idx];
+	struct wlr_backend_output_state *backend_state = &ctx->states[output_idx];
+	struct wlr_output_state *state = &backend_state->base;
+	struct wlr_output *wlr_output = backend_state->output;
 
 	const struct wlr_drm_format_set *primary_formats =
 		wlr_output_get_primary_formats(wlr_output, WLR_BUFFER_CAP_DMABUF);
 	enum render_bit_depth needed_bits = RENDER_BIT_DEPTH_8;
 	if (cfg->config && cfg->config->render_bit_depth != RENDER_BIT_DEPTH_DEFAULT) {
 		needed_bits = cfg->config->render_bit_depth;
 	}
-	for (size_t idx = 0; fmts[idx] != DRM_FORMAT_INVALID; idx++) {
-		enum render_bit_depth format_bits = bit_depth_from_format(fmts[idx]);
-		if (needed_bits < format_bits) {
-			continue;
-		}
-		if (!wlr_drm_format_set_get(primary_formats, fmts[idx])) {
-			// This is not a supported format for this output
+
+	// Try one supported format from each category
+	for (size_t mapidx = 0; mapidx < sizeof(map); mapidx++) {
+		const struct fmtmap_entry *entry = &map[mapidx];
+		if (needed_bits < entry->bit_depth) {
+			// This format category has too many bits
 			continue;
 		}
-		wlr_output_state_set_render_format(state, fmts[idx]);
-		if (search_mode(ctx, output_idx)) {
-			return true;
+		for (size_t idx = 0; entry->fmts[idx] != DRM_FORMAT_INVALID; idx++) {
+			if (wlr_drm_format_set_get(primary_formats, entry->fmts[idx])) {
+				wlr_output_state_set_render_format(state, entry->fmts[idx]);
+				if (search_mode(ctx, output_idx)) {
+					return true;
+				}
+				break;
+			}
 		}
 	}
 	return false;