JBR-5973 Implement rendering of no-AA shapes with Vulkan pipeline

Get rid of maxTextureSize in Vulkan code. This concept was introduced to fix macOS-specific bugs and don't map well to Vulkan implementation, as this value is tied to specific device and texture format, so get rid of it for now and see whether we need it at all.

Refactored native surface data hierarchy. There was a C-style "inheritance" model with VKSDOps having an SurfaceDataOps as its first member and conversions back and forth between them. And then also privOps - pointer to the platform-specific part (WLVK). This was refactored into plain inheritance: SurfaceDataOps -> VKSurfaceData -> VKSwapchainSurfaceData -> WLVKSurfaceData

State management, synchronization & layout transition. Now using dynamic rendering and synchronization2 extensions.
Each device has a single timeline semaphore (basically 64-bit counter), monotonically increasing as device executes our commands, allowing us to track the state of the submitted batches and reuse resources which are no longer in use.

Split command recording into primary and secondary command buffers.
This allows us to record commands "in the past", before current render pass started, which gives possibility for some heavy optimizations:
1. When we suddenly need some texture in the middle of the render pass - no need to stop render pass in order to insert necessary synchronization - we can do it as if we knew it beforehand.
2. When we draw something and then clear the surface - just erase all commands inside current render pass we recorded earlier, so the actual drawing will never happen.

Shaders are compiled with glslc or glslangValidator and bytecode is inlined directly into libawt_wlawt

Memory management via VMA, vertex buffer pool, shader push constants.

Other refactoring.

make/autoconf/lib-wayland.m4

@@ -99,6 +99,9 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
     AC_ARG_WITH(vulkan-include, [AS_HELP_STRING([--with-vulkan-include],
       [specify directory for the vulkan include files])])
 
+    AC_ARG_WITH(vulkan-shader-compiler, [AS_HELP_STRING([--with-vulkan-shader-compiler],
+      [specify which shader compiler to use: glslc/glslangValidator])])
+
     if test "x$SUPPORTS_LIB_VULKAN" = xfalse; then
 
       if (test "x${with_vulkan}" != x && test "x${with_vulkan}" != xno) || \
@@ -120,7 +123,6 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
           AC_CHECK_HEADERS([${with_vulkan_include}/include/vulkan/vulkan.h],
             [ VULKAN_FOUND=yes
               VULKAN_FLAGS="-DVK_USE_PLATFORM_WAYLAND_KHR -I${with_vulkan_include}/include -DVULKAN_ENABLED"
-              VULKAN_ENABLED=true
             ],
             [ AC_MSG_ERROR([Can't find 'vulkan/vulkan.h' under '${with_vulkan_include}']) ]
           )
@@ -131,7 +133,6 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
           AC_CHECK_HEADERS([$VULKAN_SDK/include/vulkan/vulkan.h],
             [ VULKAN_FOUND=yes
               VULKAN_FLAGS="-DVK_USE_PLATFORM_WAYLAND_KHR -I${VULKAN_SDK}/include -DVULKAN_ENABLED"
-              VULKAN_ENABLED=true
             ],
             [ VULKAN_FOUND=no; break ]
           )
@@ -142,7 +143,6 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
           AC_CHECK_HEADERS([vulkan/vulkan.h],
             [ VULKAN_FOUND=yes
               VULKAN_FLAGS="-DVK_USE_PLATFORM_WAYLAND_KHR -DVULKAN_ENABLED"
-              VULKAN_ENABLED=true
             ],
             [ VULKAN_FOUND=no; break ]
           )
@@ -151,11 +151,32 @@ AC_DEFUN_ONCE([LIB_SETUP_WAYLAND],
         if test "x$VULKAN_FOUND" = xno; then
           HELP_MSG_MISSING_DEPENDENCY([vulkan])
           AC_MSG_ERROR([Could not find vulkan! $HELP_MSG ])
+        else
+          # Find shader compiler - glslc or glslangValidator
+          if (test "x${with_vulkan_shader_compiler}" = x || test "x${with_vulkan_shader_compiler}" = xglslc); then
+            UTIL_LOOKUP_PROGS(GLSLC, glslc)
+            SHADER_COMPILER="$GLSLC"
+            VULKAN_SHADER_COMPILER="glslc --target-env=vulkan1.2 -mfmt=num -o"
+          fi
+
+          if (test "x${with_vulkan_shader_compiler}" = x || test "x${with_vulkan_shader_compiler}" = xglslangValidator) && \
+              test "x$SHADER_COMPILER" = x; then
+            UTIL_LOOKUP_PROGS(GLSLANG, glslangValidator)
+            SHADER_COMPILER="$GLSLANG"
+            VULKAN_SHADER_COMPILER="glslangValidator --target-env vulkan1.2 -x -o"
+          fi
+
+          if test "x$SHADER_COMPILER" != x; then
+            VULKAN_ENABLED=true
+          else
+            AC_MSG_ERROR([Can't find shader compiler])
+          fi
         fi
       fi
     fi
   fi
   AC_SUBST(VULKAN_FLAGS)
+  AC_SUBST(VULKAN_SHADER_COMPILER)
   AC_SUBST(VULKAN_ENABLED)
   AC_SUBST(WAYLAND_CFLAGS)
   AC_SUBST(WAYLAND_LIBS)

make/autoconf/spec.gmk.template

@@ -490,6 +490,7 @@ A11Y_JAWS_ANNOUNCING_ENABLED:=@A11Y_JAWS_ANNOUNCING_ENABLED@
 WAYLAND_CFLAGS:=@WAYLAND_CFLAGS@
 WAYLAND_LIBS:=@WAYLAND_LIBS@
 VULKAN_FLAGS:=@VULKAN_FLAGS@
+VULKAN_SHADER_COMPILER:=@VULKAN_SHADER_COMPILER@
 VULKAN_ENABLED:=@VULKAN_ENABLED@
 
 # The lowest required version of macosx

make/modules/java.desktop/lib/AwtLibraries.gmk

@@ -194,6 +194,34 @@ ifeq ($(call isTargetOs, windows)+$(ENABLE_HEADLESS_ONLY)+$(A11Y_NVDA_ANNOUNCING
   TARGETS += $(COPY_NVDA_DEPENDENCIES)
 endif
 
+# Compile Vulkan shaders
+define compile-spirv
+  $(call MakeTargetDir)
+  $(VULKAN_SHADER_COMPILER) '$(call DecodeSpace, $@)' '$(call DecodeSpace, $<)'
+endef
+spirv-name = $(strip $1).h
+
+ifeq ($(VULKAN_ENABLED), true)
+  $(eval $(call SetupCopyFiles, COMPILE_VULKAN_SHADERS, \
+      SRC := $(TOPDIR)/src/$(MODULE)/share/glsl/vulkan, \
+      FILES := $(call FindFiles, $(TOPDIR)/src/$(MODULE)/share/glsl/vulkan), \
+      DEST := $(SUPPORT_OUTPUTDIR)/headers/java.desktop/vulkan/spirv, \
+      MACRO := compile-spirv, \
+      NAME_MACRO := spirv-name, \
+  ))
+  VULKAN_SHADER_LIST = $(SUPPORT_OUTPUTDIR)/headers/java.desktop/vulkan/shader_list.h
+  $(VULKAN_SHADER_LIST): $(COMPILE_VULKAN_SHADERS)
+	> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	$(foreach f, $(patsubst $(TOPDIR)/src/$(MODULE)/share/glsl/vulkan/%,%,$(call FindFiles, $(TOPDIR)/src/$(MODULE)/share/glsl/vulkan)), \
+	$(ECHO) SHADER_ENTRY\($(subst .,$(COMMA),$(subst /,_,$f))\) >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	$(ECHO) '#ifdef INCLUDE_BYTECODE' >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	$(ECHO) '#include "spirv/$f.h"' >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	$(ECHO) BYTECODE_END >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+	$(ECHO) '#endif' >> $(VULKAN_SHADER_LIST) $(NEWLINE) \
+)
+  $(BUILD_LIBAWT): $(VULKAN_SHADER_LIST)
+endif
+
 TARGETS += $(BUILD_LIBAWT)
 
 # Mac and Windows only use the native AWT lib, do not build libawt_headless
@@ -384,6 +412,7 @@ ifeq ($(call isTargetOs, windows macosx), false)
         common/java2d/vulkan \
         java.base:libjvm \
         java.base:libjava \
+        libvmahpp \
         #
 
     # Enable 'wakefield' extension for java.awt.Robot support
@@ -425,6 +454,8 @@ ifeq ($(call isTargetOs, windows macosx), false)
         DISABLED_WARNINGS_CXX_gcc := undef, \
         DISABLED_WARNINGS_clang := parentheses format undef \
             logical-op-parentheses format-nonliteral int-conversion, \
+        DISABLED_WARNINGS_gcc_VKMemory.cpp := missing-field-initializers implicit-fallthrough parentheses, \
+        DISABLED_WARNINGS_clang_VKMemory.cpp := missing-field-initializers implicit-fallthrough parentheses, \
         JDK_LIBS := libawt java.base:libjava java.base:libjvm, \
         LIBS := $(WAYLAND_LIBS), \
         LIBS_unix := $(LIBDL) $(LIBM), \

src/java.desktop/share/glsl/vulkan/test.frag

@@ -0,0 +1,8 @@
+#version 450
+
+layout(location = 0) in vec4 inColor;
+layout(location = 0) out vec4 outColor;
+
+void main() {
+    outColor = inColor;
+}

src/java.desktop/share/glsl/vulkan/test.vert

@@ -0,0 +1,21 @@
+#version 450
+
+layout(push_constant) uniform Push {
+    vec2 invViewport2; // 2.0/viewport
+} push;
+
+vec4 colors[3] = vec4[](
+vec4(1,0,0,1),
+vec4(0,1,0,1),
+vec4(0,0,1,1)
+);
+
+layout(location = 0) in vec2 inPosition;
+
+layout(location = 0) out vec4 outColor;
+
+void main() {
+    outColor = colors[gl_VertexIndex % 3];
+    gl_Position = vec4(inPosition * push.invViewport2 - 1.0, 0.0, 1.0);
+    gl_PointSize = 1.0f;
+}