added in-flight frames, window resizing and minimization

- made every swapchain image have it's own render finished semaphore
- renamed draw fence to in-flight fence (fence_draw to fences_in_flight)
- made present complete semaphores and in-flight fences "per in-flight
  frames", meaning that if we have at most 2 in-flight frames, we use 2
  present complete semaphores and 2 in-flight fences
- recreate swapchain when needed
- more refactoring
- added fps counter

1 files changed, 336 insertions, 211 deletions

diff --git a/src/engine.cpp b/src/engine.cpp
index 0cf829a..582eb1d 100644
--- a/src/engine.cpp
+++ b/src/engine.cpp
@@ -20,6 +20,7 @@
 #include <ios>
 #include <fstream>
 #include <map>
+#include <chrono>
 
 #define GLFW_INCLUDE_VULKAN
 #include <GLFW/glfw3.h>
@@ -311,11 +312,15 @@ static const std::vector<const char*> physical_device_ext_names = {
 };
 
 #ifdef NDEBUG
-const bool enable_validation_layers = false;
+constexpr bool enable_validation_layers = false;
 #else
-const bool enable_validation_layers = true;
+constexpr bool enable_validation_layers = true;
 #endif
 
+constexpr size_t max_frames_in_flight = 2;
+
+constexpr bool show_fps = false;
+
 struct PhysicalDeviceEntry {
     uint32_t idx;
     VkPhysicalDevice physical_device;
@@ -446,9 +451,19 @@ static int main_graphical() {
     // init window
     glfwInit();
     glfwWindowHint(GLFW_CLIENT_API, GLFW_NO_API);
-    glfwWindowHint(GLFW_RESIZABLE, GLFW_FALSE);
+
+    bool fb_resized = false;
+
     GLFWwindow* window = glfwCreateWindow(SCREEN_WIDTH, SCREEN_HEIGHT, "Engine", nullptr, nullptr);
 
+    // TODO: might have to improve, because if in the future we need more values in a glfw callback,
+    // this approch wouldn't work
+    glfwSetWindowUserPointer(window, &fb_resized);
+
+    glfwSetFramebufferSizeCallback(window, [](GLFWwindow* window, int /* width */, int /* height */) {
+            *reinterpret_cast<bool*>(glfwGetWindowUserPointer(window)) = true;
+        });
+
     // init Vulkan
     std::cout << "Vulkan loader version: " << engine::myvk::api { VK_HEADER_VERSION_COMPLETE } << std::endl;
 
@@ -680,7 +695,12 @@ static int main_graphical() {
 
                 return std::optional<unsigned> { score };
             }();
-            std::cout << "    is suitable: " << std::boolalpha << static_cast<bool>(score) << std::noboolalpha << std::endl;
+            std::cout << "    score: ";
+            if (score)
+                std::cout << *score;
+            else
+                std::cout << "not suitable";
+            std::cout << std::endl;
 
             if (score)
                 physical_devices.insert({ *score,
@@ -798,160 +818,195 @@ static int main_graphical() {
     }();
 
     // create swap chain
-    // TODO: should probably use version 2 of theses functions, but glfwCreateWindowSurface return
-    // version 1, so for now we will use version 1
-    auto surface_capabilities = [&]() {
-        VkSurfaceCapabilitiesKHR surface_capabilities;
-        if (VkResult res = vkGetPhysicalDeviceSurfaceCapabilitiesKHR(physical_device, surface, &surface_capabilities); res != VK_SUCCESS) {
-            std::cerr << "failed to get physical device surface capabilities, error code: " << string_VkResult(res) << std::endl;
-            std::exit(EXIT_FAILURE);
-        }
-        return surface_capabilities;
-    }();
-
-    auto swapchain_extent = [&]() {
-        if (surface_capabilities.currentExtent.width != std::numeric_limits<uint32_t>::max())
-            return surface_capabilities.currentExtent;
-        int width, height;
-        glfwGetFramebufferSize(window, &width, &height);
-        return VkExtent2D{
-            .width  = std::clamp(static_cast<uint32_t>(width),  surface_capabilities.minImageExtent.width,  surface_capabilities.maxImageExtent.width),
-            .height = std::clamp(static_cast<uint32_t>(height), surface_capabilities.minImageExtent.height, surface_capabilities.maxImageExtent.height),
-        };
-    }();
+    // TODO: when we recreate the swapchain, I don't know if the number of images in it can change
+    // between the old and new swapchain. Right now we assume that it doesn't, which might cause
+    // problem (but I'm not sure)
+    // TODO: pass the old swapchain to VkSwapchainCreateInfoKHR so the new swapchain can be created
+    // while the old one is in-flight
+    VkExtent2D swapchain_extent;
+    VkSurfaceFormatKHR surface_format;
+    VkSwapchainKHR swapchain = nullptr;
+    std::vector<VkImage> swapchain_imgs;
+    std::vector<VkImageView> swapchain_img_views;
+
+    const auto destroy_swapchain = [&]() {
+        for (auto it_img_view = swapchain_img_views.rbegin(); it_img_view != swapchain_img_views.rend(); ++it_img_view)
+            vkDestroyImageView(device, *it_img_view, nullptr);
+        vkDestroySwapchainKHR(device, swapchain, nullptr);
+    };
 
-    auto surface_format = [&]() {
-        auto surface_formats = [&]() {
-            uint32_t surface_formats_count;
-            if (VkResult res = vkGetPhysicalDeviceSurfaceFormatsKHR(physical_device, surface, &surface_formats_count, nullptr); res != VK_SUCCESS) {
-                std::cerr << "failed to get physical device surface formats, error code: " << string_VkResult(res) << std::endl;
+    const auto recreate_swapchain = [&]() {
+        if (swapchain != nullptr) {
+            if (VkResult res = vkDeviceWaitIdle(device); res != VK_SUCCESS) {
+                std::cerr << "failed to wait idle for device, error code: " << string_VkResult(res) << std::endl;
                 std::exit(EXIT_FAILURE);
             }
-            std::vector<VkSurfaceFormatKHR> surface_formats(surface_formats_count);
-            if (VkResult res = vkGetPhysicalDeviceSurfaceFormatsKHR(physical_device, surface, &surface_formats_count, surface_formats.data()); res != VK_SUCCESS) {
-                std::cerr << "failed to get physical device surface formats, error code: " << string_VkResult(res) << std::endl;
+            destroy_swapchain();
+        }
+
+        // TODO: should probably use version 2 of theses functions, but glfwCreateWindowSurface
+        // return version 1, so for now we will use version 1
+        auto surface_capabilities = [&]() {
+            VkSurfaceCapabilitiesKHR surface_capabilities;
+            if (VkResult res = vkGetPhysicalDeviceSurfaceCapabilitiesKHR(physical_device, surface, &surface_capabilities); res != VK_SUCCESS) {
+                std::cerr << "failed to get physical device surface capabilities, error code: " << string_VkResult(res) << std::endl;
                 std::exit(EXIT_FAILURE);
             }
-            return surface_formats;
+            return surface_capabilities;
         }();
 
-        for (const auto& surface_format : surface_formats) {
-            if (surface_format.format == VK_FORMAT_B8G8R8A8_SRGB
-                    && surface_format.colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR)
-                return surface_format;
-        }
+        swapchain_extent = [&]() {
+            if (surface_capabilities.currentExtent.width != std::numeric_limits<uint32_t>::max())
+                return surface_capabilities.currentExtent;
+            int width, height;
+            glfwGetFramebufferSize(window, &width, &height);
+            // TODO: improve minimization handling. I'm on wayland so it's impossible to know if a
+            // window is minimized, therefore I can't really test it properly right now
+            while (width == 0 || height == 0) {
+                glfwGetFramebufferSize(window, &width, &height);
+                glfwWaitEvents();
+            }
+            return VkExtent2D{
+                .width  = std::clamp(static_cast<uint32_t>(width),  surface_capabilities.minImageExtent.width,  surface_capabilities.maxImageExtent.width),
+                .height = std::clamp(static_cast<uint32_t>(height), surface_capabilities.minImageExtent.height, surface_capabilities.maxImageExtent.height),
+            };
+        }();
 
-        // not found
-        std::cerr << "surface format not found (VK_FORMAT_B8G8R8_SRGB and VK_COLOR_SPACE_SRGB_NONLINEAR_KHR)" << std::endl;
-        std::exit(EXIT_FAILURE);
-    }();
+        surface_format = [&]() {
+            auto surface_formats = [&]() {
+                uint32_t surface_formats_count;
+                if (VkResult res = vkGetPhysicalDeviceSurfaceFormatsKHR(physical_device, surface, &surface_formats_count, nullptr); res != VK_SUCCESS) {
+                    std::cerr << "failed to get physical device surface formats, error code: " << string_VkResult(res) << std::endl;
+                    std::exit(EXIT_FAILURE);
+                }
+                std::vector<VkSurfaceFormatKHR> surface_formats(surface_formats_count);
+                if (VkResult res = vkGetPhysicalDeviceSurfaceFormatsKHR(physical_device, surface, &surface_formats_count, surface_formats.data()); res != VK_SUCCESS) {
+                    std::cerr << "failed to get physical device surface formats, error code: " << string_VkResult(res) << std::endl;
+                    std::exit(EXIT_FAILURE);
+                }
+                return surface_formats;
+            }();
 
-    auto present_mode = [&]() {
-        uint32_t present_modes_count;
-        if (VkResult res = vkGetPhysicalDeviceSurfacePresentModesKHR(physical_device, surface, &present_modes_count, nullptr); res != VK_SUCCESS) {
-            std::cerr << "failed to get physical device present modes, error code: " << string_VkResult(res) << std::endl;
-            std::exit(EXIT_FAILURE);
-        }
-        std::vector<VkPresentModeKHR> present_modes(present_modes_count);
-        if (VkResult res = vkGetPhysicalDeviceSurfacePresentModesKHR(physical_device, surface, &present_modes_count, present_modes.data()); res != VK_SUCCESS) {
-            std::cerr << "failed to get physical device present modes, error code: " << string_VkResult(res) << std::endl;
-            std::exit(EXIT_FAILURE);
-        }
+            for (const auto& surface_format : surface_formats) {
+                if (surface_format.format == VK_FORMAT_B8G8R8A8_SRGB
+                        && surface_format.colorSpace == VK_COLOR_SPACE_SRGB_NONLINEAR_KHR)
+                    return surface_format;
+            }
 
-        for (const auto& present_mode : present_modes)
-            if (present_mode == VK_PRESENT_MODE_MAILBOX_KHR)
-                return present_mode;
+            // not found
+            std::cerr << "surface format not found (VK_FORMAT_B8G8R8_SRGB and VK_COLOR_SPACE_SRGB_NONLINEAR_KHR)" << std::endl;
+            std::exit(EXIT_FAILURE);
+        }();
 
-        return VK_PRESENT_MODE_FIFO_KHR;
-    }();
+        [&]() {
+            auto present_mode = [&]() {
+                auto present_modes = [&]() {
+                    uint32_t present_modes_count;
+                    if (VkResult res = vkGetPhysicalDeviceSurfacePresentModesKHR(physical_device, surface, &present_modes_count, nullptr); res != VK_SUCCESS) {
+                        std::cerr << "failed to get physical device present modes, error code: " << string_VkResult(res) << std::endl;
+                        std::exit(EXIT_FAILURE);
+                    }
+                    std::vector<VkPresentModeKHR> present_modes(present_modes_count);
+                    if (VkResult res = vkGetPhysicalDeviceSurfacePresentModesKHR(physical_device, surface, &present_modes_count, present_modes.data()); res != VK_SUCCESS) {
+                        std::cerr << "failed to get physical device present modes, error code: " << string_VkResult(res) << std::endl;
+                        std::exit(EXIT_FAILURE);
+                    }
+                    return present_modes;
+                }();
 
-    auto swapchain = [&]() {
-        // TODO: remove unnecessary static_cast<uint32_t>, but at this moment I'm not sure where they
-        // are necessary
-        uint32_t min_image_count = std::max(static_cast<uint32_t>(3), surface_capabilities.minImageCount + static_cast<uint32_t>(1));
-        if (surface_capabilities.maxImageCount > 0 && surface_capabilities.maxImageCount < min_image_count)
-            min_image_count = surface_capabilities.maxImageCount;
-
-        // might not be used, but if we do, we have to keep it in memory until the call to vkCreateSwapchainKHR()
-        std::array<uint32_t, 2> queue_family_indices;
-        VkSwapchainCreateInfoKHR swapchain_ci {
-            .sType                 = VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR,
-            .pNext                 = nullptr,
-            .flags                 = {},
-            .surface               = surface,
-            .minImageCount         = min_image_count,
-            .imageFormat           = surface_format.format,
-            .imageColorSpace       = surface_format.colorSpace,
-            .imageExtent           = swapchain_extent,
-            .imageArrayLayers      = 1,
-            .imageUsage            = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT,
-            .imageSharingMode      = {},
-            .queueFamilyIndexCount = {},
-            .pQueueFamilyIndices   = {},
-            .preTransform          = surface_capabilities.currentTransform,
-            .compositeAlpha        = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR,
-            .presentMode           = present_mode,
-            .clipped               = VK_TRUE,
-            .oldSwapchain          = {},
-        };
-        if (graphics_queue_family_index == present_queue_family_index) {
-            swapchain_ci.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE;
-        } else {
-            swapchain_ci.imageSharingMode = VK_SHARING_MODE_CONCURRENT;
-            queue_family_indices[0] = graphics_queue_family_index;
-            queue_family_indices[1] = present_queue_family_index;
-            swapchain_ci.queueFamilyIndexCount = 2;
-            swapchain_ci.pQueueFamilyIndices = queue_family_indices.data();
-        }
+                for (const auto& present_mode : present_modes)
+                    if (present_mode == VK_PRESENT_MODE_MAILBOX_KHR)
+                        return present_mode;
 
-        VkSwapchainKHR swapchain;
-        if (VkResult res = vkCreateSwapchainKHR(device, &swapchain_ci, nullptr, &swapchain); res != VK_SUCCESS) {
-            std::cerr << "failed create swapchain, error code: " << string_VkResult(res) << std::endl;
-            std::exit(EXIT_FAILURE);
-        }
-        return swapchain;
-    }();
+                return VK_PRESENT_MODE_FIFO_KHR;
+            }();
 
-    auto swapchain_imgs = [&]() {
-        uint32_t swapchain_imgs_count;
-        if (VkResult res = vkGetSwapchainImagesKHR(device, swapchain, &swapchain_imgs_count, nullptr); res != VK_SUCCESS) {
-            std::cerr << "failed to get swapchain images, error code: " << string_VkResult(res) << std::endl;
-            std::exit(EXIT_FAILURE);
-        }
-        std::vector<VkImage> swapchain_imgs(swapchain_imgs_count);
-        if (VkResult res = vkGetSwapchainImagesKHR(device, swapchain, &swapchain_imgs_count, swapchain_imgs.data()); res != VK_SUCCESS) {
-            std::cerr << "failed to get swapchain images, error code: " << string_VkResult(res) << std::endl;
-            std::exit(EXIT_FAILURE);
-        }
-        return swapchain_imgs;
-    }();
+            // TODO: remove unnecessary static_cast<uint32_t>, but at this moment I'm not sure where they
+            // are necessary
+            uint32_t min_image_count = std::max(static_cast<uint32_t>(3), surface_capabilities.minImageCount + static_cast<uint32_t>(1));
+            if (surface_capabilities.maxImageCount > 0 && surface_capabilities.maxImageCount < min_image_count)
+                min_image_count = surface_capabilities.maxImageCount;
 
-    auto swapchain_img_views = [&]() {
-        std::vector<VkImageView> swapchain_img_views(swapchain_imgs.size());
-        for (uint32_t i = 0; i < swapchain_imgs.size(); i++) {
-            VkImageViewCreateInfo img_view_ci {
-                .sType            = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
-                .pNext            = {},
-                .flags            = {},
-                .image            = swapchain_imgs[i],
-                .viewType         = VK_IMAGE_VIEW_TYPE_2D,
-                .format           = surface_format.format,
-                .components       = {},
-                .subresourceRange = { // VkImageSubresourceRange
-                    .aspectMask     = VK_IMAGE_ASPECT_COLOR_BIT,
-                    .baseMipLevel   = 0,
-                    .levelCount     = 1,
-                    .baseArrayLayer = 0,
-                    .layerCount     = 1,
-                },
+            // might not be used, but if we do, we have to keep it in memory until the call to vkCreateSwapchainKHR()
+            std::array<uint32_t, 2> queue_family_indices;
+            VkSwapchainCreateInfoKHR swapchain_ci {
+                .sType                 = VK_STRUCTURE_TYPE_SWAPCHAIN_CREATE_INFO_KHR,
+                .pNext                 = nullptr,
+                .flags                 = {},
+                .surface               = surface,
+                .minImageCount         = min_image_count,
+                .imageFormat           = surface_format.format,
+                .imageColorSpace       = surface_format.colorSpace,
+                .imageExtent           = swapchain_extent,
+                .imageArrayLayers      = 1,
+                .imageUsage            = VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT,
+                .imageSharingMode      = {},
+                .queueFamilyIndexCount = {},
+                .pQueueFamilyIndices   = {},
+                .preTransform          = surface_capabilities.currentTransform,
+                .compositeAlpha        = VK_COMPOSITE_ALPHA_OPAQUE_BIT_KHR,
+                .presentMode           = present_mode,
+                .clipped               = VK_TRUE,
+                .oldSwapchain          = {},
             };
-            if (VkResult res = vkCreateImageView(device, &img_view_ci, nullptr, &swapchain_img_views[i]); res != VK_SUCCESS) {
-                std::cerr << "failed to create image view, error code: " << string_VkResult(res) << std::endl;
+            if (graphics_queue_family_index == present_queue_family_index) {
+                swapchain_ci.imageSharingMode = VK_SHARING_MODE_EXCLUSIVE;
+            } else {
+                swapchain_ci.imageSharingMode = VK_SHARING_MODE_CONCURRENT;
+                queue_family_indices[0] = graphics_queue_family_index;
+                queue_family_indices[1] = present_queue_family_index;
+                swapchain_ci.queueFamilyIndexCount = 2;
+                swapchain_ci.pQueueFamilyIndices = queue_family_indices.data();
+            }
+
+            if (VkResult res = vkCreateSwapchainKHR(device, &swapchain_ci, nullptr, &swapchain); res != VK_SUCCESS) {
+                std::cerr << "failed create swapchain, error code: " << string_VkResult(res) << std::endl;
                 std::exit(EXIT_FAILURE);
             }
-        }
-        return swapchain_img_views;
-    }();
+        }();
+
+        [&]() {
+            uint32_t swapchain_imgs_count;
+            if (VkResult res = vkGetSwapchainImagesKHR(device, swapchain, &swapchain_imgs_count, nullptr); res != VK_SUCCESS) {
+                std::cerr << "failed to get swapchain images, error code: " << string_VkResult(res) << std::endl;
+                std::exit(EXIT_FAILURE);
+            }
+            swapchain_imgs.resize(swapchain_imgs_count);
+            if (VkResult res = vkGetSwapchainImagesKHR(device, swapchain, &swapchain_imgs_count, swapchain_imgs.data()); res != VK_SUCCESS) {
+                std::cerr << "failed to get swapchain images, error code: " << string_VkResult(res) << std::endl;
+                std::exit(EXIT_FAILURE);
+            }
+        }();
+
+        [&]() {
+            swapchain_img_views.resize(swapchain_imgs.size());
+            for (uint32_t i = 0; i < swapchain_imgs.size(); i++) {
+                VkImageViewCreateInfo img_view_ci {
+                    .sType            = VK_STRUCTURE_TYPE_IMAGE_VIEW_CREATE_INFO,
+                    .pNext            = {},
+                    .flags            = {},
+                    .image            = swapchain_imgs[i],
+                    .viewType         = VK_IMAGE_VIEW_TYPE_2D,
+                    .format           = surface_format.format,
+                    .components       = {},
+                    .subresourceRange = { // VkImageSubresourceRange
+                        .aspectMask     = VK_IMAGE_ASPECT_COLOR_BIT,
+                        .baseMipLevel   = 0,
+                        .levelCount     = 1,
+                        .baseArrayLayer = 0,
+                        .layerCount     = 1,
+                    },
+                };
+                if (VkResult res = vkCreateImageView(device, &img_view_ci, nullptr, &swapchain_img_views[i]); res != VK_SUCCESS) {
+                    std::cerr << "failed to create image view, error code: " << string_VkResult(res) << std::endl;
+                    std::exit(EXIT_FAILURE);
+                }
+            }
+            return swapchain_img_views;
+        }();
+    };
+
+    recreate_swapchain();
 
     auto [pl_layout, graphics_pl] = [&]() {
         // reading shader file
@@ -1180,83 +1235,143 @@ static int main_graphical() {
         return cmd_pool;
     }();
 
-    auto cmd_buf = [&]() {
+    auto cmd_bufs = [&]() {
         VkCommandBufferAllocateInfo cmd_buf_ai {
             .sType              = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO,
             .pNext              = nullptr,
             .commandPool        = cmd_pool,
             .level              = VK_COMMAND_BUFFER_LEVEL_PRIMARY,
-            .commandBufferCount = 1,
+            .commandBufferCount = max_frames_in_flight,
         };
-        VkCommandBuffer cmd_buf;
-        if (VkResult res = vkAllocateCommandBuffers(device, &cmd_buf_ai, &cmd_buf); res != VK_SUCCESS) {
-            std::cerr << "failed to allocate command buffer, error code: " << string_VkResult(res) << std::endl;
+        std::array<VkCommandBuffer, max_frames_in_flight> cmd_bufs;
+        if (VkResult res = vkAllocateCommandBuffers(device, &cmd_buf_ai, cmd_bufs.data()); res != VK_SUCCESS) {
+            std::cerr << "failed to allocate command buffers, error code: " << string_VkResult(res) << std::endl;
             std::exit(EXIT_FAILURE);
         }
-        return cmd_buf;
+        return cmd_bufs;
     }();
 
     // create sync objects
-    auto [sem_present_complete, sem_render_finished] = [&]() {
-        const auto create_semaphore = [&](const char* name) {
-            VkSemaphoreCreateInfo sem_ci {
-                .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
-                .pNext = nullptr,
-                .flags = {},
-            };
-            VkSemaphore sem;
-            if (VkResult res = vkCreateSemaphore(device, &sem_ci, nullptr, &sem); res != VK_SUCCESS) {
-                std::cerr << "failed to create " << name << " semaphore, error code: " << string_VkResult(res) << std::endl;
-                std::exit(EXIT_FAILURE);
+    auto [sems_present_complete, sems_render_finished] = [&]() {
+        // TODO: remove duplicate code
+
+        auto sems_present_complete = [&]() {
+            std::array<VkSemaphore, max_frames_in_flight> sems_present_complete;
+            size_t num = 0;
+            for (auto& sem : sems_present_complete) {
+                VkSemaphoreCreateInfo sem_ci {
+                    .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
+                    .pNext = nullptr,
+                    .flags = {},
+                };
+                if (VkResult res = vkCreateSemaphore(device, &sem_ci, nullptr, &sem); res != VK_SUCCESS) {
+                    std::cerr << "failed to create present complete semaphore #" << (++num) << ", error code: " << string_VkResult(res) << std::endl;
+                    std::exit(EXIT_FAILURE);
+                }
             }
-            return sem;
-        };
+            return sems_present_complete;
+        }();
+
+        auto sems_render_finished = [&]() {
+            std::vector<VkSemaphore> sems_render_finished(swapchain_imgs.size());
+            size_t num = 0;
+            for (auto& sem : sems_render_finished) {
+                VkSemaphoreCreateInfo sem_ci {
+                    .sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO,
+                    .pNext = nullptr,
+                    .flags = {},
+                };
+                if (VkResult res = vkCreateSemaphore(device, &sem_ci, nullptr, &sem); res != VK_SUCCESS) {
+                    std::cerr << "failed to create render finished semaphore #" << (++num) << ", error code: " << string_VkResult(res) << std::endl;
+                    std::exit(EXIT_FAILURE);
+                }
+            }
+            return sems_render_finished;
+        }();
+
         return std::tuple {
-            create_semaphore("present complete"),
-            create_semaphore("render finished"),
+            sems_present_complete,
+            sems_render_finished,
         };
     }();
-    std::cout << "sem_present_complete: " << sem_present_complete << std::endl;
-    std::cout << "sem_render_finished: " << sem_render_finished << std::endl;
-
-    auto fence_draw = [&]() {
-        VkFenceCreateInfo fence_draw_ci {
-            .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,
-            .pNext = nullptr,
-            .flags = VK_FENCE_CREATE_SIGNALED_BIT,
-        };
-        VkFence fence_draw;
-        if (VkResult res = vkCreateFence(device, &fence_draw_ci, nullptr, &fence_draw); res != VK_SUCCESS) {
-            std::cerr << "failed to create draw semaphore, error code: " << string_VkResult(res) << std::endl;
-            std::exit(EXIT_FAILURE);
+
+    std::cout << "sems_present_complete: [ ";
+    for (auto it_sem = sems_present_complete.begin(); it_sem != sems_present_complete.end(); ++it_sem) {
+        if (it_sem != sems_present_complete.begin())
+            std::cout << ", ";
+        std::cout << *it_sem;
+    }
+    std::cout << " ]" << std::endl;
+
+    std::cout << "sems_render_finished: [ ";
+    for (auto it_sem = sems_render_finished.begin(); it_sem != sems_render_finished.end(); ++it_sem) {
+        if (it_sem != sems_render_finished.begin())
+            std::cout << ", ";
+        std::cout << *it_sem;
+    }
+    std::cout << " ]" << std::endl;
+
+    auto fences_in_flight = [&]() {
+        std::array<VkFence, max_frames_in_flight> fences_in_flight;
+        size_t num = 0;
+        for (auto& fence_in_flight : fences_in_flight) {
+            VkFenceCreateInfo fence_in_flight_ci {
+                .sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO,
+                .pNext = nullptr,
+                .flags = VK_FENCE_CREATE_SIGNALED_BIT,
+            };
+            if (VkResult res = vkCreateFence(device, &fence_in_flight_ci, nullptr, &fence_in_flight); res != VK_SUCCESS) {
+                std::cerr << "failed to create draw fence #" << (++num) << ", error code: " << string_VkResult(res) << std::endl;
+                std::exit(EXIT_FAILURE);
+            }
         }
-        return fence_draw;
+        return fences_in_flight;
     }();
-    std::cout << "fence_draw: " << fence_draw << std::endl;
+
+    std::cout << "fences_in_flight: [ ";
+    for (auto it_fence = fences_in_flight.begin(); it_fence != fences_in_flight.end(); ++it_fence) {
+        if (it_fence != fences_in_flight.begin())
+            std::cout << ", ";
+        std::cout << *it_fence;
+    }
+    std::cout << " ]" << std::endl;
+
+    auto last_time = std::chrono::system_clock::now();
+
+    uint32_t frame_idx = 0;
 
     // main loop
     while (!glfwWindowShouldClose(window)) {
         glfwPollEvents();
 
-        if (VkResult res = vkWaitForFences(device, 1, &fence_draw, VK_TRUE, std::numeric_limits<uint64_t>::max()); res != VK_SUCCESS) {
-            std::cerr << "failed to wait for draw fence, error code: " << string_VkResult(res) << std::endl;
-            std::exit(EXIT_FAILURE);
-        }
+        auto cur_time = std::chrono::system_clock::now();
+        auto ellapsed_time = cur_time - last_time;
+        last_time = cur_time;
+
+        if (show_fps)
+            std::cout << "\rfps: " << (1'000'000'000. / static_cast<double>(ellapsed_time.count()));
 
-        if (VkResult res = vkQueueWaitIdle(present_queue); res != VK_SUCCESS) {
-            std::cerr << "failed to wait idle for graphics queue, error code: " << string_VkResult(res) << std::endl;
+        if (VkResult res = vkWaitForFences(device, 1, &fences_in_flight[frame_idx], VK_TRUE, std::numeric_limits<uint64_t>::max()); res != VK_SUCCESS) {
+            std::cerr << "failed to wait for draw fence, error code: " << string_VkResult(res) << std::endl;
             std::exit(EXIT_FAILURE);
         }
 
-        auto img_idx = [&]() {
-            uint32_t img_idx;
-            if (VkResult res = vkAcquireNextImageKHR(device, swapchain,
-                        std::numeric_limits<uint64_t>::max(), sem_present_complete, nullptr, &img_idx); res != VK_SUCCESS) {
+        uint32_t img_idx;
+        {
+            VkResult res = vkAcquireNextImageKHR(device, swapchain,
+                    std::numeric_limits<uint64_t>::max(), sems_present_complete[frame_idx], nullptr, &img_idx);
+            switch (res) {
+            case VK_SUBOPTIMAL_KHR:
+            case VK_SUCCESS:
+                break;
+            case VK_ERROR_OUT_OF_DATE_KHR:
+                recreate_swapchain();
+                continue;
+            default:
                 std::cerr << "failed to acquire next image, error code: " << string_VkResult(res) << std::endl;
                 std::exit(EXIT_FAILURE);
             }
-            return img_idx;
-        }();
+        }
 
         // record command buffer
         {
@@ -1266,13 +1381,13 @@ static int main_graphical() {
                 .flags            = {},
                 .pInheritanceInfo = {},
             };
-            if (VkResult res = vkBeginCommandBuffer(cmd_buf, &cmd_buf_bi); res != VK_SUCCESS) {
+            if (VkResult res = vkBeginCommandBuffer(cmd_bufs[frame_idx], &cmd_buf_bi); res != VK_SUCCESS) {
                 std::cerr << "failed to begin command buffer, error code: " << string_VkResult(res) << std::endl;
                 std::exit(EXIT_FAILURE);
             }
         }
 
-        transition_image_layout(cmd_buf, swapchain_imgs[img_idx],
+        transition_image_layout(cmd_bufs[frame_idx], swapchain_imgs[img_idx],
             VK_IMAGE_LAYOUT_UNDEFINED, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL,
             {}, VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT,
             VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT);
@@ -1303,10 +1418,10 @@ static int main_graphical() {
                 .pDepthAttachment     = {},
                 .pStencilAttachment   = {},
             };
-            vkCmdBeginRendering(cmd_buf, &render_info);
+            vkCmdBeginRendering(cmd_bufs[frame_idx], &render_info);
         }
 
-        vkCmdBindPipeline(cmd_buf, VK_PIPELINE_BIND_POINT_GRAPHICS, graphics_pl);
+        vkCmdBindPipeline(cmd_bufs[frame_idx], VK_PIPELINE_BIND_POINT_GRAPHICS, graphics_pl);
 
         {
             VkViewport viewport {
@@ -1317,7 +1432,7 @@ static int main_graphical() {
                 .minDepth = 0.f,
                 .maxDepth = 1.f,
             };
-            vkCmdSetViewport(cmd_buf, 0, 1, &viewport);
+            vkCmdSetViewport(cmd_bufs[frame_idx], 0, 1, &viewport);
         }
 
         {
@@ -1325,24 +1440,24 @@ static int main_graphical() {
                 .offset = { 0, 0 },
                 .extent = swapchain_extent,
             };
-            vkCmdSetScissor(cmd_buf, 0, 1, &scissor);
+            vkCmdSetScissor(cmd_bufs[frame_idx], 0, 1, &scissor);
         }
 
-        vkCmdDraw(cmd_buf, 3, 1, 0, 0);
+        vkCmdDraw(cmd_bufs[frame_idx], 3, 1, 0, 0);
 
-        vkCmdEndRendering(cmd_buf);
+        vkCmdEndRendering(cmd_bufs[frame_idx]);
 
-        transition_image_layout(cmd_buf, swapchain_imgs[img_idx],
+        transition_image_layout(cmd_bufs[frame_idx], swapchain_imgs[img_idx],
             VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_PRESENT_SRC_KHR,
             VK_ACCESS_2_COLOR_ATTACHMENT_WRITE_BIT, {},
             VK_PIPELINE_STAGE_2_COLOR_ATTACHMENT_OUTPUT_BIT, VK_PIPELINE_STAGE_2_BOTTOM_OF_PIPE_BIT);
 
-        if (VkResult res = vkEndCommandBuffer(cmd_buf); res != VK_SUCCESS) {
+        if (VkResult res = vkEndCommandBuffer(cmd_bufs[frame_idx]); res != VK_SUCCESS) {
             std::cerr << "failed to end command buffer, error code: " << string_VkResult(res) << std::endl;
             std::exit(EXIT_FAILURE);
         }
 
-        if (VkResult res = vkResetFences(device, 1, &fence_draw); res != VK_SUCCESS) {
+        if (VkResult res = vkResetFences(device, 1, &fences_in_flight[frame_idx]); res != VK_SUCCESS) {
             std::cerr << "failed to reset draw fence, error code: " << string_VkResult(res) << std::endl;
             std::exit(EXIT_FAILURE);
         }
@@ -1353,14 +1468,14 @@ static int main_graphical() {
                 .sType                = VK_STRUCTURE_TYPE_SUBMIT_INFO,
                 .pNext                = nullptr,
                 .waitSemaphoreCount   = 1,
-                .pWaitSemaphores      = &sem_present_complete,
+                .pWaitSemaphores      = &sems_present_complete[frame_idx],
                 .pWaitDstStageMask    = &pl_stage_flags,
                 .commandBufferCount   = 1,
-                .pCommandBuffers      = &cmd_buf,
+                .pCommandBuffers      = &cmd_bufs[frame_idx],
                 .signalSemaphoreCount = 1,
-                .pSignalSemaphores    = &sem_render_finished,
+                .pSignalSemaphores    = &sems_render_finished[img_idx],
             };
-            if (VkResult res = vkQueueSubmit(graphics_queue, 1, &submit_info, fence_draw); res != VK_SUCCESS) {
+            if (VkResult res = vkQueueSubmit(graphics_queue, 1, &submit_info, fences_in_flight[frame_idx]); res != VK_SUCCESS) {
                 std::cerr << "failed to submit queue, error code: " << string_VkResult(res) << std::endl;
                 std::exit(EXIT_FAILURE);
             }
@@ -1371,34 +1486,44 @@ static int main_graphical() {
                 .sType              = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR,
                 .pNext              = nullptr,
                 .waitSemaphoreCount = 1,
-                .pWaitSemaphores    = &sem_render_finished,
+                .pWaitSemaphores    = &sems_render_finished[img_idx],
                 .swapchainCount     = 1,
                 .pSwapchains        = &swapchain,
                 .pImageIndices      = &img_idx,
                 .pResults           = nullptr,
             };
-            if (VkResult res = vkQueuePresentKHR(present_queue, &present_info); res != VK_SUCCESS) {
+            VkResult res = vkQueuePresentKHR(present_queue, &present_info);
+            if (res == VK_SUBOPTIMAL_KHR || res == VK_ERROR_OUT_OF_DATE_KHR || fb_resized) {
+                fb_resized = false;
+                recreate_swapchain();
+            } else if (res != VK_SUCCESS) {
                 std::cerr << "failed to present, error code: " << string_VkResult(res) << std::endl;
                 std::exit(EXIT_FAILURE);
             }
         }
+
+        frame_idx = (frame_idx + 1) % max_frames_in_flight;
     }
 
+    if (show_fps)
+        std::cout << std::endl;
+
     if (VkResult res = vkDeviceWaitIdle(device); res != VK_SUCCESS) {
         std::cerr << "failed to wait idle for device, error code: " << string_VkResult(res) << std::endl;
         std::exit(EXIT_FAILURE);
     }
 
     // cleanup
-    vkDestroyFence(device, fence_draw, nullptr);
-    vkDestroySemaphore(device, sem_render_finished, nullptr);
-    vkDestroySemaphore(device, sem_present_complete, nullptr);
+    for (auto it_fence = fences_in_flight.rbegin(); it_fence != fences_in_flight.rend(); ++it_fence)
+        vkDestroyFence(device, *it_fence, nullptr);
+    for (auto it_sem = sems_render_finished.rbegin(); it_sem != sems_render_finished.rend(); ++it_sem)
+        vkDestroySemaphore(device, *it_sem, nullptr);
+    for (auto it_sem = sems_present_complete.rbegin(); it_sem != sems_present_complete.rend(); ++it_sem)
+        vkDestroySemaphore(device, *it_sem, nullptr);
     vkDestroyCommandPool(device, cmd_pool, nullptr);
     vkDestroyPipeline(device, graphics_pl, nullptr);
     vkDestroyPipelineLayout(device, pl_layout, nullptr);
-    for (auto it_img_view = swapchain_img_views.rbegin(); it_img_view != swapchain_img_views.rend(); ++it_img_view)
-        vkDestroyImageView(device, *it_img_view, nullptr);
-    vkDestroySwapchainKHR(device, swapchain, nullptr);
+    destroy_swapchain();
     vkDestroySurfaceKHR(instance, surface, nullptr);
     vkDestroyDevice(device, nullptr);