diff options
58 files changed, 3807 insertions, 1671 deletions
@@ -32,6 +32,7 @@ Erik Selecký <35656626+rxlecky@users.noreply.github.com> <35656626+SeleckyErik@ Fabian <supagu@gmail.com> Ferenc Arn <tagcup@yahoo.com> Ferenc Arn <tagcup@yahoo.com> <tagcup@users.noreply.github.com> +foxydevloper <12120644+foxydevloper@users.noreply.github.com> Fredia Huya-Kouadio <fhuyakou@gmail.com> Fredia Huya-Kouadio <fhuyakou@gmail.com> <fhuya@google.com> Geequlim <geequlim@gmail.com> @@ -56,6 +57,7 @@ Indah Sylvia <ISylvox@yahoo.com> J08nY <johny@neuromancer.sk> <jancar.jj@gmail.com> J08nY <johny@neuromancer.sk> <J08nY@users.noreply.github.com> Jakub Grzesik <kubecz3k@gmail.com> +janglee <merupatel123@gmail.com> Jérôme Gully <jerome.gully0@gmail.com> JFonS <joan.fonssanchez@gmail.com> Juan Linietsky <reduzio@gmail.com> @@ -92,6 +94,7 @@ Michael Alexsander <michaelalexsander@protonmail.com> Nathan Franke <natfra@pm.me> <nathanwfranke@gmail.com> Nathan Lovato <nathan@gdquest.com> Nathan Warden <nathan@nathanwarden.com> <nathanwardenlee@icloud.com> +Nicholas Huelin <62965063+SirQuartz@users.noreply.github.com> Nils ANDRÉ-CHANG <nils@nilsand.re> Nils ANDRÉ-CHANG <nils@nilsand.re> <nils.andre.chang@gmail.com> Nuno Donato <nunodonato@gmail.com> <n.donato@estrelasustentavel.pt> diff --git a/AUTHORS.md b/AUTHORS.md index c5a971b345..5147da3700 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -35,6 +35,7 @@ name is available. Andrea Catania (AndreaCatania) Andrii Doroshenko (Xrayez) Andy Moss (MillionOstrich) + Angad Kambli (angad-k) Anish Bhobe (KidRigger) Anton Yabchinskiy (a12n) Anutrix @@ -44,6 +45,7 @@ name is available. Ben Brookshire (sheepandshepherd) Benjamin Larsson (Nallebeorn) Bernard Liebl (poke1024) + Bhuvan Vemula (Bhu1-V) Błażej Szczygieł (zaps166) Bojidar Marinov (bojidar-bg) Bruno Lourenço (MadEqua) @@ -65,6 +67,7 @@ name is available. Dmitry Koteroff (Krakean) Dominik Jasiński (dreamsComeTrue) DualMatrix + Ellen Poe (ellenhp) Emmanuel Barroga (codecustard) Emmanuel Leblond (touilleMan) Eoin O'Neill (Eoin-ONeill-Yokai) @@ -75,7 +78,11 @@ name is available. est31 Fabian Mathews (supagu) Fabio Alessandrelli (Faless) + fabriceci Ferenc Arn (tagcup) + follower + foxydevloper + François Belair (Razoric480) Franklin Sobrinho (TheHX) Fredia Huya-Kouadio (m4gr3d) Geequlim @@ -87,8 +94,11 @@ name is available. Hanif Bin Ariffin (hbina) Haoyu Qiu (timothyqiu) Hein-Pieter van Braam-Stewart (hpvb) + Hendrik Brucker (Geometror) + hilfazer Hiroshi Ogawa (hi-ogawa) homer666 + hoontee Hubert Jarosz (Marqin) Hugo Locurcio (Calinou) Ian Bishop (ianb96) @@ -108,6 +118,7 @@ name is available. Joshua Grams (JoshuaGrams) Juan Linietsky (reduz) Julian Murgia (StraToN) + Julien Nguyen (Blackiris) Justo Delgado (mrcdk) Kelly Thomas (KellyThomas) kleonc @@ -139,6 +150,7 @@ name is available. Matthias Hölzl (hoelzl) Max Hilbrunner (mhilbrunner) merumelu + Meru Patel (Janglee123) Michael Alexsander (YeldhamDev) MichiRecRoom (LikeLakers2) mrezai @@ -146,6 +158,7 @@ name is available. Nathan Franke (nathanfranke) Nathan Lovato (NathanLovato) Nathan Warden (NathanWarden) + Nicholas Huelin (SirQuartz) Nils André-Chang (NilsIrl) Noah Beard (TwistedTwigleg) Nuno Donato (nunodonato) @@ -190,6 +203,7 @@ name is available. Timo (toger5) Tomasz Chabora (KoBeWi) Twarit Waikar (IronicallySerious) + Umang Kalra (theoway) Vinzenz Feenstra (vinzenz) 박한얼 (volzhs) V. Vamsi Krishna (vkbsb) diff --git a/COPYRIGHT.txt b/COPYRIGHT.txt index 5bd67960da..ef444721b2 100644 --- a/COPYRIGHT.txt +++ b/COPYRIGHT.txt @@ -453,7 +453,7 @@ License: Apache-2.0 Files: ./thirdparty/vulkan/vk_mem_alloc.h Comment: Vulkan Memory Allocator -Copyright: 2017-2019, Advanced Micro Devices, Inc. +Copyright: 2017-2021, Advanced Micro Devices, Inc. License: Expat Files: ./thirdparty/wslay/ @@ -46,8 +46,8 @@ generous deed immortalized in the next stable release of Godot Engine. anti666 blurp Christian Baune - Christoffer Sundbom Christopher Montesano + Christopher Shifflett Daniel Edwards Darrin Massena David Mydlarz @@ -59,14 +59,13 @@ generous deed immortalized in the next stable release of Godot Engine. GameDev.net Hein-Pieter van Braam Jasper Brooks - Javary Co. Jay Sistar Jeffery Chiu John G Gentzel Jonah Stich Justin Arnold Justo Delgado Baudí - Kamil Brzezinski + Kamil Brzezinskip Marcel Kräml Marek Belski Matthieu Huvé @@ -77,8 +76,8 @@ generous deed immortalized in the next stable release of Godot Engine. Ninja_5tyl3 Patrick Horn Patrick Schmidt - Péter Magyar Rami + Relintai Ronnie Cheng Slobodan Milnovic Stephan Lanfermann @@ -99,9 +98,9 @@ generous deed immortalized in the next stable release of Godot Engine. Asher Glick Barugon Carlo Cabanilla - Chris Goddard Daniel James David Gehrig + David Graham David Snopek Don B Ed Morley @@ -123,7 +122,6 @@ generous deed immortalized in the next stable release of Godot Engine. Mason Bially Matthew Hillier Michael - m kaersten Monster Vial Officine Pixel S.n.c. Petrus Prinsloo @@ -135,7 +133,6 @@ generous deed immortalized in the next stable release of Godot Engine. Sarksus Scott B Sean - segfault-god Sergey Sofox Stephen Molyneaux @@ -150,13 +147,13 @@ generous deed immortalized in the next stable release of Godot Engine. Zaven Muradyan Adam Nakonieczny - Adrian Adamiak Alexander J Maynard Alex de la Mare Alexey Dyadchenko Alex Khayrullin alice gambrell Andrew Cunningham + Andrew Farr Antanas Paskauskas Antoni Batchelli Arch Henderson III @@ -174,11 +171,13 @@ generous deed immortalized in the next stable release of Godot Engine. Christian Leth Jeppesen Cow Craig Ostrin + Craig Scarborough Craig Smith Cristopher CzechBlueBear D dan didenko + Daniel Hernández Alcojor Daniel Tebbutt Darrian Little Dennis Belfrage @@ -219,6 +218,7 @@ generous deed immortalized in the next stable release of Godot Engine. Joel Fivat Johnathan Kupferer John Knight + Jonathan Turner Jose Malheiro Jose Manuel Muñoz Perez Joseph Crane @@ -244,7 +244,6 @@ generous deed immortalized in the next stable release of Godot Engine. Liam Smyth LoparPanda Luca Vazzano - Luc-Frédéric Langis MadScientistCarl Marcus Dobler Marcus Richter @@ -263,6 +262,7 @@ generous deed immortalized in the next stable release of Godot Engine. Michael Policastro MightyPossum MikadoSC + Mike Barbee minz1 MuffinManKen nate etan @@ -327,10 +327,10 @@ generous deed immortalized in the next stable release of Godot Engine. Vincent Cloutier Vlad Ceru Opran VoidPointer - Vramexon Winston Wojciech Chojnacki xzibiting + Yifan Lai Yuancheng Zhang Zie Weaver Zoran Kukulj @@ -338,9 +338,8 @@ generous deed immortalized in the next stable release of Godot Engine. ## Silver donors 1D_Inc - Abraham Haskins + Aaron Oldenburg Actual_Dio - Adam Adam Brunnmeier Adam Carr Adam Long @@ -372,9 +371,9 @@ generous deed immortalized in the next stable release of Godot Engine. Allan Davis Allen Schade Aloehart - Amanda Anders Marstein Kruke Andre Stackhouse + Andrew Groot andrew james morris Andrew Thomas Ano Nim @@ -388,10 +387,10 @@ generous deed immortalized in the next stable release of Godot Engine. Ashley Claymore Astier Mickael Aubrey Falconer - Audun Borgersen aurelien condomines AzulCrescent Balázs Batári + Baptiste Le Bourhis Bartosz Bielecki Benedikt Benoit Jauvin-Girard @@ -405,7 +404,6 @@ generous deed immortalized in the next stable release of Godot Engine. Blair Allen Bobby CC Wong Borkzilla - Boyd Trolinger Bram brian Brian Klein @@ -419,7 +417,6 @@ generous deed immortalized in the next stable release of Godot Engine. Cameron Meyer Carlos Cejudo Carl van der Geest - Cas Brugman Casey Cassidy James Cédric Givord @@ -440,6 +437,7 @@ generous deed immortalized in the next stable release of Godot Engine. Corchari Craig Maloney Craig Post + C. R. Messen damucz Daniel Cheney Daniel Johnson @@ -452,6 +450,7 @@ generous deed immortalized in the next stable release of Godot Engine. deadwithbread Devin Carraway Diego Pereira + Dima Fedotov Dmitry Fisher Dmytro Korchynskyi Dominik Wetzel @@ -468,6 +467,7 @@ generous deed immortalized in the next stable release of Godot Engine. Egon Elbre Elgenzay Elias Nykrem + Emerson MX Ephemeral Eric Stokes Eric Walkingshaw @@ -479,11 +479,11 @@ generous deed immortalized in the next stable release of Godot Engine. fby Fekinox Felix Bohmann - Flaredown Forty Doubleu Francisco Garcia Florez Francois Holland Frank + FrostMarble Game Endeavor Gary Thomas George Marques @@ -494,7 +494,6 @@ generous deed immortalized in the next stable release of Godot Engine. Greyson Richey Grid Grominet - Guillaume Audirac Guillaume Pham Ngoc Guldoman Hal A @@ -516,7 +515,6 @@ generous deed immortalized in the next stable release of Godot Engine. Jako Danar James James A F Manley - James Gary James Guardino James Quincy James Thomas @@ -527,7 +525,6 @@ generous deed immortalized in the next stable release of Godot Engine. Jason Bolton Jason Malcolm-Herzmark Jason Uechi - Jeferson R. P. Belmiro Jeff Hungerford Jeff Messer Jeffrey Berube @@ -542,6 +539,7 @@ generous deed immortalized in the next stable release of Godot Engine. John Szevin Jonah Branch Jonas + Jonas Arndt Jonas Bernemann Jonas Rudlang Jonas Yamazaki @@ -571,6 +569,7 @@ generous deed immortalized in the next stable release of Godot Engine. Justin Spedding Justin W. Flory KaDokta + Karol Porzycki Keedong Park Keinan Powers Keith Bradner @@ -582,6 +581,7 @@ generous deed immortalized in the next stable release of Godot Engine. Kevin van Rooijen Kiri Jolly Kjetil Haugland + Kodera Software Kolandrious Konstantin Goncharov kormai @@ -594,12 +594,10 @@ generous deed immortalized in the next stable release of Godot Engine. Kyle Burnett Kyle Jacobs Kyuppin - Lady Grach Lasse le Dous Laurent CHEA Laurent Tréguier Laxman Pradhan - LE BOURHIS LEMMiNO Leonardo Dimano Lin Chear @@ -625,10 +623,10 @@ generous deed immortalized in the next stable release of Godot Engine. Matt Edwards Matthew Booe Matt Sylvia + Maverick Max Fiedler Maxime Blade Maxwell - MC Melissa Mears Merlyn Morgan-Graham mewin @@ -644,14 +642,12 @@ generous deed immortalized in the next stable release of Godot Engine. Mike Mike Birkhead Mike Copley - Mike Cunningham Mitchell J. Wagner MJacred ModularMind Molinghu Molly Jameson MoltenGears - Morwyn MrAZIE Mrjemandem Nathan Fish @@ -677,9 +673,10 @@ generous deed immortalized in the next stable release of Godot Engine. oceoh Okatima Oleg Reva - Olle Soprani Omar Delarosa + Orfist Oriol Muñoz Princep + oscar1000108 Oscar Domingo Pascal Patrick Brock @@ -706,7 +703,6 @@ generous deed immortalized in the next stable release of Godot Engine. Rammeow RAMupgrade Remi Rampin - Rémi Verschelde Reneator Riccardo Marini Richard Hayes @@ -716,21 +712,25 @@ generous deed immortalized in the next stable release of Godot Engine. Robert Farr (Larington) Rob Ruana Rodrigo Loli + Roger Smith Roland Rząsa Roman Tinkov Ronald Ho Hip (CrimsonZA) Ronan Ross Squires + Roy Scayged Ryan Groom Sam Caulfield Sam Edson Scott Longley Sean Lynch Sebastian Michailidis + segfault-god SeongWan Kim SeungJong k Shaidak Shane + Shane Sicienski Shane Spoor simdee Simon Jonas Larsen @@ -741,6 +741,7 @@ generous deed immortalized in the next stable release of Godot Engine. smbe19 smo1704 Solene Waked + Spencer Everhart Squirrel Stéphane Roussel Steve Cloete @@ -757,7 +758,6 @@ generous deed immortalized in the next stable release of Godot Engine. thomas Thomas Bechtold Thomas Detoy - Thomas Horwath Tianren Qin Till1805 Tim Drumheller @@ -769,6 +769,7 @@ generous deed immortalized in the next stable release of Godot Engine. Toadile Tobias Bradtke Tom Coxon + Tom Webster Torgeir Lilleskog Torsten Crass toupeira @@ -779,7 +780,6 @@ generous deed immortalized in the next stable release of Godot Engine. Troy Bonneau Tryggve Sollid Turgut Temucin - Tyler Compton Tyler Stafos UltyX Uther @@ -802,7 +802,9 @@ generous deed immortalized in the next stable release of Godot Engine. Yan Shi Yegor Smirnov Zak Stephens + Zher Huei Lee 蕭惟允 + 貴宏 小松 郝晨煜 ## Bronze donors diff --git a/core/input/input.cpp b/core/input/input.cpp index 2da50b7dff..8ba8b892ac 100644 --- a/core/input/input.cpp +++ b/core/input/input.cpp @@ -460,10 +460,6 @@ Vector3 Input::get_gyroscope() const { return gyroscope; } -void Input::parse_input_event(const Ref<InputEvent> &p_event) { - _parse_input_event_impl(p_event, false); -} - void Input::_parse_input_event_impl(const Ref<InputEvent> &p_event, bool p_is_emulated) { // Notes on mouse-touch emulation: // - Emulated mouse events are parsed, that is, re-routed to this method, so they make the same effects @@ -472,8 +468,6 @@ void Input::_parse_input_event_impl(const Ref<InputEvent> &p_event, bool p_is_em // - Emulated touch events are handed right to the main loop (i.e., the SceneTree) because they don't // require additional handling by this class. - _THREAD_SAFE_METHOD_ - Ref<InputEventKey> k = p_event; if (k.is_valid() && !k->is_echo() && k->get_keycode() != 0) { if (k->is_pressed()) { @@ -838,25 +832,37 @@ void Input::set_custom_mouse_cursor(const RES &p_cursor, CursorShape p_shape, co set_custom_mouse_cursor_func(p_cursor, p_shape, p_hotspot); } -void Input::accumulate_input_event(const Ref<InputEvent> &p_event) { +void Input::parse_input_event(const Ref<InputEvent> &p_event) { + _THREAD_SAFE_METHOD_ + ERR_FAIL_COND(p_event.is_null()); - if (!use_accumulated_input) { - parse_input_event(p_event); - return; + if (use_accumulated_input) { + if (buffered_events.is_empty() || !buffered_events.back()->get()->accumulate(p_event)) { + buffered_events.push_back(p_event); + } + } else if (use_input_buffering) { + buffered_events.push_back(p_event); + } else { + _parse_input_event_impl(p_event, false); } - if (!accumulated_events.is_empty() && accumulated_events.back()->get()->accumulate(p_event)) { - return; //event was accumulated, exit +} + +void Input::flush_buffered_events() { + _THREAD_SAFE_METHOD_ + + while (buffered_events.front()) { + _parse_input_event_impl(buffered_events.front()->get(), false); + buffered_events.pop_front(); } +} - accumulated_events.push_back(p_event); +bool Input::is_using_input_buffering() { + return use_input_buffering; } -void Input::flush_accumulated_events() { - while (accumulated_events.front()) { - parse_input_event(accumulated_events.front()->get()); - accumulated_events.pop_front(); - } +void Input::set_use_input_buffering(bool p_enable) { + use_input_buffering = p_enable; } void Input::set_use_accumulated_input(bool p_enable) { @@ -864,7 +870,7 @@ void Input::set_use_accumulated_input(bool p_enable) { } void Input::release_pressed_events() { - flush_accumulated_events(); // this is needed to release actions strengths + flush_buffered_events(); // this is needed to release actions strengths keys_pressed.clear(); joy_buttons_pressed.clear(); diff --git a/core/input/input.h b/core/input/input.h index ee991aa725..6819fc8eb0 100644 --- a/core/input/input.h +++ b/core/input/input.h @@ -111,6 +111,7 @@ private: bool emulate_touch_from_mouse = false; bool emulate_mouse_from_touch = false; + bool use_input_buffering = false; bool use_accumulated_input = false; int mouse_from_touch_index = -1; @@ -213,7 +214,7 @@ private: void _parse_input_event_impl(const Ref<InputEvent> &p_event, bool p_is_emulated); - List<Ref<InputEvent>> accumulated_events; + List<Ref<InputEvent>> buffered_events; friend class DisplayServer; @@ -323,8 +324,9 @@ public: String get_joy_guid(int p_device) const; void set_fallback_mapping(String p_guid); - void accumulate_input_event(const Ref<InputEvent> &p_event); - void flush_accumulated_events(); + void flush_buffered_events(); + bool is_using_input_buffering(); + void set_use_input_buffering(bool p_enable); void set_use_accumulated_input(bool p_enable); void release_pressed_events(); diff --git a/core/input/input_event.cpp b/core/input/input_event.cpp index 6e5c1a58ae..16bb92d94b 100644 --- a/core/input/input_event.cpp +++ b/core/input/input_event.cpp @@ -1210,6 +1210,22 @@ String InputEventScreenDrag::to_string() { return vformat("InputEventScreenDrag: index=%d, position=(%s), relative=(%s), speed=(%s)", index, String(get_position()), String(get_relative()), String(get_speed())); } +bool InputEventScreenDrag::accumulate(const Ref<InputEvent> &p_event) { + Ref<InputEventScreenDrag> drag = p_event; + if (drag.is_null()) + return false; + + if (get_index() != drag->get_index()) { + return false; + } + + set_position(drag->get_position()); + set_speed(drag->get_speed()); + relative += drag->get_relative(); + + return true; +} + void InputEventScreenDrag::_bind_methods() { ClassDB::bind_method(D_METHOD("set_index", "index"), &InputEventScreenDrag::set_index); ClassDB::bind_method(D_METHOD("get_index"), &InputEventScreenDrag::get_index); diff --git a/core/input/input_event.h b/core/input/input_event.h index 57b6091123..517d63eb40 100644 --- a/core/input/input_event.h +++ b/core/input/input_event.h @@ -410,6 +410,8 @@ public: virtual String as_text() const override; virtual String to_string() override; + virtual bool accumulate(const Ref<InputEvent> &p_event) override; + InputEventScreenDrag() {} }; diff --git a/doc/classes/ProjectSettings.xml b/doc/classes/ProjectSettings.xml index f769ace836..0d1fa0e70f 100644 --- a/doc/classes/ProjectSettings.xml +++ b/doc/classes/ProjectSettings.xml @@ -717,6 +717,12 @@ Default [InputEventAction] to move up in the UI. [b]Note:[/b] Default [code]ui_*[/code] actions cannot be removed as they are necessary for the internal logic of several [Control]s. The events assigned to the action can however be modified. </member> + <member name="input_devices/buffering/agile_event_flushing" type="bool" setter="" getter="" default="false"> + If [code]true[/code], key/touch/joystick events will be flushed just before every idle and physics frame. + If [code]false[/code], such events will be flushed only once per process frame, between iterations of the engine. + Enabling this can greatly improve the responsiveness to input, specially in devices that need to run multiple physics frames per visible (process) frame, because they can't run at the target frame rate. + [b]Note:[/b] Currently implemented only on Android. + </member> <member name="input_devices/pen_tablet/driver" type="String" setter="" getter=""> Specifies the tablet driver to use. If left empty, the default driver will be used. </member> diff --git a/doc/classes/Viewport.xml b/doc/classes/Viewport.xml index 7b5cb2c459..4a62d3ec7b 100644 --- a/doc/classes/Viewport.xml +++ b/doc/classes/Viewport.xml @@ -296,19 +296,19 @@ Represents the size of the [enum ShadowAtlasQuadrantSubdiv] enum. </constant> <constant name="MSAA_DISABLED" value="0" enum="MSAA"> - Multisample antialiasing mode disabled. This is the default value, and also the fastest setting. + Multisample antialiasing mode disabled. This is the default value, and is also the fastest setting. </constant> <constant name="MSAA_2X" value="1" enum="MSAA"> - Use 2x Multisample Antialiasing. + Use 2× Multisample Antialiasing. This has a moderate performance cost. It helps reduce aliasing noticeably, but 4× MSAA still looks substantially better. </constant> <constant name="MSAA_4X" value="2" enum="MSAA"> - Use 4x Multisample Antialiasing. + Use 4× Multisample Antialiasing. This has a significant performance cost, and is generally a good compromise between performance and quality. </constant> <constant name="MSAA_8X" value="3" enum="MSAA"> - Use 8x Multisample Antialiasing. Likely unsupported on low-end and older hardware. + Use 8× Multisample Antialiasing. This has a very high performance cost. The difference between 4× and 8× MSAA may not always be visible in real gameplay conditions. Likely unsupported on low-end and older hardware. </constant> <constant name="MSAA_16X" value="4" enum="MSAA"> - Use 16x Multisample Antialiasing. Likely unsupported on medium and low-end hardware. + Use 16× Multisample Antialiasing. This has a very high performance cost. The difference between 8× and 16× MSAA may not always be visible in real gameplay conditions. Likely unsupported on medium and low-end hardware. </constant> <constant name="MSAA_MAX" value="5" enum="MSAA"> Represents the size of the [enum MSAA] enum. diff --git a/drivers/vulkan/SCsub b/drivers/vulkan/SCsub index 3e0f5788c3..8fe75367a8 100644 --- a/drivers/vulkan/SCsub +++ b/drivers/vulkan/SCsub @@ -6,24 +6,13 @@ thirdparty_obj = [] thirdparty_dir = "#thirdparty/vulkan" thirdparty_volk_dir = "#thirdparty/volk" +# Use bundled Vulkan headers +env.Prepend(CPPPATH=[thirdparty_dir, thirdparty_dir + "/include"]) + if env["use_volk"]: env.AppendUnique(CPPDEFINES=["USE_VOLK"]) env.Prepend(CPPPATH=[thirdparty_volk_dir]) -if env["platform"] == "android" and not env["use_volk"]: - # Use NDK Vulkan headers - ndk_vulkan_dir = env["ANDROID_NDK_ROOT"] + "/sources/third_party/vulkan/src" - thirdparty_includes = [ - ndk_vulkan_dir, - ndk_vulkan_dir + "/include", - ndk_vulkan_dir + "/layers", - ndk_vulkan_dir + "/layers/generated", - ] - env.Prepend(CPPPATH=thirdparty_includes) -else: - # Use bundled Vulkan headers - env.Prepend(CPPPATH=[thirdparty_dir, thirdparty_dir + "/include"]) - if env["platform"] == "android": env.AppendUnique(CPPDEFINES=["VK_USE_PLATFORM_ANDROID_KHR"]) elif env["platform"] == "iphone": @@ -47,6 +36,10 @@ if env["use_volk"]: thirdparty_sources_volk = [thirdparty_volk_dir + "/volk.c"] env_thirdparty_volk.add_source_files(thirdparty_obj, thirdparty_sources_volk) +elif env["platform"] == "android": + # Our current NDK version only provides old Vulkan headers, + # so we have to limit VMA. + env_thirdparty_vma.AppendUnique(CPPDEFINES=["VMA_VULKAN_VERSION=1000000"]) env_thirdparty_vma.add_source_files(thirdparty_obj, thirdparty_sources_vma) diff --git a/drivers/vulkan/rendering_device_vulkan.cpp b/drivers/vulkan/rendering_device_vulkan.cpp index f6677e2da4..dbe932f74c 100644 --- a/drivers/vulkan/rendering_device_vulkan.cpp +++ b/drivers/vulkan/rendering_device_vulkan.cpp @@ -8794,6 +8794,7 @@ void RenderingDeviceVulkan::initialize(VulkanContext *p_context, bool p_local_de memset(&allocatorInfo, 0, sizeof(VmaAllocatorCreateInfo)); allocatorInfo.physicalDevice = p_context->get_physical_device(); allocatorInfo.device = device; + allocatorInfo.instance = p_context->get_instance(); vmaCreateAllocator(&allocatorInfo, &allocator); } diff --git a/drivers/vulkan/vulkan_context.h b/drivers/vulkan/vulkan_context.h index 1690b853e3..fe09d4c497 100644 --- a/drivers/vulkan/vulkan_context.h +++ b/drivers/vulkan/vulkan_context.h @@ -233,10 +233,6 @@ protected: Error _get_preferred_validation_layers(uint32_t *count, const char *const **names); - VkInstance _get_instance() { - return inst; - } - public: uint32_t get_vulkan_major() const { return vulkan_major; }; uint32_t get_vulkan_minor() const { return vulkan_minor; }; @@ -245,6 +241,7 @@ public: VkDevice get_device(); VkPhysicalDevice get_physical_device(); + VkInstance get_instance() { return inst; } int get_swapchain_image_count() const; uint32_t get_graphics_queue() const; diff --git a/editor/editor_node.cpp b/editor/editor_node.cpp index 30edbd1e87..74ee9099f2 100644 --- a/editor/editor_node.cpp +++ b/editor/editor_node.cpp @@ -613,8 +613,10 @@ void EditorNode::_notification(int p_what) { _editor_select(EDITOR_3D); } - // Save the project after opening to mark it as last modified. - ProjectSettings::get_singleton()->save(); + // Save the project after opening to mark it as last modified, except in headless mode. + if (DisplayServer::get_singleton()->window_can_draw()) { + ProjectSettings::get_singleton()->save(); + } /* DO NOT LOAD SCENES HERE, WAIT FOR FILE SCANNING AND REIMPORT TO COMPLETE */ } break; diff --git a/editor/import/resource_importer_layered_texture.cpp b/editor/import/resource_importer_layered_texture.cpp index 2ac8b8bd7d..d5bb21443c 100644 --- a/editor/import/resource_importer_layered_texture.cpp +++ b/editor/import/resource_importer_layered_texture.cpp @@ -341,10 +341,7 @@ Error ResourceImporterLayeredTexture::import(const String &p_source_file, const if (compress_mode == COMPRESS_VRAM_COMPRESSED) { mipmaps = true; - } - //optimize - if (compress_mode == COMPRESS_VRAM_COMPRESSED) { //if using video ram, optimize if (channel_pack == 0) { //remove alpha if not needed, so compression is more efficient diff --git a/main/main.cpp b/main/main.cpp index 7351dafa22..6764332f16 100644 --- a/main/main.cpp +++ b/main/main.cpp @@ -1735,6 +1735,8 @@ Error Main::setup2(Thread::ID p_main_tid_override) { Input *id = Input::get_singleton(); if (id) { + agile_input_event_flushing = GLOBAL_DEF("input_devices/buffering/agile_event_flushing", false); + if (bool(GLOBAL_DEF("input_devices/pointing/emulate_touch_from_mouse", false)) && !(editor || project_manager)) { bool found_touchscreen = false; @@ -2442,6 +2444,7 @@ uint32_t Main::frames = 0; uint32_t Main::frame = 0; bool Main::force_redraw_requested = false; int Main::iterating = 0; +bool Main::agile_input_event_flushing = false; bool Main::is_iterating() { return iterating > 0; @@ -2491,9 +2494,13 @@ bool Main::iteration() { bool exit = false; - Engine::get_singleton()->_in_physics = true; - for (int iters = 0; iters < advance.physics_steps; ++iters) { + if (Input::get_singleton()->is_using_input_buffering() && agile_input_event_flushing) { + Input::get_singleton()->flush_buffered_events(); + } + + Engine::get_singleton()->_in_physics = true; + uint64_t physics_begin = OS::get_singleton()->get_ticks_usec(); PhysicsServer3D::get_singleton()->sync(); @@ -2522,9 +2529,13 @@ bool Main::iteration() { physics_process_ticks = MAX(physics_process_ticks, OS::get_singleton()->get_ticks_usec() - physics_begin); // keep the largest one for reference physics_process_max = MAX(OS::get_singleton()->get_ticks_usec() - physics_begin, physics_process_max); Engine::get_singleton()->_physics_frames++; + + Engine::get_singleton()->_in_physics = false; } - Engine::get_singleton()->_in_physics = false; + if (Input::get_singleton()->is_using_input_buffering() && agile_input_event_flushing) { + Input::get_singleton()->flush_buffered_events(); + } uint64_t process_begin = OS::get_singleton()->get_ticks_usec(); @@ -2587,6 +2598,11 @@ bool Main::iteration() { iterating--; + // Needed for OSs using input buffering regardless accumulation (like Android) + if (Input::get_singleton()->is_using_input_buffering() && !agile_input_event_flushing) { + Input::get_singleton()->flush_buffered_events(); + } + if (fixed_fps != -1) { return exit; } diff --git a/main/main.h b/main/main.h index 84077137ba..4911ff42b4 100644 --- a/main/main.h +++ b/main/main.h @@ -42,6 +42,7 @@ class Main { static uint32_t frame; static bool force_redraw_requested; static int iterating; + static bool agile_input_event_flushing; public: static bool is_project_manager(); diff --git a/modules/visual_script/visual_script_editor.cpp b/modules/visual_script/visual_script_editor.cpp index c37fbb6bb3..f3b6d74b7d 100644 --- a/modules/visual_script/visual_script_editor.cpp +++ b/modules/visual_script/visual_script_editor.cpp @@ -2899,9 +2899,8 @@ void VisualScriptEditor::_graph_connected(const String &p_from, int p_from_slot, if (!converted) { undo_redo->add_do_method(script.ptr(), "data_connect", p_from.to_int(), from_port, p_to.to_int(), to_port); undo_redo->add_undo_method(script.ptr(), "data_disconnect", p_from.to_int(), from_port, p_to.to_int(), to_port); - } - // Update nodes in graph - if (!converted) { + + // Update nodes in graph undo_redo->add_do_method(this, "_update_graph", p_from.to_int()); undo_redo->add_do_method(this, "_update_graph", p_to.to_int()); undo_redo->add_undo_method(this, "_update_graph", p_from.to_int()); diff --git a/platform/android/SCsub b/platform/android/SCsub index 56fbd2f7e4..ecc72019e5 100644 --- a/platform/android/SCsub +++ b/platform/android/SCsub @@ -4,6 +4,7 @@ Import("env") android_files = [ "os_android.cpp", + "android_input_handler.cpp", "file_access_android.cpp", "audio_driver_opensl.cpp", "dir_access_jandroid.cpp", diff --git a/platform/android/android_input_handler.cpp b/platform/android/android_input_handler.cpp new file mode 100644 index 0000000000..b9004c4989 --- /dev/null +++ b/platform/android/android_input_handler.cpp @@ -0,0 +1,395 @@ +/*************************************************************************/ +/* android_input_handler.cpp */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#include "android_input_handler.h" + +#include "android_keys_utils.h" +#include "display_server_android.h" + +void AndroidInputHandler::process_joy_event(AndroidInputHandler::JoypadEvent p_event) { + switch (p_event.type) { + case JOY_EVENT_BUTTON: + Input::get_singleton()->joy_button(p_event.device, (JoyButton)p_event.index, p_event.pressed); + break; + case JOY_EVENT_AXIS: + Input::JoyAxisValue value; + value.min = -1; + value.value = p_event.value; + Input::get_singleton()->joy_axis(p_event.device, (JoyAxis)p_event.index, value); + break; + case JOY_EVENT_HAT: + Input::get_singleton()->joy_hat(p_event.device, (HatMask)p_event.hat); + break; + default: + return; + } +} + +void AndroidInputHandler::_set_key_modifier_state(Ref<InputEventWithModifiers> ev) { + ev->set_shift_pressed(shift_mem); + ev->set_alt_pressed(alt_mem); + ev->set_meta_pressed(meta_mem); + ev->set_ctrl_pressed(control_mem); +} + +void AndroidInputHandler::process_key_event(int p_keycode, int p_scancode, int p_unicode_char, bool p_pressed) { + static char32_t prev_wc = 0; + char32_t unicode = p_unicode_char; + if ((p_unicode_char & 0xfffffc00) == 0xd800) { + if (prev_wc != 0) { + ERR_PRINT("invalid utf16 surrogate input"); + } + prev_wc = unicode; + return; // Skip surrogate. + } else if ((unicode & 0xfffffc00) == 0xdc00) { + if (prev_wc == 0) { + ERR_PRINT("invalid utf16 surrogate input"); + return; // Skip invalid surrogate. + } + unicode = (prev_wc << 10UL) + unicode - ((0xd800 << 10UL) + 0xdc00 - 0x10000); + prev_wc = 0; + } else { + prev_wc = 0; + } + + Ref<InputEventKey> ev; + ev.instantiate(); + int val = unicode; + int keycode = android_get_keysym(p_keycode); + int phy_keycode = android_get_keysym(p_scancode); + + if (keycode == KEY_SHIFT) { + shift_mem = p_pressed; + } + if (keycode == KEY_ALT) { + alt_mem = p_pressed; + } + if (keycode == KEY_CTRL) { + control_mem = p_pressed; + } + if (keycode == KEY_META) { + meta_mem = p_pressed; + } + + ev->set_keycode((Key)keycode); + ev->set_physical_keycode((Key)phy_keycode); + ev->set_unicode(val); + ev->set_pressed(p_pressed); + + _set_key_modifier_state(ev); + + if (val == '\n') { + ev->set_keycode(KEY_ENTER); + } else if (val == 61448) { + ev->set_keycode(KEY_BACKSPACE); + ev->set_unicode(KEY_BACKSPACE); + } else if (val == 61453) { + ev->set_keycode(KEY_ENTER); + ev->set_unicode(KEY_ENTER); + } else if (p_keycode == 4) { + if (DisplayServerAndroid *dsa = Object::cast_to<DisplayServerAndroid>(DisplayServer::get_singleton())) { + dsa->send_window_event(DisplayServer::WINDOW_EVENT_GO_BACK_REQUEST, true); + } + } + + Input::get_singleton()->parse_input_event(ev); +} + +void AndroidInputHandler::process_touch(int p_event, int p_pointer, const Vector<AndroidInputHandler::TouchPos> &p_points) { + switch (p_event) { + case AMOTION_EVENT_ACTION_DOWN: { //gesture begin + if (touch.size()) { + //end all if exist + for (int i = 0; i < touch.size(); i++) { + Ref<InputEventScreenTouch> ev; + ev.instantiate(); + ev->set_index(touch[i].id); + ev->set_pressed(false); + ev->set_position(touch[i].pos); + Input::get_singleton()->parse_input_event(ev); + } + } + + touch.resize(p_points.size()); + for (int i = 0; i < p_points.size(); i++) { + touch.write[i].id = p_points[i].id; + touch.write[i].pos = p_points[i].pos; + } + + //send touch + for (int i = 0; i < touch.size(); i++) { + Ref<InputEventScreenTouch> ev; + ev.instantiate(); + ev->set_index(touch[i].id); + ev->set_pressed(true); + ev->set_position(touch[i].pos); + Input::get_singleton()->parse_input_event(ev); + } + + } break; + case AMOTION_EVENT_ACTION_MOVE: { //motion + ERR_FAIL_COND(touch.size() != p_points.size()); + + for (int i = 0; i < touch.size(); i++) { + int idx = -1; + for (int j = 0; j < p_points.size(); j++) { + if (touch[i].id == p_points[j].id) { + idx = j; + break; + } + } + + ERR_CONTINUE(idx == -1); + + if (touch[i].pos == p_points[idx].pos) + continue; //no move unncesearily + + Ref<InputEventScreenDrag> ev; + ev.instantiate(); + ev->set_index(touch[i].id); + ev->set_position(p_points[idx].pos); + ev->set_relative(p_points[idx].pos - touch[i].pos); + Input::get_singleton()->parse_input_event(ev); + touch.write[i].pos = p_points[idx].pos; + } + + } break; + case AMOTION_EVENT_ACTION_CANCEL: + case AMOTION_EVENT_ACTION_UP: { //release + if (touch.size()) { + //end all if exist + for (int i = 0; i < touch.size(); i++) { + Ref<InputEventScreenTouch> ev; + ev.instantiate(); + ev->set_index(touch[i].id); + ev->set_pressed(false); + ev->set_position(touch[i].pos); + Input::get_singleton()->parse_input_event(ev); + } + touch.clear(); + } + } break; + case AMOTION_EVENT_ACTION_POINTER_DOWN: { // add touch + for (int i = 0; i < p_points.size(); i++) { + if (p_points[i].id == p_pointer) { + TouchPos tp = p_points[i]; + touch.push_back(tp); + + Ref<InputEventScreenTouch> ev; + ev.instantiate(); + + ev->set_index(tp.id); + ev->set_pressed(true); + ev->set_position(tp.pos); + Input::get_singleton()->parse_input_event(ev); + + break; + } + } + } break; + case AMOTION_EVENT_ACTION_POINTER_UP: { // remove touch + for (int i = 0; i < touch.size(); i++) { + if (touch[i].id == p_pointer) { + Ref<InputEventScreenTouch> ev; + ev.instantiate(); + ev->set_index(touch[i].id); + ev->set_pressed(false); + ev->set_position(touch[i].pos); + Input::get_singleton()->parse_input_event(ev); + touch.remove(i); + + break; + } + } + } break; + } +} + +void AndroidInputHandler::process_hover(int p_type, Point2 p_pos) { + // https://developer.android.com/reference/android/view/MotionEvent.html#ACTION_HOVER_ENTER + switch (p_type) { + case AMOTION_EVENT_ACTION_HOVER_MOVE: // hover move + case AMOTION_EVENT_ACTION_HOVER_ENTER: // hover enter + case AMOTION_EVENT_ACTION_HOVER_EXIT: { // hover exit + Ref<InputEventMouseMotion> ev; + ev.instantiate(); + _set_key_modifier_state(ev); + ev->set_position(p_pos); + ev->set_global_position(p_pos); + ev->set_relative(p_pos - hover_prev_pos); + Input::get_singleton()->parse_input_event(ev); + hover_prev_pos = p_pos; + } break; + } +} + +void AndroidInputHandler::process_mouse_event(int input_device, int event_action, int event_android_buttons_mask, Point2 event_pos, float event_vertical_factor, float event_horizontal_factor) { + MouseButton event_buttons_mask = _android_button_mask_to_godot_button_mask(event_android_buttons_mask); + switch (event_action) { + case AMOTION_EVENT_ACTION_BUTTON_PRESS: + case AMOTION_EVENT_ACTION_BUTTON_RELEASE: { + Ref<InputEventMouseButton> ev; + ev.instantiate(); + _set_key_modifier_state(ev); + if ((input_device & AINPUT_SOURCE_MOUSE) == AINPUT_SOURCE_MOUSE) { + ev->set_position(event_pos); + ev->set_global_position(event_pos); + } else { + ev->set_position(hover_prev_pos); + ev->set_global_position(hover_prev_pos); + } + ev->set_pressed(event_action == AMOTION_EVENT_ACTION_BUTTON_PRESS); + MouseButton changed_button_mask = MouseButton(buttons_state ^ event_buttons_mask); + + buttons_state = event_buttons_mask; + + ev->set_button_index(_button_index_from_mask(changed_button_mask)); + ev->set_button_mask(event_buttons_mask); + Input::get_singleton()->parse_input_event(ev); + } break; + + case AMOTION_EVENT_ACTION_MOVE: { + Ref<InputEventMouseMotion> ev; + ev.instantiate(); + _set_key_modifier_state(ev); + if ((input_device & AINPUT_SOURCE_MOUSE) == AINPUT_SOURCE_MOUSE) { + ev->set_position(event_pos); + ev->set_global_position(event_pos); + ev->set_relative(event_pos - hover_prev_pos); + hover_prev_pos = event_pos; + } else { + ev->set_position(hover_prev_pos); + ev->set_global_position(hover_prev_pos); + ev->set_relative(event_pos); + } + ev->set_button_mask(event_buttons_mask); + Input::get_singleton()->parse_input_event(ev); + } break; + case AMOTION_EVENT_ACTION_SCROLL: { + Ref<InputEventMouseButton> ev; + ev.instantiate(); + if ((input_device & AINPUT_SOURCE_MOUSE) == AINPUT_SOURCE_MOUSE) { + ev->set_position(event_pos); + ev->set_global_position(event_pos); + } else { + ev->set_position(hover_prev_pos); + ev->set_global_position(hover_prev_pos); + } + ev->set_pressed(true); + buttons_state = event_buttons_mask; + if (event_vertical_factor > 0) { + _wheel_button_click(event_buttons_mask, ev, MOUSE_BUTTON_WHEEL_UP, event_vertical_factor); + } else if (event_vertical_factor < 0) { + _wheel_button_click(event_buttons_mask, ev, MOUSE_BUTTON_WHEEL_DOWN, -event_vertical_factor); + } + + if (event_horizontal_factor > 0) { + _wheel_button_click(event_buttons_mask, ev, MOUSE_BUTTON_WHEEL_RIGHT, event_horizontal_factor); + } else if (event_horizontal_factor < 0) { + _wheel_button_click(event_buttons_mask, ev, MOUSE_BUTTON_WHEEL_LEFT, -event_horizontal_factor); + } + } break; + } +} + +void AndroidInputHandler::_wheel_button_click(MouseButton event_buttons_mask, const Ref<InputEventMouseButton> &ev, MouseButton wheel_button, float factor) { + Ref<InputEventMouseButton> evd = ev->duplicate(); + _set_key_modifier_state(evd); + evd->set_button_index(wheel_button); + evd->set_button_mask(MouseButton(event_buttons_mask ^ (1 << (wheel_button - 1)))); + evd->set_factor(factor); + Input::get_singleton()->parse_input_event(evd); + Ref<InputEventMouseButton> evdd = evd->duplicate(); + evdd->set_pressed(false); + evdd->set_button_mask(event_buttons_mask); + Input::get_singleton()->parse_input_event(evdd); +} + +void AndroidInputHandler::process_double_tap(int event_android_button_mask, Point2 p_pos) { + MouseButton event_button_mask = _android_button_mask_to_godot_button_mask(event_android_button_mask); + Ref<InputEventMouseButton> ev; + ev.instantiate(); + _set_key_modifier_state(ev); + ev->set_position(p_pos); + ev->set_global_position(p_pos); + ev->set_pressed(event_button_mask != 0); + ev->set_button_index(_button_index_from_mask(event_button_mask)); + ev->set_button_mask(event_button_mask); + ev->set_double_click(true); + Input::get_singleton()->parse_input_event(ev); +} + +MouseButton AndroidInputHandler::_button_index_from_mask(MouseButton button_mask) { + switch (button_mask) { + case MOUSE_BUTTON_MASK_LEFT: + return MOUSE_BUTTON_LEFT; + case MOUSE_BUTTON_MASK_RIGHT: + return MOUSE_BUTTON_RIGHT; + case MOUSE_BUTTON_MASK_MIDDLE: + return MOUSE_BUTTON_MIDDLE; + case MOUSE_BUTTON_MASK_XBUTTON1: + return MOUSE_BUTTON_XBUTTON1; + case MOUSE_BUTTON_MASK_XBUTTON2: + return MOUSE_BUTTON_XBUTTON2; + default: + return MOUSE_BUTTON_NONE; + } +} + +MouseButton AndroidInputHandler::_android_button_mask_to_godot_button_mask(int android_button_mask) { + MouseButton godot_button_mask = MOUSE_BUTTON_NONE; + if (android_button_mask & AMOTION_EVENT_BUTTON_PRIMARY) { + godot_button_mask |= MOUSE_BUTTON_MASK_LEFT; + } + if (android_button_mask & AMOTION_EVENT_BUTTON_SECONDARY) { + godot_button_mask |= MOUSE_BUTTON_MASK_RIGHT; + } + if (android_button_mask & AMOTION_EVENT_BUTTON_TERTIARY) { + godot_button_mask |= MOUSE_BUTTON_MASK_MIDDLE; + } + if (android_button_mask & AMOTION_EVENT_BUTTON_BACK) { + godot_button_mask |= MOUSE_BUTTON_MASK_XBUTTON1; + } + if (android_button_mask & AMOTION_EVENT_BUTTON_SECONDARY) { + godot_button_mask |= MOUSE_BUTTON_MASK_XBUTTON2; + } + + return godot_button_mask; +} + +void AndroidInputHandler::process_scroll(Point2 p_pos) { + Ref<InputEventPanGesture> ev; + ev.instantiate(); + _set_key_modifier_state(ev); + ev->set_position(p_pos); + ev->set_delta(p_pos - scroll_prev_pos); + Input::get_singleton()->parse_input_event(ev); + scroll_prev_pos = p_pos; +} diff --git a/platform/android/android_input_handler.h b/platform/android/android_input_handler.h new file mode 100644 index 0000000000..2918ca300b --- /dev/null +++ b/platform/android/android_input_handler.h @@ -0,0 +1,91 @@ +/*************************************************************************/ +/* android_input_handler.h */ +/*************************************************************************/ +/* This file is part of: */ +/* GODOT ENGINE */ +/* https://godotengine.org */ +/*************************************************************************/ +/* Copyright (c) 2007-2021 Juan Linietsky, Ariel Manzur. */ +/* Copyright (c) 2014-2021 Godot Engine contributors (cf. AUTHORS.md). */ +/* */ +/* Permission is hereby granted, free of charge, to any person obtaining */ +/* a copy of this software and associated documentation files (the */ +/* "Software"), to deal in the Software without restriction, including */ +/* without limitation the rights to use, copy, modify, merge, publish, */ +/* distribute, sublicense, and/or sell copies of the Software, and to */ +/* permit persons to whom the Software is furnished to do so, subject to */ +/* the following conditions: */ +/* */ +/* The above copyright notice and this permission notice shall be */ +/* included in all copies or substantial portions of the Software. */ +/* */ +/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, */ +/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF */ +/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/ +/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY */ +/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, */ +/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE */ +/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +/*************************************************************************/ + +#ifndef ANDROID_INPUT_HANDLER_H +#define ANDROID_INPUT_HANDLER_H + +#include "core/input/input.h" + +// This class encapsulates all the handling of input events that come from the Android UI thread. +// Remarks: +// - It's not thread-safe by itself, so its functions must only be called on a single thread, which is the Android UI thread. +// - Its functions must only call thread-safe methods. +class AndroidInputHandler { +public: + struct TouchPos { + int id = 0; + Point2 pos; + }; + + enum { + JOY_EVENT_BUTTON = 0, + JOY_EVENT_AXIS = 1, + JOY_EVENT_HAT = 2 + }; + + struct JoypadEvent { + int device = 0; + int type = 0; + int index = 0; + bool pressed = false; + float value = 0; + int hat = 0; + }; + +private: + bool alt_mem = false; + bool shift_mem = false; + bool control_mem = false; + bool meta_mem = false; + + MouseButton buttons_state = MOUSE_BUTTON_NONE; + + Vector<TouchPos> touch; + Point2 hover_prev_pos; // needed to calculate the relative position on hover events + Point2 scroll_prev_pos; // needed to calculate the relative position on scroll events + + void _set_key_modifier_state(Ref<InputEventWithModifiers> ev); + + static MouseButton _button_index_from_mask(MouseButton button_mask); + static MouseButton _android_button_mask_to_godot_button_mask(int android_button_mask); + + void _wheel_button_click(MouseButton event_buttons_mask, const Ref<InputEventMouseButton> &ev, MouseButton wheel_button, float factor); + +public: + void process_touch(int p_event, int p_pointer, const Vector<TouchPos> &p_points); + void process_hover(int p_type, Point2 p_pos); + void process_mouse_event(int input_device, int event_action, int event_android_buttons_mask, Point2 event_pos, float event_vertical_factor = 0, float event_horizontal_factor = 0); + void process_double_tap(int event_android_button_mask, Point2 p_pos); + void process_scroll(Point2 p_pos); + void process_joy_event(JoypadEvent p_event); + void process_key_event(int p_keycode, int p_scancode, int p_unicode_char, bool p_pressed); +}; + +#endif diff --git a/platform/android/detect.py b/platform/android/detect.py index 9be6f33b93..7a993e9ca6 100644 --- a/platform/android/detect.py +++ b/platform/android/detect.py @@ -54,7 +54,6 @@ def get_android_ndk_root(): def get_flags(): return [ ("tools", False), - ("use_volk", False), ] diff --git a/platform/android/display_server_android.cpp b/platform/android/display_server_android.cpp index d200d024c5..720752d28f 100644 --- a/platform/android/display_server_android.cpp +++ b/platform/android/display_server_android.cpp @@ -30,7 +30,6 @@ #include "display_server_android.h" -#include "android_keys_utils.h" #include "core/config/project_settings.h" #include "java_godot_io_wrapper.h" #include "java_godot_wrapper.h" @@ -203,17 +202,21 @@ void DisplayServerAndroid::window_set_drop_files_callback(const Callable &p_call // Not supported on Android. } -void DisplayServerAndroid::_window_callback(const Callable &p_callable, const Variant &p_arg) const { +void DisplayServerAndroid::_window_callback(const Callable &p_callable, const Variant &p_arg, bool p_deferred) const { if (!p_callable.is_null()) { const Variant *argp = &p_arg; Variant ret; Callable::CallError ce; - p_callable.call((const Variant **)&argp, 1, ret, ce); + if (p_deferred) { + p_callable.call((const Variant **)&argp, 1, ret, ce); + } else { + p_callable.call_deferred((const Variant **)&argp, 1); + } } } -void DisplayServerAndroid::send_window_event(DisplayServer::WindowEvent p_event) const { - _window_callback(window_event_callback, int(p_event)); +void DisplayServerAndroid::send_window_event(DisplayServer::WindowEvent p_event, bool p_deferred) const { + _window_callback(window_event_callback, int(p_event), p_deferred); } void DisplayServerAndroid::send_input_event(const Ref<InputEvent> &p_event) const { @@ -335,7 +338,7 @@ bool DisplayServerAndroid::can_any_window_draw() const { } void DisplayServerAndroid::process_events() { - Input::get_singleton()->flush_accumulated_events(); + Input::get_singleton()->flush_buffered_events(); } Vector<String> DisplayServerAndroid::get_rendering_drivers_func() { @@ -454,6 +457,7 @@ DisplayServerAndroid::DisplayServerAndroid(const String &p_rendering_driver, Dis #endif Input::get_singleton()->set_event_dispatch_function(_dispatch_input_events); + Input::get_singleton()->set_use_input_buffering(true); // Needed because events will come directly from the UI thread r_error = OK; } @@ -473,344 +477,6 @@ DisplayServerAndroid::~DisplayServerAndroid() { #endif } -void DisplayServerAndroid::process_joy_event(DisplayServerAndroid::JoypadEvent p_event) { - switch (p_event.type) { - case JOY_EVENT_BUTTON: - Input::get_singleton()->joy_button(p_event.device, (JoyButton)p_event.index, p_event.pressed); - break; - case JOY_EVENT_AXIS: - Input::JoyAxisValue value; - value.min = -1; - value.value = p_event.value; - Input::get_singleton()->joy_axis(p_event.device, (JoyAxis)p_event.index, value); - break; - case JOY_EVENT_HAT: - Input::get_singleton()->joy_hat(p_event.device, (HatMask)p_event.hat); - break; - default: - return; - } -} - -void DisplayServerAndroid::_set_key_modifier_state(Ref<InputEventWithModifiers> ev) { - ev->set_shift_pressed(shift_mem); - ev->set_alt_pressed(alt_mem); - ev->set_meta_pressed(meta_mem); - ev->set_ctrl_pressed(control_mem); -} - -void DisplayServerAndroid::process_key_event(int p_keycode, int p_scancode, int p_unicode_char, bool p_pressed) { - static char32_t prev_wc = 0; - char32_t unicode = p_unicode_char; - if ((p_unicode_char & 0xfffffc00) == 0xd800) { - if (prev_wc != 0) { - ERR_PRINT("invalid utf16 surrogate input"); - } - prev_wc = unicode; - return; // Skip surrogate. - } else if ((unicode & 0xfffffc00) == 0xdc00) { - if (prev_wc == 0) { - ERR_PRINT("invalid utf16 surrogate input"); - return; // Skip invalid surrogate. - } - unicode = (prev_wc << 10UL) + unicode - ((0xd800 << 10UL) + 0xdc00 - 0x10000); - prev_wc = 0; - } else { - prev_wc = 0; - } - - Ref<InputEventKey> ev; - ev.instantiate(); - int val = unicode; - int keycode = android_get_keysym(p_keycode); - int phy_keycode = android_get_keysym(p_scancode); - - if (keycode == KEY_SHIFT) { - shift_mem = p_pressed; - } - if (keycode == KEY_ALT) { - alt_mem = p_pressed; - } - if (keycode == KEY_CTRL) { - control_mem = p_pressed; - } - if (keycode == KEY_META) { - meta_mem = p_pressed; - } - - ev->set_keycode((Key)keycode); - ev->set_physical_keycode((Key)phy_keycode); - ev->set_unicode(val); - ev->set_pressed(p_pressed); - - _set_key_modifier_state(ev); - - if (val == '\n') { - ev->set_keycode(KEY_ENTER); - } else if (val == 61448) { - ev->set_keycode(KEY_BACKSPACE); - ev->set_unicode(KEY_BACKSPACE); - } else if (val == 61453) { - ev->set_keycode(KEY_ENTER); - ev->set_unicode(KEY_ENTER); - } else if (p_keycode == 4) { - OS_Android::get_singleton()->main_loop_request_go_back(); - } - - Input::get_singleton()->accumulate_input_event(ev); -} - -void DisplayServerAndroid::process_touch(int p_event, int p_pointer, const Vector<DisplayServerAndroid::TouchPos> &p_points) { - switch (p_event) { - case AMOTION_EVENT_ACTION_DOWN: { //gesture begin - if (touch.size()) { - //end all if exist - for (int i = 0; i < touch.size(); i++) { - Ref<InputEventScreenTouch> ev; - ev.instantiate(); - ev->set_index(touch[i].id); - ev->set_pressed(false); - ev->set_position(touch[i].pos); - Input::get_singleton()->accumulate_input_event(ev); - } - } - - touch.resize(p_points.size()); - for (int i = 0; i < p_points.size(); i++) { - touch.write[i].id = p_points[i].id; - touch.write[i].pos = p_points[i].pos; - } - - //send touch - for (int i = 0; i < touch.size(); i++) { - Ref<InputEventScreenTouch> ev; - ev.instantiate(); - ev->set_index(touch[i].id); - ev->set_pressed(true); - ev->set_position(touch[i].pos); - Input::get_singleton()->accumulate_input_event(ev); - } - - } break; - case AMOTION_EVENT_ACTION_MOVE: { //motion - ERR_FAIL_COND(touch.size() != p_points.size()); - - for (int i = 0; i < touch.size(); i++) { - int idx = -1; - for (int j = 0; j < p_points.size(); j++) { - if (touch[i].id == p_points[j].id) { - idx = j; - break; - } - } - - ERR_CONTINUE(idx == -1); - - if (touch[i].pos == p_points[idx].pos) - continue; //no move unncesearily - - Ref<InputEventScreenDrag> ev; - ev.instantiate(); - ev->set_index(touch[i].id); - ev->set_position(p_points[idx].pos); - ev->set_relative(p_points[idx].pos - touch[i].pos); - Input::get_singleton()->accumulate_input_event(ev); - touch.write[i].pos = p_points[idx].pos; - } - - } break; - case AMOTION_EVENT_ACTION_CANCEL: - case AMOTION_EVENT_ACTION_UP: { //release - if (touch.size()) { - //end all if exist - for (int i = 0; i < touch.size(); i++) { - Ref<InputEventScreenTouch> ev; - ev.instantiate(); - ev->set_index(touch[i].id); - ev->set_pressed(false); - ev->set_position(touch[i].pos); - Input::get_singleton()->accumulate_input_event(ev); - } - touch.clear(); - } - } break; - case AMOTION_EVENT_ACTION_POINTER_DOWN: { // add touch - for (int i = 0; i < p_points.size(); i++) { - if (p_points[i].id == p_pointer) { - TouchPos tp = p_points[i]; - touch.push_back(tp); - - Ref<InputEventScreenTouch> ev; - ev.instantiate(); - - ev->set_index(tp.id); - ev->set_pressed(true); - ev->set_position(tp.pos); - Input::get_singleton()->accumulate_input_event(ev); - - break; - } - } - } break; - case AMOTION_EVENT_ACTION_POINTER_UP: { // remove touch - for (int i = 0; i < touch.size(); i++) { - if (touch[i].id == p_pointer) { - Ref<InputEventScreenTouch> ev; - ev.instantiate(); - ev->set_index(touch[i].id); - ev->set_pressed(false); - ev->set_position(touch[i].pos); - Input::get_singleton()->accumulate_input_event(ev); - touch.remove(i); - - break; - } - } - } break; - } -} - -void DisplayServerAndroid::process_hover(int p_type, Point2 p_pos) { - // https://developer.android.com/reference/android/view/MotionEvent.html#ACTION_HOVER_ENTER - switch (p_type) { - case AMOTION_EVENT_ACTION_HOVER_MOVE: // hover move - case AMOTION_EVENT_ACTION_HOVER_ENTER: // hover enter - case AMOTION_EVENT_ACTION_HOVER_EXIT: { // hover exit - Ref<InputEventMouseMotion> ev; - ev.instantiate(); - _set_key_modifier_state(ev); - ev->set_position(p_pos); - ev->set_global_position(p_pos); - ev->set_relative(p_pos - hover_prev_pos); - Input::get_singleton()->accumulate_input_event(ev); - hover_prev_pos = p_pos; - } break; - } -} - -void DisplayServerAndroid::process_mouse_event(int input_device, int event_action, int event_android_buttons_mask, Point2 event_pos, float event_vertical_factor, float event_horizontal_factor) { - MouseButton event_buttons_mask = _android_button_mask_to_godot_button_mask(event_android_buttons_mask); - switch (event_action) { - case AMOTION_EVENT_ACTION_BUTTON_PRESS: - case AMOTION_EVENT_ACTION_BUTTON_RELEASE: { - Ref<InputEventMouseButton> ev; - ev.instantiate(); - _set_key_modifier_state(ev); - if ((input_device & AINPUT_SOURCE_MOUSE) == AINPUT_SOURCE_MOUSE) { - ev->set_position(event_pos); - ev->set_global_position(event_pos); - } else { - ev->set_position(hover_prev_pos); - ev->set_global_position(hover_prev_pos); - } - ev->set_pressed(event_action == AMOTION_EVENT_ACTION_BUTTON_PRESS); - MouseButton changed_button_mask = MouseButton(buttons_state ^ event_buttons_mask); - - buttons_state = event_buttons_mask; - - ev->set_button_index(_button_index_from_mask(changed_button_mask)); - ev->set_button_mask(event_buttons_mask); - Input::get_singleton()->accumulate_input_event(ev); - } break; - - case AMOTION_EVENT_ACTION_MOVE: { - Ref<InputEventMouseMotion> ev; - ev.instantiate(); - _set_key_modifier_state(ev); - if ((input_device & AINPUT_SOURCE_MOUSE) == AINPUT_SOURCE_MOUSE) { - ev->set_position(event_pos); - ev->set_global_position(event_pos); - ev->set_relative(event_pos - hover_prev_pos); - hover_prev_pos = event_pos; - } else { - ev->set_position(hover_prev_pos); - ev->set_global_position(hover_prev_pos); - ev->set_relative(event_pos); - } - ev->set_button_mask(event_buttons_mask); - Input::get_singleton()->accumulate_input_event(ev); - } break; - case AMOTION_EVENT_ACTION_SCROLL: { - Ref<InputEventMouseButton> ev; - ev.instantiate(); - if ((input_device & AINPUT_SOURCE_MOUSE) == AINPUT_SOURCE_MOUSE) { - ev->set_position(event_pos); - ev->set_global_position(event_pos); - } else { - ev->set_position(hover_prev_pos); - ev->set_global_position(hover_prev_pos); - } - ev->set_pressed(true); - buttons_state = event_buttons_mask; - if (event_vertical_factor > 0) { - _wheel_button_click(event_buttons_mask, ev, MOUSE_BUTTON_WHEEL_UP, event_vertical_factor); - } else if (event_vertical_factor < 0) { - _wheel_button_click(event_buttons_mask, ev, MOUSE_BUTTON_WHEEL_DOWN, -event_vertical_factor); - } - - if (event_horizontal_factor > 0) { - _wheel_button_click(event_buttons_mask, ev, MOUSE_BUTTON_WHEEL_RIGHT, event_horizontal_factor); - } else if (event_horizontal_factor < 0) { - _wheel_button_click(event_buttons_mask, ev, MOUSE_BUTTON_WHEEL_LEFT, -event_horizontal_factor); - } - } break; - } -} - -void DisplayServerAndroid::_wheel_button_click(MouseButton event_buttons_mask, const Ref<InputEventMouseButton> &ev, MouseButton wheel_button, float factor) { - Ref<InputEventMouseButton> evd = ev->duplicate(); - _set_key_modifier_state(evd); - evd->set_button_index(wheel_button); - evd->set_button_mask(MouseButton(event_buttons_mask ^ (1 << (wheel_button - 1)))); - evd->set_factor(factor); - Input::get_singleton()->accumulate_input_event(evd); - Ref<InputEventMouseButton> evdd = evd->duplicate(); - evdd->set_pressed(false); - evdd->set_button_mask(event_buttons_mask); - Input::get_singleton()->accumulate_input_event(evdd); -} - -void DisplayServerAndroid::process_double_tap(int event_android_button_mask, Point2 p_pos) { - MouseButton event_button_mask = _android_button_mask_to_godot_button_mask(event_android_button_mask); - Ref<InputEventMouseButton> ev; - ev.instantiate(); - _set_key_modifier_state(ev); - ev->set_position(p_pos); - ev->set_global_position(p_pos); - ev->set_pressed(event_button_mask != 0); - ev->set_button_index(_button_index_from_mask(event_button_mask)); - ev->set_button_mask(event_button_mask); - ev->set_double_click(true); - Input::get_singleton()->accumulate_input_event(ev); -} - -MouseButton DisplayServerAndroid::_button_index_from_mask(MouseButton button_mask) { - switch (button_mask) { - case MOUSE_BUTTON_MASK_LEFT: - return MOUSE_BUTTON_LEFT; - case MOUSE_BUTTON_MASK_RIGHT: - return MOUSE_BUTTON_RIGHT; - case MOUSE_BUTTON_MASK_MIDDLE: - return MOUSE_BUTTON_MIDDLE; - case MOUSE_BUTTON_MASK_XBUTTON1: - return MOUSE_BUTTON_XBUTTON1; - case MOUSE_BUTTON_MASK_XBUTTON2: - return MOUSE_BUTTON_XBUTTON2; - default: - return MOUSE_BUTTON_NONE; - } -} - -void DisplayServerAndroid::process_scroll(Point2 p_pos) { - Ref<InputEventPanGesture> ev; - ev.instantiate(); - _set_key_modifier_state(ev); - ev->set_position(p_pos); - ev->set_delta(p_pos - scroll_prev_pos); - Input::get_singleton()->accumulate_input_event(ev); - scroll_prev_pos = p_pos; -} - void DisplayServerAndroid::process_accelerometer(const Vector3 &p_accelerometer) { Input::get_singleton()->set_accelerometer(p_accelerometer); } @@ -852,32 +518,11 @@ DisplayServer::MouseMode DisplayServerAndroid::mouse_get_mode() const { } Point2i DisplayServerAndroid::mouse_get_position() const { - return hover_prev_pos; + return Input::get_singleton()->get_mouse_position(); } MouseButton DisplayServerAndroid::mouse_get_button_state() const { - return buttons_state; -} - -MouseButton DisplayServerAndroid::_android_button_mask_to_godot_button_mask(int android_button_mask) { - MouseButton godot_button_mask = MOUSE_BUTTON_NONE; - if (android_button_mask & AMOTION_EVENT_BUTTON_PRIMARY) { - godot_button_mask |= MOUSE_BUTTON_MASK_LEFT; - } - if (android_button_mask & AMOTION_EVENT_BUTTON_SECONDARY) { - godot_button_mask |= MOUSE_BUTTON_MASK_RIGHT; - } - if (android_button_mask & AMOTION_EVENT_BUTTON_TERTIARY) { - godot_button_mask |= MOUSE_BUTTON_MASK_MIDDLE; - } - if (android_button_mask & AMOTION_EVENT_BUTTON_BACK) { - godot_button_mask |= MOUSE_BUTTON_MASK_XBUTTON1; - } - if (android_button_mask & AMOTION_EVENT_BUTTON_SECONDARY) { - godot_button_mask |= MOUSE_BUTTON_MASK_XBUTTON2; - } - - return godot_button_mask; + return (MouseButton)Input::get_singleton()->get_mouse_button_mask(); } void DisplayServerAndroid::cursor_set_shape(DisplayServer::CursorShape p_shape) { diff --git a/platform/android/display_server_android.h b/platform/android/display_server_android.h index 9b9f5e99f6..669a1c80e4 100644 --- a/platform/android/display_server_android.h +++ b/platform/android/display_server_android.h @@ -39,37 +39,8 @@ class RenderingDeviceVulkan; #endif class DisplayServerAndroid : public DisplayServer { -public: - struct TouchPos { - int id = 0; - Point2 pos; - }; - - enum { - JOY_EVENT_BUTTON = 0, - JOY_EVENT_AXIS = 1, - JOY_EVENT_HAT = 2 - }; - - struct JoypadEvent { - int device = 0; - int type = 0; - int index = 0; - bool pressed = false; - float value = 0; - int hat = 0; - }; - -private: String rendering_driver; - bool alt_mem = false; - bool shift_mem = false; - bool control_mem = false; - bool meta_mem = false; - - MouseButton buttons_state = MOUSE_BUTTON_NONE; - // https://developer.android.com/reference/android/view/PointerIcon // mapping between Godot's cursor shape to Android's' int android_cursors[CURSOR_MAX] = { @@ -96,10 +67,6 @@ private: bool keep_screen_on; - Vector<TouchPos> touch; - Point2 hover_prev_pos; // needed to calculate the relative position on hover events - Point2 scroll_prev_pos; // needed to calculate the relative position on scroll events - CursorShape cursor_shape = CursorShape::CURSOR_ARROW; #if defined(VULKAN_ENABLED) @@ -114,18 +81,10 @@ private: Callable input_text_callback; Callable rect_changed_callback; - void _window_callback(const Callable &p_callable, const Variant &p_arg) const; + void _window_callback(const Callable &p_callable, const Variant &p_arg, bool p_deferred = false) const; static void _dispatch_input_events(const Ref<InputEvent> &p_event); - void _set_key_modifier_state(Ref<InputEventWithModifiers> ev); - - static MouseButton _button_index_from_mask(MouseButton button_mask); - - static MouseButton _android_button_mask_to_godot_button_mask(int android_button_mask); - - void _wheel_button_click(MouseButton event_buttons_mask, const Ref<InputEventMouseButton> &ev, MouseButton wheel_button, float factor); - public: static DisplayServerAndroid *get_singleton(); @@ -158,7 +117,7 @@ public: virtual void window_set_rect_changed_callback(const Callable &p_callable, WindowID p_window = MAIN_WINDOW_ID) override; virtual void window_set_drop_files_callback(const Callable &p_callable, WindowID p_window = MAIN_WINDOW_ID) override; - void send_window_event(WindowEvent p_event) const; + void send_window_event(WindowEvent p_event, bool p_deferred = false) const; void send_input_event(const Ref<InputEvent> &p_event) const; void send_input_text(const String &p_text) const; @@ -210,13 +169,6 @@ public: void process_gravity(const Vector3 &p_gravity); void process_magnetometer(const Vector3 &p_magnetometer); void process_gyroscope(const Vector3 &p_gyroscope); - void process_touch(int p_event, int p_pointer, const Vector<TouchPos> &p_points); - void process_hover(int p_type, Point2 p_pos); - void process_mouse_event(int input_device, int event_action, int event_android_buttons_mask, Point2 event_pos, float event_vertical_factor = 0, float event_horizontal_factor = 0); - void process_double_tap(int event_android_button_mask, Point2 p_pos); - void process_scroll(Point2 p_pos); - void process_joy_event(JoypadEvent p_event); - void process_key_event(int p_keycode, int p_scancode, int p_unicode_char, bool p_pressed); virtual void cursor_set_shape(CursorShape p_shape) override; virtual CursorShape cursor_get_shape() const override; diff --git a/platform/android/export/export_plugin.cpp b/platform/android/export/export_plugin.cpp index 6c2e481856..17ee173855 100644 --- a/platform/android/export/export_plugin.cpp +++ b/platform/android/export/export_plugin.cpp @@ -819,6 +819,7 @@ void EditorExportPlatformAndroid::_fix_manifest(const Ref<EditorExportPreset> &p bool backup_allowed = p_preset->get("user_data_backup/allow"); bool classify_as_game = p_preset->get("package/classify_as_game"); + bool retain_data_on_uninstall = p_preset->get("package/retain_data_on_uninstall"); Vector<String> perms; // Write permissions into the perms variable. @@ -920,6 +921,10 @@ void EditorExportPlatformAndroid::_fix_manifest(const Ref<EditorExportPreset> &p encode_uint32(classify_as_game, &p_manifest.write[iofs + 16]); } + if (tname == "application" && attrname == "hasFragileUserData") { + encode_uint32(retain_data_on_uninstall, &p_manifest.write[iofs + 16]); + } + if (tname == "instrumentation" && attrname == "targetPackage") { string_table.write[attr_value] = get_package_name(package_name); } @@ -1635,6 +1640,7 @@ void EditorExportPlatformAndroid::get_export_options(List<ExportOption> *r_optio r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, "package/name", PROPERTY_HINT_PLACEHOLDER_TEXT, "Game Name [default if blank]"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "package/signed"), true)); r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "package/classify_as_game"), true)); + r_options->push_back(ExportOption(PropertyInfo(Variant::BOOL, "package/retain_data_on_uninstall"), false)); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, launcher_icon_option, PROPERTY_HINT_FILE, "*.png"), "")); r_options->push_back(ExportOption(PropertyInfo(Variant::STRING, launcher_adaptive_icon_foreground_option, PROPERTY_HINT_FILE, "*.png"), "")); diff --git a/platform/android/export/gradle_export_util.cpp b/platform/android/export/gradle_export_util.cpp index 354287d872..76512226bf 100644 --- a/platform/android/export/gradle_export_util.cpp +++ b/platform/android/export/gradle_export_util.cpp @@ -241,10 +241,12 @@ String _get_application_tag(const Ref<EditorExportPreset> &p_preset) { " android:allowBackup=\"%s\"\n" " android:icon=\"@mipmap/icon\"\n" " android:isGame=\"%s\"\n" - " tools:replace=\"android:allowBackup,android:isGame\"\n" + " android:hasFragileUserData=\"%s\"\n" + " tools:replace=\"android:allowBackup,android:isGame,android:hasFragileUserData\"\n" " tools:ignore=\"GoogleAppIndexingWarning\">\n\n", bool_to_string(p_preset->get("user_data_backup/allow")), - bool_to_string(p_preset->get("package/classify_as_game"))); + bool_to_string(p_preset->get("package/classify_as_game")), + bool_to_string(p_preset->get("package/retain_data_on_uninstall"))); manifest_application_text += _get_activity_tag(p_preset); manifest_application_text += " </application>\n"; diff --git a/platform/android/java/app/AndroidManifest.xml b/platform/android/java/app/AndroidManifest.xml index 0874d77645..467a0dc3c0 100644 --- a/platform/android/java/app/AndroidManifest.xml +++ b/platform/android/java/app/AndroidManifest.xml @@ -21,6 +21,7 @@ android:allowBackup="false" android:icon="@mipmap/icon" android:isGame="true" + android:hasFragileUserData="false" tools:ignore="GoogleAppIndexingWarning" > <!-- Records the version of the Godot editor used for building --> diff --git a/platform/android/java/lib/src/org/godotengine/godot/FullScreenGodotApp.java b/platform/android/java/lib/src/org/godotengine/godot/FullScreenGodotApp.java index 89497d1526..ad7048cbf3 100644 --- a/platform/android/java/lib/src/org/godotengine/godot/FullScreenGodotApp.java +++ b/platform/android/java/lib/src/org/godotengine/godot/FullScreenGodotApp.java @@ -32,11 +32,12 @@ package org.godotengine.godot; import android.content.Intent; import android.os.Bundle; -import android.view.KeyEvent; +import android.util.Log; import androidx.annotation.CallSuper; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import androidx.fragment.app.Fragment; import androidx.fragment.app.FragmentActivity; /** @@ -46,6 +47,8 @@ import androidx.fragment.app.FragmentActivity; * within an Android app. */ public abstract class FullScreenGodotApp extends FragmentActivity implements GodotHost { + private static final String TAG = FullScreenGodotApp.class.getSimpleName(); + @Nullable private Godot godotFragment; @@ -53,12 +56,33 @@ public abstract class FullScreenGodotApp extends FragmentActivity implements God public void onCreate(Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.godot_app_layout); - godotFragment = initGodotInstance(); - if (godotFragment == null) { - throw new IllegalStateException("Godot instance must be non-null."); + + Fragment currentFragment = getSupportFragmentManager().findFragmentById(R.id.godot_fragment_container); + if (currentFragment instanceof Godot) { + Log.v(TAG, "Reusing existing Godot fragment instance."); + godotFragment = (Godot)currentFragment; + } else { + Log.v(TAG, "Creating new Godot fragment instance."); + godotFragment = initGodotInstance(); + if (godotFragment == null) { + throw new IllegalStateException("Godot instance must be non-null."); + } + + getSupportFragmentManager().beginTransaction().replace(R.id.godot_fragment_container, godotFragment).setPrimaryNavigationFragment(godotFragment).commitNowAllowingStateLoss(); } + } - getSupportFragmentManager().beginTransaction().replace(R.id.godot_fragment_container, godotFragment).setPrimaryNavigationFragment(godotFragment).commitNowAllowingStateLoss(); + @Override + public void onDestroy() { + super.onDestroy(); + onGodotForceQuit(godotFragment); + } + + @Override + public final void onGodotForceQuit(Godot instance) { + if (instance == godotFragment) { + System.exit(0); + } } @Override diff --git a/platform/android/java/lib/src/org/godotengine/godot/Godot.java b/platform/android/java/lib/src/org/godotengine/godot/Godot.java index 01eb1f1ec8..76751a886c 100644 --- a/platform/android/java/lib/src/org/godotengine/godot/Godot.java +++ b/platform/android/java/lib/src/org/godotengine/godot/Godot.java @@ -657,8 +657,6 @@ public class Godot extends Fragment implements SensorEventListener, IDownloaderC super.onDestroy(); - // TODO: This is a temp solution. The proper fix will involve tracking down and properly shutting down each - // native Godot components that is started in Godot#onVideoInit. forceQuit(); } @@ -842,8 +840,11 @@ public class Godot extends Fragment implements SensorEventListener, IDownloaderC } private void forceQuit() { - getActivity().finish(); - System.exit(0); + // TODO: This is a temp solution. The proper fix will involve tracking down and properly shutting down each + // native Godot components that is started in Godot#onVideoInit. + if (godotHost != null) { + godotHost.onGodotForceQuit(this); + } } private boolean obbIsCorrupted(String f, String main_pack_md5) { diff --git a/platform/android/java/lib/src/org/godotengine/godot/GodotHost.java b/platform/android/java/lib/src/org/godotengine/godot/GodotHost.java index 317fd13535..58e982c569 100644 --- a/platform/android/java/lib/src/org/godotengine/godot/GodotHost.java +++ b/platform/android/java/lib/src/org/godotengine/godot/GodotHost.java @@ -53,4 +53,9 @@ public interface GodotHost { * Invoked on the render thread when the Godot main loop has started. */ default void onGodotMainLoopStarted() {} + + /** + * Invoked on the UI thread as the last step of the Godot instance clean up phase. + */ + default void onGodotForceQuit(Godot instance) {} } diff --git a/platform/android/java/lib/src/org/godotengine/godot/input/GodotGestureHandler.java b/platform/android/java/lib/src/org/godotengine/godot/input/GodotGestureHandler.java index 1d60c21c60..6b248fd034 100644 --- a/platform/android/java/lib/src/org/godotengine/godot/input/GodotGestureHandler.java +++ b/platform/android/java/lib/src/org/godotengine/godot/input/GodotGestureHandler.java @@ -75,7 +75,7 @@ public class GodotGestureHandler extends GestureDetector.SimpleOnGestureListener final int x = Math.round(event.getX()); final int y = Math.round(event.getY()); final int buttonMask = event.getButtonState(); - queueEvent(() -> GodotLib.doubleTap(buttonMask, x, y)); + GodotLib.doubleTap(buttonMask, x, y); return true; } @@ -84,7 +84,7 @@ public class GodotGestureHandler extends GestureDetector.SimpleOnGestureListener //Log.i("GodotGesture", "onScroll"); final int x = Math.round(distanceX); final int y = Math.round(distanceY); - queueEvent(() -> GodotLib.scroll(x, y)); + GodotLib.scroll(x, y); return true; } diff --git a/platform/android/java/lib/src/org/godotengine/godot/input/GodotInputHandler.java b/platform/android/java/lib/src/org/godotengine/godot/input/GodotInputHandler.java index 4dc9157545..fc0b84b392 100644 --- a/platform/android/java/lib/src/org/godotengine/godot/input/GodotInputHandler.java +++ b/platform/android/java/lib/src/org/godotengine/godot/input/GodotInputHandler.java @@ -68,10 +68,6 @@ public class GodotInputHandler implements InputDeviceListener { mInputManager.registerInputDeviceListener(this, null); } - private void queueEvent(Runnable task) { - mRenderView.queueOnRenderThread(task); - } - private boolean isKeyEvent_GameDevice(int source) { // Note that keyboards are often (SOURCE_KEYBOARD | SOURCE_DPAD) if (source == (InputDevice.SOURCE_KEYBOARD | InputDevice.SOURCE_DPAD)) @@ -96,13 +92,12 @@ public class GodotInputHandler implements InputDeviceListener { if (mJoystickIds.indexOfKey(deviceId) >= 0) { final int button = getGodotButton(keyCode); final int godotJoyId = mJoystickIds.get(deviceId); - - queueEvent(() -> GodotLib.joybutton(godotJoyId, button, false)); + GodotLib.joybutton(godotJoyId, button, false); } } else { final int scanCode = event.getScanCode(); final int chr = event.getUnicodeChar(0); - queueEvent(() -> GodotLib.key(keyCode, scanCode, chr, false)); + GodotLib.key(keyCode, scanCode, chr, false); } return true; @@ -132,13 +127,12 @@ public class GodotInputHandler implements InputDeviceListener { if (mJoystickIds.indexOfKey(deviceId) >= 0) { final int button = getGodotButton(keyCode); final int godotJoyId = mJoystickIds.get(deviceId); - - queueEvent(() -> GodotLib.joybutton(godotJoyId, button, true)); + GodotLib.joybutton(godotJoyId, button, true); } } else { final int scanCode = event.getScanCode(); final int chr = event.getUnicodeChar(0); - queueEvent(() -> GodotLib.key(keyCode, scanCode, chr, true)); + GodotLib.key(keyCode, scanCode, chr, true); } return true; @@ -170,18 +164,16 @@ public class GodotInputHandler implements InputDeviceListener { final int action = event.getActionMasked(); final int pointer_idx = event.getPointerId(event.getActionIndex()); - mRenderView.queueOnRenderThread(() -> { - switch (action) { - case MotionEvent.ACTION_DOWN: - case MotionEvent.ACTION_CANCEL: - case MotionEvent.ACTION_UP: - case MotionEvent.ACTION_MOVE: - case MotionEvent.ACTION_POINTER_UP: - case MotionEvent.ACTION_POINTER_DOWN: { - GodotLib.touch(event.getSource(), action, pointer_idx, evcount, arr); - } break; - } - }); + switch (action) { + case MotionEvent.ACTION_DOWN: + case MotionEvent.ACTION_CANCEL: + case MotionEvent.ACTION_UP: + case MotionEvent.ACTION_MOVE: + case MotionEvent.ACTION_POINTER_UP: + case MotionEvent.ACTION_POINTER_DOWN: { + GodotLib.touch(event.getSource(), action, pointer_idx, evcount, arr); + } break; + } } return true; } @@ -205,7 +197,7 @@ public class GodotInputHandler implements InputDeviceListener { // save value to prevent repeats joystick.axesValues.put(axis, value); final int godotAxisIdx = i; - queueEvent(() -> GodotLib.joyaxis(godotJoyId, godotAxisIdx, value)); + GodotLib.joyaxis(godotJoyId, godotAxisIdx, value); } } @@ -215,7 +207,7 @@ public class GodotInputHandler implements InputDeviceListener { if (joystick.hatX != hatX || joystick.hatY != hatY) { joystick.hatX = hatX; joystick.hatY = hatY; - queueEvent(() -> GodotLib.joyhat(godotJoyId, hatX, hatY)); + GodotLib.joyhat(godotJoyId, hatX, hatY); } } return true; @@ -224,7 +216,7 @@ public class GodotInputHandler implements InputDeviceListener { final float x = event.getX(); final float y = event.getY(); final int type = event.getAction(); - queueEvent(() -> GodotLib.hover(type, x, y)); + GodotLib.hover(type, x, y); return true; } else if (event.isFromSource(InputDevice.SOURCE_MOUSE) || event.isFromSource(InputDevice.SOURCE_MOUSE_RELATIVE)) { @@ -316,7 +308,7 @@ public class GodotInputHandler implements InputDeviceListener { } mJoysticksDevices.put(deviceId, joystick); - queueEvent(() -> GodotLib.joyconnectionchanged(id, true, joystick.name)); + GodotLib.joyconnectionchanged(id, true, joystick.name); } @Override @@ -328,8 +320,7 @@ public class GodotInputHandler implements InputDeviceListener { final int godotJoyId = mJoystickIds.get(deviceId); mJoystickIds.delete(deviceId); mJoysticksDevices.delete(deviceId); - - queueEvent(() -> GodotLib.joyconnectionchanged(godotJoyId, false, "")); + GodotLib.joyconnectionchanged(godotJoyId, false, ""); } @Override @@ -418,7 +409,7 @@ public class GodotInputHandler implements InputDeviceListener { final float x = event.getX(); final float y = event.getY(); final int type = event.getAction(); - queueEvent(() -> GodotLib.hover(type, x, y)); + GodotLib.hover(type, x, y); return true; } case MotionEvent.ACTION_BUTTON_PRESS: @@ -428,7 +419,7 @@ public class GodotInputHandler implements InputDeviceListener { final float y = event.getY(); final int buttonsMask = event.getButtonState(); final int action = event.getAction(); - queueEvent(() -> GodotLib.touch(event.getSource(), action, 0, 1, new float[] { 0, x, y }, buttonsMask)); + GodotLib.touch(event.getSource(), action, 0, 1, new float[] { 0, x, y }, buttonsMask); return true; } case MotionEvent.ACTION_SCROLL: { @@ -438,7 +429,7 @@ public class GodotInputHandler implements InputDeviceListener { final int action = event.getAction(); final float verticalFactor = event.getAxisValue(MotionEvent.AXIS_VSCROLL); final float horizontalFactor = event.getAxisValue(MotionEvent.AXIS_HSCROLL); - queueEvent(() -> GodotLib.touch(event.getSource(), action, 0, 1, new float[] { 0, x, y }, buttonsMask, verticalFactor, horizontalFactor)); + GodotLib.touch(event.getSource(), action, 0, 1, new float[] { 0, x, y }, buttonsMask, verticalFactor, horizontalFactor); } case MotionEvent.ACTION_DOWN: case MotionEvent.ACTION_UP: { diff --git a/platform/android/java/lib/src/org/godotengine/godot/input/GodotTextInputWrapper.java b/platform/android/java/lib/src/org/godotengine/godot/input/GodotTextInputWrapper.java index 020870a110..002a75277d 100644 --- a/platform/android/java/lib/src/org/godotengine/godot/input/GodotTextInputWrapper.java +++ b/platform/android/java/lib/src/org/godotengine/godot/input/GodotTextInputWrapper.java @@ -94,17 +94,15 @@ public class GodotTextInputWrapper implements TextWatcher, OnEditorActionListene public void beforeTextChanged(final CharSequence pCharSequence, final int start, final int count, final int after) { //Log.d(TAG, "beforeTextChanged(" + pCharSequence + ")start: " + start + ",count: " + count + ",after: " + after); - mRenderView.queueOnRenderThread(() -> { - for (int i = 0; i < count; ++i) { - GodotLib.key(KeyEvent.KEYCODE_DEL, KeyEvent.KEYCODE_DEL, 0, true); - GodotLib.key(KeyEvent.KEYCODE_DEL, KeyEvent.KEYCODE_DEL, 0, false); - - if (mHasSelection) { - mHasSelection = false; - break; - } + for (int i = 0; i < count; ++i) { + GodotLib.key(KeyEvent.KEYCODE_DEL, KeyEvent.KEYCODE_DEL, 0, true); + GodotLib.key(KeyEvent.KEYCODE_DEL, KeyEvent.KEYCODE_DEL, 0, false); + + if (mHasSelection) { + mHasSelection = false; + break; } - }); + } } @Override @@ -115,17 +113,15 @@ public class GodotTextInputWrapper implements TextWatcher, OnEditorActionListene for (int i = start; i < start + count; ++i) { newChars[i - start] = pCharSequence.charAt(i); } - mRenderView.queueOnRenderThread(() -> { - for (int i = 0; i < count; ++i) { - int key = newChars[i]; - if ((key == '\n') && !mEdit.isMultiline()) { - // Return keys are handled through action events - continue; - } - GodotLib.key(0, 0, key, true); - GodotLib.key(0, 0, key, false); + for (int i = 0; i < count; ++i) { + int key = newChars[i]; + if ((key == '\n') && !mEdit.isMultiline()) { + // Return keys are handled through action events + continue; } - }); + GodotLib.key(0, 0, key, true); + GodotLib.key(0, 0, key, false); + } } @Override @@ -133,13 +129,11 @@ public class GodotTextInputWrapper implements TextWatcher, OnEditorActionListene if (mEdit == pTextView && isFullScreenEdit()) { final String characters = pKeyEvent.getCharacters(); - mRenderView.queueOnRenderThread(() -> { - for (int i = 0; i < characters.length(); i++) { - final int ch = characters.codePointAt(i); - GodotLib.key(0, 0, ch, true); - GodotLib.key(0, 0, ch, false); - } - }); + for (int i = 0; i < characters.length(); i++) { + final int ch = characters.codePointAt(i); + GodotLib.key(0, 0, ch, true); + GodotLib.key(0, 0, ch, false); + } } if (pActionID == EditorInfo.IME_ACTION_DONE) { diff --git a/platform/android/java_godot_lib_jni.cpp b/platform/android/java_godot_lib_jni.cpp index ce7a49e53c..d971727269 100644 --- a/platform/android/java_godot_lib_jni.cpp +++ b/platform/android/java_godot_lib_jni.cpp @@ -34,6 +34,7 @@ #include "java_godot_wrapper.h" #include "android/asset_manager_jni.h" +#include "android_input_handler.h" #include "api/java_class_wrapper.h" #include "api/jni_singleton.h" #include "core/config/engine.h" @@ -56,11 +57,12 @@ static JavaClassWrapper *java_class_wrapper = nullptr; static OS_Android *os_android = nullptr; +static AndroidInputHandler *input_handler = nullptr; static GodotJavaWrapper *godot_java = nullptr; static GodotIOJavaWrapper *godot_io_java = nullptr; static bool initialized = false; -static int step = 0; +static SafeNumeric<int> step; // Shared between UI and render threads static Size2 new_size; static Vector3 accelerometer; @@ -111,6 +113,9 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_ondestroy(JNIEnv *env if (godot_java) { delete godot_java; } + if (input_handler) { + delete input_handler; + } if (os_android) { delete os_android; } @@ -165,7 +170,7 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_resize(JNIEnv *env, j os_android->set_display_size(Size2i(p_width, p_height)); // No need to reset the surface during startup - if (step > 0) { + if (step.get() > 0) { if (p_surface) { ANativeWindow *native_window = ANativeWindow_fromSurface(env, p_surface); os_android->set_native_window(native_window); @@ -179,7 +184,7 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_resize(JNIEnv *env, j JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_newcontext(JNIEnv *env, jclass clazz, jobject p_surface, jboolean p_32_bits) { if (os_android) { - if (step == 0) { + if (step.get() == 0) { // During startup os_android->set_context_is_16_bits(!p_32_bits); if (p_surface) { @@ -188,33 +193,36 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_newcontext(JNIEnv *en } } else { // Rendering context recreated because it was lost; restart app to let it reload everything + step.set(-1); // Ensure no further steps are attempted and no further events are sent os_android->main_loop_end(); godot_java->restart(env); - step = -1; // Ensure no further steps are attempted } } } JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_back(JNIEnv *env, jclass clazz) { - if (step == 0) + if (step.get() == 0) return; - os_android->main_loop_request_go_back(); + if (DisplayServerAndroid *dsa = Object::cast_to<DisplayServerAndroid>(DisplayServer::get_singleton())) { + dsa->send_window_event(DisplayServer::WINDOW_EVENT_GO_BACK_REQUEST); + } } JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_step(JNIEnv *env, jclass clazz) { - if (step == -1) + if (step.get() == -1) return; - if (step == 0) { + if (step.get() == 0) { // Since Godot is initialized on the UI thread, main_thread_id was set to that thread's id, // but for Godot purposes, the main thread is the one running the game loop Main::setup2(Thread::get_caller_id()); - ++step; + input_handler = new AndroidInputHandler(); + step.increment(); return; } - if (step == 1) { + if (step.get() == 1) { if (!Main::start()) { return; // should exit instead and print the error } @@ -222,7 +230,7 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_step(JNIEnv *env, jcl godot_java->on_godot_setup_completed(env); os_android->main_loop_begin(); godot_java->on_godot_main_loop_started(env); - ++step; + step.increment(); } DisplayServerAndroid::get_singleton()->process_accelerometer(accelerometer); @@ -236,91 +244,100 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_step(JNIEnv *env, jcl } void touch_preprocessing(JNIEnv *env, jclass clazz, jint input_device, jint ev, jint pointer, jint pointer_count, jfloatArray positions, jint buttons_mask, jfloat vertical_factor, jfloat horizontal_factor) { - if (step == 0) + if (step.get() <= 0) return; - Vector<DisplayServerAndroid::TouchPos> points; + Vector<AndroidInputHandler::TouchPos> points; for (int i = 0; i < pointer_count; i++) { jfloat p[3]; env->GetFloatArrayRegion(positions, i * 3, 3, p); - DisplayServerAndroid::TouchPos tp; + AndroidInputHandler::TouchPos tp; tp.pos = Point2(p[1], p[2]); tp.id = (int)p[0]; points.push_back(tp); } if ((input_device & AINPUT_SOURCE_MOUSE) == AINPUT_SOURCE_MOUSE || (input_device & AINPUT_SOURCE_MOUSE_RELATIVE) == AINPUT_SOURCE_MOUSE_RELATIVE) { - DisplayServerAndroid::get_singleton()->process_mouse_event(input_device, ev, buttons_mask, points[0].pos, vertical_factor, horizontal_factor); + input_handler->process_mouse_event(input_device, ev, buttons_mask, points[0].pos, vertical_factor, horizontal_factor); } else { - DisplayServerAndroid::get_singleton()->process_touch(ev, pointer, points); + input_handler->process_touch(ev, pointer, points); } } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_touch__IIII_3F(JNIEnv *env, jclass clazz, jint input_device, jint ev, jint pointer, jint pointer_count, jfloatArray position) { touch_preprocessing(env, clazz, input_device, ev, pointer, pointer_count, position); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_touch__IIII_3FI(JNIEnv *env, jclass clazz, jint input_device, jint ev, jint pointer, jint pointer_count, jfloatArray position, jint buttons_mask) { touch_preprocessing(env, clazz, input_device, ev, pointer, pointer_count, position, buttons_mask); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_touch__IIII_3FIFF(JNIEnv *env, jclass clazz, jint input_device, jint ev, jint pointer, jint pointer_count, jfloatArray position, jint buttons_mask, jfloat vertical_factor, jfloat horizontal_factor) { touch_preprocessing(env, clazz, input_device, ev, pointer, pointer_count, position, buttons_mask, vertical_factor, horizontal_factor); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_hover(JNIEnv *env, jclass clazz, jint p_type, jfloat p_x, jfloat p_y) { - if (step == 0) + if (step.get() <= 0) return; - DisplayServerAndroid::get_singleton()->process_hover(p_type, Point2(p_x, p_y)); + input_handler->process_hover(p_type, Point2(p_x, p_y)); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_doubleTap(JNIEnv *env, jclass clazz, jint p_button_mask, jint p_x, jint p_y) { - if (step == 0) + if (step.get() <= 0) return; - DisplayServerAndroid::get_singleton()->process_double_tap(p_button_mask, Point2(p_x, p_y)); + input_handler->process_double_tap(p_button_mask, Point2(p_x, p_y)); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_scroll(JNIEnv *env, jclass clazz, jint p_x, jint p_y) { - if (step == 0) + if (step.get() <= 0) return; - DisplayServerAndroid::get_singleton()->process_scroll(Point2(p_x, p_y)); + input_handler->process_scroll(Point2(p_x, p_y)); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_joybutton(JNIEnv *env, jclass clazz, jint p_device, jint p_button, jboolean p_pressed) { - if (step == 0) + if (step.get() <= 0) return; - DisplayServerAndroid::JoypadEvent jevent; + AndroidInputHandler::JoypadEvent jevent; jevent.device = p_device; - jevent.type = DisplayServerAndroid::JOY_EVENT_BUTTON; + jevent.type = AndroidInputHandler::JOY_EVENT_BUTTON; jevent.index = p_button; jevent.pressed = p_pressed; - DisplayServerAndroid::get_singleton()->process_joy_event(jevent); + input_handler->process_joy_event(jevent); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_joyaxis(JNIEnv *env, jclass clazz, jint p_device, jint p_axis, jfloat p_value) { - if (step == 0) + if (step.get() <= 0) return; - DisplayServerAndroid::JoypadEvent jevent; + AndroidInputHandler::JoypadEvent jevent; jevent.device = p_device; - jevent.type = DisplayServerAndroid::JOY_EVENT_AXIS; + jevent.type = AndroidInputHandler::JOY_EVENT_AXIS; jevent.index = p_axis; jevent.value = p_value; - DisplayServerAndroid::get_singleton()->process_joy_event(jevent); + input_handler->process_joy_event(jevent); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_joyhat(JNIEnv *env, jclass clazz, jint p_device, jint p_hat_x, jint p_hat_y) { - if (step == 0) + if (step.get() <= 0) return; - DisplayServerAndroid::JoypadEvent jevent; + AndroidInputHandler::JoypadEvent jevent; jevent.device = p_device; - jevent.type = DisplayServerAndroid::JOY_EVENT_HAT; + jevent.type = AndroidInputHandler::JOY_EVENT_HAT; int hat = 0; if (p_hat_x != 0) { if (p_hat_x < 0) @@ -336,9 +353,10 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_joyhat(JNIEnv *env, j } jevent.hat = hat; - DisplayServerAndroid::get_singleton()->process_joy_event(jevent); + input_handler->process_joy_event(jevent); } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_joyconnectionchanged(JNIEnv *env, jclass clazz, jint p_device, jboolean p_connected, jstring p_name) { if (os_android) { String name = jstring_to_string(p_name, env); @@ -346,11 +364,12 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_joyconnectionchanged( } } +// Called on the UI thread JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_key(JNIEnv *env, jclass clazz, jint p_keycode, jint p_scancode, jint p_unicode_char, jboolean p_pressed) { - if (step == 0) + if (step.get() <= 0) return; - DisplayServerAndroid::get_singleton()->process_key_event(p_keycode, p_scancode, p_unicode_char, p_pressed); + input_handler->process_key_event(p_keycode, p_scancode, p_unicode_char, p_pressed); } JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_accelerometer(JNIEnv *env, jclass clazz, jfloat x, jfloat y, jfloat z) { @@ -370,14 +389,14 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_gyroscope(JNIEnv *env } JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_focusin(JNIEnv *env, jclass clazz) { - if (step == 0) + if (step.get() <= 0) return; os_android->main_loop_focusin(); } JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_focusout(JNIEnv *env, jclass clazz) { - if (step == 0) + if (step.get() <= 0) return; os_android->main_loop_focusout(); @@ -456,7 +475,7 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_requestPermissionResu } JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_onRendererResumed(JNIEnv *env, jclass clazz) { - if (step == 0) + if (step.get() <= 0) return; if (os_android->get_main_loop()) { @@ -465,7 +484,7 @@ JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_onRendererResumed(JNI } JNIEXPORT void JNICALL Java_org_godotengine_godot_GodotLib_onRendererPaused(JNIEnv *env, jclass clazz) { - if (step == 0) + if (step.get() <= 0) return; if (os_android->get_main_loop()) { diff --git a/platform/android/os_android.cpp b/platform/android/os_android.cpp index 792a390e36..c2e12442b3 100644 --- a/platform/android/os_android.cpp +++ b/platform/android/os_android.cpp @@ -188,10 +188,6 @@ void OS_Android::main_loop_focusin() { audio_driver_android.set_pause(false); } -void OS_Android::main_loop_request_go_back() { - DisplayServerAndroid::get_singleton()->send_window_event(DisplayServer::WINDOW_EVENT_GO_BACK_REQUEST); -} - Error OS_Android::shell_open(String p_uri) { return godot_io_java->open_uri(p_uri); } diff --git a/platform/android/os_android.h b/platform/android/os_android.h index 38f0f3edc7..a5b995a775 100644 --- a/platform/android/os_android.h +++ b/platform/android/os_android.h @@ -95,7 +95,6 @@ public: void main_loop_begin(); bool main_loop_iterate(); - void main_loop_request_go_back(); void main_loop_end(); void main_loop_focusout(); void main_loop_focusin(); diff --git a/platform/android/vulkan/vulkan_context_android.cpp b/platform/android/vulkan/vulkan_context_android.cpp index f06a30f3a4..e24d1a4527 100644 --- a/platform/android/vulkan/vulkan_context_android.cpp +++ b/platform/android/vulkan/vulkan_context_android.cpp @@ -48,7 +48,7 @@ int VulkanContextAndroid::window_create(ANativeWindow *p_window, DisplayServer:: createInfo.window = p_window; VkSurfaceKHR surface; - VkResult err = vkCreateAndroidSurfaceKHR(_get_instance(), &createInfo, nullptr, &surface); + VkResult err = vkCreateAndroidSurfaceKHR(get_instance(), &createInfo, nullptr, &surface); if (err != VK_SUCCESS) { ERR_FAIL_V_MSG(-1, "vkCreateAndroidSurfaceKHR failed with error " + itos(err)); } diff --git a/platform/iphone/vulkan_context_iphone.mm b/platform/iphone/vulkan_context_iphone.mm index 5432cdc033..547cee9570 100644 --- a/platform/iphone/vulkan_context_iphone.mm +++ b/platform/iphone/vulkan_context_iphone.mm @@ -48,7 +48,7 @@ Error VulkanContextIPhone::window_create(DisplayServer::WindowID p_window_id, Di VkSurfaceKHR surface; VkResult err = - vkCreateIOSSurfaceMVK(_get_instance(), &createInfo, nullptr, &surface); + vkCreateIOSSurfaceMVK(get_instance(), &createInfo, nullptr, &surface); ERR_FAIL_COND_V(err, ERR_CANT_CREATE); return _window_create(p_window_id, p_vsync_mode, surface, p_width, p_height); diff --git a/platform/linuxbsd/display_server_x11.cpp b/platform/linuxbsd/display_server_x11.cpp index 5b26b23806..a39941339a 100644 --- a/platform/linuxbsd/display_server_x11.cpp +++ b/platform/linuxbsd/display_server_x11.cpp @@ -2247,7 +2247,7 @@ void DisplayServerX11::_handle_key_event(WindowID p_window, XKeyEvent *p_event, k->set_shift_pressed(true); } - Input::get_singleton()->accumulate_input_event(k); + Input::get_singleton()->parse_input_event(k); } memfree(utf8string); return; @@ -2396,7 +2396,7 @@ void DisplayServerX11::_handle_key_event(WindowID p_window, XKeyEvent *p_event, } } - Input::get_singleton()->accumulate_input_event(k); + Input::get_singleton()->parse_input_event(k); } Atom DisplayServerX11::_process_selection_request_target(Atom p_target, Window p_requestor, Atom p_property) const { @@ -2883,13 +2883,13 @@ void DisplayServerX11::process_events() { // in a spurious mouse motion event being sent to Godot; remember it to be able to filter it out xi.mouse_pos_to_filter = pos; } - Input::get_singleton()->accumulate_input_event(st); + Input::get_singleton()->parse_input_event(st); } else { if (!xi.state.has(index)) { // Defensive break; } xi.state.erase(index); - Input::get_singleton()->accumulate_input_event(st); + Input::get_singleton()->parse_input_event(st); } } break; @@ -2906,7 +2906,7 @@ void DisplayServerX11::process_events() { sd->set_index(index); sd->set_position(pos); sd->set_relative(pos - curr_pos_elem->value()); - Input::get_singleton()->accumulate_input_event(sd); + Input::get_singleton()->parse_input_event(sd); curr_pos_elem->value() = pos; } @@ -3058,7 +3058,7 @@ void DisplayServerX11::process_events() { st->set_index(E->key()); st->set_window_id(window_id); st->set_position(E->get()); - Input::get_singleton()->accumulate_input_event(st); + Input::get_singleton()->parse_input_event(st); } xi.state.clear(); #endif @@ -3156,7 +3156,7 @@ void DisplayServerX11::process_events() { mb->set_window_id(window_id_other); mb->set_position(Vector2(x, y)); mb->set_global_position(mb->get_position()); - Input::get_singleton()->accumulate_input_event(mb); + Input::get_singleton()->parse_input_event(mb); } break; } @@ -3164,7 +3164,7 @@ void DisplayServerX11::process_events() { } } - Input::get_singleton()->accumulate_input_event(mb); + Input::get_singleton()->parse_input_event(mb); } break; case MotionNotify: { @@ -3280,7 +3280,7 @@ void DisplayServerX11::process_events() { // this is so that the relative motion doesn't get messed up // after we regain focus. if (focused) { - Input::get_singleton()->accumulate_input_event(mm); + Input::get_singleton()->parse_input_event(mm); } else { // Propagate the event to the focused window, // because it's received only on the topmost window. @@ -3300,7 +3300,7 @@ void DisplayServerX11::process_events() { mm->set_position(pos_focused); mm->set_global_position(pos_focused); mm->set_speed(Input::get_singleton()->get_last_mouse_speed()); - Input::get_singleton()->accumulate_input_event(mm); + Input::get_singleton()->parse_input_event(mm); break; } @@ -3433,7 +3433,7 @@ void DisplayServerX11::process_events() { */ } - Input::get_singleton()->flush_accumulated_events(); + Input::get_singleton()->flush_buffered_events(); } void DisplayServerX11::release_rendering_thread() { diff --git a/platform/linuxbsd/vulkan_context_x11.cpp b/platform/linuxbsd/vulkan_context_x11.cpp index ebaef984e0..4d58e4999b 100644 --- a/platform/linuxbsd/vulkan_context_x11.cpp +++ b/platform/linuxbsd/vulkan_context_x11.cpp @@ -48,7 +48,7 @@ Error VulkanContextX11::window_create(DisplayServer::WindowID p_window_id, Displ createInfo.window = p_window; VkSurfaceKHR surface; - VkResult err = vkCreateXlibSurfaceKHR(_get_instance(), &createInfo, nullptr, &surface); + VkResult err = vkCreateXlibSurfaceKHR(get_instance(), &createInfo, nullptr, &surface); ERR_FAIL_COND_V(err, ERR_CANT_CREATE); return _window_create(p_window_id, p_vsync_mode, surface, p_width, p_height); } diff --git a/platform/osx/display_server_osx.mm b/platform/osx/display_server_osx.mm index cc8400980a..43b7d7c1e0 100644 --- a/platform/osx/display_server_osx.mm +++ b/platform/osx/display_server_osx.mm @@ -690,7 +690,7 @@ static void _mouseDownEvent(DisplayServer::WindowID window_id, NSEvent *event, M mb->set_double_click([event clickCount] == 2); } - Input::get_singleton()->accumulate_input_event(mb); + Input::get_singleton()->parse_input_event(mb); } - (void)mouseDown:(NSEvent *)event { @@ -799,7 +799,7 @@ static void _mouseDownEvent(DisplayServer::WindowID window_id, NSEvent *event, M _get_key_modifier_state([event modifierFlags], mm); Input::get_singleton()->set_mouse_position(wd.mouse_pos); - Input::get_singleton()->accumulate_input_event(mm); + Input::get_singleton()->parse_input_event(mm); } - (void)rightMouseDown:(NSEvent *)event { @@ -875,7 +875,7 @@ static void _mouseDownEvent(DisplayServer::WindowID window_id, NSEvent *event, M ev->set_position(_get_mouse_pos(wd, [event locationInWindow])); ev->set_factor([event magnification] + 1.0); - Input::get_singleton()->accumulate_input_event(ev); + Input::get_singleton()->parse_input_event(ev); } - (void)viewDidChangeBackingProperties { @@ -1357,7 +1357,7 @@ inline void sendScrollEvent(DisplayServer::WindowID window_id, MouseButton butto DS_OSX->last_button_state |= (MouseButton)mask; sc->set_button_mask(DS_OSX->last_button_state); - Input::get_singleton()->accumulate_input_event(sc); + Input::get_singleton()->parse_input_event(sc); sc.instantiate(); sc->set_window_id(window_id); @@ -1369,7 +1369,7 @@ inline void sendScrollEvent(DisplayServer::WindowID window_id, MouseButton butto DS_OSX->last_button_state &= (MouseButton)~mask; sc->set_button_mask(DS_OSX->last_button_state); - Input::get_singleton()->accumulate_input_event(sc); + Input::get_singleton()->parse_input_event(sc); } inline void sendPanEvent(DisplayServer::WindowID window_id, double dx, double dy, int modifierFlags) { @@ -1384,7 +1384,7 @@ inline void sendPanEvent(DisplayServer::WindowID window_id, double dx, double dy pg->set_position(wd.mouse_pos); pg->set_delta(Vector2(-dx, -dy)); - Input::get_singleton()->accumulate_input_event(pg); + Input::get_singleton()->parse_input_event(pg); } - (void)scrollWheel:(NSEvent *)event { @@ -3198,7 +3198,7 @@ String DisplayServerOSX::keyboard_get_layout_name(int p_index) const { void DisplayServerOSX::_push_input(const Ref<InputEvent> &p_event) { Ref<InputEvent> ev = p_event; - Input::get_singleton()->accumulate_input_event(ev); + Input::get_singleton()->parse_input_event(ev); } void DisplayServerOSX::_release_pressed_events() { @@ -3253,7 +3253,7 @@ void DisplayServerOSX::_send_event(NSEvent *p_event) { k->set_physical_keycode(KEY_PERIOD); k->set_echo([p_event isARepeat]); - Input::get_singleton()->accumulate_input_event(k); + Input::get_singleton()->parse_input_event(k); } } } @@ -3331,7 +3331,7 @@ void DisplayServerOSX::process_events() { if (!drop_events) { _process_key_events(); - Input::get_singleton()->flush_accumulated_events(); + Input::get_singleton()->flush_buffered_events(); } for (Map<WindowID, WindowData>::Element *E = windows.front(); E; E = E->next()) { diff --git a/platform/osx/export/export_plugin.cpp b/platform/osx/export/export_plugin.cpp index 53ae1ea6fe..5b959d6da4 100644 --- a/platform/osx/export/export_plugin.cpp +++ b/platform/osx/export/export_plugin.cpp @@ -193,16 +193,16 @@ void EditorExportPlatformOSX::_make_icon(const Ref<Image> &p_icon, Vector<uint8_ }; static const MacOSIconInfo icon_infos[] = { - { "ic10", "", true, 1024 }, //1024x1024 32-bit PNG and 512x512@2x 32-bit "retina" PNG + { "ic10", "", true, 1024 }, //1024×1024 32-bit PNG and 512×512@2x 32-bit "retina" PNG { "ic09", "", true, 512 }, //512×512 32-bit PNG - { "ic14", "", true, 512 }, //256x256@2x 32-bit "retina" PNG + { "ic14", "", true, 512 }, //256×256@2x 32-bit "retina" PNG { "ic08", "", true, 256 }, //256×256 32-bit PNG - { "ic13", "", true, 256 }, //128x128@2x 32-bit "retina" PNG - { "ic07", "", true, 128 }, //128x128 32-bit PNG - { "ic12", "", true, 64 }, //32x32@2x 32-bit "retina" PNG - { "ic11", "", true, 32 }, //16x16@2x 32-bit "retina" PNG - { "il32", "l8mk", false, 32 }, //32x32 24-bit RLE + 8-bit uncompressed mask - { "is32", "s8mk", false, 16 } //16x16 24-bit RLE + 8-bit uncompressed mask + { "ic13", "", true, 256 }, //128×128@2x 32-bit "retina" PNG + { "ic07", "", true, 128 }, //128×128 32-bit PNG + { "ic12", "", true, 64 }, //32×32@2× 32-bit "retina" PNG + { "ic11", "", true, 32 }, //16×16@2× 32-bit "retina" PNG + { "il32", "l8mk", false, 32 }, //32×32 24-bit RLE + 8-bit uncompressed mask + { "is32", "s8mk", false, 16 } //16×16 24-bit RLE + 8-bit uncompressed mask }; for (uint64_t i = 0; i < (sizeof(icon_infos) / sizeof(icon_infos[0])); ++i) { diff --git a/platform/osx/vulkan_context_osx.mm b/platform/osx/vulkan_context_osx.mm index 504955149b..36c02c2497 100644 --- a/platform/osx/vulkan_context_osx.mm +++ b/platform/osx/vulkan_context_osx.mm @@ -47,7 +47,7 @@ Error VulkanContextOSX::window_create(DisplayServer::WindowID p_window_id, Displ createInfo.pView = p_window; VkSurfaceKHR surface; - VkResult err = vkCreateMacOSSurfaceMVK(_get_instance(), &createInfo, nullptr, &surface); + VkResult err = vkCreateMacOSSurfaceMVK(get_instance(), &createInfo, nullptr, &surface); ERR_FAIL_COND_V(err, ERR_CANT_CREATE); return _window_create(p_window_id, p_vsync_mode, surface, p_width, p_height); } diff --git a/platform/windows/display_server_windows.cpp b/platform/windows/display_server_windows.cpp index 42a517879b..b6489e7a95 100644 --- a/platform/windows/display_server_windows.cpp +++ b/platform/windows/display_server_windows.cpp @@ -1527,7 +1527,7 @@ void DisplayServerWindows::process_events() { if (!drop_events) { _process_key_events(); - Input::get_singleton()->flush_accumulated_events(); + Input::get_singleton()->flush_buffered_events(); } } @@ -1738,7 +1738,7 @@ void DisplayServerWindows::_touch_event(WindowID p_window, bool p_pressed, float event->set_pressed(p_pressed); event->set_position(Vector2(p_x, p_y)); - Input::get_singleton()->accumulate_input_event(event); + Input::get_singleton()->parse_input_event(event); } void DisplayServerWindows::_drag_event(WindowID p_window, float p_x, float p_y, int idx) { @@ -1757,7 +1757,7 @@ void DisplayServerWindows::_drag_event(WindowID p_window, float p_x, float p_y, event->set_position(Vector2(p_x, p_y)); event->set_relative(Vector2(p_x, p_y) - curr->get()); - Input::get_singleton()->accumulate_input_event(event); + Input::get_singleton()->parse_input_event(event); curr->get() = Vector2(p_x, p_y); } @@ -2022,7 +2022,7 @@ LRESULT DisplayServerWindows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARA } if (windows[window_id].window_has_focus && mm->get_relative() != Vector2()) - Input::get_singleton()->accumulate_input_event(mm); + Input::get_singleton()->parse_input_event(mm); } delete[] lpb; } break; @@ -2111,7 +2111,7 @@ LRESULT DisplayServerWindows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARA old_x = mm->get_position().x; old_y = mm->get_position().y; if (windows[window_id].window_has_focus) - Input::get_singleton()->accumulate_input_event(mm); + Input::get_singleton()->parse_input_event(mm); } return 0; } @@ -2258,7 +2258,7 @@ LRESULT DisplayServerWindows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARA old_x = mm->get_position().x; old_y = mm->get_position().y; if (windows[window_id].window_has_focus) { - Input::get_singleton()->accumulate_input_event(mm); + Input::get_singleton()->parse_input_event(mm); } return 0; //Pointer event handled return 0 to avoid duplicate WM_MOUSEMOVE event @@ -2364,7 +2364,7 @@ LRESULT DisplayServerWindows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARA old_x = mm->get_position().x; old_y = mm->get_position().y; if (windows[window_id].window_has_focus) - Input::get_singleton()->accumulate_input_event(mm); + Input::get_singleton()->parse_input_event(mm); } break; case WM_LBUTTONDOWN: @@ -2533,7 +2533,7 @@ LRESULT DisplayServerWindows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARA mb->set_global_position(mb->get_position()); - Input::get_singleton()->accumulate_input_event(mb); + Input::get_singleton()->parse_input_event(mb); if (mb->is_pressed() && mb->get_button_index() > 3 && mb->get_button_index() < 8) { //send release for mouse wheel Ref<InputEventMouseButton> mbd = mb->duplicate(); @@ -2541,7 +2541,7 @@ LRESULT DisplayServerWindows::WndProc(HWND hWnd, UINT uMsg, WPARAM wParam, LPARA last_button_state &= (MouseButton) ~(1 << (mbd->get_button_index() - 1)); mbd->set_button_mask(last_button_state); mbd->set_pressed(false); - Input::get_singleton()->accumulate_input_event(mbd); + Input::get_singleton()->parse_input_event(mbd); } } break; @@ -2866,7 +2866,7 @@ void DisplayServerWindows::_process_key_events() { if (k->get_unicode() < 32) k->set_unicode(0); - Input::get_singleton()->accumulate_input_event(k); + Input::get_singleton()->parse_input_event(k); } //do nothing @@ -2924,7 +2924,7 @@ void DisplayServerWindows::_process_key_events() { k->set_echo((ke.uMsg == WM_KEYDOWN && (ke.lParam & (1 << 30)))); - Input::get_singleton()->accumulate_input_event(k); + Input::get_singleton()->parse_input_event(k); } break; } diff --git a/platform/windows/vulkan_context_win.cpp b/platform/windows/vulkan_context_win.cpp index d720c7e8fe..db5e6466be 100644 --- a/platform/windows/vulkan_context_win.cpp +++ b/platform/windows/vulkan_context_win.cpp @@ -47,7 +47,7 @@ int VulkanContextWindows::window_create(DisplayServer::WindowID p_window_id, Dis createInfo.hinstance = p_instance; createInfo.hwnd = p_window; VkSurfaceKHR surface; - VkResult err = vkCreateWin32SurfaceKHR(_get_instance(), &createInfo, nullptr, &surface); + VkResult err = vkCreateWin32SurfaceKHR(get_instance(), &createInfo, nullptr, &surface); ERR_FAIL_COND_V(err, -1); return _window_create(p_window_id, p_vsync_mode, surface, p_width, p_height); } diff --git a/scene/3d/visual_instance_3d.cpp b/scene/3d/visual_instance_3d.cpp index 28560641eb..73c2887983 100644 --- a/scene/3d/visual_instance_3d.cpp +++ b/scene/3d/visual_instance_3d.cpp @@ -414,7 +414,7 @@ void GeometryInstance3D::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::BOOL, "ignore_occlusion_culling"), "set_ignore_occlusion_culling", "is_ignoring_occlusion_culling"); ADD_GROUP("Global Illumination", "gi_"); ADD_PROPERTY(PropertyInfo(Variant::INT, "gi_mode", PROPERTY_HINT_ENUM, "Disabled,Baked,Dynamic"), "set_gi_mode", "get_gi_mode"); - ADD_PROPERTY(PropertyInfo(Variant::INT, "gi_lightmap_scale", PROPERTY_HINT_ENUM, "1x,2x,4x,8x"), "set_lightmap_scale", "get_lightmap_scale"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "gi_lightmap_scale", PROPERTY_HINT_ENUM, String::utf8("1×,2×,4×,8×")), "set_lightmap_scale", "get_lightmap_scale"); ADD_GROUP("Visibility Range", "visibility_range_"); ADD_PROPERTY(PropertyInfo(Variant::FLOAT, "visibility_range_begin", PROPERTY_HINT_RANGE, "0.0,4096.0,0.01"), "set_visibility_range_begin", "get_visibility_range_begin"); diff --git a/scene/main/scene_tree.cpp b/scene/main/scene_tree.cpp index 420516cbcd..a76b6c91ee 100644 --- a/scene/main/scene_tree.cpp +++ b/scene/main/scene_tree.cpp @@ -1346,7 +1346,7 @@ SceneTree::SceneTree() { current_scene = nullptr; const int msaa_mode = GLOBAL_DEF("rendering/anti_aliasing/quality/msaa", 0); - ProjectSettings::get_singleton()->set_custom_property_info("rendering/anti_aliasing/quality/msaa", PropertyInfo(Variant::INT, "rendering/anti_aliasing/quality/msaa", PROPERTY_HINT_ENUM, "Disabled (Fastest),2x (Fast),4x (Average),8x (Slow),16x (Slower)")); + ProjectSettings::get_singleton()->set_custom_property_info("rendering/anti_aliasing/quality/msaa", PropertyInfo(Variant::INT, "rendering/anti_aliasing/quality/msaa", PROPERTY_HINT_ENUM, String::utf8("Disabled (Fastest),2× (Fast),4× (Average),8× (Slow),16× (Slower)"))); root->set_msaa(Viewport::MSAA(msaa_mode)); const int ssaa_mode = GLOBAL_DEF("rendering/anti_aliasing/quality/screen_space_aa", 0); diff --git a/scene/main/viewport.cpp b/scene/main/viewport.cpp index 146502db3b..b85f1cc0b8 100644 --- a/scene/main/viewport.cpp +++ b/scene/main/viewport.cpp @@ -3593,7 +3593,7 @@ void Viewport::_bind_methods() { ADD_PROPERTY(PropertyInfo(Variant::BOOL, "snap_2d_transforms_to_pixel"), "set_snap_2d_transforms_to_pixel", "is_snap_2d_transforms_to_pixel_enabled"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "snap_2d_vertices_to_pixel"), "set_snap_2d_vertices_to_pixel", "is_snap_2d_vertices_to_pixel_enabled"); ADD_GROUP("Rendering", ""); - ADD_PROPERTY(PropertyInfo(Variant::INT, "msaa", PROPERTY_HINT_ENUM, "Disabled (Fastest),2x (Fast),4x (Average),8x (Slow),16x (Slower)"), "set_msaa", "get_msaa"); + ADD_PROPERTY(PropertyInfo(Variant::INT, "msaa", PROPERTY_HINT_ENUM, String::utf8("Disabled (Fastest),2× (Fast),4× (Average),8× (Slow),16× (Slower)")), "set_msaa", "get_msaa"); ADD_PROPERTY(PropertyInfo(Variant::INT, "screen_space_aa", PROPERTY_HINT_ENUM, "Disabled (Fastest),FXAA (Fast)"), "set_screen_space_aa", "get_screen_space_aa"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "use_debanding"), "set_use_debanding", "is_using_debanding"); ADD_PROPERTY(PropertyInfo(Variant::BOOL, "use_occlusion_culling"), "set_use_occlusion_culling", "is_using_occlusion_culling"); diff --git a/scene/resources/texture.cpp b/scene/resources/texture.cpp index 2ea55843ad..4ad5f2a506 100644 --- a/scene/resources/texture.cpp +++ b/scene/resources/texture.cpp @@ -2587,7 +2587,10 @@ RID CameraTexture::get_rid() const { if (feed.is_valid()) { return feed->get_texture(which_feed); } else { - return RID(); + if (_texture.is_null()) { + _texture = RenderingServer::get_singleton()->texture_2d_placeholder_create(); + } + return _texture; } } @@ -2643,5 +2646,7 @@ bool CameraTexture::get_camera_active() const { CameraTexture::CameraTexture() {} CameraTexture::~CameraTexture() { - // nothing to do here yet + if (_texture.is_valid()) { + RenderingServer::get_singleton()->free(_texture); + } } diff --git a/scene/resources/texture.h b/scene/resources/texture.h index 98aa61138d..2e97c2deb1 100644 --- a/scene/resources/texture.h +++ b/scene/resources/texture.h @@ -812,6 +812,7 @@ class CameraTexture : public Texture2D { GDCLASS(CameraTexture, Texture2D); private: + mutable RID _texture; int camera_feed_id = 0; CameraServer::FeedImage which_feed = CameraServer::FEED_RGBA_IMAGE; diff --git a/servers/camera/camera_feed.cpp b/servers/camera/camera_feed.cpp index 13afc803e4..9f8e8a8106 100644 --- a/servers/camera/camera_feed.cpp +++ b/servers/camera/camera_feed.cpp @@ -138,11 +138,15 @@ RID CameraFeed::get_texture(CameraServer::FeedImage p_which) { CameraFeed::CameraFeed() { // initialize our feed id = CameraServer::get_singleton()->get_free_id(); + base_width = 0; + base_height = 0; name = "???"; active = false; datatype = CameraFeed::FEED_RGB; position = CameraFeed::FEED_UNSPECIFIED; transform = Transform2D(1.0, 0.0, 0.0, -1.0, 0.0, 1.0); + texture[CameraServer::FEED_Y_IMAGE] = RenderingServer::get_singleton()->texture_2d_placeholder_create(); + texture[CameraServer::FEED_CBCR_IMAGE] = RenderingServer::get_singleton()->texture_2d_placeholder_create(); } CameraFeed::CameraFeed(String p_name, FeedPosition p_position) { @@ -155,16 +159,14 @@ CameraFeed::CameraFeed(String p_name, FeedPosition p_position) { datatype = CameraFeed::FEED_NOIMAGE; position = p_position; transform = Transform2D(1.0, 0.0, 0.0, -1.0, 0.0, 1.0); + texture[CameraServer::FEED_Y_IMAGE] = RenderingServer::get_singleton()->texture_2d_placeholder_create(); + texture[CameraServer::FEED_CBCR_IMAGE] = RenderingServer::get_singleton()->texture_2d_placeholder_create(); } CameraFeed::~CameraFeed() { // Free our textures - if (texture[CameraServer::FEED_Y_IMAGE].is_valid()) { - RenderingServer::get_singleton()->free(texture[CameraServer::FEED_Y_IMAGE]); - } - if (texture[CameraServer::FEED_CBCR_IMAGE].is_valid()) { - RenderingServer::get_singleton()->free(texture[CameraServer::FEED_CBCR_IMAGE]); - } + RenderingServer::get_singleton()->free(texture[CameraServer::FEED_Y_IMAGE]); + RenderingServer::get_singleton()->free(texture[CameraServer::FEED_CBCR_IMAGE]); } void CameraFeed::set_RGB_img(const Ref<Image> &p_rgb_img) { @@ -177,12 +179,9 @@ void CameraFeed::set_RGB_img(const Ref<Image> &p_rgb_img) { // We're assuming here that our camera image doesn't change around formats etc, allocate the whole lot... base_width = new_width; base_height = new_height; - if (texture[CameraServer::FEED_RGBA_IMAGE].is_null()) { - texture[CameraServer::FEED_RGBA_IMAGE] = RenderingServer::get_singleton()->texture_2d_create(p_rgb_img); - } else { - RID new_texture = RenderingServer::get_singleton()->texture_2d_create(p_rgb_img); - RenderingServer::get_singleton()->texture_replace(texture[CameraServer::FEED_RGBA_IMAGE], new_texture); - } + + RID new_texture = RenderingServer::get_singleton()->texture_2d_create(p_rgb_img); + RenderingServer::get_singleton()->texture_replace(texture[CameraServer::FEED_RGBA_IMAGE], new_texture); } else { RenderingServer::get_singleton()->texture_2d_update(texture[CameraServer::FEED_RGBA_IMAGE], p_rgb_img); } @@ -201,12 +200,9 @@ void CameraFeed::set_YCbCr_img(const Ref<Image> &p_ycbcr_img) { // We're assuming here that our camera image doesn't change around formats etc, allocate the whole lot... base_width = new_width; base_height = new_height; - if (texture[CameraServer::FEED_RGBA_IMAGE].is_null()) { - texture[CameraServer::FEED_RGBA_IMAGE] = RenderingServer::get_singleton()->texture_2d_create(p_ycbcr_img); - } else { - RID new_texture = RenderingServer::get_singleton()->texture_2d_create(p_ycbcr_img); - RenderingServer::get_singleton()->texture_replace(texture[CameraServer::FEED_RGBA_IMAGE], new_texture); - } + + RID new_texture = RenderingServer::get_singleton()->texture_2d_create(p_ycbcr_img); + RenderingServer::get_singleton()->texture_replace(texture[CameraServer::FEED_RGBA_IMAGE], new_texture); } else { RenderingServer::get_singleton()->texture_2d_update(texture[CameraServer::FEED_RGBA_IMAGE], p_ycbcr_img); } @@ -230,16 +226,11 @@ void CameraFeed::set_YCbCr_imgs(const Ref<Image> &p_y_img, const Ref<Image> &p_c // We're assuming here that our camera image doesn't change around formats etc, allocate the whole lot... base_width = new_y_width; base_height = new_y_height; - if (texture[CameraServer::FEED_Y_IMAGE].is_null()) { - texture[CameraServer::FEED_Y_IMAGE] = RenderingServer::get_singleton()->texture_2d_create(p_y_img); - } else { + { RID new_texture = RenderingServer::get_singleton()->texture_2d_create(p_y_img); RenderingServer::get_singleton()->texture_replace(texture[CameraServer::FEED_Y_IMAGE], new_texture); } - - if (texture[CameraServer::FEED_CBCR_IMAGE].is_null()) { - texture[CameraServer::FEED_CBCR_IMAGE] = RenderingServer::get_singleton()->texture_2d_create(p_cbcr_img); - } else { + { RID new_texture = RenderingServer::get_singleton()->texture_2d_create(p_cbcr_img); RenderingServer::get_singleton()->texture_replace(texture[CameraServer::FEED_CBCR_IMAGE], new_texture); } diff --git a/servers/rendering/renderer_rd/shader_compiler_rd.cpp b/servers/rendering/renderer_rd/shader_compiler_rd.cpp index 9c1068ea2e..8aa03b6426 100644 --- a/servers/rendering/renderer_rd/shader_compiler_rd.cpp +++ b/servers/rendering/renderer_rd/shader_compiler_rd.cpp @@ -887,7 +887,7 @@ String ShaderCompilerRD::_dump_node_code(const SL::Node *p_node, int p_level, Ge SL::VariableNode *vnode = (SL::VariableNode *)p_node; bool use_fragment_varying = false; - if (!(p_actions.entry_point_stages.has(current_func_name) && p_actions.entry_point_stages[current_func_name] == STAGE_VERTEX)) { + if (!vnode->is_local && !(p_actions.entry_point_stages.has(current_func_name) && p_actions.entry_point_stages[current_func_name] == STAGE_VERTEX)) { if (p_assigning) { if (shader->varyings.has(vnode->name)) { use_fragment_varying = true; @@ -1037,7 +1037,7 @@ String ShaderCompilerRD::_dump_node_code(const SL::Node *p_node, int p_level, Ge SL::ArrayNode *anode = (SL::ArrayNode *)p_node; bool use_fragment_varying = false; - if (!(p_actions.entry_point_stages.has(current_func_name) && p_actions.entry_point_stages[current_func_name] == STAGE_VERTEX)) { + if (!anode->is_local && !(p_actions.entry_point_stages.has(current_func_name) && p_actions.entry_point_stages[current_func_name] == STAGE_VERTEX)) { if (anode->assign_expression != nullptr && shader->varyings.has(anode->name)) { use_fragment_varying = true; } else { diff --git a/servers/rendering/shader_language.cpp b/servers/rendering/shader_language.cpp index 47c6317962..20fcb1396d 100644 --- a/servers/rendering/shader_language.cpp +++ b/servers/rendering/shader_language.cpp @@ -4228,6 +4228,7 @@ ShaderLanguage::Node *ShaderLanguage::_parse_expression(BlockNode *p_block, cons IdentifierType ident_type; int array_size = 0; StringName struct_name; + bool is_local = false; if (p_block && p_block->block_tag != SubClassTag::TAG_GLOBAL) { int idx = 0; @@ -4284,6 +4285,8 @@ ShaderLanguage::Node *ShaderLanguage::_parse_expression(BlockNode *p_block, cons } else { last_type = ident_type; } + + is_local = ident_type == IDENTIFIER_LOCAL_VAR || ident_type == IDENTIFIER_FUNCTION_ARGUMENT; } Node *index_expression = nullptr; @@ -4358,6 +4361,7 @@ ShaderLanguage::Node *ShaderLanguage::_parse_expression(BlockNode *p_block, cons arrname->assign_expression = assign_expression; arrname->is_const = is_const; arrname->array_size = array_size; + arrname->is_local = is_local; expr = arrname; } else { VariableNode *varname = alloc_node<VariableNode>(); @@ -4365,6 +4369,7 @@ ShaderLanguage::Node *ShaderLanguage::_parse_expression(BlockNode *p_block, cons varname->datatype_cache = data_type; varname->is_const = is_const; varname->struct_name = struct_name; + varname->is_local = is_local; expr = varname; } #ifdef DEBUG_ENABLED diff --git a/servers/rendering/shader_language.h b/servers/rendering/shader_language.h index fae7fa54aa..9e0a63f0f7 100644 --- a/servers/rendering/shader_language.h +++ b/servers/rendering/shader_language.h @@ -409,6 +409,7 @@ public: StringName name; StringName struct_name; bool is_const = false; + bool is_local = false; virtual DataType get_datatype() const override { return datatype_cache; } virtual String get_datatype_name() const override { return String(struct_name); } @@ -444,6 +445,7 @@ public: Node *assign_expression = nullptr; bool is_const = false; int array_size = 0; + bool is_local = false; virtual DataType get_datatype() const override { return datatype_cache; } virtual String get_datatype_name() const override { return String(struct_name); } diff --git a/servers/rendering_server.cpp b/servers/rendering_server.cpp index 6c4fb31efd..222ea9e622 100644 --- a/servers/rendering_server.cpp +++ b/servers/rendering_server.cpp @@ -2828,7 +2828,7 @@ RenderingServer::RenderingServer() { GLOBAL_DEF_RST("rendering/textures/default_filters/use_nearest_mipmap_filter", false); GLOBAL_DEF_RST("rendering/textures/default_filters/anisotropic_filtering_level", 2); - ProjectSettings::get_singleton()->set_custom_property_info("rendering/textures/default_filters/anisotropic_filtering_level", PropertyInfo(Variant::INT, "rendering/textures/default_filters/anisotropic_filtering_level", PROPERTY_HINT_ENUM, "Disabled (Fastest),2x (Faster),4x (Fast),8x (Average),16x (Slow)")); + ProjectSettings::get_singleton()->set_custom_property_info("rendering/textures/default_filters/anisotropic_filtering_level", PropertyInfo(Variant::INT, "rendering/textures/default_filters/anisotropic_filtering_level", PROPERTY_HINT_ENUM, String::utf8("Disabled (Fastest),2× (Faster),4× (Fast),8× (Average),16× (Slow)"))); GLOBAL_DEF("rendering/camera/depth_of_field/depth_of_field_bokeh_shape", 1); ProjectSettings::get_singleton()->set_custom_property_info("rendering/camera/depth_of_field/depth_of_field_bokeh_shape", PropertyInfo(Variant::INT, "rendering/camera/depth_of_field/depth_of_field_bokeh_shape", PROPERTY_HINT_ENUM, "Box (Fast),Hexagon (Average),Circle (Slowest)")); diff --git a/thirdparty/README.md b/thirdparty/README.md index ebf9c5ac0e..e5b4af627a 100644 --- a/thirdparty/README.md +++ b/thirdparty/README.md @@ -692,8 +692,8 @@ Files extracted from upstream source: SDK release: https://github.com/KhronosGroup/Vulkan-ValidationLayers/blob/master/layers/generated/vk_enum_string_helper.h `vk_mem_alloc.h` is taken from https://github.com/GPUOpen-LibrariesAndSDKs/VulkanMemoryAllocator -Version: 2.3.0 (2019) -`vk_mem_alloc.cpp` and `android/vk_mem_alloc.cpp` are Godot files and should be preserved on updates. +Version: 3.0.0-development (2021-06-21), branch `feature-small-buffers`, commit `cfea2f72851f9ee4a399769f18865047b83711f1` +`vk_mem_alloc.cpp` is a Godot file and should be preserved on updates. Patches in the `patches` directory should be re-applied after updates. diff --git a/thirdparty/vulkan/patches/VMA-assert-remove.patch b/thirdparty/vulkan/patches/VMA-assert-remove.patch deleted file mode 100644 index 3d57ab7d42..0000000000 --- a/thirdparty/vulkan/patches/VMA-assert-remove.patch +++ /dev/null @@ -1,29 +0,0 @@ -diff --git a/thirdparty/vulkan/vk_mem_alloc.h b/thirdparty/vulkan/vk_mem_alloc.h -index 0dfb66efc6..8a42699e7f 100644 ---- a/thirdparty/vulkan/vk_mem_alloc.h -+++ b/thirdparty/vulkan/vk_mem_alloc.h -@@ -17508,24 +17508,6 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateBuffer( - allocator->GetBufferMemoryRequirements(*pBuffer, vkMemReq, - requiresDedicatedAllocation, prefersDedicatedAllocation); - -- // Make sure alignment requirements for specific buffer usages reported -- // in Physical Device Properties are included in alignment reported by memory requirements. -- if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_TEXEL_BUFFER_BIT) != 0) -- { -- VMA_ASSERT(vkMemReq.alignment % -- allocator->m_PhysicalDeviceProperties.limits.minTexelBufferOffsetAlignment == 0); -- } -- if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_UNIFORM_BUFFER_BIT) != 0) -- { -- VMA_ASSERT(vkMemReq.alignment % -- allocator->m_PhysicalDeviceProperties.limits.minUniformBufferOffsetAlignment == 0); -- } -- if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_STORAGE_BUFFER_BIT) != 0) -- { -- VMA_ASSERT(vkMemReq.alignment % -- allocator->m_PhysicalDeviceProperties.limits.minStorageBufferOffsetAlignment == 0); -- } -- - // 3. Allocate memory using allocator. - res = allocator->AllocateMemory( - vkMemReq, diff --git a/thirdparty/vulkan/vk_mem_alloc.h b/thirdparty/vulkan/vk_mem_alloc.h index 26f9faa6e4..9890f20f7c 100644 --- a/thirdparty/vulkan/vk_mem_alloc.h +++ b/thirdparty/vulkan/vk_mem_alloc.h @@ -1,5 +1,5 @@ // -// Copyright (c) 2017-2019 Advanced Micro Devices, Inc. All rights reserved. +// Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. // // Permission is hereby granted, free of charge, to any person obtaining a copy // of this software and associated documentation files (the "Software"), to deal @@ -23,15 +23,11 @@ #ifndef AMD_VULKAN_MEMORY_ALLOCATOR_H #define AMD_VULKAN_MEMORY_ALLOCATOR_H -#ifdef __cplusplus -extern "C" { -#endif - /** \mainpage Vulkan Memory Allocator -<b>Version 2.3.0</b> (2019-12-04) +<b>Version 3.0.0-development</b> (2021-06-21) -Copyright (c) 2017-2019 Advanced Micro Devices, Inc. All rights reserved. \n +Copyright (c) 2017-2021 Advanced Micro Devices, Inc. All rights reserved. \n License: MIT Documentation of all members: vk_mem_alloc.h @@ -57,6 +53,7 @@ Documentation of all members: vk_mem_alloc.h - \subpage staying_within_budget - [Querying for budget](@ref staying_within_budget_querying_for_budget) - [Controlling memory usage](@ref staying_within_budget_controlling_memory_usage) + - \subpage resource_aliasing - \subpage custom_memory_pools - [Choosing memory type index](@ref custom_memory_pools_MemTypeIndex) - [Linear allocation algorithm](@ref linear_algorithm) @@ -66,10 +63,10 @@ Documentation of all members: vk_mem_alloc.h - [Ring buffer](@ref linear_algorithm_ring_buffer) - [Buddy allocation algorithm](@ref buddy_algorithm) - \subpage defragmentation - - [Defragmenting CPU memory](@ref defragmentation_cpu) - - [Defragmenting GPU memory](@ref defragmentation_gpu) - - [Additional notes](@ref defragmentation_additional_notes) - - [Writing custom allocation algorithm](@ref defragmentation_custom_algorithm) + - [Defragmenting CPU memory](@ref defragmentation_cpu) + - [Defragmenting GPU memory](@ref defragmentation_gpu) + - [Additional notes](@ref defragmentation_additional_notes) + - [Writing custom allocation algorithm](@ref defragmentation_custom_algorithm) - \subpage lost_allocations - \subpage statistics - [Numeric statistics](@ref statistics_numeric_statistics) @@ -92,6 +89,8 @@ Documentation of all members: vk_mem_alloc.h - [Device memory allocation callbacks](@ref allocation_callbacks) - [Device heap memory limit](@ref heap_memory_limit) - \subpage vk_khr_dedicated_allocation + - \subpage enabling_buffer_device_address + - \subpage vk_amd_device_coherent_memory - \subpage general_considerations - [Thread safety](@ref general_considerations_thread_safety) - [Validation layer warnings](@ref general_considerations_validation_layer_warnings) @@ -123,7 +122,7 @@ To do it properly: -# Include "vk_mem_alloc.h" file in each CPP file where you want to use the library. This includes declarations of all members of the library. --# In exacly one CPP file define following macro before this include. +-# In exactly one CPP file define following macro before this include. It enables also internal definitions. \code @@ -143,24 +142,42 @@ before including these headers (like `WIN32_LEAN_AND_MEAN` or `WINVER` for Windows, `VK_USE_PLATFORM_WIN32_KHR` for Vulkan), you must define them before every `#include` of this library. +You may need to configure the way you import Vulkan functions. + +- By default, VMA assumes you you link statically with Vulkan API. If this is not the case, + `#define VMA_STATIC_VULKAN_FUNCTIONS 0` before `#include` of the VMA implementation and use another way. +- You can `#define VMA_DYNAMIC_VULKAN_FUNCTIONS 1` and make sure `vkGetInstanceProcAddr` and `vkGetDeviceProcAddr` globals are defined. + All the remaining Vulkan functions will be fetched automatically. +- Finally, you can provide your own pointers to all Vulkan functions needed by VMA using structure member + VmaAllocatorCreateInfo::pVulkanFunctions, if you fetched them in some custom way e.g. using some loader like [Volk](https://github.com/zeux/volk). + \section quick_start_initialization Initialization At program startup: --# Initialize Vulkan to have `VkPhysicalDevice` and `VkDevice` object. +-# Initialize Vulkan to have `VkPhysicalDevice`, `VkDevice` and `VkInstance` object. -# Fill VmaAllocatorCreateInfo structure and create #VmaAllocator object by calling vmaCreateAllocator(). \code VmaAllocatorCreateInfo allocatorInfo = {}; +allocatorInfo.vulkanApiVersion = VK_API_VERSION_1_2; allocatorInfo.physicalDevice = physicalDevice; allocatorInfo.device = device; +allocatorInfo.instance = instance; VmaAllocator allocator; vmaCreateAllocator(&allocatorInfo, &allocator); \endcode +Only members `physicalDevice`, `device`, `instance` are required. +However, you should inform the library which Vulkan version do you use by setting +VmaAllocatorCreateInfo::vulkanApiVersion and which extensions did you enable +by setting VmaAllocatorCreateInfo::flags (like #VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT for VK_KHR_buffer_device_address). +Otherwise, VMA would use only features of Vulkan 1.0 core with no extensions. + + \section quick_start_resource_allocation Resource allocation When you want to create a buffer or image: @@ -302,6 +319,7 @@ VmaAllocation allocation; vmaCreateBuffer(allocator, &bufferInfo, &allocInfo, &buffer, &allocation, nullptr); \endcode + \section choosing_memory_type_custom_memory_pools Custom memory pools If you allocate from custom memory pool, all the ways of specifying memory @@ -422,12 +440,10 @@ There are some exceptions though, when you should consider mapping memory only f block is migrated by WDDM to system RAM, which degrades performance. It doesn't matter if that particular memory block is actually used by the command buffer being submitted. -- On Mac/MoltenVK there is a known bug - [Issue #175](https://github.com/KhronosGroup/MoltenVK/issues/175) - which requires unmapping before GPU can see updated texture. - Keeping many large memory blocks mapped may impact performance or stability of some debugging tools. \section memory_mapping_cache_control Cache flush and invalidate - + Memory in Vulkan doesn't need to be unmapped before using it on GPU, but unless a memory types has `VK_MEMORY_PROPERTY_HOST_COHERENT_BIT` flag set, you need to manually **invalidate** cache before reading of mapped pointer @@ -436,7 +452,8 @@ Map/unmap operations don't do that automatically. Vulkan provides following functions for this purpose `vkFlushMappedMemoryRanges()`, `vkInvalidateMappedMemoryRanges()`, but this library provides more convenient functions that refer to given allocation object: vmaFlushAllocation(), -vmaInvalidateAllocation(). +vmaInvalidateAllocation(), +or multiple objects at once: vmaFlushAllocations(), vmaInvalidateAllocations(). Regions of memory specified for flush/invalidate must be aligned to `VkPhysicalDeviceLimits::nonCoherentAtomSize`. This is automatically ensured by the library. @@ -478,7 +495,7 @@ vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allo VkMemoryPropertyFlags memFlags; vmaGetMemoryTypeProperties(allocator, allocInfo.memoryType, &memFlags); -if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) == 0) +if((memFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0) { // Allocation ended up in mappable memory. You can map it and access it directly. void* mappedData; @@ -513,7 +530,7 @@ VmaAllocation alloc; VmaAllocationInfo allocInfo; vmaCreateBuffer(allocator, &bufCreateInfo, &allocCreateInfo, &buf, &alloc, &allocInfo); -if(allocInfo.pUserData != nullptr) +if(allocInfo.pMappedData != nullptr) { // Allocation ended up in mappable memory. // It's persistently mapped. You can access it directly. @@ -599,6 +616,114 @@ set to more than 0 will try to allocate memory blocks without checking whether t fit within budget. +\page resource_aliasing Resource aliasing (overlap) + +New explicit graphics APIs (Vulkan and Direct3D 12), thanks to manual memory +management, give an opportunity to alias (overlap) multiple resources in the +same region of memory - a feature not available in the old APIs (Direct3D 11, OpenGL). +It can be useful to save video memory, but it must be used with caution. + +For example, if you know the flow of your whole render frame in advance, you +are going to use some intermediate textures or buffers only during a small range of render passes, +and you know these ranges don't overlap in time, you can bind these resources to +the same place in memory, even if they have completely different parameters (width, height, format etc.). + + + +Such scenario is possible using VMA, but you need to create your images manually. +Then you need to calculate parameters of an allocation to be made using formula: + +- allocation size = max(size of each image) +- allocation alignment = max(alignment of each image) +- allocation memoryTypeBits = bitwise AND(memoryTypeBits of each image) + +Following example shows two different images bound to the same place in memory, +allocated to fit largest of them. + +\code +// A 512x512 texture to be sampled. +VkImageCreateInfo img1CreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; +img1CreateInfo.imageType = VK_IMAGE_TYPE_2D; +img1CreateInfo.extent.width = 512; +img1CreateInfo.extent.height = 512; +img1CreateInfo.extent.depth = 1; +img1CreateInfo.mipLevels = 10; +img1CreateInfo.arrayLayers = 1; +img1CreateInfo.format = VK_FORMAT_R8G8B8A8_SRGB; +img1CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; +img1CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; +img1CreateInfo.usage = VK_IMAGE_USAGE_TRANSFER_DST_BIT | VK_IMAGE_USAGE_SAMPLED_BIT; +img1CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + +// A full screen texture to be used as color attachment. +VkImageCreateInfo img2CreateInfo = { VK_STRUCTURE_TYPE_IMAGE_CREATE_INFO }; +img2CreateInfo.imageType = VK_IMAGE_TYPE_2D; +img2CreateInfo.extent.width = 1920; +img2CreateInfo.extent.height = 1080; +img2CreateInfo.extent.depth = 1; +img2CreateInfo.mipLevels = 1; +img2CreateInfo.arrayLayers = 1; +img2CreateInfo.format = VK_FORMAT_R8G8B8A8_UNORM; +img2CreateInfo.tiling = VK_IMAGE_TILING_OPTIMAL; +img2CreateInfo.initialLayout = VK_IMAGE_LAYOUT_UNDEFINED; +img2CreateInfo.usage = VK_IMAGE_USAGE_SAMPLED_BIT | VK_IMAGE_USAGE_COLOR_ATTACHMENT_BIT; +img2CreateInfo.samples = VK_SAMPLE_COUNT_1_BIT; + +VkImage img1; +res = vkCreateImage(device, &img1CreateInfo, nullptr, &img1); +VkImage img2; +res = vkCreateImage(device, &img2CreateInfo, nullptr, &img2); + +VkMemoryRequirements img1MemReq; +vkGetImageMemoryRequirements(device, img1, &img1MemReq); +VkMemoryRequirements img2MemReq; +vkGetImageMemoryRequirements(device, img2, &img2MemReq); + +VkMemoryRequirements finalMemReq = {}; +finalMemReq.size = std::max(img1MemReq.size, img2MemReq.size); +finalMemReq.alignment = std::max(img1MemReq.alignment, img2MemReq.alignment); +finalMemReq.memoryTypeBits = img1MemReq.memoryTypeBits & img2MemReq.memoryTypeBits; +// Validate if(finalMemReq.memoryTypeBits != 0) + +VmaAllocationCreateInfo allocCreateInfo = {}; +allocCreateInfo.usage = VMA_MEMORY_USAGE_GPU_ONLY; + +VmaAllocation alloc; +res = vmaAllocateMemory(allocator, &finalMemReq, &allocCreateInfo, &alloc, nullptr); + +res = vmaBindImageMemory(allocator, alloc, img1); +res = vmaBindImageMemory(allocator, alloc, img2); + +// You can use img1, img2 here, but not at the same time! + +vmaFreeMemory(allocator, alloc); +vkDestroyImage(allocator, img2, nullptr); +vkDestroyImage(allocator, img1, nullptr); +\endcode + +Remember that using resources that alias in memory requires proper synchronization. +You need to issue a memory barrier to make sure commands that use `img1` and `img2` +don't overlap on GPU timeline. +You also need to treat a resource after aliasing as uninitialized - containing garbage data. +For example, if you use `img1` and then want to use `img2`, you need to issue +an image memory barrier for `img2` with `oldLayout` = `VK_IMAGE_LAYOUT_UNDEFINED`. + +Additional considerations: + +- Vulkan also allows to interpret contents of memory between aliasing resources consistently in some cases. +See chapter 11.8. "Memory Aliasing" of Vulkan specification or `VK_IMAGE_CREATE_ALIAS_BIT` flag. +- You can create more complex layout where different images and buffers are bound +at different offsets inside one large allocation. For example, one can imagine +a big texture used in some render passes, aliasing with a set of many small buffers +used between in some further passes. To bind a resource at non-zero offset of an allocation, +use vmaBindBufferMemory2() / vmaBindImageMemory2(). +- Before allocating memory for the resources you want to alias, check `memoryTypeBits` +returned in memory requirements of each resource to make sure the bits overlap. +Some GPUs may expose multiple memory types suitable e.g. only for buffers or +images with `COLOR_ATTACHMENT` usage, so the sets of memory types supported by your +resources may be disjoint. Aliasing them is not possible in that case. + + \page custom_memory_pools Custom memory pools A memory pool contains a number of `VkDeviceMemory` blocks. @@ -822,7 +947,7 @@ allocations. To mitigate this problem, you can use defragmentation feature: structure #VmaDefragmentationInfo2, function vmaDefragmentationBegin(), vmaDefragmentationEnd(). -Given set of allocations, +Given set of allocations, this function can move them to compact used memory, ensure more continuous free space and possibly also free some `VkDeviceMemory` blocks. @@ -888,9 +1013,9 @@ for(uint32_t i = 0; i < allocCount; ++i) // Create new buffer with same parameters. VkBufferCreateInfo bufferInfo = ...; vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]); - + // You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning. - + // Bind new buffer to new memory region. Data contained in it is already moved. VmaAllocationInfo allocInfo; vmaGetAllocationInfo(allocator, allocations[i], &allocInfo); @@ -966,9 +1091,9 @@ for(uint32_t i = 0; i < allocCount; ++i) // Create new buffer with same parameters. VkBufferCreateInfo bufferInfo = ...; vkCreateBuffer(device, &bufferInfo, nullptr, &buffers[i]); - + // You can make dummy call to vkGetBufferMemoryRequirements here to silence validation layer warning. - + // Bind new buffer to new memory region. Data contained in it is already moved. VmaAllocationInfo allocInfo; vmaGetAllocationInfo(allocator, allocations[i], &allocInfo); @@ -1005,7 +1130,7 @@ See [Validation layer warnings](@ref general_considerations_validation_layer_war Please don't expect memory to be fully compacted after defragmentation. Algorithms inside are based on some heuristics that try to maximize number of Vulkan -memory blocks to make totally empty to release them, as well as to maximimze continuous +memory blocks to make totally empty to release them, as well as to maximize continuous empty space inside remaining blocks, while minimizing the number and size of allocations that need to be moved. Some fragmentation may still remain - this is normal. @@ -1268,6 +1393,9 @@ printf("Image name: %s\n", imageName); That string is also printed in JSON report created by vmaBuildStatsString(). +\note Passing string name to VMA allocation doesn't automatically set it to the Vulkan buffer or image created with it. +You must do it manually using an extension like VK_EXT_debug_utils, which is independent of this library. + \page debugging_memory_usage Debugging incorrect memory usage @@ -1359,7 +1487,7 @@ which indicates a serious bug. You can also explicitly request checking margins of all allocations in all memory blocks that belong to specified memory types by using function vmaCheckCorruption(), -or in memory blocks that belong to specified custom pool, by using function +or in memory blocks that belong to specified custom pool, by using function vmaCheckPoolCorruption(). Margin validation (corruption detection) works only for memory types that are @@ -1500,6 +1628,7 @@ This is a more complex situation. Different solutions are possible, and the best one depends on specific GPU type, but you can use this simple approach for the start. Prefer to write to such resource sequentially (e.g. using `memcpy`). Don't perform random access or any reads from it on CPU, as it may be very slow. +Also note that textures written directly from the host through a mapped pointer need to be in LINEAR not OPTIMAL layout. \subsection usage_patterns_readback Readback @@ -1532,17 +1661,17 @@ directly instead of submitting explicit transfer (see below). For resources that you frequently write on CPU and read on GPU, many solutions are possible: -# Create one copy in video memory using #VMA_MEMORY_USAGE_GPU_ONLY, - second copy in system memory using #VMA_MEMORY_USAGE_CPU_ONLY and submit explicit tranfer each time. --# Create just single copy using #VMA_MEMORY_USAGE_CPU_TO_GPU, map it and fill it on CPU, + second copy in system memory using #VMA_MEMORY_USAGE_CPU_ONLY and submit explicit transfer each time. +-# Create just a single copy using #VMA_MEMORY_USAGE_CPU_TO_GPU, map it and fill it on CPU, read it directly on GPU. --# Create just single copy using #VMA_MEMORY_USAGE_CPU_ONLY, map it and fill it on CPU, +-# Create just a single copy using #VMA_MEMORY_USAGE_CPU_ONLY, map it and fill it on CPU, read it directly on GPU. Which solution is the most efficient depends on your resource and especially on the GPU. It is best to measure it and then make the decision. Some general recommendations: -- On integrated graphics use (2) or (3) to avoid unnecesary time and memory overhead +- On integrated graphics use (2) or (3) to avoid unnecessary time and memory overhead related to using a second copy and making transfer. - For small resources (e.g. constant buffers) use (2). Discrete AMD cards have special 256 MiB pool of video memory that is directly mappable. @@ -1563,6 +1692,10 @@ solutions are possible: You should take some measurements to decide which option is faster in case of your specific resource. +Note that textures accessed directly from the host through a mapped pointer need to be in LINEAR layout, +which may slow down their usage on the device. +Textures accessed only by the device and transfer operations can use OPTIMAL layout. + If you don't want to specialize your code for specific types of GPUs, you can still make an simple optimization for cases when your resource ends up in mappable memory to use it directly in this case instead of creating CPU-side staging copy. @@ -1585,12 +1718,31 @@ and empty otherwise. \section config_Vulkan_functions Pointers to Vulkan functions -The library uses Vulkan functions straight from the `vulkan.h` header by default. -If you want to provide your own pointers to these functions, e.g. fetched using -`vkGetInstanceProcAddr()` and `vkGetDeviceProcAddr()`: +There are multiple ways to import pointers to Vulkan functions in the library. +In the simplest case you don't need to do anything. +If the compilation or linking of your program or the initialization of the #VmaAllocator +doesn't work for you, you can try to reconfigure it. + +First, the allocator tries to fetch pointers to Vulkan functions linked statically, +like this: + +\code +m_VulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkAllocateMemory; +\endcode + +If you want to disable this feature, set configuration macro: `#define VMA_STATIC_VULKAN_FUNCTIONS 0`. + +Second, you can provide the pointers yourself by setting member VmaAllocatorCreateInfo::pVulkanFunctions. +You can fetch them e.g. using functions `vkGetInstanceProcAddr` and `vkGetDeviceProcAddr` or +by using a helper library like [volk](https://github.com/zeux/volk). + +Third, VMA tries to fetch remaining pointers that are still null by calling +`vkGetInstanceProcAddr` and `vkGetDeviceProcAddr` on its own. +If you want to disable this feature, set configuration macro: `#define VMA_DYNAMIC_VULKAN_FUNCTIONS 0`. + +Finally, all the function pointers required by the library (considering selected +Vulkan version and enabled extensions) are checked with `VMA_ASSERT` if they are not null. --# Define `VMA_STATIC_VULKAN_FUNCTIONS 0`. --# Provide valid pointers through VmaAllocatorCreateInfo::pVulkanFunctions. \section custom_memory_allocator Custom host memory allocator @@ -1612,7 +1764,7 @@ VmaAllocatorCreateInfo::pDeviceMemoryCallbacks. When device memory of certain heap runs out of free space, new allocations may fail (returning error code) or they may succeed, silently pushing some existing memory blocks from GPU VRAM to system RAM (which degrades performance). This -behavior is implementation-dependant - it depends on GPU vendor and graphics +behavior is implementation-dependent - it depends on GPU vendor and graphics driver. On AMD cards it can be controlled while creating Vulkan device object by using @@ -1670,11 +1822,115 @@ unaware of it. To learn more about this extension, see: -- [VK_KHR_dedicated_allocation in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.0-extensions/html/vkspec.html#VK_KHR_dedicated_allocation) +- [VK_KHR_dedicated_allocation in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap50.html#VK_KHR_dedicated_allocation) - [VK_KHR_dedicated_allocation unofficial manual](http://asawicki.info/articles/VK_KHR_dedicated_allocation.php5) +\page vk_amd_device_coherent_memory VK_AMD_device_coherent_memory + +VK_AMD_device_coherent_memory is a device extension that enables access to +additional memory types with `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` and +`VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD` flag. It is useful mostly for +allocation of buffers intended for writing "breadcrumb markers" in between passes +or draw calls, which in turn are useful for debugging GPU crash/hang/TDR cases. + +When the extension is available but has not been enabled, Vulkan physical device +still exposes those memory types, but their usage is forbidden. VMA automatically +takes care of that - it returns `VK_ERROR_FEATURE_NOT_PRESENT` when an attempt +to allocate memory of such type is made. + +If you want to use this extension in connection with VMA, follow these steps: + +\section vk_amd_device_coherent_memory_initialization Initialization + +1) Call `vkEnumerateDeviceExtensionProperties` for the physical device. +Check if the extension is supported - if returned array of `VkExtensionProperties` contains "VK_AMD_device_coherent_memory". + +2) Call `vkGetPhysicalDeviceFeatures2` for the physical device instead of old `vkGetPhysicalDeviceFeatures`. +Attach additional structure `VkPhysicalDeviceCoherentMemoryFeaturesAMD` to `VkPhysicalDeviceFeatures2::pNext` to be returned. +Check if the device feature is really supported - check if `VkPhysicalDeviceCoherentMemoryFeaturesAMD::deviceCoherentMemory` is true. + +3) While creating device with `vkCreateDevice`, enable this extension - add "VK_AMD_device_coherent_memory" +to the list passed as `VkDeviceCreateInfo::ppEnabledExtensionNames`. + +4) While creating the device, also don't set `VkDeviceCreateInfo::pEnabledFeatures`. +Fill in `VkPhysicalDeviceFeatures2` structure instead and pass it as `VkDeviceCreateInfo::pNext`. +Enable this device feature - attach additional structure `VkPhysicalDeviceCoherentMemoryFeaturesAMD` to +`VkPhysicalDeviceFeatures2::pNext` and set its member `deviceCoherentMemory` to `VK_TRUE`. + +5) While creating #VmaAllocator with vmaCreateAllocator() inform VMA that you +have enabled this extension and feature - add #VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT +to VmaAllocatorCreateInfo::flags. + +\section vk_amd_device_coherent_memory_usage Usage + +After following steps described above, you can create VMA allocations and custom pools +out of the special `DEVICE_COHERENT` and `DEVICE_UNCACHED` memory types on eligible +devices. There are multiple ways to do it, for example: + +- You can request or prefer to allocate out of such memory types by adding + `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` to VmaAllocationCreateInfo::requiredFlags + or VmaAllocationCreateInfo::preferredFlags. Those flags can be freely mixed with + other ways of \ref choosing_memory_type, like setting VmaAllocationCreateInfo::usage. +- If you manually found memory type index to use for this purpose, force allocation + from this specific index by setting VmaAllocationCreateInfo::memoryTypeBits `= 1u << index`. + +\section vk_amd_device_coherent_memory_more_information More information + +To learn more about this extension, see [VK_AMD_device_coherent_memory in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap44.html#VK_AMD_device_coherent_memory) + +Example use of this extension can be found in the code of the sample and test suite +accompanying this library. + + +\page enabling_buffer_device_address Enabling buffer device address + +Device extension VK_KHR_buffer_device_address +allow to fetch raw GPU pointer to a buffer and pass it for usage in a shader code. +It is promoted to core Vulkan 1.2. + +If you want to use this feature in connection with VMA, follow these steps: + +\section enabling_buffer_device_address_initialization Initialization + +1) (For Vulkan version < 1.2) Call `vkEnumerateDeviceExtensionProperties` for the physical device. +Check if the extension is supported - if returned array of `VkExtensionProperties` contains +"VK_KHR_buffer_device_address". + +2) Call `vkGetPhysicalDeviceFeatures2` for the physical device instead of old `vkGetPhysicalDeviceFeatures`. +Attach additional structure `VkPhysicalDeviceBufferDeviceAddressFeatures*` to `VkPhysicalDeviceFeatures2::pNext` to be returned. +Check if the device feature is really supported - check if `VkPhysicalDeviceBufferDeviceAddressFeatures*::bufferDeviceAddress` is true. + +3) (For Vulkan version < 1.2) While creating device with `vkCreateDevice`, enable this extension - add +"VK_KHR_buffer_device_address" to the list passed as `VkDeviceCreateInfo::ppEnabledExtensionNames`. + +4) While creating the device, also don't set `VkDeviceCreateInfo::pEnabledFeatures`. +Fill in `VkPhysicalDeviceFeatures2` structure instead and pass it as `VkDeviceCreateInfo::pNext`. +Enable this device feature - attach additional structure `VkPhysicalDeviceBufferDeviceAddressFeatures*` to +`VkPhysicalDeviceFeatures2::pNext` and set its member `bufferDeviceAddress` to `VK_TRUE`. + +5) While creating #VmaAllocator with vmaCreateAllocator() inform VMA that you +have enabled this feature - add #VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT +to VmaAllocatorCreateInfo::flags. + +\section enabling_buffer_device_address_usage Usage + +After following steps described above, you can create buffers with `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT*` using VMA. +The library automatically adds `VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT*` to +allocated memory blocks wherever it might be needed. + +Please note that the library supports only `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT*`. +The second part of this functionality related to "capture and replay" is not supported, +as it is intended for usage in debugging tools like RenderDoc, not in everyday Vulkan usage. + +\section enabling_buffer_device_address_more_information More information + +To learn more about this extension, see [VK_KHR_buffer_device_address in Vulkan specification](https://www.khronos.org/registry/vulkan/specs/1.2-extensions/html/chap46.html#VK_KHR_buffer_device_address) + +Example use of this extension can be found in the code of the sample and test suite +accompanying this library. + \page general_considerations General considerations \section general_considerations_thread_safety Thread safety @@ -1708,7 +1964,7 @@ to just ignore them. up together, especially on GPUs with unified memory like Intel. - *Non-linear image 0xebc91 is aliased with linear buffer 0xeb8e4 which may indicate a bug.* - It happens when you use lost allocations, and a new image or buffer is - created in place of an existing object that bacame lost. + created in place of an existing object that became lost. - It may happen also when you use [defragmentation](@ref defragmentation). \section general_considerations_allocation_algorithm Allocation algorithm @@ -1731,7 +1987,7 @@ The library uses following algorithm for allocation, in order: Features deliberately excluded from the scope of this library: -- Data transfer. Uploading (straming) and downloading data of buffers and images +- Data transfer. Uploading (streaming) and downloading data of buffers and images between CPU and GPU memory and related synchronization is responsibility of the user. Defining some "texture" object that would automatically stream its data from a staging copy in CPU memory to GPU memory would rather be a feature of another, @@ -1740,6 +1996,8 @@ Features deliberately excluded from the scope of this library: explicit memory type index and dedicated allocation anyway, so they don't interact with main features of this library. Such special purpose allocations should be made manually, using `vkCreateBuffer()` and `vkAllocateMemory()`. +- Sub-allocation of parts of one large buffer. Although recommended as a good practice, + it is the user's responsibility to implement such logic on top of VMA. - Recreation of buffers and images. Although the library has functions for buffer and image creation (vmaCreateBuffer(), vmaCreateImage()), you need to recreate these objects yourself after defragmentation. That's because the big @@ -1749,8 +2007,9 @@ Features deliberately excluded from the scope of this library: objects in CPU memory (not Vulkan memory), allocation failures are not checked and handled gracefully, because that would complicate code significantly and is usually not needed in desktop PC applications anyway. + Success of an allocation is just checked with an assert. - Code free of any compiler warnings. Maintaining the library to compile and - work correctly on so many different platforms is hard enough. Being free of + work correctly on so many different platforms is hard enough. Being free of any warnings, on any version of any compiler, is simply not feasible. - This is a C++ library with C interface. Bindings or ports to any other programming languages are welcomed as external projects and @@ -1758,6 +2017,10 @@ Features deliberately excluded from the scope of this library: */ +#ifdef __cplusplus +extern "C" { +#endif + /* Define this macro to 0/1 to disable/enable support for recording functionality, available through VmaAllocatorCreateInfo::pRecordSettings. @@ -1766,10 +2029,39 @@ available through VmaAllocatorCreateInfo::pRecordSettings. #define VMA_RECORDING_ENABLED 0 #endif -#ifndef NOMINMAX +#if !defined(NOMINMAX) && defined(VMA_IMPLEMENTATION) #define NOMINMAX // For windows.h #endif +#if defined(__ANDROID__) && defined(VK_NO_PROTOTYPES) && VMA_STATIC_VULKAN_FUNCTIONS + extern PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr; + extern PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr; + extern PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties; + extern PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties; + extern PFN_vkAllocateMemory vkAllocateMemory; + extern PFN_vkFreeMemory vkFreeMemory; + extern PFN_vkMapMemory vkMapMemory; + extern PFN_vkUnmapMemory vkUnmapMemory; + extern PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges; + extern PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges; + extern PFN_vkBindBufferMemory vkBindBufferMemory; + extern PFN_vkBindImageMemory vkBindImageMemory; + extern PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements; + extern PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements; + extern PFN_vkCreateBuffer vkCreateBuffer; + extern PFN_vkDestroyBuffer vkDestroyBuffer; + extern PFN_vkCreateImage vkCreateImage; + extern PFN_vkDestroyImage vkDestroyImage; + extern PFN_vkCmdCopyBuffer vkCmdCopyBuffer; + #if VMA_VULKAN_VERSION >= 1001000 + extern PFN_vkGetBufferMemoryRequirements2 vkGetBufferMemoryRequirements2; + extern PFN_vkGetImageMemoryRequirements2 vkGetImageMemoryRequirements2; + extern PFN_vkBindBufferMemory2 vkBindBufferMemory2; + extern PFN_vkBindImageMemory2 vkBindImageMemory2; + extern PFN_vkGetPhysicalDeviceMemoryProperties2 vkGetPhysicalDeviceMemoryProperties2; + #endif // #if VMA_VULKAN_VERSION >= 1001000 +#endif // #if defined(__ANDROID__) && VMA_STATIC_VULKAN_FUNCTIONS && VK_NO_PROTOTYPES + #ifndef VULKAN_H_ #ifdef USE_VOLK #include <volk.h> @@ -1778,15 +2070,13 @@ available through VmaAllocatorCreateInfo::pRecordSettings. #endif #endif -#if VMA_RECORDING_ENABLED - #include <windows.h> -#endif - // Define this macro to declare maximum supported Vulkan version in format AAABBBCCC, // where AAA = major, BBB = minor, CCC = patch. // If you want to use version > 1.0, it still needs to be enabled via VmaAllocatorCreateInfo::vulkanApiVersion. #if !defined(VMA_VULKAN_VERSION) - #if defined(VK_VERSION_1_1) + #if defined(VK_VERSION_1_2) + #define VMA_VULKAN_VERSION 1002000 + #elif defined(VK_VERSION_1_1) #define VMA_VULKAN_VERSION 1001000 #else #define VMA_VULKAN_VERSION 1000000 @@ -1817,9 +2107,36 @@ available through VmaAllocatorCreateInfo::pRecordSettings. #endif #endif +// Defined to 1 when VK_KHR_buffer_device_address device extension or equivalent core Vulkan 1.2 feature is defined in its headers. +#if !defined(VMA_BUFFER_DEVICE_ADDRESS) + #if VK_KHR_buffer_device_address || VMA_VULKAN_VERSION >= 1002000 + #define VMA_BUFFER_DEVICE_ADDRESS 1 + #else + #define VMA_BUFFER_DEVICE_ADDRESS 0 + #endif +#endif + +// Defined to 1 when VK_EXT_memory_priority device extension is defined in Vulkan headers. +#if !defined(VMA_MEMORY_PRIORITY) + #if VK_EXT_memory_priority + #define VMA_MEMORY_PRIORITY 1 + #else + #define VMA_MEMORY_PRIORITY 0 + #endif +#endif + +// Defined to 1 when VK_KHR_external_memory device extension is defined in Vulkan headers. +#if !defined(VMA_EXTERNAL_MEMORY) + #if VK_KHR_external_memory + #define VMA_EXTERNAL_MEMORY 1 + #else + #define VMA_EXTERNAL_MEMORY 0 + #endif +#endif + // Define these macros to decorate all public functions with additional code, // before and after returned type, appropriately. This may be useful for -// exporing the functions when compiling VMA as a separate library. Example: +// exporting the functions when compiling VMA as a separate library. Example: // #define VMA_CALL_PRE __declspec(dllexport) // #define VMA_CALL_POST __cdecl #ifndef VMA_CALL_PRE @@ -1829,6 +2146,59 @@ available through VmaAllocatorCreateInfo::pRecordSettings. #define VMA_CALL_POST #endif +// Define this macro to decorate pointers with an attribute specifying the +// length of the array they point to if they are not null. +// +// The length may be one of +// - The name of another parameter in the argument list where the pointer is declared +// - The name of another member in the struct where the pointer is declared +// - The name of a member of a struct type, meaning the value of that member in +// the context of the call. For example +// VMA_LEN_IF_NOT_NULL("VkPhysicalDeviceMemoryProperties::memoryHeapCount"), +// this means the number of memory heaps available in the device associated +// with the VmaAllocator being dealt with. +#ifndef VMA_LEN_IF_NOT_NULL + #define VMA_LEN_IF_NOT_NULL(len) +#endif + +// The VMA_NULLABLE macro is defined to be _Nullable when compiling with Clang. +// see: https://clang.llvm.org/docs/AttributeReference.html#nullable +#ifndef VMA_NULLABLE + #ifdef __clang__ + #define VMA_NULLABLE _Nullable + #else + #define VMA_NULLABLE + #endif +#endif + +// The VMA_NOT_NULL macro is defined to be _Nonnull when compiling with Clang. +// see: https://clang.llvm.org/docs/AttributeReference.html#nonnull +#ifndef VMA_NOT_NULL + #ifdef __clang__ + #define VMA_NOT_NULL _Nonnull + #else + #define VMA_NOT_NULL + #endif +#endif + +// If non-dispatchable handles are represented as pointers then we can give +// then nullability annotations +#ifndef VMA_NOT_NULL_NON_DISPATCHABLE + #if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__) ) || defined(_M_X64) || defined(__ia64) || defined (_M_IA64) || defined(__aarch64__) || defined(__powerpc64__) + #define VMA_NOT_NULL_NON_DISPATCHABLE VMA_NOT_NULL + #else + #define VMA_NOT_NULL_NON_DISPATCHABLE + #endif +#endif + +#ifndef VMA_NULLABLE_NON_DISPATCHABLE + #if defined(__LP64__) || defined(_WIN64) || (defined(__x86_64__) && !defined(__ILP32__) ) || defined(_M_X64) || defined(__ia64) || defined (_M_IA64) || defined(__aarch64__) || defined(__powerpc64__) + #define VMA_NULLABLE_NON_DISPATCHABLE VMA_NULLABLE + #else + #define VMA_NULLABLE_NON_DISPATCHABLE + #endif +#endif + /** \struct VmaAllocator \brief Represents main object of this library initialized. @@ -1842,16 +2212,18 @@ VK_DEFINE_HANDLE(VmaAllocator) /// Callback function called after successful vkAllocateMemory. typedef void (VKAPI_PTR *PFN_vmaAllocateDeviceMemoryFunction)( - VmaAllocator allocator, - uint32_t memoryType, - VkDeviceMemory memory, - VkDeviceSize size); + VmaAllocator VMA_NOT_NULL allocator, + uint32_t memoryType, + VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory, + VkDeviceSize size, + void* VMA_NULLABLE pUserData); /// Callback function called before vkFreeMemory. typedef void (VKAPI_PTR *PFN_vmaFreeDeviceMemoryFunction)( - VmaAllocator allocator, - uint32_t memoryType, - VkDeviceMemory memory, - VkDeviceSize size); + VmaAllocator VMA_NOT_NULL allocator, + uint32_t memoryType, + VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory, + VkDeviceSize size, + void* VMA_NULLABLE pUserData); /** \brief Set of callbacks that the library will call for `vkAllocateMemory` and `vkFreeMemory`. @@ -1862,9 +2234,11 @@ Used in VmaAllocatorCreateInfo::pDeviceMemoryCallbacks. */ typedef struct VmaDeviceMemoryCallbacks { /// Optional, can be null. - PFN_vmaAllocateDeviceMemoryFunction pfnAllocate; + PFN_vmaAllocateDeviceMemoryFunction VMA_NULLABLE pfnAllocate; + /// Optional, can be null. + PFN_vmaFreeDeviceMemoryFunction VMA_NULLABLE pfnFree; /// Optional, can be null. - PFN_vmaFreeDeviceMemoryFunction pfnFree; + void* VMA_NULLABLE pUserData; } VmaDeviceMemoryCallbacks; /// Flags for created #VmaAllocator. @@ -1879,7 +2253,7 @@ typedef enum VmaAllocatorCreateFlagBits { The flag works only if VmaAllocatorCreateInfo::vulkanApiVersion `== VK_API_VERSION_1_0`. When it's `VK_API_VERSION_1_1`, the flag is ignored because the extension has been promoted to Vulkan 1.1. - Using this extenion will automatically allocate dedicated blocks of memory for + Using this extension will automatically allocate dedicated blocks of memory for some buffers and images instead of suballocating place for them out of bigger memory blocks (as if you explicitly used #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT flag) when it is recommended by the driver. It may improve performance on some @@ -1926,6 +2300,59 @@ typedef enum VmaAllocatorCreateFlagBits { be more accurate than an estimation used by the library otherwise. */ VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT = 0x00000008, + /** + Enables usage of VK_AMD_device_coherent_memory extension. + + You may set this flag only if you: + + - found out that this device extension is supported and enabled it while creating Vulkan device passed as VmaAllocatorCreateInfo::device, + - checked that `VkPhysicalDeviceCoherentMemoryFeaturesAMD::deviceCoherentMemory` is true and set it while creating the Vulkan device, + - want it to be used internally by this library. + + The extension and accompanying device feature provide access to memory types with + `VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD` and `VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD` flags. + They are useful mostly for writing breadcrumb markers - a common method for debugging GPU crash/hang/TDR. + + When the extension is not enabled, such memory types are still enumerated, but their usage is illegal. + To protect from this error, if you don't create the allocator with this flag, it will refuse to allocate any memory or create a custom pool in such memory type, + returning `VK_ERROR_FEATURE_NOT_PRESENT`. + */ + VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT = 0x00000010, + /** + Enables usage of "buffer device address" feature, which allows you to use function + `vkGetBufferDeviceAddress*` to get raw GPU pointer to a buffer and pass it for usage inside a shader. + + You may set this flag only if you: + + 1. (For Vulkan version < 1.2) Found as available and enabled device extension + VK_KHR_buffer_device_address. + This extension is promoted to core Vulkan 1.2. + 2. Found as available and enabled device feature `VkPhysicalDeviceBufferDeviceAddressFeatures::bufferDeviceAddress`. + + When this flag is set, you can create buffers with `VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT` using VMA. + The library automatically adds `VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT` to + allocated memory blocks wherever it might be needed. + + For more information, see documentation chapter \ref enabling_buffer_device_address. + */ + VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT = 0x00000020, + /** + Enables usage of VK_EXT_memory_priority extension in the library. + + You may set this flag only if you found available and enabled this device extension, + along with `VkPhysicalDeviceMemoryPriorityFeaturesEXT::memoryPriority == VK_TRUE`, + while creating Vulkan device passed as VmaAllocatorCreateInfo::device. + + When this flag is used, VmaAllocationCreateInfo::priority and VmaPoolCreateInfo::priority + are used to set priorities of allocated Vulkan memory. Without it, these variables are ignored. + + A priority must be a floating-point value between 0 and 1, indicating the priority of the allocation relative to other memory allocations. + Larger values are higher priority. The granularity of the priorities is implementation-dependent. + It is automatically passed to every call to `vkAllocateMemory` done by the library using structure `VkMemoryPriorityAllocateInfoEXT`. + The value to be used for default priority is 0.5. + For more details, see the documentation of the VK_EXT_memory_priority extension. + */ + VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT = 0x00000040, VMA_ALLOCATOR_CREATE_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF } VmaAllocatorCreateFlagBits; @@ -1936,33 +2363,33 @@ typedef VkFlags VmaAllocatorCreateFlags; Used in VmaAllocatorCreateInfo::pVulkanFunctions. */ typedef struct VmaVulkanFunctions { - PFN_vkGetPhysicalDeviceProperties vkGetPhysicalDeviceProperties; - PFN_vkGetPhysicalDeviceMemoryProperties vkGetPhysicalDeviceMemoryProperties; - PFN_vkAllocateMemory vkAllocateMemory; - PFN_vkFreeMemory vkFreeMemory; - PFN_vkMapMemory vkMapMemory; - PFN_vkUnmapMemory vkUnmapMemory; - PFN_vkFlushMappedMemoryRanges vkFlushMappedMemoryRanges; - PFN_vkInvalidateMappedMemoryRanges vkInvalidateMappedMemoryRanges; - PFN_vkBindBufferMemory vkBindBufferMemory; - PFN_vkBindImageMemory vkBindImageMemory; - PFN_vkGetBufferMemoryRequirements vkGetBufferMemoryRequirements; - PFN_vkGetImageMemoryRequirements vkGetImageMemoryRequirements; - PFN_vkCreateBuffer vkCreateBuffer; - PFN_vkDestroyBuffer vkDestroyBuffer; - PFN_vkCreateImage vkCreateImage; - PFN_vkDestroyImage vkDestroyImage; - PFN_vkCmdCopyBuffer vkCmdCopyBuffer; + PFN_vkGetPhysicalDeviceProperties VMA_NULLABLE vkGetPhysicalDeviceProperties; + PFN_vkGetPhysicalDeviceMemoryProperties VMA_NULLABLE vkGetPhysicalDeviceMemoryProperties; + PFN_vkAllocateMemory VMA_NULLABLE vkAllocateMemory; + PFN_vkFreeMemory VMA_NULLABLE vkFreeMemory; + PFN_vkMapMemory VMA_NULLABLE vkMapMemory; + PFN_vkUnmapMemory VMA_NULLABLE vkUnmapMemory; + PFN_vkFlushMappedMemoryRanges VMA_NULLABLE vkFlushMappedMemoryRanges; + PFN_vkInvalidateMappedMemoryRanges VMA_NULLABLE vkInvalidateMappedMemoryRanges; + PFN_vkBindBufferMemory VMA_NULLABLE vkBindBufferMemory; + PFN_vkBindImageMemory VMA_NULLABLE vkBindImageMemory; + PFN_vkGetBufferMemoryRequirements VMA_NULLABLE vkGetBufferMemoryRequirements; + PFN_vkGetImageMemoryRequirements VMA_NULLABLE vkGetImageMemoryRequirements; + PFN_vkCreateBuffer VMA_NULLABLE vkCreateBuffer; + PFN_vkDestroyBuffer VMA_NULLABLE vkDestroyBuffer; + PFN_vkCreateImage VMA_NULLABLE vkCreateImage; + PFN_vkDestroyImage VMA_NULLABLE vkDestroyImage; + PFN_vkCmdCopyBuffer VMA_NULLABLE vkCmdCopyBuffer; #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 - PFN_vkGetBufferMemoryRequirements2KHR vkGetBufferMemoryRequirements2KHR; - PFN_vkGetImageMemoryRequirements2KHR vkGetImageMemoryRequirements2KHR; + PFN_vkGetBufferMemoryRequirements2KHR VMA_NULLABLE vkGetBufferMemoryRequirements2KHR; + PFN_vkGetImageMemoryRequirements2KHR VMA_NULLABLE vkGetImageMemoryRequirements2KHR; #endif #if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000 - PFN_vkBindBufferMemory2KHR vkBindBufferMemory2KHR; - PFN_vkBindImageMemory2KHR vkBindImageMemory2KHR; + PFN_vkBindBufferMemory2KHR VMA_NULLABLE vkBindBufferMemory2KHR; + PFN_vkBindImageMemory2KHR VMA_NULLABLE vkBindImageMemory2KHR; #endif #if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000 - PFN_vkGetPhysicalDeviceMemoryProperties2KHR vkGetPhysicalDeviceMemoryProperties2KHR; + PFN_vkGetPhysicalDeviceMemoryProperties2KHR VMA_NULLABLE vkGetPhysicalDeviceMemoryProperties2KHR; #endif } VmaVulkanFunctions; @@ -1974,7 +2401,7 @@ typedef enum VmaRecordFlagBits { It may degrade performance though. */ VMA_RECORD_FLUSH_AFTER_CALL_BIT = 0x00000001, - + VMA_RECORD_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF } VmaRecordFlagBits; typedef VkFlags VmaRecordFlags; @@ -1991,7 +2418,7 @@ typedef struct VmaRecordSettings It will be opened for the whole time #VmaAllocator object is alive. If opening this file fails, creation of the whole allocator object fails. */ - const char* pFilePath; + const char* VMA_NOT_NULL pFilePath; } VmaRecordSettings; /// Description of a Allocator to be created. @@ -2001,19 +2428,19 @@ typedef struct VmaAllocatorCreateInfo VmaAllocatorCreateFlags flags; /// Vulkan physical device. /** It must be valid throughout whole lifetime of created allocator. */ - VkPhysicalDevice physicalDevice; + VkPhysicalDevice VMA_NOT_NULL physicalDevice; /// Vulkan device. /** It must be valid throughout whole lifetime of created allocator. */ - VkDevice device; + VkDevice VMA_NOT_NULL device; /// Preferred size of a single `VkDeviceMemory` block to be allocated from large heaps > 1 GiB. Optional. /** Set to 0 to use default, which is currently 256 MiB. */ VkDeviceSize preferredLargeHeapBlockSize; /// Custom CPU memory allocation callbacks. Optional. /** Optional, can be null. When specified, will also be used for all CPU-side memory allocations. */ - const VkAllocationCallbacks* pAllocationCallbacks; + const VkAllocationCallbacks* VMA_NULLABLE pAllocationCallbacks; /// Informative callbacks for `vkAllocateMemory`, `vkFreeMemory`. Optional. /** Optional, can be null. */ - const VmaDeviceMemoryCallbacks* pDeviceMemoryCallbacks; + const VmaDeviceMemoryCallbacks* VMA_NULLABLE pDeviceMemoryCallbacks; /** \brief Maximum number of additional frames that are in use at the same time as current frame. This value is used only when you make allocations with @@ -2052,67 +2479,100 @@ typedef struct VmaAllocatorCreateInfo blocks to system RAM. This driver behavior can also be controlled using VK_AMD_memory_overallocation_behavior extension. */ - const VkDeviceSize* pHeapSizeLimit; - /** \brief Pointers to Vulkan functions. Can be null if you leave define `VMA_STATIC_VULKAN_FUNCTIONS 1`. - - If you leave define `VMA_STATIC_VULKAN_FUNCTIONS 1` in configuration section, - you can pass null as this member, because the library will fetch pointers to - Vulkan functions internally in a static way, like: + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL("VkPhysicalDeviceMemoryProperties::memoryHeapCount") pHeapSizeLimit; - vulkanFunctions.vkAllocateMemory = &vkAllocateMemory; + /** \brief Pointers to Vulkan functions. Can be null. - Fill this member if you want to provide your own pointers to Vulkan functions, - e.g. fetched using `vkGetInstanceProcAddr()` and `vkGetDeviceProcAddr()`. + For details see [Pointers to Vulkan functions](@ref config_Vulkan_functions). */ - const VmaVulkanFunctions* pVulkanFunctions; + const VmaVulkanFunctions* VMA_NULLABLE pVulkanFunctions; /** \brief Parameters for recording of VMA calls. Can be null. If not null, it enables recording of calls to VMA functions to a file. If support for recording is not enabled using `VMA_RECORDING_ENABLED` macro, creation of the allocator object fails with `VK_ERROR_FEATURE_NOT_PRESENT`. */ - const VmaRecordSettings* pRecordSettings; - /** \brief Optional handle to Vulkan instance object. + const VmaRecordSettings* VMA_NULLABLE pRecordSettings; + /** \brief Handle to Vulkan instance object. - Optional, can be null. Must be set if #VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT flas is used - or if `vulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)`. + Starting from version 3.0.0 this member is no longer optional, it must be set! */ - VkInstance instance; + VkInstance VMA_NOT_NULL instance; /** \brief Optional. The highest version of Vulkan that the application is designed to use. - + It must be a value in the format as created by macro `VK_MAKE_VERSION` or a constant like: `VK_API_VERSION_1_1`, `VK_API_VERSION_1_0`. The patch version number specified is ignored. Only the major and minor versions are considered. - It must be less or euqal (preferably equal) to value as passed to `vkCreateInstance` as `VkApplicationInfo::apiVersion`. - Only versions 1.0 and 1.1 are supported by the current implementation. + It must be less or equal (preferably equal) to value as passed to `vkCreateInstance` as `VkApplicationInfo::apiVersion`. + Only versions 1.0, 1.1, 1.2 are supported by the current implementation. Leaving it initialized to zero is equivalent to `VK_API_VERSION_1_0`. */ uint32_t vulkanApiVersion; +#if VMA_EXTERNAL_MEMORY + /** \brief Either null or a pointer to an array of external memory handle types for each Vulkan memory type. + + If not NULL, it must be a pointer to an array of `VkPhysicalDeviceMemoryProperties::memoryTypeCount` + elements, defining external memory handle types of particular Vulkan memory type, + to be passed using `VkExportMemoryAllocateInfoKHR`. + + Any of the elements may be equal to 0, which means not to use `VkExportMemoryAllocateInfoKHR` on this memory type. + This is also the default in case of `pTypeExternalMemoryHandleTypes` = NULL. + */ + const VkExternalMemoryHandleTypeFlagsKHR* VMA_NULLABLE VMA_LEN_IF_NOT_NULL("VkPhysicalDeviceMemoryProperties::memoryTypeCount") pTypeExternalMemoryHandleTypes; +#endif // #if VMA_EXTERNAL_MEMORY } VmaAllocatorCreateInfo; /// Creates Allocator object. VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAllocator( - const VmaAllocatorCreateInfo* pCreateInfo, - VmaAllocator* pAllocator); + const VmaAllocatorCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaAllocator VMA_NULLABLE * VMA_NOT_NULL pAllocator); /// Destroys allocator object. VMA_CALL_PRE void VMA_CALL_POST vmaDestroyAllocator( - VmaAllocator allocator); + VmaAllocator VMA_NULLABLE allocator); + +/** \brief Information about existing #VmaAllocator object. +*/ +typedef struct VmaAllocatorInfo +{ + /** \brief Handle to Vulkan instance object. + + This is the same value as has been passed through VmaAllocatorCreateInfo::instance. + */ + VkInstance VMA_NOT_NULL instance; + /** \brief Handle to Vulkan physical device object. + + This is the same value as has been passed through VmaAllocatorCreateInfo::physicalDevice. + */ + VkPhysicalDevice VMA_NOT_NULL physicalDevice; + /** \brief Handle to Vulkan device object. + + This is the same value as has been passed through VmaAllocatorCreateInfo::device. + */ + VkDevice VMA_NOT_NULL device; +} VmaAllocatorInfo; + +/** \brief Returns information about existing #VmaAllocator object - handle to Vulkan device etc. + +It might be useful if you want to keep just the #VmaAllocator handle and fetch other required handles to +`VkPhysicalDevice`, `VkDevice` etc. every time using this function. +*/ +VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocatorInfo(VmaAllocator VMA_NOT_NULL allocator, VmaAllocatorInfo* VMA_NOT_NULL pAllocatorInfo); /** PhysicalDeviceProperties are fetched from physicalDevice by the allocator. You can access it here, without fetching it again on your own. */ VMA_CALL_PRE void VMA_CALL_POST vmaGetPhysicalDeviceProperties( - VmaAllocator allocator, - const VkPhysicalDeviceProperties** ppPhysicalDeviceProperties); + VmaAllocator VMA_NOT_NULL allocator, + const VkPhysicalDeviceProperties* VMA_NULLABLE * VMA_NOT_NULL ppPhysicalDeviceProperties); /** PhysicalDeviceMemoryProperties are fetched from physicalDevice by the allocator. You can access it here, without fetching it again on your own. */ VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryProperties( - VmaAllocator allocator, - const VkPhysicalDeviceMemoryProperties** ppPhysicalDeviceMemoryProperties); + VmaAllocator VMA_NOT_NULL allocator, + const VkPhysicalDeviceMemoryProperties* VMA_NULLABLE * VMA_NOT_NULL ppPhysicalDeviceMemoryProperties); /** \brief Given Memory Type Index, returns Property Flags of this memory type. @@ -2121,9 +2581,9 @@ This is just a convenience function. Same information can be obtained using vmaGetMemoryProperties(). */ VMA_CALL_PRE void VMA_CALL_POST vmaGetMemoryTypeProperties( - VmaAllocator allocator, + VmaAllocator VMA_NOT_NULL allocator, uint32_t memoryTypeIndex, - VkMemoryPropertyFlags* pFlags); + VkMemoryPropertyFlags* VMA_NOT_NULL pFlags); /** \brief Sets index of the current frame. @@ -2134,7 +2594,7 @@ when a new frame begins. Allocations queried using vmaGetAllocationInfo() cannot become lost in the current frame. */ VMA_CALL_PRE void VMA_CALL_POST vmaSetCurrentFrameIndex( - VmaAllocator allocator, + VmaAllocator VMA_NOT_NULL allocator, uint32_t frameIndex); /** \brief Calculated statistics of memory usage in entire allocator. @@ -2173,8 +2633,8 @@ Note that when using allocator from multiple threads, returned information may i become outdated. */ VMA_CALL_PRE void VMA_CALL_POST vmaCalculateStats( - VmaAllocator allocator, - VmaStats* pStats); + VmaAllocator VMA_NOT_NULL allocator, + VmaStats* VMA_NOT_NULL pStats); /** \brief Statistics of current memory usage and available budget, in bytes, for specific memory heap. */ @@ -2183,32 +2643,32 @@ typedef struct VmaBudget /** \brief Sum size of all `VkDeviceMemory` blocks allocated from particular heap, in bytes. */ VkDeviceSize blockBytes; - + /** \brief Sum size of all allocations created in particular heap, in bytes. - + Usually less or equal than `blockBytes`. Difference `blockBytes - allocationBytes` is the amount of memory allocated but unused - available for new allocations or wasted due to fragmentation. - + It might be greater than `blockBytes` if there are some allocations in lost state, as they account to this value as well. */ VkDeviceSize allocationBytes; - + /** \brief Estimated current memory usage of the program, in bytes. - + Fetched from system using `VK_EXT_memory_budget` extension if enabled. - + It might be different than `blockBytes` (usually higher) due to additional implicit objects also occupying the memory, like swapchain, pipelines, descriptor heaps, command buffers, or `VkDeviceMemory` blocks allocated outside of this library, if any. */ VkDeviceSize usage; - + /** \brief Estimated amount of memory available to the program, in bytes. - + Fetched from system using `VK_EXT_memory_budget` extension if enabled. - + It might be different (most probably smaller) than `VkMemoryHeap::size[heapIndex]` due to factors external to the program, like other programs also consuming system resources. Difference `budget - usage` is the amount of additional memory that can probably @@ -2228,8 +2688,8 @@ Note that when using allocator from multiple threads, returned information may i become outdated. */ VMA_CALL_PRE void VMA_CALL_POST vmaGetBudget( - VmaAllocator allocator, - VmaBudget* pBudget); + VmaAllocator VMA_NOT_NULL allocator, + VmaBudget* VMA_NOT_NULL pBudget); #ifndef VMA_STATS_STRING_ENABLED #define VMA_STATS_STRING_ENABLED 1 @@ -2241,13 +2701,13 @@ VMA_CALL_PRE void VMA_CALL_POST vmaGetBudget( /** @param[out] ppStatsString Must be freed using vmaFreeStatsString() function. */ VMA_CALL_PRE void VMA_CALL_POST vmaBuildStatsString( - VmaAllocator allocator, - char** ppStatsString, + VmaAllocator VMA_NOT_NULL allocator, + char* VMA_NULLABLE * VMA_NOT_NULL ppStatsString, VkBool32 detailedMap); VMA_CALL_PRE void VMA_CALL_POST vmaFreeStatsString( - VmaAllocator allocator, - char* pStatsString); + VmaAllocator VMA_NOT_NULL allocator, + char* VMA_NULLABLE pStatsString); #endif // #if VMA_STATS_STRING_ENABLED @@ -2273,7 +2733,7 @@ typedef enum VmaMemoryUsage It is roughly equivalent of `D3D12_HEAP_TYPE_DEFAULT`. Usage: - + - Resources written and read by device, e.g. images used as attachments. - Resources transferred from host once (immutable) or infrequently and read by device multiple times, e.g. textures to be sampled, vertex buffers, uniform @@ -2298,7 +2758,7 @@ typedef enum VmaMemoryUsage Memory that is both mappable on host (guarantees to be `HOST_VISIBLE`) and preferably fast to access by GPU. CPU access is typically uncached. Writes may be write-combined. - Usage: Resources written frequently by host (dynamic), read by device. E.g. textures, vertex buffers, uniform buffers updated every frame or every draw call. + Usage: Resources written frequently by host (dynamic), read by device. E.g. textures (with LINEAR layout), vertex buffers, uniform buffers updated every frame or every draw call. */ VMA_MEMORY_USAGE_CPU_TO_GPU = 3, /** Memory mappable on host (guarantees to be `HOST_VISIBLE`) and cached. @@ -2318,7 +2778,7 @@ typedef enum VmaMemoryUsage VMA_MEMORY_USAGE_CPU_COPY = 5, /** Lazily allocated GPU memory having `VK_MEMORY_PROPERTY_LAZILY_ALLOCATED_BIT`. Exists mostly on mobile platforms. Using it on desktop PC or other GPUs with no such memory type present will fail the allocation. - + Usage: Memory for transient attachment images (color attachments, depth attachments etc.), created with `VK_IMAGE_USAGE_TRANSIENT_ATTACHMENT_BIT`. Allocations with this usage are always created as dedicated - it implies #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT. @@ -2331,28 +2791,28 @@ typedef enum VmaMemoryUsage /// Flags to be passed as VmaAllocationCreateInfo::flags. typedef enum VmaAllocationCreateFlagBits { /** \brief Set this flag if the allocation should have its own memory block. - + Use it for special, big resources, like fullscreen images used as attachments. - + You should not use this flag if VmaAllocationCreateInfo::pool is not null. */ VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT = 0x00000001, /** \brief Set this flag to only try to allocate from existing `VkDeviceMemory` blocks and never create new such block. - + If new allocation cannot be placed in any of the existing blocks, allocation fails with `VK_ERROR_OUT_OF_DEVICE_MEMORY` error. - + You should not use #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT and #VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT at the same time. It makes no sense. - + If VmaAllocationCreateInfo::pool is not null, this flag is implied and ignored. */ VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT = 0x00000002, /** \brief Set this flag to use a memory that will be persistently mapped and retrieve pointer to it. - + Pointer to mapped memory will be returned through VmaAllocationInfo::pMappedData. - Is it valid to use this flag for allocation made from memory type that is not + It is valid to use this flag for allocation made from memory type that is not `HOST_VISIBLE`. This flag is then ignored and memory is not mapped. This is useful if you need an allocation that is efficient to use on GPU (`DEVICE_LOCAL`) and still want to map it directly if possible on platforms that @@ -2445,19 +2905,19 @@ typedef struct VmaAllocationCreateInfo /// Use #VmaAllocationCreateFlagBits enum. VmaAllocationCreateFlags flags; /** \brief Intended usage of memory. - + You can leave #VMA_MEMORY_USAGE_UNKNOWN if you specify memory requirements in other way. \n If `pool` is not null, this member is ignored. */ VmaMemoryUsage usage; /** \brief Flags that must be set in a Memory Type chosen for an allocation. - + Leave 0 if you specify memory requirements in other way. \n If `pool` is not null, this member is ignored.*/ VkMemoryPropertyFlags requiredFlags; /** \brief Flags that preferably should be set in a memory type chosen for an allocation. - - Set to 0 if no additional flags are prefered. \n + + Set to 0 if no additional flags are preferred. \n If `pool` is not null, this member is ignored. */ VkMemoryPropertyFlags preferredFlags; /** \brief Bitmask containing one bit set for every memory type acceptable for this allocation. @@ -2473,14 +2933,21 @@ typedef struct VmaAllocationCreateInfo Leave `VK_NULL_HANDLE` to allocate from default pool. If not null, members: `usage`, `requiredFlags`, `preferredFlags`, `memoryTypeBits` are ignored. */ - VmaPool pool; + VmaPool VMA_NULLABLE pool; /** \brief Custom general-purpose pointer that will be stored in #VmaAllocation, can be read as VmaAllocationInfo::pUserData and changed using vmaSetAllocationUserData(). - + If #VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT is used, it must be either null or pointer to a null-terminated string. The string will be then copied to internal buffer, so it doesn't need to be valid after allocation call. */ - void* pUserData; + void* VMA_NULLABLE pUserData; + /** \brief A floating-point value between 0 and 1, indicating the priority of the allocation relative to other memory allocations. + + It is used only when #VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT flag was used during creation of the #VmaAllocator object + and this allocation ends up as dedicated or is explicitly forced as dedicated using #VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT. + Otherwise, it has the priority of a memory block where it is placed and this variable is ignored. + */ + float priority; } VmaAllocationCreateInfo; /** @@ -2500,10 +2967,10 @@ type of resource you want to use it for. Please check parameters of your resource, like image layout (OPTIMAL versus LINEAR) or mip level count. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndex( - VmaAllocator allocator, + VmaAllocator VMA_NOT_NULL allocator, uint32_t memoryTypeBits, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - uint32_t* pMemoryTypeIndex); + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + uint32_t* VMA_NOT_NULL pMemoryTypeIndex); /** \brief Helps to find memoryTypeIndex, given VkBufferCreateInfo and VmaAllocationCreateInfo. @@ -2518,10 +2985,10 @@ It is just a convenience function, equivalent to calling: - `vkDestroyBuffer` */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForBufferInfo( - VmaAllocator allocator, - const VkBufferCreateInfo* pBufferCreateInfo, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - uint32_t* pMemoryTypeIndex); + VmaAllocator VMA_NOT_NULL allocator, + const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo, + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + uint32_t* VMA_NOT_NULL pMemoryTypeIndex); /** \brief Helps to find memoryTypeIndex, given VkImageCreateInfo and VmaAllocationCreateInfo. @@ -2536,10 +3003,10 @@ It is just a convenience function, equivalent to calling: - `vkDestroyImage` */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForImageInfo( - VmaAllocator allocator, - const VkImageCreateInfo* pImageCreateInfo, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - uint32_t* pMemoryTypeIndex); + VmaAllocator VMA_NOT_NULL allocator, + const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo, + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + uint32_t* VMA_NOT_NULL pMemoryTypeIndex); /// Flags to be passed as VmaPoolCreateInfo::flags. typedef enum VmaPoolCreateFlagBits { @@ -2626,7 +3093,7 @@ typedef struct VmaPoolCreateInfo { /** \brief Maximum number of blocks that can be allocated in this pool. Optional. Set to 0 to use default, which is `SIZE_MAX`, which means no limit. - + Set to same value as VmaPoolCreateInfo::minBlockCount to have fixed amount of memory allocated throughout whole lifetime of this pool. */ @@ -2645,6 +3112,29 @@ typedef struct VmaPoolCreateInfo { become lost, set this value to 0. */ uint32_t frameInUseCount; + /** \brief A floating-point value between 0 and 1, indicating the priority of the allocations in this pool relative to other memory allocations. + + It is used only when #VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT flag was used during creation of the #VmaAllocator object. + Otherwise, this variable is ignored. + */ + float priority; + /** \brief Additional minimum alignment to be used for all allocations created from this pool. Can be 0. + + Leave 0 (default) not to impose any additional alignment. If not 0, it must be a power of two. + It can be useful in cases where alignment returned by Vulkan by functions like `vkGetBufferMemoryRequirements` is not enough, + e.g. when doing interop with OpenGL. + */ + VkDeviceSize minAllocationAlignment; + /** \brief Additional `pNext` chain to be attached to `VkMemoryAllocateInfo` used for every allocation made by this pool. Optional. + + Optional, can be null. If not null, it must point to a `pNext` chain of structures that can be attached to `VkMemoryAllocateInfo`. + It can be useful for special needs such as adding `VkExportMemoryAllocateInfoKHR`. + Structures pointed by this member must remain alive and unchanged for the whole lifetime of the custom pool. + + Please note that some structures, e.g. `VkMemoryPriorityAllocateInfoEXT`, `VkMemoryDedicatedAllocateInfoKHR`, + can be attached automatically by this library when using other, more convenient of its features. + */ + void* VMA_NULLABLE pMemoryAllocateNext; } VmaPoolCreateInfo; /** \brief Describes parameter of existing #VmaPool. @@ -2681,15 +3171,15 @@ typedef struct VmaPoolStats { @param[out] pPool Handle to created pool. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreatePool( - VmaAllocator allocator, - const VmaPoolCreateInfo* pCreateInfo, - VmaPool* pPool); + VmaAllocator VMA_NOT_NULL allocator, + const VmaPoolCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaPool VMA_NULLABLE * VMA_NOT_NULL pPool); /** \brief Destroys #VmaPool object and frees Vulkan device memory. */ VMA_CALL_PRE void VMA_CALL_POST vmaDestroyPool( - VmaAllocator allocator, - VmaPool pool); + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NULLABLE pool); /** \brief Retrieves statistics of existing #VmaPool object. @@ -2698,9 +3188,9 @@ VMA_CALL_PRE void VMA_CALL_POST vmaDestroyPool( @param[out] pPoolStats Statistics of specified pool. */ VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolStats( - VmaAllocator allocator, - VmaPool pool, - VmaPoolStats* pPoolStats); + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NOT_NULL pool, + VmaPoolStats* VMA_NOT_NULL pPoolStats); /** \brief Marks all allocations in given pool as lost if they are not used in current frame or VmaPoolCreateInfo::frameInUseCount back from now. @@ -2709,9 +3199,9 @@ VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolStats( @param[out] pLostAllocationCount Number of allocations marked as lost. Optional - pass null if you don't need this information. */ VMA_CALL_PRE void VMA_CALL_POST vmaMakePoolAllocationsLost( - VmaAllocator allocator, - VmaPool pool, - size_t* pLostAllocationCount); + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NOT_NULL pool, + size_t* VMA_NULLABLE pLostAllocationCount); /** \brief Checks magic number in margins around all allocations in given memory pool in search for corruptions. @@ -2727,7 +3217,7 @@ Possible return values: `VMA_ASSERT` is also fired in that case. - Other value: Error returned by Vulkan, e.g. memory mapping failure. */ -VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckPoolCorruption(VmaAllocator allocator, VmaPool pool); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckPoolCorruption(VmaAllocator VMA_NOT_NULL allocator, VmaPool VMA_NOT_NULL pool); /** \brief Retrieves name of a custom pool. @@ -2736,9 +3226,9 @@ containing name of the pool that was previously set. The pointer becomes invalid destroyed or its name is changed using vmaSetPoolName(). */ VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolName( - VmaAllocator allocator, - VmaPool pool, - const char** ppName); + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NOT_NULL pool, + const char* VMA_NULLABLE * VMA_NOT_NULL ppName); /** \brief Sets name of a custom pool. @@ -2746,9 +3236,9 @@ VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolName( Function makes internal copy of the string, so it can be changed or freed immediately after this call. */ VMA_CALL_PRE void VMA_CALL_POST vmaSetPoolName( - VmaAllocator allocator, - VmaPool pool, - const char* pName); + VmaAllocator VMA_NOT_NULL allocator, + VmaPool VMA_NOT_NULL pool, + const char* VMA_NULLABLE pName); /** \struct VmaAllocation \brief Represents single memory allocation. @@ -2780,20 +3270,25 @@ VK_DEFINE_HANDLE(VmaAllocation) */ typedef struct VmaAllocationInfo { /** \brief Memory type index that this allocation was allocated from. - + It never changes. */ uint32_t memoryType; /** \brief Handle to Vulkan memory object. Same memory object can be shared by multiple allocations. - + It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost. If the allocation is lost, it is equal to `VK_NULL_HANDLE`. */ - VkDeviceMemory deviceMemory; - /** \brief Offset into deviceMemory object to the beginning of this allocation, in bytes. (deviceMemory, offset) pair is unique to this allocation. + VkDeviceMemory VMA_NULLABLE_NON_DISPATCHABLE deviceMemory; + /** \brief Offset in `VkDeviceMemory` object to the beginning of this allocation, in bytes. `(deviceMemory, offset)` pair is unique to this allocation. + + You usually don't need to use this offset. If you create a buffer or an image together with the allocation using e.g. function + vmaCreateBuffer(), vmaCreateImage(), functions that operate on these resources refer to the beginning of the buffer or image, + not entire device memory block. Functions like vmaMapMemory(), vmaBindBufferMemory() also refer to the beginning of the allocation + and apply this offset automatically. It can change after call to vmaDefragment() if this allocation is passed to the function, or if allocation is lost. */ @@ -2801,22 +3296,28 @@ typedef struct VmaAllocationInfo { /** \brief Size of this allocation, in bytes. It never changes, unless allocation is lost. + + \note Allocation size returned in this variable may be greater than the size + requested for the resource e.g. as `VkBufferCreateInfo::size`. Whole size of the + allocation is accessible for operations on memory e.g. using a pointer after + mapping with vmaMapMemory(), but operations on the resource e.g. using + `vkCmdCopyBuffer` must be limited to the size of the resource. */ VkDeviceSize size; /** \brief Pointer to the beginning of this allocation as mapped data. If the allocation hasn't been mapped using vmaMapMemory() and hasn't been - created with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag, this value null. + created with #VMA_ALLOCATION_CREATE_MAPPED_BIT flag, this value is null. It can change after call to vmaMapMemory(), vmaUnmapMemory(). It can also change after call to vmaDefragment() if this allocation is passed to the function. */ - void* pMappedData; + void* VMA_NULLABLE pMappedData; /** \brief Custom general-purpose pointer that was passed as VmaAllocationCreateInfo::pUserData or set using vmaSetAllocationUserData(). It can change after call to vmaSetAllocationUserData() for this allocation. */ - void* pUserData; + void* VMA_NULLABLE pUserData; } VmaAllocationInfo; /** \brief General purpose memory allocation. @@ -2830,11 +3331,11 @@ It is recommended to use vmaAllocateMemoryForBuffer(), vmaAllocateMemoryForImage vmaCreateBuffer(), vmaCreateImage() instead whenever possible. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemory( - VmaAllocator allocator, - const VkMemoryRequirements* pVkMemoryRequirements, - const VmaAllocationCreateInfo* pCreateInfo, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); + VmaAllocator VMA_NOT_NULL allocator, + const VkMemoryRequirements* VMA_NOT_NULL pVkMemoryRequirements, + const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); /** \brief General purpose memory allocation for multiple allocation objects at once. @@ -2856,12 +3357,12 @@ If any allocation fails, all allocations already made within this function call returned result is not `VK_SUCCESS`, `pAllocation` array is always entirely filled with `VK_NULL_HANDLE`. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages( - VmaAllocator allocator, - const VkMemoryRequirements* pVkMemoryRequirements, - const VmaAllocationCreateInfo* pCreateInfo, + VmaAllocator VMA_NOT_NULL allocator, + const VkMemoryRequirements* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pVkMemoryRequirements, + const VmaAllocationCreateInfo* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pCreateInfo, size_t allocationCount, - VmaAllocation* pAllocations, - VmaAllocationInfo* pAllocationInfo); + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations, + VmaAllocationInfo* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationInfo); /** @param[out] pAllocation Handle to allocated memory. @@ -2870,27 +3371,27 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages( You should free the memory using vmaFreeMemory(). */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForBuffer( - VmaAllocator allocator, - VkBuffer buffer, - const VmaAllocationCreateInfo* pCreateInfo, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); + VmaAllocator VMA_NOT_NULL allocator, + VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer, + const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); /// Function similar to vmaAllocateMemoryForBuffer(). VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForImage( - VmaAllocator allocator, - VkImage image, - const VmaAllocationCreateInfo* pCreateInfo, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); + VmaAllocator VMA_NOT_NULL allocator, + VkImage VMA_NOT_NULL_NON_DISPATCHABLE image, + const VmaAllocationCreateInfo* VMA_NOT_NULL pCreateInfo, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); /** \brief Frees memory previously allocated using vmaAllocateMemory(), vmaAllocateMemoryForBuffer(), or vmaAllocateMemoryForImage(). Passing `VK_NULL_HANDLE` as `allocation` is valid. Such function call is just skipped. */ VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory( - VmaAllocator allocator, - VmaAllocation allocation); + VmaAllocator VMA_NOT_NULL allocator, + const VmaAllocation VMA_NULLABLE allocation); /** \brief Frees memory and destroys multiple allocations. @@ -2903,24 +3404,13 @@ Allocations in `pAllocations` array can come from any memory pools and types. Passing `VK_NULL_HANDLE` as elements of `pAllocations` array is valid. Such entries are just skipped. */ VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages( - VmaAllocator allocator, + VmaAllocator VMA_NOT_NULL allocator, size_t allocationCount, - VmaAllocation* pAllocations); - -/** \brief Deprecated. - -In version 2.2.0 it used to try to change allocation's size without moving or reallocating it. -In current version it returns `VK_SUCCESS` only if `newSize` equals current allocation's size. -Otherwise returns `VK_ERROR_OUT_OF_POOL_MEMORY`, indicating that allocation's size could not be changed. -*/ -VMA_CALL_PRE VkResult VMA_CALL_POST vmaResizeAllocation( - VmaAllocator allocator, - VmaAllocation allocation, - VkDeviceSize newSize); + const VmaAllocation VMA_NULLABLE * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations); /** \brief Returns current information about specified allocation and atomically marks it as used in current frame. -Current paramters of given allocation are returned in `pAllocationInfo`. +Current paramteres of given allocation are returned in `pAllocationInfo`. This function also atomically "touches" allocation - marks it as used in current frame, just like vmaTouchAllocation(). @@ -2935,9 +3425,9 @@ you can avoid calling it too often. - If you just want to check if allocation is not lost, vmaTouchAllocation() will work faster. */ VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocationInfo( - VmaAllocator allocator, - VmaAllocation allocation, - VmaAllocationInfo* pAllocationInfo); + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VmaAllocationInfo* VMA_NOT_NULL pAllocationInfo); /** \brief Returns `VK_TRUE` if allocation is not lost and atomically marks it as used in current frame. @@ -2954,8 +3444,8 @@ If the allocation has been created without #VMA_ALLOCATION_CREATE_CAN_BECOME_LOS this function always returns `VK_TRUE`. */ VMA_CALL_PRE VkBool32 VMA_CALL_POST vmaTouchAllocation( - VmaAllocator allocator, - VmaAllocation allocation); + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation); /** \brief Sets pUserData in given allocation to new value. @@ -2971,9 +3461,9 @@ allocation's `pUserData`. It is opaque, so you can use it however you want - e.g as a pointer, ordinal number or some handle to you own data. */ VMA_CALL_PRE void VMA_CALL_POST vmaSetAllocationUserData( - VmaAllocator allocator, - VmaAllocation allocation, - void* pUserData); + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + void* VMA_NULLABLE pUserData); /** \brief Creates new allocation that is in lost state from the beginning. @@ -2986,15 +3476,15 @@ not bound to any image or buffer. It has size = 0. It cannot be turned into a real, non-empty allocation. */ VMA_CALL_PRE void VMA_CALL_POST vmaCreateLostAllocation( - VmaAllocator allocator, - VmaAllocation* pAllocation); + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation); /** \brief Maps memory represented by given allocation and returns pointer to it. Maps memory represented by given allocation to make it accessible to CPU code. When succeeded, `*ppData` contains pointer to first byte of this memory. If the allocation is part of bigger `VkDeviceMemory` block, the pointer is -correctly offseted to the beginning of region assigned to this particular +correctly offsetted to the beginning of region assigned to this particular allocation. Mapping is internally reference-counted and synchronized, so despite raw Vulkan @@ -3028,9 +3518,9 @@ If the allocation is made from a memory types that is not `HOST_COHERENT`, you also need to use vmaInvalidateAllocation() / vmaFlushAllocation(), as required by Vulkan specification. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaMapMemory( - VmaAllocator allocator, - VmaAllocation allocation, - void** ppData); + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + void* VMA_NULLABLE * VMA_NOT_NULL ppData); /** \brief Unmaps memory represented by given allocation, mapped previously using vmaMapMemory(). @@ -3041,8 +3531,8 @@ If the allocation is made from a memory types that is not `HOST_COHERENT`, you also need to use vmaInvalidateAllocation() / vmaFlushAllocation(), as required by Vulkan specification. */ VMA_CALL_PRE void VMA_CALL_POST vmaUnmapMemory( - VmaAllocator allocator, - VmaAllocation allocation); + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation); /** \brief Flushes memory of given allocation. @@ -3061,8 +3551,15 @@ Unmap operation doesn't do that automatically. Warning! `offset` and `size` are relative to the contents of given `allocation`. If you mean whole allocation, you can pass 0 and `VK_WHOLE_SIZE`, respectively. Do not pass allocation's offset as `offset`!!! + +This function returns the `VkResult` from `vkFlushMappedMemoryRanges` if it is +called, otherwise `VK_SUCCESS`. */ -VMA_CALL_PRE void VMA_CALL_POST vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocation( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize offset, + VkDeviceSize size); /** \brief Invalidates memory of given allocation. @@ -3081,8 +3578,57 @@ Map operation doesn't do that automatically. Warning! `offset` and `size` are relative to the contents of given `allocation`. If you mean whole allocation, you can pass 0 and `VK_WHOLE_SIZE`, respectively. Do not pass allocation's offset as `offset`!!! + +This function returns the `VkResult` from `vkInvalidateMappedMemoryRanges` if +it is called, otherwise `VK_SUCCESS`. */ -VMA_CALL_PRE void VMA_CALL_POST vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocation( + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkDeviceSize offset, + VkDeviceSize size); + +/** \brief Flushes memory of given set of allocations. + +Calls `vkFlushMappedMemoryRanges()` for memory associated with given ranges of given allocations. +For more information, see documentation of vmaFlushAllocation(). + +\param allocator +\param allocationCount +\param allocations +\param offsets If not null, it must point to an array of offsets of regions to flush, relative to the beginning of respective allocations. Null means all ofsets are zero. +\param sizes If not null, it must point to an array of sizes of regions to flush in respective allocations. Null means `VK_WHOLE_SIZE` for all allocations. + +This function returns the `VkResult` from `vkFlushMappedMemoryRanges` if it is +called, otherwise `VK_SUCCESS`. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocations( + VmaAllocator VMA_NOT_NULL allocator, + uint32_t allocationCount, + const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) allocations, + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) offsets, + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) sizes); + +/** \brief Invalidates memory of given set of allocations. + +Calls `vkInvalidateMappedMemoryRanges()` for memory associated with given ranges of given allocations. +For more information, see documentation of vmaInvalidateAllocation(). + +\param allocator +\param allocationCount +\param allocations +\param offsets If not null, it must point to an array of offsets of regions to flush, relative to the beginning of respective allocations. Null means all ofsets are zero. +\param sizes If not null, it must point to an array of sizes of regions to flush in respective allocations. Null means `VK_WHOLE_SIZE` for all allocations. + +This function returns the `VkResult` from `vkInvalidateMappedMemoryRanges` if it is +called, otherwise `VK_SUCCESS`. +*/ +VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocations( + VmaAllocator VMA_NOT_NULL allocator, + uint32_t allocationCount, + const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) allocations, + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) offsets, + const VkDeviceSize* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) sizes); /** \brief Checks magic number in margins around all allocations in given memory types (in both default and custom pools) in search for corruptions. @@ -3100,7 +3646,7 @@ Possible return values: `VMA_ASSERT` is also fired in that case. - Other value: Error returned by Vulkan, e.g. memory mapping failure. */ -VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckCorruption(VmaAllocator VMA_NOT_NULL allocator, uint32_t memoryTypeBits); /** \struct VmaDefragmentationContext \brief Represents Opaque object that represents started defragmentation process. @@ -3112,6 +3658,7 @@ VK_DEFINE_HANDLE(VmaDefragmentationContext) /// Flags to be used in vmaDefragmentationBegin(). None at the moment. Reserved for future use. typedef enum VmaDefragmentationFlagBits { + VMA_DEFRAGMENTATION_FLAG_INCREMENTAL = 0x1, VMA_DEFRAGMENTATION_FLAG_BITS_MAX_ENUM = 0x7FFFFFFF } VmaDefragmentationFlagBits; typedef VkFlags VmaDefragmentationFlags; @@ -3135,13 +3682,13 @@ typedef struct VmaDefragmentationInfo2 { It is safe to pass allocations that are in the lost state - they are ignored. All allocations not present in this array are considered non-moveable during this defragmentation. */ - VmaAllocation* pAllocations; + const VmaAllocation VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations; /** \brief Optional, output. Pointer to array that will be filled with information whether the allocation at certain index has been changed during defragmentation. The array should have `allocationCount` elements. You can pass null if you are not interested in this information. */ - VkBool32* pAllocationsChanged; + VkBool32* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationsChanged; /** \brief Numer of pools in `pPools` array. */ uint32_t poolCount; @@ -3160,9 +3707,9 @@ typedef struct VmaDefragmentationInfo2 { Using this array is equivalent to specifying all allocations from the pools in `pAllocations`. It might be more efficient. */ - VmaPool* pPools; + const VmaPool VMA_NOT_NULL * VMA_NULLABLE VMA_LEN_IF_NOT_NULL(poolCount) pPools; /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places using transfers on CPU side, like `memcpy()`, `memmove()`. - + `VK_WHOLE_SIZE` means no limit. */ VkDeviceSize maxCpuBytesToMove; @@ -3172,7 +3719,7 @@ typedef struct VmaDefragmentationInfo2 { */ uint32_t maxCpuAllocationsToMove; /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places using transfers on GPU side, posted to `commandBuffer`. - + `VK_WHOLE_SIZE` means no limit. */ VkDeviceSize maxGpuBytesToMove; @@ -3189,16 +3736,31 @@ typedef struct VmaDefragmentationInfo2 { Passing null means that only CPU defragmentation will be performed. */ - VkCommandBuffer commandBuffer; + VkCommandBuffer VMA_NULLABLE commandBuffer; } VmaDefragmentationInfo2; +typedef struct VmaDefragmentationPassMoveInfo { + VmaAllocation VMA_NOT_NULL allocation; + VkDeviceMemory VMA_NOT_NULL_NON_DISPATCHABLE memory; + VkDeviceSize offset; +} VmaDefragmentationPassMoveInfo; + +/** \brief Parameters for incremental defragmentation steps. + +To be used with function vmaBeginDefragmentationPass(). +*/ +typedef struct VmaDefragmentationPassInfo { + uint32_t moveCount; + VmaDefragmentationPassMoveInfo* VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(moveCount) pMoves; +} VmaDefragmentationPassInfo; + /** \brief Deprecated. Optional configuration parameters to be passed to function vmaDefragment(). \deprecated This is a part of the old interface. It is recommended to use structure #VmaDefragmentationInfo2 and function vmaDefragmentationBegin() instead. */ typedef struct VmaDefragmentationInfo { /** \brief Maximum total numbers of bytes that can be copied while moving allocations to different places. - + Default is `VK_WHOLE_SIZE`, which means no limit. */ VkDeviceSize maxBytesToMove; @@ -3251,10 +3813,10 @@ For more information and important limitations regarding defragmentation, see do [Defragmentation](@ref defragmentation). */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationBegin( - VmaAllocator allocator, - const VmaDefragmentationInfo2* pInfo, - VmaDefragmentationStats* pStats, - VmaDefragmentationContext *pContext); + VmaAllocator VMA_NOT_NULL allocator, + const VmaDefragmentationInfo2* VMA_NOT_NULL pInfo, + VmaDefragmentationStats* VMA_NULLABLE pStats, + VmaDefragmentationContext VMA_NULLABLE * VMA_NOT_NULL pContext); /** \brief Ends defragmentation process. @@ -3262,8 +3824,18 @@ Use this function to finish defragmentation started by vmaDefragmentationBegin() It is safe to pass `context == null`. The function then does nothing. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationEnd( - VmaAllocator allocator, - VmaDefragmentationContext context); + VmaAllocator VMA_NOT_NULL allocator, + VmaDefragmentationContext VMA_NULLABLE context); + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBeginDefragmentationPass( + VmaAllocator VMA_NOT_NULL allocator, + VmaDefragmentationContext VMA_NULLABLE context, + VmaDefragmentationPassInfo* VMA_NOT_NULL pInfo +); +VMA_CALL_PRE VkResult VMA_CALL_POST vmaEndDefragmentationPass( + VmaAllocator VMA_NOT_NULL allocator, + VmaDefragmentationContext VMA_NULLABLE context +); /** \brief Deprecated. Compacts memory by moving allocations. @@ -3306,12 +3878,12 @@ you should measure that on your platform. For more information, see [Defragmentation](@ref defragmentation) chapter. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragment( - VmaAllocator allocator, - VmaAllocation* pAllocations, + VmaAllocator VMA_NOT_NULL allocator, + const VmaAllocation VMA_NOT_NULL * VMA_NOT_NULL VMA_LEN_IF_NOT_NULL(allocationCount) pAllocations, size_t allocationCount, - VkBool32* pAllocationsChanged, - const VmaDefragmentationInfo *pDefragmentationInfo, - VmaDefragmentationStats* pDefragmentationStats); + VkBool32* VMA_NULLABLE VMA_LEN_IF_NOT_NULL(allocationCount) pAllocationsChanged, + const VmaDefragmentationInfo* VMA_NULLABLE pDefragmentationInfo, + VmaDefragmentationStats* VMA_NULLABLE pDefragmentationStats); /** \brief Binds buffer to allocation. @@ -3326,26 +3898,26 @@ allocations, calls to `vkBind*Memory()` or `vkMapMemory()` won't happen from mul It is recommended to use function vmaCreateBuffer() instead of this one. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory( - VmaAllocator allocator, - VmaAllocation allocation, - VkBuffer buffer); + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer); /** \brief Binds buffer to allocation with additional parameters. -@param allocationLocalOffset Additional offset to be added while binding, relative to the beginnig of the `allocation`. Normally it should be 0. +@param allocationLocalOffset Additional offset to be added while binding, relative to the beginning of the `allocation`. Normally it should be 0. @param pNext A chain of structures to be attached to `VkBindBufferMemoryInfoKHR` structure used internally. Normally it should be null. This function is similar to vmaBindBufferMemory(), but it provides additional parameters. If `pNext` is not null, #VmaAllocator object must have been created with #VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT flag -or with VmaAllocatorCreateInfo::vulkanApiVersion `== VK_API_VERSION_1_1`. Otherwise the call fails. +or with VmaAllocatorCreateInfo::vulkanApiVersion `>= VK_API_VERSION_1_1`. Otherwise the call fails. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory2( - VmaAllocator allocator, - VmaAllocation allocation, + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, VkDeviceSize allocationLocalOffset, - VkBuffer buffer, - const void* pNext); + VkBuffer VMA_NOT_NULL_NON_DISPATCHABLE buffer, + const void* VMA_NULLABLE pNext); /** \brief Binds image to allocation. @@ -3360,26 +3932,26 @@ allocations, calls to `vkBind*Memory()` or `vkMapMemory()` won't happen from mul It is recommended to use function vmaCreateImage() instead of this one. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory( - VmaAllocator allocator, - VmaAllocation allocation, - VkImage image); + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, + VkImage VMA_NOT_NULL_NON_DISPATCHABLE image); /** \brief Binds image to allocation with additional parameters. -@param allocationLocalOffset Additional offset to be added while binding, relative to the beginnig of the `allocation`. Normally it should be 0. +@param allocationLocalOffset Additional offset to be added while binding, relative to the beginning of the `allocation`. Normally it should be 0. @param pNext A chain of structures to be attached to `VkBindImageMemoryInfoKHR` structure used internally. Normally it should be null. This function is similar to vmaBindImageMemory(), but it provides additional parameters. If `pNext` is not null, #VmaAllocator object must have been created with #VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT flag -or with VmaAllocatorCreateInfo::vulkanApiVersion `== VK_API_VERSION_1_1`. Otherwise the call fails. +or with VmaAllocatorCreateInfo::vulkanApiVersion `>= VK_API_VERSION_1_1`. Otherwise the call fails. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindImageMemory2( - VmaAllocator allocator, - VmaAllocation allocation, + VmaAllocator VMA_NOT_NULL allocator, + VmaAllocation VMA_NOT_NULL allocation, VkDeviceSize allocationLocalOffset, - VkImage image, - const void* pNext); + VkImage VMA_NOT_NULL_NON_DISPATCHABLE image, + const void* VMA_NULLABLE pNext); /** @param[out] pBuffer Buffer that was created. @@ -3399,21 +3971,25 @@ If the function succeeded, you must destroy both buffer and allocation when you no longer need them using either convenience function vmaDestroyBuffer() or separately, using `vkDestroyBuffer()` and vmaFreeMemory(). -If VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT flag was used, +If #VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT flag was used, VK_KHR_dedicated_allocation extension is used internally to query driver whether it requires or prefers the new buffer to have dedicated allocation. If yes, and if dedicated allocation is possible (VmaAllocationCreateInfo::pool is null -and VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT is not used), it creates dedicated +and #VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT is not used), it creates dedicated allocation for this buffer, just like when using -VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT. +#VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT. + +\note This function creates a new `VkBuffer`. Sub-allocation of parts of one large buffer, +although recommended as a good practice, is out of scope of this library and could be implemented +by the user as a higher-level logic on top of VMA. */ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateBuffer( - VmaAllocator allocator, - const VkBufferCreateInfo* pBufferCreateInfo, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - VkBuffer* pBuffer, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); + VmaAllocator VMA_NOT_NULL allocator, + const VkBufferCreateInfo* VMA_NOT_NULL pBufferCreateInfo, + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + VkBuffer VMA_NULLABLE_NON_DISPATCHABLE * VMA_NOT_NULL pBuffer, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); /** \brief Destroys Vulkan buffer and frees allocated memory. @@ -3427,18 +4003,18 @@ vmaFreeMemory(allocator, allocation); It it safe to pass null as buffer and/or allocation. */ VMA_CALL_PRE void VMA_CALL_POST vmaDestroyBuffer( - VmaAllocator allocator, - VkBuffer buffer, - VmaAllocation allocation); + VmaAllocator VMA_NOT_NULL allocator, + VkBuffer VMA_NULLABLE_NON_DISPATCHABLE buffer, + VmaAllocation VMA_NULLABLE allocation); /// Function similar to vmaCreateBuffer(). VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateImage( - VmaAllocator allocator, - const VkImageCreateInfo* pImageCreateInfo, - const VmaAllocationCreateInfo* pAllocationCreateInfo, - VkImage* pImage, - VmaAllocation* pAllocation, - VmaAllocationInfo* pAllocationInfo); + VmaAllocator VMA_NOT_NULL allocator, + const VkImageCreateInfo* VMA_NOT_NULL pImageCreateInfo, + const VmaAllocationCreateInfo* VMA_NOT_NULL pAllocationCreateInfo, + VkImage VMA_NULLABLE_NON_DISPATCHABLE * VMA_NOT_NULL pImage, + VmaAllocation VMA_NULLABLE * VMA_NOT_NULL pAllocation, + VmaAllocationInfo* VMA_NULLABLE pAllocationInfo); /** \brief Destroys Vulkan image and frees allocated memory. @@ -3452,9 +4028,9 @@ vmaFreeMemory(allocator, allocation); It it safe to pass null as image and/or allocation. */ VMA_CALL_PRE void VMA_CALL_POST vmaDestroyImage( - VmaAllocator allocator, - VkImage image, - VmaAllocation allocation); + VmaAllocator VMA_NOT_NULL allocator, + VkImage VMA_NULLABLE_NON_DISPATCHABLE image, + VmaAllocation VMA_NULLABLE allocation); #ifdef __cplusplus } @@ -3473,6 +4049,17 @@ VMA_CALL_PRE void VMA_CALL_POST vmaDestroyImage( #include <cstdint> #include <cstdlib> #include <cstring> +#include <utility> + +#if VMA_RECORDING_ENABLED + #include <chrono> + #if defined(_WIN32) + #include <windows.h> + #else + #include <sstream> + #include <thread> + #endif +#endif /******************************************************************************* CONFIGURATION SECTION @@ -3486,12 +4073,23 @@ Define this macro to 1 to make the library fetch pointers to Vulkan functions internally, like: vulkanFunctions.vkAllocateMemory = &vkAllocateMemory; - -Define to 0 if you are going to provide you own pointers to Vulkan functions via -VmaAllocatorCreateInfo::pVulkanFunctions. */ #if !defined(VMA_STATIC_VULKAN_FUNCTIONS) && !defined(VK_NO_PROTOTYPES) -#define VMA_STATIC_VULKAN_FUNCTIONS 1 + #define VMA_STATIC_VULKAN_FUNCTIONS 1 +#endif + +/* +Define this macro to 1 to make the library fetch pointers to Vulkan functions +internally, like: + + vulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkGetDeviceProcAddr(m_hDevice, vkAllocateMemory); +*/ +#if !defined(VMA_DYNAMIC_VULKAN_FUNCTIONS) + #define VMA_DYNAMIC_VULKAN_FUNCTIONS 1 + #if defined(VK_NO_PROTOTYPES) + extern PFN_vkGetInstanceProcAddr vkGetInstanceProcAddr; + extern PFN_vkGetDeviceProcAddr vkGetDeviceProcAddr; + #endif #endif // Define this macro to 1 to make the library use STL containers instead of its own implementation. @@ -3554,7 +4152,7 @@ remove them if not needed. #if defined(__ANDROID_API__) && (__ANDROID_API__ < 16) #include <cstdlib> -void *aligned_alloc(size_t alignment, size_t size) +static void* vma_aligned_alloc(size_t alignment, size_t size) { // alignment must be >= sizeof(void*) if(alignment < sizeof(void*)) @@ -3566,8 +4164,25 @@ void *aligned_alloc(size_t alignment, size_t size) } #elif defined(__APPLE__) || defined(__ANDROID__) || (defined(__linux__) && defined(__GLIBCXX__) && !defined(_GLIBCXX_HAVE_ALIGNED_ALLOC)) #include <cstdlib> -void *aligned_alloc(size_t alignment, size_t size) -{ + +#if defined(__APPLE__) +#include <AvailabilityMacros.h> +#endif + +static void* vma_aligned_alloc(size_t alignment, size_t size) +{ +#if defined(__APPLE__) && (defined(MAC_OS_X_VERSION_10_16) || defined(__IPHONE_14_0)) +#if MAC_OS_X_VERSION_MAX_ALLOWED >= MAC_OS_X_VERSION_10_16 || __IPHONE_OS_VERSION_MAX_ALLOWED >= __IPHONE_14_0 + // For C++14, usr/include/malloc/_malloc.h declares aligned_alloc()) only + // with the MacOSX11.0 SDK in Xcode 12 (which is what adds + // MAC_OS_X_VERSION_10_16), even though the function is marked + // availabe for 10.15. That's why the preprocessor checks for 10.16 but + // the __builtin_available checks for 10.15. + // People who use C++17 could call aligned_alloc with the 10.15 SDK already. + if (__builtin_available(macOS 10.15, iOS 13, *)) + return aligned_alloc(alignment, size); +#endif +#endif // alignment must be >= sizeof(void*) if(alignment < sizeof(void*)) { @@ -3579,6 +4194,28 @@ void *aligned_alloc(size_t alignment, size_t size) return pointer; return VMA_NULL; } +#elif defined(_WIN32) +static void* vma_aligned_alloc(size_t alignment, size_t size) +{ + return _aligned_malloc(size, alignment); +} +#else +static void* vma_aligned_alloc(size_t alignment, size_t size) +{ + return aligned_alloc(alignment, size); +} +#endif + +#if defined(_WIN32) +static void vma_aligned_free(void* ptr) +{ + _aligned_free(ptr); +} +#else +static void vma_aligned_free(void* VMA_NULLABLE ptr) +{ + free(ptr); +} #endif // If your compiler is not compatible with C++11 and definition of @@ -3588,20 +4225,20 @@ void *aligned_alloc(size_t alignment, size_t size) // Normal assert to check for programmer's errors, especially in Debug configuration. #ifndef VMA_ASSERT - #ifdef _DEBUG - #define VMA_ASSERT(expr) assert(expr) - #else + #ifdef NDEBUG #define VMA_ASSERT(expr) + #else + #define VMA_ASSERT(expr) assert(expr) #endif #endif // Assert that will be called very often, like inside data structures e.g. operator[]. // Making it non-empty can make program slow. #ifndef VMA_HEAVY_ASSERT - #ifdef _DEBUG - #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr) - #else + #ifdef NDEBUG #define VMA_HEAVY_ASSERT(expr) + #else + #define VMA_HEAVY_ASSERT(expr) //VMA_ASSERT(expr) #endif #endif @@ -3610,19 +4247,16 @@ void *aligned_alloc(size_t alignment, size_t size) #endif #ifndef VMA_SYSTEM_ALIGNED_MALLOC - #if defined(_WIN32) - #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (_aligned_malloc((size), (alignment))) - #else - #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) (aligned_alloc((alignment), (size) )) - #endif + #define VMA_SYSTEM_ALIGNED_MALLOC(size, alignment) vma_aligned_alloc((alignment), (size)) #endif -#ifndef VMA_SYSTEM_FREE - #if defined(_WIN32) - #define VMA_SYSTEM_FREE(ptr) _aligned_free(ptr) +#ifndef VMA_SYSTEM_ALIGNED_FREE + // VMA_SYSTEM_FREE is the old name, but might have been defined by the user + #if defined(VMA_SYSTEM_FREE) + #define VMA_SYSTEM_ALIGNED_FREE(ptr) VMA_SYSTEM_FREE(ptr) #else - #define VMA_SYSTEM_FREE(ptr) free(ptr) - #endif + #define VMA_SYSTEM_ALIGNED_FREE(ptr) vma_aligned_free(ptr) + #endif #endif #ifndef VMA_MIN @@ -3653,15 +4287,15 @@ void *aligned_alloc(size_t alignment, size_t size) // Define this macro to 1 to enable functions: vmaBuildStatsString, vmaFreeStatsString. #if VMA_STATS_STRING_ENABLED - static inline void VmaUint32ToStr(char* outStr, size_t strLen, uint32_t num) + static inline void VmaUint32ToStr(char* VMA_NOT_NULL outStr, size_t strLen, uint32_t num) { snprintf(outStr, strLen, "%u", static_cast<unsigned int>(num)); } - static inline void VmaUint64ToStr(char* outStr, size_t strLen, uint64_t num) + static inline void VmaUint64ToStr(char* VMA_NOT_NULL outStr, size_t strLen, uint64_t num) { snprintf(outStr, strLen, "%llu", static_cast<unsigned long long>(num)); } - static inline void VmaPtrToStr(char* outStr, size_t strLen, const void* ptr) + static inline void VmaPtrToStr(char* VMA_NOT_NULL outStr, size_t strLen, const void* ptr) { snprintf(outStr, strLen, "%p", ptr); } @@ -3673,6 +4307,7 @@ void *aligned_alloc(size_t alignment, size_t size) public: void Lock() { m_Mutex.lock(); } void Unlock() { m_Mutex.unlock(); } + bool TryLock() { return m_Mutex.try_lock(); } private: std::mutex m_Mutex; }; @@ -3689,8 +4324,10 @@ void *aligned_alloc(size_t alignment, size_t size) public: void LockRead() { m_Mutex.lock_shared(); } void UnlockRead() { m_Mutex.unlock_shared(); } + bool TryLockRead() { return m_Mutex.try_lock_shared(); } void LockWrite() { m_Mutex.lock(); } void UnlockWrite() { m_Mutex.unlock(); } + bool TryLockWrite() { return m_Mutex.try_lock(); } private: std::shared_mutex m_Mutex; }; @@ -3704,8 +4341,10 @@ void *aligned_alloc(size_t alignment, size_t size) VmaRWMutex() { InitializeSRWLock(&m_Lock); } void LockRead() { AcquireSRWLockShared(&m_Lock); } void UnlockRead() { ReleaseSRWLockShared(&m_Lock); } + bool TryLockRead() { return TryAcquireSRWLockShared(&m_Lock) != FALSE; } void LockWrite() { AcquireSRWLockExclusive(&m_Lock); } void UnlockWrite() { ReleaseSRWLockExclusive(&m_Lock); } + bool TryLockWrite() { return TryAcquireSRWLockExclusive(&m_Lock) != FALSE; } private: SRWLOCK m_Lock; }; @@ -3717,8 +4356,10 @@ void *aligned_alloc(size_t alignment, size_t size) public: void LockRead() { m_Mutex.Lock(); } void UnlockRead() { m_Mutex.Unlock(); } + bool TryLockRead() { return m_Mutex.TryLock(); } void LockWrite() { m_Mutex.Lock(); } void UnlockWrite() { m_Mutex.Unlock(); } + bool TryLockWrite() { return m_Mutex.TryLock(); } private: VMA_MUTEX m_Mutex; }; @@ -3747,12 +4388,16 @@ If providing your own implementation, you need to implement a subset of std::ato #define VMA_DEBUG_ALWAYS_DEDICATED_MEMORY (0) #endif -#ifndef VMA_DEBUG_ALIGNMENT +#ifndef VMA_MIN_ALIGNMENT /** Minimum alignment of all allocations, in bytes. - Set to more than 1 for debugging purposes only. Must be power of two. + Set to more than 1 for debugging purposes. Must be power of two. */ - #define VMA_DEBUG_ALIGNMENT (1) + #ifdef VMA_DEBUG_ALIGNMENT // Old name + #define VMA_MIN_ALIGNMENT VMA_DEBUG_ALIGNMENT + #else + #define VMA_MIN_ALIGNMENT (1) + #endif #endif #ifndef VMA_DEBUG_MARGIN @@ -3796,6 +4441,14 @@ If providing your own implementation, you need to implement a subset of std::ato #define VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY (1) #endif +#ifndef VMA_DEBUG_DONT_EXCEED_MAX_MEMORY_ALLOCATION_COUNT + /* + Set this to 1 to make VMA never exceed VkPhysicalDeviceLimits::maxMemoryAllocationCount + and return error instead of leaving up to Vulkan implementation what to do in such cases. + */ + #define VMA_DEBUG_DONT_EXCEED_MAX_MEMORY_ALLOCATION_COUNT (0) +#endif + #ifndef VMA_SMALL_HEAP_MAX_SIZE /// Maximum size of a memory heap in Vulkan to consider it "small". #define VMA_SMALL_HEAP_MAX_SIZE (1024ull * 1024 * 1024) @@ -3825,6 +4478,12 @@ static const uint8_t VMA_ALLOCATION_FILL_PATTERN_DESTROYED = 0xEF; END OF CONFIGURATION */ +// # Copy of some Vulkan definitions so we don't need to check their existence just to handle few constants. + +static const uint32_t VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY = 0x00000040; +static const uint32_t VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY = 0x00000080; +static const uint32_t VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_COPY = 0x00020000; + static const uint32_t VMA_ALLOCATION_INTERNAL_STRATEGY_MIN_OFFSET = 0x10000000u; static VkAllocationCallbacks VmaEmptyAllocationCallbacks = { @@ -3833,51 +4492,53 @@ static VkAllocationCallbacks VmaEmptyAllocationCallbacks = { // Returns number of bits set to 1 in (v). static inline uint32_t VmaCountBitsSet(uint32_t v) { - uint32_t c = v - ((v >> 1) & 0x55555555); - c = ((c >> 2) & 0x33333333) + (c & 0x33333333); - c = ((c >> 4) + c) & 0x0F0F0F0F; - c = ((c >> 8) + c) & 0x00FF00FF; - c = ((c >> 16) + c) & 0x0000FFFF; - return c; + uint32_t c = v - ((v >> 1) & 0x55555555); + c = ((c >> 2) & 0x33333333) + (c & 0x33333333); + c = ((c >> 4) + c) & 0x0F0F0F0F; + c = ((c >> 8) + c) & 0x00FF00FF; + c = ((c >> 16) + c) & 0x0000FFFF; + return c; +} + +/* +Returns true if given number is a power of two. +T must be unsigned integer number or signed integer but always nonnegative. +For 0 returns true. +*/ +template <typename T> +inline bool VmaIsPow2(T x) +{ + return (x & (x-1)) == 0; } // Aligns given value up to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 16. // Use types like uint32_t, uint64_t as T. template <typename T> -static inline T VmaAlignUp(T val, T align) +static inline T VmaAlignUp(T val, T alignment) { - return (val + align - 1) / align * align; + VMA_HEAVY_ASSERT(VmaIsPow2(alignment)); + return (val + alignment - 1) & ~(alignment - 1); } // Aligns given value down to nearest multiply of align value. For example: VmaAlignUp(11, 8) = 8. // Use types like uint32_t, uint64_t as T. template <typename T> -static inline T VmaAlignDown(T val, T align) +static inline T VmaAlignDown(T val, T alignment) { - return val / align * align; + VMA_HEAVY_ASSERT(VmaIsPow2(alignment)); + return val & ~(alignment - 1); } // Division with mathematical rounding to nearest number. template <typename T> static inline T VmaRoundDiv(T x, T y) { - return (x + (y / (T)2)) / y; -} - -/* -Returns true if given number is a power of two. -T must be unsigned integer number or signed integer but always nonnegative. -For 0 returns true. -*/ -template <typename T> -inline bool VmaIsPow2(T x) -{ - return (x & (x-1)) == 0; + return (x + (y / (T)2)) / y; } // Returns smallest power of 2 greater or equal to v. static inline uint32_t VmaNextPow2(uint32_t v) { - v--; + v--; v |= v >> 1; v |= v >> 2; v |= v >> 4; @@ -3888,7 +4549,7 @@ static inline uint32_t VmaNextPow2(uint32_t v) } static inline uint64_t VmaNextPow2(uint64_t v) { - v--; + v--; v |= v >> 1; v |= v >> 2; v |= v >> 4; @@ -4033,7 +4694,7 @@ static inline bool VmaIsBufferImageGranularityConflict( { VMA_SWAP(suballocType1, suballocType2); } - + switch(suballocType1) { case VMA_SUBALLOCATION_TYPE_FREE: @@ -4167,7 +4828,7 @@ static IterT VmaBinaryFindFirstNotLess(IterT beg, IterT end, const KeyT &key, co size_t down = 0, up = (end - beg); while(down < up) { - const size_t mid = (down + up) / 2; + const size_t mid = down + (up - down) / 2; // Overflow-safe midpoint calculation if(cmp(*(beg+mid), key)) { down = mid + 1; @@ -4219,15 +4880,23 @@ static bool VmaValidatePointerArray(uint32_t count, const T* arr) return true; } +template<typename MainT, typename NewT> +static inline void VmaPnextChainPushFront(MainT* mainStruct, NewT* newStruct) +{ + newStruct->pNext = mainStruct->pNext; + mainStruct->pNext = newStruct; +} + //////////////////////////////////////////////////////////////////////////////// // Memory allocation static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t size, size_t alignment) { + void* result = VMA_NULL; if((pAllocationCallbacks != VMA_NULL) && (pAllocationCallbacks->pfnAllocation != VMA_NULL)) { - return (*pAllocationCallbacks->pfnAllocation)( + result = (*pAllocationCallbacks->pfnAllocation)( pAllocationCallbacks->pUserData, size, alignment, @@ -4235,8 +4904,10 @@ static void* VmaMalloc(const VkAllocationCallbacks* pAllocationCallbacks, size_t } else { - return VMA_SYSTEM_ALIGNED_MALLOC(size, alignment); + result = VMA_SYSTEM_ALIGNED_MALLOC(size, alignment); } + VMA_ASSERT(result != VMA_NULL && "CPU memory allocation failed."); + return result; } static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr) @@ -4248,7 +4919,7 @@ static void VmaFree(const VkAllocationCallbacks* pAllocationCallbacks, void* ptr } else { - VMA_SYSTEM_FREE(ptr); + VMA_SYSTEM_ALIGNED_FREE(ptr); } } @@ -4319,7 +4990,7 @@ class VmaStlAllocator public: const VkAllocationCallbacks* const m_pCallbacks; typedef T value_type; - + VmaStlAllocator(const VkAllocationCallbacks* pCallbacks) : m_pCallbacks(pCallbacks) { } template<typename U> VmaStlAllocator(const VmaStlAllocator<U>& src) : m_pCallbacks(src.m_pCallbacks) { } @@ -4338,6 +5009,7 @@ public: } VmaStlAllocator& operator=(const VmaStlAllocator& x) = delete; + VmaStlAllocator(const VmaStlAllocator&) = default; }; #if VMA_USE_STL_VECTOR @@ -4382,12 +5054,12 @@ public: m_Capacity(count) { } - + // This version of the constructor is here for compatibility with pre-C++14 std::vector. // value is unused. VmaVector(size_t count, const T& value, const AllocatorT& allocator) : VmaVector(count, allocator) {} - + VmaVector(const VmaVector<T, AllocatorT>& src) : m_Allocator(src.m_Allocator), m_pArray(src.m_Count ? (T*)VmaAllocateArray<T>(src.m_Allocator.m_pCallbacks, src.m_Count) : VMA_NULL), @@ -4399,7 +5071,7 @@ public: memcpy(m_pArray, src.m_pArray, m_Count * sizeof(T)); } } - + ~VmaVector() { VmaFree(m_Allocator.m_pCallbacks, m_pArray); @@ -4417,12 +5089,12 @@ public: } return *this; } - + bool empty() const { return m_Count == 0; } size_t size() const { return m_Count; } T* data() { return m_pArray; } const T* data() const { return m_pArray; } - + T& operator[](size_t index) { VMA_HEAVY_ASSERT(index < m_Count); @@ -4458,12 +5130,12 @@ public: void reserve(size_t newCapacity, bool freeMemory = false) { newCapacity = VMA_MAX(newCapacity, m_Count); - + if((newCapacity < m_Capacity) && !freeMemory) { newCapacity = m_Capacity; } - + if(newCapacity != m_Capacity) { T* const newArray = newCapacity ? VmaAllocateArray<T>(m_Allocator, newCapacity) : VMA_NULL; @@ -4477,17 +5149,13 @@ public: } } - void resize(size_t newCount, bool freeMemory = false) + void resize(size_t newCount) { size_t newCapacity = m_Capacity; if(newCount > m_Capacity) { newCapacity = VMA_MAX(newCount, VMA_MAX(m_Capacity * 3 / 2, (size_t)8)); } - else if(freeMemory) - { - newCapacity = newCount; - } if(newCapacity != m_Capacity) { @@ -4505,9 +5173,25 @@ public: m_Count = newCount; } - void clear(bool freeMemory = false) + void clear() + { + resize(0); + } + + void shrink_to_fit() { - resize(0, freeMemory); + if(m_Capacity > m_Count) + { + T* newArray = VMA_NULL; + if(m_Count > 0) + { + newArray = VmaAllocateArray<T>(m_Allocator.m_pCallbacks, m_Count); + memcpy(newArray, m_pArray, m_Count * sizeof(T)); + } + VmaFree(m_Allocator.m_pCallbacks, m_pArray); + m_Capacity = m_Count; + m_pArray = newArray; + } } void insert(size_t index, const T& src) @@ -4558,9 +5242,14 @@ public: } typedef T* iterator; + typedef const T* const_iterator; iterator begin() { return m_pArray; } iterator end() { return m_pArray + m_Count; } + const_iterator cbegin() const { return m_pArray; } + const_iterator cend() const { return m_pArray + m_Count; } + const_iterator begin() const { return cbegin(); } + const_iterator end() const { return cend(); } private: AllocatorT m_Allocator; @@ -4614,6 +5303,187 @@ bool VmaVectorRemoveSorted(VectorT& vector, const typename VectorT::value_type& } //////////////////////////////////////////////////////////////////////////////// +// class VmaSmallVector + +/* +This is a vector (a variable-sized array), optimized for the case when the array is small. + +It contains some number of elements in-place, which allows it to avoid heap allocation +when the actual number of elements is below that threshold. This allows normal "small" +cases to be fast without losing generality for large inputs. +*/ + +template<typename T, typename AllocatorT, size_t N> +class VmaSmallVector +{ +public: + typedef T value_type; + + VmaSmallVector(const AllocatorT& allocator) : + m_Count(0), + m_DynamicArray(allocator) + { + } + VmaSmallVector(size_t count, const AllocatorT& allocator) : + m_Count(count), + m_DynamicArray(count > N ? count : 0, allocator) + { + } + template<typename SrcT, typename SrcAllocatorT, size_t SrcN> + VmaSmallVector(const VmaSmallVector<SrcT, SrcAllocatorT, SrcN>& src) = delete; + template<typename SrcT, typename SrcAllocatorT, size_t SrcN> + VmaSmallVector<T, AllocatorT, N>& operator=(const VmaSmallVector<SrcT, SrcAllocatorT, SrcN>& rhs) = delete; + + bool empty() const { return m_Count == 0; } + size_t size() const { return m_Count; } + T* data() { return m_Count > N ? m_DynamicArray.data() : m_StaticArray; } + const T* data() const { return m_Count > N ? m_DynamicArray.data() : m_StaticArray; } + + T& operator[](size_t index) + { + VMA_HEAVY_ASSERT(index < m_Count); + return data()[index]; + } + const T& operator[](size_t index) const + { + VMA_HEAVY_ASSERT(index < m_Count); + return data()[index]; + } + + T& front() + { + VMA_HEAVY_ASSERT(m_Count > 0); + return data()[0]; + } + const T& front() const + { + VMA_HEAVY_ASSERT(m_Count > 0); + return data()[0]; + } + T& back() + { + VMA_HEAVY_ASSERT(m_Count > 0); + return data()[m_Count - 1]; + } + const T& back() const + { + VMA_HEAVY_ASSERT(m_Count > 0); + return data()[m_Count - 1]; + } + + void resize(size_t newCount, bool freeMemory = false) + { + if(newCount > N && m_Count > N) + { + // Any direction, staying in m_DynamicArray + m_DynamicArray.resize(newCount); + if(freeMemory) + { + m_DynamicArray.shrink_to_fit(); + } + } + else if(newCount > N && m_Count <= N) + { + // Growing, moving from m_StaticArray to m_DynamicArray + m_DynamicArray.resize(newCount); + if(m_Count > 0) + { + memcpy(m_DynamicArray.data(), m_StaticArray, m_Count * sizeof(T)); + } + } + else if(newCount <= N && m_Count > N) + { + // Shrinking, moving from m_DynamicArray to m_StaticArray + if(newCount > 0) + { + memcpy(m_StaticArray, m_DynamicArray.data(), newCount * sizeof(T)); + } + m_DynamicArray.resize(0); + if(freeMemory) + { + m_DynamicArray.shrink_to_fit(); + } + } + else + { + // Any direction, staying in m_StaticArray - nothing to do here + } + m_Count = newCount; + } + + void clear(bool freeMemory = false) + { + m_DynamicArray.clear(); + if(freeMemory) + { + m_DynamicArray.shrink_to_fit(); + } + m_Count = 0; + } + + void insert(size_t index, const T& src) + { + VMA_HEAVY_ASSERT(index <= m_Count); + const size_t oldCount = size(); + resize(oldCount + 1); + T* const dataPtr = data(); + if(index < oldCount) + { + // I know, this could be more optimal for case where memmove can be memcpy directly from m_StaticArray to m_DynamicArray. + memmove(dataPtr + (index + 1), dataPtr + index, (oldCount - index) * sizeof(T)); + } + dataPtr[index] = src; + } + + void remove(size_t index) + { + VMA_HEAVY_ASSERT(index < m_Count); + const size_t oldCount = size(); + if(index < oldCount - 1) + { + // I know, this could be more optimal for case where memmove can be memcpy directly from m_DynamicArray to m_StaticArray. + T* const dataPtr = data(); + memmove(dataPtr + index, dataPtr + (index + 1), (oldCount - index - 1) * sizeof(T)); + } + resize(oldCount - 1); + } + + void push_back(const T& src) + { + const size_t newIndex = size(); + resize(newIndex + 1); + data()[newIndex] = src; + } + + void pop_back() + { + VMA_HEAVY_ASSERT(m_Count > 0); + resize(size() - 1); + } + + void push_front(const T& src) + { + insert(0, src); + } + + void pop_front() + { + VMA_HEAVY_ASSERT(m_Count > 0); + remove(0); + } + + typedef T* iterator; + + iterator begin() { return data(); } + iterator end() { return data() + m_Count; } + +private: + size_t m_Count; + T m_StaticArray[N]; // Used when m_Size <= N + VmaVector<T, AllocatorT> m_DynamicArray; // Used when m_Size > N +}; + +//////////////////////////////////////////////////////////////////////////////// // class VmaPoolAllocator /* @@ -4628,7 +5498,7 @@ class VmaPoolAllocator public: VmaPoolAllocator(const VkAllocationCallbacks* pAllocationCallbacks, uint32_t firstBlockCapacity); ~VmaPoolAllocator(); - T* Alloc(); + template<typename... Types> T* Alloc(Types... args); void Free(T* ptr); private: @@ -4644,7 +5514,7 @@ private: uint32_t Capacity; uint32_t FirstFreeIndex; }; - + const VkAllocationCallbacks* m_pAllocationCallbacks; const uint32_t m_FirstBlockCapacity; VmaVector< ItemBlock, VmaStlAllocator<ItemBlock> > m_ItemBlocks; @@ -4670,7 +5540,7 @@ VmaPoolAllocator<T>::~VmaPoolAllocator() } template<typename T> -T* VmaPoolAllocator<T>::Alloc() +template<typename... Types> T* VmaPoolAllocator<T>::Alloc(Types... args) { for(size_t i = m_ItemBlocks.size(); i--; ) { @@ -4681,7 +5551,7 @@ T* VmaPoolAllocator<T>::Alloc() Item* const pItem = &block.pItems[block.FirstFreeIndex]; block.FirstFreeIndex = pItem->NextFreeIndex; T* result = (T*)&pItem->Value; - new(result)T(); // Explicit constructor call. + new(result)T(std::forward<Types>(args)...); // Explicit constructor call. return result; } } @@ -4691,7 +5561,7 @@ T* VmaPoolAllocator<T>::Alloc() Item* const pItem = &newBlock.pItems[0]; newBlock.FirstFreeIndex = pItem->NextFreeIndex; T* result = (T*)&pItem->Value; - new(result)T(); // Explicit constructor call. + new(result)T(std::forward<Types>(args)...); // Explicit constructor call. return result; } @@ -4702,11 +5572,11 @@ void VmaPoolAllocator<T>::Free(T* ptr) for(size_t i = m_ItemBlocks.size(); i--; ) { ItemBlock& block = m_ItemBlocks[i]; - + // Casting to union. Item* pItemPtr; memcpy(&pItemPtr, &ptr, sizeof(pItemPtr)); - + // Check if pItemPtr is in address range of this block. if((pItemPtr >= block.pItems) && (pItemPtr < block.pItems + block.Capacity)) { @@ -4783,7 +5653,7 @@ public: ItemType* PushFront(const T& value); void PopBack(); void PopFront(); - + // Item can be null - it means PushBack. ItemType* InsertBefore(ItemType* pItem); // Item can be null - it means PushFront. @@ -5093,7 +5963,7 @@ public: VMA_HEAVY_ASSERT(m_pList == rhs.m_pList); return m_pItem != rhs.m_pItem; } - + private: VmaRawList<T>* m_pList; VmaListItem<T>* m_pItem; @@ -5121,7 +5991,7 @@ public: m_pItem(src.m_pItem) { } - + const T& operator*() const { VMA_HEAVY_ASSERT(m_pItem != VMA_NULL); @@ -5176,7 +6046,7 @@ public: VMA_HEAVY_ASSERT(m_pList == rhs.m_pList); return m_pItem != rhs.m_pItem; } - + private: const_iterator(const VmaRawList<T>* pList, const VmaListItem<T>* pItem) : m_pList(pList), @@ -5201,6 +6071,9 @@ public: const_iterator cbegin() const { return const_iterator(&m_RawList, m_RawList.Front()); } const_iterator cend() const { return const_iterator(&m_RawList, VMA_NULL); } + const_iterator begin() const { return cbegin(); } + const_iterator end() const { return cend(); } + void clear() { m_RawList.Clear(); } void push_back(const T& value) { m_RawList.PushBack(value); } void erase(iterator it) { m_RawList.Remove(it.m_pItem); } @@ -5213,6 +6086,222 @@ private: #endif // #if VMA_USE_STL_LIST //////////////////////////////////////////////////////////////////////////////// +// class VmaIntrusiveLinkedList + +/* +Expected interface of ItemTypeTraits: +struct MyItemTypeTraits +{ + typedef MyItem ItemType; + static ItemType* GetPrev(const ItemType* item) { return item->myPrevPtr; } + static ItemType* GetNext(const ItemType* item) { return item->myNextPtr; } + static ItemType*& AccessPrev(ItemType* item) { return item->myPrevPtr; } + static ItemType*& AccessNext(ItemType* item) { return item->myNextPtr; } +}; +*/ +template<typename ItemTypeTraits> +class VmaIntrusiveLinkedList +{ +public: + typedef typename ItemTypeTraits::ItemType ItemType; + static ItemType* GetPrev(const ItemType* item) { return ItemTypeTraits::GetPrev(item); } + static ItemType* GetNext(const ItemType* item) { return ItemTypeTraits::GetNext(item); } + // Movable, not copyable. + VmaIntrusiveLinkedList() { } + VmaIntrusiveLinkedList(const VmaIntrusiveLinkedList<ItemTypeTraits>& src) = delete; + VmaIntrusiveLinkedList(VmaIntrusiveLinkedList<ItemTypeTraits>&& src) : + m_Front(src.m_Front), m_Back(src.m_Back), m_Count(src.m_Count) + { + src.m_Front = src.m_Back = VMA_NULL; + src.m_Count = 0; + } + ~VmaIntrusiveLinkedList() + { + VMA_HEAVY_ASSERT(IsEmpty()); + } + VmaIntrusiveLinkedList<ItemTypeTraits>& operator=(const VmaIntrusiveLinkedList<ItemTypeTraits>& src) = delete; + VmaIntrusiveLinkedList<ItemTypeTraits>& operator=(VmaIntrusiveLinkedList<ItemTypeTraits>&& src) + { + if(&src != this) + { + VMA_HEAVY_ASSERT(IsEmpty()); + m_Front = src.m_Front; + m_Back = src.m_Back; + m_Count = src.m_Count; + src.m_Front = src.m_Back = VMA_NULL; + src.m_Count = 0; + } + return *this; + } + void RemoveAll() + { + if(!IsEmpty()) + { + ItemType* item = m_Back; + while(item != VMA_NULL) + { + ItemType* const prevItem = ItemTypeTraits::AccessPrev(item); + ItemTypeTraits::AccessPrev(item) = VMA_NULL; + ItemTypeTraits::AccessNext(item) = VMA_NULL; + item = prevItem; + } + m_Front = VMA_NULL; + m_Back = VMA_NULL; + m_Count = 0; + } + } + size_t GetCount() const { return m_Count; } + bool IsEmpty() const { return m_Count == 0; } + ItemType* Front() { return m_Front; } + const ItemType* Front() const { return m_Front; } + ItemType* Back() { return m_Back; } + const ItemType* Back() const { return m_Back; } + void PushBack(ItemType* item) + { + VMA_HEAVY_ASSERT(ItemTypeTraits::GetPrev(item) == VMA_NULL && ItemTypeTraits::GetNext(item) == VMA_NULL); + if(IsEmpty()) + { + m_Front = item; + m_Back = item; + m_Count = 1; + } + else + { + ItemTypeTraits::AccessPrev(item) = m_Back; + ItemTypeTraits::AccessNext(m_Back) = item; + m_Back = item; + ++m_Count; + } + } + void PushFront(ItemType* item) + { + VMA_HEAVY_ASSERT(ItemTypeTraits::GetPrev(item) == VMA_NULL && ItemTypeTraits::GetNext(item) == VMA_NULL); + if(IsEmpty()) + { + m_Front = item; + m_Back = item; + m_Count = 1; + } + else + { + ItemTypeTraits::AccessNext(item) = m_Front; + ItemTypeTraits::AccessPrev(m_Front) = item; + m_Front = item; + ++m_Count; + } + } + ItemType* PopBack() + { + VMA_HEAVY_ASSERT(m_Count > 0); + ItemType* const backItem = m_Back; + ItemType* const prevItem = ItemTypeTraits::GetPrev(backItem); + if(prevItem != VMA_NULL) + { + ItemTypeTraits::AccessNext(prevItem) = VMA_NULL; + } + m_Back = prevItem; + --m_Count; + ItemTypeTraits::AccessPrev(backItem) = VMA_NULL; + ItemTypeTraits::AccessNext(backItem) = VMA_NULL; + return backItem; + } + ItemType* PopFront() + { + VMA_HEAVY_ASSERT(m_Count > 0); + ItemType* const frontItem = m_Front; + ItemType* const nextItem = ItemTypeTraits::GetNext(frontItem); + if(nextItem != VMA_NULL) + { + ItemTypeTraits::AccessPrev(nextItem) = VMA_NULL; + } + m_Front = nextItem; + --m_Count; + ItemTypeTraits::AccessPrev(frontItem) = VMA_NULL; + ItemTypeTraits::AccessNext(frontItem) = VMA_NULL; + return frontItem; + } + + // MyItem can be null - it means PushBack. + void InsertBefore(ItemType* existingItem, ItemType* newItem) + { + VMA_HEAVY_ASSERT(newItem != VMA_NULL && ItemTypeTraits::GetPrev(newItem) == VMA_NULL && ItemTypeTraits::GetNext(newItem) == VMA_NULL); + if(existingItem != VMA_NULL) + { + ItemType* const prevItem = ItemTypeTraits::GetPrev(existingItem); + ItemTypeTraits::AccessPrev(newItem) = prevItem; + ItemTypeTraits::AccessNext(newItem) = existingItem; + ItemTypeTraits::AccessPrev(existingItem) = newItem; + if(prevItem != VMA_NULL) + { + ItemTypeTraits::AccessNext(prevItem) = newItem; + } + else + { + VMA_HEAVY_ASSERT(m_Front == existingItem); + m_Front = newItem; + } + ++m_Count; + } + else + PushBack(newItem); + } + // MyItem can be null - it means PushFront. + void InsertAfter(ItemType* existingItem, ItemType* newItem) + { + VMA_HEAVY_ASSERT(newItem != VMA_NULL && ItemTypeTraits::GetPrev(newItem) == VMA_NULL && ItemTypeTraits::GetNext(newItem) == VMA_NULL); + if(existingItem != VMA_NULL) + { + ItemType* const nextItem = ItemTypeTraits::GetNext(existingItem); + ItemTypeTraits::AccessNext(newItem) = nextItem; + ItemTypeTraits::AccessPrev(newItem) = existingItem; + ItemTypeTraits::AccessNext(existingItem) = newItem; + if(nextItem != VMA_NULL) + { + ItemTypeTraits::AccessPrev(nextItem) = newItem; + } + else + { + VMA_HEAVY_ASSERT(m_Back == existingItem); + m_Back = newItem; + } + ++m_Count; + } + else + return PushFront(newItem); + } + void Remove(ItemType* item) + { + VMA_HEAVY_ASSERT(item != VMA_NULL && m_Count > 0); + if(ItemTypeTraits::GetPrev(item) != VMA_NULL) + { + ItemTypeTraits::AccessNext(ItemTypeTraits::AccessPrev(item)) = ItemTypeTraits::GetNext(item); + } + else + { + VMA_HEAVY_ASSERT(m_Front == item); + m_Front = ItemTypeTraits::GetNext(item); + } + + if(ItemTypeTraits::GetNext(item) != VMA_NULL) + { + ItemTypeTraits::AccessPrev(ItemTypeTraits::AccessNext(item)) = ItemTypeTraits::GetPrev(item); + } + else + { + VMA_HEAVY_ASSERT(m_Back == item); + m_Back = ItemTypeTraits::GetPrev(item); + } + ItemTypeTraits::AccessPrev(item) = VMA_NULL; + ItemTypeTraits::AccessNext(item) = VMA_NULL; + --m_Count; + } +private: + ItemType* m_Front = VMA_NULL; + ItemType* m_Back = VMA_NULL; + size_t m_Count = 0; +}; + +//////////////////////////////////////////////////////////////////////////////// // class VmaMap // Unused in this version. @@ -5255,7 +6344,7 @@ public: void insert(const PairType& pair); iterator find(const KeyT& key); void erase(iterator it); - + private: VmaVector< PairType, VmaStlAllocator<PairType> > m_Vector; }; @@ -5342,25 +6431,24 @@ public: This struct is allocated using VmaPoolAllocator. */ - void Ctor(uint32_t currentFrameIndex, bool userDataString) + VmaAllocation_T(uint32_t currentFrameIndex, bool userDataString) : + m_Alignment{1}, + m_Size{0}, + m_pUserData{VMA_NULL}, + m_LastUseFrameIndex{currentFrameIndex}, + m_MemoryTypeIndex{0}, + m_Type{(uint8_t)ALLOCATION_TYPE_NONE}, + m_SuballocationType{(uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN}, + m_MapCount{0}, + m_Flags{userDataString ? (uint8_t)FLAG_USER_DATA_STRING : (uint8_t)0} { - m_Alignment = 1; - m_Size = 0; - m_MemoryTypeIndex = 0; - m_pUserData = VMA_NULL; - m_LastUseFrameIndex = currentFrameIndex; - m_Type = (uint8_t)ALLOCATION_TYPE_NONE; - m_SuballocationType = (uint8_t)VMA_SUBALLOCATION_TYPE_UNKNOWN; - m_MapCount = 0; - m_Flags = userDataString ? (uint8_t)FLAG_USER_DATA_STRING : 0; - #if VMA_STATS_STRING_ENABLED m_CreationFrameIndex = currentFrameIndex; m_BufferImageUsage = 0; #endif } - void Dtor() + ~VmaAllocation_T() { VMA_ASSERT((m_MapCount & ~MAP_COUNT_FLAG_PERSISTENT_MAP) == 0 && "Allocation was not unmapped before destruction."); @@ -5405,7 +6493,7 @@ public: void ChangeBlockAllocation( VmaAllocator hAllocator, VmaDeviceMemoryBlock* block, - VkDeviceSize offset); + VkDeviceSize offset); void ChangeOffset(VkDeviceSize newOffset); @@ -5427,6 +6515,8 @@ public: m_MapCount = (pMappedData != VMA_NULL) ? MAP_COUNT_FLAG_PERSISTENT_MAP : 0; m_DedicatedAllocation.m_hMemory = hMemory; m_DedicatedAllocation.m_pMappedData = pMappedData; + m_DedicatedAllocation.m_Prev = VMA_NULL; + m_DedicatedAllocation.m_Next = VMA_NULL; } ALLOCATION_TYPE GetType() const { return (ALLOCATION_TYPE)m_Type; } @@ -5448,7 +6538,7 @@ public: bool IsPersistentMap() const { return (m_MapCount & MAP_COUNT_FLAG_PERSISTENT_MAP) != 0; } void* GetMappedData() const; bool CanBecomeLost() const; - + uint32_t GetLastUseFrameIndex() const { return m_LastUseFrameIndex.load(); @@ -5461,7 +6551,7 @@ public: - If hAllocation.LastUseFrameIndex + frameInUseCount < allocator.CurrentFrameIndex, makes it lost by setting LastUseFrameIndex = VMA_FRAME_INDEX_LOST and returns true. - Else, returns false. - + If hAllocation is already lost, assert - you should not call it then. If hAllocation was not created with CAN_BECOME_LOST_BIT, assert. */ @@ -5524,6 +6614,8 @@ private: { VkDeviceMemory m_hMemory; void* m_pMappedData; // Not null means memory is mapped. + VmaAllocation_T* m_Prev; + VmaAllocation_T* m_Next; }; union @@ -5540,6 +6632,32 @@ private: #endif void FreeUserDataString(VmaAllocator hAllocator); + + friend struct VmaDedicatedAllocationListItemTraits; +}; + +struct VmaDedicatedAllocationListItemTraits +{ + typedef VmaAllocation_T ItemType; + static ItemType* GetPrev(const ItemType* item) + { + VMA_HEAVY_ASSERT(item->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED); + return item->m_DedicatedAllocation.m_Prev; + } + static ItemType* GetNext(const ItemType* item) + { + VMA_HEAVY_ASSERT(item->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED); + return item->m_DedicatedAllocation.m_Next; + } + static ItemType*& AccessPrev(ItemType* item) + { + VMA_HEAVY_ASSERT(item->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED); + return item->m_DedicatedAllocation.m_Prev; + } + static ItemType*& AccessNext(ItemType* item){ + VMA_HEAVY_ASSERT(item->GetType() == VmaAllocation_T::ALLOCATION_TYPE_DEDICATED); + return item->m_DedicatedAllocation.m_Next; + } }; /* @@ -5757,7 +6875,7 @@ public: //////////////////////////////////////////////////////////////////////////////// // For defragmentation - + bool IsBufferImageGranularityConflictPossible( VkDeviceSize bufferImageGranularity, VmaSuballocationType& inOutPrevSuballocType) const; @@ -5968,7 +7086,7 @@ private: SuballocationVectorType& AccessSuballocations2nd() { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; } const SuballocationVectorType& AccessSuballocations1st() const { return m_1stVectorIndex ? m_Suballocations1 : m_Suballocations0; } const SuballocationVectorType& AccessSuballocations2nd() const { return m_1stVectorIndex ? m_Suballocations0 : m_Suballocations1; } - + // Number of items in 1st vector with hAllocation = null at the beginning. size_t m_1stNullItemsBeginCount; // Number of other items in 1st vector with hAllocation = null somewhere in the middle. @@ -6006,7 +7124,7 @@ private: - m_UsableSize is this size aligned down to a power of two. All allocations and calculations happen relative to m_UsableSize. - GetUnusableSize() is the difference between them. - It is repoted as separate, unused range, not available for allocations. + It is reported as separate, unused range, not available for allocations. Node at level 0 has size = m_UsableSize. Each next level contains nodes with size 2 times smaller than current level. @@ -6148,6 +7266,8 @@ private: #endif }; +struct VmaBlockVector; + /* Represents a single block of device memory (`VkDeviceMemory`) with all the data about its regions (aka suballocations, #VmaAllocation), assigned and free. @@ -6171,6 +7291,7 @@ public: // Always call after construction. void Init( VmaAllocator hAllocator, + VmaBlockVector* parentBlockVector, VmaPool hParentPool, uint32_t newMemoryTypeIndex, VkDeviceMemory newMemory, @@ -6179,7 +7300,8 @@ public: uint32_t algorithm); // Always call before destruction. void Destroy(VmaAllocator allocator); - + + VmaBlockVector* GetParentBlockVector() const { return m_ParentBlockVector; } VmaPool GetParentPool() const { return m_hParentPool; } VkDeviceMemory GetDeviceMemory() const { return m_hMemory; } uint32_t GetMemoryTypeIndex() const { return m_MemoryTypeIndex; } @@ -6212,10 +7334,11 @@ public: const void* pNext); private: - VmaPool m_hParentPool; // VK_NULL_HANDLE if not belongs to custom pool. - uint32_t m_MemoryTypeIndex; - uint32_t m_Id; - VkDeviceMemory m_hMemory; + VmaBlockVector* m_ParentBlockVector = VMA_NULL; + VmaPool m_hParentPool = VK_NULL_HANDLE; // VK_NULL_HANDLE if not belongs to custom pool. + uint32_t m_MemoryTypeIndex = UINT32_MAX; + uint32_t m_Id = 0; + VkDeviceMemory m_hMemory = VK_NULL_HANDLE; /* Protects access to m_hMemory so it's not used by multiple threads simultaneously, e.g. vkMapMemory, vkBindBufferMemory. @@ -6223,16 +7346,8 @@ private: Allocations, deallocations, any change in m_pMetadata is protected by parent's VmaBlockVector::m_Mutex. */ VMA_MUTEX m_Mutex; - uint32_t m_MapCount; - void* m_pMappedData; -}; - -struct VmaPointerLess -{ - bool operator()(const void* lhs, const void* rhs) const - { - return lhs < rhs; - } + uint32_t m_MapCount = 0; + void* m_pMappedData = VMA_NULL; }; struct VmaDefragmentationMove @@ -6242,6 +7357,9 @@ struct VmaDefragmentationMove VkDeviceSize srcOffset; VkDeviceSize dstOffset; VkDeviceSize size; + VmaAllocation hAllocation; + VmaDeviceMemoryBlock* pSrcBlock; + VmaDeviceMemoryBlock* pDstBlock; }; class VmaDefragmentationAlgorithm; @@ -6266,7 +7384,10 @@ public: VkDeviceSize bufferImageGranularity, uint32_t frameInUseCount, bool explicitBlockSize, - uint32_t algorithm); + uint32_t algorithm, + float priority, + VkDeviceSize minAllocationAlignment, + void* pMemoryAllocateNext); ~VmaBlockVector(); VkResult CreateMinBlocks(); @@ -6311,12 +7432,21 @@ public: // Saves results in pCtx->res. void Defragment( class VmaBlockVectorDefragmentationContext* pCtx, - VmaDefragmentationStats* pStats, + VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags, VkDeviceSize& maxCpuBytesToMove, uint32_t& maxCpuAllocationsToMove, VkDeviceSize& maxGpuBytesToMove, uint32_t& maxGpuAllocationsToMove, VkCommandBuffer commandBuffer); void DefragmentationEnd( class VmaBlockVectorDefragmentationContext* pCtx, + uint32_t flags, + VmaDefragmentationStats* pStats); + + uint32_t ProcessDefragmentations( + class VmaBlockVectorDefragmentationContext *pCtx, + VmaDefragmentationPassMoveInfo* pMove, uint32_t maxMoves); + + void CommitDefragmentations( + class VmaBlockVectorDefragmentationContext *pCtx, VmaDefragmentationStats* pStats); //////////////////////////////////////////////////////////////////////////////// @@ -6340,6 +7470,9 @@ private: const uint32_t m_FrameInUseCount; const bool m_ExplicitBlockSize; const uint32_t m_Algorithm; + const float m_Priority; + const VkDeviceSize m_MinAllocationAlignment; + void* const m_pMemoryAllocateNext; VMA_RW_MUTEX m_Mutex; /* There can be at most one allocation that is completely empty (except when minBlockCount > 0) - @@ -6387,7 +7520,7 @@ private: // Saves result to pCtx->res. void ApplyDefragmentationMovesGpu( class VmaBlockVectorDefragmentationContext* pDefragCtx, - const VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, + VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkCommandBuffer commandBuffer); /* @@ -6424,6 +7557,18 @@ public: private: uint32_t m_Id; char* m_Name; + VmaPool_T* m_PrevPool = VMA_NULL; + VmaPool_T* m_NextPool = VMA_NULL; + friend struct VmaPoolListItemTraits; +}; + +struct VmaPoolListItemTraits +{ + typedef VmaPool_T ItemType; + static ItemType* GetPrev(const ItemType* item) { return item->m_PrevPool; } + static ItemType* GetNext(const ItemType* item) { return item->m_NextPool; } + static ItemType*& AccessPrev(ItemType* item) { return item->m_PrevPool; } + static ItemType*& AccessNext(ItemType* item) { return item->m_NextPool; } }; /* @@ -6456,7 +7601,8 @@ public: virtual VkResult Defragment( VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove) = 0; + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags) = 0; virtual VkDeviceSize GetBytesMoved() const = 0; virtual uint32_t GetAllocationsMoved() const = 0; @@ -6501,7 +7647,8 @@ public: virtual VkResult Defragment( VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove); + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags); virtual VkDeviceSize GetBytesMoved() const { return m_BytesMoved; } virtual uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; } @@ -6602,7 +7749,8 @@ private: VkResult DefragmentRound( VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove); + uint32_t maxAllocationsToMove, + bool freeOldAllocations); size_t CalcBlocksWithNonMovableCount() const; @@ -6628,7 +7776,8 @@ public: virtual VkResult Defragment( VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove); + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags); virtual VkDeviceSize GetBytesMoved() const { return m_BytesMoved; } virtual uint32_t GetAllocationsMoved() const { return m_AllocationsMoved; } @@ -6708,7 +7857,7 @@ private: } } } - + if(bestIndex != SIZE_MAX) { outBlockInfoIndex = m_FreeSpaces[bestIndex].blockInfoIndex; @@ -6776,6 +7925,10 @@ public: VkResult res; bool mutexLocked; VmaVector< VmaBlockDefragmentationContext, VmaStlAllocator<VmaBlockDefragmentationContext> > blockContexts; + VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> > defragmentationMoves; + uint32_t defragmentationMovesProcessed; + uint32_t defragmentationMovesCommitted; + bool hasDefragmentationPlan; VmaBlockVectorDefragmentationContext( VmaAllocator hAllocator, @@ -6791,7 +7944,7 @@ public: void AddAllocation(VmaAllocation hAlloc, VkBool32* pChanged); void AddAll() { m_AllAllocations = true; } - void Begin(bool overlappingMoveSupported); + void Begin(bool overlappingMoveSupported, VmaDefragmentationFlags flags); private: const VmaAllocator m_hAllocator; @@ -6825,28 +7978,37 @@ public: VmaDefragmentationStats* pStats); ~VmaDefragmentationContext_T(); - void AddPools(uint32_t poolCount, VmaPool* pPools); + void AddPools(uint32_t poolCount, const VmaPool* pPools); void AddAllocations( uint32_t allocationCount, - VmaAllocation* pAllocations, + const VmaAllocation* pAllocations, VkBool32* pAllocationsChanged); /* Returns: - `VK_SUCCESS` if succeeded and object can be destroyed immediately. - `VK_NOT_READY` if succeeded but the object must remain alive until vmaDefragmentationEnd(). - - Negative value if error occured and object can be destroyed immediately. + - Negative value if error occurred and object can be destroyed immediately. */ VkResult Defragment( VkDeviceSize maxCpuBytesToMove, uint32_t maxCpuAllocationsToMove, VkDeviceSize maxGpuBytesToMove, uint32_t maxGpuAllocationsToMove, - VkCommandBuffer commandBuffer, VmaDefragmentationStats* pStats); + VkCommandBuffer commandBuffer, VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags); + + VkResult DefragmentPassBegin(VmaDefragmentationPassInfo* pInfo); + VkResult DefragmentPassEnd(); private: const VmaAllocator m_hAllocator; const uint32_t m_CurrFrameIndex; const uint32_t m_Flags; VmaDefragmentationStats* const m_pStats; + + VkDeviceSize m_MaxCpuBytesToMove; + uint32_t m_MaxCpuAllocationsToMove; + VkDeviceSize m_MaxGpuBytesToMove; + uint32_t m_MaxGpuAllocationsToMove; + // Owner of these objects. VmaBlockVectorDefragmentationContext* m_DefaultPoolContexts[VK_MAX_MEMORY_TYPES]; // Owner of these objects. @@ -6866,7 +8028,8 @@ public: uint32_t vulkanApiVersion, bool dedicatedAllocationExtensionEnabled, bool bindMemory2ExtensionEnabled, - bool memoryBudgetExtensionEnabled); + bool memoryBudgetExtensionEnabled, + bool deviceCoherentMemoryExtensionEnabled); ~VmaRecorder(); void RecordCreateAllocator(uint32_t frameIndex); @@ -6963,8 +8126,7 @@ private: VmaRecordFlags m_Flags; FILE* m_File; VMA_MUTEX m_FileMutex; - int64_t m_Freq; - int64_t m_StartCounter; + std::chrono::time_point<std::chrono::high_resolution_clock> m_RecordingStartTime; void GetBasicParams(CallParams& outParams); @@ -6997,7 +8159,7 @@ class VmaAllocationObjectAllocator public: VmaAllocationObjectAllocator(const VkAllocationCallbacks* pAllocationCallbacks); - VmaAllocation Allocate(); + template<typename... Types> VmaAllocation Allocate(Types... args); void Free(VmaAllocation hAlloc); private: @@ -7064,13 +8226,16 @@ public: bool m_UseKhrDedicatedAllocation; // Can be set only if m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0). bool m_UseKhrBindMemory2; // Can be set only if m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0). bool m_UseExtMemoryBudget; + bool m_UseAmdDeviceCoherentMemory; + bool m_UseKhrBufferDeviceAddress; + bool m_UseExtMemoryPriority; VkDevice m_hDevice; VkInstance m_hInstance; bool m_AllocationCallbacksSpecified; VkAllocationCallbacks m_AllocationCallbacks; VmaDeviceMemoryCallbacks m_DeviceMemoryCallbacks; VmaAllocationObjectAllocator m_AllocationObjectAllocator; - + // Each bit (1 << i) is set if HeapSizeLimit is enabled for that heap, so cannot allocate more than the heap size. uint32_t m_HeapSizeLimitMask; @@ -7079,13 +8244,14 @@ public: // Default pools. VmaBlockVector* m_pBlockVectors[VK_MAX_MEMORY_TYPES]; + VmaBlockVector* m_pSmallBufferBlockVectors[VK_MAX_MEMORY_TYPES]; - // Each vector is sorted by memory (handle value). - typedef VmaVector< VmaAllocation, VmaStlAllocator<VmaAllocation> > AllocationVectorType; - AllocationVectorType* m_pDedicatedAllocations[VK_MAX_MEMORY_TYPES]; + typedef VmaIntrusiveLinkedList<VmaDedicatedAllocationListItemTraits> DedicatedAllocationLinkedList; + DedicatedAllocationLinkedList m_DedicatedAllocations[VK_MAX_MEMORY_TYPES]; VMA_RW_MUTEX m_DedicatedAllocationsMutex[VK_MAX_MEMORY_TYPES]; VmaCurrentBudgetData m_Budget; + VMA_ATOMIC_UINT32 m_DeviceMemoryCount; // Total number of VkDeviceMemory objects. VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo); VkResult Init(const VmaAllocatorCreateInfo* pCreateInfo); @@ -7100,6 +8266,8 @@ public: return m_VulkanFunctions; } + VkPhysicalDevice GetPhysicalDevice() const { return m_PhysicalDevice; } + VkDeviceSize GetBufferImageGranularity() const { return VMA_MAX( @@ -7125,8 +8293,8 @@ public: VkDeviceSize GetMemoryTypeMinAlignment(uint32_t memTypeIndex) const { return IsMemoryTypeNonCoherent(memTypeIndex) ? - VMA_MAX((VkDeviceSize)VMA_DEBUG_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) : - (VkDeviceSize)VMA_DEBUG_ALIGNMENT; + VMA_MAX((VkDeviceSize)VMA_MIN_ALIGNMENT, m_PhysicalDeviceProperties.limits.nonCoherentAtomSize) : + (VkDeviceSize)VMA_MIN_ALIGNMENT; } bool IsIntegratedGpu() const @@ -7134,6 +8302,8 @@ public: return m_PhysicalDeviceProperties.deviceType == VK_PHYSICAL_DEVICE_TYPE_INTEGRATED_GPU; } + uint32_t GetGlobalMemoryTypeBits() const { return m_GlobalMemoryTypeBits; } + #if VMA_RECORDING_ENABLED VmaRecorder* GetRecorder() const { return m_pRecorder; } #endif @@ -7155,6 +8325,7 @@ public: bool requiresDedicatedAllocation, bool prefersDedicatedAllocation, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, // UINT32_MAX when unknown. VkImage dedicatedImage, const VmaAllocationCreateInfo& createInfo, VmaSuballocationType suballocType, @@ -7166,10 +8337,6 @@ public: size_t allocationCount, const VmaAllocation* pAllocations); - VkResult ResizeAllocation( - const VmaAllocation alloc, - VkDeviceSize newSize); - void CalculateStats(VmaStats* pStats); void GetBudget( @@ -7186,6 +8353,12 @@ public: VkResult DefragmentationEnd( VmaDefragmentationContext context); + VkResult DefragmentationPassBegin( + VmaDefragmentationPassInfo* pInfo, + VmaDefragmentationContext context); + VkResult DefragmentationPassEnd( + VmaDefragmentationContext context); + void GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo); bool TouchAllocation(VmaAllocation hAllocation); @@ -7235,10 +8408,15 @@ public: VkImage hImage, const void* pNext); - void FlushOrInvalidateAllocation( + VkResult FlushOrInvalidateAllocation( VmaAllocation hAllocation, VkDeviceSize offset, VkDeviceSize size, VMA_CACHE_OPERATION op); + VkResult FlushOrInvalidateAllocations( + uint32_t allocationCount, + const VmaAllocation* allocations, + const VkDeviceSize* offsets, const VkDeviceSize* sizes, + VMA_CACHE_OPERATION op); void FillAllocation(const VmaAllocation hAllocation, uint8_t pattern); @@ -7248,26 +8426,52 @@ public: */ uint32_t GetGpuDefragmentationMemoryTypeBits(); +#if VMA_EXTERNAL_MEMORY + VkExternalMemoryHandleTypeFlagsKHR GetExternalMemoryHandleTypeFlags(uint32_t memTypeIndex) const + { + return m_TypeExternalMemoryHandleTypes[memTypeIndex]; + } +#endif // #if VMA_EXTERNAL_MEMORY + private: VkDeviceSize m_PreferredLargeHeapBlockSize; VkPhysicalDevice m_PhysicalDevice; VMA_ATOMIC_UINT32 m_CurrentFrameIndex; VMA_ATOMIC_UINT32 m_GpuDefragmentationMemoryTypeBits; // UINT32_MAX means uninitialized. - +#if VMA_EXTERNAL_MEMORY + VkExternalMemoryHandleTypeFlagsKHR m_TypeExternalMemoryHandleTypes[VK_MAX_MEMORY_TYPES]; +#endif // #if VMA_EXTERNAL_MEMORY + VMA_RW_MUTEX m_PoolsMutex; - // Protected by m_PoolsMutex. Sorted by pointer value. - VmaVector<VmaPool, VmaStlAllocator<VmaPool> > m_Pools; + typedef VmaIntrusiveLinkedList<VmaPoolListItemTraits> PoolList; + // Protected by m_PoolsMutex. + PoolList m_Pools; uint32_t m_NextPoolId; VmaVulkanFunctions m_VulkanFunctions; + // Global bit mask AND-ed with any memoryTypeBits to disallow certain memory types. + uint32_t m_GlobalMemoryTypeBits; + #if VMA_RECORDING_ENABLED VmaRecorder* m_pRecorder; #endif void ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions); +#if VMA_STATIC_VULKAN_FUNCTIONS == 1 + void ImportVulkanFunctions_Static(); +#endif + + void ImportVulkanFunctions_Custom(const VmaVulkanFunctions* pVulkanFunctions); + +#if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1 + void ImportVulkanFunctions_Dynamic(); +#endif + + void ValidateVulkanFunctions(); + VkDeviceSize CalcPreferredBlockSize(uint32_t memTypeIndex); VkResult AllocateMemoryOfType( @@ -7275,6 +8479,7 @@ private: VkDeviceSize alignment, bool dedicatedAllocation, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, const VmaAllocationCreateInfo& createInfo, uint32_t memTypeIndex, @@ -7302,7 +8507,9 @@ private: bool map, bool isUserDataString, void* pUserData, + float priority, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, size_t allocationCount, VmaAllocation* pAllocations); @@ -7315,6 +8522,13 @@ private: */ uint32_t CalculateGpuDefragmentationMemoryTypeBits() const; + uint32_t CalculateGlobalMemoryTypeBits() const; + + bool GetFlushOrInvalidateRange( + VmaAllocation allocation, + VkDeviceSize offset, VkDeviceSize size, + VkMappedMemoryRange& outRange) const; + #if VMA_MEMORY_BUDGET void UpdateVulkanBudget(); #endif // #if VMA_MEMORY_BUDGET @@ -7451,10 +8665,10 @@ public: void BeginObject(bool singleLine = false); void EndObject(); - + void BeginArray(bool singleLine = false); void EndArray(); - + void WriteString(const char* pStr); void BeginString(const char* pStr = VMA_NULL); void ContinueString(const char* pStr); @@ -7462,7 +8676,7 @@ public: void ContinueString(uint64_t n); void ContinueString_Pointer(const void* ptr); void EndString(const char* pStr = VMA_NULL); - + void WriteNumber(uint32_t n); void WriteNumber(uint64_t n); void WriteBool(bool b); @@ -7710,7 +8924,7 @@ void VmaJsonWriter::WriteIndent(bool oneLess) if(!m_Stack.empty() && !m_Stack.back().singleLineMode) { m_SB.AddNewLine(); - + size_t count = m_Stack.size(); if(count > 0 && oneLess) { @@ -8124,7 +9338,7 @@ void VmaBlockMetadata::PrintDetailedMap_Allocation(class VmaJsonWriter& json, VmaAllocation hAllocation) const { json.BeginObject(true); - + json.WriteString("Offset"); json.WriteNumber(offset); @@ -8138,7 +9352,7 @@ void VmaBlockMetadata::PrintDetailedMap_UnusedRange(class VmaJsonWriter& json, VkDeviceSize size) const { json.BeginObject(true); - + json.WriteString("Offset"); json.WriteNumber(offset); @@ -8198,7 +9412,7 @@ void VmaBlockMetadata_Generic::Init(VkDeviceSize size) bool VmaBlockMetadata_Generic::Validate() const { VMA_VALIDATE(!m_Suballocations.empty()); - + // Expected offset of new suballocation as calculated from previous ones. VkDeviceSize calculatedOffset = 0; // Expected number of free suballocations as calculated from traversing their list. @@ -8211,12 +9425,8 @@ bool VmaBlockMetadata_Generic::Validate() const // True if previous visited suballocation was free. bool prevFree = false; - for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin(); - suballocItem != m_Suballocations.cend(); - ++suballocItem) + for(const auto& subAlloc : m_Suballocations) { - const VmaSuballocation& subAlloc = *suballocItem; - // Actual offset of this suballocation doesn't match expected one. VMA_VALIDATE(subAlloc.offset == calculatedOffset); @@ -8259,7 +9469,7 @@ bool VmaBlockMetadata_Generic::Validate() const for(size_t i = 0; i < m_FreeSuballocationsBySize.size(); ++i) { VmaSuballocationList::iterator suballocItem = m_FreeSuballocationsBySize[i]; - + // Only free suballocations can be registered in m_FreeSuballocationsBySize. VMA_VALIDATE(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE); // They must be sorted by size ascending. @@ -8268,7 +9478,7 @@ bool VmaBlockMetadata_Generic::Validate() const lastSize = suballocItem->size; } - // Check if totals match calculacted values. + // Check if totals match calculated values. VMA_VALIDATE(ValidateFreeSuballocationList()); VMA_VALIDATE(calculatedOffset == GetSize()); VMA_VALIDATE(calculatedSumFreeSize == m_SumFreeSize); @@ -8301,7 +9511,7 @@ void VmaBlockMetadata_Generic::CalcAllocationStatInfo(VmaStatInfo& outInfo) cons const uint32_t rangeCount = (uint32_t)m_Suballocations.size(); outInfo.allocationCount = rangeCount - m_FreeCount; outInfo.unusedRangeCount = m_FreeCount; - + outInfo.unusedBytes = m_SumFreeSize; outInfo.usedBytes = GetSize() - outInfo.unusedBytes; @@ -8310,11 +9520,8 @@ void VmaBlockMetadata_Generic::CalcAllocationStatInfo(VmaStatInfo& outInfo) cons outInfo.unusedRangeSizeMin = UINT64_MAX; outInfo.unusedRangeSizeMax = 0; - for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin(); - suballocItem != m_Suballocations.cend(); - ++suballocItem) + for(const auto& suballoc : m_Suballocations) { - const VmaSuballocation& suballoc = *suballocItem; if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE) { outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size); @@ -8349,17 +9556,15 @@ void VmaBlockMetadata_Generic::PrintDetailedMap(class VmaJsonWriter& json) const m_FreeCount); // unusedRangeCount size_t i = 0; - for(VmaSuballocationList::const_iterator suballocItem = m_Suballocations.cbegin(); - suballocItem != m_Suballocations.cend(); - ++suballocItem, ++i) + for(const auto& suballoc : m_Suballocations) { - if(suballocItem->type == VMA_SUBALLOCATION_TYPE_FREE) + if(suballoc.type == VMA_SUBALLOCATION_TYPE_FREE) { - PrintDetailedMap_UnusedRange(json, suballocItem->offset, suballocItem->size); + PrintDetailedMap_UnusedRange(json, suballoc.offset, suballoc.size); } else { - PrintDetailedMap_Allocation(json, suballocItem->offset, suballocItem->hAllocation); + PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation); } } @@ -8560,7 +9765,7 @@ bool VmaBlockMetadata_Generic::MakeRequestedAllocationsLost( VMA_HEAVY_ASSERT(Validate()); VMA_ASSERT(pAllocationRequest->item != m_Suballocations.end()); VMA_ASSERT(pAllocationRequest->item->type == VMA_SUBALLOCATION_TYPE_FREE); - + return true; } @@ -8584,18 +9789,16 @@ uint32_t VmaBlockMetadata_Generic::MakeAllocationsLost(uint32_t currentFrameInde VkResult VmaBlockMetadata_Generic::CheckCorruption(const void* pBlockData) { - for(VmaSuballocationList::iterator it = m_Suballocations.begin(); - it != m_Suballocations.end(); - ++it) + for(auto& suballoc : m_Suballocations) { - if(it->type != VMA_SUBALLOCATION_TYPE_FREE) + if(suballoc.type != VMA_SUBALLOCATION_TYPE_FREE) { - if(!VmaValidateMagicValue(pBlockData, it->offset - VMA_DEBUG_MARGIN)) + if(!VmaValidateMagicValue(pBlockData, suballoc.offset - VMA_DEBUG_MARGIN)) { VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED BEFORE VALIDATED ALLOCATION!"); return VK_ERROR_VALIDATION_FAILED_EXT; } - if(!VmaValidateMagicValue(pBlockData, it->offset + it->size)) + if(!VmaValidateMagicValue(pBlockData, suballoc.offset + suballoc.size)) { VMA_ASSERT(0 && "MEMORY CORRUPTION DETECTED AFTER VALIDATED ALLOCATION!"); return VK_ERROR_VALIDATION_FAILED_EXT; @@ -8737,7 +9940,7 @@ bool VmaBlockMetadata_Generic::CheckAllocation( VMA_ASSERT(allocType != VMA_SUBALLOCATION_TYPE_FREE); VMA_ASSERT(suballocItem != m_Suballocations.cend()); VMA_ASSERT(pOffset != VMA_NULL); - + *itemsToMakeLostCount = 0; *pSumFreeSize = 0; *pSumItemSize = 0; @@ -8770,19 +9973,19 @@ bool VmaBlockMetadata_Generic::CheckAllocation( // Start from offset equal to beginning of this suballocation. *pOffset = suballocItem->offset; - + // Apply VMA_DEBUG_MARGIN at the beginning. if(VMA_DEBUG_MARGIN > 0) { *pOffset += VMA_DEBUG_MARGIN; } - + // Apply alignment. *pOffset = VmaAlignUp(*pOffset, allocAlignment); // Check previous suballocations for BufferImageGranularity conflicts. // Make bigger alignment if necessary. - if(bufferImageGranularity > 1) + if(bufferImageGranularity > 1 && bufferImageGranularity != allocAlignment) { bool bufferImageGranularityConflict = false; VmaSuballocationList::const_iterator prevSuballocItem = suballocItem; @@ -8807,14 +10010,14 @@ bool VmaBlockMetadata_Generic::CheckAllocation( *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity); } } - + // Now that we have final *pOffset, check if we are past suballocItem. // If yes, return false - this function should be called for another suballocItem as starting point. if(*pOffset >= suballocItem->offset + suballocItem->size) { return false; } - + // Calculate padding at the beginning based on current offset. const VkDeviceSize paddingBegin = *pOffset - suballocItem->offset; @@ -8866,7 +10069,7 @@ bool VmaBlockMetadata_Generic::CheckAllocation( // Check next suballocations for BufferImageGranularity conflicts. // If conflict exists, we must mark more allocations lost or fail. - if(bufferImageGranularity > 1) + if(allocSize % bufferImageGranularity || *pOffset % bufferImageGranularity) { VmaSuballocationList::const_iterator nextSuballocItem = lastSuballocItem; ++nextSuballocItem; @@ -8913,19 +10116,19 @@ bool VmaBlockMetadata_Generic::CheckAllocation( // Start from offset equal to beginning of this suballocation. *pOffset = suballoc.offset; - + // Apply VMA_DEBUG_MARGIN at the beginning. if(VMA_DEBUG_MARGIN > 0) { *pOffset += VMA_DEBUG_MARGIN; } - + // Apply alignment. *pOffset = VmaAlignUp(*pOffset, allocAlignment); - + // Check previous suballocations for BufferImageGranularity conflicts. // Make bigger alignment if necessary. - if(bufferImageGranularity > 1) + if(bufferImageGranularity > 1 && bufferImageGranularity != allocAlignment) { bool bufferImageGranularityConflict = false; VmaSuballocationList::const_iterator prevSuballocItem = suballocItem; @@ -8950,7 +10153,7 @@ bool VmaBlockMetadata_Generic::CheckAllocation( *pOffset = VmaAlignUp(*pOffset, bufferImageGranularity); } } - + // Calculate padding at the beginning based on current offset. const VkDeviceSize paddingBegin = *pOffset - suballoc.offset; @@ -8965,7 +10168,7 @@ bool VmaBlockMetadata_Generic::CheckAllocation( // Check next suballocations for BufferImageGranularity conflicts. // If conflict exists, allocation cannot be made here. - if(bufferImageGranularity > 1) + if(allocSize % bufferImageGranularity || *pOffset % bufferImageGranularity) { VmaSuballocationList::const_iterator nextSuballocItem = suballocItem; ++nextSuballocItem; @@ -8997,7 +10200,7 @@ void VmaBlockMetadata_Generic::MergeFreeWithNext(VmaSuballocationList::iterator { VMA_ASSERT(item != m_Suballocations.end()); VMA_ASSERT(item->type == VMA_SUBALLOCATION_TYPE_FREE); - + VmaSuballocationList::iterator nextItem = item; ++nextItem; VMA_ASSERT(nextItem != m_Suballocations.end()); @@ -9014,7 +10217,7 @@ VmaSuballocationList::iterator VmaBlockMetadata_Generic::FreeSuballocation(VmaSu VmaSuballocation& suballoc = *suballocItem; suballoc.type = VMA_SUBALLOCATION_TYPE_FREE; suballoc.hAllocation = VK_NULL_HANDLE; - + // Update totals. ++m_FreeCount; m_SumFreeSize += suballoc.size; @@ -9022,7 +10225,7 @@ VmaSuballocationList::iterator VmaBlockMetadata_Generic::FreeSuballocation(VmaSu // Merge with previous and/or next suballocation if it's also free. bool mergeWithNext = false; bool mergeWithPrev = false; - + VmaSuballocationList::iterator nextItem = suballocItem; ++nextItem; if((nextItem != m_Suballocations.end()) && (nextItem->type == VMA_SUBALLOCATION_TYPE_FREE)) @@ -9129,14 +10332,12 @@ bool VmaBlockMetadata_Generic::IsBufferImageGranularityConflictPossible( VkDeviceSize minAlignment = VK_WHOLE_SIZE; bool typeConflictFound = false; - for(VmaSuballocationList::const_iterator it = m_Suballocations.cbegin(); - it != m_Suballocations.cend(); - ++it) + for(const auto& suballoc : m_Suballocations) { - const VmaSuballocationType suballocType = it->type; + const VmaSuballocationType suballocType = suballoc.type; if(suballocType != VMA_SUBALLOCATION_TYPE_FREE) { - minAlignment = VMA_MIN(minAlignment, it->hAllocation->GetAlignment()); + minAlignment = VMA_MIN(minAlignment, suballoc.hAllocation->GetAlignment()); if(VmaIsBufferImageGranularityConflict(inOutPrevSuballocType, suballocType)) { typeConflictFound = true; @@ -9320,7 +10521,7 @@ VkDeviceSize VmaBlockMetadata_Linear::GetUnusedRangeSizeMax() const { return size; } - + const SuballocationVectorType& suballocations1st = AccessSuballocations1st(); switch(m_2ndVectorMode) @@ -9328,7 +10529,7 @@ VkDeviceSize VmaBlockMetadata_Linear::GetUnusedRangeSizeMax() const case SECOND_VECTOR_EMPTY: /* Available space is after end of 1st, as well as before beginning of 1st (which - whould make it a ring buffer). + would make it a ring buffer). */ { const size_t suballocations1stCount = suballocations1st.size(); @@ -9407,7 +10608,7 @@ void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const if(nextAlloc2ndIndex < suballoc2ndCount) { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -9418,13 +10619,13 @@ void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. outInfo.usedBytes += suballoc.size; outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size); outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size); - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc2ndIndex; @@ -9464,7 +10665,7 @@ void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const if(nextAlloc1stIndex < suballoc1stCount) { const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -9475,13 +10676,13 @@ void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. outInfo.usedBytes += suballoc.size; outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size); outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size); - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc1stIndex; @@ -9520,7 +10721,7 @@ void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const if(nextAlloc2ndIndex != SIZE_MAX) { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -9531,13 +10732,13 @@ void VmaBlockMetadata_Linear::CalcAllocationStatInfo(VmaStatInfo& outInfo) const outInfo.unusedRangeSizeMin = VMA_MIN(outInfo.unusedRangeSizeMin, unusedRangeSize); outInfo.unusedRangeSizeMax = VMA_MIN(outInfo.unusedRangeSizeMax, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. outInfo.usedBytes += suballoc.size; outInfo.allocationSizeMin = VMA_MIN(outInfo.allocationSizeMin, suballoc.size); outInfo.allocationSizeMax = VMA_MIN(outInfo.allocationSizeMax, suballoc.size); - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; --nextAlloc2ndIndex; @@ -9593,7 +10794,7 @@ void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const if(nextAlloc2ndIndex < suballoc2ndCount) { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -9603,11 +10804,11 @@ void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const ++inoutStats.unusedRangeCount; inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++inoutStats.allocationCount; - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc2ndIndex; @@ -9646,7 +10847,7 @@ void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const if(nextAlloc1stIndex < suballoc1stCount) { const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -9656,11 +10857,11 @@ void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const ++inoutStats.unusedRangeCount; inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++inoutStats.allocationCount; - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc1stIndex; @@ -9698,7 +10899,7 @@ void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const if(nextAlloc2ndIndex != SIZE_MAX) { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -9708,11 +10909,11 @@ void VmaBlockMetadata_Linear::AddPoolStats(VmaPoolStats& inoutStats) const ++inoutStats.unusedRangeCount; inoutStats.unusedRangeSizeMax = VMA_MAX(inoutStats.unusedRangeSizeMax, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++inoutStats.allocationCount; - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; --nextAlloc2ndIndex; @@ -9770,19 +10971,19 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const if(nextAlloc2ndIndex < suballoc2ndCount) { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { // There is free space from lastOffset to suballoc.offset. ++unusedRangeCount; } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++alloc2ndCount; usedBytes += suballoc.size; - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc2ndIndex; @@ -9819,19 +11020,19 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const if(nextAlloc1stIndex < suballoc1stCount) { const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { // There is free space from lastOffset to suballoc.offset. ++unusedRangeCount; } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++alloc1stCount; usedBytes += suballoc.size; - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc1stIndex; @@ -9866,19 +11067,19 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const if(nextAlloc2ndIndex != SIZE_MAX) { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { // There is free space from lastOffset to suballoc.offset. ++unusedRangeCount; } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. ++alloc2ndCount; usedBytes += suballoc.size; - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; --nextAlloc2ndIndex; @@ -9921,7 +11122,7 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const if(nextAlloc2ndIndex < suballoc2ndCount) { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -9929,11 +11130,11 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation); - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc2ndIndex; @@ -9968,7 +11169,7 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const if(nextAlloc1stIndex < suballoc1stCount) { const VmaSuballocation& suballoc = suballocations1st[nextAlloc1stIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -9976,11 +11177,11 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation); - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; ++nextAlloc1stIndex; @@ -10016,7 +11217,7 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const if(nextAlloc2ndIndex != SIZE_MAX) { const VmaSuballocation& suballoc = suballocations2nd[nextAlloc2ndIndex]; - + // 1. Process free space before this allocation. if(lastOffset < suballoc.offset) { @@ -10024,11 +11225,11 @@ void VmaBlockMetadata_Linear::PrintDetailedMap(class VmaJsonWriter& json) const const VkDeviceSize unusedRangeSize = suballoc.offset - lastOffset; PrintDetailedMap_UnusedRange(json, lastOffset, unusedRangeSize); } - + // 2. Process this allocation. // There is allocation with suballoc.offset, suballoc.size. PrintDetailedMap_Allocation(json, suballoc.offset, suballoc.hAllocation); - + // 3. Prepare for next iteration. lastOffset = suballoc.offset + suballoc.size; --nextAlloc2ndIndex; @@ -10133,7 +11334,7 @@ bool VmaBlockMetadata_Linear::CreateAllocationRequest_UpperAddress( // Check next suballocations from 2nd for BufferImageGranularity conflicts. // Make bigger alignment if necessary. - if(bufferImageGranularity > 1 && !suballocations2nd.empty()) + if(bufferImageGranularity > 1 && bufferImageGranularity != allocAlignment && !suballocations2nd.empty()) { bool bufferImageGranularityConflict = false; for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; ) @@ -10238,7 +11439,7 @@ bool VmaBlockMetadata_Linear::CreateAllocationRequest_LowerAddress( // Check previous suballocations for BufferImageGranularity conflicts. // Make bigger alignment if necessary. - if(bufferImageGranularity > 1 && !suballocations1st.empty()) + if(bufferImageGranularity > 1 && bufferImageGranularity != allocAlignment && !suballocations1st.empty()) { bool bufferImageGranularityConflict = false; for(size_t prevSuballocIndex = suballocations1st.size(); prevSuballocIndex--; ) @@ -10270,7 +11471,7 @@ bool VmaBlockMetadata_Linear::CreateAllocationRequest_LowerAddress( { // Check next suballocations for BufferImageGranularity conflicts. // If conflict exists, allocation cannot be made here. - if(bufferImageGranularity > 1 && m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) + if((allocSize % bufferImageGranularity || resultOffset % bufferImageGranularity) && m_2ndVectorMode == SECOND_VECTOR_DOUBLE_STACK) { for(size_t nextSuballocIndex = suballocations2nd.size(); nextSuballocIndex--; ) { @@ -10328,7 +11529,7 @@ bool VmaBlockMetadata_Linear::CreateAllocationRequest_LowerAddress( // Check previous suballocations for BufferImageGranularity conflicts. // Make bigger alignment if necessary. - if(bufferImageGranularity > 1 && !suballocations2nd.empty()) + if(bufferImageGranularity > 1 && bufferImageGranularity != allocAlignment && !suballocations2nd.empty()) { bool bufferImageGranularityConflict = false; for(size_t prevSuballocIndex = suballocations2nd.size(); prevSuballocIndex--; ) @@ -10386,7 +11587,7 @@ bool VmaBlockMetadata_Linear::CreateAllocationRequest_LowerAddress( // Check next suballocations for BufferImageGranularity conflicts. // If conflict exists, we must mark more allocations lost or fail. - if(bufferImageGranularity > 1) + if(allocSize % bufferImageGranularity || resultOffset % bufferImageGranularity) { while(index1st < suballocations1st.size()) { @@ -10432,7 +11633,7 @@ bool VmaBlockMetadata_Linear::CreateAllocationRequest_LowerAddress( { // Check next suballocations for BufferImageGranularity conflicts. // If conflict exists, allocation cannot be made here. - if(bufferImageGranularity > 1) + if(allocSize % bufferImageGranularity || resultOffset % bufferImageGranularity) { for(size_t nextSuballocIndex = index1st; nextSuballocIndex < suballocations1st.size(); @@ -10480,7 +11681,7 @@ bool VmaBlockMetadata_Linear::MakeRequestedAllocationsLost( } VMA_ASSERT(m_2ndVectorMode == SECOND_VECTOR_EMPTY || m_2ndVectorMode == SECOND_VECTOR_RING_BUFFER); - + // We always start from 1st. SuballocationVectorType* suballocations = &AccessSuballocations1st(); size_t index = m_1stNullItemsBeginCount; @@ -10528,15 +11729,15 @@ bool VmaBlockMetadata_Linear::MakeRequestedAllocationsLost( } CleanupAfterFree(); - //VMA_HEAVY_ASSERT(Validate()); // Already called by ClanupAfterFree(). - + //VMA_HEAVY_ASSERT(Validate()); // Already called by CleanupAfterFree(). + return true; } uint32_t VmaBlockMetadata_Linear::MakeAllocationsLost(uint32_t currentFrameIndex, uint32_t frameInUseCount) { uint32_t lostAllocationCount = 0; - + SuballocationVectorType& suballocations1st = AccessSuballocations1st(); for(size_t i = m_1stNullItemsBeginCount, count = suballocations1st.size(); i < count; ++i) { @@ -10960,7 +12161,7 @@ bool VmaBlockMetadata_Buddy::Validate() const node = node->free.next) { VMA_VALIDATE(node->type == Node::TYPE_FREE); - + if(node->free.next == VMA_NULL) { VMA_VALIDATE(m_FreeList[level].back == node); @@ -11146,7 +12347,7 @@ void VmaBlockMetadata_Buddy::Alloc( const uint32_t targetLevel = AllocSizeToLevel(allocSize); uint32_t currLevel = (uint32_t)(uintptr_t)request.customData; - + Node* currNode = m_FreeList[currLevel].front; VMA_ASSERT(currNode != VMA_NULL && currNode->type == Node::TYPE_FREE); while(currNode->offset != request.offset) @@ -11154,14 +12355,14 @@ void VmaBlockMetadata_Buddy::Alloc( currNode = currNode->free.next; VMA_ASSERT(currNode != VMA_NULL && currNode->type == Node::TYPE_FREE); } - + // Go down, splitting free nodes. while(currLevel < targetLevel) { // currNode is already first free node at currLevel. // Remove it from list of free nodes at this currLevel. RemoveFromFreeList(currLevel, currNode); - + const uint32_t childrenLevel = currLevel + 1; // Create two free sub-nodes. @@ -11323,7 +12524,7 @@ void VmaBlockMetadata_Buddy::FreeAtOffset(VmaAllocation alloc, VkDeviceSize offs vma_delete(GetAllocationCallbacks(), node->buddy); vma_delete(GetAllocationCallbacks(), node); parent->type = Node::TYPE_FREE; - + node = parent; --level; //m_SumFreeSize += LevelToNodeSize(level) % 2; // Useful only when level node sizes can be non power of 2. @@ -11437,7 +12638,7 @@ void VmaBlockMetadata_Buddy::PrintDetailedMapNode(class VmaJsonWriter& json, con PrintDetailedMap_UnusedRange(json, node->offset, levelNodeSize); break; case Node::TYPE_ALLOCATION: - { + { PrintDetailedMap_Allocation(json, node->offset, node->allocation.alloc); const VkDeviceSize allocSize = node->allocation.alloc->GetSize(); if(allocSize < levelNodeSize) @@ -11465,18 +12666,13 @@ void VmaBlockMetadata_Buddy::PrintDetailedMapNode(class VmaJsonWriter& json, con //////////////////////////////////////////////////////////////////////////////// // class VmaDeviceMemoryBlock -VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) : - m_pMetadata(VMA_NULL), - m_MemoryTypeIndex(UINT32_MAX), - m_Id(0), - m_hMemory(VK_NULL_HANDLE), - m_MapCount(0), - m_pMappedData(VMA_NULL) +VmaDeviceMemoryBlock::VmaDeviceMemoryBlock(VmaAllocator hAllocator) { } void VmaDeviceMemoryBlock::Init( VmaAllocator hAllocator, + VmaBlockVector* parentBlockVector, VmaPool hParentPool, uint32_t newMemoryTypeIndex, VkDeviceMemory newMemory, @@ -11484,8 +12680,10 @@ void VmaDeviceMemoryBlock::Init( uint32_t id, uint32_t algorithm) { + VMA_ASSERT(parentBlockVector != VMA_NULL); VMA_ASSERT(m_hMemory == VK_NULL_HANDLE); + m_ParentBlockVector = parentBlockVector; m_hParentPool = hParentPool; m_MemoryTypeIndex = newMemoryTypeIndex; m_Id = id; @@ -11526,7 +12724,7 @@ bool VmaDeviceMemoryBlock::Validate() const { VMA_VALIDATE((m_hMemory != VK_NULL_HANDLE) && (m_pMetadata->GetSize() != 0)); - + return m_pMetadata->Validate(); } @@ -11731,7 +12929,10 @@ VmaPool_T::VmaPool_T( (createInfo.flags & VMA_POOL_CREATE_IGNORE_BUFFER_IMAGE_GRANULARITY_BIT) != 0 ? 1 : hAllocator->GetBufferImageGranularity(), createInfo.frameInUseCount, createInfo.blockSize != 0, // explicitBlockSize - createInfo.flags & VMA_POOL_CREATE_ALGORITHM_MASK), // algorithm + createInfo.flags & VMA_POOL_CREATE_ALGORITHM_MASK, // algorithm + createInfo.priority, + VMA_MAX(hAllocator->GetMemoryTypeMinAlignment(createInfo.memoryTypeIndex), createInfo.minAllocationAlignment), + createInfo.pMemoryAllocateNext), m_Id(0), m_Name(VMA_NULL) { @@ -11739,13 +12940,14 @@ VmaPool_T::VmaPool_T( VmaPool_T::~VmaPool_T() { + VMA_ASSERT(m_PrevPool == VMA_NULL && m_NextPool == VMA_NULL); } void VmaPool_T::SetName(const char* pName) { const VkAllocationCallbacks* allocs = m_BlockVector.GetAllocator()->GetAllocationCallbacks(); VmaFreeString(allocs, m_Name); - + if(pName != VMA_NULL) { m_Name = VmaCreateStringCopy(allocs, pName); @@ -11770,7 +12972,10 @@ VmaBlockVector::VmaBlockVector( VkDeviceSize bufferImageGranularity, uint32_t frameInUseCount, bool explicitBlockSize, - uint32_t algorithm) : + uint32_t algorithm, + float priority, + VkDeviceSize minAllocationAlignment, + void* pMemoryAllocateNext) : m_hAllocator(hAllocator), m_hParentPool(hParentPool), m_MemoryTypeIndex(memoryTypeIndex), @@ -11781,6 +12986,9 @@ VmaBlockVector::VmaBlockVector( m_FrameInUseCount(frameInUseCount), m_ExplicitBlockSize(explicitBlockSize), m_Algorithm(algorithm), + m_Priority(priority), + m_MinAllocationAlignment(minAllocationAlignment), + m_pMemoryAllocateNext(pMemoryAllocateNext), m_HasEmptyBlock(false), m_Blocks(VmaStlAllocator<VmaDeviceMemoryBlock*>(hAllocator->GetAllocationCallbacks())), m_NextBlockId(0) @@ -11860,6 +13068,8 @@ VkResult VmaBlockVector::Allocate( size_t allocIndex; VkResult res = VK_SUCCESS; + alignment = VMA_MAX(alignment, m_MinAllocationAlignment); + if(IsCorruptionDetectionEnabled()) { size = VmaAlignUp<VkDeviceSize>(size, sizeof(VMA_CORRUPTION_DETECTION_MAGIC_VALUE)); @@ -11887,9 +13097,13 @@ VkResult VmaBlockVector::Allocate( if(res != VK_SUCCESS) { // Free all already created allocations. + const uint32_t heapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex); while(allocIndex--) { - Free(pAllocations[allocIndex]); + VmaAllocation_T* const alloc = pAllocations[allocIndex]; + const VkDeviceSize allocSize = alloc->GetSize(); + Free(alloc); + m_hAllocator->m_Budget.RemoveAllocation(heapIndex, allocSize); } memset(pAllocations, 0, sizeof(VmaAllocation) * allocationCount); } @@ -11909,8 +13123,7 @@ VkResult VmaBlockVector::AllocatePage( bool canMakeOtherLost = (createInfo.flags & VMA_ALLOCATION_CREATE_CAN_MAKE_OTHER_LOST_BIT) != 0; const bool mapped = (createInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0; const bool isUserDataString = (createInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0; - - const bool withinBudget = (createInfo.flags & VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) != 0; + VkDeviceSize freeMemory; { const uint32_t heapIndex = m_hAllocator->MemoryTypeIndexToHeapIndex(m_MemoryTypeIndex); @@ -11918,7 +13131,7 @@ VkResult VmaBlockVector::AllocatePage( m_hAllocator->GetBudget(&heapBudget, heapIndex, 1); freeMemory = (heapBudget.usage < heapBudget.budget) ? (heapBudget.budget - heapBudget.usage) : 0; } - + const bool canFallbackToDedicated = !IsCustomPool(); const bool canCreateNewBlock = ((createInfo.flags & VMA_ALLOCATION_CREATE_NEVER_ALLOCATE_BIT) == 0) && @@ -12229,8 +13442,7 @@ VkResult VmaBlockVector::AllocatePage( &bestRequest)) { // Allocate from this pBlock. - *pAllocation = m_hAllocator->m_AllocationObjectAllocator.Allocate(); - (*pAllocation)->Ctor(currentFrameIndex, isUserDataString); + *pAllocation = m_hAllocator->m_AllocationObjectAllocator.Allocate(currentFrameIndex, isUserDataString); pBestRequestBlock->m_pMetadata->Alloc(bestRequest, suballocType, size, *pAllocation); UpdateHasEmptyBlock(); (*pAllocation)->InitBlockAllocation( @@ -12280,6 +13492,8 @@ VkResult VmaBlockVector::AllocatePage( void VmaBlockVector::Free( const VmaAllocation hAllocation) { + VMA_ASSERT(hAllocation->GetBlock()->GetParentBlockVector() == this); + VmaDeviceMemoryBlock* pBlockToDelete = VMA_NULL; bool budgetExceeded = false; @@ -12433,9 +13647,8 @@ VkResult VmaBlockVector::AllocateFromBlock( return res; } } - - *pAllocation = m_hAllocator->m_AllocationObjectAllocator.Allocate(); - (*pAllocation)->Ctor(currentFrameIndex, isUserDataString); + + *pAllocation = m_hAllocator->m_AllocationObjectAllocator.Allocate(currentFrameIndex, isUserDataString); pBlock->m_pMetadata->Alloc(currRequest, suballocType, size, *pAllocation); UpdateHasEmptyBlock(); (*pAllocation)->InitBlockAllocation( @@ -12467,8 +13680,39 @@ VkResult VmaBlockVector::AllocateFromBlock( VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIndex) { VkMemoryAllocateInfo allocInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO }; + allocInfo.pNext = m_pMemoryAllocateNext; allocInfo.memoryTypeIndex = m_MemoryTypeIndex; allocInfo.allocationSize = blockSize; + +#if VMA_BUFFER_DEVICE_ADDRESS + // Every standalone block can potentially contain a buffer with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT - always enable the feature. + VkMemoryAllocateFlagsInfoKHR allocFlagsInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO_KHR }; + if(m_hAllocator->m_UseKhrBufferDeviceAddress) + { + allocFlagsInfo.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT_KHR; + VmaPnextChainPushFront(&allocInfo, &allocFlagsInfo); + } +#endif // #if VMA_BUFFER_DEVICE_ADDRESS + +#if VMA_MEMORY_PRIORITY + VkMemoryPriorityAllocateInfoEXT priorityInfo = { VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT }; + if(m_hAllocator->m_UseExtMemoryPriority) + { + priorityInfo.priority = m_Priority; + VmaPnextChainPushFront(&allocInfo, &priorityInfo); + } +#endif // #if VMA_MEMORY_PRIORITY + +#if VMA_EXTERNAL_MEMORY + // Attach VkExportMemoryAllocateInfoKHR if necessary. + VkExportMemoryAllocateInfoKHR exportMemoryAllocInfo = { VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO_KHR }; + exportMemoryAllocInfo.handleTypes = m_hAllocator->GetExternalMemoryHandleTypeFlags(m_MemoryTypeIndex); + if(exportMemoryAllocInfo.handleTypes != 0) + { + VmaPnextChainPushFront(&allocInfo, &exportMemoryAllocInfo); + } +#endif // #if VMA_EXTERNAL_MEMORY + VkDeviceMemory mem = VK_NULL_HANDLE; VkResult res = m_hAllocator->AllocateVulkanMemory(&allocInfo, &mem); if(res < 0) @@ -12482,6 +13726,7 @@ VkResult VmaBlockVector::CreateBlock(VkDeviceSize blockSize, size_t* pNewBlockIn VmaDeviceMemoryBlock* const pBlock = vma_new(m_hAllocator, VmaDeviceMemoryBlock)(m_hAllocator); pBlock->Init( m_hAllocator, + this, // parentBlockVector m_hParentPool, m_MemoryTypeIndex, mem, @@ -12619,7 +13864,7 @@ void VmaBlockVector::ApplyDefragmentationMovesCpu( void VmaBlockVector::ApplyDefragmentationMovesGpu( class VmaBlockVectorDefragmentationContext* pDefragCtx, - const VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, + VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkCommandBuffer commandBuffer) { const size_t blockCount = m_Blocks.size(); @@ -12632,8 +13877,13 @@ void VmaBlockVector::ApplyDefragmentationMovesGpu( for(size_t moveIndex = 0; moveIndex < moveCount; ++moveIndex) { const VmaDefragmentationMove& move = moves[moveIndex]; - pDefragCtx->blockContexts[move.srcBlockIndex].flags |= VmaBlockDefragmentationContext::BLOCK_FLAG_USED; - pDefragCtx->blockContexts[move.dstBlockIndex].flags |= VmaBlockDefragmentationContext::BLOCK_FLAG_USED; + + //if(move.type == VMA_ALLOCATION_TYPE_UNKNOWN) + { + // Old school move still require us to map the whole block + pDefragCtx->blockContexts[move.srcBlockIndex].flags |= VmaBlockDefragmentationContext::BLOCK_FLAG_USED; + pDefragCtx->blockContexts[move.dstBlockIndex].flags |= VmaBlockDefragmentationContext::BLOCK_FLAG_USED; + } } VMA_ASSERT(pDefragCtx->res == VK_SUCCESS); @@ -12807,13 +14057,13 @@ void VmaBlockVector::PrintDetailedMap(class VmaJsonWriter& json) void VmaBlockVector::Defragment( class VmaBlockVectorDefragmentationContext* pCtx, - VmaDefragmentationStats* pStats, + VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags, VkDeviceSize& maxCpuBytesToMove, uint32_t& maxCpuAllocationsToMove, VkDeviceSize& maxGpuBytesToMove, uint32_t& maxGpuAllocationsToMove, VkCommandBuffer commandBuffer) { pCtx->res = VK_SUCCESS; - + const VkMemoryPropertyFlags memPropFlags = m_hAllocator->m_MemProps.memoryTypes[m_MemoryTypeIndex].propertyFlags; const bool isHostVisible = (memPropFlags & VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT) != 0; @@ -12844,19 +14094,28 @@ void VmaBlockVector::Defragment( if(m_hAllocator->m_UseMutex) { - m_Mutex.LockWrite(); - pCtx->mutexLocked = true; + if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL) + { + if(!m_Mutex.TryLockWrite()) + { + pCtx->res = VK_ERROR_INITIALIZATION_FAILED; + return; + } + } + else + { + m_Mutex.LockWrite(); + pCtx->mutexLocked = true; + } } - pCtx->Begin(overlappingMoveSupported); + pCtx->Begin(overlappingMoveSupported, flags); // Defragment. const VkDeviceSize maxBytesToMove = defragmentOnGpu ? maxGpuBytesToMove : maxCpuBytesToMove; const uint32_t maxAllocationsToMove = defragmentOnGpu ? maxGpuAllocationsToMove : maxCpuAllocationsToMove; - VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> > moves = - VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >(VmaStlAllocator<VmaDefragmentationMove>(m_hAllocator->GetAllocationCallbacks())); - pCtx->res = pCtx->GetAlgorithm()->Defragment(moves, maxBytesToMove, maxAllocationsToMove); + pCtx->res = pCtx->GetAlgorithm()->Defragment(pCtx->defragmentationMoves, maxBytesToMove, maxAllocationsToMove, flags); // Accumulate statistics. if(pStats != VMA_NULL) @@ -12878,16 +14137,27 @@ void VmaBlockVector::Defragment( maxCpuAllocationsToMove -= allocationsMoved; } } - + + if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL) + { + if(m_hAllocator->m_UseMutex) + m_Mutex.UnlockWrite(); + + if(pCtx->res >= VK_SUCCESS && !pCtx->defragmentationMoves.empty()) + pCtx->res = VK_NOT_READY; + + return; + } + if(pCtx->res >= VK_SUCCESS) { if(defragmentOnGpu) { - ApplyDefragmentationMovesGpu(pCtx, moves, commandBuffer); + ApplyDefragmentationMovesGpu(pCtx, pCtx->defragmentationMoves, commandBuffer); } else { - ApplyDefragmentationMovesCpu(pCtx, moves); + ApplyDefragmentationMovesCpu(pCtx, pCtx->defragmentationMoves); } } } @@ -12895,22 +14165,36 @@ void VmaBlockVector::Defragment( void VmaBlockVector::DefragmentationEnd( class VmaBlockVectorDefragmentationContext* pCtx, + uint32_t flags, VmaDefragmentationStats* pStats) { - // Destroy buffers. - for(size_t blockIndex = pCtx->blockContexts.size(); blockIndex--; ) + if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL && m_hAllocator->m_UseMutex) { - VmaBlockDefragmentationContext& blockCtx = pCtx->blockContexts[blockIndex]; - if(blockCtx.hBuffer) - { - (*m_hAllocator->GetVulkanFunctions().vkDestroyBuffer)( - m_hAllocator->m_hDevice, blockCtx.hBuffer, m_hAllocator->GetAllocationCallbacks()); - } + VMA_ASSERT(pCtx->mutexLocked == false); + + // Incremental defragmentation doesn't hold the lock, so when we enter here we don't actually have any + // lock protecting us. Since we mutate state here, we have to take the lock out now + m_Mutex.LockWrite(); + pCtx->mutexLocked = true; } - if(pCtx->res >= VK_SUCCESS) + // If the mutex isn't locked we didn't do any work and there is nothing to delete. + if(pCtx->mutexLocked || !m_hAllocator->m_UseMutex) { - FreeEmptyBlocks(pStats); + // Destroy buffers. + for(size_t blockIndex = pCtx->blockContexts.size(); blockIndex--;) + { + VmaBlockDefragmentationContext &blockCtx = pCtx->blockContexts[blockIndex]; + if(blockCtx.hBuffer) + { + (*m_hAllocator->GetVulkanFunctions().vkDestroyBuffer)(m_hAllocator->m_hDevice, blockCtx.hBuffer, m_hAllocator->GetAllocationCallbacks()); + } + } + + if(pCtx->res >= VK_SUCCESS) + { + FreeEmptyBlocks(pStats); + } } if(pCtx->mutexLocked) @@ -12920,6 +14204,48 @@ void VmaBlockVector::DefragmentationEnd( } } +uint32_t VmaBlockVector::ProcessDefragmentations( + class VmaBlockVectorDefragmentationContext *pCtx, + VmaDefragmentationPassMoveInfo* pMove, uint32_t maxMoves) +{ + VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex); + + const uint32_t moveCount = VMA_MIN(uint32_t(pCtx->defragmentationMoves.size()) - pCtx->defragmentationMovesProcessed, maxMoves); + + for(uint32_t i = 0; i < moveCount; ++ i) + { + VmaDefragmentationMove& move = pCtx->defragmentationMoves[pCtx->defragmentationMovesProcessed + i]; + + pMove->allocation = move.hAllocation; + pMove->memory = move.pDstBlock->GetDeviceMemory(); + pMove->offset = move.dstOffset; + + ++ pMove; + } + + pCtx->defragmentationMovesProcessed += moveCount; + + return moveCount; +} + +void VmaBlockVector::CommitDefragmentations( + class VmaBlockVectorDefragmentationContext *pCtx, + VmaDefragmentationStats* pStats) +{ + VmaMutexLockWrite lock(m_Mutex, m_hAllocator->m_UseMutex); + + for(uint32_t i = pCtx->defragmentationMovesCommitted; i < pCtx->defragmentationMovesProcessed; ++ i) + { + const VmaDefragmentationMove &move = pCtx->defragmentationMoves[i]; + + move.pSrcBlock->m_pMetadata->FreeAtOffset(move.srcOffset); + move.hAllocation->ChangeBlockAllocation(m_hAllocator, move.pDstBlock, move.dstOffset); + } + + pCtx->defragmentationMovesCommitted = pCtx->defragmentationMovesProcessed; + FreeEmptyBlocks(pStats); +} + size_t VmaBlockVector::CalcAllocationCount() const { size_t result = 0; @@ -13070,7 +14396,8 @@ void VmaDefragmentationAlgorithm_Generic::AddAllocation(VmaAllocation hAlloc, Vk VkResult VmaDefragmentationAlgorithm_Generic::DefragmentRound( VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove) + uint32_t maxAllocationsToMove, + bool freeOldAllocations) { if(m_Blocks.empty()) { @@ -13125,7 +14452,7 @@ VkResult VmaDefragmentationAlgorithm_Generic::DefragmentRound( srcAllocIndex = m_Blocks[srcBlockIndex]->m_Allocations.size() - 1; } } - + BlockInfo* pSrcBlockInfo = m_Blocks[srcBlockIndex]; AllocationInfo& allocInfo = pSrcBlockInfo->m_Allocations[srcAllocIndex]; @@ -13162,12 +14489,16 @@ VkResult VmaDefragmentationAlgorithm_Generic::DefragmentRound( return VK_SUCCESS; } - VmaDefragmentationMove move; + VmaDefragmentationMove move = {}; move.srcBlockIndex = pSrcBlockInfo->m_OriginalBlockIndex; move.dstBlockIndex = pDstBlockInfo->m_OriginalBlockIndex; move.srcOffset = srcOffset; move.dstOffset = dstAllocRequest.offset; move.size = size; + move.hAllocation = allocInfo.m_hAllocation; + move.pSrcBlock = pSrcBlockInfo->m_pBlock; + move.pDstBlock = pDstBlockInfo->m_pBlock; + moves.push_back(move); pDstBlockInfo->m_pBlock->m_pMetadata->Alloc( @@ -13175,9 +14506,12 @@ VkResult VmaDefragmentationAlgorithm_Generic::DefragmentRound( suballocType, size, allocInfo.m_hAllocation); - pSrcBlockInfo->m_pBlock->m_pMetadata->FreeAtOffset(srcOffset); - - allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset); + + if(freeOldAllocations) + { + pSrcBlockInfo->m_pBlock->m_pMetadata->FreeAtOffset(srcOffset); + allocInfo.m_hAllocation->ChangeBlockAllocation(m_hAllocator, pDstBlockInfo->m_pBlock, dstAllocRequest.offset); + } if(allocInfo.m_pChanged != VMA_NULL) { @@ -13230,7 +14564,8 @@ size_t VmaDefragmentationAlgorithm_Generic::CalcBlocksWithNonMovableCount() cons VkResult VmaDefragmentationAlgorithm_Generic::Defragment( VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove) + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags) { if(!m_AllAllocations && m_AllocationCount == 0) { @@ -13258,7 +14593,7 @@ VkResult VmaDefragmentationAlgorithm_Generic::Defragment( } pBlockInfo->CalcHasNonMovableAllocations(); - + // This is a choice based on research. // Option 1: pBlockInfo->SortAllocationsByOffsetDescending(); @@ -13276,7 +14611,7 @@ VkResult VmaDefragmentationAlgorithm_Generic::Defragment( VkResult result = VK_SUCCESS; for(uint32_t round = 0; (round < roundCount) && (result == VK_SUCCESS); ++round) { - result = DefragmentRound(moves, maxBytesToMove, maxAllocationsToMove); + result = DefragmentRound(moves, maxBytesToMove, maxAllocationsToMove, !(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL)); } return result; @@ -13328,7 +14663,8 @@ VmaDefragmentationAlgorithm_Fast::~VmaDefragmentationAlgorithm_Fast() VkResult VmaDefragmentationAlgorithm_Fast::Defragment( VmaVector< VmaDefragmentationMove, VmaStlAllocator<VmaDefragmentationMove> >& moves, VkDeviceSize maxBytesToMove, - uint32_t maxAllocationsToMove) + uint32_t maxAllocationsToMove, + VmaDefragmentationFlags flags) { VMA_ASSERT(m_AllAllocations || m_pBlockVector->CalcAllocationCount() == m_AllocationCount); @@ -13384,6 +14720,7 @@ VkResult VmaDefragmentationAlgorithm_Fast::Defragment( } const VkDeviceSize srcAllocOffset = srcSuballocIt->offset; + VmaDefragmentationMove move = {}; // Try to place it in one of free spaces from the database. size_t freeSpaceInfoIndex; VkDeviceSize dstAllocOffset; @@ -13406,7 +14743,7 @@ VkResult VmaDefragmentationAlgorithm_Fast::Defragment( suballoc.hAllocation->ChangeOffset(dstAllocOffset); m_BytesMoved += srcAllocSize; ++m_AllocationsMoved; - + VmaSuballocationList::iterator nextSuballocIt = srcSuballocIt; ++nextSuballocIt; pSrcMetadata->m_Suballocations.erase(srcSuballocIt); @@ -13414,10 +14751,12 @@ VkResult VmaDefragmentationAlgorithm_Fast::Defragment( InsertSuballoc(pFreeSpaceMetadata, suballoc); - VmaDefragmentationMove move = { - srcOrigBlockIndex, freeSpaceOrigBlockIndex, - srcAllocOffset, dstAllocOffset, - srcAllocSize }; + move.srcBlockIndex = srcOrigBlockIndex; + move.dstBlockIndex = freeSpaceOrigBlockIndex; + move.srcOffset = srcAllocOffset; + move.dstOffset = dstAllocOffset; + move.size = srcAllocSize; + moves.push_back(move); } // Different block @@ -13440,10 +14779,12 @@ VkResult VmaDefragmentationAlgorithm_Fast::Defragment( InsertSuballoc(pFreeSpaceMetadata, suballoc); - VmaDefragmentationMove move = { - srcOrigBlockIndex, freeSpaceOrigBlockIndex, - srcAllocOffset, dstAllocOffset, - srcAllocSize }; + move.srcBlockIndex = srcOrigBlockIndex; + move.dstBlockIndex = freeSpaceOrigBlockIndex; + move.srcOffset = srcAllocOffset; + move.dstOffset = dstAllocOffset; + move.size = srcAllocSize; + moves.push_back(move); } } @@ -13498,10 +14839,13 @@ VkResult VmaDefragmentationAlgorithm_Fast::Defragment( m_BytesMoved += srcAllocSize; ++m_AllocationsMoved; ++srcSuballocIt; - VmaDefragmentationMove move = { - srcOrigBlockIndex, dstOrigBlockIndex, - srcAllocOffset, dstAllocOffset, - srcAllocSize }; + + move.srcBlockIndex = srcOrigBlockIndex; + move.dstBlockIndex = dstOrigBlockIndex; + move.srcOffset = srcAllocOffset; + move.dstOffset = dstAllocOffset; + move.size = srcAllocSize; + moves.push_back(move); } } @@ -13527,10 +14871,12 @@ VkResult VmaDefragmentationAlgorithm_Fast::Defragment( pDstMetadata->m_Suballocations.push_back(suballoc); - VmaDefragmentationMove move = { - srcOrigBlockIndex, dstOrigBlockIndex, - srcAllocOffset, dstAllocOffset, - srcAllocSize }; + move.srcBlockIndex = srcOrigBlockIndex; + move.dstBlockIndex = dstOrigBlockIndex; + move.srcOffset = srcAllocOffset; + move.dstOffset = dstAllocOffset; + move.size = srcAllocSize; + moves.push_back(move); } } @@ -13538,7 +14884,7 @@ VkResult VmaDefragmentationAlgorithm_Fast::Defragment( } m_BlockInfos.clear(); - + PostprocessMetadata(); return VK_SUCCESS; @@ -13580,7 +14926,7 @@ void VmaDefragmentationAlgorithm_Fast::PostprocessMetadata() VmaBlockMetadata_Generic* const pMetadata = (VmaBlockMetadata_Generic*)m_pBlockVector->GetBlock(blockIndex)->m_pMetadata; const VkDeviceSize blockSize = pMetadata->GetSize(); - + // No allocations in this block - entire area is free. if(pMetadata->m_Suballocations.empty()) { @@ -13680,6 +15026,10 @@ VmaBlockVectorDefragmentationContext::VmaBlockVectorDefragmentationContext( res(VK_SUCCESS), mutexLocked(false), blockContexts(VmaStlAllocator<VmaBlockDefragmentationContext>(hAllocator->GetAllocationCallbacks())), + defragmentationMoves(VmaStlAllocator<VmaDefragmentationMove>(hAllocator->GetAllocationCallbacks())), + defragmentationMovesProcessed(0), + defragmentationMovesCommitted(0), + hasDefragmentationPlan(0), m_hAllocator(hAllocator), m_hCustomPool(hCustomPool), m_pBlockVector(pBlockVector), @@ -13701,7 +15051,7 @@ void VmaBlockVectorDefragmentationContext::AddAllocation(VmaAllocation hAlloc, V m_Allocations.push_back(info); } -void VmaBlockVectorDefragmentationContext::Begin(bool overlappingMoveSupported) +void VmaBlockVectorDefragmentationContext::Begin(bool overlappingMoveSupported, VmaDefragmentationFlags flags) { const bool allAllocations = m_AllAllocations || m_Allocations.size() == m_pBlockVector->CalcAllocationCount(); @@ -13715,10 +15065,12 @@ void VmaBlockVectorDefragmentationContext::Begin(bool overlappingMoveSupported) - VMA_DEBUG_MARGIN is 0. - All allocations in this block vector are moveable. - There is no possibility of image/buffer granularity conflict. + - The defragmentation is not incremental */ if(VMA_DEBUG_MARGIN == 0 && allAllocations && - !m_pBlockVector->IsBufferImageGranularityConflictPossible()) + !m_pBlockVector->IsBufferImageGranularityConflictPossible() && + !(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL)) { m_pAlgorithm = vma_new(m_hAllocator, VmaDefragmentationAlgorithm_Fast)( m_hAllocator, m_pBlockVector, m_CurrFrameIndex, overlappingMoveSupported); @@ -13764,7 +15116,7 @@ VmaDefragmentationContext_T::~VmaDefragmentationContext_T() for(size_t i = m_CustomPoolContexts.size(); i--; ) { VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_CustomPoolContexts[i]; - pBlockVectorCtx->GetBlockVector()->DefragmentationEnd(pBlockVectorCtx, m_pStats); + pBlockVectorCtx->GetBlockVector()->DefragmentationEnd(pBlockVectorCtx, m_Flags, m_pStats); vma_delete(m_hAllocator, pBlockVectorCtx); } for(size_t i = m_hAllocator->m_MemProps.memoryTypeCount; i--; ) @@ -13772,13 +15124,13 @@ VmaDefragmentationContext_T::~VmaDefragmentationContext_T() VmaBlockVectorDefragmentationContext* pBlockVectorCtx = m_DefaultPoolContexts[i]; if(pBlockVectorCtx) { - pBlockVectorCtx->GetBlockVector()->DefragmentationEnd(pBlockVectorCtx, m_pStats); + pBlockVectorCtx->GetBlockVector()->DefragmentationEnd(pBlockVectorCtx, m_Flags, m_pStats); vma_delete(m_hAllocator, pBlockVectorCtx); } } } -void VmaDefragmentationContext_T::AddPools(uint32_t poolCount, VmaPool* pPools) +void VmaDefragmentationContext_T::AddPools(uint32_t poolCount, const VmaPool* pPools) { for(uint32_t poolIndex = 0; poolIndex < poolCount; ++poolIndex) { @@ -13788,7 +15140,7 @@ void VmaDefragmentationContext_T::AddPools(uint32_t poolCount, VmaPool* pPools) if(pool->m_BlockVector.GetAlgorithm() == 0) { VmaBlockVectorDefragmentationContext* pBlockVectorDefragCtx = VMA_NULL; - + for(size_t i = m_CustomPoolContexts.size(); i--; ) { if(m_CustomPoolContexts[i]->GetCustomPool() == pool) @@ -13797,7 +15149,7 @@ void VmaDefragmentationContext_T::AddPools(uint32_t poolCount, VmaPool* pPools) break; } } - + if(!pBlockVectorDefragCtx) { pBlockVectorDefragCtx = vma_new(m_hAllocator, VmaBlockVectorDefragmentationContext)( @@ -13815,7 +15167,7 @@ void VmaDefragmentationContext_T::AddPools(uint32_t poolCount, VmaPool* pPools) void VmaDefragmentationContext_T::AddAllocations( uint32_t allocationCount, - VmaAllocation* pAllocations, + const VmaAllocation* pAllocations, VkBool32* pAllocationsChanged) { // Dispatch pAllocations among defragmentators. Create them when necessary. @@ -13885,13 +15237,30 @@ void VmaDefragmentationContext_T::AddAllocations( VkResult VmaDefragmentationContext_T::Defragment( VkDeviceSize maxCpuBytesToMove, uint32_t maxCpuAllocationsToMove, VkDeviceSize maxGpuBytesToMove, uint32_t maxGpuAllocationsToMove, - VkCommandBuffer commandBuffer, VmaDefragmentationStats* pStats) + VkCommandBuffer commandBuffer, VmaDefragmentationStats* pStats, VmaDefragmentationFlags flags) { if(pStats) { memset(pStats, 0, sizeof(VmaDefragmentationStats)); } + if(flags & VMA_DEFRAGMENTATION_FLAG_INCREMENTAL) + { + // For incremental defragmetnations, we just earmark how much we can move + // The real meat is in the defragmentation steps + m_MaxCpuBytesToMove = maxCpuBytesToMove; + m_MaxCpuAllocationsToMove = maxCpuAllocationsToMove; + + m_MaxGpuBytesToMove = maxGpuBytesToMove; + m_MaxGpuAllocationsToMove = maxGpuAllocationsToMove; + + if(m_MaxCpuBytesToMove == 0 && m_MaxCpuAllocationsToMove == 0 && + m_MaxGpuBytesToMove == 0 && m_MaxGpuAllocationsToMove == 0) + return VK_SUCCESS; + + return VK_NOT_READY; + } + if(commandBuffer == VK_NULL_HANDLE) { maxGpuBytesToMove = 0; @@ -13911,7 +15280,7 @@ VkResult VmaDefragmentationContext_T::Defragment( VMA_ASSERT(pBlockVectorCtx->GetBlockVector()); pBlockVectorCtx->GetBlockVector()->Defragment( pBlockVectorCtx, - pStats, + pStats, flags, maxCpuBytesToMove, maxCpuAllocationsToMove, maxGpuBytesToMove, maxGpuAllocationsToMove, commandBuffer); @@ -13931,7 +15300,7 @@ VkResult VmaDefragmentationContext_T::Defragment( VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector()); pBlockVectorCtx->GetBlockVector()->Defragment( pBlockVectorCtx, - pStats, + pStats, flags, maxCpuBytesToMove, maxCpuAllocationsToMove, maxGpuBytesToMove, maxGpuAllocationsToMove, commandBuffer); @@ -13944,6 +15313,132 @@ VkResult VmaDefragmentationContext_T::Defragment( return res; } +VkResult VmaDefragmentationContext_T::DefragmentPassBegin(VmaDefragmentationPassInfo* pInfo) +{ + VmaDefragmentationPassMoveInfo* pCurrentMove = pInfo->pMoves; + uint32_t movesLeft = pInfo->moveCount; + + // Process default pools. + for(uint32_t memTypeIndex = 0; + memTypeIndex < m_hAllocator->GetMemoryTypeCount(); + ++memTypeIndex) + { + VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_DefaultPoolContexts[memTypeIndex]; + if(pBlockVectorCtx) + { + VMA_ASSERT(pBlockVectorCtx->GetBlockVector()); + + if(!pBlockVectorCtx->hasDefragmentationPlan) + { + pBlockVectorCtx->GetBlockVector()->Defragment( + pBlockVectorCtx, + m_pStats, m_Flags, + m_MaxCpuBytesToMove, m_MaxCpuAllocationsToMove, + m_MaxGpuBytesToMove, m_MaxGpuAllocationsToMove, + VK_NULL_HANDLE); + + if(pBlockVectorCtx->res < VK_SUCCESS) + continue; + + pBlockVectorCtx->hasDefragmentationPlan = true; + } + + const uint32_t processed = pBlockVectorCtx->GetBlockVector()->ProcessDefragmentations( + pBlockVectorCtx, + pCurrentMove, movesLeft); + + movesLeft -= processed; + pCurrentMove += processed; + } + } + + // Process custom pools. + for(size_t customCtxIndex = 0, customCtxCount = m_CustomPoolContexts.size(); + customCtxIndex < customCtxCount; + ++customCtxIndex) + { + VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_CustomPoolContexts[customCtxIndex]; + VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector()); + + if(!pBlockVectorCtx->hasDefragmentationPlan) + { + pBlockVectorCtx->GetBlockVector()->Defragment( + pBlockVectorCtx, + m_pStats, m_Flags, + m_MaxCpuBytesToMove, m_MaxCpuAllocationsToMove, + m_MaxGpuBytesToMove, m_MaxGpuAllocationsToMove, + VK_NULL_HANDLE); + + if(pBlockVectorCtx->res < VK_SUCCESS) + continue; + + pBlockVectorCtx->hasDefragmentationPlan = true; + } + + const uint32_t processed = pBlockVectorCtx->GetBlockVector()->ProcessDefragmentations( + pBlockVectorCtx, + pCurrentMove, movesLeft); + + movesLeft -= processed; + pCurrentMove += processed; + } + + pInfo->moveCount = pInfo->moveCount - movesLeft; + + return VK_SUCCESS; +} +VkResult VmaDefragmentationContext_T::DefragmentPassEnd() +{ + VkResult res = VK_SUCCESS; + + // Process default pools. + for(uint32_t memTypeIndex = 0; + memTypeIndex < m_hAllocator->GetMemoryTypeCount(); + ++memTypeIndex) + { + VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_DefaultPoolContexts[memTypeIndex]; + if(pBlockVectorCtx) + { + VMA_ASSERT(pBlockVectorCtx->GetBlockVector()); + + if(!pBlockVectorCtx->hasDefragmentationPlan) + { + res = VK_NOT_READY; + continue; + } + + pBlockVectorCtx->GetBlockVector()->CommitDefragmentations( + pBlockVectorCtx, m_pStats); + + if(pBlockVectorCtx->defragmentationMoves.size() != pBlockVectorCtx->defragmentationMovesCommitted) + res = VK_NOT_READY; + } + } + + // Process custom pools. + for(size_t customCtxIndex = 0, customCtxCount = m_CustomPoolContexts.size(); + customCtxIndex < customCtxCount; + ++customCtxIndex) + { + VmaBlockVectorDefragmentationContext *pBlockVectorCtx = m_CustomPoolContexts[customCtxIndex]; + VMA_ASSERT(pBlockVectorCtx && pBlockVectorCtx->GetBlockVector()); + + if(!pBlockVectorCtx->hasDefragmentationPlan) + { + res = VK_NOT_READY; + continue; + } + + pBlockVectorCtx->GetBlockVector()->CommitDefragmentations( + pBlockVectorCtx, m_pStats); + + if(pBlockVectorCtx->defragmentationMoves.size() != pBlockVectorCtx->defragmentationMovesCommitted) + res = VK_NOT_READY; + } + + return res; +} + //////////////////////////////////////////////////////////////////////////////// // VmaRecorder @@ -13953,8 +15448,7 @@ VmaRecorder::VmaRecorder() : m_UseMutex(true), m_Flags(0), m_File(VMA_NULL), - m_Freq(INT64_MAX), - m_StartCounter(INT64_MAX) + m_RecordingStartTime(std::chrono::high_resolution_clock::now()) { } @@ -13963,15 +15457,23 @@ VkResult VmaRecorder::Init(const VmaRecordSettings& settings, bool useMutex) m_UseMutex = useMutex; m_Flags = settings.flags; - QueryPerformanceFrequency((LARGE_INTEGER*)&m_Freq); - QueryPerformanceCounter((LARGE_INTEGER*)&m_StartCounter); - +#if defined(_WIN32) // Open file for writing. errno_t err = fopen_s(&m_File, settings.pFilePath, "wb"); + if(err != 0) { return VK_ERROR_INITIALIZATION_FAILED; } +#else + // Open file for writing. + m_File = fopen(settings.pFilePath, "wb"); + + if(m_File == 0) + { + return VK_ERROR_INITIALIZATION_FAILED; + } +#endif // Write header. fprintf(m_File, "%s\n", "Vulkan Memory Allocator,Calls recording"); @@ -14431,7 +15933,8 @@ VmaRecorder::UserDataString::UserDataString(VmaAllocationCreateFlags allocFlags, } else { - sprintf_s(m_PtrStr, "%p", pUserData); + // If VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT is not specified, convert the string's memory address to a string and store it. + snprintf(m_PtrStr, 17, "%p", pUserData); m_Str = m_PtrStr; } } @@ -14447,7 +15950,8 @@ void VmaRecorder::WriteConfiguration( uint32_t vulkanApiVersion, bool dedicatedAllocationExtensionEnabled, bool bindMemory2ExtensionEnabled, - bool memoryBudgetExtensionEnabled) + bool memoryBudgetExtensionEnabled, + bool deviceCoherentMemoryExtensionEnabled) { fprintf(m_File, "Config,Begin\n"); @@ -14480,9 +15984,10 @@ void VmaRecorder::WriteConfiguration( fprintf(m_File, "Extension,VK_KHR_dedicated_allocation,%u\n", dedicatedAllocationExtensionEnabled ? 1 : 0); fprintf(m_File, "Extension,VK_KHR_bind_memory2,%u\n", bindMemory2ExtensionEnabled ? 1 : 0); fprintf(m_File, "Extension,VK_EXT_memory_budget,%u\n", memoryBudgetExtensionEnabled ? 1 : 0); + fprintf(m_File, "Extension,VK_AMD_device_coherent_memory,%u\n", deviceCoherentMemoryExtensionEnabled ? 1 : 0); fprintf(m_File, "Macro,VMA_DEBUG_ALWAYS_DEDICATED_MEMORY,%u\n", VMA_DEBUG_ALWAYS_DEDICATED_MEMORY ? 1 : 0); - fprintf(m_File, "Macro,VMA_DEBUG_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_DEBUG_ALIGNMENT); + fprintf(m_File, "Macro,VMA_MIN_ALIGNMENT,%llu\n", (VkDeviceSize)VMA_MIN_ALIGNMENT); fprintf(m_File, "Macro,VMA_DEBUG_MARGIN,%llu\n", (VkDeviceSize)VMA_DEBUG_MARGIN); fprintf(m_File, "Macro,VMA_DEBUG_INITIALIZE_ALLOCATIONS,%u\n", VMA_DEBUG_INITIALIZE_ALLOCATIONS ? 1 : 0); fprintf(m_File, "Macro,VMA_DEBUG_DETECT_CORRUPTION,%u\n", VMA_DEBUG_DETECT_CORRUPTION ? 1 : 0); @@ -14496,11 +16001,22 @@ void VmaRecorder::WriteConfiguration( void VmaRecorder::GetBasicParams(CallParams& outParams) { - outParams.threadId = GetCurrentThreadId(); + #if defined(_WIN32) + outParams.threadId = GetCurrentThreadId(); + #else + // Use C++11 features to get thread id and convert it to uint32_t. + // There is room for optimization since sstream is quite slow. + // Is there a better way to convert std::this_thread::get_id() to uint32_t? + std::thread::id thread_id = std::this_thread::get_id(); + std::stringstream thread_id_to_string_converter; + thread_id_to_string_converter << thread_id; + std::string thread_id_as_string = thread_id_to_string_converter.str(); + outParams.threadId = static_cast<uint32_t>(std::stoi(thread_id_as_string.c_str())); + #endif + + auto current_time = std::chrono::high_resolution_clock::now(); - LARGE_INTEGER counter; - QueryPerformanceCounter(&counter); - outParams.time = (double)(counter.QuadPart - m_StartCounter) / (double)m_Freq; + outParams.time = std::chrono::duration<double, std::chrono::seconds::period>(current_time - m_RecordingStartTime).count(); } void VmaRecorder::PrintPointerList(uint64_t count, const VmaAllocation* pItems) @@ -14533,10 +16049,10 @@ VmaAllocationObjectAllocator::VmaAllocationObjectAllocator(const VkAllocationCal { } -VmaAllocation VmaAllocationObjectAllocator::Allocate() +template<typename... Types> VmaAllocation VmaAllocationObjectAllocator::Allocate(Types... args) { VmaMutexLock mutexLock(m_Mutex); - return m_Allocator.Alloc(); + return m_Allocator.Alloc<Types...>(std::forward<Types>(args)...); } void VmaAllocationObjectAllocator::Free(VmaAllocation hAlloc) @@ -14554,6 +16070,9 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : m_UseKhrDedicatedAllocation((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_DEDICATED_ALLOCATION_BIT) != 0), m_UseKhrBindMemory2((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_KHR_BIND_MEMORY2_BIT) != 0), m_UseExtMemoryBudget((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT) != 0), + m_UseAmdDeviceCoherentMemory((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_AMD_DEVICE_COHERENT_MEMORY_BIT) != 0), + m_UseKhrBufferDeviceAddress((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT) != 0), + m_UseExtMemoryPriority((pCreateInfo->flags & VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT) != 0), m_hDevice(pCreateInfo->device), m_hInstance(pCreateInfo->instance), m_AllocationCallbacksSpecified(pCreateInfo->pAllocationCallbacks != VMA_NULL), @@ -14561,12 +16080,13 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : *pCreateInfo->pAllocationCallbacks : VmaEmptyAllocationCallbacks), m_AllocationObjectAllocator(&m_AllocationCallbacks), m_HeapSizeLimitMask(0), + m_DeviceMemoryCount(0), m_PreferredLargeHeapBlockSize(0), m_PhysicalDevice(pCreateInfo->physicalDevice), m_CurrentFrameIndex(0), m_GpuDefragmentationMemoryTypeBits(UINT32_MAX), - m_Pools(VmaStlAllocator<VmaPool>(GetAllocationCallbacks())), - m_NextPoolId(0) + m_NextPoolId(0), + m_GlobalMemoryTypeBits(UINT32_MAX) #if VMA_RECORDING_ENABLED ,m_pRecorder(VMA_NULL) #endif @@ -14583,7 +16103,7 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : VMA_ASSERT(VMA_DEBUG_MARGIN % sizeof(uint32_t) == 0); } - VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device); + VMA_ASSERT(pCreateInfo->physicalDevice && pCreateInfo->device && pCreateInfo->instance); if(m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0)) { @@ -14606,23 +16126,46 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_EXT_MEMORY_BUDGET_BIT set but required extension is disabled by preprocessor macros."); } #endif +#if !(VMA_BUFFER_DEVICE_ADDRESS) + if(m_UseKhrBufferDeviceAddress) + { + VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT is set but required extension or Vulkan 1.2 is not available in your Vulkan header or its support in VMA has been disabled by a preprocessor macro."); + } +#endif +#if VMA_VULKAN_VERSION < 1002000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 2, 0)) + { + VMA_ASSERT(0 && "vulkanApiVersion >= VK_API_VERSION_1_2 but required Vulkan version is disabled by preprocessor macros."); + } +#endif #if VMA_VULKAN_VERSION < 1001000 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) { VMA_ASSERT(0 && "vulkanApiVersion >= VK_API_VERSION_1_1 but required Vulkan version is disabled by preprocessor macros."); } #endif +#if !(VMA_MEMORY_PRIORITY) + if(m_UseExtMemoryPriority) + { + VMA_ASSERT(0 && "VMA_ALLOCATOR_CREATE_EXT_MEMORY_PRIORITY_BIT is set but required extension is not available in your Vulkan header or its support in VMA has been disabled by a preprocessor macro."); + } +#endif memset(&m_DeviceMemoryCallbacks, 0 ,sizeof(m_DeviceMemoryCallbacks)); memset(&m_PhysicalDeviceProperties, 0, sizeof(m_PhysicalDeviceProperties)); memset(&m_MemProps, 0, sizeof(m_MemProps)); - + memset(&m_pBlockVectors, 0, sizeof(m_pBlockVectors)); - memset(&m_pDedicatedAllocations, 0, sizeof(m_pDedicatedAllocations)); + memset(&m_pSmallBufferBlockVectors, 0, sizeof(m_pSmallBufferBlockVectors)); memset(&m_VulkanFunctions, 0, sizeof(m_VulkanFunctions)); +#if VMA_EXTERNAL_MEMORY + memset(&m_TypeExternalMemoryHandleTypes, 0, sizeof(m_TypeExternalMemoryHandleTypes)); +#endif // #if VMA_EXTERNAL_MEMORY + if(pCreateInfo->pDeviceMemoryCallbacks != VMA_NULL) { + m_DeviceMemoryCallbacks.pUserData = pCreateInfo->pDeviceMemoryCallbacks->pUserData; m_DeviceMemoryCallbacks.pfnAllocate = pCreateInfo->pDeviceMemoryCallbacks->pfnAllocate; m_DeviceMemoryCallbacks.pfnFree = pCreateInfo->pDeviceMemoryCallbacks->pfnFree; } @@ -14632,7 +16175,7 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : (*m_VulkanFunctions.vkGetPhysicalDeviceProperties)(m_PhysicalDevice, &m_PhysicalDeviceProperties); (*m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties)(m_PhysicalDevice, &m_MemProps); - VMA_ASSERT(VmaIsPow2(VMA_DEBUG_ALIGNMENT)); + VMA_ASSERT(VmaIsPow2(VMA_MIN_ALIGNMENT)); VMA_ASSERT(VmaIsPow2(VMA_DEBUG_MIN_BUFFER_IMAGE_GRANULARITY)); VMA_ASSERT(VmaIsPow2(m_PhysicalDeviceProperties.limits.bufferImageGranularity)); VMA_ASSERT(VmaIsPow2(m_PhysicalDeviceProperties.limits.nonCoherentAtomSize)); @@ -14640,6 +16183,16 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : m_PreferredLargeHeapBlockSize = (pCreateInfo->preferredLargeHeapBlockSize != 0) ? pCreateInfo->preferredLargeHeapBlockSize : static_cast<VkDeviceSize>(VMA_DEFAULT_LARGE_HEAP_BLOCK_SIZE); + m_GlobalMemoryTypeBits = CalculateGlobalMemoryTypeBits(); + +#if VMA_EXTERNAL_MEMORY + if(pCreateInfo->pTypeExternalMemoryHandleTypes != VMA_NULL) + { + memcpy(m_TypeExternalMemoryHandleTypes, pCreateInfo->pTypeExternalMemoryHandleTypes, + sizeof(VkExternalMemoryHandleTypeFlagsKHR) * GetMemoryTypeCount()); + } +#endif // #if VMA_EXTERNAL_MEMORY + if(pCreateInfo->pHeapSizeLimit != VMA_NULL) { for(uint32_t heapIndex = 0; heapIndex < GetMemoryHeapCount(); ++heapIndex) @@ -14670,11 +16223,26 @@ VmaAllocator_T::VmaAllocator_T(const VmaAllocatorCreateInfo* pCreateInfo) : GetBufferImageGranularity(), pCreateInfo->frameInUseCount, false, // explicitBlockSize - false); // linearAlgorithm + false, // linearAlgorithm + 0.5f, // priority (0.5 is the default per Vulkan spec) + GetMemoryTypeMinAlignment(memTypeIndex), // minAllocationAlignment + VMA_NULL); // // pMemoryAllocateNext + m_pSmallBufferBlockVectors[memTypeIndex] = vma_new(this, VmaBlockVector)( + this, + VK_NULL_HANDLE, // hParentPool + memTypeIndex, + preferredBlockSize, + 0, + SIZE_MAX, + 1, // bufferImageGranularity forced to 1 !!! + pCreateInfo->frameInUseCount, + false, // explicitBlockSize + false, // linearAlgorithm + 0.5f, // priority (0.5 is the default per Vulkan spec) + GetMemoryTypeMinAlignment(memTypeIndex), // minAllocationAlignment + VMA_NULL); // // pMemoryAllocateNext // No need to call m_pBlockVectors[memTypeIndex][blockVectorTypeIndex]->CreateMinBlocks here, // becase minBlockCount is 0. - m_pDedicatedAllocations[memTypeIndex] = vma_new(this, AllocationVectorType)(VmaStlAllocator<VmaAllocation>(GetAllocationCallbacks())); - } } @@ -14698,7 +16266,8 @@ VkResult VmaAllocator_T::Init(const VmaAllocatorCreateInfo* pCreateInfo) m_VulkanApiVersion, m_UseKhrDedicatedAllocation, m_UseKhrBindMemory2, - m_UseExtMemoryBudget); + m_UseExtMemoryBudget, + m_UseAmdDeviceCoherentMemory); m_pRecorder->RecordCreateAllocator(GetCurrentFrameIndex()); #else VMA_ASSERT(0 && "VmaAllocatorCreateInfo::pRecordSettings used, but not supported due to VMA_RECORDING_ENABLED not defined to 1."); @@ -14725,24 +16294,44 @@ VmaAllocator_T::~VmaAllocator_T() vma_delete(this, m_pRecorder); } #endif - - VMA_ASSERT(m_Pools.empty()); - for(size_t i = GetMemoryTypeCount(); i--; ) + VMA_ASSERT(m_Pools.IsEmpty()); + + for(size_t memTypeIndex = GetMemoryTypeCount(); memTypeIndex--; ) { - if(m_pDedicatedAllocations[i] != VMA_NULL && !m_pDedicatedAllocations[i]->empty()) + if(!m_DedicatedAllocations[memTypeIndex].IsEmpty()) { VMA_ASSERT(0 && "Unfreed dedicated allocations found."); } - vma_delete(this, m_pDedicatedAllocations[i]); - vma_delete(this, m_pBlockVectors[i]); + vma_delete(this, m_pSmallBufferBlockVectors[memTypeIndex]); + vma_delete(this, m_pBlockVectors[memTypeIndex]); } } void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunctions) { #if VMA_STATIC_VULKAN_FUNCTIONS == 1 + ImportVulkanFunctions_Static(); +#endif + + if(pVulkanFunctions != VMA_NULL) + { + ImportVulkanFunctions_Custom(pVulkanFunctions); + } + +#if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1 + ImportVulkanFunctions_Dynamic(); +#endif + + ValidateVulkanFunctions(); +} + +#if VMA_STATIC_VULKAN_FUNCTIONS == 1 + +void VmaAllocator_T::ImportVulkanFunctions_Static() +{ + // Vulkan 1.0 m_VulkanFunctions.vkGetPhysicalDeviceProperties = (PFN_vkGetPhysicalDeviceProperties)vkGetPhysicalDeviceProperties; m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties = (PFN_vkGetPhysicalDeviceMemoryProperties)vkGetPhysicalDeviceMemoryProperties; m_VulkanFunctions.vkAllocateMemory = (PFN_vkAllocateMemory)vkAllocateMemory; @@ -14760,89 +16349,137 @@ void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunc m_VulkanFunctions.vkCreateImage = (PFN_vkCreateImage)vkCreateImage; m_VulkanFunctions.vkDestroyImage = (PFN_vkDestroyImage)vkDestroyImage; m_VulkanFunctions.vkCmdCopyBuffer = (PFN_vkCmdCopyBuffer)vkCmdCopyBuffer; + + // Vulkan 1.1 #if VMA_VULKAN_VERSION >= 1001000 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) { - VMA_ASSERT(m_hInstance != VK_NULL_HANDLE); - m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR = - (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2"); - m_VulkanFunctions.vkGetImageMemoryRequirements2KHR = - (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2"); - m_VulkanFunctions.vkBindBufferMemory2KHR = - (PFN_vkBindBufferMemory2KHR)vkGetDeviceProcAddr(m_hDevice, "vkBindBufferMemory2"); - m_VulkanFunctions.vkBindImageMemory2KHR = - (PFN_vkBindImageMemory2KHR)vkGetDeviceProcAddr(m_hDevice, "vkBindImageMemory2"); - m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties2KHR = - (PFN_vkGetPhysicalDeviceMemoryProperties2KHR)vkGetInstanceProcAddr(m_hInstance, "vkGetPhysicalDeviceMemoryProperties2"); + m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR = (PFN_vkGetBufferMemoryRequirements2)vkGetBufferMemoryRequirements2; + m_VulkanFunctions.vkGetImageMemoryRequirements2KHR = (PFN_vkGetImageMemoryRequirements2)vkGetImageMemoryRequirements2; + m_VulkanFunctions.vkBindBufferMemory2KHR = (PFN_vkBindBufferMemory2)vkBindBufferMemory2; + m_VulkanFunctions.vkBindImageMemory2KHR = (PFN_vkBindImageMemory2)vkBindImageMemory2; + m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties2KHR = (PFN_vkGetPhysicalDeviceMemoryProperties2)vkGetPhysicalDeviceMemoryProperties2; } #endif +} + +#endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1 + +void VmaAllocator_T::ImportVulkanFunctions_Custom(const VmaVulkanFunctions* pVulkanFunctions) +{ + VMA_ASSERT(pVulkanFunctions != VMA_NULL); + +#define VMA_COPY_IF_NOT_NULL(funcName) \ + if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName; + + VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties); + VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties); + VMA_COPY_IF_NOT_NULL(vkAllocateMemory); + VMA_COPY_IF_NOT_NULL(vkFreeMemory); + VMA_COPY_IF_NOT_NULL(vkMapMemory); + VMA_COPY_IF_NOT_NULL(vkUnmapMemory); + VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges); + VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges); + VMA_COPY_IF_NOT_NULL(vkBindBufferMemory); + VMA_COPY_IF_NOT_NULL(vkBindImageMemory); + VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements); + VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements); + VMA_COPY_IF_NOT_NULL(vkCreateBuffer); + VMA_COPY_IF_NOT_NULL(vkDestroyBuffer); + VMA_COPY_IF_NOT_NULL(vkCreateImage); + VMA_COPY_IF_NOT_NULL(vkDestroyImage); + VMA_COPY_IF_NOT_NULL(vkCmdCopyBuffer); + +#if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 + VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR); + VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR); +#endif + +#if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000 + VMA_COPY_IF_NOT_NULL(vkBindBufferMemory2KHR); + VMA_COPY_IF_NOT_NULL(vkBindImageMemory2KHR); +#endif + +#if VMA_MEMORY_BUDGET + VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties2KHR); +#endif + +#undef VMA_COPY_IF_NOT_NULL +} + +#if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1 + +void VmaAllocator_T::ImportVulkanFunctions_Dynamic() +{ +#define VMA_FETCH_INSTANCE_FUNC(memberName, functionPointerType, functionNameString) \ + if(m_VulkanFunctions.memberName == VMA_NULL) \ + m_VulkanFunctions.memberName = \ + (functionPointerType)vkGetInstanceProcAddr(m_hInstance, functionNameString); +#define VMA_FETCH_DEVICE_FUNC(memberName, functionPointerType, functionNameString) \ + if(m_VulkanFunctions.memberName == VMA_NULL) \ + m_VulkanFunctions.memberName = \ + (functionPointerType)vkGetDeviceProcAddr(m_hDevice, functionNameString); + + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceProperties, PFN_vkGetPhysicalDeviceProperties, "vkGetPhysicalDeviceProperties"); + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties, PFN_vkGetPhysicalDeviceMemoryProperties, "vkGetPhysicalDeviceMemoryProperties"); + VMA_FETCH_DEVICE_FUNC(vkAllocateMemory, PFN_vkAllocateMemory, "vkAllocateMemory"); + VMA_FETCH_DEVICE_FUNC(vkFreeMemory, PFN_vkFreeMemory, "vkFreeMemory"); + VMA_FETCH_DEVICE_FUNC(vkMapMemory, PFN_vkMapMemory, "vkMapMemory"); + VMA_FETCH_DEVICE_FUNC(vkUnmapMemory, PFN_vkUnmapMemory, "vkUnmapMemory"); + VMA_FETCH_DEVICE_FUNC(vkFlushMappedMemoryRanges, PFN_vkFlushMappedMemoryRanges, "vkFlushMappedMemoryRanges"); + VMA_FETCH_DEVICE_FUNC(vkInvalidateMappedMemoryRanges, PFN_vkInvalidateMappedMemoryRanges, "vkInvalidateMappedMemoryRanges"); + VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory, PFN_vkBindBufferMemory, "vkBindBufferMemory"); + VMA_FETCH_DEVICE_FUNC(vkBindImageMemory, PFN_vkBindImageMemory, "vkBindImageMemory"); + VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements, PFN_vkGetBufferMemoryRequirements, "vkGetBufferMemoryRequirements"); + VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements, PFN_vkGetImageMemoryRequirements, "vkGetImageMemoryRequirements"); + VMA_FETCH_DEVICE_FUNC(vkCreateBuffer, PFN_vkCreateBuffer, "vkCreateBuffer"); + VMA_FETCH_DEVICE_FUNC(vkDestroyBuffer, PFN_vkDestroyBuffer, "vkDestroyBuffer"); + VMA_FETCH_DEVICE_FUNC(vkCreateImage, PFN_vkCreateImage, "vkCreateImage"); + VMA_FETCH_DEVICE_FUNC(vkDestroyImage, PFN_vkDestroyImage, "vkDestroyImage"); + VMA_FETCH_DEVICE_FUNC(vkCmdCopyBuffer, PFN_vkCmdCopyBuffer, "vkCmdCopyBuffer"); + +#if VMA_VULKAN_VERSION >= 1001000 + if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) + { + VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements2KHR, PFN_vkGetBufferMemoryRequirements2, "vkGetBufferMemoryRequirements2"); + VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements2KHR, PFN_vkGetImageMemoryRequirements2, "vkGetImageMemoryRequirements2"); + VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory2KHR, PFN_vkBindBufferMemory2, "vkBindBufferMemory2"); + VMA_FETCH_DEVICE_FUNC(vkBindImageMemory2KHR, PFN_vkBindImageMemory2, "vkBindImageMemory2"); + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2, "vkGetPhysicalDeviceMemoryProperties2"); + } +#endif + #if VMA_DEDICATED_ALLOCATION if(m_UseKhrDedicatedAllocation) { - m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR = - (PFN_vkGetBufferMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetBufferMemoryRequirements2KHR"); - m_VulkanFunctions.vkGetImageMemoryRequirements2KHR = - (PFN_vkGetImageMemoryRequirements2KHR)vkGetDeviceProcAddr(m_hDevice, "vkGetImageMemoryRequirements2KHR"); + VMA_FETCH_DEVICE_FUNC(vkGetBufferMemoryRequirements2KHR, PFN_vkGetBufferMemoryRequirements2KHR, "vkGetBufferMemoryRequirements2KHR"); + VMA_FETCH_DEVICE_FUNC(vkGetImageMemoryRequirements2KHR, PFN_vkGetImageMemoryRequirements2KHR, "vkGetImageMemoryRequirements2KHR"); } #endif + #if VMA_BIND_MEMORY2 if(m_UseKhrBindMemory2) { - m_VulkanFunctions.vkBindBufferMemory2KHR = - (PFN_vkBindBufferMemory2KHR)vkGetDeviceProcAddr(m_hDevice, "vkBindBufferMemory2KHR"); - m_VulkanFunctions.vkBindImageMemory2KHR = - (PFN_vkBindImageMemory2KHR)vkGetDeviceProcAddr(m_hDevice, "vkBindImageMemory2KHR"); + VMA_FETCH_DEVICE_FUNC(vkBindBufferMemory2KHR, PFN_vkBindBufferMemory2KHR, "vkBindBufferMemory2KHR"); + VMA_FETCH_DEVICE_FUNC(vkBindImageMemory2KHR, PFN_vkBindImageMemory2KHR, "vkBindImageMemory2KHR"); } #endif // #if VMA_BIND_MEMORY2 + #if VMA_MEMORY_BUDGET - if(m_UseExtMemoryBudget && m_VulkanApiVersion < VK_MAKE_VERSION(1, 1, 0)) + if(m_UseExtMemoryBudget) { - VMA_ASSERT(m_hInstance != VK_NULL_HANDLE); - m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties2KHR = - (PFN_vkGetPhysicalDeviceMemoryProperties2KHR)vkGetInstanceProcAddr(m_hInstance, "vkGetPhysicalDeviceMemoryProperties2KHR"); + VMA_FETCH_INSTANCE_FUNC(vkGetPhysicalDeviceMemoryProperties2KHR, PFN_vkGetPhysicalDeviceMemoryProperties2KHR, "vkGetPhysicalDeviceMemoryProperties2KHR"); } #endif // #if VMA_MEMORY_BUDGET -#endif // #if VMA_STATIC_VULKAN_FUNCTIONS == 1 - -#define VMA_COPY_IF_NOT_NULL(funcName) \ - if(pVulkanFunctions->funcName != VMA_NULL) m_VulkanFunctions.funcName = pVulkanFunctions->funcName; - if(pVulkanFunctions != VMA_NULL) - { - VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceProperties); - VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties); - VMA_COPY_IF_NOT_NULL(vkAllocateMemory); - VMA_COPY_IF_NOT_NULL(vkFreeMemory); - VMA_COPY_IF_NOT_NULL(vkMapMemory); - VMA_COPY_IF_NOT_NULL(vkUnmapMemory); - VMA_COPY_IF_NOT_NULL(vkFlushMappedMemoryRanges); - VMA_COPY_IF_NOT_NULL(vkInvalidateMappedMemoryRanges); - VMA_COPY_IF_NOT_NULL(vkBindBufferMemory); - VMA_COPY_IF_NOT_NULL(vkBindImageMemory); - VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements); - VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements); - VMA_COPY_IF_NOT_NULL(vkCreateBuffer); - VMA_COPY_IF_NOT_NULL(vkDestroyBuffer); - VMA_COPY_IF_NOT_NULL(vkCreateImage); - VMA_COPY_IF_NOT_NULL(vkDestroyImage); - VMA_COPY_IF_NOT_NULL(vkCmdCopyBuffer); -#if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 - VMA_COPY_IF_NOT_NULL(vkGetBufferMemoryRequirements2KHR); - VMA_COPY_IF_NOT_NULL(vkGetImageMemoryRequirements2KHR); -#endif -#if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000 - VMA_COPY_IF_NOT_NULL(vkBindBufferMemory2KHR); - VMA_COPY_IF_NOT_NULL(vkBindImageMemory2KHR); -#endif -#if VMA_MEMORY_BUDGET - VMA_COPY_IF_NOT_NULL(vkGetPhysicalDeviceMemoryProperties2KHR); -#endif - } +#undef VMA_FETCH_DEVICE_FUNC +#undef VMA_FETCH_INSTANCE_FUNC +} -#undef VMA_COPY_IF_NOT_NULL +#endif // #if VMA_DYNAMIC_VULKAN_FUNCTIONS == 1 - // If these asserts are hit, you must either #define VMA_STATIC_VULKAN_FUNCTIONS 1 - // or pass valid pointers as VmaAllocatorCreateInfo::pVulkanFunctions. +void VmaAllocator_T::ValidateVulkanFunctions() +{ VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceProperties != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkGetPhysicalDeviceMemoryProperties != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkAllocateMemory != VMA_NULL); @@ -14860,6 +16497,7 @@ void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunc VMA_ASSERT(m_VulkanFunctions.vkCreateImage != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkDestroyImage != VMA_NULL); VMA_ASSERT(m_VulkanFunctions.vkCmdCopyBuffer != VMA_NULL); + #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0) || m_UseKhrDedicatedAllocation) { @@ -14867,6 +16505,7 @@ void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunc VMA_ASSERT(m_VulkanFunctions.vkGetImageMemoryRequirements2KHR != VMA_NULL); } #endif + #if VMA_BIND_MEMORY2 || VMA_VULKAN_VERSION >= 1001000 if(m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0) || m_UseKhrBindMemory2) { @@ -14874,6 +16513,7 @@ void VmaAllocator_T::ImportVulkanFunctions(const VmaVulkanFunctions* pVulkanFunc VMA_ASSERT(m_VulkanFunctions.vkBindImageMemory2KHR != VMA_NULL); } #endif + #if VMA_MEMORY_BUDGET || VMA_VULKAN_VERSION >= 1001000 if(m_UseExtMemoryBudget || m_VulkanApiVersion >= VK_MAKE_VERSION(1, 1, 0)) { @@ -14895,6 +16535,7 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( VkDeviceSize alignment, bool dedicatedAllocation, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, const VmaAllocationCreateInfo& createInfo, uint32_t memTypeIndex, @@ -14919,7 +16560,8 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( finalCreateInfo.flags |= VMA_ALLOCATION_CREATE_DEDICATED_MEMORY_BIT; } - VmaBlockVector* const blockVector = m_pBlockVectors[memTypeIndex]; + bool isSmallBuffer = dedicatedBuffer != VK_NULL_HANDLE && size <= 4096; // TODO + VmaBlockVector* const blockVector = isSmallBuffer ? m_pSmallBufferBlockVectors[memTypeIndex] : m_pBlockVectors[memTypeIndex]; VMA_ASSERT(blockVector); const VkDeviceSize preferredBlockSize = blockVector->GetPreferredBlockSize(); @@ -14952,7 +16594,9 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0, (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0, finalCreateInfo.pUserData, + finalCreateInfo.priority, dedicatedBuffer, + dedicatedBufferUsage, dedicatedImage, allocationCount, pAllocations); @@ -14978,32 +16622,40 @@ VkResult VmaAllocator_T::AllocateMemoryOfType( { return VK_ERROR_OUT_OF_DEVICE_MEMORY; } + + // Protection against creating each allocation as dedicated when we reach or exceed heap size/budget, + // which can quickly deplete maxMemoryAllocationCount: Don't try dedicated allocations when above + // 3/4 of the maximum allocation count. + if(m_DeviceMemoryCount.load() > m_PhysicalDeviceProperties.limits.maxMemoryAllocationCount * 3 / 4) + { + return VK_ERROR_OUT_OF_DEVICE_MEMORY; + } + + res = AllocateDedicatedMemory( + size, + suballocType, + memTypeIndex, + (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) != 0, + (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0, + (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0, + finalCreateInfo.pUserData, + finalCreateInfo.priority, + dedicatedBuffer, + dedicatedBufferUsage, + dedicatedImage, + allocationCount, + pAllocations); + if(res == VK_SUCCESS) + { + // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here. + VMA_DEBUG_LOG(" Allocated as DedicatedMemory"); + return VK_SUCCESS; + } else { - res = AllocateDedicatedMemory( - size, - suballocType, - memTypeIndex, - (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_WITHIN_BUDGET_BIT) != 0, - (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0, - (finalCreateInfo.flags & VMA_ALLOCATION_CREATE_USER_DATA_COPY_STRING_BIT) != 0, - finalCreateInfo.pUserData, - dedicatedBuffer, - dedicatedImage, - allocationCount, - pAllocations); - if(res == VK_SUCCESS) - { - // Succeeded: AllocateDedicatedMemory function already filld pMemory, nothing more to do here. - VMA_DEBUG_LOG(" Allocated as DedicatedMemory"); - return VK_SUCCESS; - } - else - { - // Everything failed: Return error code. - VMA_DEBUG_LOG(" vkAllocateMemory FAILED"); - return res; - } + // Everything failed: Return error code. + VMA_DEBUG_LOG(" vkAllocateMemory FAILED"); + return res; } } } @@ -15016,7 +16668,9 @@ VkResult VmaAllocator_T::AllocateDedicatedMemory( bool map, bool isUserDataString, void* pUserData, + float priority, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, size_t allocationCount, VmaAllocation* pAllocations) @@ -15046,16 +16700,57 @@ VkResult VmaAllocator_T::AllocateDedicatedMemory( { VMA_ASSERT(dedicatedImage == VK_NULL_HANDLE); dedicatedAllocInfo.buffer = dedicatedBuffer; - allocInfo.pNext = &dedicatedAllocInfo; + VmaPnextChainPushFront(&allocInfo, &dedicatedAllocInfo); } else if(dedicatedImage != VK_NULL_HANDLE) { dedicatedAllocInfo.image = dedicatedImage; - allocInfo.pNext = &dedicatedAllocInfo; + VmaPnextChainPushFront(&allocInfo, &dedicatedAllocInfo); } } #endif // #if VMA_DEDICATED_ALLOCATION || VMA_VULKAN_VERSION >= 1001000 +#if VMA_BUFFER_DEVICE_ADDRESS + VkMemoryAllocateFlagsInfoKHR allocFlagsInfo = { VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_FLAGS_INFO_KHR }; + if(m_UseKhrBufferDeviceAddress) + { + bool canContainBufferWithDeviceAddress = true; + if(dedicatedBuffer != VK_NULL_HANDLE) + { + canContainBufferWithDeviceAddress = dedicatedBufferUsage == UINT32_MAX || // Usage flags unknown + (dedicatedBufferUsage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_EXT) != 0; + } + else if(dedicatedImage != VK_NULL_HANDLE) + { + canContainBufferWithDeviceAddress = false; + } + if(canContainBufferWithDeviceAddress) + { + allocFlagsInfo.flags = VK_MEMORY_ALLOCATE_DEVICE_ADDRESS_BIT_KHR; + VmaPnextChainPushFront(&allocInfo, &allocFlagsInfo); + } + } +#endif // #if VMA_BUFFER_DEVICE_ADDRESS + +#if VMA_MEMORY_PRIORITY + VkMemoryPriorityAllocateInfoEXT priorityInfo = { VK_STRUCTURE_TYPE_MEMORY_PRIORITY_ALLOCATE_INFO_EXT }; + if(m_UseExtMemoryPriority) + { + priorityInfo.priority = priority; + VmaPnextChainPushFront(&allocInfo, &priorityInfo); + } +#endif // #if VMA_MEMORY_PRIORITY + +#if VMA_EXTERNAL_MEMORY + // Attach VkExportMemoryAllocateInfoKHR if necessary. + VkExportMemoryAllocateInfoKHR exportMemoryAllocInfo = { VK_STRUCTURE_TYPE_EXPORT_MEMORY_ALLOCATE_INFO_KHR }; + exportMemoryAllocInfo.handleTypes = GetExternalMemoryHandleTypeFlags(memTypeIndex); + if(exportMemoryAllocInfo.handleTypes != 0) + { + VmaPnextChainPushFront(&allocInfo, &exportMemoryAllocInfo); + } +#endif // #if VMA_EXTERNAL_MEMORY + size_t allocIndex; VkResult res = VK_SUCCESS; for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex) @@ -15077,14 +16772,13 @@ VkResult VmaAllocator_T::AllocateDedicatedMemory( if(res == VK_SUCCESS) { - // Register them in m_pDedicatedAllocations. + // Register them in m_DedicatedAllocations. { VmaMutexLockWrite lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); - AllocationVectorType* pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex]; - VMA_ASSERT(pDedicatedAllocations); + DedicatedAllocationLinkedList& dedicatedAllocations = m_DedicatedAllocations[memTypeIndex]; for(allocIndex = 0; allocIndex < allocationCount; ++allocIndex) { - VmaVectorInsertSorted<VmaPointerLess>(*pDedicatedAllocations, pAllocations[allocIndex]); + dedicatedAllocations.PushBack(pAllocations[allocIndex]); } } @@ -15097,7 +16791,7 @@ VkResult VmaAllocator_T::AllocateDedicatedMemory( { VmaAllocation currAlloc = pAllocations[allocIndex]; VkDeviceMemory hMemory = currAlloc->GetMemory(); - + /* There is no need to call this, because Vulkan spec allows to skip vkUnmapMemory before vkFreeMemory. @@ -15107,11 +16801,10 @@ VkResult VmaAllocator_T::AllocateDedicatedMemory( (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory); } */ - + FreeVulkanMemory(memTypeIndex, currAlloc->GetSize(), hMemory); m_Budget.RemoveAllocation(MemoryTypeIndexToHeapIndex(memTypeIndex), currAlloc->GetSize()); currAlloc->SetUserData(this, VMA_NULL); - currAlloc->Dtor(); m_AllocationObjectAllocator.Free(currAlloc); } @@ -15157,8 +16850,7 @@ VkResult VmaAllocator_T::AllocateDedicatedMemoryPage( } } - *pAllocation = m_AllocationObjectAllocator.Allocate(); - (*pAllocation)->Ctor(m_CurrentFrameIndex.load(), isUserDataString); + *pAllocation = m_AllocationObjectAllocator.Allocate(m_CurrentFrameIndex.load(), isUserDataString); (*pAllocation)->InitDedicatedAllocation(memTypeIndex, hMemory, suballocType, pMappedData, size); (*pAllocation)->SetUserData(this, pUserData); m_Budget.AddAllocation(MemoryTypeIndexToHeapIndex(memTypeIndex), size); @@ -15185,7 +16877,7 @@ void VmaAllocator_T::GetBufferMemoryRequirements( VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR }; VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR }; - memReq2.pNext = &memDedicatedReq; + VmaPnextChainPushFront(&memReq2, &memDedicatedReq); (*m_VulkanFunctions.vkGetBufferMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2); @@ -15217,7 +16909,7 @@ void VmaAllocator_T::GetImageMemoryRequirements( VkMemoryDedicatedRequirementsKHR memDedicatedReq = { VK_STRUCTURE_TYPE_MEMORY_DEDICATED_REQUIREMENTS_KHR }; VkMemoryRequirements2KHR memReq2 = { VK_STRUCTURE_TYPE_MEMORY_REQUIREMENTS_2_KHR }; - memReq2.pNext = &memDedicatedReq; + VmaPnextChainPushFront(&memReq2, &memDedicatedReq); (*m_VulkanFunctions.vkGetImageMemoryRequirements2KHR)(m_hDevice, &memReqInfo, &memReq2); @@ -15239,6 +16931,7 @@ VkResult VmaAllocator_T::AllocateMemory( bool requiresDedicatedAllocation, bool prefersDedicatedAllocation, VkBuffer dedicatedBuffer, + VkBufferUsageFlags dedicatedBufferUsage, VkImage dedicatedImage, const VmaAllocationCreateInfo& createInfo, VmaSuballocationType suballocType, @@ -15287,10 +16980,6 @@ VkResult VmaAllocator_T::AllocateMemory( if(createInfo.pool != VK_NULL_HANDLE) { - const VkDeviceSize alignmentForPool = VMA_MAX( - vkMemReq.alignment, - GetMemoryTypeMinAlignment(createInfo.pool->m_BlockVector.GetMemoryTypeIndex())); - VmaAllocationCreateInfo createInfoForPool = createInfo; // If memory type is not HOST_VISIBLE, disable MAPPED. if((createInfoForPool.flags & VMA_ALLOCATION_CREATE_MAPPED_BIT) != 0 && @@ -15302,7 +16991,7 @@ VkResult VmaAllocator_T::AllocateMemory( return createInfo.pool->m_BlockVector.Allocate( m_CurrentFrameIndex.load(), vkMemReq.size, - alignmentForPool, + vkMemReq.alignment, createInfoForPool, suballocType, allocationCount, @@ -15316,15 +17005,12 @@ VkResult VmaAllocator_T::AllocateMemory( VkResult res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex); if(res == VK_SUCCESS) { - VkDeviceSize alignmentForMemType = VMA_MAX( - vkMemReq.alignment, - GetMemoryTypeMinAlignment(memTypeIndex)); - res = AllocateMemoryOfType( vkMemReq.size, - alignmentForMemType, + vkMemReq.alignment, requiresDedicatedAllocation || prefersDedicatedAllocation, dedicatedBuffer, + dedicatedBufferUsage, dedicatedImage, createInfo, memTypeIndex, @@ -15347,15 +17033,12 @@ VkResult VmaAllocator_T::AllocateMemory( res = vmaFindMemoryTypeIndex(this, memoryTypeBits, &createInfo, &memTypeIndex); if(res == VK_SUCCESS) { - alignmentForMemType = VMA_MAX( - vkMemReq.alignment, - GetMemoryTypeMinAlignment(memTypeIndex)); - res = AllocateMemoryOfType( vkMemReq.size, - alignmentForMemType, + vkMemReq.alignment, requiresDedicatedAllocation || prefersDedicatedAllocation, dedicatedBuffer, + dedicatedBufferUsage, dedicatedImage, createInfo, memTypeIndex, @@ -15415,8 +17098,7 @@ void VmaAllocator_T::FreeMemory( } else { - const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); - pBlockVector = m_pBlockVectors[memTypeIndex]; + pBlockVector = allocation->GetBlock()->GetParentBlockVector(); } pBlockVector->Free(allocation); } @@ -15432,28 +17114,11 @@ void VmaAllocator_T::FreeMemory( // Do this regardless of whether the allocation is lost. Lost allocations still account to Budget.AllocationBytes. m_Budget.RemoveAllocation(MemoryTypeIndexToHeapIndex(allocation->GetMemoryTypeIndex()), allocation->GetSize()); allocation->SetUserData(this, VMA_NULL); - allocation->Dtor(); m_AllocationObjectAllocator.Free(allocation); } } } -VkResult VmaAllocator_T::ResizeAllocation( - const VmaAllocation alloc, - VkDeviceSize newSize) -{ - // This function is deprecated and so it does nothing. It's left for backward compatibility. - if(newSize == 0 || alloc->GetLastUseFrameIndex() == VMA_FRAME_INDEX_LOST) - { - return VK_ERROR_VALIDATION_FAILED_EXT; - } - if(newSize == alloc->GetSize()) - { - return VK_SUCCESS; - } - return VK_ERROR_OUT_OF_POOL_MEMORY; -} - void VmaAllocator_T::CalculateStats(VmaStats* pStats) { // Initialize. @@ -15462,21 +17127,25 @@ void VmaAllocator_T::CalculateStats(VmaStats* pStats) InitStatInfo(pStats->memoryType[i]); for(size_t i = 0; i < VK_MAX_MEMORY_HEAPS; ++i) InitStatInfo(pStats->memoryHeap[i]); - + // Process default pools. for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) { VmaBlockVector* const pBlockVector = m_pBlockVectors[memTypeIndex]; VMA_ASSERT(pBlockVector); pBlockVector->AddStats(pStats); + + VmaBlockVector* const pSmallBufferBlockVector = m_pSmallBufferBlockVectors[memTypeIndex]; + VMA_ASSERT(pSmallBufferBlockVector); + pSmallBufferBlockVector->AddStats(pStats); } // Process custom pools. { VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex); - for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex) + for(VmaPool pool = m_Pools.Front(); pool != VMA_NULL; pool = m_Pools.GetNext(pool)) { - m_Pools[poolIndex]->m_BlockVector.AddStats(pStats); + pool->m_BlockVector.AddStats(pStats); } } @@ -15485,12 +17154,12 @@ void VmaAllocator_T::CalculateStats(VmaStats* pStats) { const uint32_t memHeapIndex = MemoryTypeIndexToHeapIndex(memTypeIndex); VmaMutexLockRead dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); - AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex]; - VMA_ASSERT(pDedicatedAllocVector); - for(size_t allocIndex = 0, allocCount = pDedicatedAllocVector->size(); allocIndex < allocCount; ++allocIndex) + DedicatedAllocationLinkedList& dedicatedAllocList = m_DedicatedAllocations[memTypeIndex]; + for(VmaAllocation alloc = dedicatedAllocList.Front(); + alloc != VMA_NULL; alloc = dedicatedAllocList.GetNext(alloc)) { VmaStatInfo allocationStatInfo; - (*pDedicatedAllocVector)[allocIndex]->DedicatedAllocCalcStatsInfo(allocationStatInfo); + alloc->DedicatedAllocCalcStatsInfo(allocationStatInfo); VmaAddStatInfo(pStats->total, allocationStatInfo); VmaAddStatInfo(pStats->memoryType[memTypeIndex], allocationStatInfo); VmaAddStatInfo(pStats->memoryHeap[memHeapIndex], allocationStatInfo); @@ -15579,7 +17248,7 @@ VkResult VmaAllocator_T::DefragmentationBegin( VkResult res = (*pContext)->Defragment( info.maxCpuBytesToMove, info.maxCpuAllocationsToMove, info.maxGpuBytesToMove, info.maxGpuAllocationsToMove, - info.commandBuffer, pStats); + info.commandBuffer, pStats, info.flags); if(res != VK_NOT_READY) { @@ -15597,6 +17266,19 @@ VkResult VmaAllocator_T::DefragmentationEnd( return VK_SUCCESS; } +VkResult VmaAllocator_T::DefragmentationPassBegin( + VmaDefragmentationPassInfo* pInfo, + VmaDefragmentationContext context) +{ + return context->DefragmentPassBegin(pInfo); +} +VkResult VmaAllocator_T::DefragmentationPassEnd( + VmaDefragmentationContext context) +{ + return context->DefragmentPassEnd(); + +} + void VmaAllocator_T::GetAllocationInfo(VmaAllocation hAllocation, VmaAllocationInfo* pAllocationInfo) { if(hAllocation->CanBecomeLost()) @@ -15727,6 +17409,12 @@ VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPoo VmaPoolCreateInfo newCreateInfo = *pCreateInfo; + // Protection against uninitialized new structure member. If garbage data are left there, this pointer dereference would crash. + if(pCreateInfo->pMemoryAllocateNext) + { + VMA_ASSERT(((const VkBaseInStructure*)pCreateInfo->pMemoryAllocateNext)->sType != 0); + } + if(newCreateInfo.maxBlockCount == 0) { newCreateInfo.maxBlockCount = SIZE_MAX; @@ -15735,6 +17423,16 @@ VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPoo { return VK_ERROR_INITIALIZATION_FAILED; } + // Memory type index out of range or forbidden. + if(pCreateInfo->memoryTypeIndex >= GetMemoryTypeCount() || + ((1u << pCreateInfo->memoryTypeIndex) & m_GlobalMemoryTypeBits) == 0) + { + return VK_ERROR_FEATURE_NOT_PRESENT; + } + if(newCreateInfo.minAllocationAlignment > 0) + { + VMA_ASSERT(VmaIsPow2(newCreateInfo.minAllocationAlignment)); + } const VkDeviceSize preferredBlockSize = CalcPreferredBlockSize(newCreateInfo.memoryTypeIndex); @@ -15752,7 +17450,7 @@ VkResult VmaAllocator_T::CreatePool(const VmaPoolCreateInfo* pCreateInfo, VmaPoo { VmaMutexLockWrite lock(m_PoolsMutex, m_UseMutex); (*pPool)->SetId(m_NextPoolId++); - VmaVectorInsertSorted<VmaPointerLess>(m_Pools, *pPool); + m_Pools.PushBack(*pPool); } return VK_SUCCESS; @@ -15763,8 +17461,7 @@ void VmaAllocator_T::DestroyPool(VmaPool pool) // Remove from m_Pools. { VmaMutexLockWrite lock(m_PoolsMutex, m_UseMutex); - bool success = VmaVectorRemoveSorted<VmaPointerLess>(m_Pools, pool); - VMA_ASSERT(success && "Pool not found in Allocator."); + m_Pools.Remove(pool); } vma_delete(this, pool); @@ -15829,11 +17526,11 @@ VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits) // Process custom pools. { VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex); - for(size_t poolIndex = 0, poolCount = m_Pools.size(); poolIndex < poolCount; ++poolIndex) + for(VmaPool pool = m_Pools.Front(); pool != VMA_NULL; pool = m_Pools.GetNext(pool)) { - if(((1u << m_Pools[poolIndex]->m_BlockVector.GetMemoryTypeIndex()) & memoryTypeBits) != 0) + if(((1u << pool->m_BlockVector.GetMemoryTypeIndex()) & memoryTypeBits) != 0) { - VkResult localRes = m_Pools[poolIndex]->m_BlockVector.CheckCorruption(); + VkResult localRes = pool->m_BlockVector.CheckCorruption(); switch(localRes) { case VK_ERROR_FEATURE_NOT_PRESENT: @@ -15853,13 +17550,46 @@ VkResult VmaAllocator_T::CheckCorruption(uint32_t memoryTypeBits) void VmaAllocator_T::CreateLostAllocation(VmaAllocation* pAllocation) { - *pAllocation = m_AllocationObjectAllocator.Allocate(); - (*pAllocation)->Ctor(VMA_FRAME_INDEX_LOST, false); + *pAllocation = m_AllocationObjectAllocator.Allocate(VMA_FRAME_INDEX_LOST, false); (*pAllocation)->InitLost(); } +// An object that increments given atomic but decrements it back in the destructor unless Commit() is called. +template<typename T> +struct AtomicTransactionalIncrement +{ +public: + typedef std::atomic<T> AtomicT; + ~AtomicTransactionalIncrement() + { + if(m_Atomic) + --(*m_Atomic); + } + T Increment(AtomicT* atomic) + { + m_Atomic = atomic; + return m_Atomic->fetch_add(1); + } + void Commit() + { + m_Atomic = nullptr; + } + +private: + AtomicT* m_Atomic = nullptr; +}; + VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAllocateInfo, VkDeviceMemory* pMemory) { + AtomicTransactionalIncrement<uint32_t> deviceMemoryCountIncrement; + const uint64_t prevDeviceMemoryCount = deviceMemoryCountIncrement.Increment(&m_DeviceMemoryCount); +#if VMA_DEBUG_DONT_EXCEED_MAX_MEMORY_ALLOCATION_COUNT + if(prevDeviceMemoryCount >= m_PhysicalDeviceProperties.limits.maxMemoryAllocationCount) + { + return VK_ERROR_TOO_MANY_OBJECTS; + } +#endif + const uint32_t heapIndex = MemoryTypeIndexToHeapIndex(pAllocateInfo->memoryTypeIndex); // HeapSizeLimit is in effect for this heap. @@ -15897,8 +17627,10 @@ VkResult VmaAllocator_T::AllocateVulkanMemory(const VkMemoryAllocateInfo* pAlloc // Informative callback. if(m_DeviceMemoryCallbacks.pfnAllocate != VMA_NULL) { - (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize); + (*m_DeviceMemoryCallbacks.pfnAllocate)(this, pAllocateInfo->memoryTypeIndex, *pMemory, pAllocateInfo->allocationSize, m_DeviceMemoryCallbacks.pUserData); } + + deviceMemoryCountIncrement.Commit(); } else { @@ -15913,13 +17645,15 @@ void VmaAllocator_T::FreeVulkanMemory(uint32_t memoryType, VkDeviceSize size, Vk // Informative callback. if(m_DeviceMemoryCallbacks.pfnFree != VMA_NULL) { - (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size); + (*m_DeviceMemoryCallbacks.pfnFree)(this, memoryType, hMemory, size, m_DeviceMemoryCallbacks.pUserData); } // VULKAN CALL vkFreeMemory. (*m_VulkanFunctions.vkFreeMemory)(m_hDevice, hMemory, GetAllocationCallbacks()); m_Budget.m_BlockBytes[MemoryTypeIndexToHeapIndex(memoryType)] -= size; + + --m_DeviceMemoryCount; } VkResult VmaAllocator_T::BindVulkanBuffer( @@ -16082,80 +17816,71 @@ VkResult VmaAllocator_T::BindImageMemory( return res; } -void VmaAllocator_T::FlushOrInvalidateAllocation( +VkResult VmaAllocator_T::FlushOrInvalidateAllocation( VmaAllocation hAllocation, VkDeviceSize offset, VkDeviceSize size, VMA_CACHE_OPERATION op) { - const uint32_t memTypeIndex = hAllocation->GetMemoryTypeIndex(); - if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex)) - { - const VkDeviceSize allocationSize = hAllocation->GetSize(); - VMA_ASSERT(offset <= allocationSize); - - const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize; + VkResult res = VK_SUCCESS; - VkMappedMemoryRange memRange = { VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE }; - memRange.memory = hAllocation->GetMemory(); - - switch(hAllocation->GetType()) + VkMappedMemoryRange memRange = {}; + if(GetFlushOrInvalidateRange(hAllocation, offset, size, memRange)) + { + switch(op) { - case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: - memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize); - if(size == VK_WHOLE_SIZE) - { - memRange.size = allocationSize - memRange.offset; - } - else - { - VMA_ASSERT(offset + size <= allocationSize); - memRange.size = VMA_MIN( - VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize), - allocationSize - memRange.offset); - } + case VMA_CACHE_FLUSH: + res = (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange); break; - - case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: - { - // 1. Still within this allocation. - memRange.offset = VmaAlignDown(offset, nonCoherentAtomSize); - if(size == VK_WHOLE_SIZE) - { - size = allocationSize - offset; - } - else - { - VMA_ASSERT(offset + size <= allocationSize); - } - memRange.size = VmaAlignUp(size + (offset - memRange.offset), nonCoherentAtomSize); - - // 2. Adjust to whole block. - const VkDeviceSize allocationOffset = hAllocation->GetOffset(); - VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0); - const VkDeviceSize blockSize = hAllocation->GetBlock()->m_pMetadata->GetSize(); - memRange.offset += allocationOffset; - memRange.size = VMA_MIN(memRange.size, blockSize - memRange.offset); - + case VMA_CACHE_INVALIDATE: + res = (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange); break; - } - default: VMA_ASSERT(0); } + } + // else: Just ignore this call. + return res; +} + +VkResult VmaAllocator_T::FlushOrInvalidateAllocations( + uint32_t allocationCount, + const VmaAllocation* allocations, + const VkDeviceSize* offsets, const VkDeviceSize* sizes, + VMA_CACHE_OPERATION op) +{ + typedef VmaStlAllocator<VkMappedMemoryRange> RangeAllocator; + typedef VmaSmallVector<VkMappedMemoryRange, RangeAllocator, 16> RangeVector; + RangeVector ranges = RangeVector(RangeAllocator(GetAllocationCallbacks())); + + for(uint32_t allocIndex = 0; allocIndex < allocationCount; ++allocIndex) + { + const VmaAllocation alloc = allocations[allocIndex]; + const VkDeviceSize offset = offsets != VMA_NULL ? offsets[allocIndex] : 0; + const VkDeviceSize size = sizes != VMA_NULL ? sizes[allocIndex] : VK_WHOLE_SIZE; + VkMappedMemoryRange newRange; + if(GetFlushOrInvalidateRange(alloc, offset, size, newRange)) + { + ranges.push_back(newRange); + } + } + VkResult res = VK_SUCCESS; + if(!ranges.empty()) + { switch(op) { case VMA_CACHE_FLUSH: - (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, 1, &memRange); + res = (*GetVulkanFunctions().vkFlushMappedMemoryRanges)(m_hDevice, (uint32_t)ranges.size(), ranges.data()); break; case VMA_CACHE_INVALIDATE: - (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, 1, &memRange); + res = (*GetVulkanFunctions().vkInvalidateMappedMemoryRanges)(m_hDevice, (uint32_t)ranges.size(), ranges.data()); break; default: VMA_ASSERT(0); } } // else: Just ignore this call. + return res; } void VmaAllocator_T::FreeDedicatedMemory(const VmaAllocation allocation) @@ -16165,14 +17890,12 @@ void VmaAllocator_T::FreeDedicatedMemory(const VmaAllocation allocation) const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); { VmaMutexLockWrite lock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); - AllocationVectorType* const pDedicatedAllocations = m_pDedicatedAllocations[memTypeIndex]; - VMA_ASSERT(pDedicatedAllocations); - bool success = VmaVectorRemoveSorted<VmaPointerLess>(*pDedicatedAllocations, allocation); - VMA_ASSERT(success); + DedicatedAllocationLinkedList& dedicatedAllocations = m_DedicatedAllocations[memTypeIndex]; + dedicatedAllocations.Remove(allocation); } VkDeviceMemory hMemory = allocation->GetMemory(); - + /* There is no need to call this, because Vulkan spec allows to skip vkUnmapMemory before vkFreeMemory. @@ -16182,7 +17905,7 @@ void VmaAllocator_T::FreeDedicatedMemory(const VmaAllocation allocation) (*m_VulkanFunctions.vkUnmapMemory)(m_hDevice, hMemory); } */ - + FreeVulkanMemory(memTypeIndex, allocation->GetSize(), hMemory); VMA_DEBUG_LOG(" Freed DedicatedMemory MemoryTypeIndex=%u", memTypeIndex); @@ -16213,6 +17936,91 @@ uint32_t VmaAllocator_T::CalculateGpuDefragmentationMemoryTypeBits() const return memoryTypeBits; } +uint32_t VmaAllocator_T::CalculateGlobalMemoryTypeBits() const +{ + // Make sure memory information is already fetched. + VMA_ASSERT(GetMemoryTypeCount() > 0); + + uint32_t memoryTypeBits = UINT32_MAX; + + if(!m_UseAmdDeviceCoherentMemory) + { + // Exclude memory types that have VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD. + for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) + { + if((m_MemProps.memoryTypes[memTypeIndex].propertyFlags & VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY) != 0) + { + memoryTypeBits &= ~(1u << memTypeIndex); + } + } + } + + return memoryTypeBits; +} + +bool VmaAllocator_T::GetFlushOrInvalidateRange( + VmaAllocation allocation, + VkDeviceSize offset, VkDeviceSize size, + VkMappedMemoryRange& outRange) const +{ + const uint32_t memTypeIndex = allocation->GetMemoryTypeIndex(); + if(size > 0 && IsMemoryTypeNonCoherent(memTypeIndex)) + { + const VkDeviceSize nonCoherentAtomSize = m_PhysicalDeviceProperties.limits.nonCoherentAtomSize; + const VkDeviceSize allocationSize = allocation->GetSize(); + VMA_ASSERT(offset <= allocationSize); + + outRange.sType = VK_STRUCTURE_TYPE_MAPPED_MEMORY_RANGE; + outRange.pNext = VMA_NULL; + outRange.memory = allocation->GetMemory(); + + switch(allocation->GetType()) + { + case VmaAllocation_T::ALLOCATION_TYPE_DEDICATED: + outRange.offset = VmaAlignDown(offset, nonCoherentAtomSize); + if(size == VK_WHOLE_SIZE) + { + outRange.size = allocationSize - outRange.offset; + } + else + { + VMA_ASSERT(offset + size <= allocationSize); + outRange.size = VMA_MIN( + VmaAlignUp(size + (offset - outRange.offset), nonCoherentAtomSize), + allocationSize - outRange.offset); + } + break; + case VmaAllocation_T::ALLOCATION_TYPE_BLOCK: + { + // 1. Still within this allocation. + outRange.offset = VmaAlignDown(offset, nonCoherentAtomSize); + if(size == VK_WHOLE_SIZE) + { + size = allocationSize - offset; + } + else + { + VMA_ASSERT(offset + size <= allocationSize); + } + outRange.size = VmaAlignUp(size + (offset - outRange.offset), nonCoherentAtomSize); + + // 2. Adjust to whole block. + const VkDeviceSize allocationOffset = allocation->GetOffset(); + VMA_ASSERT(allocationOffset % nonCoherentAtomSize == 0); + const VkDeviceSize blockSize = allocation->GetBlock()->m_pMetadata->GetSize(); + outRange.offset += allocationOffset; + outRange.size = VMA_MIN(outRange.size, blockSize - outRange.offset); + + break; + } + default: + VMA_ASSERT(0); + } + return true; + } + return false; +} + #if VMA_MEMORY_BUDGET void VmaAllocator_T::UpdateVulkanBudget() @@ -16222,7 +18030,7 @@ void VmaAllocator_T::UpdateVulkanBudget() VkPhysicalDeviceMemoryProperties2KHR memProps = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_PROPERTIES_2_KHR }; VkPhysicalDeviceMemoryBudgetPropertiesEXT budgetProps = { VK_STRUCTURE_TYPE_PHYSICAL_DEVICE_MEMORY_BUDGET_PROPERTIES_EXT }; - memProps.pNext = &budgetProps; + VmaPnextChainPushFront(&memProps, &budgetProps); GetVulkanFunctions().vkGetPhysicalDeviceMemoryProperties2KHR(m_PhysicalDevice, &memProps); @@ -16234,6 +18042,20 @@ void VmaAllocator_T::UpdateVulkanBudget() m_Budget.m_VulkanUsage[heapIndex] = budgetProps.heapUsage[heapIndex]; m_Budget.m_VulkanBudget[heapIndex] = budgetProps.heapBudget[heapIndex]; m_Budget.m_BlockBytesAtBudgetFetch[heapIndex] = m_Budget.m_BlockBytes[heapIndex].load(); + + // Some bugged drivers return the budget incorrectly, e.g. 0 or much bigger than heap size. + if(m_Budget.m_VulkanBudget[heapIndex] == 0) + { + m_Budget.m_VulkanBudget[heapIndex] = m_MemProps.memoryHeaps[heapIndex].size * 8 / 10; // 80% heuristics. + } + else if(m_Budget.m_VulkanBudget[heapIndex] > m_MemProps.memoryHeaps[heapIndex].size) + { + m_Budget.m_VulkanBudget[heapIndex] = m_MemProps.memoryHeaps[heapIndex].size; + } + if(m_Budget.m_VulkanUsage[heapIndex] == 0 && m_Budget.m_BlockBytesAtBudgetFetch[heapIndex] > 0) + { + m_Budget.m_VulkanUsage[heapIndex] = m_Budget.m_BlockBytesAtBudgetFetch[heapIndex]; + } } m_Budget.m_OperationsSinceBudgetFetch = 0; } @@ -16281,9 +18103,8 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) { VmaMutexLockRead dedicatedAllocationsLock(m_DedicatedAllocationsMutex[memTypeIndex], m_UseMutex); - AllocationVectorType* const pDedicatedAllocVector = m_pDedicatedAllocations[memTypeIndex]; - VMA_ASSERT(pDedicatedAllocVector); - if(pDedicatedAllocVector->empty() == false) + DedicatedAllocationLinkedList& dedicatedAllocList = m_DedicatedAllocations[memTypeIndex]; + if(!dedicatedAllocList.IsEmpty()) { if(dedicatedAllocationsStarted == false) { @@ -16295,14 +18116,14 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) json.BeginString("Type "); json.ContinueString(memTypeIndex); json.EndString(); - + json.BeginArray(); - for(size_t i = 0; i < pDedicatedAllocVector->size(); ++i) + for(VmaAllocation alloc = dedicatedAllocList.Front(); + alloc != VMA_NULL; alloc = dedicatedAllocList.GetNext(alloc)) { json.BeginObject(true); - const VmaAllocation hAlloc = (*pDedicatedAllocVector)[i]; - hAlloc->PrintParameters(json); + alloc->PrintParameters(json); json.EndObject(); } @@ -16314,6 +18135,7 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) json.EndObject(); } + // Default pools { bool allocationsStarted = false; for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) @@ -16340,21 +18162,47 @@ void VmaAllocator_T::PrintDetailedMap(VmaJsonWriter& json) } } + // Small buffer pools + { + bool allocationsStarted = false; + for(uint32_t memTypeIndex = 0; memTypeIndex < GetMemoryTypeCount(); ++memTypeIndex) + { + if(m_pSmallBufferBlockVectors[memTypeIndex]->IsEmpty() == false) + { + if(allocationsStarted == false) + { + allocationsStarted = true; + json.WriteString("SmallBufferPools"); + json.BeginObject(); + } + + json.BeginString("Type "); + json.ContinueString(memTypeIndex); + json.EndString(); + + m_pSmallBufferBlockVectors[memTypeIndex]->PrintDetailedMap(json); + } + } + if(allocationsStarted) + { + json.EndObject(); + } + } + // Custom pools { VmaMutexLockRead lock(m_PoolsMutex, m_UseMutex); - const size_t poolCount = m_Pools.size(); - if(poolCount > 0) + if(!m_Pools.IsEmpty()) { json.WriteString("Pools"); json.BeginObject(); - for(size_t poolIndex = 0; poolIndex < poolCount; ++poolIndex) + for(VmaPool pool = m_Pools.Front(); pool != VMA_NULL; pool = m_Pools.GetNext(pool)) { json.BeginString(); - json.ContinueString(m_Pools[poolIndex]->GetId()); + json.ContinueString(pool->GetId()); json.EndString(); - m_Pools[poolIndex]->m_BlockVector.PrintDetailedMap(json); + pool->m_BlockVector.PrintDetailedMap(json); } json.EndObject(); } @@ -16372,7 +18220,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateAllocator( { VMA_ASSERT(pCreateInfo && pAllocator); VMA_ASSERT(pCreateInfo->vulkanApiVersion == 0 || - (VK_VERSION_MAJOR(pCreateInfo->vulkanApiVersion) == 1 && VK_VERSION_MINOR(pCreateInfo->vulkanApiVersion) <= 1)); + (VK_VERSION_MAJOR(pCreateInfo->vulkanApiVersion) == 1 && VK_VERSION_MINOR(pCreateInfo->vulkanApiVersion) <= 2)); VMA_DEBUG_LOG("vmaCreateAllocator"); *pAllocator = vma_new(pCreateInfo->pAllocationCallbacks, VmaAllocator_T)(pCreateInfo); return (*pAllocator)->Init(pCreateInfo); @@ -16389,6 +18237,14 @@ VMA_CALL_PRE void VMA_CALL_POST vmaDestroyAllocator( } } +VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocatorInfo(VmaAllocator allocator, VmaAllocatorInfo* pAllocatorInfo) +{ + VMA_ASSERT(allocator && pAllocatorInfo); + pAllocatorInfo->instance = allocator->m_hInstance; + pAllocatorInfo->physicalDevice = allocator->GetPhysicalDevice(); + pAllocatorInfo->device = allocator->m_hDevice; +} + VMA_CALL_PRE void VMA_CALL_POST vmaGetPhysicalDeviceProperties( VmaAllocator allocator, const VkPhysicalDeviceProperties **ppPhysicalDeviceProperties) @@ -16468,7 +18324,7 @@ VMA_CALL_PRE void VMA_CALL_POST vmaBuildStatsString( json.WriteString("Total"); VmaPrintStatInfo(json, stats.total); - + for(uint32_t heapIndex = 0; heapIndex < allocator->GetMemoryHeapCount(); ++heapIndex) { json.BeginString("Heap "); @@ -16540,6 +18396,22 @@ VMA_CALL_PRE void VMA_CALL_POST vmaBuildStatsString( { json.WriteString("LAZILY_ALLOCATED"); } +#if VMA_VULKAN_VERSION >= 1001000 + if((flags & VK_MEMORY_PROPERTY_PROTECTED_BIT) != 0) + { + json.WriteString("PROTECTED"); + } +#endif // #if VMA_VULKAN_VERSION >= 1001000 +#if VK_AMD_device_coherent_memory + if((flags & VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY) != 0) + { + json.WriteString("DEVICE_COHERENT"); + } + if((flags & VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY) != 0) + { + json.WriteString("DEVICE_UNCACHED"); + } +#endif // #if VK_AMD_device_coherent_memory json.EndArray(); if(stats.memoryType[typeIndex].blockCount > 0) @@ -16599,11 +18471,13 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndex( VMA_ASSERT(pAllocationCreateInfo != VMA_NULL); VMA_ASSERT(pMemoryTypeIndex != VMA_NULL); + memoryTypeBits &= allocator->GetGlobalMemoryTypeBits(); + if(pAllocationCreateInfo->memoryTypeBits != 0) { memoryTypeBits &= pAllocationCreateInfo->memoryTypeBits; } - + uint32_t requiredFlags = pAllocationCreateInfo->requiredFlags; uint32_t preferredFlags = pAllocationCreateInfo->preferredFlags; uint32_t notPreferredFlags = 0; @@ -16644,6 +18518,13 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndex( break; } + // Avoid DEVICE_COHERENT unless explicitly requested. + if(((pAllocationCreateInfo->requiredFlags | pAllocationCreateInfo->preferredFlags) & + (VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY | VK_MEMORY_PROPERTY_DEVICE_UNCACHED_BIT_AMD_COPY)) == 0) + { + notPreferredFlags |= VK_MEMORY_PROPERTY_DEVICE_COHERENT_BIT_AMD_COPY; + } + *pMemoryTypeIndex = UINT32_MAX; uint32_t minCost = UINT32_MAX; for(uint32_t memTypeIndex = 0, memTypeBit = 1; @@ -16744,25 +18625,25 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaFindMemoryTypeIndexForImageInfo( } VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreatePool( - VmaAllocator allocator, - const VmaPoolCreateInfo* pCreateInfo, - VmaPool* pPool) + VmaAllocator allocator, + const VmaPoolCreateInfo* pCreateInfo, + VmaPool* pPool) { VMA_ASSERT(allocator && pCreateInfo && pPool); - + VMA_DEBUG_LOG("vmaCreatePool"); - + VMA_DEBUG_GLOBAL_MUTEX_LOCK - + VkResult res = allocator->CreatePool(pCreateInfo, pPool); - + #if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { allocator->GetRecorder()->RecordCreatePool(allocator->GetCurrentFrameIndex(), *pCreateInfo, *pPool); } #endif - + return res; } @@ -16771,16 +18652,16 @@ VMA_CALL_PRE void VMA_CALL_POST vmaDestroyPool( VmaPool pool) { VMA_ASSERT(allocator); - + if(pool == VK_NULL_HANDLE) { return; } - + VMA_DEBUG_LOG("vmaDestroyPool"); - + VMA_DEBUG_GLOBAL_MUTEX_LOCK - + #if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) { @@ -16838,8 +18719,8 @@ VMA_CALL_PRE void VMA_CALL_POST vmaGetPoolName( VmaPool pool, const char** ppName) { - VMA_ASSERT(allocator && pool); - + VMA_ASSERT(allocator && pool && ppName); + VMA_DEBUG_LOG("vmaGetPoolName"); VMA_DEBUG_GLOBAL_MUTEX_LOCK @@ -16881,11 +18762,12 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemory( VMA_DEBUG_GLOBAL_MUTEX_LOCK - VkResult result = allocator->AllocateMemory( + VkResult result = allocator->AllocateMemory( *pVkMemoryRequirements, false, // requiresDedicatedAllocation false, // prefersDedicatedAllocation VK_NULL_HANDLE, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage VK_NULL_HANDLE, // dedicatedImage *pCreateInfo, VMA_SUBALLOCATION_TYPE_UNKNOWN, @@ -16902,13 +18784,13 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemory( *pAllocation); } #endif - + if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS) { allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } - return result; + return result; } VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages( @@ -16930,11 +18812,12 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages( VMA_DEBUG_GLOBAL_MUTEX_LOCK - VkResult result = allocator->AllocateMemory( + VkResult result = allocator->AllocateMemory( *pVkMemoryRequirements, false, // requiresDedicatedAllocation false, // prefersDedicatedAllocation VK_NULL_HANDLE, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage VK_NULL_HANDLE, // dedicatedImage *pCreateInfo, VMA_SUBALLOCATION_TYPE_UNKNOWN, @@ -16952,7 +18835,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages( pAllocations); } #endif - + if(pAllocationInfo != VMA_NULL && result == VK_SUCCESS) { for(size_t i = 0; i < allocationCount; ++i) @@ -16961,7 +18844,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryPages( } } - return result; + return result; } VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForBuffer( @@ -16989,6 +18872,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForBuffer( requiresDedicatedAllocation, prefersDedicatedAllocation, buffer, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage VK_NULL_HANDLE, // dedicatedImage *pCreateInfo, VMA_SUBALLOCATION_TYPE_BUFFER, @@ -17013,7 +18897,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForBuffer( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } - return result; + return result; } VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForImage( @@ -17040,6 +18924,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForImage( requiresDedicatedAllocation, prefersDedicatedAllocation, VK_NULL_HANDLE, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage image, // dedicatedImage *pCreateInfo, VMA_SUBALLOCATION_TYPE_IMAGE_UNKNOWN, @@ -17064,7 +18949,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaAllocateMemoryForImage( allocator->GetAllocationInfo(*pAllocation, pAllocationInfo); } - return result; + return result; } VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory( @@ -17072,14 +18957,14 @@ VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory( VmaAllocation allocation) { VMA_ASSERT(allocator); - + if(allocation == VK_NULL_HANDLE) { return; } - + VMA_DEBUG_LOG("vmaFreeMemory"); - + VMA_DEBUG_GLOBAL_MUTEX_LOCK #if VMA_RECORDING_ENABLED @@ -17090,7 +18975,7 @@ VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory( allocation); } #endif - + allocator->FreeMemory( 1, // allocationCount &allocation); @@ -17099,7 +18984,7 @@ VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemory( VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages( VmaAllocator allocator, size_t allocationCount, - VmaAllocation* pAllocations) + const VmaAllocation* pAllocations) { if(allocationCount == 0) { @@ -17107,9 +18992,9 @@ VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages( } VMA_ASSERT(allocator); - + VMA_DEBUG_LOG("vmaFreeMemoryPages"); - + VMA_DEBUG_GLOBAL_MUTEX_LOCK #if VMA_RECORDING_ENABLED @@ -17121,22 +19006,8 @@ VMA_CALL_PRE void VMA_CALL_POST vmaFreeMemoryPages( pAllocations); } #endif - - allocator->FreeMemory(allocationCount, pAllocations); -} -VMA_CALL_PRE VkResult VMA_CALL_POST vmaResizeAllocation( - VmaAllocator allocator, - VmaAllocation allocation, - VkDeviceSize newSize) -{ - VMA_ASSERT(allocator && allocation); - - VMA_DEBUG_LOG("vmaResizeAllocation"); - - VMA_DEBUG_GLOBAL_MUTEX_LOCK - - return allocator->ResizeAllocation(allocation, newSize); + allocator->FreeMemory(allocationCount, pAllocations); } VMA_CALL_PRE void VMA_CALL_POST vmaGetAllocationInfo( @@ -17265,7 +19136,7 @@ VMA_CALL_PRE void VMA_CALL_POST vmaUnmapMemory( allocator->Unmap(allocation); } -VMA_CALL_PRE void VMA_CALL_POST vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) { VMA_ASSERT(allocator && allocation); @@ -17273,7 +19144,7 @@ VMA_CALL_PRE void VMA_CALL_POST vmaFlushAllocation(VmaAllocator allocator, VmaAl VMA_DEBUG_GLOBAL_MUTEX_LOCK - allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH); + const VkResult res = allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_FLUSH); #if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) @@ -17283,9 +19154,11 @@ VMA_CALL_PRE void VMA_CALL_POST vmaFlushAllocation(VmaAllocator allocator, VmaAl allocation, offset, size); } #endif + + return res; } -VMA_CALL_PRE void VMA_CALL_POST vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) +VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocation(VmaAllocator allocator, VmaAllocation allocation, VkDeviceSize offset, VkDeviceSize size) { VMA_ASSERT(allocator && allocation); @@ -17293,7 +19166,7 @@ VMA_CALL_PRE void VMA_CALL_POST vmaInvalidateAllocation(VmaAllocator allocator, VMA_DEBUG_GLOBAL_MUTEX_LOCK - allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE); + const VkResult res = allocator->FlushOrInvalidateAllocation(allocation, offset, size, VMA_CACHE_INVALIDATE); #if VMA_RECORDING_ENABLED if(allocator->GetRecorder() != VMA_NULL) @@ -17303,6 +19176,72 @@ VMA_CALL_PRE void VMA_CALL_POST vmaInvalidateAllocation(VmaAllocator allocator, allocation, offset, size); } #endif + + return res; +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaFlushAllocations( + VmaAllocator allocator, + uint32_t allocationCount, + const VmaAllocation* allocations, + const VkDeviceSize* offsets, + const VkDeviceSize* sizes) +{ + VMA_ASSERT(allocator); + + if(allocationCount == 0) + { + return VK_SUCCESS; + } + + VMA_ASSERT(allocations); + + VMA_DEBUG_LOG("vmaFlushAllocations"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + const VkResult res = allocator->FlushOrInvalidateAllocations(allocationCount, allocations, offsets, sizes, VMA_CACHE_FLUSH); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + //TODO + } +#endif + + return res; +} + +VMA_CALL_PRE VkResult VMA_CALL_POST vmaInvalidateAllocations( + VmaAllocator allocator, + uint32_t allocationCount, + const VmaAllocation* allocations, + const VkDeviceSize* offsets, + const VkDeviceSize* sizes) +{ + VMA_ASSERT(allocator); + + if(allocationCount == 0) + { + return VK_SUCCESS; + } + + VMA_ASSERT(allocations); + + VMA_DEBUG_LOG("vmaInvalidateAllocations"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + const VkResult res = allocator->FlushOrInvalidateAllocations(allocationCount, allocations, offsets, sizes, VMA_CACHE_INVALIDATE); + +#if VMA_RECORDING_ENABLED + if(allocator->GetRecorder() != VMA_NULL) + { + //TODO + } +#endif + + return res; } VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckCorruption(VmaAllocator allocator, uint32_t memoryTypeBits) @@ -17318,7 +19257,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCheckCorruption(VmaAllocator allocator, u VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragment( VmaAllocator allocator, - VmaAllocation* pAllocations, + const VmaAllocation* pAllocations, size_t allocationCount, VkBool32* pAllocationsChanged, const VmaDefragmentationInfo *pDefragmentationInfo, @@ -17415,6 +19354,42 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaDefragmentationEnd( } } +VMA_CALL_PRE VkResult VMA_CALL_POST vmaBeginDefragmentationPass( + VmaAllocator allocator, + VmaDefragmentationContext context, + VmaDefragmentationPassInfo* pInfo + ) +{ + VMA_ASSERT(allocator); + VMA_ASSERT(pInfo); + + VMA_DEBUG_LOG("vmaBeginDefragmentationPass"); + + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + if(context == VK_NULL_HANDLE) + { + pInfo->moveCount = 0; + return VK_SUCCESS; + } + + return allocator->DefragmentationPassBegin(pInfo, context); +} +VMA_CALL_PRE VkResult VMA_CALL_POST vmaEndDefragmentationPass( + VmaAllocator allocator, + VmaDefragmentationContext context) +{ + VMA_ASSERT(allocator); + + VMA_DEBUG_LOG("vmaEndDefragmentationPass"); + VMA_DEBUG_GLOBAL_MUTEX_LOCK + + if(context == VK_NULL_HANDLE) + return VK_SUCCESS; + + return allocator->DefragmentationPassEnd(context); +} + VMA_CALL_PRE VkResult VMA_CALL_POST vmaBindBufferMemory( VmaAllocator allocator, VmaAllocation allocation, @@ -17489,9 +19464,15 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateBuffer( { return VK_ERROR_VALIDATION_FAILED_EXT; } - + if((pBufferCreateInfo->usage & VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT_COPY) != 0 && + !allocator->m_UseKhrBufferDeviceAddress) + { + VMA_ASSERT(0 && "Creating a buffer with VK_BUFFER_USAGE_SHADER_DEVICE_ADDRESS_BIT is not valid if VMA_ALLOCATOR_CREATE_BUFFER_DEVICE_ADDRESS_BIT was not used."); + return VK_ERROR_VALIDATION_FAILED_EXT; + } + VMA_DEBUG_LOG("vmaCreateBuffer"); - + VMA_DEBUG_GLOBAL_MUTEX_LOCK *pBuffer = VK_NULL_HANDLE; @@ -17518,6 +19499,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateBuffer( requiresDedicatedAllocation, prefersDedicatedAllocation, *pBuffer, // dedicatedBuffer + pBufferCreateInfo->usage, // dedicatedBufferUsage VK_NULL_HANDLE, // dedicatedImage *pAllocationCreateInfo, VMA_SUBALLOCATION_TYPE_BUFFER, @@ -17645,7 +19627,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateImage( VmaSuballocationType suballocType = pImageCreateInfo->tiling == VK_IMAGE_TILING_OPTIMAL ? VMA_SUBALLOCATION_TYPE_IMAGE_OPTIMAL : VMA_SUBALLOCATION_TYPE_IMAGE_LINEAR; - + // 2. Allocate memory using allocator. VkMemoryRequirements vkMemReq = {}; bool requiresDedicatedAllocation = false; @@ -17658,6 +19640,7 @@ VMA_CALL_PRE VkResult VMA_CALL_POST vmaCreateImage( requiresDedicatedAllocation, prefersDedicatedAllocation, VK_NULL_HANDLE, // dedicatedBuffer + UINT32_MAX, // dedicatedBufferUsage *pImage, // dedicatedImage *pAllocationCreateInfo, suballocType, |