diff options
| author | reduz <juan@okamstudio.com> | 2015-10-08 15:00:40 -0300 |
|---|---|---|
| committer | reduz <juan@okamstudio.com> | 2015-10-08 15:00:40 -0300 |
| commit | aad2bbdb6fb7c8217d7e75480b38e45f00cb3abd (patch) | |
| tree | 64244b44491a06c518571857e37c6bb98e4e6173 /drivers | |
| parent | 3514a87b371385cdc71a2ed36deb40cd0275fff5 (diff) | |
newline fixes
Diffstat (limited to 'drivers')
| -rw-r--r-- | drivers/etc1/rg_etc1.cpp | 4908 | ||||
| -rw-r--r-- | drivers/etc1/rg_etc1.h | 152 | ||||
| -rw-r--r-- | drivers/nedmalloc/malloc.c.h | 11628 | ||||
| -rw-r--r-- | drivers/nedmalloc/nedmalloc.cpp | 2934 | ||||
| -rw-r--r-- | drivers/nedmalloc/nedmalloc.h | 604 | ||||
| -rw-r--r-- | drivers/openssl/register_openssl.cpp | 38 | ||||
| -rw-r--r-- | drivers/openssl/register_openssl.h | 22 | ||||
| -rw-r--r-- | drivers/rtaudio/RtAudio.cpp | 20468 | ||||
| -rw-r--r-- | drivers/speex/config.h | 104 | ||||
| -rw-r--r-- | drivers/speex/lsp.h | 128 | ||||
| -rw-r--r-- | drivers/speex/speex_bind.cpp | 128 | ||||
| -rw-r--r-- | drivers/speex/speex_bind.h | 96 |
12 files changed, 20605 insertions, 20605 deletions
diff --git a/drivers/etc1/rg_etc1.cpp b/drivers/etc1/rg_etc1.cpp index fd109f003c..47dcb57e6b 100644 --- a/drivers/etc1/rg_etc1.cpp +++ b/drivers/etc1/rg_etc1.cpp @@ -1,2454 +1,2454 @@ -// File: rg_etc1.cpp - Fast, high quality ETC1 block packer/unpacker - Rich Geldreich <richgel99@gmail.com>
-// Please see ZLIB license at the end of rg_etc1.h.
-//
-// For more information Ericsson Texture Compression (ETC/ETC1), see:
-// http://www.khronos.org/registry/gles/extensions/OES/OES_compressed_ETC1_RGB8_texture.txt
-//
-// v1.03 - 5/12/13 - Initial public release
-#include "rg_etc1.h"
-
-#include <stdlib.h>
-#include <string.h>
-#include <assert.h>
-//#include <stdio.h>
-#include <math.h>
-#include <stdio.h>
-#pragma warning (disable: 4201) // nonstandard extension used : nameless struct/union
-
-#if defined(_DEBUG) || defined(DEBUG)
-#define RG_ETC1_BUILD_DEBUG
-#endif
-
-#define RG_ETC1_ASSERT assert
-
-namespace rg_etc1
-{
-
- inline long labs(long val) {
- return val < 0 ? -val : val;
- }
-
- inline int intabs(int val) {
-
- return val<0?-val:val;
- }
-
- typedef unsigned char uint8;
- typedef unsigned short uint16;
- typedef unsigned int uint;
- typedef unsigned int uint32;
- typedef long long int64;
- typedef unsigned long long uint64;
-
- const uint32 cUINT32_MAX = 0xFFFFFFFFU;
- const uint64 cUINT64_MAX = 0xFFFFFFFFFFFFFFFFULL; //0xFFFFFFFFFFFFFFFFui64;
-
- template<typename T> inline T minimum(T a, T b) { return (a < b) ? a : b; }
- template<typename T> inline T minimum(T a, T b, T c) { return minimum(minimum(a, b), c); }
- template<typename T> inline T maximum(T a, T b) { return (a > b) ? a : b; }
- template<typename T> inline T maximum(T a, T b, T c) { return maximum(maximum(a, b), c); }
- template<typename T> inline T clamp(T value, T low, T high) { return (value < low) ? low : ((value > high) ? high : value); }
- template<typename T> inline T square(T value) { return value * value; }
- template<typename T> inline void zero_object(T& obj) { memset((void*)&obj, 0, sizeof(obj)); }
- template<typename T> inline void zero_this(T* pObj) { memset((void*)pObj, 0, sizeof(*pObj)); }
-
- template<class T, size_t N> T decay_array_to_subtype(T (&a)[N]);
-
-#define RG_ETC1_ARRAY_SIZE(X) (sizeof(X) / sizeof(decay_array_to_subtype(X)))
-
- enum eNoClamp { cNoClamp };
-
- struct color_quad_u8
- {
- static inline int clamp(int v) { if (v & 0xFFFFFF00U) v = (~(static_cast<int>(v) >> 31)) & 0xFF; return v; }
-
- struct component_traits { enum { cSigned = false, cFloat = false, cMin = 0U, cMax = 255U }; };
-
- public:
- typedef unsigned char component_t;
- typedef int parameter_t;
-
- enum { cNumComps = 4 };
-
- union
- {
- struct
- {
- component_t r;
- component_t g;
- component_t b;
- component_t a;
- };
-
- component_t c[cNumComps];
-
- uint32 m_u32;
- };
-
- inline color_quad_u8()
- {
- }
-
- inline color_quad_u8(const color_quad_u8& other) : m_u32(other.m_u32)
- {
- }
-
- explicit inline color_quad_u8(parameter_t y, parameter_t alpha = component_traits::cMax)
- {
- set(y, alpha);
- }
-
- inline color_quad_u8(parameter_t red, parameter_t green, parameter_t blue, parameter_t alpha = component_traits::cMax)
- {
- set(red, green, blue, alpha);
- }
-
- explicit inline color_quad_u8(eNoClamp, parameter_t y, parameter_t alpha = component_traits::cMax)
- {
- set_noclamp_y_alpha(y, alpha);
- }
-
- inline color_quad_u8(eNoClamp, parameter_t red, parameter_t green, parameter_t blue, parameter_t alpha = component_traits::cMax)
- {
- set_noclamp_rgba(red, green, blue, alpha);
- }
-
- inline void clear()
- {
- m_u32 = 0;
- }
-
- inline color_quad_u8& operator= (const color_quad_u8& other)
- {
- m_u32 = other.m_u32;
- return *this;
- }
-
- inline color_quad_u8& set_rgb(const color_quad_u8& other)
- {
- r = other.r;
- g = other.g;
- b = other.b;
- return *this;
- }
-
- inline color_quad_u8& operator= (parameter_t y)
- {
- set(y, component_traits::cMax);
- return *this;
- }
-
- inline color_quad_u8& set(parameter_t y, parameter_t alpha = component_traits::cMax)
- {
- y = clamp(y);
- alpha = clamp(alpha);
- r = static_cast<component_t>(y);
- g = static_cast<component_t>(y);
- b = static_cast<component_t>(y);
- a = static_cast<component_t>(alpha);
- return *this;
- }
-
- inline color_quad_u8& set_noclamp_y_alpha(parameter_t y, parameter_t alpha = component_traits::cMax)
- {
- RG_ETC1_ASSERT( (y >= component_traits::cMin) && (y <= component_traits::cMax) );
- RG_ETC1_ASSERT( (alpha >= component_traits::cMin) && (alpha <= component_traits::cMax) );
-
- r = static_cast<component_t>(y);
- g = static_cast<component_t>(y);
- b = static_cast<component_t>(y);
- a = static_cast<component_t>(alpha);
- return *this;
- }
-
- inline color_quad_u8& set(parameter_t red, parameter_t green, parameter_t blue, parameter_t alpha = component_traits::cMax)
- {
- r = static_cast<component_t>(clamp(red));
- g = static_cast<component_t>(clamp(green));
- b = static_cast<component_t>(clamp(blue));
- a = static_cast<component_t>(clamp(alpha));
- return *this;
- }
-
- inline color_quad_u8& set_noclamp_rgba(parameter_t red, parameter_t green, parameter_t blue, parameter_t alpha)
- {
- RG_ETC1_ASSERT( (red >= component_traits::cMin) && (red <= component_traits::cMax) );
- RG_ETC1_ASSERT( (green >= component_traits::cMin) && (green <= component_traits::cMax) );
- RG_ETC1_ASSERT( (blue >= component_traits::cMin) && (blue <= component_traits::cMax) );
- RG_ETC1_ASSERT( (alpha >= component_traits::cMin) && (alpha <= component_traits::cMax) );
-
- r = static_cast<component_t>(red);
- g = static_cast<component_t>(green);
- b = static_cast<component_t>(blue);
- a = static_cast<component_t>(alpha);
- return *this;
- }
-
- inline color_quad_u8& set_noclamp_rgb(parameter_t red, parameter_t green, parameter_t blue)
- {
- RG_ETC1_ASSERT( (red >= component_traits::cMin) && (red <= component_traits::cMax) );
- RG_ETC1_ASSERT( (green >= component_traits::cMin) && (green <= component_traits::cMax) );
- RG_ETC1_ASSERT( (blue >= component_traits::cMin) && (blue <= component_traits::cMax) );
-
- r = static_cast<component_t>(red);
- g = static_cast<component_t>(green);
- b = static_cast<component_t>(blue);
- return *this;
- }
-
- static inline parameter_t get_min_comp() { return component_traits::cMin; }
- static inline parameter_t get_max_comp() { return component_traits::cMax; }
- static inline bool get_comps_are_signed() { return component_traits::cSigned; }
-
- inline component_t operator[] (uint i) const { RG_ETC1_ASSERT(i < cNumComps); return c[i]; }
- inline component_t& operator[] (uint i) { RG_ETC1_ASSERT(i < cNumComps); return c[i]; }
-
- inline color_quad_u8& set_component(uint i, parameter_t f)
- {
- RG_ETC1_ASSERT(i < cNumComps);
-
- c[i] = static_cast<component_t>(clamp(f));
-
- return *this;
- }
-
- inline color_quad_u8& set_grayscale(parameter_t l)
- {
- component_t x = static_cast<component_t>(clamp(l));
- c[0] = x;
- c[1] = x;
- c[2] = x;
- return *this;
- }
-
- inline color_quad_u8& clamp(const color_quad_u8& l, const color_quad_u8& h)
- {
- for (uint i = 0; i < cNumComps; i++)
- c[i] = static_cast<component_t>(rg_etc1::clamp<parameter_t>(c[i], l[i], h[i]));
- return *this;
- }
-
- inline color_quad_u8& clamp(parameter_t l, parameter_t h)
- {
- for (uint i = 0; i < cNumComps; i++)
- c[i] = static_cast<component_t>(rg_etc1::clamp<parameter_t>(c[i], l, h));
- return *this;
- }
-
- // Returns CCIR 601 luma (consistent with color_utils::RGB_To_Y).
- inline parameter_t get_luma() const
- {
- return static_cast<parameter_t>((19595U * r + 38470U * g + 7471U * b + 32768U) >> 16U);
- }
-
- // Returns REC 709 luma.
- inline parameter_t get_luma_rec709() const
- {
- return static_cast<parameter_t>((13938U * r + 46869U * g + 4729U * b + 32768U) >> 16U);
- }
-
- inline uint squared_distance_rgb(const color_quad_u8& c) const
- {
- return rg_etc1::square(r - c.r) + rg_etc1::square(g - c.g) + rg_etc1::square(b - c.b);
- }
-
- inline uint squared_distance_rgba(const color_quad_u8& c) const
- {
- return rg_etc1::square(r - c.r) + rg_etc1::square(g - c.g) + rg_etc1::square(b - c.b) + rg_etc1::square(a - c.a);
- }
-
- inline bool rgb_equals(const color_quad_u8& rhs) const
- {
- return (r == rhs.r) && (g == rhs.g) && (b == rhs.b);
- }
-
- inline bool operator== (const color_quad_u8& rhs) const
- {
- return m_u32 == rhs.m_u32;
- }
-
- color_quad_u8& operator+= (const color_quad_u8& other)
- {
- for (uint i = 0; i < 4; i++)
- c[i] = static_cast<component_t>(clamp(c[i] + other.c[i]));
- return *this;
- }
-
- color_quad_u8& operator-= (const color_quad_u8& other)
- {
- for (uint i = 0; i < 4; i++)
- c[i] = static_cast<component_t>(clamp(c[i] - other.c[i]));
- return *this;
- }
-
- friend color_quad_u8 operator+ (const color_quad_u8& lhs, const color_quad_u8& rhs)
- {
- color_quad_u8 result(lhs);
- result += rhs;
- return result;
- }
-
- friend color_quad_u8 operator- (const color_quad_u8& lhs, const color_quad_u8& rhs)
- {
- color_quad_u8 result(lhs);
- result -= rhs;
- return result;
- }
- }; // class color_quad_u8
-
- struct vec3F
- {
- float m_s[3];
-
- inline vec3F() { }
- inline vec3F(float s) { m_s[0] = s; m_s[1] = s; m_s[2] = s; }
- inline vec3F(float x, float y, float z) { m_s[0] = x; m_s[1] = y; m_s[2] = z; }
-
- inline float operator[] (uint i) const { RG_ETC1_ASSERT(i < 3); return m_s[i]; }
-
- inline vec3F& operator += (const vec3F& other) { for (uint i = 0; i < 3; i++) m_s[i] += other.m_s[i]; return *this; }
-
- inline vec3F& operator *= (float s) { for (uint i = 0; i < 3; i++) m_s[i] *= s; return *this; }
- };
-
- enum etc_constants
- {
- cETC1BytesPerBlock = 8U,
-
- cETC1SelectorBits = 2U,
- cETC1SelectorValues = 1U << cETC1SelectorBits,
- cETC1SelectorMask = cETC1SelectorValues - 1U,
-
- cETC1BlockShift = 2U,
- cETC1BlockSize = 1U << cETC1BlockShift,
-
- cETC1LSBSelectorIndicesBitOffset = 0,
- cETC1MSBSelectorIndicesBitOffset = 16,
-
- cETC1FlipBitOffset = 32,
- cETC1DiffBitOffset = 33,
-
- cETC1IntenModifierNumBits = 3,
- cETC1IntenModifierValues = 1 << cETC1IntenModifierNumBits,
- cETC1RightIntenModifierTableBitOffset = 34,
- cETC1LeftIntenModifierTableBitOffset = 37,
-
- // Base+Delta encoding (5 bit bases, 3 bit delta)
- cETC1BaseColorCompNumBits = 5,
- cETC1BaseColorCompMax = 1 << cETC1BaseColorCompNumBits,
-
- cETC1DeltaColorCompNumBits = 3,
- cETC1DeltaColorComp = 1 << cETC1DeltaColorCompNumBits,
- cETC1DeltaColorCompMax = 1 << cETC1DeltaColorCompNumBits,
-
- cETC1BaseColor5RBitOffset = 59,
- cETC1BaseColor5GBitOffset = 51,
- cETC1BaseColor5BBitOffset = 43,
-
- cETC1DeltaColor3RBitOffset = 56,
- cETC1DeltaColor3GBitOffset = 48,
- cETC1DeltaColor3BBitOffset = 40,
-
- // Absolute (non-delta) encoding (two 4-bit per component bases)
- cETC1AbsColorCompNumBits = 4,
- cETC1AbsColorCompMax = 1 << cETC1AbsColorCompNumBits,
-
- cETC1AbsColor4R1BitOffset = 60,
- cETC1AbsColor4G1BitOffset = 52,
- cETC1AbsColor4B1BitOffset = 44,
-
- cETC1AbsColor4R2BitOffset = 56,
- cETC1AbsColor4G2BitOffset = 48,
- cETC1AbsColor4B2BitOffset = 40,
-
- cETC1ColorDeltaMin = -4,
- cETC1ColorDeltaMax = 3,
-
- // Delta3:
- // 0 1 2 3 4 5 6 7
- // 000 001 010 011 100 101 110 111
- // 0 1 2 3 -4 -3 -2 -1
- };
-
- static uint8 g_quant5_tab[256+16];
-
-
- static const int g_etc1_inten_tables[cETC1IntenModifierValues][cETC1SelectorValues] =
- {
- { -8, -2, 2, 8 }, { -17, -5, 5, 17 }, { -29, -9, 9, 29 }, { -42, -13, 13, 42 },
- { -60, -18, 18, 60 }, { -80, -24, 24, 80 }, { -106, -33, 33, 106 }, { -183, -47, 47, 183 }
- };
-
- static const uint8 g_etc1_to_selector_index[cETC1SelectorValues] = { 2, 3, 1, 0 };
- static const uint8 g_selector_index_to_etc1[cETC1SelectorValues] = { 3, 2, 0, 1 };
-
- // Given an ETC1 diff/inten_table/selector, and an 8-bit desired color, this table encodes the best packed_color in the low byte, and the abs error in the high byte.
- static uint16 g_etc1_inverse_lookup[2*8*4][256]; // [diff/inten_table/selector][desired_color]
-
- // g_color8_to_etc_block_config[color][table_index] = Supplies for each 8-bit color value a list of packed ETC1 diff/intensity table/selectors/packed_colors that map to that color.
- // To pack: diff | (inten << 1) | (selector << 4) | (packed_c << 8)
- static const uint16 g_color8_to_etc_block_config_0_255[2][33] =
- {
- { 0x0000, 0x0010, 0x0002, 0x0012, 0x0004, 0x0014, 0x0006, 0x0016, 0x0008, 0x0018, 0x000A, 0x001A, 0x000C, 0x001C, 0x000E, 0x001E,
- 0x0001, 0x0011, 0x0003, 0x0013, 0x0005, 0x0015, 0x0007, 0x0017, 0x0009, 0x0019, 0x000B, 0x001B, 0x000D, 0x001D, 0x000F, 0x001F, 0xFFFF },
- { 0x0F20, 0x0F30, 0x0E32, 0x0F22, 0x0E34, 0x0F24, 0x0D36, 0x0F26, 0x0C38, 0x0E28, 0x0B3A, 0x0E2A, 0x093C, 0x0E2C, 0x053E, 0x0D2E,
- 0x1E31, 0x1F21, 0x1D33, 0x1F23, 0x1C35, 0x1E25, 0x1A37, 0x1E27, 0x1839, 0x1D29, 0x163B, 0x1C2B, 0x133D, 0x1B2D, 0x093F, 0x1A2F, 0xFFFF },
- };
-
- // Really only [254][11].
- static const uint16 g_color8_to_etc_block_config_1_to_254[254][12] =
- {
- { 0x021C, 0x0D0D, 0xFFFF }, { 0x0020, 0x0021, 0x0A0B, 0x061F, 0xFFFF }, { 0x0113, 0x0217, 0xFFFF }, { 0x0116, 0x031E,
- 0x0B0E, 0x0405, 0xFFFF }, { 0x0022, 0x0204, 0x050A, 0x0023, 0xFFFF }, { 0x0111, 0x0319, 0x0809, 0x170F, 0xFFFF }, {
- 0x0303, 0x0215, 0x0607, 0xFFFF }, { 0x0030, 0x0114, 0x0408, 0x0031, 0x0201, 0x051D, 0xFFFF }, { 0x0100, 0x0024, 0x0306,
- 0x0025, 0x041B, 0x0E0D, 0xFFFF }, { 0x021A, 0x0121, 0x0B0B, 0x071F, 0xFFFF }, { 0x0213, 0x0317, 0xFFFF }, { 0x0112,
- 0x0505, 0xFFFF }, { 0x0026, 0x070C, 0x0123, 0x0027, 0xFFFF }, { 0x0211, 0x0909, 0xFFFF }, { 0x0110, 0x0315, 0x0707,
- 0x0419, 0x180F, 0xFFFF }, { 0x0218, 0x0131, 0x0301, 0x0403, 0x061D, 0xFFFF }, { 0x0032, 0x0202, 0x0033, 0x0125, 0x051B,
- 0x0F0D, 0xFFFF }, { 0x0028, 0x031C, 0x0221, 0x0029, 0xFFFF }, { 0x0120, 0x0313, 0x0C0B, 0x081F, 0xFFFF }, { 0x0605,
- 0x0417, 0xFFFF }, { 0x0216, 0x041E, 0x0C0E, 0x0223, 0x0127, 0xFFFF }, { 0x0122, 0x0304, 0x060A, 0x0311, 0x0A09, 0xFFFF
- }, { 0x0519, 0x190F, 0xFFFF }, { 0x002A, 0x0231, 0x0503, 0x0415, 0x0807, 0x002B, 0x071D, 0xFFFF }, { 0x0130, 0x0214,
- 0x0508, 0x0401, 0x0133, 0x0225, 0x061B, 0xFFFF }, { 0x0200, 0x0124, 0x0406, 0x0321, 0x0129, 0x100D, 0xFFFF }, { 0x031A,
- 0x0D0B, 0x091F, 0xFFFF }, { 0x0413, 0x0705, 0x0517, 0xFFFF }, { 0x0212, 0x0034, 0x0323, 0x0035, 0x0227, 0xFFFF }, {
- 0x0126, 0x080C, 0x0B09, 0xFFFF }, { 0x0411, 0x0619, 0x1A0F, 0xFFFF }, { 0x0210, 0x0331, 0x0603, 0x0515, 0x0907, 0x012B,
- 0xFFFF }, { 0x0318, 0x002C, 0x0501, 0x0233, 0x0325, 0x071B, 0x002D, 0x081D, 0xFFFF }, { 0x0132, 0x0302, 0x0229, 0x110D,
- 0xFFFF }, { 0x0128, 0x041C, 0x0421, 0x0E0B, 0x0A1F, 0xFFFF }, { 0x0220, 0x0513, 0x0617, 0xFFFF }, { 0x0135, 0x0805,
- 0x0327, 0xFFFF }, { 0x0316, 0x051E, 0x0D0E, 0x0423, 0xFFFF }, { 0x0222, 0x0404, 0x070A, 0x0511, 0x0719, 0x0C09, 0x1B0F,
- 0xFFFF }, { 0x0703, 0x0615, 0x0A07, 0x022B, 0xFFFF }, { 0x012A, 0x0431, 0x0601, 0x0333, 0x012D, 0x091D, 0xFFFF }, {
- 0x0230, 0x0314, 0x0036, 0x0608, 0x0425, 0x0037, 0x0329, 0x081B, 0x120D, 0xFFFF }, { 0x0300, 0x0224, 0x0506, 0x0521,
- 0x0F0B, 0x0B1F, 0xFFFF }, { 0x041A, 0x0613, 0x0717, 0xFFFF }, { 0x0235, 0x0905, 0xFFFF }, { 0x0312, 0x0134, 0x0523,
- 0x0427, 0xFFFF }, { 0x0226, 0x090C, 0x002E, 0x0611, 0x0D09, 0x002F, 0xFFFF }, { 0x0715, 0x0B07, 0x0819, 0x032B, 0x1C0F,
- 0xFFFF }, { 0x0310, 0x0531, 0x0701, 0x0803, 0x022D, 0x0A1D, 0xFFFF }, { 0x0418, 0x012C, 0x0433, 0x0525, 0x0137, 0x091B,
- 0x130D, 0xFFFF }, { 0x0232, 0x0402, 0x0621, 0x0429, 0xFFFF }, { 0x0228, 0x051C, 0x0713, 0x100B, 0x0C1F, 0xFFFF }, {
- 0x0320, 0x0335, 0x0A05, 0x0817, 0xFFFF }, { 0x0623, 0x0527, 0xFFFF }, { 0x0416, 0x061E, 0x0E0E, 0x0711, 0x0E09, 0x012F,
- 0xFFFF }, { 0x0322, 0x0504, 0x080A, 0x0919, 0x1D0F, 0xFFFF }, { 0x0631, 0x0903, 0x0815, 0x0C07, 0x042B, 0x032D, 0x0B1D,
- 0xFFFF }, { 0x022A, 0x0801, 0x0533, 0x0625, 0x0237, 0x0A1B, 0xFFFF }, { 0x0330, 0x0414, 0x0136, 0x0708, 0x0721, 0x0529,
- 0x140D, 0xFFFF }, { 0x0400, 0x0324, 0x0606, 0x0038, 0x0039, 0x110B, 0x0D1F, 0xFFFF }, { 0x051A, 0x0813, 0x0B05, 0x0917,
- 0xFFFF }, { 0x0723, 0x0435, 0x0627, 0xFFFF }, { 0x0412, 0x0234, 0x0F09, 0x022F, 0xFFFF }, { 0x0326, 0x0A0C, 0x012E,
- 0x0811, 0x0A19, 0x1E0F, 0xFFFF }, { 0x0731, 0x0A03, 0x0915, 0x0D07, 0x052B, 0xFFFF }, { 0x0410, 0x0901, 0x0633, 0x0725,
- 0x0337, 0x0B1B, 0x042D, 0x0C1D, 0xFFFF }, { 0x0518, 0x022C, 0x0629, 0x150D, 0xFFFF }, { 0x0332, 0x0502, 0x0821, 0x0139,
- 0x120B, 0x0E1F, 0xFFFF }, { 0x0328, 0x061C, 0x0913, 0x0A17, 0xFFFF }, { 0x0420, 0x0535, 0x0C05, 0x0727, 0xFFFF }, {
- 0x0823, 0x032F, 0xFFFF }, { 0x0516, 0x071E, 0x0F0E, 0x0911, 0x0B19, 0x1009, 0x1F0F, 0xFFFF }, { 0x0422, 0x0604, 0x090A,
- 0x0B03, 0x0A15, 0x0E07, 0x062B, 0xFFFF }, { 0x0831, 0x0A01, 0x0733, 0x052D, 0x0D1D, 0xFFFF }, { 0x032A, 0x0825, 0x0437,
- 0x0729, 0x0C1B, 0x160D, 0xFFFF }, { 0x0430, 0x0514, 0x0236, 0x0808, 0x0921, 0x0239, 0x130B, 0x0F1F, 0xFFFF }, { 0x0500,
- 0x0424, 0x0706, 0x0138, 0x0A13, 0x0B17, 0xFFFF }, { 0x061A, 0x0635, 0x0D05, 0xFFFF }, { 0x0923, 0x0827, 0xFFFF }, {
- 0x0512, 0x0334, 0x003A, 0x0A11, 0x1109, 0x003B, 0x042F, 0xFFFF }, { 0x0426, 0x0B0C, 0x022E, 0x0B15, 0x0F07, 0x0C19,
- 0x072B, 0xFFFF }, { 0x0931, 0x0B01, 0x0C03, 0x062D, 0x0E1D, 0xFFFF }, { 0x0510, 0x0833, 0x0925, 0x0537, 0x0D1B, 0x170D,
- 0xFFFF }, { 0x0618, 0x032C, 0x0A21, 0x0339, 0x0829, 0xFFFF }, { 0x0432, 0x0602, 0x0B13, 0x140B, 0x101F, 0xFFFF }, {
- 0x0428, 0x071C, 0x0735, 0x0E05, 0x0C17, 0xFFFF }, { 0x0520, 0x0A23, 0x0927, 0xFFFF }, { 0x0B11, 0x1209, 0x013B, 0x052F,
- 0xFFFF }, { 0x0616, 0x081E, 0x0D19, 0xFFFF }, { 0x0522, 0x0704, 0x0A0A, 0x0A31, 0x0D03, 0x0C15, 0x1007, 0x082B, 0x072D,
- 0x0F1D, 0xFFFF }, { 0x0C01, 0x0933, 0x0A25, 0x0637, 0x0E1B, 0xFFFF }, { 0x042A, 0x0B21, 0x0929, 0x180D, 0xFFFF }, {
- 0x0530, 0x0614, 0x0336, 0x0908, 0x0439, 0x150B, 0x111F, 0xFFFF }, { 0x0600, 0x0524, 0x0806, 0x0238, 0x0C13, 0x0F05,
- 0x0D17, 0xFFFF }, { 0x071A, 0x0B23, 0x0835, 0x0A27, 0xFFFF }, { 0x1309, 0x023B, 0x062F, 0xFFFF }, { 0x0612, 0x0434,
- 0x013A, 0x0C11, 0x0E19, 0xFFFF }, { 0x0526, 0x0C0C, 0x032E, 0x0B31, 0x0E03, 0x0D15, 0x1107, 0x092B, 0xFFFF }, { 0x0D01,
- 0x0A33, 0x0B25, 0x0737, 0x0F1B, 0x082D, 0x101D, 0xFFFF }, { 0x0610, 0x0A29, 0x190D, 0xFFFF }, { 0x0718, 0x042C, 0x0C21,
- 0x0539, 0x160B, 0x121F, 0xFFFF }, { 0x0532, 0x0702, 0x0D13, 0x0E17, 0xFFFF }, { 0x0528, 0x081C, 0x0935, 0x1005, 0x0B27,
- 0xFFFF }, { 0x0620, 0x0C23, 0x033B, 0x072F, 0xFFFF }, { 0x0D11, 0x0F19, 0x1409, 0xFFFF }, { 0x0716, 0x003C, 0x091E,
- 0x0F03, 0x0E15, 0x1207, 0x0A2B, 0x003D, 0xFFFF }, { 0x0622, 0x0804, 0x0B0A, 0x0C31, 0x0E01, 0x0B33, 0x092D, 0x111D,
- 0xFFFF }, { 0x0C25, 0x0837, 0x0B29, 0x101B, 0x1A0D, 0xFFFF }, { 0x052A, 0x0D21, 0x0639, 0x170B, 0x131F, 0xFFFF }, {
- 0x0630, 0x0714, 0x0436, 0x0A08, 0x0E13, 0x0F17, 0xFFFF }, { 0x0700, 0x0624, 0x0906, 0x0338, 0x0A35, 0x1105, 0xFFFF }, {
- 0x081A, 0x0D23, 0x0C27, 0xFFFF }, { 0x0E11, 0x1509, 0x043B, 0x082F, 0xFFFF }, { 0x0712, 0x0534, 0x023A, 0x0F15, 0x1307,
- 0x1019, 0x0B2B, 0x013D, 0xFFFF }, { 0x0626, 0x0D0C, 0x042E, 0x0D31, 0x0F01, 0x1003, 0x0A2D, 0x121D, 0xFFFF }, { 0x0C33,
- 0x0D25, 0x0937, 0x111B, 0x1B0D, 0xFFFF }, { 0x0710, 0x0E21, 0x0739, 0x0C29, 0xFFFF }, { 0x0818, 0x052C, 0x0F13, 0x180B,
- 0x141F, 0xFFFF }, { 0x0632, 0x0802, 0x0B35, 0x1205, 0x1017, 0xFFFF }, { 0x0628, 0x091C, 0x0E23, 0x0D27, 0xFFFF }, {
- 0x0720, 0x0F11, 0x1609, 0x053B, 0x092F, 0xFFFF }, { 0x1119, 0x023D, 0xFFFF }, { 0x0816, 0x013C, 0x0A1E, 0x0E31, 0x1103,
- 0x1015, 0x1407, 0x0C2B, 0x0B2D, 0x131D, 0xFFFF }, { 0x0722, 0x0904, 0x0C0A, 0x1001, 0x0D33, 0x0E25, 0x0A37, 0x121B,
- 0xFFFF }, { 0x0F21, 0x0D29, 0x1C0D, 0xFFFF }, { 0x062A, 0x0839, 0x190B, 0x151F, 0xFFFF }, { 0x0730, 0x0814, 0x0536,
- 0x0B08, 0x1013, 0x1305, 0x1117, 0xFFFF }, { 0x0800, 0x0724, 0x0A06, 0x0438, 0x0F23, 0x0C35, 0x0E27, 0xFFFF }, { 0x091A,
- 0x1709, 0x063B, 0x0A2F, 0xFFFF }, { 0x1011, 0x1219, 0x033D, 0xFFFF }, { 0x0812, 0x0634, 0x033A, 0x0F31, 0x1203, 0x1115,
- 0x1507, 0x0D2B, 0xFFFF }, { 0x0726, 0x0E0C, 0x052E, 0x1101, 0x0E33, 0x0F25, 0x0B37, 0x131B, 0x0C2D, 0x141D, 0xFFFF }, {
- 0x0E29, 0x1D0D, 0xFFFF }, { 0x0810, 0x1021, 0x0939, 0x1A0B, 0x161F, 0xFFFF }, { 0x0918, 0x062C, 0x1113, 0x1217, 0xFFFF
- }, { 0x0732, 0x0902, 0x0D35, 0x1405, 0x0F27, 0xFFFF }, { 0x0728, 0x0A1C, 0x1023, 0x073B, 0x0B2F, 0xFFFF }, { 0x0820,
- 0x1111, 0x1319, 0x1809, 0xFFFF }, { 0x1303, 0x1215, 0x1607, 0x0E2B, 0x043D, 0xFFFF }, { 0x0916, 0x023C, 0x0B1E, 0x1031,
- 0x1201, 0x0F33, 0x0D2D, 0x151D, 0xFFFF }, { 0x0822, 0x0A04, 0x0D0A, 0x1025, 0x0C37, 0x0F29, 0x141B, 0x1E0D, 0xFFFF }, {
- 0x1121, 0x0A39, 0x1B0B, 0x171F, 0xFFFF }, { 0x072A, 0x1213, 0x1317, 0xFFFF }, { 0x0830, 0x0914, 0x0636, 0x0C08, 0x0E35,
- 0x1505, 0xFFFF }, { 0x0900, 0x0824, 0x0B06, 0x0538, 0x1123, 0x1027, 0xFFFF }, { 0x0A1A, 0x1211, 0x1909, 0x083B, 0x0C2F,
- 0xFFFF }, { 0x1315, 0x1707, 0x1419, 0x0F2B, 0x053D, 0xFFFF }, { 0x0912, 0x0734, 0x043A, 0x1131, 0x1301, 0x1403, 0x0E2D,
- 0x161D, 0xFFFF }, { 0x0826, 0x0F0C, 0x062E, 0x1033, 0x1125, 0x0D37, 0x151B, 0x1F0D, 0xFFFF }, { 0x1221, 0x0B39, 0x1029,
- 0xFFFF }, { 0x0910, 0x1313, 0x1C0B, 0x181F, 0xFFFF }, { 0x0A18, 0x072C, 0x0F35, 0x1605, 0x1417, 0xFFFF }, { 0x0832,
- 0x0A02, 0x1223, 0x1127, 0xFFFF }, { 0x0828, 0x0B1C, 0x1311, 0x1A09, 0x093B, 0x0D2F, 0xFFFF }, { 0x0920, 0x1519, 0x063D,
- 0xFFFF }, { 0x1231, 0x1503, 0x1415, 0x1807, 0x102B, 0x0F2D, 0x171D, 0xFFFF }, { 0x0A16, 0x033C, 0x0C1E, 0x1401, 0x1133,
- 0x1225, 0x0E37, 0x161B, 0xFFFF }, { 0x0922, 0x0B04, 0x0E0A, 0x1321, 0x1129, 0xFFFF }, { 0x0C39, 0x1D0B, 0x191F, 0xFFFF
- }, { 0x082A, 0x1413, 0x1705, 0x1517, 0xFFFF }, { 0x0930, 0x0A14, 0x0736, 0x0D08, 0x1323, 0x1035, 0x1227, 0xFFFF }, {
- 0x0A00, 0x0924, 0x0C06, 0x0638, 0x1B09, 0x0A3B, 0x0E2F, 0xFFFF }, { 0x0B1A, 0x1411, 0x1619, 0x073D, 0xFFFF }, { 0x1331,
- 0x1603, 0x1515, 0x1907, 0x112B, 0xFFFF }, { 0x0A12, 0x0834, 0x053A, 0x1501, 0x1233, 0x1325, 0x0F37, 0x171B, 0x102D,
- 0x181D, 0xFFFF }, { 0x0926, 0x072E, 0x1229, 0xFFFF }, { 0x1421, 0x0D39, 0x1E0B, 0x1A1F, 0xFFFF }, { 0x0A10, 0x1513,
- 0x1617, 0xFFFF }, { 0x0B18, 0x082C, 0x1135, 0x1805, 0x1327, 0xFFFF }, { 0x0932, 0x0B02, 0x1423, 0x0B3B, 0x0F2F, 0xFFFF
- }, { 0x0928, 0x0C1C, 0x1511, 0x1719, 0x1C09, 0xFFFF }, { 0x0A20, 0x1703, 0x1615, 0x1A07, 0x122B, 0x083D, 0xFFFF }, {
- 0x1431, 0x1601, 0x1333, 0x112D, 0x191D, 0xFFFF }, { 0x0B16, 0x043C, 0x0D1E, 0x1425, 0x1037, 0x1329, 0x181B, 0xFFFF }, {
- 0x0A22, 0x0C04, 0x0F0A, 0x1521, 0x0E39, 0x1F0B, 0x1B1F, 0xFFFF }, { 0x1613, 0x1717, 0xFFFF }, { 0x092A, 0x1235, 0x1905,
- 0xFFFF }, { 0x0A30, 0x0B14, 0x0836, 0x0E08, 0x1523, 0x1427, 0xFFFF }, { 0x0B00, 0x0A24, 0x0D06, 0x0738, 0x1611, 0x1D09,
- 0x0C3B, 0x102F, 0xFFFF }, { 0x0C1A, 0x1715, 0x1B07, 0x1819, 0x132B, 0x093D, 0xFFFF }, { 0x1531, 0x1701, 0x1803, 0x122D,
- 0x1A1D, 0xFFFF }, { 0x0B12, 0x0934, 0x063A, 0x1433, 0x1525, 0x1137, 0x191B, 0xFFFF }, { 0x0A26, 0x003E, 0x082E, 0x1621,
- 0x0F39, 0x1429, 0x003F, 0xFFFF }, { 0x1713, 0x1C1F, 0xFFFF }, { 0x0B10, 0x1335, 0x1A05, 0x1817, 0xFFFF }, { 0x0C18,
- 0x092C, 0x1623, 0x1527, 0xFFFF }, { 0x0A32, 0x0C02, 0x1711, 0x1E09, 0x0D3B, 0x112F, 0xFFFF }, { 0x0A28, 0x0D1C, 0x1919,
- 0x0A3D, 0xFFFF }, { 0x0B20, 0x1631, 0x1903, 0x1815, 0x1C07, 0x142B, 0x132D, 0x1B1D, 0xFFFF }, { 0x1801, 0x1533, 0x1625,
- 0x1237, 0x1A1B, 0xFFFF }, { 0x0C16, 0x053C, 0x0E1E, 0x1721, 0x1529, 0x013F, 0xFFFF }, { 0x0B22, 0x0D04, 0x1039, 0x1D1F,
- 0xFFFF }, { 0x1813, 0x1B05, 0x1917, 0xFFFF }, { 0x0A2A, 0x1723, 0x1435, 0x1627, 0xFFFF }, { 0x0B30, 0x0C14, 0x0936,
- 0x0F08, 0x1F09, 0x0E3B, 0x122F, 0xFFFF }, { 0x0C00, 0x0B24, 0x0E06, 0x0838, 0x1811, 0x1A19, 0x0B3D, 0xFFFF }, { 0x0D1A,
- 0x1731, 0x1A03, 0x1915, 0x1D07, 0x152B, 0xFFFF }, { 0x1901, 0x1633, 0x1725, 0x1337, 0x1B1B, 0x142D, 0x1C1D, 0xFFFF }, {
- 0x0C12, 0x0A34, 0x073A, 0x1629, 0x023F, 0xFFFF }, { 0x0B26, 0x013E, 0x092E, 0x1821, 0x1139, 0x1E1F, 0xFFFF }, { 0x1913,
- 0x1A17, 0xFFFF }, { 0x0C10, 0x1535, 0x1C05, 0x1727, 0xFFFF }, { 0x0D18, 0x0A2C, 0x1823, 0x0F3B, 0x132F, 0xFFFF }, {
- 0x0B32, 0x0D02, 0x1911, 0x1B19, 0xFFFF }, { 0x0B28, 0x0E1C, 0x1B03, 0x1A15, 0x1E07, 0x162B, 0x0C3D, 0xFFFF }, { 0x0C20,
- 0x1831, 0x1A01, 0x1733, 0x152D, 0x1D1D, 0xFFFF }, { 0x1825, 0x1437, 0x1729, 0x1C1B, 0x033F, 0xFFFF }, { 0x0D16, 0x063C,
- 0x0F1E, 0x1921, 0x1239, 0x1F1F, 0xFFFF }, { 0x0C22, 0x0E04, 0x1A13, 0x1B17, 0xFFFF }, { 0x1635, 0x1D05, 0xFFFF }, {
- 0x0B2A, 0x1923, 0x1827, 0xFFFF }, { 0x0C30, 0x0D14, 0x0A36, 0x1A11, 0x103B, 0x142F, 0xFFFF }, { 0x0D00, 0x0C24, 0x0F06,
- 0x0938, 0x1B15, 0x1F07, 0x1C19, 0x172B, 0x0D3D, 0xFFFF }, { 0x0E1A, 0x1931, 0x1B01, 0x1C03, 0x162D, 0x1E1D, 0xFFFF }, {
- 0x1833, 0x1925, 0x1537, 0x1D1B, 0xFFFF }, { 0x0D12, 0x0B34, 0x083A, 0x1A21, 0x1339, 0x1829, 0x043F, 0xFFFF }, { 0x0C26,
- 0x023E, 0x0A2E, 0x1B13, 0xFFFF }, { 0x1735, 0x1E05, 0x1C17, 0xFFFF }, { 0x0D10, 0x1A23, 0x1927, 0xFFFF }, { 0x0E18,
- 0x0B2C, 0x1B11, 0x113B, 0x152F, 0xFFFF }, { 0x0C32, 0x0E02, 0x1D19, 0x0E3D, 0xFFFF }, { 0x0C28, 0x0F1C, 0x1A31, 0x1D03,
- 0x1C15, 0x182B, 0x172D, 0x1F1D, 0xFFFF }, { 0x0D20, 0x1C01, 0x1933, 0x1A25, 0x1637, 0x1E1B, 0xFFFF }, { 0x1B21, 0x1929,
- 0x053F, 0xFFFF }, { 0x0E16, 0x073C, 0x1439, 0xFFFF }, { 0x0D22, 0x0F04, 0x1C13, 0x1F05, 0x1D17, 0xFFFF }, { 0x1B23,
- 0x1835, 0x1A27, 0xFFFF }, { 0x0C2A, 0x123B, 0x162F, 0xFFFF }, { 0x0D30, 0x0E14, 0x0B36, 0x1C11, 0x1E19, 0x0F3D, 0xFFFF
- }, { 0x0E00, 0x0D24, 0x0A38, 0x1B31, 0x1E03, 0x1D15, 0x192B, 0xFFFF }, { 0x0F1A, 0x1D01, 0x1A33, 0x1B25, 0x1737, 0x1F1B,
- 0x182D, 0xFFFF }, { 0x1A29, 0x063F, 0xFFFF }, { 0x0E12, 0x0C34, 0x093A, 0x1C21, 0x1539, 0xFFFF }, { 0x0D26, 0x033E,
- 0x0B2E, 0x1D13, 0x1E17, 0xFFFF }, { 0x1935, 0x1B27, 0xFFFF }, { 0x0E10, 0x1C23, 0x133B, 0x172F, 0xFFFF }, { 0x0F18,
- 0x0C2C, 0x1D11, 0x1F19, 0xFFFF }, { 0x0D32, 0x0F02, 0x1F03, 0x1E15, 0x1A2B, 0x103D, 0xFFFF }, { 0x0D28, 0x1C31, 0x1E01,
- 0x1B33, 0x192D, 0xFFFF }, { 0x0E20, 0x1C25, 0x1837, 0x1B29, 0x073F, 0xFFFF }, { 0x1D21, 0x1639, 0xFFFF }, { 0x0F16,
- 0x083C, 0x1E13, 0x1F17, 0xFFFF }, { 0x0E22, 0x1A35, 0xFFFF }, { 0x1D23, 0x1C27, 0xFFFF }, { 0x0D2A, 0x1E11, 0x143B,
- 0x182F, 0xFFFF }, { 0x0E30, 0x0F14, 0x0C36, 0x1F15, 0x1B2B, 0x113D, 0xFFFF }, { 0x0F00, 0x0E24, 0x0B38, 0x1D31, 0x1F01,
- 0x1A2D, 0xFFFF }, { 0x1C33, 0x1D25, 0x1937, 0xFFFF }, { 0x1E21, 0x1739, 0x1C29, 0x083F, 0xFFFF }, { 0x0F12, 0x0D34,
- 0x0A3A, 0x1F13, 0xFFFF }, { 0x0E26, 0x043E, 0x0C2E, 0x1B35, 0xFFFF }, { 0x1E23, 0x1D27, 0xFFFF }, { 0x0F10, 0x1F11,
- 0x153B, 0x192F, 0xFFFF }, { 0x0D2C, 0x123D, 0xFFFF },
- };
-
- struct etc1_block
- {
- // big endian uint64:
- // bit ofs: 56 48 40 32 24 16 8 0
- // byte ofs: b0, b1, b2, b3, b4, b5, b6, b7
- union
- {
- uint64 m_uint64;
- uint8 m_bytes[8];
- };
-
- uint8 m_low_color[2];
- uint8 m_high_color[2];
-
- enum { cNumSelectorBytes = 4 };
- uint8 m_selectors[cNumSelectorBytes];
-
- inline void clear()
- {
- zero_this(this);
- }
-
- inline uint get_byte_bits(uint ofs, uint num) const
- {
- RG_ETC1_ASSERT((ofs + num) <= 64U);
- RG_ETC1_ASSERT(num && (num <= 8U));
- RG_ETC1_ASSERT((ofs >> 3) == ((ofs + num - 1) >> 3));
- const uint byte_ofs = 7 - (ofs >> 3);
- const uint byte_bit_ofs = ofs & 7;
- return (m_bytes[byte_ofs] >> byte_bit_ofs) & ((1 << num) - 1);
- }
-
- inline void set_byte_bits(uint ofs, uint num, uint bits)
- {
- RG_ETC1_ASSERT((ofs + num) <= 64U);
- RG_ETC1_ASSERT(num && (num < 32U));
- RG_ETC1_ASSERT((ofs >> 3) == ((ofs + num - 1) >> 3));
- RG_ETC1_ASSERT(bits < (1U << num));
- const uint byte_ofs = 7 - (ofs >> 3);
- const uint byte_bit_ofs = ofs & 7;
- const uint mask = (1 << num) - 1;
- m_bytes[byte_ofs] &= ~(mask << byte_bit_ofs);
- m_bytes[byte_ofs] |= (bits << byte_bit_ofs);
- }
-
- // false = left/right subblocks
- // true = upper/lower subblocks
- inline bool get_flip_bit() const
- {
- return (m_bytes[3] & 1) != 0;
- }
-
- inline void set_flip_bit(bool flip)
- {
- m_bytes[3] &= ~1;
- m_bytes[3] |= static_cast<uint8>(flip);
- }
-
- inline bool get_diff_bit() const
- {
- return (m_bytes[3] & 2) != 0;
- }
-
- inline void set_diff_bit(bool diff)
- {
- m_bytes[3] &= ~2;
- m_bytes[3] |= (static_cast<uint>(diff) << 1);
- }
-
- // Returns intensity modifier table (0-7) used by subblock subblock_id.
- // subblock_id=0 left/top (CW 1), 1=right/bottom (CW 2)
- inline uint get_inten_table(uint subblock_id) const
- {
- RG_ETC1_ASSERT(subblock_id < 2);
- const uint ofs = subblock_id ? 2 : 5;
- return (m_bytes[3] >> ofs) & 7;
- }
-
- // Sets intensity modifier table (0-7) used by subblock subblock_id (0 or 1)
- inline void set_inten_table(uint subblock_id, uint t)
- {
- RG_ETC1_ASSERT(subblock_id < 2);
- RG_ETC1_ASSERT(t < 8);
- const uint ofs = subblock_id ? 2 : 5;
- m_bytes[3] &= ~(7 << ofs);
- m_bytes[3] |= (t << ofs);
- }
-
- // Returned selector value ranges from 0-3 and is a direct index into g_etc1_inten_tables.
- inline uint get_selector(uint x, uint y) const
- {
- RG_ETC1_ASSERT((x | y) < 4);
-
- const uint bit_index = x * 4 + y;
- const uint byte_bit_ofs = bit_index & 7;
- const uint8 *p = &m_bytes[7 - (bit_index >> 3)];
- const uint lsb = (p[0] >> byte_bit_ofs) & 1;
- const uint msb = (p[-2] >> byte_bit_ofs) & 1;
- const uint val = lsb | (msb << 1);
-
- return g_etc1_to_selector_index[val];
- }
-
- // Selector "val" ranges from 0-3 and is a direct index into g_etc1_inten_tables.
- inline void set_selector(uint x, uint y, uint val)
- {
- RG_ETC1_ASSERT((x | y | val) < 4);
- const uint bit_index = x * 4 + y;
-
- uint8 *p = &m_bytes[7 - (bit_index >> 3)];
-
- const uint byte_bit_ofs = bit_index & 7;
- const uint mask = 1 << byte_bit_ofs;
-
- const uint etc1_val = g_selector_index_to_etc1[val];
-
- const uint lsb = etc1_val & 1;
- const uint msb = etc1_val >> 1;
-
- p[0] &= ~mask;
- p[0] |= (lsb << byte_bit_ofs);
-
- p[-2] &= ~mask;
- p[-2] |= (msb << byte_bit_ofs);
- }
-
- inline void set_base4_color(uint idx, uint16 c)
- {
- if (idx)
- {
- set_byte_bits(cETC1AbsColor4R2BitOffset, 4, (c >> 8) & 15);
- set_byte_bits(cETC1AbsColor4G2BitOffset, 4, (c >> 4) & 15);
- set_byte_bits(cETC1AbsColor4B2BitOffset, 4, c & 15);
- }
- else
- {
- set_byte_bits(cETC1AbsColor4R1BitOffset, 4, (c >> 8) & 15);
- set_byte_bits(cETC1AbsColor4G1BitOffset, 4, (c >> 4) & 15);
- set_byte_bits(cETC1AbsColor4B1BitOffset, 4, c & 15);
- }
- }
-
- inline uint16 get_base4_color(uint idx) const
- {
- uint r, g, b;
- if (idx)
- {
- r = get_byte_bits(cETC1AbsColor4R2BitOffset, 4);
- g = get_byte_bits(cETC1AbsColor4G2BitOffset, 4);
- b = get_byte_bits(cETC1AbsColor4B2BitOffset, 4);
- }
- else
- {
- r = get_byte_bits(cETC1AbsColor4R1BitOffset, 4);
- g = get_byte_bits(cETC1AbsColor4G1BitOffset, 4);
- b = get_byte_bits(cETC1AbsColor4B1BitOffset, 4);
- }
- return static_cast<uint16>(b | (g << 4U) | (r << 8U));
- }
-
- inline void set_base5_color(uint16 c)
- {
- set_byte_bits(cETC1BaseColor5RBitOffset, 5, (c >> 10) & 31);
- set_byte_bits(cETC1BaseColor5GBitOffset, 5, (c >> 5) & 31);
- set_byte_bits(cETC1BaseColor5BBitOffset, 5, c & 31);
- }
-
- inline uint16 get_base5_color() const
- {
- const uint r = get_byte_bits(cETC1BaseColor5RBitOffset, 5);
- const uint g = get_byte_bits(cETC1BaseColor5GBitOffset, 5);
- const uint b = get_byte_bits(cETC1BaseColor5BBitOffset, 5);
- return static_cast<uint16>(b | (g << 5U) | (r << 10U));
- }
-
- void set_delta3_color(uint16 c)
- {
- set_byte_bits(cETC1DeltaColor3RBitOffset, 3, (c >> 6) & 7);
- set_byte_bits(cETC1DeltaColor3GBitOffset, 3, (c >> 3) & 7);
- set_byte_bits(cETC1DeltaColor3BBitOffset, 3, c & 7);
- }
-
- inline uint16 get_delta3_color() const
- {
- const uint r = get_byte_bits(cETC1DeltaColor3RBitOffset, 3);
- const uint g = get_byte_bits(cETC1DeltaColor3GBitOffset, 3);
- const uint b = get_byte_bits(cETC1DeltaColor3BBitOffset, 3);
- return static_cast<uint16>(b | (g << 3U) | (r << 6U));
- }
-
- // Base color 5
- static uint16 pack_color5(const color_quad_u8& color, bool scaled, uint bias = 127U);
- static uint16 pack_color5(uint r, uint g, uint b, bool scaled, uint bias = 127U);
-
- static color_quad_u8 unpack_color5(uint16 packed_color5, bool scaled, uint alpha = 255U);
- static void unpack_color5(uint& r, uint& g, uint& b, uint16 packed_color, bool scaled);
-
- static bool unpack_color5(color_quad_u8& result, uint16 packed_color5, uint16 packed_delta3, bool scaled, uint alpha = 255U);
- static bool unpack_color5(uint& r, uint& g, uint& b, uint16 packed_color5, uint16 packed_delta3, bool scaled, uint alpha = 255U);
-
- // Delta color 3
- // Inputs range from -4 to 3 (cETC1ColorDeltaMin to cETC1ColorDeltaMax)
- static uint16 pack_delta3(int r, int g, int b);
-
- // Results range from -4 to 3 (cETC1ColorDeltaMin to cETC1ColorDeltaMax)
- static void unpack_delta3(int& r, int& g, int& b, uint16 packed_delta3);
-
- // Abs color 4
- static uint16 pack_color4(const color_quad_u8& color, bool scaled, uint bias = 127U);
- static uint16 pack_color4(uint r, uint g, uint b, bool scaled, uint bias = 127U);
-
- static color_quad_u8 unpack_color4(uint16 packed_color4, bool scaled, uint alpha = 255U);
- static void unpack_color4(uint& r, uint& g, uint& b, uint16 packed_color4, bool scaled);
-
- // subblock colors
- static void get_diff_subblock_colors(color_quad_u8* pDst, uint16 packed_color5, uint table_idx);
- static bool get_diff_subblock_colors(color_quad_u8* pDst, uint16 packed_color5, uint16 packed_delta3, uint table_idx);
- static void get_abs_subblock_colors(color_quad_u8* pDst, uint16 packed_color4, uint table_idx);
-
- static inline void unscaled_to_scaled_color(color_quad_u8& dst, const color_quad_u8& src, bool color4)
- {
- if (color4)
- {
- dst.r = src.r | (src.r << 4);
- dst.g = src.g | (src.g << 4);
- dst.b = src.b | (src.b << 4);
- }
- else
- {
- dst.r = (src.r >> 2) | (src.r << 3);
- dst.g = (src.g >> 2) | (src.g << 3);
- dst.b = (src.b >> 2) | (src.b << 3);
- }
- dst.a = src.a;
- }
- };
-
- // Returns pointer to sorted array.
- template<typename T, typename Q>
- T* indirect_radix_sort(uint num_indices, T* pIndices0, T* pIndices1, const Q* pKeys, uint key_ofs, uint key_size, bool init_indices)
- {
- RG_ETC1_ASSERT((key_ofs >= 0) && (key_ofs < sizeof(T)));
- RG_ETC1_ASSERT((key_size >= 1) && (key_size <= 4));
-
- if (init_indices)
- {
- T* p = pIndices0;
- T* q = pIndices0 + (num_indices >> 1) * 2;
- uint i;
- for (i = 0; p != q; p += 2, i += 2)
- {
- p[0] = static_cast<T>(i);
- p[1] = static_cast<T>(i + 1);
- }
-
- if (num_indices & 1)
- *p = static_cast<T>(i);
- }
-
- uint hist[256 * 4];
-
- memset(hist, 0, sizeof(hist[0]) * 256 * key_size);
-
-#define RG_ETC1_GET_KEY(p) (*(const uint*)((const uint8*)(pKeys + *(p)) + key_ofs))
-#define RG_ETC1_GET_KEY_FROM_INDEX(i) (*(const uint*)((const uint8*)(pKeys + (i)) + key_ofs))
-
- if (key_size == 4)
- {
- T* p = pIndices0;
- T* q = pIndices0 + num_indices;
- for ( ; p != q; p++)
- {
- const uint key = RG_ETC1_GET_KEY(p);
-
- hist[ key & 0xFF]++;
- hist[256 + ((key >> 8) & 0xFF)]++;
- hist[512 + ((key >> 16) & 0xFF)]++;
- hist[768 + ((key >> 24) & 0xFF)]++;
- }
- }
- else if (key_size == 3)
- {
- T* p = pIndices0;
- T* q = pIndices0 + num_indices;
- for ( ; p != q; p++)
- {
- const uint key = RG_ETC1_GET_KEY(p);
-
- hist[ key & 0xFF]++;
- hist[256 + ((key >> 8) & 0xFF)]++;
- hist[512 + ((key >> 16) & 0xFF)]++;
- }
- }
- else if (key_size == 2)
- {
- T* p = pIndices0;
- T* q = pIndices0 + (num_indices >> 1) * 2;
-
- for ( ; p != q; p += 2)
- {
- const uint key0 = RG_ETC1_GET_KEY(p);
- const uint key1 = RG_ETC1_GET_KEY(p+1);
-
- hist[ key0 & 0xFF]++;
- hist[256 + ((key0 >> 8) & 0xFF)]++;
-
- hist[ key1 & 0xFF]++;
- hist[256 + ((key1 >> 8) & 0xFF)]++;
- }
-
- if (num_indices & 1)
- {
- const uint key = RG_ETC1_GET_KEY(p);
-
- hist[ key & 0xFF]++;
- hist[256 + ((key >> 8) & 0xFF)]++;
- }
- }
- else
- {
- RG_ETC1_ASSERT(key_size == 1);
- if (key_size != 1)
- return NULL;
-
- T* p = pIndices0;
- T* q = pIndices0 + (num_indices >> 1) * 2;
-
- for ( ; p != q; p += 2)
- {
- const uint key0 = RG_ETC1_GET_KEY(p);
- const uint key1 = RG_ETC1_GET_KEY(p+1);
-
- hist[key0 & 0xFF]++;
- hist[key1 & 0xFF]++;
- }
-
- if (num_indices & 1)
- {
- const uint key = RG_ETC1_GET_KEY(p);
-
- hist[key & 0xFF]++;
- }
- }
-
- T* pCur = pIndices0;
- T* pNew = pIndices1;
-
- for (uint pass = 0; pass < key_size; pass++)
- {
- const uint* pHist = &hist[pass << 8];
-
- uint offsets[256];
-
- uint cur_ofs = 0;
- for (uint i = 0; i < 256; i += 2)
- {
- offsets[i] = cur_ofs;
- cur_ofs += pHist[i];
-
- offsets[i+1] = cur_ofs;
- cur_ofs += pHist[i+1];
- }
-
- const uint pass_shift = pass << 3;
-
- T* p = pCur;
- T* q = pCur + (num_indices >> 1) * 2;
-
- for ( ; p != q; p += 2)
- {
- uint index0 = p[0];
- uint index1 = p[1];
-
- uint c0 = (RG_ETC1_GET_KEY_FROM_INDEX(index0) >> pass_shift) & 0xFF;
- uint c1 = (RG_ETC1_GET_KEY_FROM_INDEX(index1) >> pass_shift) & 0xFF;
-
- if (c0 == c1)
- {
- uint dst_offset0 = offsets[c0];
-
- offsets[c0] = dst_offset0 + 2;
-
- pNew[dst_offset0] = static_cast<T>(index0);
- pNew[dst_offset0 + 1] = static_cast<T>(index1);
- }
- else
- {
- uint dst_offset0 = offsets[c0]++;
- uint dst_offset1 = offsets[c1]++;
-
- pNew[dst_offset0] = static_cast<T>(index0);
- pNew[dst_offset1] = static_cast<T>(index1);
- }
- }
-
- if (num_indices & 1)
- {
- uint index = *p;
- uint c = (RG_ETC1_GET_KEY_FROM_INDEX(index) >> pass_shift) & 0xFF;
-
- uint dst_offset = offsets[c];
- offsets[c] = dst_offset + 1;
-
- pNew[dst_offset] = static_cast<T>(index);
- }
-
- T* t = pCur;
- pCur = pNew;
- pNew = t;
- }
-
- return pCur;
- }
-
-#undef RG_ETC1_GET_KEY
-#undef RG_ETC1_GET_KEY_FROM_INDEX
-
- uint16 etc1_block::pack_color5(const color_quad_u8& color, bool scaled, uint bias)
- {
- return pack_color5(color.r, color.g, color.b, scaled, bias);
- }
-
- uint16 etc1_block::pack_color5(uint r, uint g, uint b, bool scaled, uint bias)
- {
- if (scaled)
- {
- r = (r * 31U + bias) / 255U;
- g = (g * 31U + bias) / 255U;
- b = (b * 31U + bias) / 255U;
- }
-
- r = rg_etc1::minimum(r, 31U);
- g = rg_etc1::minimum(g, 31U);
- b = rg_etc1::minimum(b, 31U);
-
- return static_cast<uint16>(b | (g << 5U) | (r << 10U));
- }
-
- color_quad_u8 etc1_block::unpack_color5(uint16 packed_color5, bool scaled, uint alpha)
- {
- uint b = packed_color5 & 31U;
- uint g = (packed_color5 >> 5U) & 31U;
- uint r = (packed_color5 >> 10U) & 31U;
-
- if (scaled)
- {
- b = (b << 3U) | (b >> 2U);
- g = (g << 3U) | (g >> 2U);
- r = (r << 3U) | (r >> 2U);
- }
-
- return color_quad_u8(cNoClamp, r, g, b, rg_etc1::minimum(alpha, 255U));
- }
-
- void etc1_block::unpack_color5(uint& r, uint& g, uint& b, uint16 packed_color5, bool scaled)
- {
- color_quad_u8 c(unpack_color5(packed_color5, scaled, 0));
- r = c.r;
- g = c.g;
- b = c.b;
- }
-
- bool etc1_block::unpack_color5(color_quad_u8& result, uint16 packed_color5, uint16 packed_delta3, bool scaled, uint alpha)
- {
- int dc_r, dc_g, dc_b;
- unpack_delta3(dc_r, dc_g, dc_b, packed_delta3);
-
- int b = (packed_color5 & 31U) + dc_b;
- int g = ((packed_color5 >> 5U) & 31U) + dc_g;
- int r = ((packed_color5 >> 10U) & 31U) + dc_r;
-
- bool success = true;
- if (static_cast<uint>(r | g | b) > 31U)
- {
- success = false;
- r = rg_etc1::clamp<int>(r, 0, 31);
- g = rg_etc1::clamp<int>(g, 0, 31);
- b = rg_etc1::clamp<int>(b, 0, 31);
- }
-
- if (scaled)
- {
- b = (b << 3U) | (b >> 2U);
- g = (g << 3U) | (g >> 2U);
- r = (r << 3U) | (r >> 2U);
- }
-
- result.set_noclamp_rgba(r, g, b, rg_etc1::minimum(alpha, 255U));
- return success;
- }
-
- bool etc1_block::unpack_color5(uint& r, uint& g, uint& b, uint16 packed_color5, uint16 packed_delta3, bool scaled, uint alpha)
- {
- color_quad_u8 result;
- const bool success = unpack_color5(result, packed_color5, packed_delta3, scaled, alpha);
- r = result.r;
- g = result.g;
- b = result.b;
- return success;
- }
-
- uint16 etc1_block::pack_delta3(int r, int g, int b)
- {
- RG_ETC1_ASSERT((r >= cETC1ColorDeltaMin) && (r <= cETC1ColorDeltaMax));
- RG_ETC1_ASSERT((g >= cETC1ColorDeltaMin) && (g <= cETC1ColorDeltaMax));
- RG_ETC1_ASSERT((b >= cETC1ColorDeltaMin) && (b <= cETC1ColorDeltaMax));
- if (r < 0) r += 8;
- if (g < 0) g += 8;
- if (b < 0) b += 8;
- return static_cast<uint16>(b | (g << 3) | (r << 6));
- }
-
- void etc1_block::unpack_delta3(int& r, int& g, int& b, uint16 packed_delta3)
- {
- r = (packed_delta3 >> 6) & 7;
- g = (packed_delta3 >> 3) & 7;
- b = packed_delta3 & 7;
- if (r >= 4) r -= 8;
- if (g >= 4) g -= 8;
- if (b >= 4) b -= 8;
- }
-
- uint16 etc1_block::pack_color4(const color_quad_u8& color, bool scaled, uint bias)
- {
- return pack_color4(color.r, color.g, color.b, scaled, bias);
- }
-
- uint16 etc1_block::pack_color4(uint r, uint g, uint b, bool scaled, uint bias)
- {
- if (scaled)
- {
- r = (r * 15U + bias) / 255U;
- g = (g * 15U + bias) / 255U;
- b = (b * 15U + bias) / 255U;
- }
-
- r = rg_etc1::minimum(r, 15U);
- g = rg_etc1::minimum(g, 15U);
- b = rg_etc1::minimum(b, 15U);
-
- return static_cast<uint16>(b | (g << 4U) | (r << 8U));
- }
-
- color_quad_u8 etc1_block::unpack_color4(uint16 packed_color4, bool scaled, uint alpha)
- {
- uint b = packed_color4 & 15U;
- uint g = (packed_color4 >> 4U) & 15U;
- uint r = (packed_color4 >> 8U) & 15U;
-
- if (scaled)
- {
- b = (b << 4U) | b;
- g = (g << 4U) | g;
- r = (r << 4U) | r;
- }
-
- return color_quad_u8(cNoClamp, r, g, b, rg_etc1::minimum(alpha, 255U));
- }
-
- void etc1_block::unpack_color4(uint& r, uint& g, uint& b, uint16 packed_color4, bool scaled)
- {
- color_quad_u8 c(unpack_color4(packed_color4, scaled, 0));
- r = c.r;
- g = c.g;
- b = c.b;
- }
-
- void etc1_block::get_diff_subblock_colors(color_quad_u8* pDst, uint16 packed_color5, uint table_idx)
- {
- RG_ETC1_ASSERT(table_idx < cETC1IntenModifierValues);
- const int *pInten_modifer_table = &g_etc1_inten_tables[table_idx][0];
-
- uint r, g, b;
- unpack_color5(r, g, b, packed_color5, true);
-
- const int ir = static_cast<int>(r), ig = static_cast<int>(g), ib = static_cast<int>(b);
-
- const int y0 = pInten_modifer_table[0];
- pDst[0].set(ir + y0, ig + y0, ib + y0);
-
- const int y1 = pInten_modifer_table[1];
- pDst[1].set(ir + y1, ig + y1, ib + y1);
-
- const int y2 = pInten_modifer_table[2];
- pDst[2].set(ir + y2, ig + y2, ib + y2);
-
- const int y3 = pInten_modifer_table[3];
- pDst[3].set(ir + y3, ig + y3, ib + y3);
- }
-
- bool etc1_block::get_diff_subblock_colors(color_quad_u8* pDst, uint16 packed_color5, uint16 packed_delta3, uint table_idx)
- {
- RG_ETC1_ASSERT(table_idx < cETC1IntenModifierValues);
- const int *pInten_modifer_table = &g_etc1_inten_tables[table_idx][0];
-
- uint r, g, b;
- bool success = unpack_color5(r, g, b, packed_color5, packed_delta3, true);
-
- const int ir = static_cast<int>(r), ig = static_cast<int>(g), ib = static_cast<int>(b);
-
- const int y0 = pInten_modifer_table[0];
- pDst[0].set(ir + y0, ig + y0, ib + y0);
-
- const int y1 = pInten_modifer_table[1];
- pDst[1].set(ir + y1, ig + y1, ib + y1);
-
- const int y2 = pInten_modifer_table[2];
- pDst[2].set(ir + y2, ig + y2, ib + y2);
-
- const int y3 = pInten_modifer_table[3];
- pDst[3].set(ir + y3, ig + y3, ib + y3);
-
- return success;
- }
-
- void etc1_block::get_abs_subblock_colors(color_quad_u8* pDst, uint16 packed_color4, uint table_idx)
- {
- RG_ETC1_ASSERT(table_idx < cETC1IntenModifierValues);
- const int *pInten_modifer_table = &g_etc1_inten_tables[table_idx][0];
-
- uint r, g, b;
- unpack_color4(r, g, b, packed_color4, true);
-
- const int ir = static_cast<int>(r), ig = static_cast<int>(g), ib = static_cast<int>(b);
-
- const int y0 = pInten_modifer_table[0];
- pDst[0].set(ir + y0, ig + y0, ib + y0);
-
- const int y1 = pInten_modifer_table[1];
- pDst[1].set(ir + y1, ig + y1, ib + y1);
-
- const int y2 = pInten_modifer_table[2];
- pDst[2].set(ir + y2, ig + y2, ib + y2);
-
- const int y3 = pInten_modifer_table[3];
- pDst[3].set(ir + y3, ig + y3, ib + y3);
- }
-
- bool unpack_etc1_block(const void* pETC1_block, unsigned int* pDst_pixels_rgba, bool preserve_alpha)
- {
- color_quad_u8* pDst = reinterpret_cast<color_quad_u8*>(pDst_pixels_rgba);
- const etc1_block& block = *static_cast<const etc1_block*>(pETC1_block);
-
- const bool diff_flag = block.get_diff_bit();
- const bool flip_flag = block.get_flip_bit();
- const uint table_index0 = block.get_inten_table(0);
- const uint table_index1 = block.get_inten_table(1);
-
- color_quad_u8 subblock_colors0[4];
- color_quad_u8 subblock_colors1[4];
- bool success = true;
-
- if (diff_flag)
- {
- const uint16 base_color5 = block.get_base5_color();
- const uint16 delta_color3 = block.get_delta3_color();
- etc1_block::get_diff_subblock_colors(subblock_colors0, base_color5, table_index0);
-
- if (!etc1_block::get_diff_subblock_colors(subblock_colors1, base_color5, delta_color3, table_index1))
- success = false;
- }
- else
- {
- const uint16 base_color4_0 = block.get_base4_color(0);
- etc1_block::get_abs_subblock_colors(subblock_colors0, base_color4_0, table_index0);
-
- const uint16 base_color4_1 = block.get_base4_color(1);
- etc1_block::get_abs_subblock_colors(subblock_colors1, base_color4_1, table_index1);
- }
-
- if (preserve_alpha)
- {
- if (flip_flag)
- {
- for (uint y = 0; y < 2; y++)
- {
- pDst[0].set_rgb(subblock_colors0[block.get_selector(0, y)]);
- pDst[1].set_rgb(subblock_colors0[block.get_selector(1, y)]);
- pDst[2].set_rgb(subblock_colors0[block.get_selector(2, y)]);
- pDst[3].set_rgb(subblock_colors0[block.get_selector(3, y)]);
- pDst += 4;
- }
-
- for (uint y = 2; y < 4; y++)
- {
- pDst[0].set_rgb(subblock_colors1[block.get_selector(0, y)]);
- pDst[1].set_rgb(subblock_colors1[block.get_selector(1, y)]);
- pDst[2].set_rgb(subblock_colors1[block.get_selector(2, y)]);
- pDst[3].set_rgb(subblock_colors1[block.get_selector(3, y)]);
- pDst += 4;
- }
- }
- else
- {
- for (uint y = 0; y < 4; y++)
- {
- pDst[0].set_rgb(subblock_colors0[block.get_selector(0, y)]);
- pDst[1].set_rgb(subblock_colors0[block.get_selector(1, y)]);
- pDst[2].set_rgb(subblock_colors1[block.get_selector(2, y)]);
- pDst[3].set_rgb(subblock_colors1[block.get_selector(3, y)]);
- pDst += 4;
- }
- }
- }
- else
- {
- if (flip_flag)
- {
- // 0000
- // 0000
- // 1111
- // 1111
- for (uint y = 0; y < 2; y++)
- {
- pDst[0] = subblock_colors0[block.get_selector(0, y)];
- pDst[1] = subblock_colors0[block.get_selector(1, y)];
- pDst[2] = subblock_colors0[block.get_selector(2, y)];
- pDst[3] = subblock_colors0[block.get_selector(3, y)];
- pDst += 4;
- }
-
- for (uint y = 2; y < 4; y++)
- {
- pDst[0] = subblock_colors1[block.get_selector(0, y)];
- pDst[1] = subblock_colors1[block.get_selector(1, y)];
- pDst[2] = subblock_colors1[block.get_selector(2, y)];
- pDst[3] = subblock_colors1[block.get_selector(3, y)];
- pDst += 4;
- }
- }
- else
- {
- // 0011
- // 0011
- // 0011
- // 0011
- for (uint y = 0; y < 4; y++)
- {
- pDst[0] = subblock_colors0[block.get_selector(0, y)];
- pDst[1] = subblock_colors0[block.get_selector(1, y)];
- pDst[2] = subblock_colors1[block.get_selector(2, y)];
- pDst[3] = subblock_colors1[block.get_selector(3, y)];
- pDst += 4;
- }
- }
- }
-
- return success;
- }
-
- struct etc1_solution_coordinates
- {
- inline etc1_solution_coordinates() :
- m_unscaled_color(0, 0, 0, 0),
- m_inten_table(0),
- m_color4(false)
- {
- }
-
- inline etc1_solution_coordinates(uint r, uint g, uint b, uint inten_table, bool color4) :
- m_unscaled_color(r, g, b, 255),
- m_inten_table(inten_table),
- m_color4(color4)
- {
- }
-
- inline etc1_solution_coordinates(const color_quad_u8& c, uint inten_table, bool color4) :
- m_unscaled_color(c),
- m_inten_table(inten_table),
- m_color4(color4)
- {
- }
-
- inline etc1_solution_coordinates(const etc1_solution_coordinates& other)
- {
- *this = other;
- }
-
- inline etc1_solution_coordinates& operator= (const etc1_solution_coordinates& rhs)
- {
- m_unscaled_color = rhs.m_unscaled_color;
- m_inten_table = rhs.m_inten_table;
- m_color4 = rhs.m_color4;
- return *this;
- }
-
- inline void clear()
- {
- m_unscaled_color.clear();
- m_inten_table = 0;
- m_color4 = false;
- }
-
- inline color_quad_u8 get_scaled_color() const
- {
- int br, bg, bb;
- if (m_color4)
- {
- br = m_unscaled_color.r | (m_unscaled_color.r << 4);
- bg = m_unscaled_color.g | (m_unscaled_color.g << 4);
- bb = m_unscaled_color.b | (m_unscaled_color.b << 4);
- }
- else
- {
- br = (m_unscaled_color.r >> 2) | (m_unscaled_color.r << 3);
- bg = (m_unscaled_color.g >> 2) | (m_unscaled_color.g << 3);
- bb = (m_unscaled_color.b >> 2) | (m_unscaled_color.b << 3);
- }
- return color_quad_u8(br, bg, bb);
- }
-
- inline void get_block_colors(color_quad_u8* pBlock_colors)
- {
- int br, bg, bb;
- if (m_color4)
- {
- br = m_unscaled_color.r | (m_unscaled_color.r << 4);
- bg = m_unscaled_color.g | (m_unscaled_color.g << 4);
- bb = m_unscaled_color.b | (m_unscaled_color.b << 4);
- }
- else
- {
- br = (m_unscaled_color.r >> 2) | (m_unscaled_color.r << 3);
- bg = (m_unscaled_color.g >> 2) | (m_unscaled_color.g << 3);
- bb = (m_unscaled_color.b >> 2) | (m_unscaled_color.b << 3);
- }
- const int* pInten_table = g_etc1_inten_tables[m_inten_table];
- pBlock_colors[0].set(br + pInten_table[0], bg + pInten_table[0], bb + pInten_table[0]);
- pBlock_colors[1].set(br + pInten_table[1], bg + pInten_table[1], bb + pInten_table[1]);
- pBlock_colors[2].set(br + pInten_table[2], bg + pInten_table[2], bb + pInten_table[2]);
- pBlock_colors[3].set(br + pInten_table[3], bg + pInten_table[3], bb + pInten_table[3]);
- }
-
- color_quad_u8 m_unscaled_color;
- uint m_inten_table;
- bool m_color4;
- };
-
- class etc1_optimizer
- {
- etc1_optimizer(const etc1_optimizer&);
- etc1_optimizer& operator= (const etc1_optimizer&);
-
- public:
- etc1_optimizer()
- {
- clear();
- }
-
- void clear()
- {
- m_pParams = NULL;
- m_pResult = NULL;
- m_pSorted_luma = NULL;
- m_pSorted_luma_indices = NULL;
- }
-
- struct params : etc1_pack_params
- {
- params()
- {
- clear();
- }
-
- params(const etc1_pack_params& base_params) :
- etc1_pack_params(base_params)
- {
- clear_optimizer_params();
- }
-
- void clear()
- {
- etc1_pack_params::clear();
- clear_optimizer_params();
- }
-
- void clear_optimizer_params()
- {
- m_num_src_pixels = 0;
- m_pSrc_pixels = 0;
-
- m_use_color4 = false;
- static const int s_default_scan_delta[] = { 0 };
- m_pScan_deltas = s_default_scan_delta;
- m_scan_delta_size = 1;
-
- m_base_color5.clear();
- m_constrain_against_base_color5 = false;
- }
-
- uint m_num_src_pixels;
- const color_quad_u8* m_pSrc_pixels;
-
- bool m_use_color4;
- const int* m_pScan_deltas;
- uint m_scan_delta_size;
-
- color_quad_u8 m_base_color5;
- bool m_constrain_against_base_color5;
- };
-
- struct results
- {
- uint64 m_error;
- color_quad_u8 m_block_color_unscaled;
- uint m_block_inten_table;
- uint m_n;
- uint8* m_pSelectors;
- bool m_block_color4;
-
- inline results& operator= (const results& rhs)
- {
- m_block_color_unscaled = rhs.m_block_color_unscaled;
- m_block_color4 = rhs.m_block_color4;
- m_block_inten_table = rhs.m_block_inten_table;
- m_error = rhs.m_error;
- RG_ETC1_ASSERT(m_n == rhs.m_n);
- memcpy(m_pSelectors, rhs.m_pSelectors, rhs.m_n);
- return *this;
- }
- };
-
- void init(const params& params, results& result);
- bool compute();
-
- private:
- struct potential_solution
- {
- potential_solution() : m_coords(), m_error(cUINT64_MAX), m_valid(false)
- {
- }
-
- etc1_solution_coordinates m_coords;
- uint8 m_selectors[8];
- uint64 m_error;
- bool m_valid;
-
- void clear()
- {
- m_coords.clear();
- m_error = cUINT64_MAX;
- m_valid = false;
- }
- };
-
- const params* m_pParams;
- results* m_pResult;
-
- int m_limit;
-
- vec3F m_avg_color;
- int m_br, m_bg, m_bb;
- uint16 m_luma[8];
- uint32 m_sorted_luma[2][8];
- const uint32* m_pSorted_luma_indices;
- uint32* m_pSorted_luma;
-
- uint8 m_selectors[8];
- uint8 m_best_selectors[8];
-
- potential_solution m_best_solution;
- potential_solution m_trial_solution;
- uint8 m_temp_selectors[8];
-
- bool evaluate_solution(const etc1_solution_coordinates& coords, potential_solution& trial_solution, potential_solution* pBest_solution);
- bool evaluate_solution_fast(const etc1_solution_coordinates& coords, potential_solution& trial_solution, potential_solution* pBest_solution);
- };
-
- bool etc1_optimizer::compute()
- {
- const uint n = m_pParams->m_num_src_pixels;
- const int scan_delta_size = m_pParams->m_scan_delta_size;
-
- // Scan through a subset of the 3D lattice centered around the avg block color trying each 3D (555 or 444) lattice point as a potential block color.
- // Each time a better solution is found try to refine the current solution's block color based of the current selectors and intensity table index.
- for (int zdi = 0; zdi < scan_delta_size; zdi++)
- {
- const int zd = m_pParams->m_pScan_deltas[zdi];
- const int mbb = m_bb + zd;
- if (mbb < 0) continue; else if (mbb > m_limit) break;
-
- for (int ydi = 0; ydi < scan_delta_size; ydi++)
- {
- const int yd = m_pParams->m_pScan_deltas[ydi];
- const int mbg = m_bg + yd;
- if (mbg < 0) continue; else if (mbg > m_limit) break;
-
- for (int xdi = 0; xdi < scan_delta_size; xdi++)
- {
- const int xd = m_pParams->m_pScan_deltas[xdi];
- const int mbr = m_br + xd;
- if (mbr < 0) continue; else if (mbr > m_limit) break;
-
- etc1_solution_coordinates coords(mbr, mbg, mbb, 0, m_pParams->m_use_color4);
- if (m_pParams->m_quality == cHighQuality)
- {
- if (!evaluate_solution(coords, m_trial_solution, &m_best_solution))
- continue;
- }
- else
- {
- if (!evaluate_solution_fast(coords, m_trial_solution, &m_best_solution))
- continue;
- }
-
- // Now we have the input block, the avg. color of the input pixels, a set of trial selector indices, and the block color+intensity index.
- // Now, for each component, attempt to refine the current solution by solving a simple linear equation. For example, for 4 colors:
- // The goal is:
- // pixel0 - (block_color+inten_table[selector0]) + pixel1 - (block_color+inten_table[selector1]) + pixel2 - (block_color+inten_table[selector2]) + pixel3 - (block_color+inten_table[selector3]) = 0
- // Rearranging this:
- // (pixel0 + pixel1 + pixel2 + pixel3) - (block_color+inten_table[selector0]) - (block_color+inten_table[selector1]) - (block_color+inten_table[selector2]) - (block_color+inten_table[selector3]) = 0
- // (pixel0 + pixel1 + pixel2 + pixel3) - block_color - inten_table[selector0] - block_color-inten_table[selector1] - block_color-inten_table[selector2] - block_color-inten_table[selector3] = 0
- // (pixel0 + pixel1 + pixel2 + pixel3) - 4*block_color - inten_table[selector0] - inten_table[selector1] - inten_table[selector2] - inten_table[selector3] = 0
- // (pixel0 + pixel1 + pixel2 + pixel3) - 4*block_color - (inten_table[selector0] + inten_table[selector1] + inten_table[selector2] + inten_table[selector3]) = 0
- // (pixel0 + pixel1 + pixel2 + pixel3)/4 - block_color - (inten_table[selector0] + inten_table[selector1] + inten_table[selector2] + inten_table[selector3])/4 = 0
- // block_color = (pixel0 + pixel1 + pixel2 + pixel3)/4 - (inten_table[selector0] + inten_table[selector1] + inten_table[selector2] + inten_table[selector3])/4
- // So what this means:
- // optimal_block_color = avg_input - avg_inten_delta
- // So the optimal block color can be computed by taking the average block color and subtracting the current average of the intensity delta.
- // Unfortunately, optimal_block_color must then be quantized to 555 or 444 so it's not always possible to improve matters using this formula.
- // Also, the above formula is for unclamped intensity deltas. The actual implementation takes into account clamping.
-
- const uint max_refinement_trials = (m_pParams->m_quality == cLowQuality) ? 2 : (((xd | yd | zd) == 0) ? 4 : 2);
- for (uint refinement_trial = 0; refinement_trial < max_refinement_trials; refinement_trial++)
- {
- const uint8* pSelectors = m_best_solution.m_selectors;
- const int* pInten_table = g_etc1_inten_tables[m_best_solution.m_coords.m_inten_table];
-
- int delta_sum_r = 0, delta_sum_g = 0, delta_sum_b = 0;
- const color_quad_u8 base_color(m_best_solution.m_coords.get_scaled_color());
- for (uint r = 0; r < n; r++)
- {
- const uint s = *pSelectors++;
- const int yd = pInten_table[s];
- // Compute actual delta being applied to each pixel, taking into account clamping.
- delta_sum_r += rg_etc1::clamp<int>(base_color.r + yd, 0, 255) - base_color.r;
- delta_sum_g += rg_etc1::clamp<int>(base_color.g + yd, 0, 255) - base_color.g;
- delta_sum_b += rg_etc1::clamp<int>(base_color.b + yd, 0, 255) - base_color.b;
- }
- if ((!delta_sum_r) && (!delta_sum_g) && (!delta_sum_b))
- break;
- const float avg_delta_r_f = static_cast<float>(delta_sum_r) / n;
- const float avg_delta_g_f = static_cast<float>(delta_sum_g) / n;
- const float avg_delta_b_f = static_cast<float>(delta_sum_b) / n;
- const int br1 = rg_etc1::clamp<int>(static_cast<uint>((m_avg_color[0] - avg_delta_r_f) * m_limit / 255.0f + .5f), 0, m_limit);
- const int bg1 = rg_etc1::clamp<int>(static_cast<uint>((m_avg_color[1] - avg_delta_g_f) * m_limit / 255.0f + .5f), 0, m_limit);
- const int bb1 = rg_etc1::clamp<int>(static_cast<uint>((m_avg_color[2] - avg_delta_b_f) * m_limit / 255.0f + .5f), 0, m_limit);
-
- bool skip = false;
-
- if ((mbr == br1) && (mbg == bg1) && (mbb == bb1))
- skip = true;
- else if ((br1 == m_best_solution.m_coords.m_unscaled_color.r) && (bg1 == m_best_solution.m_coords.m_unscaled_color.g) && (bb1 == m_best_solution.m_coords.m_unscaled_color.b))
- skip = true;
- else if ((m_br == br1) && (m_bg == bg1) && (m_bb == bb1))
- skip = true;
-
- if (skip)
- break;
-
- etc1_solution_coordinates coords1(br1, bg1, bb1, 0, m_pParams->m_use_color4);
- if (m_pParams->m_quality == cHighQuality)
- {
- if (!evaluate_solution(coords1, m_trial_solution, &m_best_solution))
- break;
- }
- else
- {
- if (!evaluate_solution_fast(coords1, m_trial_solution, &m_best_solution))
- break;
- }
-
- } // refinement_trial
-
- } // xdi
- } // ydi
- } // zdi
-
- if (!m_best_solution.m_valid)
- {
- m_pResult->m_error = cUINT32_MAX;
- return false;
- }
-
- const uint8* pSelectors = m_best_solution.m_selectors;
-
-#ifdef RG_ETC1_BUILD_DEBUG
- {
- color_quad_u8 block_colors[4];
- m_best_solution.m_coords.get_block_colors(block_colors);
-
- const color_quad_u8* pSrc_pixels = m_pParams->m_pSrc_pixels;
- uint64 actual_error = 0;
- for (uint i = 0; i < n; i++)
- actual_error += pSrc_pixels[i].squared_distance_rgb(block_colors[pSelectors[i]]);
-
- RG_ETC1_ASSERT(actual_error == m_best_solution.m_error);
- }
-#endif
-
- m_pResult->m_error = m_best_solution.m_error;
-
- m_pResult->m_block_color_unscaled = m_best_solution.m_coords.m_unscaled_color;
- m_pResult->m_block_color4 = m_best_solution.m_coords.m_color4;
-
- m_pResult->m_block_inten_table = m_best_solution.m_coords.m_inten_table;
- memcpy(m_pResult->m_pSelectors, pSelectors, n);
- m_pResult->m_n = n;
-
- return true;
- }
-
- void etc1_optimizer::init(const params& p, results& r)
- {
- // This version is hardcoded for 8 pixel subblocks.
- RG_ETC1_ASSERT(p.m_num_src_pixels == 8);
-
- m_pParams = &p;
- m_pResult = &r;
-
- const uint n = 8;
-
- m_limit = m_pParams->m_use_color4 ? 15 : 31;
-
- vec3F avg_color(0.0f);
-
- for (uint i = 0; i < n; i++)
- {
- const color_quad_u8& c = m_pParams->m_pSrc_pixels[i];
- const vec3F fc(c.r, c.g, c.b);
-
- avg_color += fc;
-
- m_luma[i] = static_cast<uint16>(c.r + c.g + c.b);
- m_sorted_luma[0][i] = i;
- }
- avg_color *= (1.0f / static_cast<float>(n));
- m_avg_color = avg_color;
-
- m_br = rg_etc1::clamp<int>(static_cast<uint>(m_avg_color[0] * m_limit / 255.0f + .5f), 0, m_limit);
- m_bg = rg_etc1::clamp<int>(static_cast<uint>(m_avg_color[1] * m_limit / 255.0f + .5f), 0, m_limit);
- m_bb = rg_etc1::clamp<int>(static_cast<uint>(m_avg_color[2] * m_limit / 255.0f + .5f), 0, m_limit);
-
- if (m_pParams->m_quality <= cMediumQuality)
- {
- m_pSorted_luma_indices = indirect_radix_sort(n, m_sorted_luma[0], m_sorted_luma[1], m_luma, 0, sizeof(m_luma[0]), false);
- m_pSorted_luma = m_sorted_luma[0];
- if (m_pSorted_luma_indices == m_sorted_luma[0])
- m_pSorted_luma = m_sorted_luma[1];
-
- for (uint i = 0; i < n; i++)
- m_pSorted_luma[i] = m_luma[m_pSorted_luma_indices[i]];
- }
-
- m_best_solution.m_coords.clear();
- m_best_solution.m_valid = false;
- m_best_solution.m_error = cUINT64_MAX;
- }
-
- bool etc1_optimizer::evaluate_solution(const etc1_solution_coordinates& coords, potential_solution& trial_solution, potential_solution* pBest_solution)
- {
- trial_solution.m_valid = false;
-
- if (m_pParams->m_constrain_against_base_color5)
- {
- const int dr = coords.m_unscaled_color.r - m_pParams->m_base_color5.r;
- const int dg = coords.m_unscaled_color.g - m_pParams->m_base_color5.g;
- const int db = coords.m_unscaled_color.b - m_pParams->m_base_color5.b;
-
- if ((rg_etc1::minimum(dr, dg, db) < cETC1ColorDeltaMin) || (rg_etc1::maximum(dr, dg, db) > cETC1ColorDeltaMax))
- return false;
- }
-
- const color_quad_u8 base_color(coords.get_scaled_color());
-
- const uint n = 8;
-
- trial_solution.m_error = cUINT64_MAX;
-
- for (uint inten_table = 0; inten_table < cETC1IntenModifierValues; inten_table++)
- {
- const int* pInten_table = g_etc1_inten_tables[inten_table];
-
- color_quad_u8 block_colors[4];
- for (uint s = 0; s < 4; s++)
- {
- const int yd = pInten_table[s];
- block_colors[s].set(base_color.r + yd, base_color.g + yd, base_color.b + yd, 0);
- }
-
- uint64 total_error = 0;
-
- const color_quad_u8* pSrc_pixels = m_pParams->m_pSrc_pixels;
- for (uint c = 0; c < n; c++)
- {
- const color_quad_u8& src_pixel = *pSrc_pixels++;
-
- uint best_selector_index = 0;
- uint best_error = rg_etc1::square(src_pixel.r - block_colors[0].r) + rg_etc1::square(src_pixel.g - block_colors[0].g) + rg_etc1::square(src_pixel.b - block_colors[0].b);
-
- uint trial_error = rg_etc1::square(src_pixel.r - block_colors[1].r) + rg_etc1::square(src_pixel.g - block_colors[1].g) + rg_etc1::square(src_pixel.b - block_colors[1].b);
- if (trial_error < best_error)
- {
- best_error = trial_error;
- best_selector_index = 1;
- }
-
- trial_error = rg_etc1::square(src_pixel.r - block_colors[2].r) + rg_etc1::square(src_pixel.g - block_colors[2].g) + rg_etc1::square(src_pixel.b - block_colors[2].b);
- if (trial_error < best_error)
- {
- best_error = trial_error;
- best_selector_index = 2;
- }
-
- trial_error = rg_etc1::square(src_pixel.r - block_colors[3].r) + rg_etc1::square(src_pixel.g - block_colors[3].g) + rg_etc1::square(src_pixel.b - block_colors[3].b);
- if (trial_error < best_error)
- {
- best_error = trial_error;
- best_selector_index = 3;
- }
-
- m_temp_selectors[c] = static_cast<uint8>(best_selector_index);
-
- total_error += best_error;
- if (total_error >= trial_solution.m_error)
- break;
- }
-
- if (total_error < trial_solution.m_error)
- {
- trial_solution.m_error = total_error;
- trial_solution.m_coords.m_inten_table = inten_table;
- memcpy(trial_solution.m_selectors, m_temp_selectors, 8);
- trial_solution.m_valid = true;
- }
- }
- trial_solution.m_coords.m_unscaled_color = coords.m_unscaled_color;
- trial_solution.m_coords.m_color4 = m_pParams->m_use_color4;
-
- bool success = false;
- if (pBest_solution)
- {
- if (trial_solution.m_error < pBest_solution->m_error)
- {
- *pBest_solution = trial_solution;
- success = true;
- }
- }
-
- return success;
- }
-
- bool etc1_optimizer::evaluate_solution_fast(const etc1_solution_coordinates& coords, potential_solution& trial_solution, potential_solution* pBest_solution)
- {
- if (m_pParams->m_constrain_against_base_color5)
- {
- const int dr = coords.m_unscaled_color.r - m_pParams->m_base_color5.r;
- const int dg = coords.m_unscaled_color.g - m_pParams->m_base_color5.g;
- const int db = coords.m_unscaled_color.b - m_pParams->m_base_color5.b;
-
- if ((rg_etc1::minimum(dr, dg, db) < cETC1ColorDeltaMin) || (rg_etc1::maximum(dr, dg, db) > cETC1ColorDeltaMax))
- {
- trial_solution.m_valid = false;
- return false;
- }
- }
-
- const color_quad_u8 base_color(coords.get_scaled_color());
-
- const uint n = 8;
-
- trial_solution.m_error = cUINT64_MAX;
-
- for (int inten_table = cETC1IntenModifierValues - 1; inten_table >= 0; --inten_table)
- {
- const int* pInten_table = g_etc1_inten_tables[inten_table];
-
- uint block_inten[4];
- color_quad_u8 block_colors[4];
- for (uint s = 0; s < 4; s++)
- {
- const int yd = pInten_table[s];
- color_quad_u8 block_color(base_color.r + yd, base_color.g + yd, base_color.b + yd, 0);
- block_colors[s] = block_color;
- block_inten[s] = block_color.r + block_color.g + block_color.b;
- }
-
- // evaluate_solution_fast() enforces/assumesd a total ordering of the input colors along the intensity (1,1,1) axis to more quickly classify the inputs to selectors.
- // The inputs colors have been presorted along the projection onto this axis, and ETC1 block colors are always ordered along the intensity axis, so this classification is fast.
- // 0 1 2 3
- // 01 12 23
- const uint block_inten_midpoints[3] = { block_inten[0] + block_inten[1], block_inten[1] + block_inten[2], block_inten[2] + block_inten[3] };
-
- uint64 total_error = 0;
- const color_quad_u8* pSrc_pixels = m_pParams->m_pSrc_pixels;
- if ((m_pSorted_luma[n - 1] * 2) < block_inten_midpoints[0])
- {
- if (block_inten[0] > m_pSorted_luma[n - 1])
- {
- const uint min_error = intabs(block_inten[0] - m_pSorted_luma[n - 1]);
- if (min_error >= trial_solution.m_error)
- continue;
- }
-
- memset(&m_temp_selectors[0], 0, n);
-
- for (uint c = 0; c < n; c++)
- total_error += block_colors[0].squared_distance_rgb(pSrc_pixels[c]);
- }
- else if ((m_pSorted_luma[0] * 2) >= block_inten_midpoints[2])
- {
- if (m_pSorted_luma[0] > block_inten[3])
- {
- const uint min_error = intabs(m_pSorted_luma[0] - block_inten[3]);
- if (min_error >= trial_solution.m_error)
- continue;
- }
-
- memset(&m_temp_selectors[0], 3, n);
-
- for (uint c = 0; c < n; c++)
- total_error += block_colors[3].squared_distance_rgb(pSrc_pixels[c]);
- }
- else
- {
- uint cur_selector = 0, c;
- for (c = 0; c < n; c++)
- {
- const uint y = m_pSorted_luma[c];
- while ((y * 2) >= block_inten_midpoints[cur_selector])
- if (++cur_selector > 2)
- goto done;
- const uint sorted_pixel_index = m_pSorted_luma_indices[c];
- m_temp_selectors[sorted_pixel_index] = static_cast<uint8>(cur_selector);
- total_error += block_colors[cur_selector].squared_distance_rgb(pSrc_pixels[sorted_pixel_index]);
- }
-done:
- while (c < n)
- {
- const uint sorted_pixel_index = m_pSorted_luma_indices[c];
- m_temp_selectors[sorted_pixel_index] = 3;
- total_error += block_colors[3].squared_distance_rgb(pSrc_pixels[sorted_pixel_index]);
- ++c;
- }
- }
-
- if (total_error < trial_solution.m_error)
- {
- trial_solution.m_error = total_error;
- trial_solution.m_coords.m_inten_table = inten_table;
- memcpy(trial_solution.m_selectors, m_temp_selectors, n);
- trial_solution.m_valid = true;
- if (!total_error)
- break;
- }
- }
- trial_solution.m_coords.m_unscaled_color = coords.m_unscaled_color;
- trial_solution.m_coords.m_color4 = m_pParams->m_use_color4;
-
- bool success = false;
- if (pBest_solution)
- {
- if (trial_solution.m_error < pBest_solution->m_error)
- {
- *pBest_solution = trial_solution;
- success = true;
- }
- }
-
- return success;
- }
-
- static uint etc1_decode_value(uint diff, uint inten, uint selector, uint packed_c)
- {
- const uint limit = diff ? 32 : 16; limit;
- RG_ETC1_ASSERT((diff < 2) && (inten < 8) && (selector < 4) && (packed_c < limit));
- int c;
- if (diff)
- c = (packed_c >> 2) | (packed_c << 3);
- else
- c = packed_c | (packed_c << 4);
- c += g_etc1_inten_tables[inten][selector];
- c = rg_etc1::clamp<int>(c, 0, 255);
- return c;
- }
-
- static inline int mul_8bit(int a, int b) { int t = a*b + 128; return (t + (t >> 8)) >> 8; }
-
- void pack_etc1_block_init()
- {
- for (uint diff = 0; diff < 2; diff++)
- {
- const uint limit = diff ? 32 : 16;
-
- for (uint inten = 0; inten < 8; inten++)
- {
- for (uint selector = 0; selector < 4; selector++)
- {
- const uint inverse_table_index = diff + (inten << 1) + (selector << 4);
- for (uint color = 0; color < 256; color++)
- {
- uint best_error = cUINT32_MAX, best_packed_c = 0;
- for (uint packed_c = 0; packed_c < limit; packed_c++)
- {
- int v = etc1_decode_value(diff, inten, selector, packed_c);
- uint err = labs(v - static_cast<int>(color));
- //printf("err: %d - %u = %u\n",v,color,err);
- if (err < best_error)
- {
- best_error = err;
- best_packed_c = packed_c;
- if (!best_error)
- break;
- }
- }
- RG_ETC1_ASSERT(best_error <= 255);
- g_etc1_inverse_lookup[inverse_table_index][color] = static_cast<uint16>(best_packed_c | (best_error << 8));
- }
- }
- }
- }
-
- uint expand5[32];
- for(int i = 0; i < 32; i++)
- expand5[i] = (i << 3) | (i >> 2);
-
- for(int i = 0; i < 256 + 16; i++)
- {
- int v = clamp<int>(i - 8, 0, 255);
- g_quant5_tab[i] = static_cast<uint8>(expand5[mul_8bit(v,31)]);
- }
- }
-
- // Packs solid color blocks efficiently using a set of small precomputed tables.
- // For random 888 inputs, MSE results are better than Erricson's ETC1 packer in "slow" mode ~9.5% of the time, is slightly worse only ~.01% of the time, and is equal the rest of the time.
- static uint64 pack_etc1_block_solid_color(etc1_block& block, const uint8* pColor, etc1_pack_params& pack_params)
- {
- pack_params;
- RG_ETC1_ASSERT(g_etc1_inverse_lookup[0][255]);
-
- static uint s_next_comp[4] = { 1, 2, 0, 1 };
-
- uint best_error = cUINT32_MAX, best_i = 0;
- int best_x = 0, best_packed_c1 = 0, best_packed_c2 = 0;
-
- // For each possible 8-bit value, there is a precomputed list of diff/inten/selector configurations that allow that 8-bit value to be encoded with no error.
- for (uint i = 0; i < 3; i++)
- {
- const uint c1 = pColor[s_next_comp[i]], c2 = pColor[s_next_comp[i + 1]];
-
- const int delta_range = 1;
- for (int delta = -delta_range; delta <= delta_range; delta++)
- {
- const int c_plus_delta = rg_etc1::clamp<int>(pColor[i] + delta, 0, 255);
-
- const uint16* pTable;
- if (!c_plus_delta)
- pTable = g_color8_to_etc_block_config_0_255[0];
- else if (c_plus_delta == 255)
- pTable = g_color8_to_etc_block_config_0_255[1];
- else
- pTable = g_color8_to_etc_block_config_1_to_254[c_plus_delta - 1];
-
- do
- {
- const uint x = *pTable++;
-
-#ifdef RG_ETC1_BUILD_DEBUG
- const uint diff = x & 1;
- const uint inten = (x >> 1) & 7;
- const uint selector = (x >> 4) & 3;
- const uint p0 = (x >> 8) & 255;
- RG_ETC1_ASSERT(etc1_decode_value(diff, inten, selector, p0) == (uint)c_plus_delta);
-#endif
-
- const uint16* pInverse_table = g_etc1_inverse_lookup[x & 0xFF];
- uint16 p1 = pInverse_table[c1];
- uint16 p2 = pInverse_table[c2];
- const uint trial_error = rg_etc1::square(c_plus_delta - pColor[i]) + rg_etc1::square(p1 >> 8) + rg_etc1::square(p2 >> 8);
- if (trial_error < best_error)
- {
- best_error = trial_error;
- best_x = x;
- best_packed_c1 = p1 & 0xFF;
- best_packed_c2 = p2 & 0xFF;
- best_i = i;
- if (!best_error)
- goto found_perfect_match;
- }
- } while (*pTable != 0xFFFF);
- }
- }
-found_perfect_match:
-
- const uint diff = best_x & 1;
- const uint inten = (best_x >> 1) & 7;
-
- block.m_bytes[3] = static_cast<uint8>(((inten | (inten << 3)) << 2) | (diff << 1));
-
- const uint etc1_selector = g_selector_index_to_etc1[(best_x >> 4) & 3];
- *reinterpret_cast<uint16*>(&block.m_bytes[4]) = (etc1_selector & 2) ? 0xFFFF : 0;
- *reinterpret_cast<uint16*>(&block.m_bytes[6]) = (etc1_selector & 1) ? 0xFFFF : 0;
-
- const uint best_packed_c0 = (best_x >> 8) & 255;
- if (diff)
- {
- block.m_bytes[best_i] = static_cast<uint8>(best_packed_c0 << 3);
- block.m_bytes[s_next_comp[best_i]] = static_cast<uint8>(best_packed_c1 << 3);
- block.m_bytes[s_next_comp[best_i+1]] = static_cast<uint8>(best_packed_c2 << 3);
- }
- else
- {
- block.m_bytes[best_i] = static_cast<uint8>(best_packed_c0 | (best_packed_c0 << 4));
- block.m_bytes[s_next_comp[best_i]] = static_cast<uint8>(best_packed_c1 | (best_packed_c1 << 4));
- block.m_bytes[s_next_comp[best_i+1]] = static_cast<uint8>(best_packed_c2 | (best_packed_c2 << 4));
- }
-
- return best_error;
- }
-
- static uint pack_etc1_block_solid_color_constrained(
- etc1_optimizer::results& results,
- uint num_colors, const uint8* pColor,
- etc1_pack_params& pack_params,
- bool use_diff,
- const color_quad_u8* pBase_color5_unscaled)
- {
- RG_ETC1_ASSERT(g_etc1_inverse_lookup[0][255]);
-
- pack_params;
- static uint s_next_comp[4] = { 1, 2, 0, 1 };
-
- uint best_error = cUINT32_MAX, best_i = 0;
- int best_x = 0, best_packed_c1 = 0, best_packed_c2 = 0;
-
- // For each possible 8-bit value, there is a precomputed list of diff/inten/selector configurations that allow that 8-bit value to be encoded with no error.
- for (uint i = 0; i < 3; i++)
- {
- const uint c1 = pColor[s_next_comp[i]], c2 = pColor[s_next_comp[i + 1]];
-
- const int delta_range = 1;
- for (int delta = -delta_range; delta <= delta_range; delta++)
- {
- const int c_plus_delta = rg_etc1::clamp<int>(pColor[i] + delta, 0, 255);
-
- const uint16* pTable;
- if (!c_plus_delta)
- pTable = g_color8_to_etc_block_config_0_255[0];
- else if (c_plus_delta == 255)
- pTable = g_color8_to_etc_block_config_0_255[1];
- else
- pTable = g_color8_to_etc_block_config_1_to_254[c_plus_delta - 1];
-
- do
- {
- const uint x = *pTable++;
- const uint diff = x & 1;
- if (static_cast<uint>(use_diff) != diff)
- {
- if (*pTable == 0xFFFF)
- break;
- continue;
- }
-
- if ((diff) && (pBase_color5_unscaled))
- {
- const int p0 = (x >> 8) & 255;
- int delta = p0 - static_cast<int>(pBase_color5_unscaled->c[i]);
- if ((delta < cETC1ColorDeltaMin) || (delta > cETC1ColorDeltaMax))
- {
- if (*pTable == 0xFFFF)
- break;
- continue;
- }
- }
-
-#ifdef RG_ETC1_BUILD_DEBUG
- {
- const uint inten = (x >> 1) & 7;
- const uint selector = (x >> 4) & 3;
- const uint p0 = (x >> 8) & 255;
- RG_ETC1_ASSERT(etc1_decode_value(diff, inten, selector, p0) == (uint)c_plus_delta);
- }
-#endif
-
- const uint16* pInverse_table = g_etc1_inverse_lookup[x & 0xFF];
- uint16 p1 = pInverse_table[c1];
- uint16 p2 = pInverse_table[c2];
-
- if ((diff) && (pBase_color5_unscaled))
- {
- int delta1 = (p1 & 0xFF) - static_cast<int>(pBase_color5_unscaled->c[s_next_comp[i]]);
- int delta2 = (p2 & 0xFF) - static_cast<int>(pBase_color5_unscaled->c[s_next_comp[i + 1]]);
- if ((delta1 < cETC1ColorDeltaMin) || (delta1 > cETC1ColorDeltaMax) || (delta2 < cETC1ColorDeltaMin) || (delta2 > cETC1ColorDeltaMax))
- {
- if (*pTable == 0xFFFF)
- break;
- continue;
- }
- }
-
- const uint trial_error = rg_etc1::square(c_plus_delta - pColor[i]) + rg_etc1::square(p1 >> 8) + rg_etc1::square(p2 >> 8);
- if (trial_error < best_error)
- {
- best_error = trial_error;
- best_x = x;
- best_packed_c1 = p1 & 0xFF;
- best_packed_c2 = p2 & 0xFF;
- best_i = i;
- if (!best_error)
- goto found_perfect_match;
- }
- } while (*pTable != 0xFFFF);
- }
- }
-found_perfect_match:
-
- if (best_error == cUINT32_MAX)
- return best_error;
-
- best_error *= num_colors;
-
- results.m_n = num_colors;
- results.m_block_color4 = !(best_x & 1);
- results.m_block_inten_table = (best_x >> 1) & 7;
- memset(results.m_pSelectors, (best_x >> 4) & 3, num_colors);
-
- const uint best_packed_c0 = (best_x >> 8) & 255;
- results.m_block_color_unscaled[best_i] = static_cast<uint8>(best_packed_c0);
- results.m_block_color_unscaled[s_next_comp[best_i]] = static_cast<uint8>(best_packed_c1);
- results.m_block_color_unscaled[s_next_comp[best_i + 1]] = static_cast<uint8>(best_packed_c2);
- results.m_error = best_error;
-
- return best_error;
- }
-
- // Function originally from RYG's public domain real-time DXT1 compressor, modified for 555.
- static void dither_block_555(color_quad_u8* dest, const color_quad_u8* block)
- {
- int err[8],*ep1 = err,*ep2 = err+4;
- uint8 *quant = g_quant5_tab+8;
-
- memset(dest, 0xFF, sizeof(color_quad_u8)*16);
-
- // process channels seperately
- for(int ch=0;ch<3;ch++)
- {
- uint8* bp = (uint8*)block;
- uint8* dp = (uint8*)dest;
-
- bp += ch; dp += ch;
-
- memset(err,0, sizeof(err));
- for(int y = 0; y < 4; y++)
- {
- // pixel 0
- dp[ 0] = quant[bp[ 0] + ((3*ep2[1] + 5*ep2[0]) >> 4)];
- ep1[0] = bp[ 0] - dp[ 0];
-
- // pixel 1
- dp[ 4] = quant[bp[ 4] + ((7*ep1[0] + 3*ep2[2] + 5*ep2[1] + ep2[0]) >> 4)];
- ep1[1] = bp[ 4] - dp[ 4];
-
- // pixel 2
- dp[ 8] = quant[bp[ 8] + ((7*ep1[1] + 3*ep2[3] + 5*ep2[2] + ep2[1]) >> 4)];
- ep1[2] = bp[ 8] - dp[ 8];
-
- // pixel 3
- dp[12] = quant[bp[12] + ((7*ep1[2] + 5*ep2[3] + ep2[2]) >> 4)];
- ep1[3] = bp[12] - dp[12];
-
- // advance to next line
- int* tmp = ep1; ep1 = ep2; ep2 = tmp;
- bp += 16;
- dp += 16;
- }
- }
- }
-
- unsigned int pack_etc1_block(void* pETC1_block, const unsigned int* pSrc_pixels_rgba, etc1_pack_params& pack_params)
- {
- const color_quad_u8* pSrc_pixels = reinterpret_cast<const color_quad_u8*>(pSrc_pixels_rgba);
- etc1_block& dst_block = *static_cast<etc1_block*>(pETC1_block);
-
-#ifdef RG_ETC1_BUILD_DEBUG
- // Ensure all alpha values are 0xFF.
- for (uint i = 0; i < 16; i++)
- {
- RG_ETC1_ASSERT(pSrc_pixels[i].a == 255);
- }
-#endif
-
- color_quad_u8 src_pixel0(pSrc_pixels[0]);
-
- // Check for solid block.
- const uint32 first_pixel_u32 = pSrc_pixels->m_u32;
- int r;
- for (r = 15; r >= 1; --r)
- if (pSrc_pixels[r].m_u32 != first_pixel_u32)
- break;
- if (!r)
- return static_cast<unsigned int>(16 * pack_etc1_block_solid_color(dst_block, &pSrc_pixels[0].r, pack_params));
-
- color_quad_u8 dithered_pixels[16];
- if (pack_params.m_dithering)
- {
- dither_block_555(dithered_pixels, pSrc_pixels);
- pSrc_pixels = dithered_pixels;
- }
-
- etc1_optimizer optimizer;
-
- uint64 best_error = cUINT64_MAX;
- uint best_flip = false, best_use_color4 = false;
-
- uint8 best_selectors[2][8];
- etc1_optimizer::results best_results[2];
- for (uint i = 0; i < 2; i++)
- {
- best_results[i].m_n = 8;
- best_results[i].m_pSelectors = best_selectors[i];
- }
-
- uint8 selectors[3][8];
- etc1_optimizer::results results[3];
-
- for (uint i = 0; i < 3; i++)
- {
- results[i].m_n = 8;
- results[i].m_pSelectors = selectors[i];
- }
-
- color_quad_u8 subblock_pixels[8];
-
- etc1_optimizer::params params(pack_params);
- params.m_num_src_pixels = 8;
- params.m_pSrc_pixels = subblock_pixels;
-
- for (uint flip = 0; flip < 2; flip++)
- {
- for (uint use_color4 = 0; use_color4 < 2; use_color4++)
- {
- uint64 trial_error = 0;
-
- uint subblock;
- for (subblock = 0; subblock < 2; subblock++)
- {
- if (flip)
- memcpy(subblock_pixels, pSrc_pixels + subblock * 8, sizeof(color_quad_u8) * 8);
- else
- {
- const color_quad_u8* pSrc_col = pSrc_pixels + subblock * 2;
- subblock_pixels[0] = pSrc_col[0]; subblock_pixels[1] = pSrc_col[4]; subblock_pixels[2] = pSrc_col[8]; subblock_pixels[3] = pSrc_col[12];
- subblock_pixels[4] = pSrc_col[1]; subblock_pixels[5] = pSrc_col[5]; subblock_pixels[6] = pSrc_col[9]; subblock_pixels[7] = pSrc_col[13];
- }
-
- results[2].m_error = cUINT64_MAX;
- if ((params.m_quality >= cMediumQuality) && ((subblock) || (use_color4)))
- {
- const uint32 subblock_pixel0_u32 = subblock_pixels[0].m_u32;
- for (r = 7; r >= 1; --r)
- if (subblock_pixels[r].m_u32 != subblock_pixel0_u32)
- break;
- if (!r)
- {
- pack_etc1_block_solid_color_constrained(results[2], 8, &subblock_pixels[0].r, pack_params, !use_color4, (subblock && !use_color4) ? &results[0].m_block_color_unscaled : NULL);
- }
- }
-
- params.m_use_color4 = (use_color4 != 0);
- params.m_constrain_against_base_color5 = false;
-
- if ((!use_color4) && (subblock))
- {
- params.m_constrain_against_base_color5 = true;
- params.m_base_color5 = results[0].m_block_color_unscaled;
- }
-
- if (params.m_quality == cHighQuality)
- {
- static const int s_scan_delta_0_to_4[] = { -4, -3, -2, -1, 0, 1, 2, 3, 4 };
- params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_0_to_4);
- params.m_pScan_deltas = s_scan_delta_0_to_4;
- }
- else if (params.m_quality == cMediumQuality)
- {
- static const int s_scan_delta_0_to_1[] = { -1, 0, 1 };
- params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_0_to_1);
- params.m_pScan_deltas = s_scan_delta_0_to_1;
- }
- else
- {
- static const int s_scan_delta_0[] = { 0 };
- params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_0);
- params.m_pScan_deltas = s_scan_delta_0;
- }
-
- optimizer.init(params, results[subblock]);
- if (!optimizer.compute())
- break;
-
- if (params.m_quality >= cMediumQuality)
- {
- // TODO: Fix fairly arbitrary/unrefined thresholds that control how far away to scan for potentially better solutions.
- const uint refinement_error_thresh0 = 3000;
- const uint refinement_error_thresh1 = 6000;
- if (results[subblock].m_error > refinement_error_thresh0)
- {
- if (params.m_quality == cMediumQuality)
- {
- static const int s_scan_delta_2_to_3[] = { -3, -2, 2, 3 };
- params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_2_to_3);
- params.m_pScan_deltas = s_scan_delta_2_to_3;
- }
- else
- {
- static const int s_scan_delta_5_to_5[] = { -5, 5 };
- static const int s_scan_delta_5_to_8[] = { -8, -7, -6, -5, 5, 6, 7, 8 };
- if (results[subblock].m_error > refinement_error_thresh1)
- {
- params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_5_to_8);
- params.m_pScan_deltas = s_scan_delta_5_to_8;
- }
- else
- {
- params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_5_to_5);
- params.m_pScan_deltas = s_scan_delta_5_to_5;
- }
- }
-
- if (!optimizer.compute())
- break;
- }
-
- if (results[2].m_error < results[subblock].m_error)
- results[subblock] = results[2];
- }
-
- trial_error += results[subblock].m_error;
- if (trial_error >= best_error)
- break;
- }
-
- if (subblock < 2)
- continue;
-
- best_error = trial_error;
- best_results[0] = results[0];
- best_results[1] = results[1];
- best_flip = flip;
- best_use_color4 = use_color4;
-
- } // use_color4
-
- } // flip
-
- int dr = best_results[1].m_block_color_unscaled.r - best_results[0].m_block_color_unscaled.r;
- int dg = best_results[1].m_block_color_unscaled.g - best_results[0].m_block_color_unscaled.g;
- int db = best_results[1].m_block_color_unscaled.b - best_results[0].m_block_color_unscaled.b;
- RG_ETC1_ASSERT(best_use_color4 || ((rg_etc1::minimum(dr, dg, db) >= cETC1ColorDeltaMin) && (rg_etc1::maximum(dr, dg, db) <= cETC1ColorDeltaMax)));
-
- if (best_use_color4)
- {
- dst_block.m_bytes[0] = static_cast<uint8>(best_results[1].m_block_color_unscaled.r | (best_results[0].m_block_color_unscaled.r << 4));
- dst_block.m_bytes[1] = static_cast<uint8>(best_results[1].m_block_color_unscaled.g | (best_results[0].m_block_color_unscaled.g << 4));
- dst_block.m_bytes[2] = static_cast<uint8>(best_results[1].m_block_color_unscaled.b | (best_results[0].m_block_color_unscaled.b << 4));
- }
- else
- {
- if (dr < 0) dr += 8; dst_block.m_bytes[0] = static_cast<uint8>((best_results[0].m_block_color_unscaled.r << 3) | dr);
- if (dg < 0) dg += 8; dst_block.m_bytes[1] = static_cast<uint8>((best_results[0].m_block_color_unscaled.g << 3) | dg);
- if (db < 0) db += 8; dst_block.m_bytes[2] = static_cast<uint8>((best_results[0].m_block_color_unscaled.b << 3) | db);
- }
-
- dst_block.m_bytes[3] = static_cast<uint8>( (best_results[1].m_block_inten_table << 2) | (best_results[0].m_block_inten_table << 5) | ((~best_use_color4 & 1) << 1) | best_flip );
-
- uint selector0 = 0, selector1 = 0;
- if (best_flip)
- {
- // flipped:
- // { 0, 0 }, { 1, 0 }, { 2, 0 }, { 3, 0 },
- // { 0, 1 }, { 1, 1 }, { 2, 1 }, { 3, 1 }
- //
- // { 0, 2 }, { 1, 2 }, { 2, 2 }, { 3, 2 },
- // { 0, 3 }, { 1, 3 }, { 2, 3 }, { 3, 3 }
- const uint8* pSelectors0 = best_results[0].m_pSelectors;
- const uint8* pSelectors1 = best_results[1].m_pSelectors;
- for (int x = 3; x >= 0; --x)
- {
- uint b;
- b = g_selector_index_to_etc1[pSelectors1[4 + x]];
- selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1);
-
- b = g_selector_index_to_etc1[pSelectors1[x]];
- selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1);
-
- b = g_selector_index_to_etc1[pSelectors0[4 + x]];
- selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1);
-
- b = g_selector_index_to_etc1[pSelectors0[x]];
- selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1);
- }
- }
- else
- {
- // non-flipped:
- // { 0, 0 }, { 0, 1 }, { 0, 2 }, { 0, 3 },
- // { 1, 0 }, { 1, 1 }, { 1, 2 }, { 1, 3 }
- //
- // { 2, 0 }, { 2, 1 }, { 2, 2 }, { 2, 3 },
- // { 3, 0 }, { 3, 1 }, { 3, 2 }, { 3, 3 }
- for (int subblock = 1; subblock >= 0; --subblock)
- {
- const uint8* pSelectors = best_results[subblock].m_pSelectors + 4;
- for (uint i = 0; i < 2; i++)
- {
- uint b;
- b = g_selector_index_to_etc1[pSelectors[3]];
- selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1);
-
- b = g_selector_index_to_etc1[pSelectors[2]];
- selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1);
-
- b = g_selector_index_to_etc1[pSelectors[1]];
- selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1);
-
- b = g_selector_index_to_etc1[pSelectors[0]];
- selector0 = (selector0 << 1) | (b & 1);selector1 = (selector1 << 1) | (b >> 1);
-
- pSelectors -= 4;
- }
- }
- }
-
- dst_block.m_bytes[4] = static_cast<uint8>(selector1 >> 8); dst_block.m_bytes[5] = static_cast<uint8>(selector1 & 0xFF);
- dst_block.m_bytes[6] = static_cast<uint8>(selector0 >> 8); dst_block.m_bytes[7] = static_cast<uint8>(selector0 & 0xFF);
-
- return static_cast<unsigned int>(best_error);
- }
-
-} // namespace rg_etc1
+// File: rg_etc1.cpp - Fast, high quality ETC1 block packer/unpacker - Rich Geldreich <richgel99@gmail.com> +// Please see ZLIB license at the end of rg_etc1.h. +// +// For more information Ericsson Texture Compression (ETC/ETC1), see: +// http://www.khronos.org/registry/gles/extensions/OES/OES_compressed_ETC1_RGB8_texture.txt +// +// v1.03 - 5/12/13 - Initial public release +#include "rg_etc1.h" + +#include <stdlib.h> +#include <string.h> +#include <assert.h> +//#include <stdio.h> +#include <math.h> +#include <stdio.h> +#pragma warning (disable: 4201) // nonstandard extension used : nameless struct/union + +#if defined(_DEBUG) || defined(DEBUG) +#define RG_ETC1_BUILD_DEBUG +#endif + +#define RG_ETC1_ASSERT assert + +namespace rg_etc1 +{ + + inline long labs(long val) { + return val < 0 ? -val : val; + } + + inline int intabs(int val) { + + return val<0?-val:val; + } + + typedef unsigned char uint8; + typedef unsigned short uint16; + typedef unsigned int uint; + typedef unsigned int uint32; + typedef long long int64; + typedef unsigned long long uint64; + + const uint32 cUINT32_MAX = 0xFFFFFFFFU; + const uint64 cUINT64_MAX = 0xFFFFFFFFFFFFFFFFULL; //0xFFFFFFFFFFFFFFFFui64; + + template<typename T> inline T minimum(T a, T b) { return (a < b) ? a : b; } + template<typename T> inline T minimum(T a, T b, T c) { return minimum(minimum(a, b), c); } + template<typename T> inline T maximum(T a, T b) { return (a > b) ? a : b; } + template<typename T> inline T maximum(T a, T b, T c) { return maximum(maximum(a, b), c); } + template<typename T> inline T clamp(T value, T low, T high) { return (value < low) ? low : ((value > high) ? high : value); } + template<typename T> inline T square(T value) { return value * value; } + template<typename T> inline void zero_object(T& obj) { memset((void*)&obj, 0, sizeof(obj)); } + template<typename T> inline void zero_this(T* pObj) { memset((void*)pObj, 0, sizeof(*pObj)); } + + template<class T, size_t N> T decay_array_to_subtype(T (&a)[N]); + +#define RG_ETC1_ARRAY_SIZE(X) (sizeof(X) / sizeof(decay_array_to_subtype(X))) + + enum eNoClamp { cNoClamp }; + + struct color_quad_u8 + { + static inline int clamp(int v) { if (v & 0xFFFFFF00U) v = (~(static_cast<int>(v) >> 31)) & 0xFF; return v; } + + struct component_traits { enum { cSigned = false, cFloat = false, cMin = 0U, cMax = 255U }; }; + + public: + typedef unsigned char component_t; + typedef int parameter_t; + + enum { cNumComps = 4 }; + + union + { + struct + { + component_t r; + component_t g; + component_t b; + component_t a; + }; + + component_t c[cNumComps]; + + uint32 m_u32; + }; + + inline color_quad_u8() + { + } + + inline color_quad_u8(const color_quad_u8& other) : m_u32(other.m_u32) + { + } + + explicit inline color_quad_u8(parameter_t y, parameter_t alpha = component_traits::cMax) + { + set(y, alpha); + } + + inline color_quad_u8(parameter_t red, parameter_t green, parameter_t blue, parameter_t alpha = component_traits::cMax) + { + set(red, green, blue, alpha); + } + + explicit inline color_quad_u8(eNoClamp, parameter_t y, parameter_t alpha = component_traits::cMax) + { + set_noclamp_y_alpha(y, alpha); + } + + inline color_quad_u8(eNoClamp, parameter_t red, parameter_t green, parameter_t blue, parameter_t alpha = component_traits::cMax) + { + set_noclamp_rgba(red, green, blue, alpha); + } + + inline void clear() + { + m_u32 = 0; + } + + inline color_quad_u8& operator= (const color_quad_u8& other) + { + m_u32 = other.m_u32; + return *this; + } + + inline color_quad_u8& set_rgb(const color_quad_u8& other) + { + r = other.r; + g = other.g; + b = other.b; + return *this; + } + + inline color_quad_u8& operator= (parameter_t y) + { + set(y, component_traits::cMax); + return *this; + } + + inline color_quad_u8& set(parameter_t y, parameter_t alpha = component_traits::cMax) + { + y = clamp(y); + alpha = clamp(alpha); + r = static_cast<component_t>(y); + g = static_cast<component_t>(y); + b = static_cast<component_t>(y); + a = static_cast<component_t>(alpha); + return *this; + } + + inline color_quad_u8& set_noclamp_y_alpha(parameter_t y, parameter_t alpha = component_traits::cMax) + { + RG_ETC1_ASSERT( (y >= component_traits::cMin) && (y <= component_traits::cMax) ); + RG_ETC1_ASSERT( (alpha >= component_traits::cMin) && (alpha <= component_traits::cMax) ); + + r = static_cast<component_t>(y); + g = static_cast<component_t>(y); + b = static_cast<component_t>(y); + a = static_cast<component_t>(alpha); + return *this; + } + + inline color_quad_u8& set(parameter_t red, parameter_t green, parameter_t blue, parameter_t alpha = component_traits::cMax) + { + r = static_cast<component_t>(clamp(red)); + g = static_cast<component_t>(clamp(green)); + b = static_cast<component_t>(clamp(blue)); + a = static_cast<component_t>(clamp(alpha)); + return *this; + } + + inline color_quad_u8& set_noclamp_rgba(parameter_t red, parameter_t green, parameter_t blue, parameter_t alpha) + { + RG_ETC1_ASSERT( (red >= component_traits::cMin) && (red <= component_traits::cMax) ); + RG_ETC1_ASSERT( (green >= component_traits::cMin) && (green <= component_traits::cMax) ); + RG_ETC1_ASSERT( (blue >= component_traits::cMin) && (blue <= component_traits::cMax) ); + RG_ETC1_ASSERT( (alpha >= component_traits::cMin) && (alpha <= component_traits::cMax) ); + + r = static_cast<component_t>(red); + g = static_cast<component_t>(green); + b = static_cast<component_t>(blue); + a = static_cast<component_t>(alpha); + return *this; + } + + inline color_quad_u8& set_noclamp_rgb(parameter_t red, parameter_t green, parameter_t blue) + { + RG_ETC1_ASSERT( (red >= component_traits::cMin) && (red <= component_traits::cMax) ); + RG_ETC1_ASSERT( (green >= component_traits::cMin) && (green <= component_traits::cMax) ); + RG_ETC1_ASSERT( (blue >= component_traits::cMin) && (blue <= component_traits::cMax) ); + + r = static_cast<component_t>(red); + g = static_cast<component_t>(green); + b = static_cast<component_t>(blue); + return *this; + } + + static inline parameter_t get_min_comp() { return component_traits::cMin; } + static inline parameter_t get_max_comp() { return component_traits::cMax; } + static inline bool get_comps_are_signed() { return component_traits::cSigned; } + + inline component_t operator[] (uint i) const { RG_ETC1_ASSERT(i < cNumComps); return c[i]; } + inline component_t& operator[] (uint i) { RG_ETC1_ASSERT(i < cNumComps); return c[i]; } + + inline color_quad_u8& set_component(uint i, parameter_t f) + { + RG_ETC1_ASSERT(i < cNumComps); + + c[i] = static_cast<component_t>(clamp(f)); + + return *this; + } + + inline color_quad_u8& set_grayscale(parameter_t l) + { + component_t x = static_cast<component_t>(clamp(l)); + c[0] = x; + c[1] = x; + c[2] = x; + return *this; + } + + inline color_quad_u8& clamp(const color_quad_u8& l, const color_quad_u8& h) + { + for (uint i = 0; i < cNumComps; i++) + c[i] = static_cast<component_t>(rg_etc1::clamp<parameter_t>(c[i], l[i], h[i])); + return *this; + } + + inline color_quad_u8& clamp(parameter_t l, parameter_t h) + { + for (uint i = 0; i < cNumComps; i++) + c[i] = static_cast<component_t>(rg_etc1::clamp<parameter_t>(c[i], l, h)); + return *this; + } + + // Returns CCIR 601 luma (consistent with color_utils::RGB_To_Y). + inline parameter_t get_luma() const + { + return static_cast<parameter_t>((19595U * r + 38470U * g + 7471U * b + 32768U) >> 16U); + } + + // Returns REC 709 luma. + inline parameter_t get_luma_rec709() const + { + return static_cast<parameter_t>((13938U * r + 46869U * g + 4729U * b + 32768U) >> 16U); + } + + inline uint squared_distance_rgb(const color_quad_u8& c) const + { + return rg_etc1::square(r - c.r) + rg_etc1::square(g - c.g) + rg_etc1::square(b - c.b); + } + + inline uint squared_distance_rgba(const color_quad_u8& c) const + { + return rg_etc1::square(r - c.r) + rg_etc1::square(g - c.g) + rg_etc1::square(b - c.b) + rg_etc1::square(a - c.a); + } + + inline bool rgb_equals(const color_quad_u8& rhs) const + { + return (r == rhs.r) && (g == rhs.g) && (b == rhs.b); + } + + inline bool operator== (const color_quad_u8& rhs) const + { + return m_u32 == rhs.m_u32; + } + + color_quad_u8& operator+= (const color_quad_u8& other) + { + for (uint i = 0; i < 4; i++) + c[i] = static_cast<component_t>(clamp(c[i] + other.c[i])); + return *this; + } + + color_quad_u8& operator-= (const color_quad_u8& other) + { + for (uint i = 0; i < 4; i++) + c[i] = static_cast<component_t>(clamp(c[i] - other.c[i])); + return *this; + } + + friend color_quad_u8 operator+ (const color_quad_u8& lhs, const color_quad_u8& rhs) + { + color_quad_u8 result(lhs); + result += rhs; + return result; + } + + friend color_quad_u8 operator- (const color_quad_u8& lhs, const color_quad_u8& rhs) + { + color_quad_u8 result(lhs); + result -= rhs; + return result; + } + }; // class color_quad_u8 + + struct vec3F + { + float m_s[3]; + + inline vec3F() { } + inline vec3F(float s) { m_s[0] = s; m_s[1] = s; m_s[2] = s; } + inline vec3F(float x, float y, float z) { m_s[0] = x; m_s[1] = y; m_s[2] = z; } + + inline float operator[] (uint i) const { RG_ETC1_ASSERT(i < 3); return m_s[i]; } + + inline vec3F& operator += (const vec3F& other) { for (uint i = 0; i < 3; i++) m_s[i] += other.m_s[i]; return *this; } + + inline vec3F& operator *= (float s) { for (uint i = 0; i < 3; i++) m_s[i] *= s; return *this; } + }; + + enum etc_constants + { + cETC1BytesPerBlock = 8U, + + cETC1SelectorBits = 2U, + cETC1SelectorValues = 1U << cETC1SelectorBits, + cETC1SelectorMask = cETC1SelectorValues - 1U, + + cETC1BlockShift = 2U, + cETC1BlockSize = 1U << cETC1BlockShift, + + cETC1LSBSelectorIndicesBitOffset = 0, + cETC1MSBSelectorIndicesBitOffset = 16, + + cETC1FlipBitOffset = 32, + cETC1DiffBitOffset = 33, + + cETC1IntenModifierNumBits = 3, + cETC1IntenModifierValues = 1 << cETC1IntenModifierNumBits, + cETC1RightIntenModifierTableBitOffset = 34, + cETC1LeftIntenModifierTableBitOffset = 37, + + // Base+Delta encoding (5 bit bases, 3 bit delta) + cETC1BaseColorCompNumBits = 5, + cETC1BaseColorCompMax = 1 << cETC1BaseColorCompNumBits, + + cETC1DeltaColorCompNumBits = 3, + cETC1DeltaColorComp = 1 << cETC1DeltaColorCompNumBits, + cETC1DeltaColorCompMax = 1 << cETC1DeltaColorCompNumBits, + + cETC1BaseColor5RBitOffset = 59, + cETC1BaseColor5GBitOffset = 51, + cETC1BaseColor5BBitOffset = 43, + + cETC1DeltaColor3RBitOffset = 56, + cETC1DeltaColor3GBitOffset = 48, + cETC1DeltaColor3BBitOffset = 40, + + // Absolute (non-delta) encoding (two 4-bit per component bases) + cETC1AbsColorCompNumBits = 4, + cETC1AbsColorCompMax = 1 << cETC1AbsColorCompNumBits, + + cETC1AbsColor4R1BitOffset = 60, + cETC1AbsColor4G1BitOffset = 52, + cETC1AbsColor4B1BitOffset = 44, + + cETC1AbsColor4R2BitOffset = 56, + cETC1AbsColor4G2BitOffset = 48, + cETC1AbsColor4B2BitOffset = 40, + + cETC1ColorDeltaMin = -4, + cETC1ColorDeltaMax = 3, + + // Delta3: + // 0 1 2 3 4 5 6 7 + // 000 001 010 011 100 101 110 111 + // 0 1 2 3 -4 -3 -2 -1 + }; + + static uint8 g_quant5_tab[256+16]; + + + static const int g_etc1_inten_tables[cETC1IntenModifierValues][cETC1SelectorValues] = + { + { -8, -2, 2, 8 }, { -17, -5, 5, 17 }, { -29, -9, 9, 29 }, { -42, -13, 13, 42 }, + { -60, -18, 18, 60 }, { -80, -24, 24, 80 }, { -106, -33, 33, 106 }, { -183, -47, 47, 183 } + }; + + static const uint8 g_etc1_to_selector_index[cETC1SelectorValues] = { 2, 3, 1, 0 }; + static const uint8 g_selector_index_to_etc1[cETC1SelectorValues] = { 3, 2, 0, 1 }; + + // Given an ETC1 diff/inten_table/selector, and an 8-bit desired color, this table encodes the best packed_color in the low byte, and the abs error in the high byte. + static uint16 g_etc1_inverse_lookup[2*8*4][256]; // [diff/inten_table/selector][desired_color] + + // g_color8_to_etc_block_config[color][table_index] = Supplies for each 8-bit color value a list of packed ETC1 diff/intensity table/selectors/packed_colors that map to that color. + // To pack: diff | (inten << 1) | (selector << 4) | (packed_c << 8) + static const uint16 g_color8_to_etc_block_config_0_255[2][33] = + { + { 0x0000, 0x0010, 0x0002, 0x0012, 0x0004, 0x0014, 0x0006, 0x0016, 0x0008, 0x0018, 0x000A, 0x001A, 0x000C, 0x001C, 0x000E, 0x001E, + 0x0001, 0x0011, 0x0003, 0x0013, 0x0005, 0x0015, 0x0007, 0x0017, 0x0009, 0x0019, 0x000B, 0x001B, 0x000D, 0x001D, 0x000F, 0x001F, 0xFFFF }, + { 0x0F20, 0x0F30, 0x0E32, 0x0F22, 0x0E34, 0x0F24, 0x0D36, 0x0F26, 0x0C38, 0x0E28, 0x0B3A, 0x0E2A, 0x093C, 0x0E2C, 0x053E, 0x0D2E, + 0x1E31, 0x1F21, 0x1D33, 0x1F23, 0x1C35, 0x1E25, 0x1A37, 0x1E27, 0x1839, 0x1D29, 0x163B, 0x1C2B, 0x133D, 0x1B2D, 0x093F, 0x1A2F, 0xFFFF }, + }; + + // Really only [254][11]. + static const uint16 g_color8_to_etc_block_config_1_to_254[254][12] = + { + { 0x021C, 0x0D0D, 0xFFFF }, { 0x0020, 0x0021, 0x0A0B, 0x061F, 0xFFFF }, { 0x0113, 0x0217, 0xFFFF }, { 0x0116, 0x031E, + 0x0B0E, 0x0405, 0xFFFF }, { 0x0022, 0x0204, 0x050A, 0x0023, 0xFFFF }, { 0x0111, 0x0319, 0x0809, 0x170F, 0xFFFF }, { + 0x0303, 0x0215, 0x0607, 0xFFFF }, { 0x0030, 0x0114, 0x0408, 0x0031, 0x0201, 0x051D, 0xFFFF }, { 0x0100, 0x0024, 0x0306, + 0x0025, 0x041B, 0x0E0D, 0xFFFF }, { 0x021A, 0x0121, 0x0B0B, 0x071F, 0xFFFF }, { 0x0213, 0x0317, 0xFFFF }, { 0x0112, + 0x0505, 0xFFFF }, { 0x0026, 0x070C, 0x0123, 0x0027, 0xFFFF }, { 0x0211, 0x0909, 0xFFFF }, { 0x0110, 0x0315, 0x0707, + 0x0419, 0x180F, 0xFFFF }, { 0x0218, 0x0131, 0x0301, 0x0403, 0x061D, 0xFFFF }, { 0x0032, 0x0202, 0x0033, 0x0125, 0x051B, + 0x0F0D, 0xFFFF }, { 0x0028, 0x031C, 0x0221, 0x0029, 0xFFFF }, { 0x0120, 0x0313, 0x0C0B, 0x081F, 0xFFFF }, { 0x0605, + 0x0417, 0xFFFF }, { 0x0216, 0x041E, 0x0C0E, 0x0223, 0x0127, 0xFFFF }, { 0x0122, 0x0304, 0x060A, 0x0311, 0x0A09, 0xFFFF + }, { 0x0519, 0x190F, 0xFFFF }, { 0x002A, 0x0231, 0x0503, 0x0415, 0x0807, 0x002B, 0x071D, 0xFFFF }, { 0x0130, 0x0214, + 0x0508, 0x0401, 0x0133, 0x0225, 0x061B, 0xFFFF }, { 0x0200, 0x0124, 0x0406, 0x0321, 0x0129, 0x100D, 0xFFFF }, { 0x031A, + 0x0D0B, 0x091F, 0xFFFF }, { 0x0413, 0x0705, 0x0517, 0xFFFF }, { 0x0212, 0x0034, 0x0323, 0x0035, 0x0227, 0xFFFF }, { + 0x0126, 0x080C, 0x0B09, 0xFFFF }, { 0x0411, 0x0619, 0x1A0F, 0xFFFF }, { 0x0210, 0x0331, 0x0603, 0x0515, 0x0907, 0x012B, + 0xFFFF }, { 0x0318, 0x002C, 0x0501, 0x0233, 0x0325, 0x071B, 0x002D, 0x081D, 0xFFFF }, { 0x0132, 0x0302, 0x0229, 0x110D, + 0xFFFF }, { 0x0128, 0x041C, 0x0421, 0x0E0B, 0x0A1F, 0xFFFF }, { 0x0220, 0x0513, 0x0617, 0xFFFF }, { 0x0135, 0x0805, + 0x0327, 0xFFFF }, { 0x0316, 0x051E, 0x0D0E, 0x0423, 0xFFFF }, { 0x0222, 0x0404, 0x070A, 0x0511, 0x0719, 0x0C09, 0x1B0F, + 0xFFFF }, { 0x0703, 0x0615, 0x0A07, 0x022B, 0xFFFF }, { 0x012A, 0x0431, 0x0601, 0x0333, 0x012D, 0x091D, 0xFFFF }, { + 0x0230, 0x0314, 0x0036, 0x0608, 0x0425, 0x0037, 0x0329, 0x081B, 0x120D, 0xFFFF }, { 0x0300, 0x0224, 0x0506, 0x0521, + 0x0F0B, 0x0B1F, 0xFFFF }, { 0x041A, 0x0613, 0x0717, 0xFFFF }, { 0x0235, 0x0905, 0xFFFF }, { 0x0312, 0x0134, 0x0523, + 0x0427, 0xFFFF }, { 0x0226, 0x090C, 0x002E, 0x0611, 0x0D09, 0x002F, 0xFFFF }, { 0x0715, 0x0B07, 0x0819, 0x032B, 0x1C0F, + 0xFFFF }, { 0x0310, 0x0531, 0x0701, 0x0803, 0x022D, 0x0A1D, 0xFFFF }, { 0x0418, 0x012C, 0x0433, 0x0525, 0x0137, 0x091B, + 0x130D, 0xFFFF }, { 0x0232, 0x0402, 0x0621, 0x0429, 0xFFFF }, { 0x0228, 0x051C, 0x0713, 0x100B, 0x0C1F, 0xFFFF }, { + 0x0320, 0x0335, 0x0A05, 0x0817, 0xFFFF }, { 0x0623, 0x0527, 0xFFFF }, { 0x0416, 0x061E, 0x0E0E, 0x0711, 0x0E09, 0x012F, + 0xFFFF }, { 0x0322, 0x0504, 0x080A, 0x0919, 0x1D0F, 0xFFFF }, { 0x0631, 0x0903, 0x0815, 0x0C07, 0x042B, 0x032D, 0x0B1D, + 0xFFFF }, { 0x022A, 0x0801, 0x0533, 0x0625, 0x0237, 0x0A1B, 0xFFFF }, { 0x0330, 0x0414, 0x0136, 0x0708, 0x0721, 0x0529, + 0x140D, 0xFFFF }, { 0x0400, 0x0324, 0x0606, 0x0038, 0x0039, 0x110B, 0x0D1F, 0xFFFF }, { 0x051A, 0x0813, 0x0B05, 0x0917, + 0xFFFF }, { 0x0723, 0x0435, 0x0627, 0xFFFF }, { 0x0412, 0x0234, 0x0F09, 0x022F, 0xFFFF }, { 0x0326, 0x0A0C, 0x012E, + 0x0811, 0x0A19, 0x1E0F, 0xFFFF }, { 0x0731, 0x0A03, 0x0915, 0x0D07, 0x052B, 0xFFFF }, { 0x0410, 0x0901, 0x0633, 0x0725, + 0x0337, 0x0B1B, 0x042D, 0x0C1D, 0xFFFF }, { 0x0518, 0x022C, 0x0629, 0x150D, 0xFFFF }, { 0x0332, 0x0502, 0x0821, 0x0139, + 0x120B, 0x0E1F, 0xFFFF }, { 0x0328, 0x061C, 0x0913, 0x0A17, 0xFFFF }, { 0x0420, 0x0535, 0x0C05, 0x0727, 0xFFFF }, { + 0x0823, 0x032F, 0xFFFF }, { 0x0516, 0x071E, 0x0F0E, 0x0911, 0x0B19, 0x1009, 0x1F0F, 0xFFFF }, { 0x0422, 0x0604, 0x090A, + 0x0B03, 0x0A15, 0x0E07, 0x062B, 0xFFFF }, { 0x0831, 0x0A01, 0x0733, 0x052D, 0x0D1D, 0xFFFF }, { 0x032A, 0x0825, 0x0437, + 0x0729, 0x0C1B, 0x160D, 0xFFFF }, { 0x0430, 0x0514, 0x0236, 0x0808, 0x0921, 0x0239, 0x130B, 0x0F1F, 0xFFFF }, { 0x0500, + 0x0424, 0x0706, 0x0138, 0x0A13, 0x0B17, 0xFFFF }, { 0x061A, 0x0635, 0x0D05, 0xFFFF }, { 0x0923, 0x0827, 0xFFFF }, { + 0x0512, 0x0334, 0x003A, 0x0A11, 0x1109, 0x003B, 0x042F, 0xFFFF }, { 0x0426, 0x0B0C, 0x022E, 0x0B15, 0x0F07, 0x0C19, + 0x072B, 0xFFFF }, { 0x0931, 0x0B01, 0x0C03, 0x062D, 0x0E1D, 0xFFFF }, { 0x0510, 0x0833, 0x0925, 0x0537, 0x0D1B, 0x170D, + 0xFFFF }, { 0x0618, 0x032C, 0x0A21, 0x0339, 0x0829, 0xFFFF }, { 0x0432, 0x0602, 0x0B13, 0x140B, 0x101F, 0xFFFF }, { + 0x0428, 0x071C, 0x0735, 0x0E05, 0x0C17, 0xFFFF }, { 0x0520, 0x0A23, 0x0927, 0xFFFF }, { 0x0B11, 0x1209, 0x013B, 0x052F, + 0xFFFF }, { 0x0616, 0x081E, 0x0D19, 0xFFFF }, { 0x0522, 0x0704, 0x0A0A, 0x0A31, 0x0D03, 0x0C15, 0x1007, 0x082B, 0x072D, + 0x0F1D, 0xFFFF }, { 0x0C01, 0x0933, 0x0A25, 0x0637, 0x0E1B, 0xFFFF }, { 0x042A, 0x0B21, 0x0929, 0x180D, 0xFFFF }, { + 0x0530, 0x0614, 0x0336, 0x0908, 0x0439, 0x150B, 0x111F, 0xFFFF }, { 0x0600, 0x0524, 0x0806, 0x0238, 0x0C13, 0x0F05, + 0x0D17, 0xFFFF }, { 0x071A, 0x0B23, 0x0835, 0x0A27, 0xFFFF }, { 0x1309, 0x023B, 0x062F, 0xFFFF }, { 0x0612, 0x0434, + 0x013A, 0x0C11, 0x0E19, 0xFFFF }, { 0x0526, 0x0C0C, 0x032E, 0x0B31, 0x0E03, 0x0D15, 0x1107, 0x092B, 0xFFFF }, { 0x0D01, + 0x0A33, 0x0B25, 0x0737, 0x0F1B, 0x082D, 0x101D, 0xFFFF }, { 0x0610, 0x0A29, 0x190D, 0xFFFF }, { 0x0718, 0x042C, 0x0C21, + 0x0539, 0x160B, 0x121F, 0xFFFF }, { 0x0532, 0x0702, 0x0D13, 0x0E17, 0xFFFF }, { 0x0528, 0x081C, 0x0935, 0x1005, 0x0B27, + 0xFFFF }, { 0x0620, 0x0C23, 0x033B, 0x072F, 0xFFFF }, { 0x0D11, 0x0F19, 0x1409, 0xFFFF }, { 0x0716, 0x003C, 0x091E, + 0x0F03, 0x0E15, 0x1207, 0x0A2B, 0x003D, 0xFFFF }, { 0x0622, 0x0804, 0x0B0A, 0x0C31, 0x0E01, 0x0B33, 0x092D, 0x111D, + 0xFFFF }, { 0x0C25, 0x0837, 0x0B29, 0x101B, 0x1A0D, 0xFFFF }, { 0x052A, 0x0D21, 0x0639, 0x170B, 0x131F, 0xFFFF }, { + 0x0630, 0x0714, 0x0436, 0x0A08, 0x0E13, 0x0F17, 0xFFFF }, { 0x0700, 0x0624, 0x0906, 0x0338, 0x0A35, 0x1105, 0xFFFF }, { + 0x081A, 0x0D23, 0x0C27, 0xFFFF }, { 0x0E11, 0x1509, 0x043B, 0x082F, 0xFFFF }, { 0x0712, 0x0534, 0x023A, 0x0F15, 0x1307, + 0x1019, 0x0B2B, 0x013D, 0xFFFF }, { 0x0626, 0x0D0C, 0x042E, 0x0D31, 0x0F01, 0x1003, 0x0A2D, 0x121D, 0xFFFF }, { 0x0C33, + 0x0D25, 0x0937, 0x111B, 0x1B0D, 0xFFFF }, { 0x0710, 0x0E21, 0x0739, 0x0C29, 0xFFFF }, { 0x0818, 0x052C, 0x0F13, 0x180B, + 0x141F, 0xFFFF }, { 0x0632, 0x0802, 0x0B35, 0x1205, 0x1017, 0xFFFF }, { 0x0628, 0x091C, 0x0E23, 0x0D27, 0xFFFF }, { + 0x0720, 0x0F11, 0x1609, 0x053B, 0x092F, 0xFFFF }, { 0x1119, 0x023D, 0xFFFF }, { 0x0816, 0x013C, 0x0A1E, 0x0E31, 0x1103, + 0x1015, 0x1407, 0x0C2B, 0x0B2D, 0x131D, 0xFFFF }, { 0x0722, 0x0904, 0x0C0A, 0x1001, 0x0D33, 0x0E25, 0x0A37, 0x121B, + 0xFFFF }, { 0x0F21, 0x0D29, 0x1C0D, 0xFFFF }, { 0x062A, 0x0839, 0x190B, 0x151F, 0xFFFF }, { 0x0730, 0x0814, 0x0536, + 0x0B08, 0x1013, 0x1305, 0x1117, 0xFFFF }, { 0x0800, 0x0724, 0x0A06, 0x0438, 0x0F23, 0x0C35, 0x0E27, 0xFFFF }, { 0x091A, + 0x1709, 0x063B, 0x0A2F, 0xFFFF }, { 0x1011, 0x1219, 0x033D, 0xFFFF }, { 0x0812, 0x0634, 0x033A, 0x0F31, 0x1203, 0x1115, + 0x1507, 0x0D2B, 0xFFFF }, { 0x0726, 0x0E0C, 0x052E, 0x1101, 0x0E33, 0x0F25, 0x0B37, 0x131B, 0x0C2D, 0x141D, 0xFFFF }, { + 0x0E29, 0x1D0D, 0xFFFF }, { 0x0810, 0x1021, 0x0939, 0x1A0B, 0x161F, 0xFFFF }, { 0x0918, 0x062C, 0x1113, 0x1217, 0xFFFF + }, { 0x0732, 0x0902, 0x0D35, 0x1405, 0x0F27, 0xFFFF }, { 0x0728, 0x0A1C, 0x1023, 0x073B, 0x0B2F, 0xFFFF }, { 0x0820, + 0x1111, 0x1319, 0x1809, 0xFFFF }, { 0x1303, 0x1215, 0x1607, 0x0E2B, 0x043D, 0xFFFF }, { 0x0916, 0x023C, 0x0B1E, 0x1031, + 0x1201, 0x0F33, 0x0D2D, 0x151D, 0xFFFF }, { 0x0822, 0x0A04, 0x0D0A, 0x1025, 0x0C37, 0x0F29, 0x141B, 0x1E0D, 0xFFFF }, { + 0x1121, 0x0A39, 0x1B0B, 0x171F, 0xFFFF }, { 0x072A, 0x1213, 0x1317, 0xFFFF }, { 0x0830, 0x0914, 0x0636, 0x0C08, 0x0E35, + 0x1505, 0xFFFF }, { 0x0900, 0x0824, 0x0B06, 0x0538, 0x1123, 0x1027, 0xFFFF }, { 0x0A1A, 0x1211, 0x1909, 0x083B, 0x0C2F, + 0xFFFF }, { 0x1315, 0x1707, 0x1419, 0x0F2B, 0x053D, 0xFFFF }, { 0x0912, 0x0734, 0x043A, 0x1131, 0x1301, 0x1403, 0x0E2D, + 0x161D, 0xFFFF }, { 0x0826, 0x0F0C, 0x062E, 0x1033, 0x1125, 0x0D37, 0x151B, 0x1F0D, 0xFFFF }, { 0x1221, 0x0B39, 0x1029, + 0xFFFF }, { 0x0910, 0x1313, 0x1C0B, 0x181F, 0xFFFF }, { 0x0A18, 0x072C, 0x0F35, 0x1605, 0x1417, 0xFFFF }, { 0x0832, + 0x0A02, 0x1223, 0x1127, 0xFFFF }, { 0x0828, 0x0B1C, 0x1311, 0x1A09, 0x093B, 0x0D2F, 0xFFFF }, { 0x0920, 0x1519, 0x063D, + 0xFFFF }, { 0x1231, 0x1503, 0x1415, 0x1807, 0x102B, 0x0F2D, 0x171D, 0xFFFF }, { 0x0A16, 0x033C, 0x0C1E, 0x1401, 0x1133, + 0x1225, 0x0E37, 0x161B, 0xFFFF }, { 0x0922, 0x0B04, 0x0E0A, 0x1321, 0x1129, 0xFFFF }, { 0x0C39, 0x1D0B, 0x191F, 0xFFFF + }, { 0x082A, 0x1413, 0x1705, 0x1517, 0xFFFF }, { 0x0930, 0x0A14, 0x0736, 0x0D08, 0x1323, 0x1035, 0x1227, 0xFFFF }, { + 0x0A00, 0x0924, 0x0C06, 0x0638, 0x1B09, 0x0A3B, 0x0E2F, 0xFFFF }, { 0x0B1A, 0x1411, 0x1619, 0x073D, 0xFFFF }, { 0x1331, + 0x1603, 0x1515, 0x1907, 0x112B, 0xFFFF }, { 0x0A12, 0x0834, 0x053A, 0x1501, 0x1233, 0x1325, 0x0F37, 0x171B, 0x102D, + 0x181D, 0xFFFF }, { 0x0926, 0x072E, 0x1229, 0xFFFF }, { 0x1421, 0x0D39, 0x1E0B, 0x1A1F, 0xFFFF }, { 0x0A10, 0x1513, + 0x1617, 0xFFFF }, { 0x0B18, 0x082C, 0x1135, 0x1805, 0x1327, 0xFFFF }, { 0x0932, 0x0B02, 0x1423, 0x0B3B, 0x0F2F, 0xFFFF + }, { 0x0928, 0x0C1C, 0x1511, 0x1719, 0x1C09, 0xFFFF }, { 0x0A20, 0x1703, 0x1615, 0x1A07, 0x122B, 0x083D, 0xFFFF }, { + 0x1431, 0x1601, 0x1333, 0x112D, 0x191D, 0xFFFF }, { 0x0B16, 0x043C, 0x0D1E, 0x1425, 0x1037, 0x1329, 0x181B, 0xFFFF }, { + 0x0A22, 0x0C04, 0x0F0A, 0x1521, 0x0E39, 0x1F0B, 0x1B1F, 0xFFFF }, { 0x1613, 0x1717, 0xFFFF }, { 0x092A, 0x1235, 0x1905, + 0xFFFF }, { 0x0A30, 0x0B14, 0x0836, 0x0E08, 0x1523, 0x1427, 0xFFFF }, { 0x0B00, 0x0A24, 0x0D06, 0x0738, 0x1611, 0x1D09, + 0x0C3B, 0x102F, 0xFFFF }, { 0x0C1A, 0x1715, 0x1B07, 0x1819, 0x132B, 0x093D, 0xFFFF }, { 0x1531, 0x1701, 0x1803, 0x122D, + 0x1A1D, 0xFFFF }, { 0x0B12, 0x0934, 0x063A, 0x1433, 0x1525, 0x1137, 0x191B, 0xFFFF }, { 0x0A26, 0x003E, 0x082E, 0x1621, + 0x0F39, 0x1429, 0x003F, 0xFFFF }, { 0x1713, 0x1C1F, 0xFFFF }, { 0x0B10, 0x1335, 0x1A05, 0x1817, 0xFFFF }, { 0x0C18, + 0x092C, 0x1623, 0x1527, 0xFFFF }, { 0x0A32, 0x0C02, 0x1711, 0x1E09, 0x0D3B, 0x112F, 0xFFFF }, { 0x0A28, 0x0D1C, 0x1919, + 0x0A3D, 0xFFFF }, { 0x0B20, 0x1631, 0x1903, 0x1815, 0x1C07, 0x142B, 0x132D, 0x1B1D, 0xFFFF }, { 0x1801, 0x1533, 0x1625, + 0x1237, 0x1A1B, 0xFFFF }, { 0x0C16, 0x053C, 0x0E1E, 0x1721, 0x1529, 0x013F, 0xFFFF }, { 0x0B22, 0x0D04, 0x1039, 0x1D1F, + 0xFFFF }, { 0x1813, 0x1B05, 0x1917, 0xFFFF }, { 0x0A2A, 0x1723, 0x1435, 0x1627, 0xFFFF }, { 0x0B30, 0x0C14, 0x0936, + 0x0F08, 0x1F09, 0x0E3B, 0x122F, 0xFFFF }, { 0x0C00, 0x0B24, 0x0E06, 0x0838, 0x1811, 0x1A19, 0x0B3D, 0xFFFF }, { 0x0D1A, + 0x1731, 0x1A03, 0x1915, 0x1D07, 0x152B, 0xFFFF }, { 0x1901, 0x1633, 0x1725, 0x1337, 0x1B1B, 0x142D, 0x1C1D, 0xFFFF }, { + 0x0C12, 0x0A34, 0x073A, 0x1629, 0x023F, 0xFFFF }, { 0x0B26, 0x013E, 0x092E, 0x1821, 0x1139, 0x1E1F, 0xFFFF }, { 0x1913, + 0x1A17, 0xFFFF }, { 0x0C10, 0x1535, 0x1C05, 0x1727, 0xFFFF }, { 0x0D18, 0x0A2C, 0x1823, 0x0F3B, 0x132F, 0xFFFF }, { + 0x0B32, 0x0D02, 0x1911, 0x1B19, 0xFFFF }, { 0x0B28, 0x0E1C, 0x1B03, 0x1A15, 0x1E07, 0x162B, 0x0C3D, 0xFFFF }, { 0x0C20, + 0x1831, 0x1A01, 0x1733, 0x152D, 0x1D1D, 0xFFFF }, { 0x1825, 0x1437, 0x1729, 0x1C1B, 0x033F, 0xFFFF }, { 0x0D16, 0x063C, + 0x0F1E, 0x1921, 0x1239, 0x1F1F, 0xFFFF }, { 0x0C22, 0x0E04, 0x1A13, 0x1B17, 0xFFFF }, { 0x1635, 0x1D05, 0xFFFF }, { + 0x0B2A, 0x1923, 0x1827, 0xFFFF }, { 0x0C30, 0x0D14, 0x0A36, 0x1A11, 0x103B, 0x142F, 0xFFFF }, { 0x0D00, 0x0C24, 0x0F06, + 0x0938, 0x1B15, 0x1F07, 0x1C19, 0x172B, 0x0D3D, 0xFFFF }, { 0x0E1A, 0x1931, 0x1B01, 0x1C03, 0x162D, 0x1E1D, 0xFFFF }, { + 0x1833, 0x1925, 0x1537, 0x1D1B, 0xFFFF }, { 0x0D12, 0x0B34, 0x083A, 0x1A21, 0x1339, 0x1829, 0x043F, 0xFFFF }, { 0x0C26, + 0x023E, 0x0A2E, 0x1B13, 0xFFFF }, { 0x1735, 0x1E05, 0x1C17, 0xFFFF }, { 0x0D10, 0x1A23, 0x1927, 0xFFFF }, { 0x0E18, + 0x0B2C, 0x1B11, 0x113B, 0x152F, 0xFFFF }, { 0x0C32, 0x0E02, 0x1D19, 0x0E3D, 0xFFFF }, { 0x0C28, 0x0F1C, 0x1A31, 0x1D03, + 0x1C15, 0x182B, 0x172D, 0x1F1D, 0xFFFF }, { 0x0D20, 0x1C01, 0x1933, 0x1A25, 0x1637, 0x1E1B, 0xFFFF }, { 0x1B21, 0x1929, + 0x053F, 0xFFFF }, { 0x0E16, 0x073C, 0x1439, 0xFFFF }, { 0x0D22, 0x0F04, 0x1C13, 0x1F05, 0x1D17, 0xFFFF }, { 0x1B23, + 0x1835, 0x1A27, 0xFFFF }, { 0x0C2A, 0x123B, 0x162F, 0xFFFF }, { 0x0D30, 0x0E14, 0x0B36, 0x1C11, 0x1E19, 0x0F3D, 0xFFFF + }, { 0x0E00, 0x0D24, 0x0A38, 0x1B31, 0x1E03, 0x1D15, 0x192B, 0xFFFF }, { 0x0F1A, 0x1D01, 0x1A33, 0x1B25, 0x1737, 0x1F1B, + 0x182D, 0xFFFF }, { 0x1A29, 0x063F, 0xFFFF }, { 0x0E12, 0x0C34, 0x093A, 0x1C21, 0x1539, 0xFFFF }, { 0x0D26, 0x033E, + 0x0B2E, 0x1D13, 0x1E17, 0xFFFF }, { 0x1935, 0x1B27, 0xFFFF }, { 0x0E10, 0x1C23, 0x133B, 0x172F, 0xFFFF }, { 0x0F18, + 0x0C2C, 0x1D11, 0x1F19, 0xFFFF }, { 0x0D32, 0x0F02, 0x1F03, 0x1E15, 0x1A2B, 0x103D, 0xFFFF }, { 0x0D28, 0x1C31, 0x1E01, + 0x1B33, 0x192D, 0xFFFF }, { 0x0E20, 0x1C25, 0x1837, 0x1B29, 0x073F, 0xFFFF }, { 0x1D21, 0x1639, 0xFFFF }, { 0x0F16, + 0x083C, 0x1E13, 0x1F17, 0xFFFF }, { 0x0E22, 0x1A35, 0xFFFF }, { 0x1D23, 0x1C27, 0xFFFF }, { 0x0D2A, 0x1E11, 0x143B, + 0x182F, 0xFFFF }, { 0x0E30, 0x0F14, 0x0C36, 0x1F15, 0x1B2B, 0x113D, 0xFFFF }, { 0x0F00, 0x0E24, 0x0B38, 0x1D31, 0x1F01, + 0x1A2D, 0xFFFF }, { 0x1C33, 0x1D25, 0x1937, 0xFFFF }, { 0x1E21, 0x1739, 0x1C29, 0x083F, 0xFFFF }, { 0x0F12, 0x0D34, + 0x0A3A, 0x1F13, 0xFFFF }, { 0x0E26, 0x043E, 0x0C2E, 0x1B35, 0xFFFF }, { 0x1E23, 0x1D27, 0xFFFF }, { 0x0F10, 0x1F11, + 0x153B, 0x192F, 0xFFFF }, { 0x0D2C, 0x123D, 0xFFFF }, + }; + + struct etc1_block + { + // big endian uint64: + // bit ofs: 56 48 40 32 24 16 8 0 + // byte ofs: b0, b1, b2, b3, b4, b5, b6, b7 + union + { + uint64 m_uint64; + uint8 m_bytes[8]; + }; + + uint8 m_low_color[2]; + uint8 m_high_color[2]; + + enum { cNumSelectorBytes = 4 }; + uint8 m_selectors[cNumSelectorBytes]; + + inline void clear() + { + zero_this(this); + } + + inline uint get_byte_bits(uint ofs, uint num) const + { + RG_ETC1_ASSERT((ofs + num) <= 64U); + RG_ETC1_ASSERT(num && (num <= 8U)); + RG_ETC1_ASSERT((ofs >> 3) == ((ofs + num - 1) >> 3)); + const uint byte_ofs = 7 - (ofs >> 3); + const uint byte_bit_ofs = ofs & 7; + return (m_bytes[byte_ofs] >> byte_bit_ofs) & ((1 << num) - 1); + } + + inline void set_byte_bits(uint ofs, uint num, uint bits) + { + RG_ETC1_ASSERT((ofs + num) <= 64U); + RG_ETC1_ASSERT(num && (num < 32U)); + RG_ETC1_ASSERT((ofs >> 3) == ((ofs + num - 1) >> 3)); + RG_ETC1_ASSERT(bits < (1U << num)); + const uint byte_ofs = 7 - (ofs >> 3); + const uint byte_bit_ofs = ofs & 7; + const uint mask = (1 << num) - 1; + m_bytes[byte_ofs] &= ~(mask << byte_bit_ofs); + m_bytes[byte_ofs] |= (bits << byte_bit_ofs); + } + + // false = left/right subblocks + // true = upper/lower subblocks + inline bool get_flip_bit() const + { + return (m_bytes[3] & 1) != 0; + } + + inline void set_flip_bit(bool flip) + { + m_bytes[3] &= ~1; + m_bytes[3] |= static_cast<uint8>(flip); + } + + inline bool get_diff_bit() const + { + return (m_bytes[3] & 2) != 0; + } + + inline void set_diff_bit(bool diff) + { + m_bytes[3] &= ~2; + m_bytes[3] |= (static_cast<uint>(diff) << 1); + } + + // Returns intensity modifier table (0-7) used by subblock subblock_id. + // subblock_id=0 left/top (CW 1), 1=right/bottom (CW 2) + inline uint get_inten_table(uint subblock_id) const + { + RG_ETC1_ASSERT(subblock_id < 2); + const uint ofs = subblock_id ? 2 : 5; + return (m_bytes[3] >> ofs) & 7; + } + + // Sets intensity modifier table (0-7) used by subblock subblock_id (0 or 1) + inline void set_inten_table(uint subblock_id, uint t) + { + RG_ETC1_ASSERT(subblock_id < 2); + RG_ETC1_ASSERT(t < 8); + const uint ofs = subblock_id ? 2 : 5; + m_bytes[3] &= ~(7 << ofs); + m_bytes[3] |= (t << ofs); + } + + // Returned selector value ranges from 0-3 and is a direct index into g_etc1_inten_tables. + inline uint get_selector(uint x, uint y) const + { + RG_ETC1_ASSERT((x | y) < 4); + + const uint bit_index = x * 4 + y; + const uint byte_bit_ofs = bit_index & 7; + const uint8 *p = &m_bytes[7 - (bit_index >> 3)]; + const uint lsb = (p[0] >> byte_bit_ofs) & 1; + const uint msb = (p[-2] >> byte_bit_ofs) & 1; + const uint val = lsb | (msb << 1); + + return g_etc1_to_selector_index[val]; + } + + // Selector "val" ranges from 0-3 and is a direct index into g_etc1_inten_tables. + inline void set_selector(uint x, uint y, uint val) + { + RG_ETC1_ASSERT((x | y | val) < 4); + const uint bit_index = x * 4 + y; + + uint8 *p = &m_bytes[7 - (bit_index >> 3)]; + + const uint byte_bit_ofs = bit_index & 7; + const uint mask = 1 << byte_bit_ofs; + + const uint etc1_val = g_selector_index_to_etc1[val]; + + const uint lsb = etc1_val & 1; + const uint msb = etc1_val >> 1; + + p[0] &= ~mask; + p[0] |= (lsb << byte_bit_ofs); + + p[-2] &= ~mask; + p[-2] |= (msb << byte_bit_ofs); + } + + inline void set_base4_color(uint idx, uint16 c) + { + if (idx) + { + set_byte_bits(cETC1AbsColor4R2BitOffset, 4, (c >> 8) & 15); + set_byte_bits(cETC1AbsColor4G2BitOffset, 4, (c >> 4) & 15); + set_byte_bits(cETC1AbsColor4B2BitOffset, 4, c & 15); + } + else + { + set_byte_bits(cETC1AbsColor4R1BitOffset, 4, (c >> 8) & 15); + set_byte_bits(cETC1AbsColor4G1BitOffset, 4, (c >> 4) & 15); + set_byte_bits(cETC1AbsColor4B1BitOffset, 4, c & 15); + } + } + + inline uint16 get_base4_color(uint idx) const + { + uint r, g, b; + if (idx) + { + r = get_byte_bits(cETC1AbsColor4R2BitOffset, 4); + g = get_byte_bits(cETC1AbsColor4G2BitOffset, 4); + b = get_byte_bits(cETC1AbsColor4B2BitOffset, 4); + } + else + { + r = get_byte_bits(cETC1AbsColor4R1BitOffset, 4); + g = get_byte_bits(cETC1AbsColor4G1BitOffset, 4); + b = get_byte_bits(cETC1AbsColor4B1BitOffset, 4); + } + return static_cast<uint16>(b | (g << 4U) | (r << 8U)); + } + + inline void set_base5_color(uint16 c) + { + set_byte_bits(cETC1BaseColor5RBitOffset, 5, (c >> 10) & 31); + set_byte_bits(cETC1BaseColor5GBitOffset, 5, (c >> 5) & 31); + set_byte_bits(cETC1BaseColor5BBitOffset, 5, c & 31); + } + + inline uint16 get_base5_color() const + { + const uint r = get_byte_bits(cETC1BaseColor5RBitOffset, 5); + const uint g = get_byte_bits(cETC1BaseColor5GBitOffset, 5); + const uint b = get_byte_bits(cETC1BaseColor5BBitOffset, 5); + return static_cast<uint16>(b | (g << 5U) | (r << 10U)); + } + + void set_delta3_color(uint16 c) + { + set_byte_bits(cETC1DeltaColor3RBitOffset, 3, (c >> 6) & 7); + set_byte_bits(cETC1DeltaColor3GBitOffset, 3, (c >> 3) & 7); + set_byte_bits(cETC1DeltaColor3BBitOffset, 3, c & 7); + } + + inline uint16 get_delta3_color() const + { + const uint r = get_byte_bits(cETC1DeltaColor3RBitOffset, 3); + const uint g = get_byte_bits(cETC1DeltaColor3GBitOffset, 3); + const uint b = get_byte_bits(cETC1DeltaColor3BBitOffset, 3); + return static_cast<uint16>(b | (g << 3U) | (r << 6U)); + } + + // Base color 5 + static uint16 pack_color5(const color_quad_u8& color, bool scaled, uint bias = 127U); + static uint16 pack_color5(uint r, uint g, uint b, bool scaled, uint bias = 127U); + + static color_quad_u8 unpack_color5(uint16 packed_color5, bool scaled, uint alpha = 255U); + static void unpack_color5(uint& r, uint& g, uint& b, uint16 packed_color, bool scaled); + + static bool unpack_color5(color_quad_u8& result, uint16 packed_color5, uint16 packed_delta3, bool scaled, uint alpha = 255U); + static bool unpack_color5(uint& r, uint& g, uint& b, uint16 packed_color5, uint16 packed_delta3, bool scaled, uint alpha = 255U); + + // Delta color 3 + // Inputs range from -4 to 3 (cETC1ColorDeltaMin to cETC1ColorDeltaMax) + static uint16 pack_delta3(int r, int g, int b); + + // Results range from -4 to 3 (cETC1ColorDeltaMin to cETC1ColorDeltaMax) + static void unpack_delta3(int& r, int& g, int& b, uint16 packed_delta3); + + // Abs color 4 + static uint16 pack_color4(const color_quad_u8& color, bool scaled, uint bias = 127U); + static uint16 pack_color4(uint r, uint g, uint b, bool scaled, uint bias = 127U); + + static color_quad_u8 unpack_color4(uint16 packed_color4, bool scaled, uint alpha = 255U); + static void unpack_color4(uint& r, uint& g, uint& b, uint16 packed_color4, bool scaled); + + // subblock colors + static void get_diff_subblock_colors(color_quad_u8* pDst, uint16 packed_color5, uint table_idx); + static bool get_diff_subblock_colors(color_quad_u8* pDst, uint16 packed_color5, uint16 packed_delta3, uint table_idx); + static void get_abs_subblock_colors(color_quad_u8* pDst, uint16 packed_color4, uint table_idx); + + static inline void unscaled_to_scaled_color(color_quad_u8& dst, const color_quad_u8& src, bool color4) + { + if (color4) + { + dst.r = src.r | (src.r << 4); + dst.g = src.g | (src.g << 4); + dst.b = src.b | (src.b << 4); + } + else + { + dst.r = (src.r >> 2) | (src.r << 3); + dst.g = (src.g >> 2) | (src.g << 3); + dst.b = (src.b >> 2) | (src.b << 3); + } + dst.a = src.a; + } + }; + + // Returns pointer to sorted array. + template<typename T, typename Q> + T* indirect_radix_sort(uint num_indices, T* pIndices0, T* pIndices1, const Q* pKeys, uint key_ofs, uint key_size, bool init_indices) + { + RG_ETC1_ASSERT((key_ofs >= 0) && (key_ofs < sizeof(T))); + RG_ETC1_ASSERT((key_size >= 1) && (key_size <= 4)); + + if (init_indices) + { + T* p = pIndices0; + T* q = pIndices0 + (num_indices >> 1) * 2; + uint i; + for (i = 0; p != q; p += 2, i += 2) + { + p[0] = static_cast<T>(i); + p[1] = static_cast<T>(i + 1); + } + + if (num_indices & 1) + *p = static_cast<T>(i); + } + + uint hist[256 * 4]; + + memset(hist, 0, sizeof(hist[0]) * 256 * key_size); + +#define RG_ETC1_GET_KEY(p) (*(const uint*)((const uint8*)(pKeys + *(p)) + key_ofs)) +#define RG_ETC1_GET_KEY_FROM_INDEX(i) (*(const uint*)((const uint8*)(pKeys + (i)) + key_ofs)) + + if (key_size == 4) + { + T* p = pIndices0; + T* q = pIndices0 + num_indices; + for ( ; p != q; p++) + { + const uint key = RG_ETC1_GET_KEY(p); + + hist[ key & 0xFF]++; + hist[256 + ((key >> 8) & 0xFF)]++; + hist[512 + ((key >> 16) & 0xFF)]++; + hist[768 + ((key >> 24) & 0xFF)]++; + } + } + else if (key_size == 3) + { + T* p = pIndices0; + T* q = pIndices0 + num_indices; + for ( ; p != q; p++) + { + const uint key = RG_ETC1_GET_KEY(p); + + hist[ key & 0xFF]++; + hist[256 + ((key >> 8) & 0xFF)]++; + hist[512 + ((key >> 16) & 0xFF)]++; + } + } + else if (key_size == 2) + { + T* p = pIndices0; + T* q = pIndices0 + (num_indices >> 1) * 2; + + for ( ; p != q; p += 2) + { + const uint key0 = RG_ETC1_GET_KEY(p); + const uint key1 = RG_ETC1_GET_KEY(p+1); + + hist[ key0 & 0xFF]++; + hist[256 + ((key0 >> 8) & 0xFF)]++; + + hist[ key1 & 0xFF]++; + hist[256 + ((key1 >> 8) & 0xFF)]++; + } + + if (num_indices & 1) + { + const uint key = RG_ETC1_GET_KEY(p); + + hist[ key & 0xFF]++; + hist[256 + ((key >> 8) & 0xFF)]++; + } + } + else + { + RG_ETC1_ASSERT(key_size == 1); + if (key_size != 1) + return NULL; + + T* p = pIndices0; + T* q = pIndices0 + (num_indices >> 1) * 2; + + for ( ; p != q; p += 2) + { + const uint key0 = RG_ETC1_GET_KEY(p); + const uint key1 = RG_ETC1_GET_KEY(p+1); + + hist[key0 & 0xFF]++; + hist[key1 & 0xFF]++; + } + + if (num_indices & 1) + { + const uint key = RG_ETC1_GET_KEY(p); + + hist[key & 0xFF]++; + } + } + + T* pCur = pIndices0; + T* pNew = pIndices1; + + for (uint pass = 0; pass < key_size; pass++) + { + const uint* pHist = &hist[pass << 8]; + + uint offsets[256]; + + uint cur_ofs = 0; + for (uint i = 0; i < 256; i += 2) + { + offsets[i] = cur_ofs; + cur_ofs += pHist[i]; + + offsets[i+1] = cur_ofs; + cur_ofs += pHist[i+1]; + } + + const uint pass_shift = pass << 3; + + T* p = pCur; + T* q = pCur + (num_indices >> 1) * 2; + + for ( ; p != q; p += 2) + { + uint index0 = p[0]; + uint index1 = p[1]; + + uint c0 = (RG_ETC1_GET_KEY_FROM_INDEX(index0) >> pass_shift) & 0xFF; + uint c1 = (RG_ETC1_GET_KEY_FROM_INDEX(index1) >> pass_shift) & 0xFF; + + if (c0 == c1) + { + uint dst_offset0 = offsets[c0]; + + offsets[c0] = dst_offset0 + 2; + + pNew[dst_offset0] = static_cast<T>(index0); + pNew[dst_offset0 + 1] = static_cast<T>(index1); + } + else + { + uint dst_offset0 = offsets[c0]++; + uint dst_offset1 = offsets[c1]++; + + pNew[dst_offset0] = static_cast<T>(index0); + pNew[dst_offset1] = static_cast<T>(index1); + } + } + + if (num_indices & 1) + { + uint index = *p; + uint c = (RG_ETC1_GET_KEY_FROM_INDEX(index) >> pass_shift) & 0xFF; + + uint dst_offset = offsets[c]; + offsets[c] = dst_offset + 1; + + pNew[dst_offset] = static_cast<T>(index); + } + + T* t = pCur; + pCur = pNew; + pNew = t; + } + + return pCur; + } + +#undef RG_ETC1_GET_KEY +#undef RG_ETC1_GET_KEY_FROM_INDEX + + uint16 etc1_block::pack_color5(const color_quad_u8& color, bool scaled, uint bias) + { + return pack_color5(color.r, color.g, color.b, scaled, bias); + } + + uint16 etc1_block::pack_color5(uint r, uint g, uint b, bool scaled, uint bias) + { + if (scaled) + { + r = (r * 31U + bias) / 255U; + g = (g * 31U + bias) / 255U; + b = (b * 31U + bias) / 255U; + } + + r = rg_etc1::minimum(r, 31U); + g = rg_etc1::minimum(g, 31U); + b = rg_etc1::minimum(b, 31U); + + return static_cast<uint16>(b | (g << 5U) | (r << 10U)); + } + + color_quad_u8 etc1_block::unpack_color5(uint16 packed_color5, bool scaled, uint alpha) + { + uint b = packed_color5 & 31U; + uint g = (packed_color5 >> 5U) & 31U; + uint r = (packed_color5 >> 10U) & 31U; + + if (scaled) + { + b = (b << 3U) | (b >> 2U); + g = (g << 3U) | (g >> 2U); + r = (r << 3U) | (r >> 2U); + } + + return color_quad_u8(cNoClamp, r, g, b, rg_etc1::minimum(alpha, 255U)); + } + + void etc1_block::unpack_color5(uint& r, uint& g, uint& b, uint16 packed_color5, bool scaled) + { + color_quad_u8 c(unpack_color5(packed_color5, scaled, 0)); + r = c.r; + g = c.g; + b = c.b; + } + + bool etc1_block::unpack_color5(color_quad_u8& result, uint16 packed_color5, uint16 packed_delta3, bool scaled, uint alpha) + { + int dc_r, dc_g, dc_b; + unpack_delta3(dc_r, dc_g, dc_b, packed_delta3); + + int b = (packed_color5 & 31U) + dc_b; + int g = ((packed_color5 >> 5U) & 31U) + dc_g; + int r = ((packed_color5 >> 10U) & 31U) + dc_r; + + bool success = true; + if (static_cast<uint>(r | g | b) > 31U) + { + success = false; + r = rg_etc1::clamp<int>(r, 0, 31); + g = rg_etc1::clamp<int>(g, 0, 31); + b = rg_etc1::clamp<int>(b, 0, 31); + } + + if (scaled) + { + b = (b << 3U) | (b >> 2U); + g = (g << 3U) | (g >> 2U); + r = (r << 3U) | (r >> 2U); + } + + result.set_noclamp_rgba(r, g, b, rg_etc1::minimum(alpha, 255U)); + return success; + } + + bool etc1_block::unpack_color5(uint& r, uint& g, uint& b, uint16 packed_color5, uint16 packed_delta3, bool scaled, uint alpha) + { + color_quad_u8 result; + const bool success = unpack_color5(result, packed_color5, packed_delta3, scaled, alpha); + r = result.r; + g = result.g; + b = result.b; + return success; + } + + uint16 etc1_block::pack_delta3(int r, int g, int b) + { + RG_ETC1_ASSERT((r >= cETC1ColorDeltaMin) && (r <= cETC1ColorDeltaMax)); + RG_ETC1_ASSERT((g >= cETC1ColorDeltaMin) && (g <= cETC1ColorDeltaMax)); + RG_ETC1_ASSERT((b >= cETC1ColorDeltaMin) && (b <= cETC1ColorDeltaMax)); + if (r < 0) r += 8; + if (g < 0) g += 8; + if (b < 0) b += 8; + return static_cast<uint16>(b | (g << 3) | (r << 6)); + } + + void etc1_block::unpack_delta3(int& r, int& g, int& b, uint16 packed_delta3) + { + r = (packed_delta3 >> 6) & 7; + g = (packed_delta3 >> 3) & 7; + b = packed_delta3 & 7; + if (r >= 4) r -= 8; + if (g >= 4) g -= 8; + if (b >= 4) b -= 8; + } + + uint16 etc1_block::pack_color4(const color_quad_u8& color, bool scaled, uint bias) + { + return pack_color4(color.r, color.g, color.b, scaled, bias); + } + + uint16 etc1_block::pack_color4(uint r, uint g, uint b, bool scaled, uint bias) + { + if (scaled) + { + r = (r * 15U + bias) / 255U; + g = (g * 15U + bias) / 255U; + b = (b * 15U + bias) / 255U; + } + + r = rg_etc1::minimum(r, 15U); + g = rg_etc1::minimum(g, 15U); + b = rg_etc1::minimum(b, 15U); + + return static_cast<uint16>(b | (g << 4U) | (r << 8U)); + } + + color_quad_u8 etc1_block::unpack_color4(uint16 packed_color4, bool scaled, uint alpha) + { + uint b = packed_color4 & 15U; + uint g = (packed_color4 >> 4U) & 15U; + uint r = (packed_color4 >> 8U) & 15U; + + if (scaled) + { + b = (b << 4U) | b; + g = (g << 4U) | g; + r = (r << 4U) | r; + } + + return color_quad_u8(cNoClamp, r, g, b, rg_etc1::minimum(alpha, 255U)); + } + + void etc1_block::unpack_color4(uint& r, uint& g, uint& b, uint16 packed_color4, bool scaled) + { + color_quad_u8 c(unpack_color4(packed_color4, scaled, 0)); + r = c.r; + g = c.g; + b = c.b; + } + + void etc1_block::get_diff_subblock_colors(color_quad_u8* pDst, uint16 packed_color5, uint table_idx) + { + RG_ETC1_ASSERT(table_idx < cETC1IntenModifierValues); + const int *pInten_modifer_table = &g_etc1_inten_tables[table_idx][0]; + + uint r, g, b; + unpack_color5(r, g, b, packed_color5, true); + + const int ir = static_cast<int>(r), ig = static_cast<int>(g), ib = static_cast<int>(b); + + const int y0 = pInten_modifer_table[0]; + pDst[0].set(ir + y0, ig + y0, ib + y0); + + const int y1 = pInten_modifer_table[1]; + pDst[1].set(ir + y1, ig + y1, ib + y1); + + const int y2 = pInten_modifer_table[2]; + pDst[2].set(ir + y2, ig + y2, ib + y2); + + const int y3 = pInten_modifer_table[3]; + pDst[3].set(ir + y3, ig + y3, ib + y3); + } + + bool etc1_block::get_diff_subblock_colors(color_quad_u8* pDst, uint16 packed_color5, uint16 packed_delta3, uint table_idx) + { + RG_ETC1_ASSERT(table_idx < cETC1IntenModifierValues); + const int *pInten_modifer_table = &g_etc1_inten_tables[table_idx][0]; + + uint r, g, b; + bool success = unpack_color5(r, g, b, packed_color5, packed_delta3, true); + + const int ir = static_cast<int>(r), ig = static_cast<int>(g), ib = static_cast<int>(b); + + const int y0 = pInten_modifer_table[0]; + pDst[0].set(ir + y0, ig + y0, ib + y0); + + const int y1 = pInten_modifer_table[1]; + pDst[1].set(ir + y1, ig + y1, ib + y1); + + const int y2 = pInten_modifer_table[2]; + pDst[2].set(ir + y2, ig + y2, ib + y2); + + const int y3 = pInten_modifer_table[3]; + pDst[3].set(ir + y3, ig + y3, ib + y3); + + return success; + } + + void etc1_block::get_abs_subblock_colors(color_quad_u8* pDst, uint16 packed_color4, uint table_idx) + { + RG_ETC1_ASSERT(table_idx < cETC1IntenModifierValues); + const int *pInten_modifer_table = &g_etc1_inten_tables[table_idx][0]; + + uint r, g, b; + unpack_color4(r, g, b, packed_color4, true); + + const int ir = static_cast<int>(r), ig = static_cast<int>(g), ib = static_cast<int>(b); + + const int y0 = pInten_modifer_table[0]; + pDst[0].set(ir + y0, ig + y0, ib + y0); + + const int y1 = pInten_modifer_table[1]; + pDst[1].set(ir + y1, ig + y1, ib + y1); + + const int y2 = pInten_modifer_table[2]; + pDst[2].set(ir + y2, ig + y2, ib + y2); + + const int y3 = pInten_modifer_table[3]; + pDst[3].set(ir + y3, ig + y3, ib + y3); + } + + bool unpack_etc1_block(const void* pETC1_block, unsigned int* pDst_pixels_rgba, bool preserve_alpha) + { + color_quad_u8* pDst = reinterpret_cast<color_quad_u8*>(pDst_pixels_rgba); + const etc1_block& block = *static_cast<const etc1_block*>(pETC1_block); + + const bool diff_flag = block.get_diff_bit(); + const bool flip_flag = block.get_flip_bit(); + const uint table_index0 = block.get_inten_table(0); + const uint table_index1 = block.get_inten_table(1); + + color_quad_u8 subblock_colors0[4]; + color_quad_u8 subblock_colors1[4]; + bool success = true; + + if (diff_flag) + { + const uint16 base_color5 = block.get_base5_color(); + const uint16 delta_color3 = block.get_delta3_color(); + etc1_block::get_diff_subblock_colors(subblock_colors0, base_color5, table_index0); + + if (!etc1_block::get_diff_subblock_colors(subblock_colors1, base_color5, delta_color3, table_index1)) + success = false; + } + else + { + const uint16 base_color4_0 = block.get_base4_color(0); + etc1_block::get_abs_subblock_colors(subblock_colors0, base_color4_0, table_index0); + + const uint16 base_color4_1 = block.get_base4_color(1); + etc1_block::get_abs_subblock_colors(subblock_colors1, base_color4_1, table_index1); + } + + if (preserve_alpha) + { + if (flip_flag) + { + for (uint y = 0; y < 2; y++) + { + pDst[0].set_rgb(subblock_colors0[block.get_selector(0, y)]); + pDst[1].set_rgb(subblock_colors0[block.get_selector(1, y)]); + pDst[2].set_rgb(subblock_colors0[block.get_selector(2, y)]); + pDst[3].set_rgb(subblock_colors0[block.get_selector(3, y)]); + pDst += 4; + } + + for (uint y = 2; y < 4; y++) + { + pDst[0].set_rgb(subblock_colors1[block.get_selector(0, y)]); + pDst[1].set_rgb(subblock_colors1[block.get_selector(1, y)]); + pDst[2].set_rgb(subblock_colors1[block.get_selector(2, y)]); + pDst[3].set_rgb(subblock_colors1[block.get_selector(3, y)]); + pDst += 4; + } + } + else + { + for (uint y = 0; y < 4; y++) + { + pDst[0].set_rgb(subblock_colors0[block.get_selector(0, y)]); + pDst[1].set_rgb(subblock_colors0[block.get_selector(1, y)]); + pDst[2].set_rgb(subblock_colors1[block.get_selector(2, y)]); + pDst[3].set_rgb(subblock_colors1[block.get_selector(3, y)]); + pDst += 4; + } + } + } + else + { + if (flip_flag) + { + // 0000 + // 0000 + // 1111 + // 1111 + for (uint y = 0; y < 2; y++) + { + pDst[0] = subblock_colors0[block.get_selector(0, y)]; + pDst[1] = subblock_colors0[block.get_selector(1, y)]; + pDst[2] = subblock_colors0[block.get_selector(2, y)]; + pDst[3] = subblock_colors0[block.get_selector(3, y)]; + pDst += 4; + } + + for (uint y = 2; y < 4; y++) + { + pDst[0] = subblock_colors1[block.get_selector(0, y)]; + pDst[1] = subblock_colors1[block.get_selector(1, y)]; + pDst[2] = subblock_colors1[block.get_selector(2, y)]; + pDst[3] = subblock_colors1[block.get_selector(3, y)]; + pDst += 4; + } + } + else + { + // 0011 + // 0011 + // 0011 + // 0011 + for (uint y = 0; y < 4; y++) + { + pDst[0] = subblock_colors0[block.get_selector(0, y)]; + pDst[1] = subblock_colors0[block.get_selector(1, y)]; + pDst[2] = subblock_colors1[block.get_selector(2, y)]; + pDst[3] = subblock_colors1[block.get_selector(3, y)]; + pDst += 4; + } + } + } + + return success; + } + + struct etc1_solution_coordinates + { + inline etc1_solution_coordinates() : + m_unscaled_color(0, 0, 0, 0), + m_inten_table(0), + m_color4(false) + { + } + + inline etc1_solution_coordinates(uint r, uint g, uint b, uint inten_table, bool color4) : + m_unscaled_color(r, g, b, 255), + m_inten_table(inten_table), + m_color4(color4) + { + } + + inline etc1_solution_coordinates(const color_quad_u8& c, uint inten_table, bool color4) : + m_unscaled_color(c), + m_inten_table(inten_table), + m_color4(color4) + { + } + + inline etc1_solution_coordinates(const etc1_solution_coordinates& other) + { + *this = other; + } + + inline etc1_solution_coordinates& operator= (const etc1_solution_coordinates& rhs) + { + m_unscaled_color = rhs.m_unscaled_color; + m_inten_table = rhs.m_inten_table; + m_color4 = rhs.m_color4; + return *this; + } + + inline void clear() + { + m_unscaled_color.clear(); + m_inten_table = 0; + m_color4 = false; + } + + inline color_quad_u8 get_scaled_color() const + { + int br, bg, bb; + if (m_color4) + { + br = m_unscaled_color.r | (m_unscaled_color.r << 4); + bg = m_unscaled_color.g | (m_unscaled_color.g << 4); + bb = m_unscaled_color.b | (m_unscaled_color.b << 4); + } + else + { + br = (m_unscaled_color.r >> 2) | (m_unscaled_color.r << 3); + bg = (m_unscaled_color.g >> 2) | (m_unscaled_color.g << 3); + bb = (m_unscaled_color.b >> 2) | (m_unscaled_color.b << 3); + } + return color_quad_u8(br, bg, bb); + } + + inline void get_block_colors(color_quad_u8* pBlock_colors) + { + int br, bg, bb; + if (m_color4) + { + br = m_unscaled_color.r | (m_unscaled_color.r << 4); + bg = m_unscaled_color.g | (m_unscaled_color.g << 4); + bb = m_unscaled_color.b | (m_unscaled_color.b << 4); + } + else + { + br = (m_unscaled_color.r >> 2) | (m_unscaled_color.r << 3); + bg = (m_unscaled_color.g >> 2) | (m_unscaled_color.g << 3); + bb = (m_unscaled_color.b >> 2) | (m_unscaled_color.b << 3); + } + const int* pInten_table = g_etc1_inten_tables[m_inten_table]; + pBlock_colors[0].set(br + pInten_table[0], bg + pInten_table[0], bb + pInten_table[0]); + pBlock_colors[1].set(br + pInten_table[1], bg + pInten_table[1], bb + pInten_table[1]); + pBlock_colors[2].set(br + pInten_table[2], bg + pInten_table[2], bb + pInten_table[2]); + pBlock_colors[3].set(br + pInten_table[3], bg + pInten_table[3], bb + pInten_table[3]); + } + + color_quad_u8 m_unscaled_color; + uint m_inten_table; + bool m_color4; + }; + + class etc1_optimizer + { + etc1_optimizer(const etc1_optimizer&); + etc1_optimizer& operator= (const etc1_optimizer&); + + public: + etc1_optimizer() + { + clear(); + } + + void clear() + { + m_pParams = NULL; + m_pResult = NULL; + m_pSorted_luma = NULL; + m_pSorted_luma_indices = NULL; + } + + struct params : etc1_pack_params + { + params() + { + clear(); + } + + params(const etc1_pack_params& base_params) : + etc1_pack_params(base_params) + { + clear_optimizer_params(); + } + + void clear() + { + etc1_pack_params::clear(); + clear_optimizer_params(); + } + + void clear_optimizer_params() + { + m_num_src_pixels = 0; + m_pSrc_pixels = 0; + + m_use_color4 = false; + static const int s_default_scan_delta[] = { 0 }; + m_pScan_deltas = s_default_scan_delta; + m_scan_delta_size = 1; + + m_base_color5.clear(); + m_constrain_against_base_color5 = false; + } + + uint m_num_src_pixels; + const color_quad_u8* m_pSrc_pixels; + + bool m_use_color4; + const int* m_pScan_deltas; + uint m_scan_delta_size; + + color_quad_u8 m_base_color5; + bool m_constrain_against_base_color5; + }; + + struct results + { + uint64 m_error; + color_quad_u8 m_block_color_unscaled; + uint m_block_inten_table; + uint m_n; + uint8* m_pSelectors; + bool m_block_color4; + + inline results& operator= (const results& rhs) + { + m_block_color_unscaled = rhs.m_block_color_unscaled; + m_block_color4 = rhs.m_block_color4; + m_block_inten_table = rhs.m_block_inten_table; + m_error = rhs.m_error; + RG_ETC1_ASSERT(m_n == rhs.m_n); + memcpy(m_pSelectors, rhs.m_pSelectors, rhs.m_n); + return *this; + } + }; + + void init(const params& params, results& result); + bool compute(); + + private: + struct potential_solution + { + potential_solution() : m_coords(), m_error(cUINT64_MAX), m_valid(false) + { + } + + etc1_solution_coordinates m_coords; + uint8 m_selectors[8]; + uint64 m_error; + bool m_valid; + + void clear() + { + m_coords.clear(); + m_error = cUINT64_MAX; + m_valid = false; + } + }; + + const params* m_pParams; + results* m_pResult; + + int m_limit; + + vec3F m_avg_color; + int m_br, m_bg, m_bb; + uint16 m_luma[8]; + uint32 m_sorted_luma[2][8]; + const uint32* m_pSorted_luma_indices; + uint32* m_pSorted_luma; + + uint8 m_selectors[8]; + uint8 m_best_selectors[8]; + + potential_solution m_best_solution; + potential_solution m_trial_solution; + uint8 m_temp_selectors[8]; + + bool evaluate_solution(const etc1_solution_coordinates& coords, potential_solution& trial_solution, potential_solution* pBest_solution); + bool evaluate_solution_fast(const etc1_solution_coordinates& coords, potential_solution& trial_solution, potential_solution* pBest_solution); + }; + + bool etc1_optimizer::compute() + { + const uint n = m_pParams->m_num_src_pixels; + const int scan_delta_size = m_pParams->m_scan_delta_size; + + // Scan through a subset of the 3D lattice centered around the avg block color trying each 3D (555 or 444) lattice point as a potential block color. + // Each time a better solution is found try to refine the current solution's block color based of the current selectors and intensity table index. + for (int zdi = 0; zdi < scan_delta_size; zdi++) + { + const int zd = m_pParams->m_pScan_deltas[zdi]; + const int mbb = m_bb + zd; + if (mbb < 0) continue; else if (mbb > m_limit) break; + + for (int ydi = 0; ydi < scan_delta_size; ydi++) + { + const int yd = m_pParams->m_pScan_deltas[ydi]; + const int mbg = m_bg + yd; + if (mbg < 0) continue; else if (mbg > m_limit) break; + + for (int xdi = 0; xdi < scan_delta_size; xdi++) + { + const int xd = m_pParams->m_pScan_deltas[xdi]; + const int mbr = m_br + xd; + if (mbr < 0) continue; else if (mbr > m_limit) break; + + etc1_solution_coordinates coords(mbr, mbg, mbb, 0, m_pParams->m_use_color4); + if (m_pParams->m_quality == cHighQuality) + { + if (!evaluate_solution(coords, m_trial_solution, &m_best_solution)) + continue; + } + else + { + if (!evaluate_solution_fast(coords, m_trial_solution, &m_best_solution)) + continue; + } + + // Now we have the input block, the avg. color of the input pixels, a set of trial selector indices, and the block color+intensity index. + // Now, for each component, attempt to refine the current solution by solving a simple linear equation. For example, for 4 colors: + // The goal is: + // pixel0 - (block_color+inten_table[selector0]) + pixel1 - (block_color+inten_table[selector1]) + pixel2 - (block_color+inten_table[selector2]) + pixel3 - (block_color+inten_table[selector3]) = 0 + // Rearranging this: + // (pixel0 + pixel1 + pixel2 + pixel3) - (block_color+inten_table[selector0]) - (block_color+inten_table[selector1]) - (block_color+inten_table[selector2]) - (block_color+inten_table[selector3]) = 0 + // (pixel0 + pixel1 + pixel2 + pixel3) - block_color - inten_table[selector0] - block_color-inten_table[selector1] - block_color-inten_table[selector2] - block_color-inten_table[selector3] = 0 + // (pixel0 + pixel1 + pixel2 + pixel3) - 4*block_color - inten_table[selector0] - inten_table[selector1] - inten_table[selector2] - inten_table[selector3] = 0 + // (pixel0 + pixel1 + pixel2 + pixel3) - 4*block_color - (inten_table[selector0] + inten_table[selector1] + inten_table[selector2] + inten_table[selector3]) = 0 + // (pixel0 + pixel1 + pixel2 + pixel3)/4 - block_color - (inten_table[selector0] + inten_table[selector1] + inten_table[selector2] + inten_table[selector3])/4 = 0 + // block_color = (pixel0 + pixel1 + pixel2 + pixel3)/4 - (inten_table[selector0] + inten_table[selector1] + inten_table[selector2] + inten_table[selector3])/4 + // So what this means: + // optimal_block_color = avg_input - avg_inten_delta + // So the optimal block color can be computed by taking the average block color and subtracting the current average of the intensity delta. + // Unfortunately, optimal_block_color must then be quantized to 555 or 444 so it's not always possible to improve matters using this formula. + // Also, the above formula is for unclamped intensity deltas. The actual implementation takes into account clamping. + + const uint max_refinement_trials = (m_pParams->m_quality == cLowQuality) ? 2 : (((xd | yd | zd) == 0) ? 4 : 2); + for (uint refinement_trial = 0; refinement_trial < max_refinement_trials; refinement_trial++) + { + const uint8* pSelectors = m_best_solution.m_selectors; + const int* pInten_table = g_etc1_inten_tables[m_best_solution.m_coords.m_inten_table]; + + int delta_sum_r = 0, delta_sum_g = 0, delta_sum_b = 0; + const color_quad_u8 base_color(m_best_solution.m_coords.get_scaled_color()); + for (uint r = 0; r < n; r++) + { + const uint s = *pSelectors++; + const int yd = pInten_table[s]; + // Compute actual delta being applied to each pixel, taking into account clamping. + delta_sum_r += rg_etc1::clamp<int>(base_color.r + yd, 0, 255) - base_color.r; + delta_sum_g += rg_etc1::clamp<int>(base_color.g + yd, 0, 255) - base_color.g; + delta_sum_b += rg_etc1::clamp<int>(base_color.b + yd, 0, 255) - base_color.b; + } + if ((!delta_sum_r) && (!delta_sum_g) && (!delta_sum_b)) + break; + const float avg_delta_r_f = static_cast<float>(delta_sum_r) / n; + const float avg_delta_g_f = static_cast<float>(delta_sum_g) / n; + const float avg_delta_b_f = static_cast<float>(delta_sum_b) / n; + const int br1 = rg_etc1::clamp<int>(static_cast<uint>((m_avg_color[0] - avg_delta_r_f) * m_limit / 255.0f + .5f), 0, m_limit); + const int bg1 = rg_etc1::clamp<int>(static_cast<uint>((m_avg_color[1] - avg_delta_g_f) * m_limit / 255.0f + .5f), 0, m_limit); + const int bb1 = rg_etc1::clamp<int>(static_cast<uint>((m_avg_color[2] - avg_delta_b_f) * m_limit / 255.0f + .5f), 0, m_limit); + + bool skip = false; + + if ((mbr == br1) && (mbg == bg1) && (mbb == bb1)) + skip = true; + else if ((br1 == m_best_solution.m_coords.m_unscaled_color.r) && (bg1 == m_best_solution.m_coords.m_unscaled_color.g) && (bb1 == m_best_solution.m_coords.m_unscaled_color.b)) + skip = true; + else if ((m_br == br1) && (m_bg == bg1) && (m_bb == bb1)) + skip = true; + + if (skip) + break; + + etc1_solution_coordinates coords1(br1, bg1, bb1, 0, m_pParams->m_use_color4); + if (m_pParams->m_quality == cHighQuality) + { + if (!evaluate_solution(coords1, m_trial_solution, &m_best_solution)) + break; + } + else + { + if (!evaluate_solution_fast(coords1, m_trial_solution, &m_best_solution)) + break; + } + + } // refinement_trial + + } // xdi + } // ydi + } // zdi + + if (!m_best_solution.m_valid) + { + m_pResult->m_error = cUINT32_MAX; + return false; + } + + const uint8* pSelectors = m_best_solution.m_selectors; + +#ifdef RG_ETC1_BUILD_DEBUG + { + color_quad_u8 block_colors[4]; + m_best_solution.m_coords.get_block_colors(block_colors); + + const color_quad_u8* pSrc_pixels = m_pParams->m_pSrc_pixels; + uint64 actual_error = 0; + for (uint i = 0; i < n; i++) + actual_error += pSrc_pixels[i].squared_distance_rgb(block_colors[pSelectors[i]]); + + RG_ETC1_ASSERT(actual_error == m_best_solution.m_error); + } +#endif + + m_pResult->m_error = m_best_solution.m_error; + + m_pResult->m_block_color_unscaled = m_best_solution.m_coords.m_unscaled_color; + m_pResult->m_block_color4 = m_best_solution.m_coords.m_color4; + + m_pResult->m_block_inten_table = m_best_solution.m_coords.m_inten_table; + memcpy(m_pResult->m_pSelectors, pSelectors, n); + m_pResult->m_n = n; + + return true; + } + + void etc1_optimizer::init(const params& p, results& r) + { + // This version is hardcoded for 8 pixel subblocks. + RG_ETC1_ASSERT(p.m_num_src_pixels == 8); + + m_pParams = &p; + m_pResult = &r; + + const uint n = 8; + + m_limit = m_pParams->m_use_color4 ? 15 : 31; + + vec3F avg_color(0.0f); + + for (uint i = 0; i < n; i++) + { + const color_quad_u8& c = m_pParams->m_pSrc_pixels[i]; + const vec3F fc(c.r, c.g, c.b); + + avg_color += fc; + + m_luma[i] = static_cast<uint16>(c.r + c.g + c.b); + m_sorted_luma[0][i] = i; + } + avg_color *= (1.0f / static_cast<float>(n)); + m_avg_color = avg_color; + + m_br = rg_etc1::clamp<int>(static_cast<uint>(m_avg_color[0] * m_limit / 255.0f + .5f), 0, m_limit); + m_bg = rg_etc1::clamp<int>(static_cast<uint>(m_avg_color[1] * m_limit / 255.0f + .5f), 0, m_limit); + m_bb = rg_etc1::clamp<int>(static_cast<uint>(m_avg_color[2] * m_limit / 255.0f + .5f), 0, m_limit); + + if (m_pParams->m_quality <= cMediumQuality) + { + m_pSorted_luma_indices = indirect_radix_sort(n, m_sorted_luma[0], m_sorted_luma[1], m_luma, 0, sizeof(m_luma[0]), false); + m_pSorted_luma = m_sorted_luma[0]; + if (m_pSorted_luma_indices == m_sorted_luma[0]) + m_pSorted_luma = m_sorted_luma[1]; + + for (uint i = 0; i < n; i++) + m_pSorted_luma[i] = m_luma[m_pSorted_luma_indices[i]]; + } + + m_best_solution.m_coords.clear(); + m_best_solution.m_valid = false; + m_best_solution.m_error = cUINT64_MAX; + } + + bool etc1_optimizer::evaluate_solution(const etc1_solution_coordinates& coords, potential_solution& trial_solution, potential_solution* pBest_solution) + { + trial_solution.m_valid = false; + + if (m_pParams->m_constrain_against_base_color5) + { + const int dr = coords.m_unscaled_color.r - m_pParams->m_base_color5.r; + const int dg = coords.m_unscaled_color.g - m_pParams->m_base_color5.g; + const int db = coords.m_unscaled_color.b - m_pParams->m_base_color5.b; + + if ((rg_etc1::minimum(dr, dg, db) < cETC1ColorDeltaMin) || (rg_etc1::maximum(dr, dg, db) > cETC1ColorDeltaMax)) + return false; + } + + const color_quad_u8 base_color(coords.get_scaled_color()); + + const uint n = 8; + + trial_solution.m_error = cUINT64_MAX; + + for (uint inten_table = 0; inten_table < cETC1IntenModifierValues; inten_table++) + { + const int* pInten_table = g_etc1_inten_tables[inten_table]; + + color_quad_u8 block_colors[4]; + for (uint s = 0; s < 4; s++) + { + const int yd = pInten_table[s]; + block_colors[s].set(base_color.r + yd, base_color.g + yd, base_color.b + yd, 0); + } + + uint64 total_error = 0; + + const color_quad_u8* pSrc_pixels = m_pParams->m_pSrc_pixels; + for (uint c = 0; c < n; c++) + { + const color_quad_u8& src_pixel = *pSrc_pixels++; + + uint best_selector_index = 0; + uint best_error = rg_etc1::square(src_pixel.r - block_colors[0].r) + rg_etc1::square(src_pixel.g - block_colors[0].g) + rg_etc1::square(src_pixel.b - block_colors[0].b); + + uint trial_error = rg_etc1::square(src_pixel.r - block_colors[1].r) + rg_etc1::square(src_pixel.g - block_colors[1].g) + rg_etc1::square(src_pixel.b - block_colors[1].b); + if (trial_error < best_error) + { + best_error = trial_error; + best_selector_index = 1; + } + + trial_error = rg_etc1::square(src_pixel.r - block_colors[2].r) + rg_etc1::square(src_pixel.g - block_colors[2].g) + rg_etc1::square(src_pixel.b - block_colors[2].b); + if (trial_error < best_error) + { + best_error = trial_error; + best_selector_index = 2; + } + + trial_error = rg_etc1::square(src_pixel.r - block_colors[3].r) + rg_etc1::square(src_pixel.g - block_colors[3].g) + rg_etc1::square(src_pixel.b - block_colors[3].b); + if (trial_error < best_error) + { + best_error = trial_error; + best_selector_index = 3; + } + + m_temp_selectors[c] = static_cast<uint8>(best_selector_index); + + total_error += best_error; + if (total_error >= trial_solution.m_error) + break; + } + + if (total_error < trial_solution.m_error) + { + trial_solution.m_error = total_error; + trial_solution.m_coords.m_inten_table = inten_table; + memcpy(trial_solution.m_selectors, m_temp_selectors, 8); + trial_solution.m_valid = true; + } + } + trial_solution.m_coords.m_unscaled_color = coords.m_unscaled_color; + trial_solution.m_coords.m_color4 = m_pParams->m_use_color4; + + bool success = false; + if (pBest_solution) + { + if (trial_solution.m_error < pBest_solution->m_error) + { + *pBest_solution = trial_solution; + success = true; + } + } + + return success; + } + + bool etc1_optimizer::evaluate_solution_fast(const etc1_solution_coordinates& coords, potential_solution& trial_solution, potential_solution* pBest_solution) + { + if (m_pParams->m_constrain_against_base_color5) + { + const int dr = coords.m_unscaled_color.r - m_pParams->m_base_color5.r; + const int dg = coords.m_unscaled_color.g - m_pParams->m_base_color5.g; + const int db = coords.m_unscaled_color.b - m_pParams->m_base_color5.b; + + if ((rg_etc1::minimum(dr, dg, db) < cETC1ColorDeltaMin) || (rg_etc1::maximum(dr, dg, db) > cETC1ColorDeltaMax)) + { + trial_solution.m_valid = false; + return false; + } + } + + const color_quad_u8 base_color(coords.get_scaled_color()); + + const uint n = 8; + + trial_solution.m_error = cUINT64_MAX; + + for (int inten_table = cETC1IntenModifierValues - 1; inten_table >= 0; --inten_table) + { + const int* pInten_table = g_etc1_inten_tables[inten_table]; + + uint block_inten[4]; + color_quad_u8 block_colors[4]; + for (uint s = 0; s < 4; s++) + { + const int yd = pInten_table[s]; + color_quad_u8 block_color(base_color.r + yd, base_color.g + yd, base_color.b + yd, 0); + block_colors[s] = block_color; + block_inten[s] = block_color.r + block_color.g + block_color.b; + } + + // evaluate_solution_fast() enforces/assumesd a total ordering of the input colors along the intensity (1,1,1) axis to more quickly classify the inputs to selectors. + // The inputs colors have been presorted along the projection onto this axis, and ETC1 block colors are always ordered along the intensity axis, so this classification is fast. + // 0 1 2 3 + // 01 12 23 + const uint block_inten_midpoints[3] = { block_inten[0] + block_inten[1], block_inten[1] + block_inten[2], block_inten[2] + block_inten[3] }; + + uint64 total_error = 0; + const color_quad_u8* pSrc_pixels = m_pParams->m_pSrc_pixels; + if ((m_pSorted_luma[n - 1] * 2) < block_inten_midpoints[0]) + { + if (block_inten[0] > m_pSorted_luma[n - 1]) + { + const uint min_error = intabs(block_inten[0] - m_pSorted_luma[n - 1]); + if (min_error >= trial_solution.m_error) + continue; + } + + memset(&m_temp_selectors[0], 0, n); + + for (uint c = 0; c < n; c++) + total_error += block_colors[0].squared_distance_rgb(pSrc_pixels[c]); + } + else if ((m_pSorted_luma[0] * 2) >= block_inten_midpoints[2]) + { + if (m_pSorted_luma[0] > block_inten[3]) + { + const uint min_error = intabs(m_pSorted_luma[0] - block_inten[3]); + if (min_error >= trial_solution.m_error) + continue; + } + + memset(&m_temp_selectors[0], 3, n); + + for (uint c = 0; c < n; c++) + total_error += block_colors[3].squared_distance_rgb(pSrc_pixels[c]); + } + else + { + uint cur_selector = 0, c; + for (c = 0; c < n; c++) + { + const uint y = m_pSorted_luma[c]; + while ((y * 2) >= block_inten_midpoints[cur_selector]) + if (++cur_selector > 2) + goto done; + const uint sorted_pixel_index = m_pSorted_luma_indices[c]; + m_temp_selectors[sorted_pixel_index] = static_cast<uint8>(cur_selector); + total_error += block_colors[cur_selector].squared_distance_rgb(pSrc_pixels[sorted_pixel_index]); + } +done: + while (c < n) + { + const uint sorted_pixel_index = m_pSorted_luma_indices[c]; + m_temp_selectors[sorted_pixel_index] = 3; + total_error += block_colors[3].squared_distance_rgb(pSrc_pixels[sorted_pixel_index]); + ++c; + } + } + + if (total_error < trial_solution.m_error) + { + trial_solution.m_error = total_error; + trial_solution.m_coords.m_inten_table = inten_table; + memcpy(trial_solution.m_selectors, m_temp_selectors, n); + trial_solution.m_valid = true; + if (!total_error) + break; + } + } + trial_solution.m_coords.m_unscaled_color = coords.m_unscaled_color; + trial_solution.m_coords.m_color4 = m_pParams->m_use_color4; + + bool success = false; + if (pBest_solution) + { + if (trial_solution.m_error < pBest_solution->m_error) + { + *pBest_solution = trial_solution; + success = true; + } + } + + return success; + } + + static uint etc1_decode_value(uint diff, uint inten, uint selector, uint packed_c) + { + const uint limit = diff ? 32 : 16; limit; + RG_ETC1_ASSERT((diff < 2) && (inten < 8) && (selector < 4) && (packed_c < limit)); + int c; + if (diff) + c = (packed_c >> 2) | (packed_c << 3); + else + c = packed_c | (packed_c << 4); + c += g_etc1_inten_tables[inten][selector]; + c = rg_etc1::clamp<int>(c, 0, 255); + return c; + } + + static inline int mul_8bit(int a, int b) { int t = a*b + 128; return (t + (t >> 8)) >> 8; } + + void pack_etc1_block_init() + { + for (uint diff = 0; diff < 2; diff++) + { + const uint limit = diff ? 32 : 16; + + for (uint inten = 0; inten < 8; inten++) + { + for (uint selector = 0; selector < 4; selector++) + { + const uint inverse_table_index = diff + (inten << 1) + (selector << 4); + for (uint color = 0; color < 256; color++) + { + uint best_error = cUINT32_MAX, best_packed_c = 0; + for (uint packed_c = 0; packed_c < limit; packed_c++) + { + int v = etc1_decode_value(diff, inten, selector, packed_c); + uint err = labs(v - static_cast<int>(color)); + //printf("err: %d - %u = %u\n",v,color,err); + if (err < best_error) + { + best_error = err; + best_packed_c = packed_c; + if (!best_error) + break; + } + } + RG_ETC1_ASSERT(best_error <= 255); + g_etc1_inverse_lookup[inverse_table_index][color] = static_cast<uint16>(best_packed_c | (best_error << 8)); + } + } + } + } + + uint expand5[32]; + for(int i = 0; i < 32; i++) + expand5[i] = (i << 3) | (i >> 2); + + for(int i = 0; i < 256 + 16; i++) + { + int v = clamp<int>(i - 8, 0, 255); + g_quant5_tab[i] = static_cast<uint8>(expand5[mul_8bit(v,31)]); + } + } + + // Packs solid color blocks efficiently using a set of small precomputed tables. + // For random 888 inputs, MSE results are better than Erricson's ETC1 packer in "slow" mode ~9.5% of the time, is slightly worse only ~.01% of the time, and is equal the rest of the time. + static uint64 pack_etc1_block_solid_color(etc1_block& block, const uint8* pColor, etc1_pack_params& pack_params) + { + pack_params; + RG_ETC1_ASSERT(g_etc1_inverse_lookup[0][255]); + + static uint s_next_comp[4] = { 1, 2, 0, 1 }; + + uint best_error = cUINT32_MAX, best_i = 0; + int best_x = 0, best_packed_c1 = 0, best_packed_c2 = 0; + + // For each possible 8-bit value, there is a precomputed list of diff/inten/selector configurations that allow that 8-bit value to be encoded with no error. + for (uint i = 0; i < 3; i++) + { + const uint c1 = pColor[s_next_comp[i]], c2 = pColor[s_next_comp[i + 1]]; + + const int delta_range = 1; + for (int delta = -delta_range; delta <= delta_range; delta++) + { + const int c_plus_delta = rg_etc1::clamp<int>(pColor[i] + delta, 0, 255); + + const uint16* pTable; + if (!c_plus_delta) + pTable = g_color8_to_etc_block_config_0_255[0]; + else if (c_plus_delta == 255) + pTable = g_color8_to_etc_block_config_0_255[1]; + else + pTable = g_color8_to_etc_block_config_1_to_254[c_plus_delta - 1]; + + do + { + const uint x = *pTable++; + +#ifdef RG_ETC1_BUILD_DEBUG + const uint diff = x & 1; + const uint inten = (x >> 1) & 7; + const uint selector = (x >> 4) & 3; + const uint p0 = (x >> 8) & 255; + RG_ETC1_ASSERT(etc1_decode_value(diff, inten, selector, p0) == (uint)c_plus_delta); +#endif + + const uint16* pInverse_table = g_etc1_inverse_lookup[x & 0xFF]; + uint16 p1 = pInverse_table[c1]; + uint16 p2 = pInverse_table[c2]; + const uint trial_error = rg_etc1::square(c_plus_delta - pColor[i]) + rg_etc1::square(p1 >> 8) + rg_etc1::square(p2 >> 8); + if (trial_error < best_error) + { + best_error = trial_error; + best_x = x; + best_packed_c1 = p1 & 0xFF; + best_packed_c2 = p2 & 0xFF; + best_i = i; + if (!best_error) + goto found_perfect_match; + } + } while (*pTable != 0xFFFF); + } + } +found_perfect_match: + + const uint diff = best_x & 1; + const uint inten = (best_x >> 1) & 7; + + block.m_bytes[3] = static_cast<uint8>(((inten | (inten << 3)) << 2) | (diff << 1)); + + const uint etc1_selector = g_selector_index_to_etc1[(best_x >> 4) & 3]; + *reinterpret_cast<uint16*>(&block.m_bytes[4]) = (etc1_selector & 2) ? 0xFFFF : 0; + *reinterpret_cast<uint16*>(&block.m_bytes[6]) = (etc1_selector & 1) ? 0xFFFF : 0; + + const uint best_packed_c0 = (best_x >> 8) & 255; + if (diff) + { + block.m_bytes[best_i] = static_cast<uint8>(best_packed_c0 << 3); + block.m_bytes[s_next_comp[best_i]] = static_cast<uint8>(best_packed_c1 << 3); + block.m_bytes[s_next_comp[best_i+1]] = static_cast<uint8>(best_packed_c2 << 3); + } + else + { + block.m_bytes[best_i] = static_cast<uint8>(best_packed_c0 | (best_packed_c0 << 4)); + block.m_bytes[s_next_comp[best_i]] = static_cast<uint8>(best_packed_c1 | (best_packed_c1 << 4)); + block.m_bytes[s_next_comp[best_i+1]] = static_cast<uint8>(best_packed_c2 | (best_packed_c2 << 4)); + } + + return best_error; + } + + static uint pack_etc1_block_solid_color_constrained( + etc1_optimizer::results& results, + uint num_colors, const uint8* pColor, + etc1_pack_params& pack_params, + bool use_diff, + const color_quad_u8* pBase_color5_unscaled) + { + RG_ETC1_ASSERT(g_etc1_inverse_lookup[0][255]); + + pack_params; + static uint s_next_comp[4] = { 1, 2, 0, 1 }; + + uint best_error = cUINT32_MAX, best_i = 0; + int best_x = 0, best_packed_c1 = 0, best_packed_c2 = 0; + + // For each possible 8-bit value, there is a precomputed list of diff/inten/selector configurations that allow that 8-bit value to be encoded with no error. + for (uint i = 0; i < 3; i++) + { + const uint c1 = pColor[s_next_comp[i]], c2 = pColor[s_next_comp[i + 1]]; + + const int delta_range = 1; + for (int delta = -delta_range; delta <= delta_range; delta++) + { + const int c_plus_delta = rg_etc1::clamp<int>(pColor[i] + delta, 0, 255); + + const uint16* pTable; + if (!c_plus_delta) + pTable = g_color8_to_etc_block_config_0_255[0]; + else if (c_plus_delta == 255) + pTable = g_color8_to_etc_block_config_0_255[1]; + else + pTable = g_color8_to_etc_block_config_1_to_254[c_plus_delta - 1]; + + do + { + const uint x = *pTable++; + const uint diff = x & 1; + if (static_cast<uint>(use_diff) != diff) + { + if (*pTable == 0xFFFF) + break; + continue; + } + + if ((diff) && (pBase_color5_unscaled)) + { + const int p0 = (x >> 8) & 255; + int delta = p0 - static_cast<int>(pBase_color5_unscaled->c[i]); + if ((delta < cETC1ColorDeltaMin) || (delta > cETC1ColorDeltaMax)) + { + if (*pTable == 0xFFFF) + break; + continue; + } + } + +#ifdef RG_ETC1_BUILD_DEBUG + { + const uint inten = (x >> 1) & 7; + const uint selector = (x >> 4) & 3; + const uint p0 = (x >> 8) & 255; + RG_ETC1_ASSERT(etc1_decode_value(diff, inten, selector, p0) == (uint)c_plus_delta); + } +#endif + + const uint16* pInverse_table = g_etc1_inverse_lookup[x & 0xFF]; + uint16 p1 = pInverse_table[c1]; + uint16 p2 = pInverse_table[c2]; + + if ((diff) && (pBase_color5_unscaled)) + { + int delta1 = (p1 & 0xFF) - static_cast<int>(pBase_color5_unscaled->c[s_next_comp[i]]); + int delta2 = (p2 & 0xFF) - static_cast<int>(pBase_color5_unscaled->c[s_next_comp[i + 1]]); + if ((delta1 < cETC1ColorDeltaMin) || (delta1 > cETC1ColorDeltaMax) || (delta2 < cETC1ColorDeltaMin) || (delta2 > cETC1ColorDeltaMax)) + { + if (*pTable == 0xFFFF) + break; + continue; + } + } + + const uint trial_error = rg_etc1::square(c_plus_delta - pColor[i]) + rg_etc1::square(p1 >> 8) + rg_etc1::square(p2 >> 8); + if (trial_error < best_error) + { + best_error = trial_error; + best_x = x; + best_packed_c1 = p1 & 0xFF; + best_packed_c2 = p2 & 0xFF; + best_i = i; + if (!best_error) + goto found_perfect_match; + } + } while (*pTable != 0xFFFF); + } + } +found_perfect_match: + + if (best_error == cUINT32_MAX) + return best_error; + + best_error *= num_colors; + + results.m_n = num_colors; + results.m_block_color4 = !(best_x & 1); + results.m_block_inten_table = (best_x >> 1) & 7; + memset(results.m_pSelectors, (best_x >> 4) & 3, num_colors); + + const uint best_packed_c0 = (best_x >> 8) & 255; + results.m_block_color_unscaled[best_i] = static_cast<uint8>(best_packed_c0); + results.m_block_color_unscaled[s_next_comp[best_i]] = static_cast<uint8>(best_packed_c1); + results.m_block_color_unscaled[s_next_comp[best_i + 1]] = static_cast<uint8>(best_packed_c2); + results.m_error = best_error; + + return best_error; + } + + // Function originally from RYG's public domain real-time DXT1 compressor, modified for 555. + static void dither_block_555(color_quad_u8* dest, const color_quad_u8* block) + { + int err[8],*ep1 = err,*ep2 = err+4; + uint8 *quant = g_quant5_tab+8; + + memset(dest, 0xFF, sizeof(color_quad_u8)*16); + + // process channels seperately + for(int ch=0;ch<3;ch++) + { + uint8* bp = (uint8*)block; + uint8* dp = (uint8*)dest; + + bp += ch; dp += ch; + + memset(err,0, sizeof(err)); + for(int y = 0; y < 4; y++) + { + // pixel 0 + dp[ 0] = quant[bp[ 0] + ((3*ep2[1] + 5*ep2[0]) >> 4)]; + ep1[0] = bp[ 0] - dp[ 0]; + + // pixel 1 + dp[ 4] = quant[bp[ 4] + ((7*ep1[0] + 3*ep2[2] + 5*ep2[1] + ep2[0]) >> 4)]; + ep1[1] = bp[ 4] - dp[ 4]; + + // pixel 2 + dp[ 8] = quant[bp[ 8] + ((7*ep1[1] + 3*ep2[3] + 5*ep2[2] + ep2[1]) >> 4)]; + ep1[2] = bp[ 8] - dp[ 8]; + + // pixel 3 + dp[12] = quant[bp[12] + ((7*ep1[2] + 5*ep2[3] + ep2[2]) >> 4)]; + ep1[3] = bp[12] - dp[12]; + + // advance to next line + int* tmp = ep1; ep1 = ep2; ep2 = tmp; + bp += 16; + dp += 16; + } + } + } + + unsigned int pack_etc1_block(void* pETC1_block, const unsigned int* pSrc_pixels_rgba, etc1_pack_params& pack_params) + { + const color_quad_u8* pSrc_pixels = reinterpret_cast<const color_quad_u8*>(pSrc_pixels_rgba); + etc1_block& dst_block = *static_cast<etc1_block*>(pETC1_block); + +#ifdef RG_ETC1_BUILD_DEBUG + // Ensure all alpha values are 0xFF. + for (uint i = 0; i < 16; i++) + { + RG_ETC1_ASSERT(pSrc_pixels[i].a == 255); + } +#endif + + color_quad_u8 src_pixel0(pSrc_pixels[0]); + + // Check for solid block. + const uint32 first_pixel_u32 = pSrc_pixels->m_u32; + int r; + for (r = 15; r >= 1; --r) + if (pSrc_pixels[r].m_u32 != first_pixel_u32) + break; + if (!r) + return static_cast<unsigned int>(16 * pack_etc1_block_solid_color(dst_block, &pSrc_pixels[0].r, pack_params)); + + color_quad_u8 dithered_pixels[16]; + if (pack_params.m_dithering) + { + dither_block_555(dithered_pixels, pSrc_pixels); + pSrc_pixels = dithered_pixels; + } + + etc1_optimizer optimizer; + + uint64 best_error = cUINT64_MAX; + uint best_flip = false, best_use_color4 = false; + + uint8 best_selectors[2][8]; + etc1_optimizer::results best_results[2]; + for (uint i = 0; i < 2; i++) + { + best_results[i].m_n = 8; + best_results[i].m_pSelectors = best_selectors[i]; + } + + uint8 selectors[3][8]; + etc1_optimizer::results results[3]; + + for (uint i = 0; i < 3; i++) + { + results[i].m_n = 8; + results[i].m_pSelectors = selectors[i]; + } + + color_quad_u8 subblock_pixels[8]; + + etc1_optimizer::params params(pack_params); + params.m_num_src_pixels = 8; + params.m_pSrc_pixels = subblock_pixels; + + for (uint flip = 0; flip < 2; flip++) + { + for (uint use_color4 = 0; use_color4 < 2; use_color4++) + { + uint64 trial_error = 0; + + uint subblock; + for (subblock = 0; subblock < 2; subblock++) + { + if (flip) + memcpy(subblock_pixels, pSrc_pixels + subblock * 8, sizeof(color_quad_u8) * 8); + else + { + const color_quad_u8* pSrc_col = pSrc_pixels + subblock * 2; + subblock_pixels[0] = pSrc_col[0]; subblock_pixels[1] = pSrc_col[4]; subblock_pixels[2] = pSrc_col[8]; subblock_pixels[3] = pSrc_col[12]; + subblock_pixels[4] = pSrc_col[1]; subblock_pixels[5] = pSrc_col[5]; subblock_pixels[6] = pSrc_col[9]; subblock_pixels[7] = pSrc_col[13]; + } + + results[2].m_error = cUINT64_MAX; + if ((params.m_quality >= cMediumQuality) && ((subblock) || (use_color4))) + { + const uint32 subblock_pixel0_u32 = subblock_pixels[0].m_u32; + for (r = 7; r >= 1; --r) + if (subblock_pixels[r].m_u32 != subblock_pixel0_u32) + break; + if (!r) + { + pack_etc1_block_solid_color_constrained(results[2], 8, &subblock_pixels[0].r, pack_params, !use_color4, (subblock && !use_color4) ? &results[0].m_block_color_unscaled : NULL); + } + } + + params.m_use_color4 = (use_color4 != 0); + params.m_constrain_against_base_color5 = false; + + if ((!use_color4) && (subblock)) + { + params.m_constrain_against_base_color5 = true; + params.m_base_color5 = results[0].m_block_color_unscaled; + } + + if (params.m_quality == cHighQuality) + { + static const int s_scan_delta_0_to_4[] = { -4, -3, -2, -1, 0, 1, 2, 3, 4 }; + params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_0_to_4); + params.m_pScan_deltas = s_scan_delta_0_to_4; + } + else if (params.m_quality == cMediumQuality) + { + static const int s_scan_delta_0_to_1[] = { -1, 0, 1 }; + params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_0_to_1); + params.m_pScan_deltas = s_scan_delta_0_to_1; + } + else + { + static const int s_scan_delta_0[] = { 0 }; + params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_0); + params.m_pScan_deltas = s_scan_delta_0; + } + + optimizer.init(params, results[subblock]); + if (!optimizer.compute()) + break; + + if (params.m_quality >= cMediumQuality) + { + // TODO: Fix fairly arbitrary/unrefined thresholds that control how far away to scan for potentially better solutions. + const uint refinement_error_thresh0 = 3000; + const uint refinement_error_thresh1 = 6000; + if (results[subblock].m_error > refinement_error_thresh0) + { + if (params.m_quality == cMediumQuality) + { + static const int s_scan_delta_2_to_3[] = { -3, -2, 2, 3 }; + params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_2_to_3); + params.m_pScan_deltas = s_scan_delta_2_to_3; + } + else + { + static const int s_scan_delta_5_to_5[] = { -5, 5 }; + static const int s_scan_delta_5_to_8[] = { -8, -7, -6, -5, 5, 6, 7, 8 }; + if (results[subblock].m_error > refinement_error_thresh1) + { + params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_5_to_8); + params.m_pScan_deltas = s_scan_delta_5_to_8; + } + else + { + params.m_scan_delta_size = RG_ETC1_ARRAY_SIZE(s_scan_delta_5_to_5); + params.m_pScan_deltas = s_scan_delta_5_to_5; + } + } + + if (!optimizer.compute()) + break; + } + + if (results[2].m_error < results[subblock].m_error) + results[subblock] = results[2]; + } + + trial_error += results[subblock].m_error; + if (trial_error >= best_error) + break; + } + + if (subblock < 2) + continue; + + best_error = trial_error; + best_results[0] = results[0]; + best_results[1] = results[1]; + best_flip = flip; + best_use_color4 = use_color4; + + } // use_color4 + + } // flip + + int dr = best_results[1].m_block_color_unscaled.r - best_results[0].m_block_color_unscaled.r; + int dg = best_results[1].m_block_color_unscaled.g - best_results[0].m_block_color_unscaled.g; + int db = best_results[1].m_block_color_unscaled.b - best_results[0].m_block_color_unscaled.b; + RG_ETC1_ASSERT(best_use_color4 || ((rg_etc1::minimum(dr, dg, db) >= cETC1ColorDeltaMin) && (rg_etc1::maximum(dr, dg, db) <= cETC1ColorDeltaMax))); + + if (best_use_color4) + { + dst_block.m_bytes[0] = static_cast<uint8>(best_results[1].m_block_color_unscaled.r | (best_results[0].m_block_color_unscaled.r << 4)); + dst_block.m_bytes[1] = static_cast<uint8>(best_results[1].m_block_color_unscaled.g | (best_results[0].m_block_color_unscaled.g << 4)); + dst_block.m_bytes[2] = static_cast<uint8>(best_results[1].m_block_color_unscaled.b | (best_results[0].m_block_color_unscaled.b << 4)); + } + else + { + if (dr < 0) dr += 8; dst_block.m_bytes[0] = static_cast<uint8>((best_results[0].m_block_color_unscaled.r << 3) | dr); + if (dg < 0) dg += 8; dst_block.m_bytes[1] = static_cast<uint8>((best_results[0].m_block_color_unscaled.g << 3) | dg); + if (db < 0) db += 8; dst_block.m_bytes[2] = static_cast<uint8>((best_results[0].m_block_color_unscaled.b << 3) | db); + } + + dst_block.m_bytes[3] = static_cast<uint8>( (best_results[1].m_block_inten_table << 2) | (best_results[0].m_block_inten_table << 5) | ((~best_use_color4 & 1) << 1) | best_flip ); + + uint selector0 = 0, selector1 = 0; + if (best_flip) + { + // flipped: + // { 0, 0 }, { 1, 0 }, { 2, 0 }, { 3, 0 }, + // { 0, 1 }, { 1, 1 }, { 2, 1 }, { 3, 1 } + // + // { 0, 2 }, { 1, 2 }, { 2, 2 }, { 3, 2 }, + // { 0, 3 }, { 1, 3 }, { 2, 3 }, { 3, 3 } + const uint8* pSelectors0 = best_results[0].m_pSelectors; + const uint8* pSelectors1 = best_results[1].m_pSelectors; + for (int x = 3; x >= 0; --x) + { + uint b; + b = g_selector_index_to_etc1[pSelectors1[4 + x]]; + selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1); + + b = g_selector_index_to_etc1[pSelectors1[x]]; + selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1); + + b = g_selector_index_to_etc1[pSelectors0[4 + x]]; + selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1); + + b = g_selector_index_to_etc1[pSelectors0[x]]; + selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1); + } + } + else + { + // non-flipped: + // { 0, 0 }, { 0, 1 }, { 0, 2 }, { 0, 3 }, + // { 1, 0 }, { 1, 1 }, { 1, 2 }, { 1, 3 } + // + // { 2, 0 }, { 2, 1 }, { 2, 2 }, { 2, 3 }, + // { 3, 0 }, { 3, 1 }, { 3, 2 }, { 3, 3 } + for (int subblock = 1; subblock >= 0; --subblock) + { + const uint8* pSelectors = best_results[subblock].m_pSelectors + 4; + for (uint i = 0; i < 2; i++) + { + uint b; + b = g_selector_index_to_etc1[pSelectors[3]]; + selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1); + + b = g_selector_index_to_etc1[pSelectors[2]]; + selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1); + + b = g_selector_index_to_etc1[pSelectors[1]]; + selector0 = (selector0 << 1) | (b & 1); selector1 = (selector1 << 1) | (b >> 1); + + b = g_selector_index_to_etc1[pSelectors[0]]; + selector0 = (selector0 << 1) | (b & 1);selector1 = (selector1 << 1) | (b >> 1); + + pSelectors -= 4; + } + } + } + + dst_block.m_bytes[4] = static_cast<uint8>(selector1 >> 8); dst_block.m_bytes[5] = static_cast<uint8>(selector1 & 0xFF); + dst_block.m_bytes[6] = static_cast<uint8>(selector0 >> 8); dst_block.m_bytes[7] = static_cast<uint8>(selector0 & 0xFF); + + return static_cast<unsigned int>(best_error); + } + +} // namespace rg_etc1 diff --git a/drivers/etc1/rg_etc1.h b/drivers/etc1/rg_etc1.h index 9a701506fd..9ce89a6cc6 100644 --- a/drivers/etc1/rg_etc1.h +++ b/drivers/etc1/rg_etc1.h @@ -1,76 +1,76 @@ -// File: rg_etc1.h - Fast, high quality ETC1 block packer/unpacker - Rich Geldreich <richgel99@gmail.com>
-// Please see ZLIB license at the end of this file.
-#pragma once
-
-namespace rg_etc1
-{
- // Unpacks an 8-byte ETC1 compressed block to a block of 4x4 32bpp RGBA pixels.
- // Returns false if the block is invalid. Invalid blocks will still be unpacked with clamping.
- // This function is thread safe, and does not dynamically allocate any memory.
- // If preserve_alpha is true, the alpha channel of the destination pixels will not be overwritten. Otherwise, alpha will be set to 255.
- bool unpack_etc1_block(const void *pETC1_block, unsigned int* pDst_pixels_rgba, bool preserve_alpha = false);
-
- // Quality setting = the higher the quality, the slower.
- // To pack large textures, it is highly recommended to call pack_etc1_block() in parallel, on different blocks, from multiple threads (particularly when using cHighQuality).
- enum etc1_quality
- {
- cLowQuality,
- cMediumQuality,
- cHighQuality,
- };
-
- struct etc1_pack_params
- {
- etc1_quality m_quality;
- bool m_dithering;
-
- inline etc1_pack_params()
- {
- clear();
- }
-
- void clear()
- {
- m_quality = cHighQuality;
- m_dithering = false;
- }
- };
-
- // Important: pack_etc1_block_init() must be called before calling pack_etc1_block().
- void pack_etc1_block_init();
-
- // Packs a 4x4 block of 32bpp RGBA pixels to an 8-byte ETC1 block.
- // 32-bit RGBA pixels must always be arranged as (R,G,B,A) (R first, A last) in memory, independent of platform endianness. A should always be 255.
- // Returns squared error of result.
- // This function is thread safe, and does not dynamically allocate any memory.
- // pack_etc1_block() does not currently support "perceptual" colorspace metrics - it primarily optimizes for RGB RMSE.
- unsigned int pack_etc1_block(void* pETC1_block, const unsigned int* pSrc_pixels_rgba, etc1_pack_params& pack_params);
-
-} // namespace rg_etc1
-
-//------------------------------------------------------------------------------
-//
-// rg_etc1 uses the ZLIB license:
-// http://opensource.org/licenses/Zlib
-//
-// Copyright (c) 2012 Rich Geldreich
-//
-// This software is provided 'as-is', without any express or implied
-// warranty. In no event will the authors be held liable for any damages
-// arising from the use of this software.
-//
-// Permission is granted to anyone to use this software for any purpose,
-// including commercial applications, and to alter it and redistribute it
-// freely, subject to the following restrictions:
-//
-// 1. The origin of this software must not be misrepresented; you must not
-// claim that you wrote the original software. If you use this software
-// in a product, an acknowledgment in the product documentation would be
-// appreciated but is not required.
-//
-// 2. Altered source versions must be plainly marked as such, and must not be
-// misrepresented as being the original software.
-//
-// 3. This notice may not be removed or altered from any source distribution.
-//
-//------------------------------------------------------------------------------
+// File: rg_etc1.h - Fast, high quality ETC1 block packer/unpacker - Rich Geldreich <richgel99@gmail.com> +// Please see ZLIB license at the end of this file. +#pragma once + +namespace rg_etc1 +{ + // Unpacks an 8-byte ETC1 compressed block to a block of 4x4 32bpp RGBA pixels. + // Returns false if the block is invalid. Invalid blocks will still be unpacked with clamping. + // This function is thread safe, and does not dynamically allocate any memory. + // If preserve_alpha is true, the alpha channel of the destination pixels will not be overwritten. Otherwise, alpha will be set to 255. + bool unpack_etc1_block(const void *pETC1_block, unsigned int* pDst_pixels_rgba, bool preserve_alpha = false); + + // Quality setting = the higher the quality, the slower. + // To pack large textures, it is highly recommended to call pack_etc1_block() in parallel, on different blocks, from multiple threads (particularly when using cHighQuality). + enum etc1_quality + { + cLowQuality, + cMediumQuality, + cHighQuality, + }; + + struct etc1_pack_params + { + etc1_quality m_quality; + bool m_dithering; + + inline etc1_pack_params() + { + clear(); + } + + void clear() + { + m_quality = cHighQuality; + m_dithering = false; + } + }; + + // Important: pack_etc1_block_init() must be called before calling pack_etc1_block(). + void pack_etc1_block_init(); + + // Packs a 4x4 block of 32bpp RGBA pixels to an 8-byte ETC1 block. + // 32-bit RGBA pixels must always be arranged as (R,G,B,A) (R first, A last) in memory, independent of platform endianness. A should always be 255. + // Returns squared error of result. + // This function is thread safe, and does not dynamically allocate any memory. + // pack_etc1_block() does not currently support "perceptual" colorspace metrics - it primarily optimizes for RGB RMSE. + unsigned int pack_etc1_block(void* pETC1_block, const unsigned int* pSrc_pixels_rgba, etc1_pack_params& pack_params); + +} // namespace rg_etc1 + +//------------------------------------------------------------------------------ +// +// rg_etc1 uses the ZLIB license: +// http://opensource.org/licenses/Zlib +// +// Copyright (c) 2012 Rich Geldreich +// +// This software is provided 'as-is', without any express or implied +// warranty. In no event will the authors be held liable for any damages +// arising from the use of this software. +// +// Permission is granted to anyone to use this software for any purpose, +// including commercial applications, and to alter it and redistribute it +// freely, subject to the following restrictions: +// +// 1. The origin of this software must not be misrepresented; you must not +// claim that you wrote the original software. If you use this software +// in a product, an acknowledgment in the product documentation would be +// appreciated but is not required. +// +// 2. Altered source versions must be plainly marked as such, and must not be +// misrepresented as being the original software. +// +// 3. This notice may not be removed or altered from any source distribution. +// +//------------------------------------------------------------------------------ diff --git a/drivers/nedmalloc/malloc.c.h b/drivers/nedmalloc/malloc.c.h index b9e65637d5..4fec5cc9d4 100644 --- a/drivers/nedmalloc/malloc.c.h +++ b/drivers/nedmalloc/malloc.c.h @@ -1,5814 +1,5814 @@ -#ifdef NEDMALLOC_ENABLED
-/*
- This is a version (aka dlmalloc) of malloc/free/realloc written by
- Doug Lea and released to the public domain, as explained at
- http://creativecommons.org/licenses/publicdomain. Send questions,
- comments, complaints, performance data, etc to dl@cs.oswego.edu
-
-* Version 2.8.4 Wed May 27 09:56:23 2009 Doug Lea (dl at gee)
-
- Note: There may be an updated version of this malloc obtainable at
- ftp://gee.cs.oswego.edu/pub/misc/malloc.c
- Check before installing!
-
-* Quickstart
-
- This library is all in one file to simplify the most common usage:
- ftp it, compile it (-O3), and link it into another program. All of
- the compile-time options default to reasonable values for use on
- most platforms. You might later want to step through various
- compile-time and dynamic tuning options.
-
- For convenience, an include file for code using this malloc is at:
- ftp://gee.cs.oswego.edu/pub/misc/malloc-2.8.4.h
- You don't really need this .h file unless you call functions not
- defined in your system include files. The .h file contains only the
- excerpts from this file needed for using this malloc on ANSI C/C++
- systems, so long as you haven't changed compile-time options about
- naming and tuning parameters. If you do, then you can create your
- own malloc.h that does include all settings by cutting at the point
- indicated below. Note that you may already by default be using a C
- library containing a malloc that is based on some version of this
- malloc (for example in linux). You might still want to use the one
- in this file to customize settings or to avoid overheads associated
- with library versions.
-
-* Vital statistics:
-
- Supported pointer/size_t representation: 4 or 8 bytes
- size_t MUST be an unsigned type of the same width as
- pointers. (If you are using an ancient system that declares
- size_t as a signed type, or need it to be a different width
- than pointers, you can use a previous release of this malloc
- (e.g. 2.7.2) supporting these.)
-
- Alignment: 8 bytes (default)
- This suffices for nearly all current machines and C compilers.
- However, you can define MALLOC_ALIGNMENT to be wider than this
- if necessary (up to 128bytes), at the expense of using more space.
-
- Minimum overhead per allocated chunk: 4 or 8 bytes (if 4byte sizes)
- 8 or 16 bytes (if 8byte sizes)
- Each malloced chunk has a hidden word of overhead holding size
- and status information, and additional cross-check word
- if FOOTERS is defined.
-
- Minimum allocated size: 4-byte ptrs: 16 bytes (including overhead)
- 8-byte ptrs: 32 bytes (including overhead)
-
- Even a request for zero bytes (i.e., malloc(0)) returns a
- pointer to something of the minimum allocatable size.
- The maximum overhead wastage (i.e., number of extra bytes
- allocated than were requested in malloc) is less than or equal
- to the minimum size, except for requests >= mmap_threshold that
- are serviced via mmap(), where the worst case wastage is about
- 32 bytes plus the remainder from a system page (the minimal
- mmap unit); typically 4096 or 8192 bytes.
-
- Security: static-safe; optionally more or less
- The "security" of malloc refers to the ability of malicious
- code to accentuate the effects of errors (for example, freeing
- space that is not currently malloc'ed or overwriting past the
- ends of chunks) in code that calls malloc. This malloc
- guarantees not to modify any memory locations below the base of
- heap, i.e., static variables, even in the presence of usage
- errors. The routines additionally detect most improper frees
- and reallocs. All this holds as long as the static bookkeeping
- for malloc itself is not corrupted by some other means. This
- is only one aspect of security -- these checks do not, and
- cannot, detect all possible programming errors.
-
- If FOOTERS is defined nonzero, then each allocated chunk
- carries an additional check word to verify that it was malloced
- from its space. These check words are the same within each
- execution of a program using malloc, but differ across
- executions, so externally crafted fake chunks cannot be
- freed. This improves security by rejecting frees/reallocs that
- could corrupt heap memory, in addition to the checks preventing
- writes to statics that are always on. This may further improve
- security at the expense of time and space overhead. (Note that
- FOOTERS may also be worth using with MSPACES.)
-
- By default detected errors cause the program to abort (calling
- "abort()"). You can override this to instead proceed past
- errors by defining PROCEED_ON_ERROR. In this case, a bad free
- has no effect, and a malloc that encounters a bad address
- caused by user overwrites will ignore the bad address by
- dropping pointers and indices to all known memory. This may
- be appropriate for programs that should continue if at all
- possible in the face of programming errors, although they may
- run out of memory because dropped memory is never reclaimed.
-
- If you don't like either of these options, you can define
- CORRUPTION_ERROR_ACTION and USAGE_ERROR_ACTION to do anything
- else. And if if you are sure that your program using malloc has
- no errors or vulnerabilities, you can define INSECURE to 1,
- which might (or might not) provide a small performance improvement.
-
- Thread-safety: NOT thread-safe unless USE_LOCKS defined
- When USE_LOCKS is defined, each public call to malloc, free,
- etc is surrounded with either a pthread mutex or a win32
- spinlock (depending on WIN32). This is not especially fast, and
- can be a major bottleneck. It is designed only to provide
- minimal protection in concurrent environments, and to provide a
- basis for extensions. If you are using malloc in a concurrent
- program, consider instead using nedmalloc
- (http://www.nedprod.com/programs/portable/nedmalloc/) or
- ptmalloc (See http://www.malloc.de), which are derived
- from versions of this malloc.
-
- System requirements: Any combination of MORECORE and/or MMAP/MUNMAP
- This malloc can use unix sbrk or any emulation (invoked using
- the CALL_MORECORE macro) and/or mmap/munmap or any emulation
- (invoked using CALL_MMAP/CALL_MUNMAP) to get and release system
- memory. On most unix systems, it tends to work best if both
- MORECORE and MMAP are enabled. On Win32, it uses emulations
- based on VirtualAlloc. It also uses common C library functions
- like memset.
-
- Compliance: I believe it is compliant with the Single Unix Specification
- (See http://www.unix.org). Also SVID/XPG, ANSI C, and probably
- others as well.
-
-* Overview of algorithms
-
- This is not the fastest, most space-conserving, most portable, or
- most tunable malloc ever written. However it is among the fastest
- while also being among the most space-conserving, portable and
- tunable. Consistent balance across these factors results in a good
- general-purpose allocator for malloc-intensive programs.
-
- In most ways, this malloc is a best-fit allocator. Generally, it
- chooses the best-fitting existing chunk for a request, with ties
- broken in approximately least-recently-used order. (This strategy
- normally maintains low fragmentation.) However, for requests less
- than 256bytes, it deviates from best-fit when there is not an
- exactly fitting available chunk by preferring to use space adjacent
- to that used for the previous small request, as well as by breaking
- ties in approximately most-recently-used order. (These enhance
- locality of series of small allocations.) And for very large requests
- (>= 256Kb by default), it relies on system memory mapping
- facilities, if supported. (This helps avoid carrying around and
- possibly fragmenting memory used only for large chunks.)
-
- All operations (except malloc_stats and mallinfo) have execution
- times that are bounded by a constant factor of the number of bits in
- a size_t, not counting any clearing in calloc or copying in realloc,
- or actions surrounding MORECORE and MMAP that have times
- proportional to the number of non-contiguous regions returned by
- system allocation routines, which is often just 1. In real-time
- applications, you can optionally suppress segment traversals using
- NO_SEGMENT_TRAVERSAL, which assures bounded execution even when
- system allocators return non-contiguous spaces, at the typical
- expense of carrying around more memory and increased fragmentation.
-
- The implementation is not very modular and seriously overuses
- macros. Perhaps someday all C compilers will do as good a job
- inlining modular code as can now be done by brute-force expansion,
- but now, enough of them seem not to.
-
- Some compilers issue a lot of warnings about code that is
- dead/unreachable only on some platforms, and also about intentional
- uses of negation on unsigned types. All known cases of each can be
- ignored.
-
- For a longer but out of date high-level description, see
- http://gee.cs.oswego.edu/dl/html/malloc.html
-
-* MSPACES
- If MSPACES is defined, then in addition to malloc, free, etc.,
- this file also defines mspace_malloc, mspace_free, etc. These
- are versions of malloc routines that take an "mspace" argument
- obtained using create_mspace, to control all internal bookkeeping.
- If ONLY_MSPACES is defined, only these versions are compiled.
- So if you would like to use this allocator for only some allocations,
- and your system malloc for others, you can compile with
- ONLY_MSPACES and then do something like...
- static mspace mymspace = create_mspace(0,0); // for example
- #define mymalloc(bytes) mspace_malloc(mymspace, bytes)
-
- (Note: If you only need one instance of an mspace, you can instead
- use "USE_DL_PREFIX" to relabel the global malloc.)
-
- You can similarly create thread-local allocators by storing
- mspaces as thread-locals. For example:
- static __thread mspace tlms = 0;
- void* tlmalloc(size_t bytes) {
- if (tlms == 0) tlms = create_mspace(0, 0);
- return mspace_malloc(tlms, bytes);
- }
- void tlfree(void* mem) { mspace_free(tlms, mem); }
-
- Unless FOOTERS is defined, each mspace is completely independent.
- You cannot allocate from one and free to another (although
- conformance is only weakly checked, so usage errors are not always
- caught). If FOOTERS is defined, then each chunk carries around a tag
- indicating its originating mspace, and frees are directed to their
- originating spaces.
-
- ------------------------- Compile-time options ---------------------------
-
-Be careful in setting #define values for numerical constants of type
-size_t. On some systems, literal values are not automatically extended
-to size_t precision unless they are explicitly casted. You can also
-use the symbolic values MAX_SIZE_T, SIZE_T_ONE, etc below.
-
-WIN32 default: defined if _WIN32 defined
- Defining WIN32 sets up defaults for MS environment and compilers.
- Otherwise defaults are for unix. Beware that there seem to be some
- cases where this malloc might not be a pure drop-in replacement for
- Win32 malloc: Random-looking failures from Win32 GDI API's (eg;
- SetDIBits()) may be due to bugs in some video driver implementations
- when pixel buffers are malloc()ed, and the region spans more than
- one VirtualAlloc()ed region. Because dlmalloc uses a small (64Kb)
- default granularity, pixel buffers may straddle virtual allocation
- regions more often than when using the Microsoft allocator. You can
- avoid this by using VirtualAlloc() and VirtualFree() for all pixel
- buffers rather than using malloc(). If this is not possible,
- recompile this malloc with a larger DEFAULT_GRANULARITY.
-
-MALLOC_ALIGNMENT default: (size_t)8
- Controls the minimum alignment for malloc'ed chunks. It must be a
- power of two and at least 8, even on machines for which smaller
- alignments would suffice. It may be defined as larger than this
- though. Note however that code and data structures are optimized for
- the case of 8-byte alignment.
-
-MSPACES default: 0 (false)
- If true, compile in support for independent allocation spaces.
- This is only supported if HAVE_MMAP is true.
-
-ONLY_MSPACES default: 0 (false)
- If true, only compile in mspace versions, not regular versions.
-
-USE_LOCKS default: 0 (false)
- Causes each call to each public routine to be surrounded with
- pthread or WIN32 mutex lock/unlock. (If set true, this can be
- overridden on a per-mspace basis for mspace versions.) If set to a
- non-zero value other than 1, locks are used, but their
- implementation is left out, so lock functions must be supplied manually,
- as described below.
-
-USE_SPIN_LOCKS default: 1 iff USE_LOCKS and on x86 using gcc or MSC
- If true, uses custom spin locks for locking. This is currently
- supported only for x86 platforms using gcc or recent MS compilers.
- Otherwise, posix locks or win32 critical sections are used.
-
-FOOTERS default: 0
- If true, provide extra checking and dispatching by placing
- information in the footers of allocated chunks. This adds
- space and time overhead.
-
-INSECURE default: 0
- If true, omit checks for usage errors and heap space overwrites.
-
-USE_DL_PREFIX default: NOT defined
- Causes compiler to prefix all public routines with the string 'dl'.
- This can be useful when you only want to use this malloc in one part
- of a program, using your regular system malloc elsewhere.
-
-ABORT default: defined as abort()
- Defines how to abort on failed checks. On most systems, a failed
- check cannot die with an "assert" or even print an informative
- message, because the underlying print routines in turn call malloc,
- which will fail again. Generally, the best policy is to simply call
- abort(). It's not very useful to do more than this because many
- errors due to overwriting will show up as address faults (null, odd
- addresses etc) rather than malloc-triggered checks, so will also
- abort. Also, most compilers know that abort() does not return, so
- can better optimize code conditionally calling it.
-
-PROCEED_ON_ERROR default: defined as 0 (false)
- Controls whether detected bad addresses cause them to bypassed
- rather than aborting. If set, detected bad arguments to free and
- realloc are ignored. And all bookkeeping information is zeroed out
- upon a detected overwrite of freed heap space, thus losing the
- ability to ever return it from malloc again, but enabling the
- application to proceed. If PROCEED_ON_ERROR is defined, the
- static variable malloc_corruption_error_count is compiled in
- and can be examined to see if errors have occurred. This option
- generates slower code than the default abort policy.
-
-DEBUG default: NOT defined
- The DEBUG setting is mainly intended for people trying to modify
- this code or diagnose problems when porting to new platforms.
- However, it may also be able to better isolate user errors than just
- using runtime checks. The assertions in the check routines spell
- out in more detail the assumptions and invariants underlying the
- algorithms. The checking is fairly extensive, and will slow down
- execution noticeably. Calling malloc_stats or mallinfo with DEBUG
- set will attempt to check every non-mmapped allocated and free chunk
- in the course of computing the summaries.
-
-ABORT_ON_ASSERT_FAILURE default: defined as 1 (true)
- Debugging assertion failures can be nearly impossible if your
- version of the assert macro causes malloc to be called, which will
- lead to a cascade of further failures, blowing the runtime stack.
- ABORT_ON_ASSERT_FAILURE cause assertions failures to call abort(),
- which will usually make debugging easier.
-
-MALLOC_FAILURE_ACTION default: sets errno to ENOMEM, or no-op on win32
- The action to take before "return 0" when malloc fails to be able to
- return memory because there is none available.
-
-HAVE_MORECORE default: 1 (true) unless win32 or ONLY_MSPACES
- True if this system supports sbrk or an emulation of it.
-
-MORECORE default: sbrk
- The name of the sbrk-style system routine to call to obtain more
- memory. See below for guidance on writing custom MORECORE
- functions. The type of the argument to sbrk/MORECORE varies across
- systems. It cannot be size_t, because it supports negative
- arguments, so it is normally the signed type of the same width as
- size_t (sometimes declared as "intptr_t"). It doesn't much matter
- though. Internally, we only call it with arguments less than half
- the max value of a size_t, which should work across all reasonable
- possibilities, although sometimes generating compiler warnings.
-
-MORECORE_CONTIGUOUS default: 1 (true) if HAVE_MORECORE
- If true, take advantage of fact that consecutive calls to MORECORE
- with positive arguments always return contiguous increasing
- addresses. This is true of unix sbrk. It does not hurt too much to
- set it true anyway, since malloc copes with non-contiguities.
- Setting it false when definitely non-contiguous saves time
- and possibly wasted space it would take to discover this though.
-
-MORECORE_CANNOT_TRIM default: NOT defined
- True if MORECORE cannot release space back to the system when given
- negative arguments. This is generally necessary only if you are
- using a hand-crafted MORECORE function that cannot handle negative
- arguments.
-
-NO_SEGMENT_TRAVERSAL default: 0
- If non-zero, suppresses traversals of memory segments
- returned by either MORECORE or CALL_MMAP. This disables
- merging of segments that are contiguous, and selectively
- releasing them to the OS if unused, but bounds execution times.
-
-HAVE_MMAP default: 1 (true)
- True if this system supports mmap or an emulation of it. If so, and
- HAVE_MORECORE is not true, MMAP is used for all system
- allocation. If set and HAVE_MORECORE is true as well, MMAP is
- primarily used to directly allocate very large blocks. It is also
- used as a backup strategy in cases where MORECORE fails to provide
- space from system. Note: A single call to MUNMAP is assumed to be
- able to unmap memory that may have be allocated using multiple calls
- to MMAP, so long as they are adjacent.
-
-HAVE_MREMAP default: 1 on linux, else 0
- If true realloc() uses mremap() to re-allocate large blocks and
- extend or shrink allocation spaces.
-
-MMAP_CLEARS default: 1 except on WINCE.
- True if mmap clears memory so calloc doesn't need to. This is true
- for standard unix mmap using /dev/zero and on WIN32 except for WINCE.
-
-USE_BUILTIN_FFS default: 0 (i.e., not used)
- Causes malloc to use the builtin ffs() function to compute indices.
- Some compilers may recognize and intrinsify ffs to be faster than the
- supplied C version. Also, the case of x86 using gcc is special-cased
- to an asm instruction, so is already as fast as it can be, and so
- this setting has no effect. Similarly for Win32 under recent MS compilers.
- (On most x86s, the asm version is only slightly faster than the C version.)
-
-malloc_getpagesize default: derive from system includes, or 4096.
- The system page size. To the extent possible, this malloc manages
- memory from the system in page-size units. This may be (and
- usually is) a function rather than a constant. This is ignored
- if WIN32, where page size is determined using getSystemInfo during
- initialization. This may be several megabytes if ENABLE_LARGE_PAGES
- is enabled.
-
-ENABLE_LARGE_PAGES default: NOT defined
- Causes the system page size to be the value of GetLargePageMinimum()
- if that function is available (Windows Server 2003/Vista or later).
- This allows the use of large page entries in the MMU which can
- significantly improve performance in large working set applications
- as TLB cache load is reduced by a factor of three. Note that enabling
- this option is equal to locking the process' memory in current
- implementations of Windows and requires the SE_LOCK_MEMORY_PRIVILEGE
- to be held by the process in order to succeed.
-
-USE_DEV_RANDOM default: 0 (i.e., not used)
- Causes malloc to use /dev/random to initialize secure magic seed for
- stamping footers. Otherwise, the current time is used.
-
-NO_MALLINFO default: 0
- If defined, don't compile "mallinfo". This can be a simple way
- of dealing with mismatches between system declarations and
- those in this file.
-
-MALLINFO_FIELD_TYPE default: size_t
- The type of the fields in the mallinfo struct. This was originally
- defined as "int" in SVID etc, but is more usefully defined as
- size_t. The value is used only if HAVE_USR_INCLUDE_MALLOC_H is not set
-
-REALLOC_ZERO_BYTES_FREES default: not defined
- This should be set if a call to realloc with zero bytes should
- be the same as a call to free. Some people think it should. Otherwise,
- since this malloc returns a unique pointer for malloc(0), so does
- realloc(p, 0).
-
-LACKS_UNISTD_H, LACKS_FCNTL_H, LACKS_SYS_PARAM_H, LACKS_SYS_MMAN_H
-LACKS_STRINGS_H, LACKS_STRING_H, LACKS_SYS_TYPES_H, LACKS_ERRNO_H
-LACKS_STDLIB_H default: NOT defined unless on WIN32
- Define these if your system does not have these header files.
- You might need to manually insert some of the declarations they provide.
-
-DEFAULT_GRANULARITY default: page size if MORECORE_CONTIGUOUS,
- system_info.dwAllocationGranularity in WIN32,
- GetLargePageMinimum() if ENABLE_LARGE_PAGES,
- otherwise 64K.
- Also settable using mallopt(M_GRANULARITY, x)
- The unit for allocating and deallocating memory from the system. On
- most systems with contiguous MORECORE, there is no reason to
- make this more than a page. However, systems with MMAP tend to
- either require or encourage larger granularities. You can increase
- this value to prevent system allocation functions to be called so
- often, especially if they are slow. The value must be at least one
- page and must be a power of two. Setting to 0 causes initialization
- to either page size or win32 region size. (Note: In previous
- versions of malloc, the equivalent of this option was called
- "TOP_PAD")
-
-DEFAULT_GRANULARITY_ALIGNED default: undefined (which means page size)
- Whether to enforce alignment when allocating and deallocating memory
- from the system i.e. the base address of all allocations will be
- aligned to DEFAULT_GRANULARITY if it is set. Note that enabling this carries
- some overhead as multiple calls must now be made when probing for a valid
- aligned value, however it does greatly ease the checking for whether
- a given memory pointer was allocated by this allocator rather than
- some other.
-
-DEFAULT_TRIM_THRESHOLD default: 2MB
- Also settable using mallopt(M_TRIM_THRESHOLD, x)
- The maximum amount of unused top-most memory to keep before
- releasing via malloc_trim in free(). Automatic trimming is mainly
- useful in long-lived programs using contiguous MORECORE. Because
- trimming via sbrk can be slow on some systems, and can sometimes be
- wasteful (in cases where programs immediately afterward allocate
- more large chunks) the value should be high enough so that your
- overall system performance would improve by releasing this much
- memory. As a rough guide, you might set to a value close to the
- average size of a process (program) running on your system.
- Releasing this much memory would allow such a process to run in
- memory. Generally, it is worth tuning trim thresholds when a
- program undergoes phases where several large chunks are allocated
- and released in ways that can reuse each other's storage, perhaps
- mixed with phases where there are no such chunks at all. The trim
- value must be greater than page size to have any useful effect. To
- disable trimming completely, you can set to MAX_SIZE_T. Note that the trick
- some people use of mallocing a huge space and then freeing it at
- program startup, in an attempt to reserve system memory, doesn't
- have the intended effect under automatic trimming, since that memory
- will immediately be returned to the system.
-
-DEFAULT_MMAP_THRESHOLD default: 256K
- Also settable using mallopt(M_MMAP_THRESHOLD, x)
- The request size threshold for using MMAP to directly service a
- request. Requests of at least this size that cannot be allocated
- using already-existing space will be serviced via mmap. (If enough
- normal freed space already exists it is used instead.) Using mmap
- segregates relatively large chunks of memory so that they can be
- individually obtained and released from the host system. A request
- serviced through mmap is never reused by any other request (at least
- not directly; the system may just so happen to remap successive
- requests to the same locations). Segregating space in this way has
- the benefits that: Mmapped space can always be individually released
- back to the system, which helps keep the system level memory demands
- of a long-lived program low. Also, mapped memory doesn't become
- `locked' between other chunks, as can happen with normally allocated
- chunks, which means that even trimming via malloc_trim would not
- release them. However, it has the disadvantage that the space
- cannot be reclaimed, consolidated, and then used to service later
- requests, as happens with normal chunks. The advantages of mmap
- nearly always outweigh disadvantages for "large" chunks, but the
- value of "large" may vary across systems. The default is an
- empirically derived value that works well in most systems. You can
- disable mmap by setting to MAX_SIZE_T.
-
-MAX_RELEASE_CHECK_RATE default: 4095 unless not HAVE_MMAP
- The number of consolidated frees between checks to release
- unused segments when freeing. When using non-contiguous segments,
- especially with multiple mspaces, checking only for topmost space
- doesn't always suffice to trigger trimming. To compensate for this,
- free() will, with a period of MAX_RELEASE_CHECK_RATE (or the
- current number of segments, if greater) try to release unused
- segments to the OS when freeing chunks that result in
- consolidation. The best value for this parameter is a compromise
- between slowing down frees with relatively costly checks that
- rarely trigger versus holding on to unused memory. To effectively
- disable, set to MAX_SIZE_T. This may lead to a very slight speed
- improvement at the expense of carrying around more memory.
-*/
-
-/* Version identifier to allow people to support multiple versions */
-#ifndef DLMALLOC_VERSION
-#define DLMALLOC_VERSION 20804
-#endif /* DLMALLOC_VERSION */
-
-#ifndef WIN32
-#ifdef _WIN32
-#define WIN32 1
-#endif /* _WIN32 */
-#ifdef _WIN32_WCE
-#define LACKS_FCNTL_H
-#define WIN32 1
-#endif /* _WIN32_WCE */
-#endif /* WIN32 */
-#ifdef WIN32
-#define WIN32_LEAN_AND_MEAN
-#include <windows.h>
-#include <tchar.h>
-#define HAVE_MMAP 1
-#define HAVE_MORECORE 0
-#define LACKS_UNISTD_H
-#define LACKS_SYS_PARAM_H
-#define LACKS_SYS_MMAN_H
-#define LACKS_STRING_H
-#define LACKS_STRINGS_H
-#define LACKS_SYS_TYPES_H
-#define LACKS_ERRNO_H
-#ifndef MALLOC_FAILURE_ACTION
-#define MALLOC_FAILURE_ACTION
-#endif /* MALLOC_FAILURE_ACTION */
-#ifdef _WIN32_WCE /* WINCE reportedly does not clear */
-#define MMAP_CLEARS 0
-#else
-#define MMAP_CLEARS 1
-#endif /* _WIN32_WCE */
-#endif /* WIN32 */
-
-#if defined(DARWIN) || defined(_DARWIN)
-/* Mac OSX docs advise not to use sbrk; it seems better to use mmap */
-#ifndef HAVE_MORECORE
-#define HAVE_MORECORE 0
-#define HAVE_MMAP 1
-/* OSX allocators provide 16 byte alignment */
-#ifndef MALLOC_ALIGNMENT
-#define MALLOC_ALIGNMENT ((size_t)16U)
-#endif
-#endif /* HAVE_MORECORE */
-#endif /* DARWIN */
-
-#ifndef LACKS_SYS_TYPES_H
-#include <sys/types.h> /* For size_t */
-#endif /* LACKS_SYS_TYPES_H */
-
-#if (defined(__GNUC__) && ((defined(__i386__) || defined(__x86_64__)))) || (defined(_MSC_VER) && _MSC_VER>=1310)
-#define SPIN_LOCKS_AVAILABLE 1
-#else
-#define SPIN_LOCKS_AVAILABLE 0
-#endif
-
-/* The maximum possible size_t value has all bits set */
-#define MAX_SIZE_T (~(size_t)0)
-
-#ifndef ONLY_MSPACES
-#define ONLY_MSPACES 0 /* define to a value */
-#else
-#define ONLY_MSPACES 1
-#endif /* ONLY_MSPACES */
-#ifndef MSPACES
-#if ONLY_MSPACES
-#define MSPACES 1
-#else /* ONLY_MSPACES */
-#define MSPACES 0
-#endif /* ONLY_MSPACES */
-#endif /* MSPACES */
-#ifndef MALLOC_ALIGNMENT
-#define MALLOC_ALIGNMENT ((size_t)8U)
-#endif /* MALLOC_ALIGNMENT */
-#ifndef FOOTERS
-#define FOOTERS 0
-#endif /* FOOTERS */
-#ifndef ABORT
-#define ABORT abort()
-#endif /* ABORT */
-#ifndef ABORT_ON_ASSERT_FAILURE
-#define ABORT_ON_ASSERT_FAILURE 1
-#endif /* ABORT_ON_ASSERT_FAILURE */
-#ifndef PROCEED_ON_ERROR
-#define PROCEED_ON_ERROR 0
-#endif /* PROCEED_ON_ERROR */
-#ifndef USE_LOCKS
-#define USE_LOCKS 0
-#endif /* USE_LOCKS */
-#ifndef USE_SPIN_LOCKS
-#if USE_LOCKS && SPIN_LOCKS_AVAILABLE
-#define USE_SPIN_LOCKS 1
-#else
-#define USE_SPIN_LOCKS 0
-#endif /* USE_LOCKS && SPIN_LOCKS_AVAILABLE. */
-#endif /* USE_SPIN_LOCKS */
-#ifndef INSECURE
-#define INSECURE 0
-#endif /* INSECURE */
-#ifndef HAVE_MMAP
-#define HAVE_MMAP 1
-#endif /* HAVE_MMAP */
-#ifndef MMAP_CLEARS
-#define MMAP_CLEARS 1
-#endif /* MMAP_CLEARS */
-#ifndef HAVE_MREMAP
-#ifdef linux
-#define HAVE_MREMAP 1
-#else /* linux */
-#define HAVE_MREMAP 0
-#endif /* linux */
-#endif /* HAVE_MREMAP */
-#ifndef MALLOC_FAILURE_ACTION
-#define MALLOC_FAILURE_ACTION errno = ENOMEM;
-#endif /* MALLOC_FAILURE_ACTION */
-#ifndef HAVE_MORECORE
-#if ONLY_MSPACES
-#define HAVE_MORECORE 0
-#else /* ONLY_MSPACES */
-#define HAVE_MORECORE 1
-#endif /* ONLY_MSPACES */
-#endif /* HAVE_MORECORE */
-#if !HAVE_MORECORE
-#define MORECORE_CONTIGUOUS 0
-#else /* !HAVE_MORECORE */
-#define MORECORE_DEFAULT sbrk
-#ifndef MORECORE_CONTIGUOUS
-#define MORECORE_CONTIGUOUS 1
-#endif /* MORECORE_CONTIGUOUS */
-#endif /* HAVE_MORECORE */
-#ifndef DEFAULT_GRANULARITY
-#if (MORECORE_CONTIGUOUS || defined(WIN32))
-#define DEFAULT_GRANULARITY (0) /* 0 means to compute in init_mparams */
-#else /* MORECORE_CONTIGUOUS */
-#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U)
-#endif /* MORECORE_CONTIGUOUS */
-#endif /* DEFAULT_GRANULARITY */
-#ifndef DEFAULT_TRIM_THRESHOLD
-#ifndef MORECORE_CANNOT_TRIM
-#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U)
-#else /* MORECORE_CANNOT_TRIM */
-#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T
-#endif /* MORECORE_CANNOT_TRIM */
-#endif /* DEFAULT_TRIM_THRESHOLD */
-#ifndef DEFAULT_MMAP_THRESHOLD
-#if HAVE_MMAP
-#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U)
-#else /* HAVE_MMAP */
-#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T
-#endif /* HAVE_MMAP */
-#endif /* DEFAULT_MMAP_THRESHOLD */
-#ifndef MAX_RELEASE_CHECK_RATE
-#if HAVE_MMAP
-#define MAX_RELEASE_CHECK_RATE 4095
-#else
-#define MAX_RELEASE_CHECK_RATE MAX_SIZE_T
-#endif /* HAVE_MMAP */
-#endif /* MAX_RELEASE_CHECK_RATE */
-#ifndef USE_BUILTIN_FFS
-#define USE_BUILTIN_FFS 0
-#endif /* USE_BUILTIN_FFS */
-#ifndef USE_DEV_RANDOM
-#define USE_DEV_RANDOM 0
-#endif /* USE_DEV_RANDOM */
-#ifndef NO_MALLINFO
-#define NO_MALLINFO 0
-#endif /* NO_MALLINFO */
-#ifndef MALLINFO_FIELD_TYPE
-#define MALLINFO_FIELD_TYPE size_t
-#endif /* MALLINFO_FIELD_TYPE */
-#ifndef NO_SEGMENT_TRAVERSAL
-#define NO_SEGMENT_TRAVERSAL 0
-#endif /* NO_SEGMENT_TRAVERSAL */
-
-/*
- mallopt tuning options. SVID/XPG defines four standard parameter
- numbers for mallopt, normally defined in malloc.h. None of these
- are used in this malloc, so setting them has no effect. But this
- malloc does support the following options.
-*/
-
-#define M_TRIM_THRESHOLD (-1)
-#define M_GRANULARITY (-2)
-#define M_MMAP_THRESHOLD (-3)
-
-/* ------------------------ Mallinfo declarations ------------------------ */
-
-#if !NO_MALLINFO
-/*
- This version of malloc supports the standard SVID/XPG mallinfo
- routine that returns a struct containing usage properties and
- statistics. It should work on any system that has a
- /usr/include/malloc.h defining struct mallinfo. The main
- declaration needed is the mallinfo struct that is returned (by-copy)
- by mallinfo(). The malloinfo struct contains a bunch of fields that
- are not even meaningful in this version of malloc. These fields are
- are instead filled by mallinfo() with other numbers that might be of
- interest.
-
- HAVE_USR_INCLUDE_MALLOC_H should be set if you have a
- /usr/include/malloc.h file that includes a declaration of struct
- mallinfo. If so, it is included; else a compliant version is
- declared below. These must be precisely the same for mallinfo() to
- work. The original SVID version of this struct, defined on most
- systems with mallinfo, declares all fields as ints. But some others
- define as unsigned long. If your system defines the fields using a
- type of different width than listed here, you MUST #include your
- system version and #define HAVE_USR_INCLUDE_MALLOC_H.
-*/
-
-/* #define HAVE_USR_INCLUDE_MALLOC_H */
-
-#ifdef HAVE_USR_INCLUDE_MALLOC_H
-#include "/usr/include/malloc.h"
-#else /* HAVE_USR_INCLUDE_MALLOC_H */
-#ifndef STRUCT_MALLINFO_DECLARED
-#define STRUCT_MALLINFO_DECLARED 1
-struct mallinfo {
- MALLINFO_FIELD_TYPE arena; /* non-mmapped space allocated from system */
- MALLINFO_FIELD_TYPE ordblks; /* number of free chunks */
- MALLINFO_FIELD_TYPE smblks; /* always 0 */
- MALLINFO_FIELD_TYPE hblks; /* always 0 */
- MALLINFO_FIELD_TYPE hblkhd; /* space in mmapped regions */
- MALLINFO_FIELD_TYPE usmblks; /* maximum total allocated space */
- MALLINFO_FIELD_TYPE fsmblks; /* always 0 */
- MALLINFO_FIELD_TYPE uordblks; /* total allocated space */
- MALLINFO_FIELD_TYPE fordblks; /* total free space */
- MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */
-};
-#endif /* STRUCT_MALLINFO_DECLARED */
-#endif /* HAVE_USR_INCLUDE_MALLOC_H */
-#endif /* NO_MALLINFO */
-
-/*
- Try to persuade compilers to inline. The most critical functions for
- inlining are defined as macros, so these aren't used for them.
-*/
-
-#ifndef FORCEINLINE
- #if defined(__GNUC__)
-#define FORCEINLINE __inline __attribute__ ((always_inline))
- #elif defined(_MSC_VER)
- #define FORCEINLINE __forceinline
- #endif
-#endif
-#ifndef NOINLINE
- #if defined(__GNUC__)
- #define NOINLINE __attribute__ ((noinline))
- #elif defined(_MSC_VER)
- #define NOINLINE __declspec(noinline)
- #else
- #define NOINLINE
- #endif
-#endif
-
-#ifdef __cplusplus
-extern "C" {
-#ifndef FORCEINLINE
- #define FORCEINLINE inline
-#endif
-#endif /* __cplusplus */
-#ifndef FORCEINLINE
- #define FORCEINLINE
-#endif
-
-#if !ONLY_MSPACES
-
-/* ------------------- Declarations of public routines ------------------- */
-
-#ifndef USE_DL_PREFIX
-#define dlcalloc calloc
-#define dlfree free
-#define dlmalloc malloc
-#define dlmemalign memalign
-#define dlrealloc realloc
-#define dlvalloc valloc
-#define dlpvalloc pvalloc
-#define dlmallinfo mallinfo
-#define dlmallopt mallopt
-#define dlmalloc_trim malloc_trim
-#define dlmalloc_stats malloc_stats
-#define dlmalloc_usable_size malloc_usable_size
-#define dlmalloc_footprint malloc_footprint
-#define dlmalloc_max_footprint malloc_max_footprint
-#define dlindependent_calloc independent_calloc
-#define dlindependent_comalloc independent_comalloc
-#endif /* USE_DL_PREFIX */
-
-
-/*
- malloc(size_t n)
- Returns a pointer to a newly allocated chunk of at least n bytes, or
- null if no space is available, in which case errno is set to ENOMEM
- on ANSI C systems.
-
- If n is zero, malloc returns a minimum-sized chunk. (The minimum
- size is 16 bytes on most 32bit systems, and 32 bytes on 64bit
- systems.) Note that size_t is an unsigned type, so calls with
- arguments that would be negative if signed are interpreted as
- requests for huge amounts of space, which will often fail. The
- maximum supported value of n differs across systems, but is in all
- cases less than the maximum representable value of a size_t.
-*/
-void* dlmalloc(size_t);
-
-/*
- free(void* p)
- Releases the chunk of memory pointed to by p, that had been previously
- allocated using malloc or a related routine such as realloc.
- It has no effect if p is null. If p was not malloced or already
- freed, free(p) will by default cause the current program to abort.
-*/
-void dlfree(void*);
-
-/*
- calloc(size_t n_elements, size_t element_size);
- Returns a pointer to n_elements * element_size bytes, with all locations
- set to zero.
-*/
-void* dlcalloc(size_t, size_t);
-
-/*
- realloc(void* p, size_t n)
- Returns a pointer to a chunk of size n that contains the same data
- as does chunk p up to the minimum of (n, p's size) bytes, or null
- if no space is available.
-
- The returned pointer may or may not be the same as p. The algorithm
- prefers extending p in most cases when possible, otherwise it
- employs the equivalent of a malloc-copy-free sequence.
-
- If p is null, realloc is equivalent to malloc.
-
- If space is not available, realloc returns null, errno is set (if on
- ANSI) and p is NOT freed.
-
- if n is for fewer bytes than already held by p, the newly unused
- space is lopped off and freed if possible. realloc with a size
- argument of zero (re)allocates a minimum-sized chunk.
-
- The old unix realloc convention of allowing the last-free'd chunk
- to be used as an argument to realloc is not supported.
-*/
-
-void* dlrealloc(void*, size_t);
-
-/*
- memalign(size_t alignment, size_t n);
- Returns a pointer to a newly allocated chunk of n bytes, aligned
- in accord with the alignment argument.
-
- The alignment argument should be a power of two. If the argument is
- not a power of two, the nearest greater power is used.
- 8-byte alignment is guaranteed by normal malloc calls, so don't
- bother calling memalign with an argument of 8 or less.
-
- Overreliance on memalign is a sure way to fragment space.
-*/
-void* dlmemalign(size_t, size_t);
-
-/*
- valloc(size_t n);
- Equivalent to memalign(pagesize, n), where pagesize is the page
- size of the system. If the pagesize is unknown, 4096 is used.
-*/
-void* dlvalloc(size_t);
-
-/*
- mallopt(int parameter_number, int parameter_value)
- Sets tunable parameters The format is to provide a
- (parameter-number, parameter-value) pair. mallopt then sets the
- corresponding parameter to the argument value if it can (i.e., so
- long as the value is meaningful), and returns 1 if successful else
- 0. To workaround the fact that mallopt is specified to use int,
- not size_t parameters, the value -1 is specially treated as the
- maximum unsigned size_t value.
-
- SVID/XPG/ANSI defines four standard param numbers for mallopt,
- normally defined in malloc.h. None of these are use in this malloc,
- so setting them has no effect. But this malloc also supports other
- options in mallopt. See below for details. Briefly, supported
- parameters are as follows (listed defaults are for "typical"
- configurations).
-
- Symbol param # default allowed param values
- M_TRIM_THRESHOLD -1 2*1024*1024 any (-1 disables)
- M_GRANULARITY -2 page size any power of 2 >= page size
- M_MMAP_THRESHOLD -3 256*1024 any (or 0 if no MMAP support)
-*/
-int dlmallopt(int, int);
-
-/*
- malloc_footprint();
- Returns the number of bytes obtained from the system. The total
- number of bytes allocated by malloc, realloc etc., is less than this
- value. Unlike mallinfo, this function returns only a precomputed
- result, so can be called frequently to monitor memory consumption.
- Even if locks are otherwise defined, this function does not use them,
- so results might not be up to date.
-*/
-size_t dlmalloc_footprint(void);
-
-/*
- malloc_max_footprint();
- Returns the maximum number of bytes obtained from the system. This
- value will be greater than current footprint if deallocated space
- has been reclaimed by the system. The peak number of bytes allocated
- by malloc, realloc etc., is less than this value. Unlike mallinfo,
- this function returns only a precomputed result, so can be called
- frequently to monitor memory consumption. Even if locks are
- otherwise defined, this function does not use them, so results might
- not be up to date.
-*/
-size_t dlmalloc_max_footprint(void);
-
-#if !NO_MALLINFO
-/*
- mallinfo()
- Returns (by copy) a struct containing various summary statistics:
-
- arena: current total non-mmapped bytes allocated from system
- ordblks: the number of free chunks
- smblks: always zero.
- hblks: current number of mmapped regions
- hblkhd: total bytes held in mmapped regions
- usmblks: the maximum total allocated space. This will be greater
- than current total if trimming has occurred.
- fsmblks: always zero
- uordblks: current total allocated space (normal or mmapped)
- fordblks: total free space
- keepcost: the maximum number of bytes that could ideally be released
- back to system via malloc_trim. ("ideally" means that
- it ignores page restrictions etc.)
-
- Because these fields are ints, but internal bookkeeping may
- be kept as longs, the reported values may wrap around zero and
- thus be inaccurate.
-*/
-struct mallinfo dlmallinfo(void);
-#endif /* NO_MALLINFO */
-
-/*
- independent_calloc(size_t n_elements, size_t element_size, void* chunks[]);
-
- independent_calloc is similar to calloc, but instead of returning a
- single cleared space, it returns an array of pointers to n_elements
- independent elements that can hold contents of size elem_size, each
- of which starts out cleared, and can be independently freed,
- realloc'ed etc. The elements are guaranteed to be adjacently
- allocated (this is not guaranteed to occur with multiple callocs or
- mallocs), which may also improve cache locality in some
- applications.
-
- The "chunks" argument is optional (i.e., may be null, which is
- probably the most typical usage). If it is null, the returned array
- is itself dynamically allocated and should also be freed when it is
- no longer needed. Otherwise, the chunks array must be of at least
- n_elements in length. It is filled in with the pointers to the
- chunks.
-
- In either case, independent_calloc returns this pointer array, or
- null if the allocation failed. If n_elements is zero and "chunks"
- is null, it returns a chunk representing an array with zero elements
- (which should be freed if not wanted).
-
- Each element must be individually freed when it is no longer
- needed. If you'd like to instead be able to free all at once, you
- should instead use regular calloc and assign pointers into this
- space to represent elements. (In this case though, you cannot
- independently free elements.)
-
- independent_calloc simplifies and speeds up implementations of many
- kinds of pools. It may also be useful when constructing large data
- structures that initially have a fixed number of fixed-sized nodes,
- but the number is not known at compile time, and some of the nodes
- may later need to be freed. For example:
-
- struct Node { int item; struct Node* next; };
-
- struct Node* build_list() {
- struct Node** pool;
- int n = read_number_of_nodes_needed();
- if (n <= 0) return 0;
- pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0);
- if (pool == 0) die();
- // organize into a linked list...
- struct Node* first = pool[0];
- for (i = 0; i < n-1; ++i)
- pool[i]->next = pool[i+1];
- free(pool); // Can now free the array (or not, if it is needed later)
- return first;
- }
-*/
-void** dlindependent_calloc(size_t, size_t, void**);
-
-/*
- independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]);
-
- independent_comalloc allocates, all at once, a set of n_elements
- chunks with sizes indicated in the "sizes" array. It returns
- an array of pointers to these elements, each of which can be
- independently freed, realloc'ed etc. The elements are guaranteed to
- be adjacently allocated (this is not guaranteed to occur with
- multiple callocs or mallocs), which may also improve cache locality
- in some applications.
-
- The "chunks" argument is optional (i.e., may be null). If it is null
- the returned array is itself dynamically allocated and should also
- be freed when it is no longer needed. Otherwise, the chunks array
- must be of at least n_elements in length. It is filled in with the
- pointers to the chunks.
-
- In either case, independent_comalloc returns this pointer array, or
- null if the allocation failed. If n_elements is zero and chunks is
- null, it returns a chunk representing an array with zero elements
- (which should be freed if not wanted).
-
- Each element must be individually freed when it is no longer
- needed. If you'd like to instead be able to free all at once, you
- should instead use a single regular malloc, and assign pointers at
- particular offsets in the aggregate space. (In this case though, you
- cannot independently free elements.)
-
- independent_comallac differs from independent_calloc in that each
- element may have a different size, and also that it does not
- automatically clear elements.
-
- independent_comalloc can be used to speed up allocation in cases
- where several structs or objects must always be allocated at the
- same time. For example:
-
- struct Head { ... }
- struct Foot { ... }
-
- void send_message(char* msg) {
- int msglen = strlen(msg);
- size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) };
- void* chunks[3];
- if (independent_comalloc(3, sizes, chunks) == 0)
- die();
- struct Head* head = (struct Head*)(chunks[0]);
- char* body = (char*)(chunks[1]);
- struct Foot* foot = (struct Foot*)(chunks[2]);
- // ...
- }
-
- In general though, independent_comalloc is worth using only for
- larger values of n_elements. For small values, you probably won't
- detect enough difference from series of malloc calls to bother.
-
- Overuse of independent_comalloc can increase overall memory usage,
- since it cannot reuse existing noncontiguous small chunks that
- might be available for some of the elements.
-*/
-void** dlindependent_comalloc(size_t, size_t*, void**);
-
-
-/*
- pvalloc(size_t n);
- Equivalent to valloc(minimum-page-that-holds(n)), that is,
- round up n to nearest pagesize.
- */
-void* dlpvalloc(size_t);
-
-/*
- malloc_trim(size_t pad);
-
- If possible, gives memory back to the system (via negative arguments
- to sbrk) if there is unused memory at the `high' end of the malloc
- pool or in unused MMAP segments. You can call this after freeing
- large blocks of memory to potentially reduce the system-level memory
- requirements of a program. However, it cannot guarantee to reduce
- memory. Under some allocation patterns, some large free blocks of
- memory will be locked between two used chunks, so they cannot be
- given back to the system.
-
- The `pad' argument to malloc_trim represents the amount of free
- trailing space to leave untrimmed. If this argument is zero, only
- the minimum amount of memory to maintain internal data structures
- will be left. Non-zero arguments can be supplied to maintain enough
- trailing space to service future expected allocations without having
- to re-obtain memory from the system.
-
- Malloc_trim returns 1 if it actually released any memory, else 0.
-*/
-int dlmalloc_trim(size_t);
-
-/*
- malloc_stats();
- Prints on stderr the amount of space obtained from the system (both
- via sbrk and mmap), the maximum amount (which may be more than
- current if malloc_trim and/or munmap got called), and the current
- number of bytes allocated via malloc (or realloc, etc) but not yet
- freed. Note that this is the number of bytes allocated, not the
- number requested. It will be larger than the number requested
- because of alignment and bookkeeping overhead. Because it includes
- alignment wastage as being in use, this figure may be greater than
- zero even when no user-level chunks are allocated.
-
- The reported current and maximum system memory can be inaccurate if
- a program makes other calls to system memory allocation functions
- (normally sbrk) outside of malloc.
-
- malloc_stats prints only the most commonly interesting statistics.
- More information can be obtained by calling mallinfo.
-*/
-void dlmalloc_stats(void);
-
-#endif /* ONLY_MSPACES */
-
-/*
- malloc_usable_size(void* p);
-
- Returns the number of bytes you can actually use in
- an allocated chunk, which may be more than you requested (although
- often not) due to alignment and minimum size constraints.
- You can use this many bytes without worrying about
- overwriting other allocated objects. This is not a particularly great
- programming practice. malloc_usable_size can be more useful in
- debugging and assertions, for example:
-
- p = malloc(n);
- assert(malloc_usable_size(p) >= 256);
-*/
-size_t dlmalloc_usable_size(void*);
-
-
-#if MSPACES
-
-/*
- mspace is an opaque type representing an independent
- region of space that supports mspace_malloc, etc.
-*/
-typedef void* mspace;
-
-/*
- create_mspace creates and returns a new independent space with the
- given initial capacity, or, if 0, the default granularity size. It
- returns null if there is no system memory available to create the
- space. If argument locked is non-zero, the space uses a separate
- lock to control access. The capacity of the space will grow
- dynamically as needed to service mspace_malloc requests. You can
- control the sizes of incremental increases of this space by
- compiling with a different DEFAULT_GRANULARITY or dynamically
- setting with mallopt(M_GRANULARITY, value).
-*/
-mspace create_mspace(size_t capacity, int locked);
-
-/*
- destroy_mspace destroys the given space, and attempts to return all
- of its memory back to the system, returning the total number of
- bytes freed. After destruction, the results of access to all memory
- used by the space become undefined.
-*/
-size_t destroy_mspace(mspace msp);
-
-/*
- create_mspace_with_base uses the memory supplied as the initial base
- of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this
- space is used for bookkeeping, so the capacity must be at least this
- large. (Otherwise 0 is returned.) When this initial space is
- exhausted, additional memory will be obtained from the system.
- Destroying this space will deallocate all additionally allocated
- space (if possible) but not the initial base.
-*/
-mspace create_mspace_with_base(void* base, size_t capacity, int locked);
-
-/*
- mspace_track_large_chunks controls whether requests for large chunks
- are allocated in their own untracked mmapped regions, separate from
- others in this mspace. By default large chunks are not tracked,
- which reduces fragmentation. However, such chunks are not
- necessarily released to the system upon destroy_mspace. Enabling
- tracking by setting to true may increase fragmentation, but avoids
- leakage when relying on destroy_mspace to release all memory
- allocated using this space. The function returns the previous
- setting.
-*/
-int mspace_track_large_chunks(mspace msp, int enable);
-
-
-/*
- mspace_malloc behaves as malloc, but operates within
- the given space.
-*/
-void* mspace_malloc(mspace msp, size_t bytes);
-
-/*
- mspace_free behaves as free, but operates within
- the given space.
-
- If compiled with FOOTERS==1, mspace_free is not actually needed.
- free may be called instead of mspace_free because freed chunks from
- any space are handled by their originating spaces.
-*/
-void mspace_free(mspace msp, void* mem);
-
-/*
- mspace_realloc behaves as realloc, but operates within
- the given space.
-
- If compiled with FOOTERS==1, mspace_realloc is not actually
- needed. realloc may be called instead of mspace_realloc because
- realloced chunks from any space are handled by their originating
- spaces.
-*/
-void* mspace_realloc(mspace msp, void* mem, size_t newsize);
-
-/*
- mspace_calloc behaves as calloc, but operates within
- the given space.
-*/
-void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size);
-
-/*
- mspace_memalign behaves as memalign, but operates within
- the given space.
-*/
-void* mspace_memalign(mspace msp, size_t alignment, size_t bytes);
-
-/*
- mspace_independent_calloc behaves as independent_calloc, but
- operates within the given space.
-*/
-void** mspace_independent_calloc(mspace msp, size_t n_elements,
- size_t elem_size, void* chunks[]);
-
-/*
- mspace_independent_comalloc behaves as independent_comalloc, but
- operates within the given space.
-*/
-void** mspace_independent_comalloc(mspace msp, size_t n_elements,
- size_t sizes[], void* chunks[]);
-
-/*
- mspace_footprint() returns the number of bytes obtained from the
- system for this space.
-*/
-size_t mspace_footprint(mspace msp);
-
-/*
- mspace_max_footprint() returns the peak number of bytes obtained from the
- system for this space.
-*/
-size_t mspace_max_footprint(mspace msp);
-
-
-#if !NO_MALLINFO
-/*
- mspace_mallinfo behaves as mallinfo, but reports properties of
- the given space.
-*/
-struct mallinfo mspace_mallinfo(mspace msp);
-#endif /* NO_MALLINFO */
-
-/*
- malloc_usable_size(void* p) behaves the same as malloc_usable_size;
-*/
- size_t mspace_usable_size(void* mem);
-
-/*
- mspace_malloc_stats behaves as malloc_stats, but reports
- properties of the given space.
-*/
-void mspace_malloc_stats(mspace msp);
-
-/*
- mspace_trim behaves as malloc_trim, but
- operates within the given space.
-*/
-int mspace_trim(mspace msp, size_t pad);
-
-/*
- An alias for mallopt.
-*/
-int mspace_mallopt(int, int);
-
-#endif /* MSPACES */
-
-#ifdef __cplusplus
-} /* end of extern "C" */
-#endif /* __cplusplus */
-
-/*
- ========================================================================
- To make a fully customizable malloc.h header file, cut everything
- above this line, put into file malloc.h, edit to suit, and #include it
- on the next line, as well as in programs that use this malloc.
- ========================================================================
-*/
-
-/* #include "malloc.h" */
-
-/*------------------------------ internal #includes ---------------------- */
-
-#ifdef WIN32
-#pragma warning( disable : 4146 ) /* no "unsigned" warnings */
-#endif /* WIN32 */
-
-#include <stdio.h> /* for printing in malloc_stats */
-
-#ifndef LACKS_ERRNO_H
-#include <errno.h> /* for MALLOC_FAILURE_ACTION */
-#endif /* LACKS_ERRNO_H */
-#if FOOTERS || DEBUG
-#include <time.h> /* for magic initialization */
-#endif /* FOOTERS */
-#ifndef LACKS_STDLIB_H
-#include <stdlib.h> /* for abort() */
-#endif /* LACKS_STDLIB_H */
-#ifdef DEBUG
-#if ABORT_ON_ASSERT_FAILURE
-#undef assert
-#define assert(x) if(!(x)) ABORT
-#else /* ABORT_ON_ASSERT_FAILURE */
-#include <assert.h>
-#endif /* ABORT_ON_ASSERT_FAILURE */
-#else /* DEBUG */
-#ifndef assert
-#define assert(x)
-#endif
-#define DEBUG 0
-#endif /* DEBUG */
-#ifndef LACKS_STRING_H
-#include <string.h> /* for memset etc */
-#endif /* LACKS_STRING_H */
-#if USE_BUILTIN_FFS
-#ifndef LACKS_STRINGS_H
-#include <strings.h> /* for ffs */
-#endif /* LACKS_STRINGS_H */
-#endif /* USE_BUILTIN_FFS */
-#if HAVE_MMAP
-#ifndef LACKS_SYS_MMAN_H
-/* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */
-#if (defined(linux) && !defined(__USE_GNU))
-#define __USE_GNU 1
-#include <sys/mman.h> /* for mmap */
-#undef __USE_GNU
-#else
-#include <sys/mman.h> /* for mmap */
-#endif /* linux */
-#endif /* LACKS_SYS_MMAN_H */
-#ifndef LACKS_FCNTL_H
-#include <fcntl.h>
-#endif /* LACKS_FCNTL_H */
-#endif /* HAVE_MMAP */
-#ifndef LACKS_UNISTD_H
-#include <unistd.h> /* for sbrk, sysconf */
-#else /* LACKS_UNISTD_H */
-#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__)
-extern void* sbrk(ptrdiff_t);
-#endif /* FreeBSD etc */
-#endif /* LACKS_UNISTD_H */
-
-/* Declarations for locking */
-#if USE_LOCKS
-#ifndef WIN32
-#include <pthread.h>
-#if defined (__SVR4) && defined (__sun) /* solaris */
-#include <thread.h>
-#endif /* solaris */
-#else
-#ifndef _M_AMD64
-/* These are already defined on AMD64 builds */
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-LONG __cdecl _InterlockedCompareExchange(LONG volatile *Dest, LONG Exchange, LONG Comp);
-LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value);
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-#endif /* _M_AMD64 */
-#pragma intrinsic (_InterlockedCompareExchange)
-#pragma intrinsic (_InterlockedExchange)
-#define interlockedcompareexchange _InterlockedCompareExchange
-#define interlockedexchange _InterlockedExchange
-#endif /* Win32 */
-#endif /* USE_LOCKS */
-
-/* Declarations for bit scanning on win32 */
-#if defined(_MSC_VER) && _MSC_VER>=1300
-#ifndef BitScanForward /* Try to avoid pulling in WinNT.h */
-#ifdef __cplusplus
-extern "C" {
-#endif /* __cplusplus */
-unsigned char _BitScanForward(unsigned long *index, unsigned long mask);
-unsigned char _BitScanReverse(unsigned long *index, unsigned long mask);
-#ifdef __cplusplus
-}
-#endif /* __cplusplus */
-
-#define BitScanForward _BitScanForward
-#define BitScanReverse _BitScanReverse
-#pragma intrinsic(_BitScanForward)
-#pragma intrinsic(_BitScanReverse)
-#endif /* BitScanForward */
-#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */
-
-#ifndef WIN32
-#ifndef malloc_getpagesize
-# ifdef _SC_PAGESIZE /* some SVR4 systems omit an underscore */
-# ifndef _SC_PAGE_SIZE
-# define _SC_PAGE_SIZE _SC_PAGESIZE
-# endif
-# endif
-# ifdef _SC_PAGE_SIZE
-# define malloc_getpagesize sysconf(_SC_PAGE_SIZE)
-# else
-# if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE)
- extern size_t getpagesize();
-# define malloc_getpagesize getpagesize()
-# else
-# ifdef WIN32 /* use supplied emulation of getpagesize */
-# define malloc_getpagesize getpagesize()
-# else
-# ifndef LACKS_SYS_PARAM_H
-# include <sys/param.h>
-# endif
-# ifdef EXEC_PAGESIZE
-# define malloc_getpagesize EXEC_PAGESIZE
-# else
-# ifdef NBPG
-# ifndef CLSIZE
-# define malloc_getpagesize NBPG
-# else
-# define malloc_getpagesize (NBPG * CLSIZE)
-# endif
-# else
-# ifdef NBPC
-# define malloc_getpagesize NBPC
-# else
-# ifdef PAGESIZE
-# define malloc_getpagesize PAGESIZE
-# else /* just guess */
-# define malloc_getpagesize ((size_t)4096U)
-# endif
-# endif
-# endif
-# endif
-# endif
-# endif
-# endif
-#endif
-#endif
-
-
-
-/* ------------------- size_t and alignment properties -------------------- */
-
-/* The byte and bit size of a size_t */
-#define SIZE_T_SIZE (sizeof(size_t))
-#define SIZE_T_BITSIZE (sizeof(size_t) << 3)
-
-/* Some constants coerced to size_t */
-/* Annoying but necessary to avoid errors on some platforms */
-#define SIZE_T_ZERO ((size_t)0)
-#define SIZE_T_ONE ((size_t)1)
-#define SIZE_T_TWO ((size_t)2)
-#define SIZE_T_FOUR ((size_t)4)
-#define TWO_SIZE_T_SIZES (SIZE_T_SIZE<<1)
-#define FOUR_SIZE_T_SIZES (SIZE_T_SIZE<<2)
-#define SIX_SIZE_T_SIZES (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES)
-#define HALF_MAX_SIZE_T (MAX_SIZE_T / 2U)
-
-/* The bit mask value corresponding to MALLOC_ALIGNMENT */
-#define CHUNK_ALIGN_MASK (MALLOC_ALIGNMENT - SIZE_T_ONE)
-
-/* True if address a has acceptable alignment */
-#define is_aligned(A) (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0)
-
-/* the number of bytes to offset an address to align it */
-#define align_offset(A)\
- ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\
- ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK))
-
-/*
- malloc_params holds global properties, including those that can be
- dynamically set using mallopt. There is a single instance, mparams,
- initialized in init_mparams. Note that the non-zeroness of "magic"
- also serves as an initialization flag.
-*/
-typedef unsigned int flag_t;
-struct malloc_params {
- volatile size_t magic;
- size_t page_size;
- size_t granularity;
- size_t mmap_threshold;
- size_t trim_threshold;
- flag_t default_mflags;
-};
-
-static struct malloc_params mparams;
-
-/* Ensure mparams initialized */
-#define ensure_initialization() (void)(mparams.magic != 0 || init_mparams())
-
-/* -------------------------- MMAP preliminaries ------------------------- */
-
-/*
- If HAVE_MORECORE or HAVE_MMAP are false, we just define calls and
- checks to fail so compiler optimizer can delete code rather than
- using so many "#if"s.
-*/
-
-
-/* MORECORE and MMAP must return MFAIL on failure */
-#define MFAIL ((void*)(MAX_SIZE_T))
-#define CMFAIL ((char*)(MFAIL)) /* defined for convenience */
-
-#if HAVE_MMAP
-
-#ifndef WIN32
-#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON)
-#define MAP_ANONYMOUS MAP_ANON
-#endif /* MAP_ANON */
-#ifdef DEFAULT_GRANULARITY_ALIGNED
-#define MMAP_IMPL mmap_aligned
-static void* lastAlignedmmap; /* Used as a hint */
-static void* mmap_aligned(void *start, size_t length, int prot, int flags, int fd, off_t offset) {
- void* baseaddress = 0;
- void* ptr = 0;
- if(!start) {
- baseaddress = lastAlignedmmap;
- for(;;) {
- if(baseaddress) flags|=MAP_FIXED;
- ptr = mmap(baseaddress, length, prot, flags, fd, offset);
- if(!ptr)
- baseaddress = (void*)((size_t)baseaddress + mparams.granularity);
- else if((size_t)ptr & (mparams.granularity - SIZE_T_ONE)) {
- munmap(ptr, length);
- baseaddress = (void*)(((size_t)ptr + mparams.granularity) & ~(mparams.granularity - SIZE_T_ONE));
- }
- else break;
- }
- }
- else ptr = mmap(start, length, prot, flags, fd, offset);
- if(ptr) lastAlignedmmap = (void*)((size_t) ptr + mparams.granularity);
- return ptr;
-}
-#else
-#define MMAP_IMPL mmap
-#endif /* DEFAULT_GRANULARITY_ALIGNED */
-#define MUNMAP_DEFAULT(a, s) munmap((a), (s))
-#define MMAP_PROT (PROT_READ|PROT_WRITE)
-#ifdef MAP_ANONYMOUS
-#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS)
-#define MMAP_DEFAULT(s) MMAP_IMPL(0, (s), MMAP_PROT, MMAP_FLAGS, -1, 0)
-#else /* MAP_ANONYMOUS */
-/*
- Nearly all versions of mmap support MAP_ANONYMOUS, so the following
- is unlikely to be needed, but is supplied just in case.
-*/
-#define MMAP_FLAGS (MAP_PRIVATE)
-static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */
-#define MMAP_DEFAULT(s) ((dev_zero_fd < 0) ? \
- (dev_zero_fd = open("/dev/zero", O_RDWR), \
- MMAP_IMPL(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) : \
- MMAP_IMPL(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0))
-#endif /* MAP_ANONYMOUS */
-
-#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s)
-
-#else /* WIN32 */
-
-/* Win32 MMAP via VirtualAlloc */
-#ifdef DEFAULT_GRANULARITY_ALIGNED
-static void* lastWin32mmap; /* Used as a hint */
-#endif /* DEFAULT_GRANULARITY_ALIGNED */
-#ifdef ENABLE_LARGE_PAGES
-static int largepagesavailable = 1;
-#endif /* ENABLE_LARGE_PAGES */
-static FORCEINLINE void* win32mmap(size_t size) {
- void* baseaddress = 0;
- void* ptr = 0;
-#ifdef ENABLE_LARGE_PAGES
- /* Note that large pages are *always* allocated on a large page boundary.
- If however granularity is small then don't waste a kernel call if size
- isn't around the size of a large page */
- if(largepagesavailable && size >= 1*1024*1024) {
- ptr = VirtualAlloc(baseaddress, size, MEM_RESERVE|MEM_COMMIT|MEM_LARGE_PAGES, PAGE_READWRITE);
- if(!ptr && ERROR_PRIVILEGE_NOT_HELD==GetLastError()) largepagesavailable=0;
- }
-#endif
- if(!ptr) {
-#ifdef DEFAULT_GRANULARITY_ALIGNED
- /* We try to avoid overhead by speculatively reserving at aligned
- addresses until we succeed */
- baseaddress = lastWin32mmap;
- for(;;) {
- void* reserveaddr = VirtualAlloc(baseaddress, size, MEM_RESERVE, PAGE_READWRITE);
- if(!reserveaddr)
- baseaddress = (void*)((size_t)baseaddress + mparams.granularity);
- else if((size_t)reserveaddr & (mparams.granularity - SIZE_T_ONE)) {
- VirtualFree(reserveaddr, 0, MEM_RELEASE);
- baseaddress = (void*)(((size_t)reserveaddr + mparams.granularity) & ~(mparams.granularity - SIZE_T_ONE));
- }
- else break;
- }
-#endif
- if(!ptr) ptr = VirtualAlloc(baseaddress, size, baseaddress ? MEM_COMMIT : MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE);
-#if DEBUG
- if(lastWin32mmap && ptr!=lastWin32mmap) printf("Non-contiguous VirtualAlloc between %p and %p\n", ptr, lastWin32mmap);
-#endif
-#ifdef DEFAULT_GRANULARITY_ALIGNED
- if(ptr) lastWin32mmap = (void*)((size_t) ptr + mparams.granularity);
-#endif
- }
-#if DEBUG
-#ifdef ENABLE_LARGE_PAGES
- printf("VirtualAlloc returns %p size %u. LargePagesAvailable=%d\n", ptr, size, largepagesavailable);
-#else
- printf("VirtualAlloc returns %p size %u\n", ptr, size);
-#endif
-#endif
- return (ptr != 0)? ptr: MFAIL;
-}
-
-/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */
-static FORCEINLINE void* win32direct_mmap(size_t size) {
- void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN,
- PAGE_READWRITE);
- return (ptr != 0)? ptr: MFAIL;
-}
-
-/* This function supports releasing coalesed segments */
-static FORCEINLINE int win32munmap(void* ptr, size_t size) {
- MEMORY_BASIC_INFORMATION minfo;
- char* cptr = (char*)ptr;
- while (size) {
- if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0)
- return -1;
- if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr ||
- minfo.State != MEM_COMMIT || minfo.RegionSize > size)
- return -1;
- if (VirtualFree(cptr, 0, MEM_RELEASE) == 0)
- return -1;
- cptr += minfo.RegionSize;
- size -= minfo.RegionSize;
- }
- return 0;
-}
-
-#define MMAP_DEFAULT(s) win32mmap(s)
-#define MUNMAP_DEFAULT(a, s) win32munmap((a), (s))
-#define DIRECT_MMAP_DEFAULT(s) win32direct_mmap(s)
-#endif /* WIN32 */
-#endif /* HAVE_MMAP */
-
-#if HAVE_MREMAP
-#ifndef WIN32
-#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv))
-#endif /* WIN32 */
-#endif /* HAVE_MREMAP */
-
-
-/**
- * Define CALL_MORECORE
- */
-#if HAVE_MORECORE
- #ifdef MORECORE
- #define CALL_MORECORE(S) MORECORE(S)
- #else /* MORECORE */
- #define CALL_MORECORE(S) MORECORE_DEFAULT(S)
- #endif /* MORECORE */
-#else /* HAVE_MORECORE */
- #define CALL_MORECORE(S) MFAIL
-#endif /* HAVE_MORECORE */
-
-/**
- * Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP
- */
-#if HAVE_MMAP
- #define USE_MMAP_BIT (SIZE_T_ONE)
-
- #ifdef MMAP
- #define CALL_MMAP(s) MMAP(s)
- #else /* MMAP */
- #define CALL_MMAP(s) MMAP_DEFAULT(s)
- #endif /* MMAP */
- #ifdef MUNMAP
- #define CALL_MUNMAP(a, s) MUNMAP((a), (s))
- #else /* MUNMAP */
- #define CALL_MUNMAP(a, s) MUNMAP_DEFAULT((a), (s))
- #endif /* MUNMAP */
- #ifdef DIRECT_MMAP
- #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
- #else /* DIRECT_MMAP */
- #define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s)
- #endif /* DIRECT_MMAP */
-#else /* HAVE_MMAP */
- #define USE_MMAP_BIT (SIZE_T_ZERO)
-
- #define MMAP(s) MFAIL
- #define MUNMAP(a, s) (-1)
- #define DIRECT_MMAP(s) MFAIL
- #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s)
- #define CALL_MMAP(s) MMAP(s)
- #define CALL_MUNMAP(a, s) MUNMAP((a), (s))
-#endif /* HAVE_MMAP */
-
-/**
- * Define CALL_MREMAP
- */
-#if HAVE_MMAP && HAVE_MREMAP
- #ifdef MREMAP
- #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv))
- #else /* MREMAP */
- #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP_DEFAULT((addr), (osz), (nsz), (mv))
- #endif /* MREMAP */
-#else /* HAVE_MMAP && HAVE_MREMAP */
- #define CALL_MREMAP(addr, osz, nsz, mv) MFAIL
-#endif /* HAVE_MMAP && HAVE_MREMAP */
-
-/* mstate bit set if continguous morecore disabled or failed */
-#define USE_NONCONTIGUOUS_BIT (4U)
-
-/* segment bit set in create_mspace_with_base */
-#define EXTERN_BIT (8U)
-
-
-/* --------------------------- Lock preliminaries ------------------------ */
-
-/*
- When locks are defined, there is one global lock, plus
- one per-mspace lock.
-
- The global lock_ensures that mparams.magic and other unique
- mparams values are initialized only once. It also protects
- sequences of calls to MORECORE. In many cases sys_alloc requires
- two calls, that should not be interleaved with calls by other
- threads. This does not protect against direct calls to MORECORE
- by other threads not using this lock, so there is still code to
- cope the best we can on interference.
-
- Per-mspace locks surround calls to malloc, free, etc. To enable use
- in layered extensions, per-mspace locks are reentrant.
-
- Because lock-protected regions generally have bounded times, it is
- OK to use the supplied simple spinlocks in the custom versions for
- x86. Spinlocks are likely to improve performance for lightly
- contended applications, but worsen performance under heavy
- contention.
-
- If USE_LOCKS is > 1, the definitions of lock routines here are
- bypassed, in which case you will need to define the type MLOCK_T,
- and at least INITIAL_LOCK, ACQUIRE_LOCK, RELEASE_LOCK and possibly
- TRY_LOCK (which is not used in this malloc, but commonly needed in
- extensions.) You must also declare a
- static MLOCK_T malloc_global_mutex = { initialization values };.
-
-*/
-
-#if USE_LOCKS == 1
-
-#if USE_SPIN_LOCKS && SPIN_LOCKS_AVAILABLE
-#ifndef WIN32
-
-/* Custom pthread-style spin locks on x86 and x64 for gcc */
-struct pthread_mlock_t {
- volatile unsigned int l;
- char cachelinepadding[64];
- unsigned int c;
- pthread_t threadid;
-};
-#define MLOCK_T struct pthread_mlock_t
-#define CURRENT_THREAD pthread_self()
-#define INITIAL_LOCK(sl) ((sl)->threadid = 0, (sl)->l = (sl)->c = 0, 0)
-#define ACQUIRE_LOCK(sl) pthread_acquire_lock(sl)
-#define RELEASE_LOCK(sl) pthread_release_lock(sl)
-#define TRY_LOCK(sl) pthread_try_lock(sl)
-#define SPINS_PER_YIELD 63
-
-static MLOCK_T malloc_global_mutex = { 0, "", 0, 0};
-
-static FORCEINLINE int pthread_acquire_lock (MLOCK_T *sl) {
- int spins = 0;
- volatile unsigned int* lp = &sl->l;
- for (;;) {
- if (*lp != 0) {
- if (sl->threadid == CURRENT_THREAD) {
- ++sl->c;
- return 0;
- }
- }
- else {
- /* place args to cmpxchgl in locals to evade oddities in some gccs */
- int cmp = 0;
- int val = 1;
- int ret;
- __asm__ __volatile__ ("lock; cmpxchgl %1, %2"
- : "=a" (ret)
- : "r" (val), "m" (*(lp)), "0"(cmp)
- : "memory", "cc");
- if (!ret) {
- assert(!sl->threadid);
- sl->threadid = CURRENT_THREAD;
- sl->c = 1;
- return 0;
- }
- }
- if ((++spins & SPINS_PER_YIELD) == 0) {
-#if defined (__SVR4) && defined (__sun) /* solaris */
- thr_yield();
-#else
-#if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)
- sched_yield();
-#else /* no-op yield on unknown systems */
- ;
-#endif /* __linux__ || __FreeBSD__ || __APPLE__ */
-#endif /* solaris */
- }
- }
-}
-
-static FORCEINLINE void pthread_release_lock (MLOCK_T *sl) {
- volatile unsigned int* lp = &sl->l;
- assert(*lp != 0);
- assert(sl->threadid == CURRENT_THREAD);
- if (--sl->c == 0) {
- sl->threadid = 0;
- int prev = 0;
- int ret;
- __asm__ __volatile__ ("lock; xchgl %0, %1"
- : "=r" (ret)
- : "m" (*(lp)), "0"(prev)
- : "memory");
- }
-}
-
-static FORCEINLINE int pthread_try_lock (MLOCK_T *sl) {
- volatile unsigned int* lp = &sl->l;
- if (*lp != 0) {
- if (sl->threadid == CURRENT_THREAD) {
- ++sl->c;
- return 1;
- }
- }
- else {
- int cmp = 0;
- int val = 1;
- int ret;
- __asm__ __volatile__ ("lock; cmpxchgl %1, %2"
- : "=a" (ret)
- : "r" (val), "m" (*(lp)), "0"(cmp)
- : "memory", "cc");
- if (!ret) {
- assert(!sl->threadid);
- sl->threadid = CURRENT_THREAD;
- sl->c = 1;
- return 1;
- }
- }
- return 0;
-}
-
-
-#else /* WIN32 */
-/* Custom win32-style spin locks on x86 and x64 for MSC */
-struct win32_mlock_t {
- volatile long l;
- char cachelinepadding[64];
- unsigned int c;
- long threadid;
-};
-
-#define MLOCK_T struct win32_mlock_t
-#define CURRENT_THREAD ((long)GetCurrentThreadId())
-#define INITIAL_LOCK(sl) ((sl)->threadid = 0, (sl)->l = (sl)->c = 0, 0)
-#define ACQUIRE_LOCK(sl) win32_acquire_lock(sl)
-#define RELEASE_LOCK(sl) win32_release_lock(sl)
-#define TRY_LOCK(sl) win32_try_lock(sl)
-#define SPINS_PER_YIELD 63
-
-static MLOCK_T malloc_global_mutex = { 0, 0, 0};
-
-static FORCEINLINE int win32_acquire_lock (MLOCK_T *sl) {
- int spins = 0;
- for (;;) {
- if (sl->l != 0) {
- if (sl->threadid == CURRENT_THREAD) {
- ++sl->c;
- return 0;
- }
- }
- else {
- if (!interlockedexchange(&sl->l, 1)) {
- assert(!sl->threadid);
- sl->threadid = CURRENT_THREAD;
- sl->c = 1;
- return 0;
- }
- }
- if ((++spins & SPINS_PER_YIELD) == 0)
- SleepEx(0, FALSE);
- }
-}
-
-static FORCEINLINE void win32_release_lock (MLOCK_T *sl) {
- assert(sl->threadid == CURRENT_THREAD);
- assert(sl->l != 0);
- if (--sl->c == 0) {
- sl->threadid = 0;
- interlockedexchange (&sl->l, 0);
- }
-}
-
-static FORCEINLINE int win32_try_lock (MLOCK_T *sl) {
- if (sl->l != 0) {
- if (sl->threadid == CURRENT_THREAD) {
- ++sl->c;
- return 1;
- }
- }
- else {
- if (!interlockedexchange(&sl->l, 1)){
- assert(!sl->threadid);
- sl->threadid = CURRENT_THREAD;
- sl->c = 1;
- return 1;
- }
- }
- return 0;
-}
-
-#endif /* WIN32 */
-#else /* USE_SPIN_LOCKS */
-
-#ifndef WIN32
-/* pthreads-based locks */
-
-#define MLOCK_T pthread_mutex_t
-#define CURRENT_THREAD pthread_self()
-#define INITIAL_LOCK(sl) pthread_init_lock(sl)
-#define ACQUIRE_LOCK(sl) pthread_mutex_lock(sl)
-#define RELEASE_LOCK(sl) pthread_mutex_unlock(sl)
-#define TRY_LOCK(sl) (!pthread_mutex_trylock(sl))
-
-static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER;
-
-/* Cope with old-style linux recursive lock initialization by adding */
-/* skipped internal declaration from pthread.h */
-#ifdef linux
-#ifndef PTHREAD_MUTEX_RECURSIVE
-extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr,
- int __kind));
-#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP
-#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y)
-#endif
-#endif
-
-static int pthread_init_lock (MLOCK_T *sl) {
- pthread_mutexattr_t attr;
- if (pthread_mutexattr_init(&attr)) return 1;
- if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1;
- if (pthread_mutex_init(sl, &attr)) return 1;
- if (pthread_mutexattr_destroy(&attr)) return 1;
- return 0;
-}
-
-#else /* WIN32 */
-/* Win32 critical sections */
-#define MLOCK_T CRITICAL_SECTION
-#define CURRENT_THREAD GetCurrentThreadId()
-#define INITIAL_LOCK(s) (!InitializeCriticalSectionAndSpinCount((s), 0x80000000|4000))
-#define ACQUIRE_LOCK(s) (EnterCriticalSection(sl), 0)
-#define RELEASE_LOCK(s) LeaveCriticalSection(sl)
-#define TRY_LOCK(s) TryEnterCriticalSection(sl)
-#define NEED_GLOBAL_LOCK_INIT
-
-static MLOCK_T malloc_global_mutex;
-static volatile long malloc_global_mutex_status;
-
-/* Use spin loop to initialize global lock */
-static void init_malloc_global_mutex() {
- for (;;) {
- long stat = malloc_global_mutex_status;
- if (stat > 0)
- return;
- /* transition to < 0 while initializing, then to > 0) */
- if (stat == 0 &&
- interlockedcompareexchange(&malloc_global_mutex_status, -1, 0) == 0) {
- InitializeCriticalSection(&malloc_global_mutex);
- interlockedexchange(&malloc_global_mutex_status,1);
- return;
- }
- SleepEx(0, FALSE);
- }
-}
-
-#endif /* WIN32 */
-#endif /* USE_SPIN_LOCKS */
-#endif /* USE_LOCKS == 1 */
-
-/* ----------------------- User-defined locks ------------------------ */
-
-#if USE_LOCKS > 1
-/* Define your own lock implementation here */
-/* #define INITIAL_LOCK(sl) ... */
-/* #define ACQUIRE_LOCK(sl) ... */
-/* #define RELEASE_LOCK(sl) ... */
-/* #define TRY_LOCK(sl) ... */
-/* static MLOCK_T malloc_global_mutex = ... */
-#endif /* USE_LOCKS > 1 */
-
-/* ----------------------- Lock-based state ------------------------ */
-
-#if USE_LOCKS
-#define USE_LOCK_BIT (2U)
-#else /* USE_LOCKS */
-#define USE_LOCK_BIT (0U)
-#define INITIAL_LOCK(l)
-#endif /* USE_LOCKS */
-
-#if USE_LOCKS
-#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK
-#define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex);
-#endif
-#ifndef RELEASE_MALLOC_GLOBAL_LOCK
-#define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex);
-#endif
-#else /* USE_LOCKS */
-#define ACQUIRE_MALLOC_GLOBAL_LOCK()
-#define RELEASE_MALLOC_GLOBAL_LOCK()
-#endif /* USE_LOCKS */
-
-
-/* ----------------------- Chunk representations ------------------------ */
-
-/*
- (The following includes lightly edited explanations by Colin Plumb.)
-
- The malloc_chunk declaration below is misleading (but accurate and
- necessary). It declares a "view" into memory allowing access to
- necessary fields at known offsets from a given base.
-
- Chunks of memory are maintained using a `boundary tag' method as
- originally described by Knuth. (See the paper by Paul Wilson
- ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a survey of such
- techniques.) Sizes of free chunks are stored both in the front of
- each chunk and at the end. This makes consolidating fragmented
- chunks into bigger chunks fast. The head fields also hold bits
- representing whether chunks are free or in use.
-
- Here are some pictures to make it clearer. They are "exploded" to
- show that the state of a chunk can be thought of as extending from
- the high 31 bits of the head field of its header through the
- prev_foot and PINUSE_BIT bit of the following chunk header.
-
- A chunk that's in use looks like:
-
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Size of previous chunk (if P = 0) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
- | Size of this chunk 1| +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | |
- +- -+
- | |
- +- -+
- | :
- +- size - sizeof(size_t) available payload bytes -+
- : |
- chunk-> +- -+
- | |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |1|
- | Size of next chunk (may or may not be in use) | +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
- And if it's free, it looks like this:
-
- chunk-> +- -+
- | User payload (must be in use, or we would have merged!) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P|
- | Size of this chunk 0| +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Next pointer |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Prev pointer |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | :
- +- size - sizeof(struct chunk) unused bytes -+
- : |
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Size of this chunk |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |0|
- | Size of next chunk (must be in use, or we would have merged)| +-+
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | :
- +- User payload -+
- : |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- |0|
- +-+
- Note that since we always merge adjacent free chunks, the chunks
- adjacent to a free chunk must be in use.
-
- Given a pointer to a chunk (which can be derived trivially from the
- payload pointer) we can, in O(1) time, find out whether the adjacent
- chunks are free, and if so, unlink them from the lists that they
- are on and merge them with the current chunk.
-
- Chunks always begin on even word boundaries, so the mem portion
- (which is returned to the user) is also on an even word boundary, and
- thus at least double-word aligned.
-
- The P (PINUSE_BIT) bit, stored in the unused low-order bit of the
- chunk size (which is always a multiple of two words), is an in-use
- bit for the *previous* chunk. If that bit is *clear*, then the
- word before the current chunk size contains the previous chunk
- size, and can be used to find the front of the previous chunk.
- The very first chunk allocated always has this bit set, preventing
- access to non-existent (or non-owned) memory. If pinuse is set for
- any given chunk, then you CANNOT determine the size of the
- previous chunk, and might even get a memory addressing fault when
- trying to do so.
-
- The C (CINUSE_BIT) bit, stored in the unused second-lowest bit of
- the chunk size redundantly records whether the current chunk is
- inuse (unless the chunk is mmapped). This redundancy enables usage
- checks within free and realloc, and reduces indirection when freeing
- and consolidating chunks.
-
- Each freshly allocated chunk must have both cinuse and pinuse set.
- That is, each allocated chunk borders either a previously allocated
- and still in-use chunk, or the base of its memory arena. This is
- ensured by making all allocations from the the `lowest' part of any
- found chunk. Further, no free chunk physically borders another one,
- so each free chunk is known to be preceded and followed by either
- inuse chunks or the ends of memory.
-
- Note that the `foot' of the current chunk is actually represented
- as the prev_foot of the NEXT chunk. This makes it easier to
- deal with alignments etc but can be very confusing when trying
- to extend or adapt this code.
-
- The exceptions to all this are
-
- 1. The special chunk `top' is the top-most available chunk (i.e.,
- the one bordering the end of available memory). It is treated
- specially. Top is never included in any bin, is used only if
- no other chunk is available, and is released back to the
- system if it is very large (see M_TRIM_THRESHOLD). In effect,
- the top chunk is treated as larger (and thus less well
- fitting) than any other available chunk. The top chunk
- doesn't update its trailing size field since there is no next
- contiguous chunk that would have to index off it. However,
- space is still allocated for it (TOP_FOOT_SIZE) to enable
- separation or merging when space is extended.
-
- 3. Chunks allocated via mmap, have both cinuse and pinuse bits
- cleared in their head fields. Because they are allocated
- one-by-one, each must carry its own prev_foot field, which is
- also used to hold the offset this chunk has within its mmapped
- region, which is needed to preserve alignment. Each mmapped
- chunk is trailed by the first two fields of a fake next-chunk
- for sake of usage checks.
-
-*/
-
-struct malloc_chunk {
- size_t prev_foot; /* Size of previous chunk (if free). */
- size_t head; /* Size and inuse bits. */
- struct malloc_chunk* fd; /* double links -- used only if free. */
- struct malloc_chunk* bk;
-};
-
-typedef struct malloc_chunk mchunk;
-typedef struct malloc_chunk* mchunkptr;
-typedef struct malloc_chunk* sbinptr; /* The type of bins of chunks */
-typedef unsigned int bindex_t; /* Described below */
-typedef unsigned int binmap_t; /* Described below */
-
-/* ------------------- Chunks sizes and alignments ----------------------- */
-
-#define MCHUNK_SIZE (sizeof(mchunk))
-
-#if FOOTERS
-#define CHUNK_OVERHEAD (TWO_SIZE_T_SIZES)
-#else /* FOOTERS */
-#define CHUNK_OVERHEAD (SIZE_T_SIZE)
-#endif /* FOOTERS */
-
-/* MMapped chunks need a second word of overhead ... */
-#define MMAP_CHUNK_OVERHEAD (TWO_SIZE_T_SIZES)
-/* ... and additional padding for fake next-chunk at foot */
-#define MMAP_FOOT_PAD (FOUR_SIZE_T_SIZES)
-
-/* The smallest size we can malloc is an aligned minimal chunk */
-#define MIN_CHUNK_SIZE\
- ((MCHUNK_SIZE + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
-
-/* conversion from malloc headers to user pointers, and back */
-#define chunk2mem(p) ((void*)((char*)(p) + TWO_SIZE_T_SIZES))
-#define mem2chunk(mem) ((mchunkptr)((char*)(mem) - TWO_SIZE_T_SIZES))
-/* chunk associated with aligned address A */
-#define align_as_chunk(A) (mchunkptr)((A) + align_offset(chunk2mem(A)))
-
-/* Bounds on request (not chunk) sizes. */
-#define MAX_REQUEST ((-MIN_CHUNK_SIZE) << 2)
-#define MIN_REQUEST (MIN_CHUNK_SIZE - CHUNK_OVERHEAD - SIZE_T_ONE)
-
-/* pad request bytes into a usable size */
-#define pad_request(req) \
- (((req) + CHUNK_OVERHEAD + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK)
-
-/* pad request, checking for minimum (but not maximum) */
-#define request2size(req) \
- (((req) < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(req))
-
-
-/* ------------------ Operations on head and foot fields ----------------- */
-
-/*
- The head field of a chunk is or'ed with PINUSE_BIT when previous
- adjacent chunk in use, and or'ed with CINUSE_BIT if this chunk is in
- use, unless mmapped, in which case both bits are cleared.
-
- FLAG4_BIT is not used by this malloc, but might be useful in extensions.
-*/
-
-#define PINUSE_BIT (SIZE_T_ONE)
-#define CINUSE_BIT (SIZE_T_TWO)
-#define FLAG4_BIT (SIZE_T_FOUR)
-#define INUSE_BITS (PINUSE_BIT|CINUSE_BIT)
-#define FLAG_BITS (PINUSE_BIT|CINUSE_BIT|FLAG4_BIT)
-
-/* Head value for fenceposts */
-#define FENCEPOST_HEAD (INUSE_BITS|SIZE_T_SIZE)
-
-/* extraction of fields from head words */
-#define cinuse(p) ((p)->head & CINUSE_BIT)
-#define pinuse(p) ((p)->head & PINUSE_BIT)
-#define is_inuse(p) (((p)->head & INUSE_BITS) != PINUSE_BIT)
-#define is_mmapped(p) (((p)->head & INUSE_BITS) == 0)
-
-#define chunksize(p) ((p)->head & ~(FLAG_BITS))
-
-#define clear_pinuse(p) ((p)->head &= ~PINUSE_BIT)
-
-/* Treat space at ptr +/- offset as a chunk */
-#define chunk_plus_offset(p, s) ((mchunkptr)(((char*)(p)) + (s)))
-#define chunk_minus_offset(p, s) ((mchunkptr)(((char*)(p)) - (s)))
-
-/* Ptr to next or previous physical malloc_chunk. */
-#define next_chunk(p) ((mchunkptr)( ((char*)(p)) + ((p)->head & ~FLAG_BITS)))
-#define prev_chunk(p) ((mchunkptr)( ((char*)(p)) - ((p)->prev_foot) ))
-
-/* extract next chunk's pinuse bit */
-#define next_pinuse(p) ((next_chunk(p)->head) & PINUSE_BIT)
-
-/* Get/set size at footer */
-#define get_foot(p, s) (((mchunkptr)((char*)(p) + (s)))->prev_foot)
-#define set_foot(p, s) (((mchunkptr)((char*)(p) + (s)))->prev_foot = (s))
-
-/* Set size, pinuse bit, and foot */
-#define set_size_and_pinuse_of_free_chunk(p, s)\
- ((p)->head = (s|PINUSE_BIT), set_foot(p, s))
-
-/* Set size, pinuse bit, foot, and clear next pinuse */
-#define set_free_with_pinuse(p, s, n)\
- (clear_pinuse(n), set_size_and_pinuse_of_free_chunk(p, s))
-
-/* Get the internal overhead associated with chunk p */
-#define overhead_for(p)\
- (is_mmapped(p)? MMAP_CHUNK_OVERHEAD : CHUNK_OVERHEAD)
-
-/* Return true if malloced space is not necessarily cleared */
-#if MMAP_CLEARS
-#define calloc_must_clear(p) (!is_mmapped(p))
-#else /* MMAP_CLEARS */
-#define calloc_must_clear(p) (1)
-#endif /* MMAP_CLEARS */
-
-/* ---------------------- Overlaid data structures ----------------------- */
-
-/*
- When chunks are not in use, they are treated as nodes of either
- lists or trees.
-
- "Small" chunks are stored in circular doubly-linked lists, and look
- like this:
-
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Size of previous chunk |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- `head:' | Size of chunk, in bytes |P|
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Forward pointer to next chunk in list |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Back pointer to previous chunk in list |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Unused space (may be 0 bytes long) .
- . .
- . |
-nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- `foot:' | Size of chunk, in bytes |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
- Larger chunks are kept in a form of bitwise digital trees (aka
- tries) keyed on chunksizes. Because malloc_tree_chunks are only for
- free chunks greater than 256 bytes, their size doesn't impose any
- constraints on user chunk sizes. Each node looks like:
-
- chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Size of previous chunk |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- `head:' | Size of chunk, in bytes |P|
- mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Forward pointer to next chunk of same size |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Back pointer to previous chunk of same size |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Pointer to left child (child[0]) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Pointer to right child (child[1]) |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Pointer to parent |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | bin index of this chunk |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- | Unused space .
- . |
-nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
- `foot:' | Size of chunk, in bytes |
- +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+
-
- Each tree holding treenodes is a tree of unique chunk sizes. Chunks
- of the same size are arranged in a circularly-linked list, with only
- the oldest chunk (the next to be used, in our FIFO ordering)
- actually in the tree. (Tree members are distinguished by a non-null
- parent pointer.) If a chunk with the same size an an existing node
- is inserted, it is linked off the existing node using pointers that
- work in the same way as fd/bk pointers of small chunks.
-
- Each tree contains a power of 2 sized range of chunk sizes (the
- smallest is 0x100 <= x < 0x180), which is is divided in half at each
- tree level, with the chunks in the smaller half of the range (0x100
- <= x < 0x140 for the top nose) in the left subtree and the larger
- half (0x140 <= x < 0x180) in the right subtree. This is, of course,
- done by inspecting individual bits.
-
- Using these rules, each node's left subtree contains all smaller
- sizes than its right subtree. However, the node at the root of each
- subtree has no particular ordering relationship to either. (The
- dividing line between the subtree sizes is based on trie relation.)
- If we remove the last chunk of a given size from the interior of the
- tree, we need to replace it with a leaf node. The tree ordering
- rules permit a node to be replaced by any leaf below it.
-
- The smallest chunk in a tree (a common operation in a best-fit
- allocator) can be found by walking a path to the leftmost leaf in
- the tree. Unlike a usual binary tree, where we follow left child
- pointers until we reach a null, here we follow the right child
- pointer any time the left one is null, until we reach a leaf with
- both child pointers null. The smallest chunk in the tree will be
- somewhere along that path.
-
- The worst case number of steps to add, find, or remove a node is
- bounded by the number of bits differentiating chunks within
- bins. Under current bin calculations, this ranges from 6 up to 21
- (for 32 bit sizes) or up to 53 (for 64 bit sizes). The typical case
- is of course much better.
-*/
-
-struct malloc_tree_chunk {
- /* The first four fields must be compatible with malloc_chunk */
- size_t prev_foot;
- size_t head;
- struct malloc_tree_chunk* fd;
- struct malloc_tree_chunk* bk;
-
- struct malloc_tree_chunk* child[2];
- struct malloc_tree_chunk* parent;
- bindex_t index;
-};
-
-typedef struct malloc_tree_chunk tchunk;
-typedef struct malloc_tree_chunk* tchunkptr;
-typedef struct malloc_tree_chunk* tbinptr; /* The type of bins of trees */
-
-/* A little helper macro for trees */
-#define leftmost_child(t) ((t)->child[0] != 0? (t)->child[0] : (t)->child[1])
-
-/* ----------------------------- Segments -------------------------------- */
-
-/*
- Each malloc space may include non-contiguous segments, held in a
- list headed by an embedded malloc_segment record representing the
- top-most space. Segments also include flags holding properties of
- the space. Large chunks that are directly allocated by mmap are not
- included in this list. They are instead independently created and
- destroyed without otherwise keeping track of them.
-
- Segment management mainly comes into play for spaces allocated by
- MMAP. Any call to MMAP might or might not return memory that is
- adjacent to an existing segment. MORECORE normally contiguously
- extends the current space, so this space is almost always adjacent,
- which is simpler and faster to deal with. (This is why MORECORE is
- used preferentially to MMAP when both are available -- see
- sys_alloc.) When allocating using MMAP, we don't use any of the
- hinting mechanisms (inconsistently) supported in various
- implementations of unix mmap, or distinguish reserving from
- committing memory. Instead, we just ask for space, and exploit
- contiguity when we get it. It is probably possible to do
- better than this on some systems, but no general scheme seems
- to be significantly better.
-
- Management entails a simpler variant of the consolidation scheme
- used for chunks to reduce fragmentation -- new adjacent memory is
- normally prepended or appended to an existing segment. However,
- there are limitations compared to chunk consolidation that mostly
- reflect the fact that segment processing is relatively infrequent
- (occurring only when getting memory from system) and that we
- don't expect to have huge numbers of segments:
-
- * Segments are not indexed, so traversal requires linear scans. (It
- would be possible to index these, but is not worth the extra
- overhead and complexity for most programs on most platforms.)
- * New segments are only appended to old ones when holding top-most
- memory; if they cannot be prepended to others, they are held in
- different segments.
-
- Except for the top-most segment of an mstate, each segment record
- is kept at the tail of its segment. Segments are added by pushing
- segment records onto the list headed by &mstate.seg for the
- containing mstate.
-
- Segment flags control allocation/merge/deallocation policies:
- * If EXTERN_BIT set, then we did not allocate this segment,
- and so should not try to deallocate or merge with others.
- (This currently holds only for the initial segment passed
- into create_mspace_with_base.)
- * If USE_MMAP_BIT set, the segment may be merged with
- other surrounding mmapped segments and trimmed/de-allocated
- using munmap.
- * If neither bit is set, then the segment was obtained using
- MORECORE so can be merged with surrounding MORECORE'd segments
- and deallocated/trimmed using MORECORE with negative arguments.
-*/
-
-struct malloc_segment {
- char* base; /* base address */
- size_t size; /* allocated size */
- struct malloc_segment* next; /* ptr to next segment */
- flag_t sflags; /* mmap and extern flag */
-};
-
-#define is_mmapped_segment(S) ((S)->sflags & USE_MMAP_BIT)
-#define is_extern_segment(S) ((S)->sflags & EXTERN_BIT)
-
-typedef struct malloc_segment msegment;
-typedef struct malloc_segment* msegmentptr;
-
-/* ---------------------------- malloc_state ----------------------------- */
-
-/*
- A malloc_state holds all of the bookkeeping for a space.
- The main fields are:
-
- Top
- The topmost chunk of the currently active segment. Its size is
- cached in topsize. The actual size of topmost space is
- topsize+TOP_FOOT_SIZE, which includes space reserved for adding
- fenceposts and segment records if necessary when getting more
- space from the system. The size at which to autotrim top is
- cached from mparams in trim_check, except that it is disabled if
- an autotrim fails.
-
- Designated victim (dv)
- This is the preferred chunk for servicing small requests that
- don't have exact fits. It is normally the chunk split off most
- recently to service another small request. Its size is cached in
- dvsize. The link fields of this chunk are not maintained since it
- is not kept in a bin.
-
- SmallBins
- An array of bin headers for free chunks. These bins hold chunks
- with sizes less than MIN_LARGE_SIZE bytes. Each bin contains
- chunks of all the same size, spaced 8 bytes apart. To simplify
- use in double-linked lists, each bin header acts as a malloc_chunk
- pointing to the real first node, if it exists (else pointing to
- itself). This avoids special-casing for headers. But to avoid
- waste, we allocate only the fd/bk pointers of bins, and then use
- repositioning tricks to treat these as the fields of a chunk.
-
- TreeBins
- Treebins are pointers to the roots of trees holding a range of
- sizes. There are 2 equally spaced treebins for each power of two
- from TREE_SHIFT to TREE_SHIFT+16. The last bin holds anything
- larger.
-
- Bin maps
- There is one bit map for small bins ("smallmap") and one for
- treebins ("treemap). Each bin sets its bit when non-empty, and
- clears the bit when empty. Bit operations are then used to avoid
- bin-by-bin searching -- nearly all "search" is done without ever
- looking at bins that won't be selected. The bit maps
- conservatively use 32 bits per map word, even if on 64bit system.
- For a good description of some of the bit-based techniques used
- here, see Henry S. Warren Jr's book "Hacker's Delight" (and
- supplement at http://hackersdelight.org/). Many of these are
- intended to reduce the branchiness of paths through malloc etc, as
- well as to reduce the number of memory locations read or written.
-
- Segments
- A list of segments headed by an embedded malloc_segment record
- representing the initial space.
-
- Address check support
- The least_addr field is the least address ever obtained from
- MORECORE or MMAP. Attempted frees and reallocs of any address less
- than this are trapped (unless INSECURE is defined).
-
- Magic tag
- A cross-check field that should always hold same value as mparams.magic.
-
- Flags
- Bits recording whether to use MMAP, locks, or contiguous MORECORE
-
- Statistics
- Each space keeps track of current and maximum system memory
- obtained via MORECORE or MMAP.
-
- Trim support
- Fields holding the amount of unused topmost memory that should trigger
- timming, and a counter to force periodic scanning to release unused
- non-topmost segments.
-
- Locking
- If USE_LOCKS is defined, the "mutex" lock is acquired and released
- around every public call using this mspace.
-
- Extension support
- A void* pointer and a size_t field that can be used to help implement
- extensions to this malloc.
-*/
-
-/* Bin types, widths and sizes */
-#define NSMALLBINS (32U)
-#define NTREEBINS (32U)
-#define SMALLBIN_SHIFT (3U)
-#define SMALLBIN_WIDTH (SIZE_T_ONE << SMALLBIN_SHIFT)
-#define TREEBIN_SHIFT (8U)
-#define MIN_LARGE_SIZE (SIZE_T_ONE << TREEBIN_SHIFT)
-#define MAX_SMALL_SIZE (MIN_LARGE_SIZE - SIZE_T_ONE)
-#define MAX_SMALL_REQUEST (MAX_SMALL_SIZE - CHUNK_ALIGN_MASK - CHUNK_OVERHEAD)
-
-struct malloc_state {
- binmap_t smallmap;
- binmap_t treemap;
- size_t dvsize;
- size_t topsize;
- char* least_addr;
- mchunkptr dv;
- mchunkptr top;
- size_t trim_check;
- size_t release_checks;
- size_t magic;
- mchunkptr smallbins[(NSMALLBINS+1)*2];
- tbinptr treebins[NTREEBINS];
- size_t footprint;
- size_t max_footprint;
- flag_t mflags;
- msegment seg;
-#if USE_LOCKS
- MLOCK_T mutex; /* locate lock among fields that rarely change */
-#endif /* USE_LOCKS */
- void* extp; /* Unused but available for extensions */
- size_t exts;
-};
-
-typedef struct malloc_state* mstate;
-
-/* ------------- Global malloc_state and malloc_params ------------------- */
-
-#if !ONLY_MSPACES
-
-/* The global malloc_state used for all non-"mspace" calls */
-static struct malloc_state _gm_;
-#define gm (&_gm_)
-#define is_global(M) ((M) == &_gm_)
-
-#endif /* !ONLY_MSPACES */
-
-#define is_initialized(M) ((M)->top != 0)
-
-/* -------------------------- system alloc setup ------------------------- */
-
-/* Operations on mflags */
-
-#define use_lock(M) ((M)->mflags & USE_LOCK_BIT)
-#define enable_lock(M) ((M)->mflags |= USE_LOCK_BIT)
-#define disable_lock(M) ((M)->mflags &= ~USE_LOCK_BIT)
-
-#define use_mmap(M) ((M)->mflags & USE_MMAP_BIT)
-#define enable_mmap(M) ((M)->mflags |= USE_MMAP_BIT)
-#define disable_mmap(M) ((M)->mflags &= ~USE_MMAP_BIT)
-
-#define use_noncontiguous(M) ((M)->mflags & USE_NONCONTIGUOUS_BIT)
-#define disable_contiguous(M) ((M)->mflags |= USE_NONCONTIGUOUS_BIT)
-
-#define set_lock(M,L)\
- ((M)->mflags = (L)?\
- ((M)->mflags | USE_LOCK_BIT) :\
- ((M)->mflags & ~USE_LOCK_BIT))
-
-/* page-align a size */
-#define page_align(S)\
- (((S) + (mparams.page_size - SIZE_T_ONE)) & ~(mparams.page_size - SIZE_T_ONE))
-
-/* granularity-align a size */
-#define granularity_align(S)\
- (((S) + (mparams.granularity - SIZE_T_ONE))\
- & ~(mparams.granularity - SIZE_T_ONE))
-
-
-/* For mmap, use granularity alignment on windows, else page-align */
-#ifdef WIN32
-#define mmap_align(S) granularity_align(S)
-#else
-#define mmap_align(S) page_align(S)
-#endif
-
-/* For sys_alloc, enough padding to ensure can malloc request on success */
-#define SYS_ALLOC_PADDING (TOP_FOOT_SIZE + MALLOC_ALIGNMENT)
-
-#define is_page_aligned(S)\
- (((size_t)(S) & (mparams.page_size - SIZE_T_ONE)) == 0)
-#define is_granularity_aligned(S)\
- (((size_t)(S) & (mparams.granularity - SIZE_T_ONE)) == 0)
-
-/* True if segment S holds address A */
-#define segment_holds(S, A)\
- ((char*)(A) >= S->base && (char*)(A) < S->base + S->size)
-
-/* Return segment holding given address */
-static msegmentptr segment_holding(mstate m, char* addr) {
- msegmentptr sp = &m->seg;
- for (;;) {
- if (addr >= sp->base && addr < sp->base + sp->size)
- return sp;
- if ((sp = sp->next) == 0)
- return 0;
- }
-}
-
-/* Return true if segment contains a segment link */
-static int has_segment_link(mstate m, msegmentptr ss) {
- msegmentptr sp = &m->seg;
- for (;;) {
- if ((char*)sp >= ss->base && (char*)sp < ss->base + ss->size)
- return 1;
- if ((sp = sp->next) == 0)
- return 0;
- }
-}
-
-#ifndef MORECORE_CANNOT_TRIM
-#define should_trim(M,s) ((s) > (M)->trim_check)
-#else /* MORECORE_CANNOT_TRIM */
-#define should_trim(M,s) (0)
-#endif /* MORECORE_CANNOT_TRIM */
-
-/*
- TOP_FOOT_SIZE is padding at the end of a segment, including space
- that may be needed to place segment records and fenceposts when new
- noncontiguous segments are added.
-*/
-#define TOP_FOOT_SIZE\
- (align_offset(chunk2mem(0))+pad_request(sizeof(struct malloc_segment))+MIN_CHUNK_SIZE)
-
-
-/* ------------------------------- Hooks -------------------------------- */
-
-/*
- PREACTION should be defined to return 0 on success, and nonzero on
- failure. If you are not using locking, you can redefine these to do
- anything you like.
-*/
-
-#if USE_LOCKS
-
-#define PREACTION(M) ((use_lock(M))? ACQUIRE_LOCK(&(M)->mutex) : 0)
-#define POSTACTION(M) { if (use_lock(M)) RELEASE_LOCK(&(M)->mutex); }
-#else /* USE_LOCKS */
-
-#ifndef PREACTION
-#define PREACTION(M) (0)
-#endif /* PREACTION */
-
-#ifndef POSTACTION
-#define POSTACTION(M)
-#endif /* POSTACTION */
-
-#endif /* USE_LOCKS */
-
-/*
- CORRUPTION_ERROR_ACTION is triggered upon detected bad addresses.
- USAGE_ERROR_ACTION is triggered on detected bad frees and
- reallocs. The argument p is an address that might have triggered the
- fault. It is ignored by the two predefined actions, but might be
- useful in custom actions that try to help diagnose errors.
-*/
-
-#if PROCEED_ON_ERROR
-
-/* A count of the number of corruption errors causing resets */
-int malloc_corruption_error_count;
-
-/* default corruption action */
-static void reset_on_error(mstate m);
-
-#define CORRUPTION_ERROR_ACTION(m) reset_on_error(m)
-#define USAGE_ERROR_ACTION(m, p)
-
-#else /* PROCEED_ON_ERROR */
-
-#ifndef CORRUPTION_ERROR_ACTION
-#define CORRUPTION_ERROR_ACTION(m) ABORT
-#endif /* CORRUPTION_ERROR_ACTION */
-
-#ifndef USAGE_ERROR_ACTION
-#define USAGE_ERROR_ACTION(m,p) ABORT
-#endif /* USAGE_ERROR_ACTION */
-
-#endif /* PROCEED_ON_ERROR */
-
-/* -------------------------- Debugging setup ---------------------------- */
-
-#if ! DEBUG
-
-#define check_free_chunk(M,P)
-#define check_inuse_chunk(M,P)
-#define check_malloced_chunk(M,P,N)
-#define check_mmapped_chunk(M,P)
-#define check_malloc_state(M)
-#define check_top_chunk(M,P)
-
-#else /* DEBUG */
-#define check_free_chunk(M,P) do_check_free_chunk(M,P)
-#define check_inuse_chunk(M,P) do_check_inuse_chunk(M,P)
-#define check_top_chunk(M,P) do_check_top_chunk(M,P)
-#define check_malloced_chunk(M,P,N) do_check_malloced_chunk(M,P,N)
-#define check_mmapped_chunk(M,P) do_check_mmapped_chunk(M,P)
-#define check_malloc_state(M) do_check_malloc_state(M)
-
-static void do_check_any_chunk(mstate m, mchunkptr p);
-static void do_check_top_chunk(mstate m, mchunkptr p);
-static void do_check_mmapped_chunk(mstate m, mchunkptr p);
-static void do_check_inuse_chunk(mstate m, mchunkptr p);
-static void do_check_free_chunk(mstate m, mchunkptr p);
-static void do_check_malloced_chunk(mstate m, void* mem, size_t s);
-static void do_check_tree(mstate m, tchunkptr t);
-static void do_check_treebin(mstate m, bindex_t i);
-static void do_check_smallbin(mstate m, bindex_t i);
-static void do_check_malloc_state(mstate m);
-static int bin_find(mstate m, mchunkptr x);
-static size_t traverse_and_check(mstate m);
-#endif /* DEBUG */
-
-/* ---------------------------- Indexing Bins ---------------------------- */
-
-#define is_small(s) (((s) >> SMALLBIN_SHIFT) < NSMALLBINS)
-#define small_index(s) (bindex_t)((s) >> SMALLBIN_SHIFT)
-#define small_index2size(i) ((i) << SMALLBIN_SHIFT)
-#define MIN_SMALL_INDEX (small_index(MIN_CHUNK_SIZE))
-
-/* addressing by index. See above about smallbin repositioning */
-#define smallbin_at(M, i) ((sbinptr)((char*)&((M)->smallbins[(i)<<1])))
-#define treebin_at(M,i) (&((M)->treebins[i]))
-
-/* assign tree index for size S to variable I. Use x86 asm if possible */
-#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
-#define compute_tree_index(S, I)\
-{\
- unsigned int X = S >> TREEBIN_SHIFT;\
- if (X == 0)\
- I = 0;\
- else if (X > 0xFFFF)\
- I = NTREEBINS-1;\
- else {\
- unsigned int K;\
- __asm__("bsrl\t%1, %0\n\t" : "=r" (K) : "g" (X));\
- I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
- }\
-}
-
-#elif defined (__INTEL_COMPILER)
-#define compute_tree_index(S, I)\
-{\
- size_t X = S >> TREEBIN_SHIFT;\
- if (X == 0)\
- I = 0;\
- else if (X > 0xFFFF)\
- I = NTREEBINS-1;\
- else {\
- unsigned int K = _bit_scan_reverse (X); \
- I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
- }\
-}
-
-#elif defined(_MSC_VER) && _MSC_VER>=1300
-#define compute_tree_index(S, I)\
-{\
- size_t X = S >> TREEBIN_SHIFT;\
- if (X == 0)\
- I = 0;\
- else if (X > 0xFFFF)\
- I = NTREEBINS-1;\
- else {\
- unsigned int K;\
- _BitScanReverse((DWORD *) &K, (DWORD) X);\
- I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\
- }\
-}
-
-#else /* GNUC */
-#define compute_tree_index(S, I)\
-{\
- size_t X = S >> TREEBIN_SHIFT;\
- if (X == 0)\
- I = 0;\
- else if (X > 0xFFFF)\
- I = NTREEBINS-1;\
- else {\
- unsigned int Y = (unsigned int)X;\
- unsigned int N = ((Y - 0x100) >> 16) & 8;\
- unsigned int K = (((Y <<= N) - 0x1000) >> 16) & 4;\
- N += K;\
- N += K = (((Y <<= K) - 0x4000) >> 16) & 2;\
- K = 14 - N + ((Y <<= K) >> 15);\
- I = (K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1));\
- }\
-}
-#endif /* GNUC */
-
-/* Bit representing maximum resolved size in a treebin at i */
-#define bit_for_tree_index(i) \
- (i == NTREEBINS-1)? (SIZE_T_BITSIZE-1) : (((i) >> 1) + TREEBIN_SHIFT - 2)
-
-/* Shift placing maximum resolved bit in a treebin at i as sign bit */
-#define leftshift_for_tree_index(i) \
- ((i == NTREEBINS-1)? 0 : \
- ((SIZE_T_BITSIZE-SIZE_T_ONE) - (((i) >> 1) + TREEBIN_SHIFT - 2)))
-
-/* The size of the smallest chunk held in bin with index i */
-#define minsize_for_tree_index(i) \
- ((SIZE_T_ONE << (((i) >> 1) + TREEBIN_SHIFT)) | \
- (((size_t)((i) & SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1)))
-
-
-/* ------------------------ Operations on bin maps ----------------------- */
-
-/* bit corresponding to given index */
-#define idx2bit(i) ((binmap_t)(1) << (i))
-
-/* Mark/Clear bits with given index */
-#define mark_smallmap(M,i) ((M)->smallmap |= idx2bit(i))
-#define clear_smallmap(M,i) ((M)->smallmap &= ~idx2bit(i))
-#define smallmap_is_marked(M,i) ((M)->smallmap & idx2bit(i))
-
-#define mark_treemap(M,i) ((M)->treemap |= idx2bit(i))
-#define clear_treemap(M,i) ((M)->treemap &= ~idx2bit(i))
-#define treemap_is_marked(M,i) ((M)->treemap & idx2bit(i))
-
-/* isolate the least set bit of a bitmap */
-#define least_bit(x) ((x) & -(x))
-
-/* mask with all bits to left of least bit of x on */
-#define left_bits(x) ((x<<1) | -(x<<1))
-
-/* mask with all bits to left of or equal to least bit of x on */
-#define same_or_left_bits(x) ((x) | -(x))
-
-/* index corresponding to given bit. Use x86 asm if possible */
-
-#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__))
-#define compute_bit2idx(X, I)\
-{\
- unsigned int J;\
- __asm__("bsfl\t%1, %0\n\t" : "=r" (J) : "g" (X));\
- I = (bindex_t)J;\
-}
-
-#elif defined (__INTEL_COMPILER)
-#define compute_bit2idx(X, I)\
-{\
- unsigned int J;\
- J = _bit_scan_forward (X); \
- I = (bindex_t)J;\
-}
-
-#elif defined(_MSC_VER) && _MSC_VER>=1300
-#define compute_bit2idx(X, I)\
-{\
- unsigned int J;\
- _BitScanForward((DWORD *) &J, X);\
- I = (bindex_t)J;\
-}
-
-#elif USE_BUILTIN_FFS
-#define compute_bit2idx(X, I) I = ffs(X)-1
-
-#else
-#define compute_bit2idx(X, I)\
-{\
- unsigned int Y = X - 1;\
- unsigned int K = Y >> (16-4) & 16;\
- unsigned int N = K; Y >>= K;\
- N += K = Y >> (8-3) & 8; Y >>= K;\
- N += K = Y >> (4-2) & 4; Y >>= K;\
- N += K = Y >> (2-1) & 2; Y >>= K;\
- N += K = Y >> (1-0) & 1; Y >>= K;\
- I = (bindex_t)(N + Y);\
-}
-#endif /* GNUC */
-
-
-/* ----------------------- Runtime Check Support ------------------------- */
-
-/*
- For security, the main invariant is that malloc/free/etc never
- writes to a static address other than malloc_state, unless static
- malloc_state itself has been corrupted, which cannot occur via
- malloc (because of these checks). In essence this means that we
- believe all pointers, sizes, maps etc held in malloc_state, but
- check all of those linked or offsetted from other embedded data
- structures. These checks are interspersed with main code in a way
- that tends to minimize their run-time cost.
-
- When FOOTERS is defined, in addition to range checking, we also
- verify footer fields of inuse chunks, which can be used guarantee
- that the mstate controlling malloc/free is intact. This is a
- streamlined version of the approach described by William Robertson
- et al in "Run-time Detection of Heap-based Overflows" LISA'03
- http://www.usenix.org/events/lisa03/tech/robertson.html The footer
- of an inuse chunk holds the xor of its mstate and a random seed,
- that is checked upon calls to free() and realloc(). This is
- (probablistically) unguessable from outside the program, but can be
- computed by any code successfully malloc'ing any chunk, so does not
- itself provide protection against code that has already broken
- security through some other means. Unlike Robertson et al, we
- always dynamically check addresses of all offset chunks (previous,
- next, etc). This turns out to be cheaper than relying on hashes.
-*/
-
-#if !INSECURE
-/* Check if address a is at least as high as any from MORECORE or MMAP */
-#define ok_address(M, a) ((char*)(a) >= (M)->least_addr)
-/* Check if address of next chunk n is higher than base chunk p */
-#define ok_next(p, n) ((char*)(p) < (char*)(n))
-/* Check if p has inuse status */
-#define ok_inuse(p) is_inuse(p)
-/* Check if p has its pinuse bit on */
-#define ok_pinuse(p) pinuse(p)
-
-#else /* !INSECURE */
-#define ok_address(M, a) (1)
-#define ok_next(b, n) (1)
-#define ok_inuse(p) (1)
-#define ok_pinuse(p) (1)
-#endif /* !INSECURE */
-
-#if (FOOTERS && !INSECURE)
-/* Check if (alleged) mstate m has expected magic field */
-#define ok_magic(M) ((M)->magic == mparams.magic)
-#else /* (FOOTERS && !INSECURE) */
-#define ok_magic(M) (1)
-#endif /* (FOOTERS && !INSECURE) */
-
-
-/* In gcc, use __builtin_expect to minimize impact of checks */
-#if !INSECURE
-#if defined(__GNUC__) && __GNUC__ >= 3
-#define RTCHECK(e) __builtin_expect(e, 1)
-#else /* GNUC */
-#define RTCHECK(e) (e)
-#endif /* GNUC */
-#else /* !INSECURE */
-#define RTCHECK(e) (1)
-#endif /* !INSECURE */
-
-/* macros to set up inuse chunks with or without footers */
-
-#if !FOOTERS
-
-#define mark_inuse_foot(M,p,s)
-
-/* Macros for setting head/foot of non-mmapped chunks */
-
-/* Set cinuse bit and pinuse bit of next chunk */
-#define set_inuse(M,p,s)\
- ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\
- ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT)
-
-/* Set cinuse and pinuse of this chunk and pinuse of next chunk */
-#define set_inuse_and_pinuse(M,p,s)\
- ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
- ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT)
-
-/* Set size, cinuse and pinuse bit of this chunk */
-#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\
- ((p)->head = (s|PINUSE_BIT|CINUSE_BIT))
-
-#else /* FOOTERS */
-
-/* Set foot of inuse chunk to be xor of mstate and seed */
-#define mark_inuse_foot(M,p,s)\
- (((mchunkptr)((char*)(p) + (s)))->prev_foot = ((size_t)(M) ^ mparams.magic))
-
-#define get_mstate_for(p)\
- ((mstate)(((mchunkptr)((char*)(p) +\
- (chunksize(p))))->prev_foot ^ mparams.magic))
-
-#define set_inuse(M,p,s)\
- ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\
- (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT), \
- mark_inuse_foot(M,p,s))
-
-#define set_inuse_and_pinuse(M,p,s)\
- ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
- (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT),\
- mark_inuse_foot(M,p,s))
-
-#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\
- ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\
- mark_inuse_foot(M, p, s))
-
-#endif /* !FOOTERS */
-
-/* ---------------------------- setting mparams -------------------------- */
-
-#ifdef ENABLE_LARGE_PAGES
-typedef size_t (WINAPI *GetLargePageMinimum_t)(void);
-#endif
-
-/* Initialize mparams */
-static int init_mparams(void) {
-#ifdef NEED_GLOBAL_LOCK_INIT
- if (malloc_global_mutex_status <= 0)
- init_malloc_global_mutex();
-#endif
-
- ACQUIRE_MALLOC_GLOBAL_LOCK();
- if (mparams.magic == 0) {
- size_t magic;
- size_t psize;
- size_t gsize;
-
-#ifndef WIN32
- psize = malloc_getpagesize;
- gsize = ((DEFAULT_GRANULARITY != 0)? DEFAULT_GRANULARITY : psize);
-#else /* WIN32 */
- {
- SYSTEM_INFO system_info;
- GetSystemInfo(&system_info);
- psize = system_info.dwPageSize;
- gsize = ((DEFAULT_GRANULARITY != 0)?
- DEFAULT_GRANULARITY : system_info.dwAllocationGranularity);
-#ifdef ENABLE_LARGE_PAGES
- {
- GetLargePageMinimum_t GetLargePageMinimum_ = (GetLargePageMinimum_t) GetProcAddress(GetModuleHandle(__T("kernel32.dll")), "GetLargePageMinimum");
- if(GetLargePageMinimum_) {
- size_t largepagesize = GetLargePageMinimum_();
- if(largepagesize) {
- psize = largepagesize;
- gsize = ((DEFAULT_GRANULARITY != 0)?
- DEFAULT_GRANULARITY : largepagesize);
- if(gsize < largepagesize) gsize = largepagesize;
- }
- }
- }
-#endif
- }
-#endif /* WIN32 */
-
- /* Sanity-check configuration:
- size_t must be unsigned and as wide as pointer type.
- ints must be at least 4 bytes.
- alignment must be at least 8.
- Alignment, min chunk size, and page size must all be powers of 2.
- */
- if ((sizeof(size_t) != sizeof(char*)) ||
- (MAX_SIZE_T < MIN_CHUNK_SIZE) ||
- (sizeof(int) < 4) ||
- (MALLOC_ALIGNMENT < (size_t)8U) ||
- ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT-SIZE_T_ONE)) != 0) ||
- ((MCHUNK_SIZE & (MCHUNK_SIZE-SIZE_T_ONE)) != 0) ||
- ((gsize & (gsize-SIZE_T_ONE)) != 0) ||
- ((psize & (psize-SIZE_T_ONE)) != 0))
- ABORT;
-
- mparams.granularity = gsize;
- mparams.page_size = psize;
- mparams.mmap_threshold = DEFAULT_MMAP_THRESHOLD;
- mparams.trim_threshold = DEFAULT_TRIM_THRESHOLD;
-#if MORECORE_CONTIGUOUS
- mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT;
-#else /* MORECORE_CONTIGUOUS */
- mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT|USE_NONCONTIGUOUS_BIT;
-#endif /* MORECORE_CONTIGUOUS */
-
-#if !ONLY_MSPACES
- /* Set up lock for main malloc area */
- gm->mflags = mparams.default_mflags;
- INITIAL_LOCK(&gm->mutex);
-#endif
-
- {
-#if USE_DEV_RANDOM
- int fd;
- unsigned char buf[sizeof(size_t)];
- /* Try to use /dev/urandom, else fall back on using time */
- if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 &&
- read(fd, buf, sizeof(buf)) == sizeof(buf)) {
- magic = *((size_t *) buf);
- close(fd);
- }
- else
-#endif /* USE_DEV_RANDOM */
-#ifdef WIN32
- magic = (size_t)(GetTickCount() ^ (size_t)0x55555555U);
-#else
- magic = (size_t)(time(0) ^ (size_t)0x55555555U);
-#endif
- magic |= (size_t)8U; /* ensure nonzero */
- magic &= ~(size_t)7U; /* improve chances of fault for bad values */
- mparams.magic = magic;
- }
- }
-
- RELEASE_MALLOC_GLOBAL_LOCK();
- return 1;
-}
-
-/* support for mallopt */
-static int change_mparam(int param_number, int value) {
- size_t val;
- ensure_initialization();
- val = (value == -1)? MAX_SIZE_T : (size_t)value;
- switch(param_number) {
- case M_TRIM_THRESHOLD:
- mparams.trim_threshold = val;
- return 1;
- case M_GRANULARITY:
- if (val >= mparams.page_size && ((val & (val-1)) == 0)) {
- mparams.granularity = val;
- return 1;
- }
- else
- return 0;
- case M_MMAP_THRESHOLD:
- mparams.mmap_threshold = val;
- return 1;
- default:
- return 0;
- }
-}
-
-#if DEBUG
-/* ------------------------- Debugging Support --------------------------- */
-
-/* Check properties of any chunk, whether free, inuse, mmapped etc */
-static void do_check_any_chunk(mstate m, mchunkptr p) {
- assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
- assert(ok_address(m, p));
-}
-
-/* Check properties of top chunk */
-static void do_check_top_chunk(mstate m, mchunkptr p) {
- msegmentptr sp = segment_holding(m, (char*)p);
- size_t sz = p->head & ~INUSE_BITS; /* third-lowest bit can be set! */
- assert(sp != 0);
- assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
- assert(ok_address(m, p));
- assert(sz == m->topsize);
- assert(sz > 0);
- assert(sz == ((sp->base + sp->size) - (char*)p) - TOP_FOOT_SIZE);
- assert(pinuse(p));
- assert(!pinuse(chunk_plus_offset(p, sz)));
-}
-
-/* Check properties of (inuse) mmapped chunks */
-static void do_check_mmapped_chunk(mstate m, mchunkptr p) {
- size_t sz = chunksize(p);
- size_t len = (sz + (p->prev_foot) + MMAP_FOOT_PAD);
- assert(is_mmapped(p));
- assert(use_mmap(m));
- assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD));
- assert(ok_address(m, p));
- assert(!is_small(sz));
- assert((len & (mparams.page_size-SIZE_T_ONE)) == 0);
- assert(chunk_plus_offset(p, sz)->head == FENCEPOST_HEAD);
- assert(chunk_plus_offset(p, sz+SIZE_T_SIZE)->head == 0);
-}
-
-/* Check properties of inuse chunks */
-static void do_check_inuse_chunk(mstate m, mchunkptr p) {
- do_check_any_chunk(m, p);
- assert(is_inuse(p));
- assert(next_pinuse(p));
- /* If not pinuse and not mmapped, previous chunk has OK offset */
- assert(is_mmapped(p) || pinuse(p) || next_chunk(prev_chunk(p)) == p);
- if (is_mmapped(p))
- do_check_mmapped_chunk(m, p);
-}
-
-/* Check properties of free chunks */
-static void do_check_free_chunk(mstate m, mchunkptr p) {
- size_t sz = chunksize(p);
- mchunkptr next = chunk_plus_offset(p, sz);
- do_check_any_chunk(m, p);
- assert(!is_inuse(p));
- assert(!next_pinuse(p));
- assert (!is_mmapped(p));
- if (p != m->dv && p != m->top) {
- if (sz >= MIN_CHUNK_SIZE) {
- assert((sz & CHUNK_ALIGN_MASK) == 0);
- assert(is_aligned(chunk2mem(p)));
- assert(next->prev_foot == sz);
- assert(pinuse(p));
- assert (next == m->top || is_inuse(next));
- assert(p->fd->bk == p);
- assert(p->bk->fd == p);
- }
- else /* markers are always of size SIZE_T_SIZE */
- assert(sz == SIZE_T_SIZE);
- }
-}
-
-/* Check properties of malloced chunks at the point they are malloced */
-static void do_check_malloced_chunk(mstate m, void* mem, size_t s) {
- if (mem != 0) {
- mchunkptr p = mem2chunk(mem);
- size_t sz = p->head & ~INUSE_BITS;
- do_check_inuse_chunk(m, p);
- assert((sz & CHUNK_ALIGN_MASK) == 0);
- assert(sz >= MIN_CHUNK_SIZE);
- assert(sz >= s);
- /* unless mmapped, size is less than MIN_CHUNK_SIZE more than request */
- assert(is_mmapped(p) || sz < (s + MIN_CHUNK_SIZE));
- }
-}
-
-/* Check a tree and its subtrees. */
-static void do_check_tree(mstate m, tchunkptr t) {
- tchunkptr head = 0;
- tchunkptr u = t;
- bindex_t tindex = t->index;
- size_t tsize = chunksize(t);
- bindex_t idx;
- compute_tree_index(tsize, idx);
- assert(tindex == idx);
- assert(tsize >= MIN_LARGE_SIZE);
- assert(tsize >= minsize_for_tree_index(idx));
- assert((idx == NTREEBINS-1) || (tsize < minsize_for_tree_index((idx+1))));
-
- do { /* traverse through chain of same-sized nodes */
- do_check_any_chunk(m, ((mchunkptr)u));
- assert(u->index == tindex);
- assert(chunksize(u) == tsize);
- assert(!is_inuse(u));
- assert(!next_pinuse(u));
- assert(u->fd->bk == u);
- assert(u->bk->fd == u);
- if (u->parent == 0) {
- assert(u->child[0] == 0);
- assert(u->child[1] == 0);
- }
- else {
- assert(head == 0); /* only one node on chain has parent */
- head = u;
- assert(u->parent != u);
- assert (u->parent->child[0] == u ||
- u->parent->child[1] == u ||
- *((tbinptr*)(u->parent)) == u);
- if (u->child[0] != 0) {
- assert(u->child[0]->parent == u);
- assert(u->child[0] != u);
- do_check_tree(m, u->child[0]);
- }
- if (u->child[1] != 0) {
- assert(u->child[1]->parent == u);
- assert(u->child[1] != u);
- do_check_tree(m, u->child[1]);
- }
- if (u->child[0] != 0 && u->child[1] != 0) {
- assert(chunksize(u->child[0]) < chunksize(u->child[1]));
- }
- }
- u = u->fd;
- } while (u != t);
- assert(head != 0);
-}
-
-/* Check all the chunks in a treebin. */
-static void do_check_treebin(mstate m, bindex_t i) {
- tbinptr* tb = treebin_at(m, i);
- tchunkptr t = *tb;
- int empty = (m->treemap & (1U << i)) == 0;
- if (t == 0)
- assert(empty);
- if (!empty)
- do_check_tree(m, t);
-}
-
-/* Check all the chunks in a smallbin. */
-static void do_check_smallbin(mstate m, bindex_t i) {
- sbinptr b = smallbin_at(m, i);
- mchunkptr p = b->bk;
- unsigned int empty = (m->smallmap & (1U << i)) == 0;
- if (p == b)
- assert(empty);
- if (!empty) {
- for (; p != b; p = p->bk) {
- size_t size = chunksize(p);
- mchunkptr q;
- /* each chunk claims to be free */
- do_check_free_chunk(m, p);
- /* chunk belongs in bin */
- assert(small_index(size) == i);
- assert(p->bk == b || chunksize(p->bk) == chunksize(p));
- /* chunk is followed by an inuse chunk */
- q = next_chunk(p);
- if (q->head != FENCEPOST_HEAD)
- do_check_inuse_chunk(m, q);
- }
- }
-}
-
-/* Find x in a bin. Used in other check functions. */
-static int bin_find(mstate m, mchunkptr x) {
- size_t size = chunksize(x);
- if (is_small(size)) {
- bindex_t sidx = small_index(size);
- sbinptr b = smallbin_at(m, sidx);
- if (smallmap_is_marked(m, sidx)) {
- mchunkptr p = b;
- do {
- if (p == x)
- return 1;
- } while ((p = p->fd) != b);
- }
- }
- else {
- bindex_t tidx;
- compute_tree_index(size, tidx);
- if (treemap_is_marked(m, tidx)) {
- tchunkptr t = *treebin_at(m, tidx);
- size_t sizebits = size << leftshift_for_tree_index(tidx);
- while (t != 0 && chunksize(t) != size) {
- t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1];
- sizebits <<= 1;
- }
- if (t != 0) {
- tchunkptr u = t;
- do {
- if (u == (tchunkptr)x)
- return 1;
- } while ((u = u->fd) != t);
- }
- }
- }
- return 0;
-}
-
-/* Traverse each chunk and check it; return total */
-static size_t traverse_and_check(mstate m) {
- size_t sum = 0;
- if (is_initialized(m)) {
- msegmentptr s = &m->seg;
- sum += m->topsize + TOP_FOOT_SIZE;
- while (s != 0) {
- mchunkptr q = align_as_chunk(s->base);
- mchunkptr lastq = 0;
- assert(pinuse(q));
- while (segment_holds(s, q) &&
- q != m->top && q->head != FENCEPOST_HEAD) {
- sum += chunksize(q);
- if (is_inuse(q)) {
- assert(!bin_find(m, q));
- do_check_inuse_chunk(m, q);
- }
- else {
- assert(q == m->dv || bin_find(m, q));
- assert(lastq == 0 || is_inuse(lastq)); /* Not 2 consecutive free */
- do_check_free_chunk(m, q);
- }
- lastq = q;
- q = next_chunk(q);
- }
- s = s->next;
- }
- }
- return sum;
-}
-
-/* Check all properties of malloc_state. */
-static void do_check_malloc_state(mstate m) {
- bindex_t i;
- size_t total;
- /* check bins */
- for (i = 0; i < NSMALLBINS; ++i)
- do_check_smallbin(m, i);
- for (i = 0; i < NTREEBINS; ++i)
- do_check_treebin(m, i);
-
- if (m->dvsize != 0) { /* check dv chunk */
- do_check_any_chunk(m, m->dv);
- assert(m->dvsize == chunksize(m->dv));
- assert(m->dvsize >= MIN_CHUNK_SIZE);
- assert(bin_find(m, m->dv) == 0);
- }
-
- if (m->top != 0) { /* check top chunk */
- do_check_top_chunk(m, m->top);
- /*assert(m->topsize == chunksize(m->top)); redundant */
- assert(m->topsize > 0);
- assert(bin_find(m, m->top) == 0);
- }
-
- total = traverse_and_check(m);
- assert(total <= m->footprint);
- assert(m->footprint <= m->max_footprint);
-}
-#endif /* DEBUG */
-
-/* ----------------------------- statistics ------------------------------ */
-
-#if !NO_MALLINFO
-static struct mallinfo internal_mallinfo(mstate m) {
- struct mallinfo nm = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 };
- ensure_initialization();
- if (!PREACTION(m)) {
- check_malloc_state(m);
- if (is_initialized(m)) {
- size_t nfree = SIZE_T_ONE; /* top always free */
- size_t mfree = m->topsize + TOP_FOOT_SIZE;
- size_t sum = mfree;
- msegmentptr s = &m->seg;
- while (s != 0) {
- mchunkptr q = align_as_chunk(s->base);
- while (segment_holds(s, q) &&
- q != m->top && q->head != FENCEPOST_HEAD) {
- size_t sz = chunksize(q);
- sum += sz;
- if (!is_inuse(q)) {
- mfree += sz;
- ++nfree;
- }
- q = next_chunk(q);
- }
- s = s->next;
- }
-
- nm.arena = sum;
- nm.ordblks = nfree;
- nm.hblkhd = m->footprint - sum;
- nm.usmblks = m->max_footprint;
- nm.uordblks = m->footprint - mfree;
- nm.fordblks = mfree;
- nm.keepcost = m->topsize;
- }
-
- POSTACTION(m);
- }
- return nm;
-}
-#endif /* !NO_MALLINFO */
-
-static void internal_malloc_stats(mstate m) {
- ensure_initialization();
- if (!PREACTION(m)) {
- size_t maxfp = 0;
- size_t fp = 0;
- size_t used = 0;
- check_malloc_state(m);
- if (is_initialized(m)) {
- msegmentptr s = &m->seg;
- maxfp = m->max_footprint;
- fp = m->footprint;
- used = fp - (m->topsize + TOP_FOOT_SIZE);
-
- while (s != 0) {
- mchunkptr q = align_as_chunk(s->base);
- while (segment_holds(s, q) &&
- q != m->top && q->head != FENCEPOST_HEAD) {
- if (!is_inuse(q))
- used -= chunksize(q);
- q = next_chunk(q);
- }
- s = s->next;
- }
- }
-
- fprintf(stderr, "max system bytes = %10lu\n", (unsigned long)(maxfp));
- fprintf(stderr, "system bytes = %10lu\n", (unsigned long)(fp));
- fprintf(stderr, "in use bytes = %10lu\n", (unsigned long)(used));
-
- POSTACTION(m);
- }
-}
-
-/* ----------------------- Operations on smallbins ----------------------- */
-
-/*
- Various forms of linking and unlinking are defined as macros. Even
- the ones for trees, which are very long but have very short typical
- paths. This is ugly but reduces reliance on inlining support of
- compilers.
-*/
-
-/* Link a free chunk into a smallbin */
-#define insert_small_chunk(M, P, S) {\
- bindex_t I = small_index(S);\
- mchunkptr B = smallbin_at(M, I);\
- mchunkptr F = B;\
- assert(S >= MIN_CHUNK_SIZE);\
- if (!smallmap_is_marked(M, I))\
- mark_smallmap(M, I);\
- else if (RTCHECK(ok_address(M, B->fd)))\
- F = B->fd;\
- else {\
- CORRUPTION_ERROR_ACTION(M);\
- }\
- B->fd = P;\
- F->bk = P;\
- P->fd = F;\
- P->bk = B;\
-}
-
-/* Unlink a chunk from a smallbin */
-#define unlink_small_chunk(M, P, S) {\
- mchunkptr F = P->fd;\
- mchunkptr B = P->bk;\
- bindex_t I = small_index(S);\
- assert(P != B);\
- assert(P != F);\
- assert(chunksize(P) == small_index2size(I));\
- if (F == B)\
- clear_smallmap(M, I);\
- else if (RTCHECK((F == smallbin_at(M,I) || ok_address(M, F)) &&\
- (B == smallbin_at(M,I) || ok_address(M, B)))) {\
- F->bk = B;\
- B->fd = F;\
- }\
- else {\
- CORRUPTION_ERROR_ACTION(M);\
- }\
-}
-
-/* Unlink the first chunk from a smallbin */
-#define unlink_first_small_chunk(M, B, P, I) {\
- mchunkptr F = P->fd;\
- assert(P != B);\
- assert(P != F);\
- assert(chunksize(P) == small_index2size(I));\
- if (B == F)\
- clear_smallmap(M, I);\
- else if (RTCHECK(ok_address(M, F))) {\
- B->fd = F;\
- F->bk = B;\
- }\
- else {\
- CORRUPTION_ERROR_ACTION(M);\
- }\
-}
-
-
-
-/* Replace dv node, binning the old one */
-/* Used only when dvsize known to be small */
-#define replace_dv(M, P, S) {\
- size_t DVS = M->dvsize;\
- if (DVS != 0) {\
- mchunkptr DV = M->dv;\
- assert(is_small(DVS));\
- insert_small_chunk(M, DV, DVS);\
- }\
- M->dvsize = S;\
- M->dv = P;\
-}
-
-/* ------------------------- Operations on trees ------------------------- */
-
-/* Insert chunk into tree */
-#define insert_large_chunk(M, X, S) {\
- tbinptr* H;\
- bindex_t I;\
- compute_tree_index(S, I);\
- H = treebin_at(M, I);\
- X->index = I;\
- X->child[0] = X->child[1] = 0;\
- if (!treemap_is_marked(M, I)) {\
- mark_treemap(M, I);\
- *H = X;\
- X->parent = (tchunkptr)H;\
- X->fd = X->bk = X;\
- }\
- else {\
- tchunkptr T = *H;\
- size_t K = S << leftshift_for_tree_index(I);\
- for (;;) {\
- if (chunksize(T) != S) {\
- tchunkptr* C = &(T->child[(K >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]);\
- K <<= 1;\
- if (*C != 0)\
- T = *C;\
- else if (RTCHECK(ok_address(M, C))) {\
- *C = X;\
- X->parent = T;\
- X->fd = X->bk = X;\
- break;\
- }\
- else {\
- CORRUPTION_ERROR_ACTION(M);\
- break;\
- }\
- }\
- else {\
- tchunkptr F = T->fd;\
- if (RTCHECK(ok_address(M, T) && ok_address(M, F))) {\
- T->fd = F->bk = X;\
- X->fd = F;\
- X->bk = T;\
- X->parent = 0;\
- break;\
- }\
- else {\
- CORRUPTION_ERROR_ACTION(M);\
- break;\
- }\
- }\
- }\
- }\
-}
-
-/*
- Unlink steps:
-
- 1. If x is a chained node, unlink it from its same-sized fd/bk links
- and choose its bk node as its replacement.
- 2. If x was the last node of its size, but not a leaf node, it must
- be replaced with a leaf node (not merely one with an open left or
- right), to make sure that lefts and rights of descendents
- correspond properly to bit masks. We use the rightmost descendent
- of x. We could use any other leaf, but this is easy to locate and
- tends to counteract removal of leftmosts elsewhere, and so keeps
- paths shorter than minimally guaranteed. This doesn't loop much
- because on average a node in a tree is near the bottom.
- 3. If x is the base of a chain (i.e., has parent links) relink
- x's parent and children to x's replacement (or null if none).
-*/
-
-#define unlink_large_chunk(M, X) {\
- tchunkptr XP = X->parent;\
- tchunkptr R;\
- if (X->bk != X) {\
- tchunkptr F = X->fd;\
- R = X->bk;\
- if (RTCHECK(ok_address(M, F))) {\
- F->bk = R;\
- R->fd = F;\
- }\
- else {\
- CORRUPTION_ERROR_ACTION(M);\
- }\
- }\
- else {\
- tchunkptr* RP;\
- if (((R = *(RP = &(X->child[1]))) != 0) ||\
- ((R = *(RP = &(X->child[0]))) != 0)) {\
- tchunkptr* CP;\
- while ((*(CP = &(R->child[1])) != 0) ||\
- (*(CP = &(R->child[0])) != 0)) {\
- R = *(RP = CP);\
- }\
- if (RTCHECK(ok_address(M, RP)))\
- *RP = 0;\
- else {\
- CORRUPTION_ERROR_ACTION(M);\
- }\
- }\
- }\
- if (XP != 0) {\
- tbinptr* H = treebin_at(M, X->index);\
- if (X == *H) {\
- if ((*H = R) == 0) \
- clear_treemap(M, X->index);\
- }\
- else if (RTCHECK(ok_address(M, XP))) {\
- if (XP->child[0] == X) \
- XP->child[0] = R;\
- else \
- XP->child[1] = R;\
- }\
- else\
- CORRUPTION_ERROR_ACTION(M);\
- if (R != 0) {\
- if (RTCHECK(ok_address(M, R))) {\
- tchunkptr C0, C1;\
- R->parent = XP;\
- if ((C0 = X->child[0]) != 0) {\
- if (RTCHECK(ok_address(M, C0))) {\
- R->child[0] = C0;\
- C0->parent = R;\
- }\
- else\
- CORRUPTION_ERROR_ACTION(M);\
- }\
- if ((C1 = X->child[1]) != 0) {\
- if (RTCHECK(ok_address(M, C1))) {\
- R->child[1] = C1;\
- C1->parent = R;\
- }\
- else\
- CORRUPTION_ERROR_ACTION(M);\
- }\
- }\
- else\
- CORRUPTION_ERROR_ACTION(M);\
- }\
- }\
-}
-
-/* Relays to large vs small bin operations */
-
-#define insert_chunk(M, P, S)\
- if (is_small(S)) insert_small_chunk(M, P, S)\
- else { tchunkptr TP = (tchunkptr)(P); insert_large_chunk(M, TP, S); }
-
-#define unlink_chunk(M, P, S)\
- if (is_small(S)) unlink_small_chunk(M, P, S)\
- else { tchunkptr TP = (tchunkptr)(P); unlink_large_chunk(M, TP); }
-
-
-/* Relays to internal calls to malloc/free from realloc, memalign etc */
-
-#if ONLY_MSPACES
-#define internal_malloc(m, b) mspace_malloc(m, b)
-#define internal_free(m, mem) mspace_free(m,mem);
-#else /* ONLY_MSPACES */
-#if MSPACES
-#define internal_malloc(m, b)\
- (m == gm)? dlmalloc(b) : mspace_malloc(m, b)
-#define internal_free(m, mem)\
- if (m == gm) dlfree(mem); else mspace_free(m,mem);
-#else /* MSPACES */
-#define internal_malloc(m, b) dlmalloc(b)
-#define internal_free(m, mem) dlfree(mem)
-#endif /* MSPACES */
-#endif /* ONLY_MSPACES */
-
-/* ----------------------- Direct-mmapping chunks ----------------------- */
-
-/*
- Directly mmapped chunks are set up with an offset to the start of
- the mmapped region stored in the prev_foot field of the chunk. This
- allows reconstruction of the required argument to MUNMAP when freed,
- and also allows adjustment of the returned chunk to meet alignment
- requirements (especially in memalign).
-*/
-
-/* Malloc using mmap */
-static void* mmap_alloc(mstate m, size_t nb) {
- size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
- if (mmsize > nb) { /* Check for wrap around 0 */
- char* mm = (char*)(CALL_DIRECT_MMAP(mmsize));
- if (mm != CMFAIL) {
- size_t offset = align_offset(chunk2mem(mm));
- size_t psize = mmsize - offset - MMAP_FOOT_PAD;
- mchunkptr p = (mchunkptr)(mm + offset);
- p->prev_foot = offset;
- p->head = psize;
- mark_inuse_foot(m, p, psize);
- chunk_plus_offset(p, psize)->head = FENCEPOST_HEAD;
- chunk_plus_offset(p, psize+SIZE_T_SIZE)->head = 0;
-
- if (m->least_addr == 0 || mm < m->least_addr)
- m->least_addr = mm;
- if ((m->footprint += mmsize) > m->max_footprint)
- m->max_footprint = m->footprint;
- assert(is_aligned(chunk2mem(p)));
- check_mmapped_chunk(m, p);
- return chunk2mem(p);
- }
- }
- return 0;
-}
-
-/* Realloc using mmap */
-static mchunkptr mmap_resize(mstate m, mchunkptr oldp, size_t nb) {
- size_t oldsize = chunksize(oldp);
- if (is_small(nb)) /* Can't shrink mmap regions below small size */
- return 0;
- /* Keep old chunk if big enough but not too big */
- if (oldsize >= nb + SIZE_T_SIZE &&
- (oldsize - nb) <= (mparams.granularity << 1))
- return oldp;
- else {
- size_t offset = oldp->prev_foot;
- size_t oldmmsize = oldsize + offset + MMAP_FOOT_PAD;
- size_t newmmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
- char* cp = (char*)CALL_MREMAP((char*)oldp - offset,
- oldmmsize, newmmsize, 1);
- if (cp != CMFAIL) {
- mchunkptr newp = (mchunkptr)(cp + offset);
- size_t psize = newmmsize - offset - MMAP_FOOT_PAD;
- newp->head = psize;
- mark_inuse_foot(m, newp, psize);
- chunk_plus_offset(newp, psize)->head = FENCEPOST_HEAD;
- chunk_plus_offset(newp, psize+SIZE_T_SIZE)->head = 0;
-
- if (cp < m->least_addr)
- m->least_addr = cp;
- if ((m->footprint += newmmsize - oldmmsize) > m->max_footprint)
- m->max_footprint = m->footprint;
- check_mmapped_chunk(m, newp);
- return newp;
- }
- }
- return 0;
-}
-
-/* -------------------------- mspace management -------------------------- */
-
-/* Initialize top chunk and its size */
-static void init_top(mstate m, mchunkptr p, size_t psize) {
- /* Ensure alignment */
- size_t offset = align_offset(chunk2mem(p));
- p = (mchunkptr)((char*)p + offset);
- psize -= offset;
-
- m->top = p;
- m->topsize = psize;
- p->head = psize | PINUSE_BIT;
- /* set size of fake trailing chunk holding overhead space only once */
- chunk_plus_offset(p, psize)->head = TOP_FOOT_SIZE;
- m->trim_check = mparams.trim_threshold; /* reset on each update */
-}
-
-/* Initialize bins for a new mstate that is otherwise zeroed out */
-static void init_bins(mstate m) {
- /* Establish circular links for smallbins */
- bindex_t i;
- for (i = 0; i < NSMALLBINS; ++i) {
- sbinptr bin = smallbin_at(m,i);
- bin->fd = bin->bk = bin;
- }
-}
-
-#if PROCEED_ON_ERROR
-
-/* default corruption action */
-static void reset_on_error(mstate m) {
- int i;
- ++malloc_corruption_error_count;
- /* Reinitialize fields to forget about all memory */
- m->smallbins = m->treebins = 0;
- m->dvsize = m->topsize = 0;
- m->seg.base = 0;
- m->seg.size = 0;
- m->seg.next = 0;
- m->top = m->dv = 0;
- for (i = 0; i < NTREEBINS; ++i)
- *treebin_at(m, i) = 0;
- init_bins(m);
-}
-#endif /* PROCEED_ON_ERROR */
-
-/* Allocate chunk and prepend remainder with chunk in successor base. */
-static void* prepend_alloc(mstate m, char* newbase, char* oldbase,
- size_t nb) {
- mchunkptr p = align_as_chunk(newbase);
- mchunkptr oldfirst = align_as_chunk(oldbase);
- size_t psize = (char*)oldfirst - (char*)p;
- mchunkptr q = chunk_plus_offset(p, nb);
- size_t qsize = psize - nb;
- set_size_and_pinuse_of_inuse_chunk(m, p, nb);
-
- assert((char*)oldfirst > (char*)q);
- assert(pinuse(oldfirst));
- assert(qsize >= MIN_CHUNK_SIZE);
-
- /* consolidate remainder with first chunk of old base */
- if (oldfirst == m->top) {
- size_t tsize = m->topsize += qsize;
- m->top = q;
- q->head = tsize | PINUSE_BIT;
- check_top_chunk(m, q);
- }
- else if (oldfirst == m->dv) {
- size_t dsize = m->dvsize += qsize;
- m->dv = q;
- set_size_and_pinuse_of_free_chunk(q, dsize);
- }
- else {
- if (!is_inuse(oldfirst)) {
- size_t nsize = chunksize(oldfirst);
- unlink_chunk(m, oldfirst, nsize);
- oldfirst = chunk_plus_offset(oldfirst, nsize);
- qsize += nsize;
- }
- set_free_with_pinuse(q, qsize, oldfirst);
- insert_chunk(m, q, qsize);
- check_free_chunk(m, q);
- }
-
- check_malloced_chunk(m, chunk2mem(p), nb);
- return chunk2mem(p);
-}
-
-/* Add a segment to hold a new noncontiguous region */
-static void add_segment(mstate m, char* tbase, size_t tsize, flag_t mmapped) {
- /* Determine locations and sizes of segment, fenceposts, old top */
- char* old_top = (char*)m->top;
- msegmentptr oldsp = segment_holding(m, old_top);
- char* old_end = oldsp->base + oldsp->size;
- size_t ssize = pad_request(sizeof(struct malloc_segment));
- char* rawsp = old_end - (ssize + FOUR_SIZE_T_SIZES + CHUNK_ALIGN_MASK);
- size_t offset = align_offset(chunk2mem(rawsp));
- char* asp = rawsp + offset;
- char* csp = (asp < (old_top + MIN_CHUNK_SIZE))? old_top : asp;
- mchunkptr sp = (mchunkptr)csp;
- msegmentptr ss = (msegmentptr)(chunk2mem(sp));
- mchunkptr tnext = chunk_plus_offset(sp, ssize);
- mchunkptr p = tnext;
- int nfences = 0;
-
- /* reset top to new space */
- init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE);
-
- /* Set up segment record */
- assert(is_aligned(ss));
- set_size_and_pinuse_of_inuse_chunk(m, sp, ssize);
- *ss = m->seg; /* Push current record */
- m->seg.base = tbase;
- m->seg.size = tsize;
- m->seg.sflags = mmapped;
- m->seg.next = ss;
-
- /* Insert trailing fenceposts */
- for (;;) {
- mchunkptr nextp = chunk_plus_offset(p, SIZE_T_SIZE);
- p->head = FENCEPOST_HEAD;
- ++nfences;
- if ((char*)(&(nextp->head)) < old_end)
- p = nextp;
- else
- break;
- }
- assert(nfences >= 2);
-
- /* Insert the rest of old top into a bin as an ordinary free chunk */
- if (csp != old_top) {
- mchunkptr q = (mchunkptr)old_top;
- size_t psize = csp - old_top;
- mchunkptr tn = chunk_plus_offset(q, psize);
- set_free_with_pinuse(q, psize, tn);
- insert_chunk(m, q, psize);
- }
-
- check_top_chunk(m, m->top);
-}
-
-/* -------------------------- System allocation -------------------------- */
-
-/* Get memory from system using MORECORE or MMAP */
-static void* sys_alloc(mstate m, size_t nb) {
- char* tbase = CMFAIL;
- size_t tsize = 0;
- flag_t mmap_flag = 0;
-
- ensure_initialization();
-
- /* Directly map large chunks, but only if already initialized */
- if (use_mmap(m) && nb >= mparams.mmap_threshold && m->topsize != 0) {
- void* mem = mmap_alloc(m, nb);
- if (mem != 0)
- return mem;
- }
-
- /*
- Try getting memory in any of three ways (in most-preferred to
- least-preferred order):
- 1. A call to MORECORE that can normally contiguously extend memory.
- (disabled if not MORECORE_CONTIGUOUS or not HAVE_MORECORE or
- or main space is mmapped or a previous contiguous call failed)
- 2. A call to MMAP new space (disabled if not HAVE_MMAP).
- Note that under the default settings, if MORECORE is unable to
- fulfill a request, and HAVE_MMAP is true, then mmap is
- used as a noncontiguous system allocator. This is a useful backup
- strategy for systems with holes in address spaces -- in this case
- sbrk cannot contiguously expand the heap, but mmap may be able to
- find space.
- 3. A call to MORECORE that cannot usually contiguously extend memory.
- (disabled if not HAVE_MORECORE)
-
- In all cases, we need to request enough bytes from system to ensure
- we can malloc nb bytes upon success, so pad with enough space for
- top_foot, plus alignment-pad to make sure we don't lose bytes if
- not on boundary, and round this up to a granularity unit.
- */
-
- if (MORECORE_CONTIGUOUS && !use_noncontiguous(m)) {
- char* br = CMFAIL;
- msegmentptr ss = (m->top == 0)? 0 : segment_holding(m, (char*)m->top);
- size_t asize = 0;
- ACQUIRE_MALLOC_GLOBAL_LOCK();
-
- if (ss == 0) { /* First time through or recovery */
- char* base = (char*)CALL_MORECORE(0);
- if (base != CMFAIL) {
- asize = granularity_align(nb + SYS_ALLOC_PADDING);
- /* Adjust to end on a page boundary */
- if (!is_page_aligned(base))
- asize += (page_align((size_t)base) - (size_t)base);
- /* Can't call MORECORE if size is negative when treated as signed */
- if (asize < HALF_MAX_SIZE_T &&
- (br = (char*)(CALL_MORECORE(asize))) == base) {
- tbase = base;
- tsize = asize;
- }
- }
- }
- else {
- /* Subtract out existing available top space from MORECORE request. */
- asize = granularity_align(nb - m->topsize + SYS_ALLOC_PADDING);
- /* Use mem here only if it did continuously extend old space */
- if (asize < HALF_MAX_SIZE_T &&
- (br = (char*)(CALL_MORECORE(asize))) == ss->base+ss->size) {
- tbase = br;
- tsize = asize;
- }
- }
-
- if (tbase == CMFAIL) { /* Cope with partial failure */
- if (br != CMFAIL) { /* Try to use/extend the space we did get */
- if (asize < HALF_MAX_SIZE_T &&
- asize < nb + SYS_ALLOC_PADDING) {
- size_t esize = granularity_align(nb + SYS_ALLOC_PADDING - asize);
- if (esize < HALF_MAX_SIZE_T) {
- char* end = (char*)CALL_MORECORE(esize);
- if (end != CMFAIL)
- asize += esize;
- else { /* Can't use; try to release */
- (void) CALL_MORECORE(-asize);
- br = CMFAIL;
- }
- }
- }
- }
- if (br != CMFAIL) { /* Use the space we did get */
- tbase = br;
- tsize = asize;
- }
- else
- disable_contiguous(m); /* Don't try contiguous path in the future */
- }
-
- RELEASE_MALLOC_GLOBAL_LOCK();
- }
-
- if (HAVE_MMAP && tbase == CMFAIL) { /* Try MMAP */
- size_t rsize = granularity_align(nb + SYS_ALLOC_PADDING);
- if (rsize > nb) { /* Fail if wraps around zero */
- char* mp = (char*)(CALL_MMAP(rsize));
- if (mp != CMFAIL) {
- tbase = mp;
- tsize = rsize;
- mmap_flag = USE_MMAP_BIT;
- }
- }
- }
-
- if (HAVE_MORECORE && tbase == CMFAIL) { /* Try noncontiguous MORECORE */
- size_t asize = granularity_align(nb + SYS_ALLOC_PADDING);
- if (asize < HALF_MAX_SIZE_T) {
- char* br = CMFAIL;
- char* end = CMFAIL;
- ACQUIRE_MALLOC_GLOBAL_LOCK();
- br = (char*)(CALL_MORECORE(asize));
- end = (char*)(CALL_MORECORE(0));
- RELEASE_MALLOC_GLOBAL_LOCK();
- if (br != CMFAIL && end != CMFAIL && br < end) {
- size_t ssize = end - br;
- if (ssize > nb + TOP_FOOT_SIZE) {
- tbase = br;
- tsize = ssize;
- }
- }
- }
- }
-
- if (tbase != CMFAIL) {
-
- if ((m->footprint += tsize) > m->max_footprint)
- m->max_footprint = m->footprint;
-
- if (!is_initialized(m)) { /* first-time initialization */
- if (m->least_addr == 0 || tbase < m->least_addr)
- m->least_addr = tbase;
- m->seg.base = tbase;
- m->seg.size = tsize;
- m->seg.sflags = mmap_flag;
- m->magic = mparams.magic;
- m->release_checks = MAX_RELEASE_CHECK_RATE;
- init_bins(m);
-#if !ONLY_MSPACES
- if (is_global(m))
- init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE);
- else
-#endif
- {
- /* Offset top by embedded malloc_state */
- mchunkptr mn = next_chunk(mem2chunk(m));
- init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) -TOP_FOOT_SIZE);
- }
- }
-
- else {
- /* Try to merge with an existing segment */
- msegmentptr sp = &m->seg;
- /* Only consider most recent segment if traversal suppressed */
- while (sp != 0 && tbase != sp->base + sp->size)
- sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next;
- if (sp != 0 &&
- !is_extern_segment(sp) &&
- (sp->sflags & USE_MMAP_BIT) == mmap_flag &&
- segment_holds(sp, m->top)) { /* append */
- sp->size += tsize;
- init_top(m, m->top, m->topsize + tsize);
- }
- else {
- if (tbase < m->least_addr)
- m->least_addr = tbase;
- sp = &m->seg;
- while (sp != 0 && sp->base != tbase + tsize)
- sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next;
- if (sp != 0 &&
- !is_extern_segment(sp) &&
- (sp->sflags & USE_MMAP_BIT) == mmap_flag) {
- char* oldbase = sp->base;
- sp->base = tbase;
- sp->size += tsize;
- return prepend_alloc(m, tbase, oldbase, nb);
- }
- else
- add_segment(m, tbase, tsize, mmap_flag);
- }
- }
-
- if (nb < m->topsize) { /* Allocate from new or extended top space */
- size_t rsize = m->topsize -= nb;
- mchunkptr p = m->top;
- mchunkptr r = m->top = chunk_plus_offset(p, nb);
- r->head = rsize | PINUSE_BIT;
- set_size_and_pinuse_of_inuse_chunk(m, p, nb);
- check_top_chunk(m, m->top);
- check_malloced_chunk(m, chunk2mem(p), nb);
- return chunk2mem(p);
- }
- }
-
- MALLOC_FAILURE_ACTION;
- return 0;
-}
-
-/* ----------------------- system deallocation -------------------------- */
-
-/* Unmap and unlink any mmapped segments that don't contain used chunks */
-static size_t release_unused_segments(mstate m) {
- size_t released = 0;
- int nsegs = 0;
- msegmentptr pred = &m->seg;
- msegmentptr sp = pred->next;
- while (sp != 0) {
- char* base = sp->base;
- size_t size = sp->size;
- msegmentptr next = sp->next;
- ++nsegs;
- if (is_mmapped_segment(sp) && !is_extern_segment(sp)) {
- mchunkptr p = align_as_chunk(base);
- size_t psize = chunksize(p);
- /* Can unmap if first chunk holds entire segment and not pinned */
- if (!is_inuse(p) && (char*)p + psize >= base + size - TOP_FOOT_SIZE) {
- tchunkptr tp = (tchunkptr)p;
- assert(segment_holds(sp, (char*)sp));
- if (p == m->dv) {
- m->dv = 0;
- m->dvsize = 0;
- }
- else {
- unlink_large_chunk(m, tp);
- }
- if (CALL_MUNMAP(base, size) == 0) {
- released += size;
- m->footprint -= size;
- /* unlink obsoleted record */
- sp = pred;
- sp->next = next;
- }
- else { /* back out if cannot unmap */
- insert_large_chunk(m, tp, psize);
- }
- }
- }
- if (NO_SEGMENT_TRAVERSAL) /* scan only first segment */
- break;
- pred = sp;
- sp = next;
- }
- /* Reset check counter */
- m->release_checks = ((nsegs > MAX_RELEASE_CHECK_RATE)?
- nsegs : MAX_RELEASE_CHECK_RATE);
- return released;
-}
-
-static int sys_trim(mstate m, size_t pad) {
- size_t released = 0;
- ensure_initialization();
- if (pad < MAX_REQUEST && is_initialized(m)) {
- pad += TOP_FOOT_SIZE; /* ensure enough room for segment overhead */
-
- if (m->topsize > pad) {
- /* Shrink top space in granularity-size units, keeping at least one */
- size_t unit = mparams.granularity;
- size_t extra = ((m->topsize - pad + (unit - SIZE_T_ONE)) / unit -
- SIZE_T_ONE) * unit;
- msegmentptr sp = segment_holding(m, (char*)m->top);
-
- if (!is_extern_segment(sp)) {
- if (is_mmapped_segment(sp)) {
- if (HAVE_MMAP &&
- sp->size >= extra &&
- !has_segment_link(m, sp)) { /* can't shrink if pinned */
- size_t newsize = sp->size - extra;
- /* Prefer mremap, fall back to munmap */
- if ((CALL_MREMAP(sp->base, sp->size, newsize, 0) != MFAIL) ||
- (CALL_MUNMAP(sp->base + newsize, extra) == 0)) {
- released = extra;
- }
- }
- }
- else if (HAVE_MORECORE) {
- if (extra >= HALF_MAX_SIZE_T) /* Avoid wrapping negative */
- extra = (HALF_MAX_SIZE_T) + SIZE_T_ONE - unit;
- ACQUIRE_MALLOC_GLOBAL_LOCK();
- {
- /* Make sure end of memory is where we last set it. */
- char* old_br = (char*)(CALL_MORECORE(0));
- if (old_br == sp->base + sp->size) {
- char* rel_br = (char*)(CALL_MORECORE(-extra));
- char* new_br = (char*)(CALL_MORECORE(0));
- if (rel_br != CMFAIL && new_br < old_br)
- released = old_br - new_br;
- }
- }
- RELEASE_MALLOC_GLOBAL_LOCK();
- }
- }
-
- if (released != 0) {
- sp->size -= released;
- m->footprint -= released;
- init_top(m, m->top, m->topsize - released);
- check_top_chunk(m, m->top);
- }
- }
-
- /* Unmap any unused mmapped segments */
- if (HAVE_MMAP)
- released += release_unused_segments(m);
-
- /* On failure, disable autotrim to avoid repeated failed future calls */
- if (released == 0 && m->topsize > m->trim_check)
- m->trim_check = MAX_SIZE_T;
- }
-
- return (released != 0)? 1 : 0;
-}
-
-
-/* ---------------------------- malloc support --------------------------- */
-
-/* allocate a large request from the best fitting chunk in a treebin */
-static void* tmalloc_large(mstate m, size_t nb) {
- tchunkptr v = 0;
- size_t rsize = -nb; /* Unsigned negation */
- tchunkptr t;
- bindex_t idx;
- compute_tree_index(nb, idx);
- if ((t = *treebin_at(m, idx)) != 0) {
- /* Traverse tree for this bin looking for node with size == nb */
- size_t sizebits = nb << leftshift_for_tree_index(idx);
- tchunkptr rst = 0; /* The deepest untaken right subtree */
- for (;;) {
- tchunkptr rt;
- size_t trem = chunksize(t) - nb;
- if (trem < rsize) {
- v = t;
- if ((rsize = trem) == 0)
- break;
- }
- rt = t->child[1];
- t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1];
- if (rt != 0 && rt != t)
- rst = rt;
- if (t == 0) {
- t = rst; /* set t to least subtree holding sizes > nb */
- break;
- }
- sizebits <<= 1;
- }
- }
- if (t == 0 && v == 0) { /* set t to root of next non-empty treebin */
- binmap_t leftbits = left_bits(idx2bit(idx)) & m->treemap;
- if (leftbits != 0) {
- bindex_t i;
- binmap_t leastbit = least_bit(leftbits);
- compute_bit2idx(leastbit, i);
- t = *treebin_at(m, i);
- }
- }
-
- while (t != 0) { /* find smallest of tree or subtree */
- size_t trem = chunksize(t) - nb;
- if (trem < rsize) {
- rsize = trem;
- v = t;
- }
- t = leftmost_child(t);
- }
-
- /* If dv is a better fit, return 0 so malloc will use it */
- if (v != 0 && rsize < (size_t)(m->dvsize - nb)) {
- if (RTCHECK(ok_address(m, v))) { /* split */
- mchunkptr r = chunk_plus_offset(v, nb);
- assert(chunksize(v) == rsize + nb);
- if (RTCHECK(ok_next(v, r))) {
- unlink_large_chunk(m, v);
- if (rsize < MIN_CHUNK_SIZE)
- set_inuse_and_pinuse(m, v, (rsize + nb));
- else {
- set_size_and_pinuse_of_inuse_chunk(m, v, nb);
- set_size_and_pinuse_of_free_chunk(r, rsize);
- insert_chunk(m, r, rsize);
- }
- return chunk2mem(v);
- }
- }
- CORRUPTION_ERROR_ACTION(m);
- }
- return 0;
-}
-
-/* allocate a small request from the best fitting chunk in a treebin */
-static void* tmalloc_small(mstate m, size_t nb) {
- tchunkptr t, v;
- size_t rsize;
- bindex_t i;
- binmap_t leastbit = least_bit(m->treemap);
- compute_bit2idx(leastbit, i);
- v = t = *treebin_at(m, i);
- rsize = chunksize(t) - nb;
-
- while ((t = leftmost_child(t)) != 0) {
- size_t trem = chunksize(t) - nb;
- if (trem < rsize) {
- rsize = trem;
- v = t;
- }
- }
-
- if (RTCHECK(ok_address(m, v))) {
- mchunkptr r = chunk_plus_offset(v, nb);
- assert(chunksize(v) == rsize + nb);
- if (RTCHECK(ok_next(v, r))) {
- unlink_large_chunk(m, v);
- if (rsize < MIN_CHUNK_SIZE)
- set_inuse_and_pinuse(m, v, (rsize + nb));
- else {
- set_size_and_pinuse_of_inuse_chunk(m, v, nb);
- set_size_and_pinuse_of_free_chunk(r, rsize);
- replace_dv(m, r, rsize);
- }
- return chunk2mem(v);
- }
- }
-
- CORRUPTION_ERROR_ACTION(m);
- return 0;
-}
-
-/* --------------------------- realloc support --------------------------- */
-
-static void* internal_realloc(mstate m, void* oldmem, size_t bytes) {
- if (bytes >= MAX_REQUEST) {
- MALLOC_FAILURE_ACTION;
- return 0;
- }
- if (!PREACTION(m)) {
- mchunkptr oldp = mem2chunk(oldmem);
- size_t oldsize = chunksize(oldp);
- mchunkptr next = chunk_plus_offset(oldp, oldsize);
- mchunkptr newp = 0;
- void* extra = 0;
-
- /* Try to either shrink or extend into top. Else malloc-copy-free */
-
- if (RTCHECK(ok_address(m, oldp) && ok_inuse(oldp) &&
- ok_next(oldp, next) && ok_pinuse(next))) {
- size_t nb = request2size(bytes);
- if (is_mmapped(oldp))
- newp = mmap_resize(m, oldp, nb);
- else if (oldsize >= nb) { /* already big enough */
- size_t rsize = oldsize - nb;
- newp = oldp;
- if (rsize >= MIN_CHUNK_SIZE) {
- mchunkptr remainder = chunk_plus_offset(newp, nb);
- set_inuse(m, newp, nb);
- set_inuse_and_pinuse(m, remainder, rsize);
- extra = chunk2mem(remainder);
- }
- }
- else if (next == m->top && oldsize + m->topsize > nb) {
- /* Expand into top */
- size_t newsize = oldsize + m->topsize;
- size_t newtopsize = newsize - nb;
- mchunkptr newtop = chunk_plus_offset(oldp, nb);
- set_inuse(m, oldp, nb);
- newtop->head = newtopsize |PINUSE_BIT;
- m->top = newtop;
- m->topsize = newtopsize;
- newp = oldp;
- }
- }
- else {
- USAGE_ERROR_ACTION(m, oldmem);
- POSTACTION(m);
- return 0;
- }
-#if DEBUG
- if (newp != 0) {
- check_inuse_chunk(m, newp); /* Check requires lock */
- }
-#endif
-
- POSTACTION(m);
-
- if (newp != 0) {
- if (extra != 0) {
- internal_free(m, extra);
- }
- return chunk2mem(newp);
- }
- else {
- void* newmem = internal_malloc(m, bytes);
- if (newmem != 0) {
- size_t oc = oldsize - overhead_for(oldp);
- memcpy(newmem, oldmem, (oc < bytes)? oc : bytes);
- internal_free(m, oldmem);
- }
- return newmem;
- }
- }
- return 0;
-}
-
-/* --------------------------- memalign support -------------------------- */
-
-static void* internal_memalign(mstate m, size_t alignment, size_t bytes) {
- if (alignment <= MALLOC_ALIGNMENT) /* Can just use malloc */
- return internal_malloc(m, bytes);
- if (alignment < MIN_CHUNK_SIZE) /* must be at least a minimum chunk size */
- alignment = MIN_CHUNK_SIZE;
- if ((alignment & (alignment-SIZE_T_ONE)) != 0) {/* Ensure a power of 2 */
- size_t a = MALLOC_ALIGNMENT << 1;
- while (a < alignment) a <<= 1;
- alignment = a;
- }
-
- if (bytes >= MAX_REQUEST - alignment) {
- if (m != 0) { /* Test isn't needed but avoids compiler warning */
- MALLOC_FAILURE_ACTION;
- }
- }
- else {
- size_t nb = request2size(bytes);
- size_t req = nb + alignment + MIN_CHUNK_SIZE - CHUNK_OVERHEAD;
- char* mem = (char*)internal_malloc(m, req);
- if (mem != 0) {
- void* leader = 0;
- void* trailer = 0;
- mchunkptr p = mem2chunk(mem);
-
- if (PREACTION(m)) return 0;
- if ((((size_t)(mem)) % alignment) != 0) { /* misaligned */
- /*
- Find an aligned spot inside chunk. Since we need to give
- back leading space in a chunk of at least MIN_CHUNK_SIZE, if
- the first calculation places us at a spot with less than
- MIN_CHUNK_SIZE leader, we can move to the next aligned spot.
- We've allocated enough total room so that this is always
- possible.
- */
- char* br = (char*)mem2chunk((size_t)(((size_t)(mem +
- alignment -
- SIZE_T_ONE)) &
- -alignment));
- char* pos = ((size_t)(br - (char*)(p)) >= MIN_CHUNK_SIZE)?
- br : br+alignment;
- mchunkptr newp = (mchunkptr)pos;
- size_t leadsize = pos - (char*)(p);
- size_t newsize = chunksize(p) - leadsize;
-
- if (is_mmapped(p)) { /* For mmapped chunks, just adjust offset */
- newp->prev_foot = p->prev_foot + leadsize;
- newp->head = newsize;
- }
- else { /* Otherwise, give back leader, use the rest */
- set_inuse(m, newp, newsize);
- set_inuse(m, p, leadsize);
- leader = chunk2mem(p);
- }
- p = newp;
- }
-
- /* Give back spare room at the end */
- if (!is_mmapped(p)) {
- size_t size = chunksize(p);
- if (size > nb + MIN_CHUNK_SIZE) {
- size_t remainder_size = size - nb;
- mchunkptr remainder = chunk_plus_offset(p, nb);
- set_inuse(m, p, nb);
- set_inuse(m, remainder, remainder_size);
- trailer = chunk2mem(remainder);
- }
- }
-
- assert (chunksize(p) >= nb);
- assert((((size_t)(chunk2mem(p))) % alignment) == 0);
- check_inuse_chunk(m, p);
- POSTACTION(m);
- if (leader != 0) {
- internal_free(m, leader);
- }
- if (trailer != 0) {
- internal_free(m, trailer);
- }
- return chunk2mem(p);
- }
- }
- return 0;
-}
-
-/* ------------------------ comalloc/coalloc support --------------------- */
-
-static void** ialloc(mstate m,
- size_t n_elements,
- size_t* sizes,
- int opts,
- void* chunks[]) {
- /*
- This provides common support for independent_X routines, handling
- all of the combinations that can result.
-
- The opts arg has:
- bit 0 set if all elements are same size (using sizes[0])
- bit 1 set if elements should be zeroed
- */
-
- size_t element_size; /* chunksize of each element, if all same */
- size_t contents_size; /* total size of elements */
- size_t array_size; /* request size of pointer array */
- void* mem; /* malloced aggregate space */
- mchunkptr p; /* corresponding chunk */
- size_t remainder_size; /* remaining bytes while splitting */
- void** marray; /* either "chunks" or malloced ptr array */
- mchunkptr array_chunk; /* chunk for malloced ptr array */
- flag_t was_enabled; /* to disable mmap */
- size_t size;
- size_t i;
-
- ensure_initialization();
- /* compute array length, if needed */
- if (chunks != 0) {
- if (n_elements == 0)
- return chunks; /* nothing to do */
- marray = chunks;
- array_size = 0;
- }
- else {
- /* if empty req, must still return chunk representing empty array */
- if (n_elements == 0)
- return (void**)internal_malloc(m, 0);
- marray = 0;
- array_size = request2size(n_elements * (sizeof(void*)));
- }
-
- /* compute total element size */
- if (opts & 0x1) { /* all-same-size */
- element_size = request2size(*sizes);
- contents_size = n_elements * element_size;
- }
- else { /* add up all the sizes */
- element_size = 0;
- contents_size = 0;
- for (i = 0; i != n_elements; ++i)
- contents_size += request2size(sizes[i]);
- }
-
- size = contents_size + array_size;
-
- /*
- Allocate the aggregate chunk. First disable direct-mmapping so
- malloc won't use it, since we would not be able to later
- free/realloc space internal to a segregated mmap region.
- */
- was_enabled = use_mmap(m);
- disable_mmap(m);
- mem = internal_malloc(m, size - CHUNK_OVERHEAD);
- if (was_enabled)
- enable_mmap(m);
- if (mem == 0)
- return 0;
-
- if (PREACTION(m)) return 0;
- p = mem2chunk(mem);
- remainder_size = chunksize(p);
-
- assert(!is_mmapped(p));
-
- if (opts & 0x2) { /* optionally clear the elements */
- memset((size_t*)mem, 0, remainder_size - SIZE_T_SIZE - array_size);
- }
-
- /* If not provided, allocate the pointer array as final part of chunk */
- if (marray == 0) {
- size_t array_chunk_size;
- array_chunk = chunk_plus_offset(p, contents_size);
- array_chunk_size = remainder_size - contents_size;
- marray = (void**) (chunk2mem(array_chunk));
- set_size_and_pinuse_of_inuse_chunk(m, array_chunk, array_chunk_size);
- remainder_size = contents_size;
- }
-
- /* split out elements */
- for (i = 0; ; ++i) {
- marray[i] = chunk2mem(p);
- if (i != n_elements-1) {
- if (element_size != 0)
- size = element_size;
- else
- size = request2size(sizes[i]);
- remainder_size -= size;
- set_size_and_pinuse_of_inuse_chunk(m, p, size);
- p = chunk_plus_offset(p, size);
- }
- else { /* the final element absorbs any overallocation slop */
- set_size_and_pinuse_of_inuse_chunk(m, p, remainder_size);
- break;
- }
- }
-
-#if DEBUG
- if (marray != chunks) {
- /* final element must have exactly exhausted chunk */
- if (element_size != 0) {
- assert(remainder_size == element_size);
- }
- else {
- assert(remainder_size == request2size(sizes[i]));
- }
- check_inuse_chunk(m, mem2chunk(marray));
- }
- for (i = 0; i != n_elements; ++i)
- check_inuse_chunk(m, mem2chunk(marray[i]));
-
-#endif /* DEBUG */
-
- POSTACTION(m);
- return marray;
-}
-
-
-/* -------------------------- public routines ---------------------------- */
-
-#if !ONLY_MSPACES
-
-void* dlmalloc(size_t bytes) {
- /*
- Basic algorithm:
- If a small request (< 256 bytes minus per-chunk overhead):
- 1. If one exists, use a remainderless chunk in associated smallbin.
- (Remainderless means that there are too few excess bytes to
- represent as a chunk.)
- 2. If it is big enough, use the dv chunk, which is normally the
- chunk adjacent to the one used for the most recent small request.
- 3. If one exists, split the smallest available chunk in a bin,
- saving remainder in dv.
- 4. If it is big enough, use the top chunk.
- 5. If available, get memory from system and use it
- Otherwise, for a large request:
- 1. Find the smallest available binned chunk that fits, and use it
- if it is better fitting than dv chunk, splitting if necessary.
- 2. If better fitting than any binned chunk, use the dv chunk.
- 3. If it is big enough, use the top chunk.
- 4. If request size >= mmap threshold, try to directly mmap this chunk.
- 5. If available, get memory from system and use it
-
- The ugly goto's here ensure that postaction occurs along all paths.
- */
-
-#if USE_LOCKS
- ensure_initialization(); /* initialize in sys_alloc if not using locks */
-#endif
-
- if (!PREACTION(gm)) {
- void* mem;
- size_t nb;
- if (bytes <= MAX_SMALL_REQUEST) {
- bindex_t idx;
- binmap_t smallbits;
- nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes);
- idx = small_index(nb);
- smallbits = gm->smallmap >> idx;
-
- if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */
- mchunkptr b, p;
- idx += ~smallbits & 1; /* Uses next bin if idx empty */
- b = smallbin_at(gm, idx);
- p = b->fd;
- assert(chunksize(p) == small_index2size(idx));
- unlink_first_small_chunk(gm, b, p, idx);
- set_inuse_and_pinuse(gm, p, small_index2size(idx));
- mem = chunk2mem(p);
- check_malloced_chunk(gm, mem, nb);
- goto postaction;
- }
-
- else if (nb > gm->dvsize) {
- if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
- mchunkptr b, p, r;
- size_t rsize;
- bindex_t i;
- binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx));
- binmap_t leastbit = least_bit(leftbits);
- compute_bit2idx(leastbit, i);
- b = smallbin_at(gm, i);
- p = b->fd;
- assert(chunksize(p) == small_index2size(i));
- unlink_first_small_chunk(gm, b, p, i);
- rsize = small_index2size(i) - nb;
- /* Fit here cannot be remainderless if 4byte sizes */
- if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE)
- set_inuse_and_pinuse(gm, p, small_index2size(i));
- else {
- set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
- r = chunk_plus_offset(p, nb);
- set_size_and_pinuse_of_free_chunk(r, rsize);
- replace_dv(gm, r, rsize);
- }
- mem = chunk2mem(p);
- check_malloced_chunk(gm, mem, nb);
- goto postaction;
- }
-
- else if (gm->treemap != 0 && (mem = tmalloc_small(gm, nb)) != 0) {
- check_malloced_chunk(gm, mem, nb);
- goto postaction;
- }
- }
- }
- else if (bytes >= MAX_REQUEST)
- nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */
- else {
- nb = pad_request(bytes);
- if (gm->treemap != 0 && (mem = tmalloc_large(gm, nb)) != 0) {
- check_malloced_chunk(gm, mem, nb);
- goto postaction;
- }
- }
-
- if (nb <= gm->dvsize) {
- size_t rsize = gm->dvsize - nb;
- mchunkptr p = gm->dv;
- if (rsize >= MIN_CHUNK_SIZE) { /* split dv */
- mchunkptr r = gm->dv = chunk_plus_offset(p, nb);
- gm->dvsize = rsize;
- set_size_and_pinuse_of_free_chunk(r, rsize);
- set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
- }
- else { /* exhaust dv */
- size_t dvs = gm->dvsize;
- gm->dvsize = 0;
- gm->dv = 0;
- set_inuse_and_pinuse(gm, p, dvs);
- }
- mem = chunk2mem(p);
- check_malloced_chunk(gm, mem, nb);
- goto postaction;
- }
-
- else if (nb < gm->topsize) { /* Split top */
- size_t rsize = gm->topsize -= nb;
- mchunkptr p = gm->top;
- mchunkptr r = gm->top = chunk_plus_offset(p, nb);
- r->head = rsize | PINUSE_BIT;
- set_size_and_pinuse_of_inuse_chunk(gm, p, nb);
- mem = chunk2mem(p);
- check_top_chunk(gm, gm->top);
- check_malloced_chunk(gm, mem, nb);
- goto postaction;
- }
-
- mem = sys_alloc(gm, nb);
-
- postaction:
- POSTACTION(gm);
- return mem;
- }
-
- return 0;
-}
-
-void dlfree(void* mem) {
- /*
- Consolidate freed chunks with preceeding or succeeding bordering
- free chunks, if they exist, and then place in a bin. Intermixed
- with special cases for top, dv, mmapped chunks, and usage errors.
- */
-
- if (mem != 0) {
- mchunkptr p = mem2chunk(mem);
-#if FOOTERS
- mstate fm = get_mstate_for(p);
- if (!ok_magic(fm)) {
- USAGE_ERROR_ACTION(fm, p);
- return;
- }
-#else /* FOOTERS */
-#define fm gm
-#endif /* FOOTERS */
- if (!PREACTION(fm)) {
- check_inuse_chunk(fm, p);
- if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) {
- size_t psize = chunksize(p);
- mchunkptr next = chunk_plus_offset(p, psize);
- if (!pinuse(p)) {
- size_t prevsize = p->prev_foot;
- if (is_mmapped(p)) {
- psize += prevsize + MMAP_FOOT_PAD;
- if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
- fm->footprint -= psize;
- goto postaction;
- }
- else {
- mchunkptr prev = chunk_minus_offset(p, prevsize);
- psize += prevsize;
- p = prev;
- if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */
- if (p != fm->dv) {
- unlink_chunk(fm, p, prevsize);
- }
- else if ((next->head & INUSE_BITS) == INUSE_BITS) {
- fm->dvsize = psize;
- set_free_with_pinuse(p, psize, next);
- goto postaction;
- }
- }
- else
- goto erroraction;
- }
- }
-
- if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) {
- if (!cinuse(next)) { /* consolidate forward */
- if (next == fm->top) {
- size_t tsize = fm->topsize += psize;
- fm->top = p;
- p->head = tsize | PINUSE_BIT;
- if (p == fm->dv) {
- fm->dv = 0;
- fm->dvsize = 0;
- }
- if (should_trim(fm, tsize))
- sys_trim(fm, 0);
- goto postaction;
- }
- else if (next == fm->dv) {
- size_t dsize = fm->dvsize += psize;
- fm->dv = p;
- set_size_and_pinuse_of_free_chunk(p, dsize);
- goto postaction;
- }
- else {
- size_t nsize = chunksize(next);
- psize += nsize;
- unlink_chunk(fm, next, nsize);
- set_size_and_pinuse_of_free_chunk(p, psize);
- if (p == fm->dv) {
- fm->dvsize = psize;
- goto postaction;
- }
- }
- }
- else
- set_free_with_pinuse(p, psize, next);
-
- if (is_small(psize)) {
- insert_small_chunk(fm, p, psize);
- check_free_chunk(fm, p);
- }
- else {
- tchunkptr tp = (tchunkptr)p;
- insert_large_chunk(fm, tp, psize);
- check_free_chunk(fm, p);
- if (--fm->release_checks == 0)
- release_unused_segments(fm);
- }
- goto postaction;
- }
- }
- erroraction:
- USAGE_ERROR_ACTION(fm, p);
- postaction:
- POSTACTION(fm);
- }
- }
-#if !FOOTERS
-#undef fm
-#endif /* FOOTERS */
-}
-
-void* dlcalloc(size_t n_elements, size_t elem_size) {
- void* mem;
- size_t req = 0;
- if (n_elements != 0) {
- req = n_elements * elem_size;
- if (((n_elements | elem_size) & ~(size_t)0xffff) &&
- (req / n_elements != elem_size))
- req = MAX_SIZE_T; /* force downstream failure on overflow */
- }
- mem = dlmalloc(req);
- if (mem != 0 && calloc_must_clear(mem2chunk(mem)))
- memset(mem, 0, req);
- return mem;
-}
-
-void* dlrealloc(void* oldmem, size_t bytes) {
- if (oldmem == 0)
- return dlmalloc(bytes);
-#ifdef REALLOC_ZERO_BYTES_FREES
- if (bytes == 0) {
- dlfree(oldmem);
- return 0;
- }
-#endif /* REALLOC_ZERO_BYTES_FREES */
- else {
-#if ! FOOTERS
- mstate m = gm;
-#else /* FOOTERS */
- mstate m = get_mstate_for(mem2chunk(oldmem));
- if (!ok_magic(m)) {
- USAGE_ERROR_ACTION(m, oldmem);
- return 0;
- }
-#endif /* FOOTERS */
- return internal_realloc(m, oldmem, bytes);
- }
-}
-
-void* dlmemalign(size_t alignment, size_t bytes) {
- return internal_memalign(gm, alignment, bytes);
-}
-
-void** dlindependent_calloc(size_t n_elements, size_t elem_size,
- void* chunks[]) {
- size_t sz = elem_size; /* serves as 1-element array */
- return ialloc(gm, n_elements, &sz, 3, chunks);
-}
-
-void** dlindependent_comalloc(size_t n_elements, size_t sizes[],
- void* chunks[]) {
- return ialloc(gm, n_elements, sizes, 0, chunks);
-}
-
-void* dlvalloc(size_t bytes) {
- size_t pagesz;
- ensure_initialization();
- pagesz = mparams.page_size;
- return dlmemalign(pagesz, bytes);
-}
-
-void* dlpvalloc(size_t bytes) {
- size_t pagesz;
- ensure_initialization();
- pagesz = mparams.page_size;
- return dlmemalign(pagesz, (bytes + pagesz - SIZE_T_ONE) & ~(pagesz - SIZE_T_ONE));
-}
-
-int dlmalloc_trim(size_t pad) {
- int result = 0;
- ensure_initialization();
- if (!PREACTION(gm)) {
- result = sys_trim(gm, pad);
- POSTACTION(gm);
- }
- return result;
-}
-
-size_t dlmalloc_footprint(void) {
- return gm->footprint;
-}
-
-size_t dlmalloc_max_footprint(void) {
- return gm->max_footprint;
-}
-
-#if !NO_MALLINFO
-struct mallinfo dlmallinfo(void) {
- return internal_mallinfo(gm);
-}
-#endif /* NO_MALLINFO */
-
-void dlmalloc_stats() {
- internal_malloc_stats(gm);
-}
-
-int dlmallopt(int param_number, int value) {
- return change_mparam(param_number, value);
-}
-
-#endif /* !ONLY_MSPACES */
-
-size_t dlmalloc_usable_size(void* mem) {
- if (mem != 0) {
- mchunkptr p = mem2chunk(mem);
- if (is_inuse(p))
- return chunksize(p) - overhead_for(p);
- }
- return 0;
-}
-
-/* ----------------------------- user mspaces ---------------------------- */
-
-#if MSPACES
-
-static mstate init_user_mstate(char* tbase, size_t tsize) {
- size_t msize = pad_request(sizeof(struct malloc_state));
- mchunkptr mn;
- mchunkptr msp = align_as_chunk(tbase);
- mstate m = (mstate)(chunk2mem(msp));
- memset(m, 0, msize);
- INITIAL_LOCK(&m->mutex);
- msp->head = (msize|INUSE_BITS);
- m->seg.base = m->least_addr = tbase;
- m->seg.size = m->footprint = m->max_footprint = tsize;
- m->magic = mparams.magic;
- m->release_checks = MAX_RELEASE_CHECK_RATE;
- m->mflags = mparams.default_mflags;
- m->extp = 0;
- m->exts = 0;
- disable_contiguous(m);
- init_bins(m);
- mn = next_chunk(mem2chunk(m));
- init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) - TOP_FOOT_SIZE);
- check_top_chunk(m, m->top);
- return m;
-}
-
-mspace create_mspace(size_t capacity, int locked) {
- mstate m = 0;
- size_t msize;
- ensure_initialization();
- msize = pad_request(sizeof(struct malloc_state));
- if (capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) {
- size_t rs = ((capacity == 0)? mparams.granularity :
- (capacity + TOP_FOOT_SIZE + msize));
- size_t tsize = granularity_align(rs);
- char* tbase = (char*)(CALL_MMAP(tsize));
- if (tbase != CMFAIL) {
- m = init_user_mstate(tbase, tsize);
- m->seg.sflags = USE_MMAP_BIT;
- set_lock(m, locked);
- }
- }
- return (mspace)m;
-}
-
-mspace create_mspace_with_base(void* base, size_t capacity, int locked) {
- mstate m = 0;
- size_t msize;
- ensure_initialization();
- msize = pad_request(sizeof(struct malloc_state));
- if (capacity > msize + TOP_FOOT_SIZE &&
- capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) {
- m = init_user_mstate((char*)base, capacity);
- m->seg.sflags = EXTERN_BIT;
- set_lock(m, locked);
- }
- return (mspace)m;
-}
-
-int mspace_track_large_chunks(mspace msp, int enable) {
- int ret = 0;
- mstate ms = (mstate)msp;
- if (!PREACTION(ms)) {
- if (!use_mmap(ms))
- ret = 1;
- if (!enable)
- enable_mmap(ms);
- else
- disable_mmap(ms);
- POSTACTION(ms);
- }
- return ret;
-}
-
-size_t destroy_mspace(mspace msp) {
- size_t freed = 0;
- mstate ms = (mstate)msp;
- if (ok_magic(ms)) {
- msegmentptr sp = &ms->seg;
- while (sp != 0) {
- char* base = sp->base;
- size_t size = sp->size;
- flag_t flag = sp->sflags;
- sp = sp->next;
- if ((flag & USE_MMAP_BIT) && !(flag & EXTERN_BIT) &&
- CALL_MUNMAP(base, size) == 0)
- freed += size;
- }
- }
- else {
- USAGE_ERROR_ACTION(ms,ms);
- }
- return freed;
-}
-
-/*
- mspace versions of routines are near-clones of the global
- versions. This is not so nice but better than the alternatives.
-*/
-
-
-void* mspace_malloc(mspace msp, size_t bytes) {
- mstate ms = (mstate)msp;
- if (!ok_magic(ms)) {
- USAGE_ERROR_ACTION(ms,ms);
- return 0;
- }
- if (!PREACTION(ms)) {
- void* mem;
- size_t nb;
- if (bytes <= MAX_SMALL_REQUEST) {
- bindex_t idx;
- binmap_t smallbits;
- nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes);
- idx = small_index(nb);
- smallbits = ms->smallmap >> idx;
-
- if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */
- mchunkptr b, p;
- idx += ~smallbits & 1; /* Uses next bin if idx empty */
- b = smallbin_at(ms, idx);
- p = b->fd;
- assert(chunksize(p) == small_index2size(idx));
- unlink_first_small_chunk(ms, b, p, idx);
- set_inuse_and_pinuse(ms, p, small_index2size(idx));
- mem = chunk2mem(p);
- check_malloced_chunk(ms, mem, nb);
- goto postaction;
- }
-
- else if (nb > ms->dvsize) {
- if (smallbits != 0) { /* Use chunk in next nonempty smallbin */
- mchunkptr b, p, r;
- size_t rsize;
- bindex_t i;
- binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx));
- binmap_t leastbit = least_bit(leftbits);
- compute_bit2idx(leastbit, i);
- b = smallbin_at(ms, i);
- p = b->fd;
- assert(chunksize(p) == small_index2size(i));
- unlink_first_small_chunk(ms, b, p, i);
- rsize = small_index2size(i) - nb;
- /* Fit here cannot be remainderless if 4byte sizes */
- if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE)
- set_inuse_and_pinuse(ms, p, small_index2size(i));
- else {
- set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
- r = chunk_plus_offset(p, nb);
- set_size_and_pinuse_of_free_chunk(r, rsize);
- replace_dv(ms, r, rsize);
- }
- mem = chunk2mem(p);
- check_malloced_chunk(ms, mem, nb);
- goto postaction;
- }
-
- else if (ms->treemap != 0 && (mem = tmalloc_small(ms, nb)) != 0) {
- check_malloced_chunk(ms, mem, nb);
- goto postaction;
- }
- }
- }
- else if (bytes >= MAX_REQUEST)
- nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */
- else {
- nb = pad_request(bytes);
- if (ms->treemap != 0 && (mem = tmalloc_large(ms, nb)) != 0) {
- check_malloced_chunk(ms, mem, nb);
- goto postaction;
- }
- }
-
- if (nb <= ms->dvsize) {
- size_t rsize = ms->dvsize - nb;
- mchunkptr p = ms->dv;
- if (rsize >= MIN_CHUNK_SIZE) { /* split dv */
- mchunkptr r = ms->dv = chunk_plus_offset(p, nb);
- ms->dvsize = rsize;
- set_size_and_pinuse_of_free_chunk(r, rsize);
- set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
- }
- else { /* exhaust dv */
- size_t dvs = ms->dvsize;
- ms->dvsize = 0;
- ms->dv = 0;
- set_inuse_and_pinuse(ms, p, dvs);
- }
- mem = chunk2mem(p);
- check_malloced_chunk(ms, mem, nb);
- goto postaction;
- }
-
- else if (nb < ms->topsize) { /* Split top */
- size_t rsize = ms->topsize -= nb;
- mchunkptr p = ms->top;
- mchunkptr r = ms->top = chunk_plus_offset(p, nb);
- r->head = rsize | PINUSE_BIT;
- set_size_and_pinuse_of_inuse_chunk(ms, p, nb);
- mem = chunk2mem(p);
- check_top_chunk(ms, ms->top);
- check_malloced_chunk(ms, mem, nb);
- goto postaction;
- }
-
- mem = sys_alloc(ms, nb);
-
- postaction:
- POSTACTION(ms);
- return mem;
- }
-
- return 0;
-}
-
-void mspace_free(mspace msp, void* mem) {
- if (mem != 0) {
- mchunkptr p = mem2chunk(mem);
-#if FOOTERS
- mstate fm = get_mstate_for(p);
- msp = msp; /* placate people compiling -Wunused */
-#else /* FOOTERS */
- mstate fm = (mstate)msp;
-#endif /* FOOTERS */
- if (!ok_magic(fm)) {
- USAGE_ERROR_ACTION(fm, p);
- return;
- }
- if (!PREACTION(fm)) {
- check_inuse_chunk(fm, p);
- if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) {
- size_t psize = chunksize(p);
- mchunkptr next = chunk_plus_offset(p, psize);
- if (!pinuse(p)) {
- size_t prevsize = p->prev_foot;
- if (is_mmapped(p)) {
- psize += prevsize + MMAP_FOOT_PAD;
- if (CALL_MUNMAP((char*)p - prevsize, psize) == 0)
- fm->footprint -= psize;
- goto postaction;
- }
- else {
- mchunkptr prev = chunk_minus_offset(p, prevsize);
- psize += prevsize;
- p = prev;
- if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */
- if (p != fm->dv) {
- unlink_chunk(fm, p, prevsize);
- }
- else if ((next->head & INUSE_BITS) == INUSE_BITS) {
- fm->dvsize = psize;
- set_free_with_pinuse(p, psize, next);
- goto postaction;
- }
- }
- else
- goto erroraction;
- }
- }
-
- if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) {
- if (!cinuse(next)) { /* consolidate forward */
- if (next == fm->top) {
- size_t tsize = fm->topsize += psize;
- fm->top = p;
- p->head = tsize | PINUSE_BIT;
- if (p == fm->dv) {
- fm->dv = 0;
- fm->dvsize = 0;
- }
- if (should_trim(fm, tsize))
- sys_trim(fm, 0);
- goto postaction;
- }
- else if (next == fm->dv) {
- size_t dsize = fm->dvsize += psize;
- fm->dv = p;
- set_size_and_pinuse_of_free_chunk(p, dsize);
- goto postaction;
- }
- else {
- size_t nsize = chunksize(next);
- psize += nsize;
- unlink_chunk(fm, next, nsize);
- set_size_and_pinuse_of_free_chunk(p, psize);
- if (p == fm->dv) {
- fm->dvsize = psize;
- goto postaction;
- }
- }
- }
- else
- set_free_with_pinuse(p, psize, next);
-
- if (is_small(psize)) {
- insert_small_chunk(fm, p, psize);
- check_free_chunk(fm, p);
- }
- else {
- tchunkptr tp = (tchunkptr)p;
- insert_large_chunk(fm, tp, psize);
- check_free_chunk(fm, p);
- if (--fm->release_checks == 0)
- release_unused_segments(fm);
- }
- goto postaction;
- }
- }
- erroraction:
- USAGE_ERROR_ACTION(fm, p);
- postaction:
- POSTACTION(fm);
- }
- }
-}
-
-void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size) {
- void* mem;
- size_t req = 0;
- mstate ms = (mstate)msp;
- if (!ok_magic(ms)) {
- USAGE_ERROR_ACTION(ms,ms);
- return 0;
- }
- if (n_elements != 0) {
- req = n_elements * elem_size;
- if (((n_elements | elem_size) & ~(size_t)0xffff) &&
- (req / n_elements != elem_size))
- req = MAX_SIZE_T; /* force downstream failure on overflow */
- }
- mem = internal_malloc(ms, req);
- if (mem != 0 && calloc_must_clear(mem2chunk(mem)))
- memset(mem, 0, req);
- return mem;
-}
-
-void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) {
- if (oldmem == 0)
- return mspace_malloc(msp, bytes);
-#ifdef REALLOC_ZERO_BYTES_FREES
- if (bytes == 0) {
- mspace_free(msp, oldmem);
- return 0;
- }
-#endif /* REALLOC_ZERO_BYTES_FREES */
- else {
-#if FOOTERS
- mchunkptr p = mem2chunk(oldmem);
- mstate ms = get_mstate_for(p);
-#else /* FOOTERS */
- mstate ms = (mstate)msp;
-#endif /* FOOTERS */
- if (!ok_magic(ms)) {
- USAGE_ERROR_ACTION(ms,ms);
- return 0;
- }
- return internal_realloc(ms, oldmem, bytes);
- }
-}
-
-void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) {
- mstate ms = (mstate)msp;
- if (!ok_magic(ms)) {
- USAGE_ERROR_ACTION(ms,ms);
- return 0;
- }
- return internal_memalign(ms, alignment, bytes);
-}
-
-void** mspace_independent_calloc(mspace msp, size_t n_elements,
- size_t elem_size, void* chunks[]) {
- size_t sz = elem_size; /* serves as 1-element array */
- mstate ms = (mstate)msp;
- if (!ok_magic(ms)) {
- USAGE_ERROR_ACTION(ms,ms);
- return 0;
- }
- return ialloc(ms, n_elements, &sz, 3, chunks);
-}
-
-void** mspace_independent_comalloc(mspace msp, size_t n_elements,
- size_t sizes[], void* chunks[]) {
- mstate ms = (mstate)msp;
- if (!ok_magic(ms)) {
- USAGE_ERROR_ACTION(ms,ms);
- return 0;
- }
- return ialloc(ms, n_elements, sizes, 0, chunks);
-}
-
-int mspace_trim(mspace msp, size_t pad) {
- int result = 0;
- mstate ms = (mstate)msp;
- if (ok_magic(ms)) {
- if (!PREACTION(ms)) {
- result = sys_trim(ms, pad);
- POSTACTION(ms);
- }
- }
- else {
- USAGE_ERROR_ACTION(ms,ms);
- }
- return result;
-}
-
-void mspace_malloc_stats(mspace msp) {
- mstate ms = (mstate)msp;
- if (ok_magic(ms)) {
- internal_malloc_stats(ms);
- }
- else {
- USAGE_ERROR_ACTION(ms,ms);
- }
-}
-
-size_t mspace_footprint(mspace msp) {
- size_t result = 0;
- mstate ms = (mstate)msp;
- if (ok_magic(ms)) {
- result = ms->footprint;
- }
- else {
- USAGE_ERROR_ACTION(ms,ms);
- }
- return result;
-}
-
-
-size_t mspace_max_footprint(mspace msp) {
- size_t result = 0;
- mstate ms = (mstate)msp;
- if (ok_magic(ms)) {
- result = ms->max_footprint;
- }
- else {
- USAGE_ERROR_ACTION(ms,ms);
- }
- return result;
-}
-
-
-#if !NO_MALLINFO
-struct mallinfo mspace_mallinfo(mspace msp) {
- mstate ms = (mstate)msp;
- if (!ok_magic(ms)) {
- USAGE_ERROR_ACTION(ms,ms);
- }
- return internal_mallinfo(ms);
-}
-#endif /* NO_MALLINFO */
-
-size_t mspace_usable_size(void* mem) {
- if (mem != 0) {
- mchunkptr p = mem2chunk(mem);
- if (is_inuse(p))
- return chunksize(p) - overhead_for(p);
- }
- return 0;
-}
-
-int mspace_mallopt(int param_number, int value) {
- return change_mparam(param_number, value);
-}
-
-#endif /* MSPACES */
-
-
-/* -------------------- Alternative MORECORE functions ------------------- */
-
-/*
- Guidelines for creating a custom version of MORECORE:
-
- * For best performance, MORECORE should allocate in multiples of pagesize.
- * MORECORE may allocate more memory than requested. (Or even less,
- but this will usually result in a malloc failure.)
- * MORECORE must not allocate memory when given argument zero, but
- instead return one past the end address of memory from previous
- nonzero call.
- * For best performance, consecutive calls to MORECORE with positive
- arguments should return increasing addresses, indicating that
- space has been contiguously extended.
- * Even though consecutive calls to MORECORE need not return contiguous
- addresses, it must be OK for malloc'ed chunks to span multiple
- regions in those cases where they do happen to be contiguous.
- * MORECORE need not handle negative arguments -- it may instead
- just return MFAIL when given negative arguments.
- Negative arguments are always multiples of pagesize. MORECORE
- must not misinterpret negative args as large positive unsigned
- args. You can suppress all such calls from even occurring by defining
- MORECORE_CANNOT_TRIM,
-
- As an example alternative MORECORE, here is a custom allocator
- kindly contributed for pre-OSX macOS. It uses virtually but not
- necessarily physically contiguous non-paged memory (locked in,
- present and won't get swapped out). You can use it by uncommenting
- this section, adding some #includes, and setting up the appropriate
- defines above:
-
- #define MORECORE osMoreCore
-
- There is also a shutdown routine that should somehow be called for
- cleanup upon program exit.
-
- #define MAX_POOL_ENTRIES 100
- #define MINIMUM_MORECORE_SIZE (64 * 1024U)
- static int next_os_pool;
- void *our_os_pools[MAX_POOL_ENTRIES];
-
- void *osMoreCore(int size)
- {
- void *ptr = 0;
- static void *sbrk_top = 0;
-
- if (size > 0)
- {
- if (size < MINIMUM_MORECORE_SIZE)
- size = MINIMUM_MORECORE_SIZE;
- if (CurrentExecutionLevel() == kTaskLevel)
- ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0);
- if (ptr == 0)
- {
- return (void *) MFAIL;
- }
- // save ptrs so they can be freed during cleanup
- our_os_pools[next_os_pool] = ptr;
- next_os_pool++;
- ptr = (void *) ((((size_t) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK);
- sbrk_top = (char *) ptr + size;
- return ptr;
- }
- else if (size < 0)
- {
- // we don't currently support shrink behavior
- return (void *) MFAIL;
- }
- else
- {
- return sbrk_top;
- }
- }
-
- // cleanup any allocated memory pools
- // called as last thing before shutting down driver
-
- void osCleanupMem(void)
- {
- void **ptr;
-
- for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++)
- if (*ptr)
- {
- PoolDeallocate(*ptr);
- *ptr = 0;
- }
- }
-
-*/
-
-
-/* -----------------------------------------------------------------------
-History:
- V2.8.4 Wed May 27 09:56:23 2009 Doug Lea (dl at gee)
- * Use zeros instead of prev foot for is_mmapped
- * Add mspace_track_large_chunks; thanks to Jean Brouwers
- * Fix set_inuse in internal_realloc; thanks to Jean Brouwers
- * Fix insufficient sys_alloc padding when using 16byte alignment
- * Fix bad error check in mspace_footprint
- * Adaptations for ptmalloc; thanks to Wolfram Gloger.
- * Reentrant spin locks; thanks to Earl Chew and others
- * Win32 improvements; thanks to Niall Douglas and Earl Chew
- * Add NO_SEGMENT_TRAVERSAL and MAX_RELEASE_CHECK_RATE options
- * Extension hook in malloc_state
- * Various small adjustments to reduce warnings on some compilers
- * Various configuration extensions/changes for more platforms. Thanks
- to all who contributed these.
-
- V2.8.3 Thu Sep 22 11:16:32 2005 Doug Lea (dl at gee)
- * Add max_footprint functions
- * Ensure all appropriate literals are size_t
- * Fix conditional compilation problem for some #define settings
- * Avoid concatenating segments with the one provided
- in create_mspace_with_base
- * Rename some variables to avoid compiler shadowing warnings
- * Use explicit lock initialization.
- * Better handling of sbrk interference.
- * Simplify and fix segment insertion, trimming and mspace_destroy
- * Reinstate REALLOC_ZERO_BYTES_FREES option from 2.7.x
- * Thanks especially to Dennis Flanagan for help on these.
-
- V2.8.2 Sun Jun 12 16:01:10 2005 Doug Lea (dl at gee)
- * Fix memalign brace error.
-
- V2.8.1 Wed Jun 8 16:11:46 2005 Doug Lea (dl at gee)
- * Fix improper #endif nesting in C++
- * Add explicit casts needed for C++
-
- V2.8.0 Mon May 30 14:09:02 2005 Doug Lea (dl at gee)
- * Use trees for large bins
- * Support mspaces
- * Use segments to unify sbrk-based and mmap-based system allocation,
- removing need for emulation on most platforms without sbrk.
- * Default safety checks
- * Optional footer checks. Thanks to William Robertson for the idea.
- * Internal code refactoring
- * Incorporate suggestions and platform-specific changes.
- Thanks to Dennis Flanagan, Colin Plumb, Niall Douglas,
- Aaron Bachmann, Emery Berger, and others.
- * Speed up non-fastbin processing enough to remove fastbins.
- * Remove useless cfree() to avoid conflicts with other apps.
- * Remove internal memcpy, memset. Compilers handle builtins better.
- * Remove some options that no one ever used and rename others.
-
- V2.7.2 Sat Aug 17 09:07:30 2002 Doug Lea (dl at gee)
- * Fix malloc_state bitmap array misdeclaration
-
- V2.7.1 Thu Jul 25 10:58:03 2002 Doug Lea (dl at gee)
- * Allow tuning of FIRST_SORTED_BIN_SIZE
- * Use PTR_UINT as type for all ptr->int casts. Thanks to John Belmonte.
- * Better detection and support for non-contiguousness of MORECORE.
- Thanks to Andreas Mueller, Conal Walsh, and Wolfram Gloger
- * Bypass most of malloc if no frees. Thanks To Emery Berger.
- * Fix freeing of old top non-contiguous chunk im sysmalloc.
- * Raised default trim and map thresholds to 256K.
- * Fix mmap-related #defines. Thanks to Lubos Lunak.
- * Fix copy macros; added LACKS_FCNTL_H. Thanks to Neal Walfield.
- * Branch-free bin calculation
- * Default trim and mmap thresholds now 256K.
-
- V2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee)
- * Introduce independent_comalloc and independent_calloc.
- Thanks to Michael Pachos for motivation and help.
- * Make optional .h file available
- * Allow > 2GB requests on 32bit systems.
- * new WIN32 sbrk, mmap, munmap, lock code from <Walter@GeNeSys-e.de>.
- Thanks also to Andreas Mueller <a.mueller at paradatec.de>,
- and Anonymous.
- * Allow override of MALLOC_ALIGNMENT (Thanks to Ruud Waij for
- helping test this.)
- * memalign: check alignment arg
- * realloc: don't try to shift chunks backwards, since this
- leads to more fragmentation in some programs and doesn't
- seem to help in any others.
- * Collect all cases in malloc requiring system memory into sysmalloc
- * Use mmap as backup to sbrk
- * Place all internal state in malloc_state
- * Introduce fastbins (although similar to 2.5.1)
- * Many minor tunings and cosmetic improvements
- * Introduce USE_PUBLIC_MALLOC_WRAPPERS, USE_MALLOC_LOCK
- * Introduce MALLOC_FAILURE_ACTION, MORECORE_CONTIGUOUS
- Thanks to Tony E. Bennett <tbennett@nvidia.com> and others.
- * Include errno.h to support default failure action.
-
- V2.6.6 Sun Dec 5 07:42:19 1999 Doug Lea (dl at gee)
- * return null for negative arguments
- * Added Several WIN32 cleanups from Martin C. Fong <mcfong at yahoo.com>
- * Add 'LACKS_SYS_PARAM_H' for those systems without 'sys/param.h'
- (e.g. WIN32 platforms)
- * Cleanup header file inclusion for WIN32 platforms
- * Cleanup code to avoid Microsoft Visual C++ compiler complaints
- * Add 'USE_DL_PREFIX' to quickly allow co-existence with existing
- memory allocation routines
- * Set 'malloc_getpagesize' for WIN32 platforms (needs more work)
- * Use 'assert' rather than 'ASSERT' in WIN32 code to conform to
- usage of 'assert' in non-WIN32 code
- * Improve WIN32 'sbrk()' emulation's 'findRegion()' routine to
- avoid infinite loop
- * Always call 'fREe()' rather than 'free()'
-
- V2.6.5 Wed Jun 17 15:57:31 1998 Doug Lea (dl at gee)
- * Fixed ordering problem with boundary-stamping
-
- V2.6.3 Sun May 19 08:17:58 1996 Doug Lea (dl at gee)
- * Added pvalloc, as recommended by H.J. Liu
- * Added 64bit pointer support mainly from Wolfram Gloger
- * Added anonymously donated WIN32 sbrk emulation
- * Malloc, calloc, getpagesize: add optimizations from Raymond Nijssen
- * malloc_extend_top: fix mask error that caused wastage after
- foreign sbrks
- * Add linux mremap support code from HJ Liu
-
- V2.6.2 Tue Dec 5 06:52:55 1995 Doug Lea (dl at gee)
- * Integrated most documentation with the code.
- * Add support for mmap, with help from
- Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
- * Use last_remainder in more cases.
- * Pack bins using idea from colin@nyx10.cs.du.edu
- * Use ordered bins instead of best-fit threshhold
- * Eliminate block-local decls to simplify tracing and debugging.
- * Support another case of realloc via move into top
- * Fix error occuring when initial sbrk_base not word-aligned.
- * Rely on page size for units instead of SBRK_UNIT to
- avoid surprises about sbrk alignment conventions.
- * Add mallinfo, mallopt. Thanks to Raymond Nijssen
- (raymond@es.ele.tue.nl) for the suggestion.
- * Add `pad' argument to malloc_trim and top_pad mallopt parameter.
- * More precautions for cases where other routines call sbrk,
- courtesy of Wolfram Gloger (Gloger@lrz.uni-muenchen.de).
- * Added macros etc., allowing use in linux libc from
- H.J. Lu (hjl@gnu.ai.mit.edu)
- * Inverted this history list
-
- V2.6.1 Sat Dec 2 14:10:57 1995 Doug Lea (dl at gee)
- * Re-tuned and fixed to behave more nicely with V2.6.0 changes.
- * Removed all preallocation code since under current scheme
- the work required to undo bad preallocations exceeds
- the work saved in good cases for most test programs.
- * No longer use return list or unconsolidated bins since
- no scheme using them consistently outperforms those that don't
- given above changes.
- * Use best fit for very large chunks to prevent some worst-cases.
- * Added some support for debugging
-
- V2.6.0 Sat Nov 4 07:05:23 1995 Doug Lea (dl at gee)
- * Removed footers when chunks are in use. Thanks to
- Paul Wilson (wilson@cs.texas.edu) for the suggestion.
-
- V2.5.4 Wed Nov 1 07:54:51 1995 Doug Lea (dl at gee)
- * Added malloc_trim, with help from Wolfram Gloger
- (wmglo@Dent.MED.Uni-Muenchen.DE).
-
- V2.5.3 Tue Apr 26 10:16:01 1994 Doug Lea (dl at g)
-
- V2.5.2 Tue Apr 5 16:20:40 1994 Doug Lea (dl at g)
- * realloc: try to expand in both directions
- * malloc: swap order of clean-bin strategy;
- * realloc: only conditionally expand backwards
- * Try not to scavenge used bins
- * Use bin counts as a guide to preallocation
- * Occasionally bin return list chunks in first scan
- * Add a few optimizations from colin@nyx10.cs.du.edu
-
- V2.5.1 Sat Aug 14 15:40:43 1993 Doug Lea (dl at g)
- * faster bin computation & slightly different binning
- * merged all consolidations to one part of malloc proper
- (eliminating old malloc_find_space & malloc_clean_bin)
- * Scan 2 returns chunks (not just 1)
- * Propagate failure in realloc if malloc returns 0
- * Add stuff to allow compilation on non-ANSI compilers
- from kpv@research.att.com
-
- V2.5 Sat Aug 7 07:41:59 1993 Doug Lea (dl at g.oswego.edu)
- * removed potential for odd address access in prev_chunk
- * removed dependency on getpagesize.h
- * misc cosmetics and a bit more internal documentation
- * anticosmetics: mangled names in macros to evade debugger strangeness
- * tested on sparc, hp-700, dec-mips, rs6000
- with gcc & native cc (hp, dec only) allowing
- Detlefs & Zorn comparison study (in SIGPLAN Notices.)
-
- Trial version Fri Aug 28 13:14:29 1992 Doug Lea (dl at g.oswego.edu)
- * Based loosely on libg++-1.2X malloc. (It retains some of the overall
- structure of old version, but most details differ.)
-
-*/
-
-#endif
+#ifdef NEDMALLOC_ENABLED +/* + This is a version (aka dlmalloc) of malloc/free/realloc written by + Doug Lea and released to the public domain, as explained at + http://creativecommons.org/licenses/publicdomain. Send questions, + comments, complaints, performance data, etc to dl@cs.oswego.edu + +* Version 2.8.4 Wed May 27 09:56:23 2009 Doug Lea (dl at gee) + + Note: There may be an updated version of this malloc obtainable at + ftp://gee.cs.oswego.edu/pub/misc/malloc.c + Check before installing! + +* Quickstart + + This library is all in one file to simplify the most common usage: + ftp it, compile it (-O3), and link it into another program. All of + the compile-time options default to reasonable values for use on + most platforms. You might later want to step through various + compile-time and dynamic tuning options. + + For convenience, an include file for code using this malloc is at: + ftp://gee.cs.oswego.edu/pub/misc/malloc-2.8.4.h + You don't really need this .h file unless you call functions not + defined in your system include files. The .h file contains only the + excerpts from this file needed for using this malloc on ANSI C/C++ + systems, so long as you haven't changed compile-time options about + naming and tuning parameters. If you do, then you can create your + own malloc.h that does include all settings by cutting at the point + indicated below. Note that you may already by default be using a C + library containing a malloc that is based on some version of this + malloc (for example in linux). You might still want to use the one + in this file to customize settings or to avoid overheads associated + with library versions. + +* Vital statistics: + + Supported pointer/size_t representation: 4 or 8 bytes + size_t MUST be an unsigned type of the same width as + pointers. (If you are using an ancient system that declares + size_t as a signed type, or need it to be a different width + than pointers, you can use a previous release of this malloc + (e.g. 2.7.2) supporting these.) + + Alignment: 8 bytes (default) + This suffices for nearly all current machines and C compilers. + However, you can define MALLOC_ALIGNMENT to be wider than this + if necessary (up to 128bytes), at the expense of using more space. + + Minimum overhead per allocated chunk: 4 or 8 bytes (if 4byte sizes) + 8 or 16 bytes (if 8byte sizes) + Each malloced chunk has a hidden word of overhead holding size + and status information, and additional cross-check word + if FOOTERS is defined. + + Minimum allocated size: 4-byte ptrs: 16 bytes (including overhead) + 8-byte ptrs: 32 bytes (including overhead) + + Even a request for zero bytes (i.e., malloc(0)) returns a + pointer to something of the minimum allocatable size. + The maximum overhead wastage (i.e., number of extra bytes + allocated than were requested in malloc) is less than or equal + to the minimum size, except for requests >= mmap_threshold that + are serviced via mmap(), where the worst case wastage is about + 32 bytes plus the remainder from a system page (the minimal + mmap unit); typically 4096 or 8192 bytes. + + Security: static-safe; optionally more or less + The "security" of malloc refers to the ability of malicious + code to accentuate the effects of errors (for example, freeing + space that is not currently malloc'ed or overwriting past the + ends of chunks) in code that calls malloc. This malloc + guarantees not to modify any memory locations below the base of + heap, i.e., static variables, even in the presence of usage + errors. The routines additionally detect most improper frees + and reallocs. All this holds as long as the static bookkeeping + for malloc itself is not corrupted by some other means. This + is only one aspect of security -- these checks do not, and + cannot, detect all possible programming errors. + + If FOOTERS is defined nonzero, then each allocated chunk + carries an additional check word to verify that it was malloced + from its space. These check words are the same within each + execution of a program using malloc, but differ across + executions, so externally crafted fake chunks cannot be + freed. This improves security by rejecting frees/reallocs that + could corrupt heap memory, in addition to the checks preventing + writes to statics that are always on. This may further improve + security at the expense of time and space overhead. (Note that + FOOTERS may also be worth using with MSPACES.) + + By default detected errors cause the program to abort (calling + "abort()"). You can override this to instead proceed past + errors by defining PROCEED_ON_ERROR. In this case, a bad free + has no effect, and a malloc that encounters a bad address + caused by user overwrites will ignore the bad address by + dropping pointers and indices to all known memory. This may + be appropriate for programs that should continue if at all + possible in the face of programming errors, although they may + run out of memory because dropped memory is never reclaimed. + + If you don't like either of these options, you can define + CORRUPTION_ERROR_ACTION and USAGE_ERROR_ACTION to do anything + else. And if if you are sure that your program using malloc has + no errors or vulnerabilities, you can define INSECURE to 1, + which might (or might not) provide a small performance improvement. + + Thread-safety: NOT thread-safe unless USE_LOCKS defined + When USE_LOCKS is defined, each public call to malloc, free, + etc is surrounded with either a pthread mutex or a win32 + spinlock (depending on WIN32). This is not especially fast, and + can be a major bottleneck. It is designed only to provide + minimal protection in concurrent environments, and to provide a + basis for extensions. If you are using malloc in a concurrent + program, consider instead using nedmalloc + (http://www.nedprod.com/programs/portable/nedmalloc/) or + ptmalloc (See http://www.malloc.de), which are derived + from versions of this malloc. + + System requirements: Any combination of MORECORE and/or MMAP/MUNMAP + This malloc can use unix sbrk or any emulation (invoked using + the CALL_MORECORE macro) and/or mmap/munmap or any emulation + (invoked using CALL_MMAP/CALL_MUNMAP) to get and release system + memory. On most unix systems, it tends to work best if both + MORECORE and MMAP are enabled. On Win32, it uses emulations + based on VirtualAlloc. It also uses common C library functions + like memset. + + Compliance: I believe it is compliant with the Single Unix Specification + (See http://www.unix.org). Also SVID/XPG, ANSI C, and probably + others as well. + +* Overview of algorithms + + This is not the fastest, most space-conserving, most portable, or + most tunable malloc ever written. However it is among the fastest + while also being among the most space-conserving, portable and + tunable. Consistent balance across these factors results in a good + general-purpose allocator for malloc-intensive programs. + + In most ways, this malloc is a best-fit allocator. Generally, it + chooses the best-fitting existing chunk for a request, with ties + broken in approximately least-recently-used order. (This strategy + normally maintains low fragmentation.) However, for requests less + than 256bytes, it deviates from best-fit when there is not an + exactly fitting available chunk by preferring to use space adjacent + to that used for the previous small request, as well as by breaking + ties in approximately most-recently-used order. (These enhance + locality of series of small allocations.) And for very large requests + (>= 256Kb by default), it relies on system memory mapping + facilities, if supported. (This helps avoid carrying around and + possibly fragmenting memory used only for large chunks.) + + All operations (except malloc_stats and mallinfo) have execution + times that are bounded by a constant factor of the number of bits in + a size_t, not counting any clearing in calloc or copying in realloc, + or actions surrounding MORECORE and MMAP that have times + proportional to the number of non-contiguous regions returned by + system allocation routines, which is often just 1. In real-time + applications, you can optionally suppress segment traversals using + NO_SEGMENT_TRAVERSAL, which assures bounded execution even when + system allocators return non-contiguous spaces, at the typical + expense of carrying around more memory and increased fragmentation. + + The implementation is not very modular and seriously overuses + macros. Perhaps someday all C compilers will do as good a job + inlining modular code as can now be done by brute-force expansion, + but now, enough of them seem not to. + + Some compilers issue a lot of warnings about code that is + dead/unreachable only on some platforms, and also about intentional + uses of negation on unsigned types. All known cases of each can be + ignored. + + For a longer but out of date high-level description, see + http://gee.cs.oswego.edu/dl/html/malloc.html + +* MSPACES + If MSPACES is defined, then in addition to malloc, free, etc., + this file also defines mspace_malloc, mspace_free, etc. These + are versions of malloc routines that take an "mspace" argument + obtained using create_mspace, to control all internal bookkeeping. + If ONLY_MSPACES is defined, only these versions are compiled. + So if you would like to use this allocator for only some allocations, + and your system malloc for others, you can compile with + ONLY_MSPACES and then do something like... + static mspace mymspace = create_mspace(0,0); // for example + #define mymalloc(bytes) mspace_malloc(mymspace, bytes) + + (Note: If you only need one instance of an mspace, you can instead + use "USE_DL_PREFIX" to relabel the global malloc.) + + You can similarly create thread-local allocators by storing + mspaces as thread-locals. For example: + static __thread mspace tlms = 0; + void* tlmalloc(size_t bytes) { + if (tlms == 0) tlms = create_mspace(0, 0); + return mspace_malloc(tlms, bytes); + } + void tlfree(void* mem) { mspace_free(tlms, mem); } + + Unless FOOTERS is defined, each mspace is completely independent. + You cannot allocate from one and free to another (although + conformance is only weakly checked, so usage errors are not always + caught). If FOOTERS is defined, then each chunk carries around a tag + indicating its originating mspace, and frees are directed to their + originating spaces. + + ------------------------- Compile-time options --------------------------- + +Be careful in setting #define values for numerical constants of type +size_t. On some systems, literal values are not automatically extended +to size_t precision unless they are explicitly casted. You can also +use the symbolic values MAX_SIZE_T, SIZE_T_ONE, etc below. + +WIN32 default: defined if _WIN32 defined + Defining WIN32 sets up defaults for MS environment and compilers. + Otherwise defaults are for unix. Beware that there seem to be some + cases where this malloc might not be a pure drop-in replacement for + Win32 malloc: Random-looking failures from Win32 GDI API's (eg; + SetDIBits()) may be due to bugs in some video driver implementations + when pixel buffers are malloc()ed, and the region spans more than + one VirtualAlloc()ed region. Because dlmalloc uses a small (64Kb) + default granularity, pixel buffers may straddle virtual allocation + regions more often than when using the Microsoft allocator. You can + avoid this by using VirtualAlloc() and VirtualFree() for all pixel + buffers rather than using malloc(). If this is not possible, + recompile this malloc with a larger DEFAULT_GRANULARITY. + +MALLOC_ALIGNMENT default: (size_t)8 + Controls the minimum alignment for malloc'ed chunks. It must be a + power of two and at least 8, even on machines for which smaller + alignments would suffice. It may be defined as larger than this + though. Note however that code and data structures are optimized for + the case of 8-byte alignment. + +MSPACES default: 0 (false) + If true, compile in support for independent allocation spaces. + This is only supported if HAVE_MMAP is true. + +ONLY_MSPACES default: 0 (false) + If true, only compile in mspace versions, not regular versions. + +USE_LOCKS default: 0 (false) + Causes each call to each public routine to be surrounded with + pthread or WIN32 mutex lock/unlock. (If set true, this can be + overridden on a per-mspace basis for mspace versions.) If set to a + non-zero value other than 1, locks are used, but their + implementation is left out, so lock functions must be supplied manually, + as described below. + +USE_SPIN_LOCKS default: 1 iff USE_LOCKS and on x86 using gcc or MSC + If true, uses custom spin locks for locking. This is currently + supported only for x86 platforms using gcc or recent MS compilers. + Otherwise, posix locks or win32 critical sections are used. + +FOOTERS default: 0 + If true, provide extra checking and dispatching by placing + information in the footers of allocated chunks. This adds + space and time overhead. + +INSECURE default: 0 + If true, omit checks for usage errors and heap space overwrites. + +USE_DL_PREFIX default: NOT defined + Causes compiler to prefix all public routines with the string 'dl'. + This can be useful when you only want to use this malloc in one part + of a program, using your regular system malloc elsewhere. + +ABORT default: defined as abort() + Defines how to abort on failed checks. On most systems, a failed + check cannot die with an "assert" or even print an informative + message, because the underlying print routines in turn call malloc, + which will fail again. Generally, the best policy is to simply call + abort(). It's not very useful to do more than this because many + errors due to overwriting will show up as address faults (null, odd + addresses etc) rather than malloc-triggered checks, so will also + abort. Also, most compilers know that abort() does not return, so + can better optimize code conditionally calling it. + +PROCEED_ON_ERROR default: defined as 0 (false) + Controls whether detected bad addresses cause them to bypassed + rather than aborting. If set, detected bad arguments to free and + realloc are ignored. And all bookkeeping information is zeroed out + upon a detected overwrite of freed heap space, thus losing the + ability to ever return it from malloc again, but enabling the + application to proceed. If PROCEED_ON_ERROR is defined, the + static variable malloc_corruption_error_count is compiled in + and can be examined to see if errors have occurred. This option + generates slower code than the default abort policy. + +DEBUG default: NOT defined + The DEBUG setting is mainly intended for people trying to modify + this code or diagnose problems when porting to new platforms. + However, it may also be able to better isolate user errors than just + using runtime checks. The assertions in the check routines spell + out in more detail the assumptions and invariants underlying the + algorithms. The checking is fairly extensive, and will slow down + execution noticeably. Calling malloc_stats or mallinfo with DEBUG + set will attempt to check every non-mmapped allocated and free chunk + in the course of computing the summaries. + +ABORT_ON_ASSERT_FAILURE default: defined as 1 (true) + Debugging assertion failures can be nearly impossible if your + version of the assert macro causes malloc to be called, which will + lead to a cascade of further failures, blowing the runtime stack. + ABORT_ON_ASSERT_FAILURE cause assertions failures to call abort(), + which will usually make debugging easier. + +MALLOC_FAILURE_ACTION default: sets errno to ENOMEM, or no-op on win32 + The action to take before "return 0" when malloc fails to be able to + return memory because there is none available. + +HAVE_MORECORE default: 1 (true) unless win32 or ONLY_MSPACES + True if this system supports sbrk or an emulation of it. + +MORECORE default: sbrk + The name of the sbrk-style system routine to call to obtain more + memory. See below for guidance on writing custom MORECORE + functions. The type of the argument to sbrk/MORECORE varies across + systems. It cannot be size_t, because it supports negative + arguments, so it is normally the signed type of the same width as + size_t (sometimes declared as "intptr_t"). It doesn't much matter + though. Internally, we only call it with arguments less than half + the max value of a size_t, which should work across all reasonable + possibilities, although sometimes generating compiler warnings. + +MORECORE_CONTIGUOUS default: 1 (true) if HAVE_MORECORE + If true, take advantage of fact that consecutive calls to MORECORE + with positive arguments always return contiguous increasing + addresses. This is true of unix sbrk. It does not hurt too much to + set it true anyway, since malloc copes with non-contiguities. + Setting it false when definitely non-contiguous saves time + and possibly wasted space it would take to discover this though. + +MORECORE_CANNOT_TRIM default: NOT defined + True if MORECORE cannot release space back to the system when given + negative arguments. This is generally necessary only if you are + using a hand-crafted MORECORE function that cannot handle negative + arguments. + +NO_SEGMENT_TRAVERSAL default: 0 + If non-zero, suppresses traversals of memory segments + returned by either MORECORE or CALL_MMAP. This disables + merging of segments that are contiguous, and selectively + releasing them to the OS if unused, but bounds execution times. + +HAVE_MMAP default: 1 (true) + True if this system supports mmap or an emulation of it. If so, and + HAVE_MORECORE is not true, MMAP is used for all system + allocation. If set and HAVE_MORECORE is true as well, MMAP is + primarily used to directly allocate very large blocks. It is also + used as a backup strategy in cases where MORECORE fails to provide + space from system. Note: A single call to MUNMAP is assumed to be + able to unmap memory that may have be allocated using multiple calls + to MMAP, so long as they are adjacent. + +HAVE_MREMAP default: 1 on linux, else 0 + If true realloc() uses mremap() to re-allocate large blocks and + extend or shrink allocation spaces. + +MMAP_CLEARS default: 1 except on WINCE. + True if mmap clears memory so calloc doesn't need to. This is true + for standard unix mmap using /dev/zero and on WIN32 except for WINCE. + +USE_BUILTIN_FFS default: 0 (i.e., not used) + Causes malloc to use the builtin ffs() function to compute indices. + Some compilers may recognize and intrinsify ffs to be faster than the + supplied C version. Also, the case of x86 using gcc is special-cased + to an asm instruction, so is already as fast as it can be, and so + this setting has no effect. Similarly for Win32 under recent MS compilers. + (On most x86s, the asm version is only slightly faster than the C version.) + +malloc_getpagesize default: derive from system includes, or 4096. + The system page size. To the extent possible, this malloc manages + memory from the system in page-size units. This may be (and + usually is) a function rather than a constant. This is ignored + if WIN32, where page size is determined using getSystemInfo during + initialization. This may be several megabytes if ENABLE_LARGE_PAGES + is enabled. + +ENABLE_LARGE_PAGES default: NOT defined + Causes the system page size to be the value of GetLargePageMinimum() + if that function is available (Windows Server 2003/Vista or later). + This allows the use of large page entries in the MMU which can + significantly improve performance in large working set applications + as TLB cache load is reduced by a factor of three. Note that enabling + this option is equal to locking the process' memory in current + implementations of Windows and requires the SE_LOCK_MEMORY_PRIVILEGE + to be held by the process in order to succeed. + +USE_DEV_RANDOM default: 0 (i.e., not used) + Causes malloc to use /dev/random to initialize secure magic seed for + stamping footers. Otherwise, the current time is used. + +NO_MALLINFO default: 0 + If defined, don't compile "mallinfo". This can be a simple way + of dealing with mismatches between system declarations and + those in this file. + +MALLINFO_FIELD_TYPE default: size_t + The type of the fields in the mallinfo struct. This was originally + defined as "int" in SVID etc, but is more usefully defined as + size_t. The value is used only if HAVE_USR_INCLUDE_MALLOC_H is not set + +REALLOC_ZERO_BYTES_FREES default: not defined + This should be set if a call to realloc with zero bytes should + be the same as a call to free. Some people think it should. Otherwise, + since this malloc returns a unique pointer for malloc(0), so does + realloc(p, 0). + +LACKS_UNISTD_H, LACKS_FCNTL_H, LACKS_SYS_PARAM_H, LACKS_SYS_MMAN_H +LACKS_STRINGS_H, LACKS_STRING_H, LACKS_SYS_TYPES_H, LACKS_ERRNO_H +LACKS_STDLIB_H default: NOT defined unless on WIN32 + Define these if your system does not have these header files. + You might need to manually insert some of the declarations they provide. + +DEFAULT_GRANULARITY default: page size if MORECORE_CONTIGUOUS, + system_info.dwAllocationGranularity in WIN32, + GetLargePageMinimum() if ENABLE_LARGE_PAGES, + otherwise 64K. + Also settable using mallopt(M_GRANULARITY, x) + The unit for allocating and deallocating memory from the system. On + most systems with contiguous MORECORE, there is no reason to + make this more than a page. However, systems with MMAP tend to + either require or encourage larger granularities. You can increase + this value to prevent system allocation functions to be called so + often, especially if they are slow. The value must be at least one + page and must be a power of two. Setting to 0 causes initialization + to either page size or win32 region size. (Note: In previous + versions of malloc, the equivalent of this option was called + "TOP_PAD") + +DEFAULT_GRANULARITY_ALIGNED default: undefined (which means page size) + Whether to enforce alignment when allocating and deallocating memory + from the system i.e. the base address of all allocations will be + aligned to DEFAULT_GRANULARITY if it is set. Note that enabling this carries + some overhead as multiple calls must now be made when probing for a valid + aligned value, however it does greatly ease the checking for whether + a given memory pointer was allocated by this allocator rather than + some other. + +DEFAULT_TRIM_THRESHOLD default: 2MB + Also settable using mallopt(M_TRIM_THRESHOLD, x) + The maximum amount of unused top-most memory to keep before + releasing via malloc_trim in free(). Automatic trimming is mainly + useful in long-lived programs using contiguous MORECORE. Because + trimming via sbrk can be slow on some systems, and can sometimes be + wasteful (in cases where programs immediately afterward allocate + more large chunks) the value should be high enough so that your + overall system performance would improve by releasing this much + memory. As a rough guide, you might set to a value close to the + average size of a process (program) running on your system. + Releasing this much memory would allow such a process to run in + memory. Generally, it is worth tuning trim thresholds when a + program undergoes phases where several large chunks are allocated + and released in ways that can reuse each other's storage, perhaps + mixed with phases where there are no such chunks at all. The trim + value must be greater than page size to have any useful effect. To + disable trimming completely, you can set to MAX_SIZE_T. Note that the trick + some people use of mallocing a huge space and then freeing it at + program startup, in an attempt to reserve system memory, doesn't + have the intended effect under automatic trimming, since that memory + will immediately be returned to the system. + +DEFAULT_MMAP_THRESHOLD default: 256K + Also settable using mallopt(M_MMAP_THRESHOLD, x) + The request size threshold for using MMAP to directly service a + request. Requests of at least this size that cannot be allocated + using already-existing space will be serviced via mmap. (If enough + normal freed space already exists it is used instead.) Using mmap + segregates relatively large chunks of memory so that they can be + individually obtained and released from the host system. A request + serviced through mmap is never reused by any other request (at least + not directly; the system may just so happen to remap successive + requests to the same locations). Segregating space in this way has + the benefits that: Mmapped space can always be individually released + back to the system, which helps keep the system level memory demands + of a long-lived program low. Also, mapped memory doesn't become + `locked' between other chunks, as can happen with normally allocated + chunks, which means that even trimming via malloc_trim would not + release them. However, it has the disadvantage that the space + cannot be reclaimed, consolidated, and then used to service later + requests, as happens with normal chunks. The advantages of mmap + nearly always outweigh disadvantages for "large" chunks, but the + value of "large" may vary across systems. The default is an + empirically derived value that works well in most systems. You can + disable mmap by setting to MAX_SIZE_T. + +MAX_RELEASE_CHECK_RATE default: 4095 unless not HAVE_MMAP + The number of consolidated frees between checks to release + unused segments when freeing. When using non-contiguous segments, + especially with multiple mspaces, checking only for topmost space + doesn't always suffice to trigger trimming. To compensate for this, + free() will, with a period of MAX_RELEASE_CHECK_RATE (or the + current number of segments, if greater) try to release unused + segments to the OS when freeing chunks that result in + consolidation. The best value for this parameter is a compromise + between slowing down frees with relatively costly checks that + rarely trigger versus holding on to unused memory. To effectively + disable, set to MAX_SIZE_T. This may lead to a very slight speed + improvement at the expense of carrying around more memory. +*/ + +/* Version identifier to allow people to support multiple versions */ +#ifndef DLMALLOC_VERSION +#define DLMALLOC_VERSION 20804 +#endif /* DLMALLOC_VERSION */ + +#ifndef WIN32 +#ifdef _WIN32 +#define WIN32 1 +#endif /* _WIN32 */ +#ifdef _WIN32_WCE +#define LACKS_FCNTL_H +#define WIN32 1 +#endif /* _WIN32_WCE */ +#endif /* WIN32 */ +#ifdef WIN32 +#define WIN32_LEAN_AND_MEAN +#include <windows.h> +#include <tchar.h> +#define HAVE_MMAP 1 +#define HAVE_MORECORE 0 +#define LACKS_UNISTD_H +#define LACKS_SYS_PARAM_H +#define LACKS_SYS_MMAN_H +#define LACKS_STRING_H +#define LACKS_STRINGS_H +#define LACKS_SYS_TYPES_H +#define LACKS_ERRNO_H +#ifndef MALLOC_FAILURE_ACTION +#define MALLOC_FAILURE_ACTION +#endif /* MALLOC_FAILURE_ACTION */ +#ifdef _WIN32_WCE /* WINCE reportedly does not clear */ +#define MMAP_CLEARS 0 +#else +#define MMAP_CLEARS 1 +#endif /* _WIN32_WCE */ +#endif /* WIN32 */ + +#if defined(DARWIN) || defined(_DARWIN) +/* Mac OSX docs advise not to use sbrk; it seems better to use mmap */ +#ifndef HAVE_MORECORE +#define HAVE_MORECORE 0 +#define HAVE_MMAP 1 +/* OSX allocators provide 16 byte alignment */ +#ifndef MALLOC_ALIGNMENT +#define MALLOC_ALIGNMENT ((size_t)16U) +#endif +#endif /* HAVE_MORECORE */ +#endif /* DARWIN */ + +#ifndef LACKS_SYS_TYPES_H +#include <sys/types.h> /* For size_t */ +#endif /* LACKS_SYS_TYPES_H */ + +#if (defined(__GNUC__) && ((defined(__i386__) || defined(__x86_64__)))) || (defined(_MSC_VER) && _MSC_VER>=1310) +#define SPIN_LOCKS_AVAILABLE 1 +#else +#define SPIN_LOCKS_AVAILABLE 0 +#endif + +/* The maximum possible size_t value has all bits set */ +#define MAX_SIZE_T (~(size_t)0) + +#ifndef ONLY_MSPACES +#define ONLY_MSPACES 0 /* define to a value */ +#else +#define ONLY_MSPACES 1 +#endif /* ONLY_MSPACES */ +#ifndef MSPACES +#if ONLY_MSPACES +#define MSPACES 1 +#else /* ONLY_MSPACES */ +#define MSPACES 0 +#endif /* ONLY_MSPACES */ +#endif /* MSPACES */ +#ifndef MALLOC_ALIGNMENT +#define MALLOC_ALIGNMENT ((size_t)8U) +#endif /* MALLOC_ALIGNMENT */ +#ifndef FOOTERS +#define FOOTERS 0 +#endif /* FOOTERS */ +#ifndef ABORT +#define ABORT abort() +#endif /* ABORT */ +#ifndef ABORT_ON_ASSERT_FAILURE +#define ABORT_ON_ASSERT_FAILURE 1 +#endif /* ABORT_ON_ASSERT_FAILURE */ +#ifndef PROCEED_ON_ERROR +#define PROCEED_ON_ERROR 0 +#endif /* PROCEED_ON_ERROR */ +#ifndef USE_LOCKS +#define USE_LOCKS 0 +#endif /* USE_LOCKS */ +#ifndef USE_SPIN_LOCKS +#if USE_LOCKS && SPIN_LOCKS_AVAILABLE +#define USE_SPIN_LOCKS 1 +#else +#define USE_SPIN_LOCKS 0 +#endif /* USE_LOCKS && SPIN_LOCKS_AVAILABLE. */ +#endif /* USE_SPIN_LOCKS */ +#ifndef INSECURE +#define INSECURE 0 +#endif /* INSECURE */ +#ifndef HAVE_MMAP +#define HAVE_MMAP 1 +#endif /* HAVE_MMAP */ +#ifndef MMAP_CLEARS +#define MMAP_CLEARS 1 +#endif /* MMAP_CLEARS */ +#ifndef HAVE_MREMAP +#ifdef linux +#define HAVE_MREMAP 1 +#else /* linux */ +#define HAVE_MREMAP 0 +#endif /* linux */ +#endif /* HAVE_MREMAP */ +#ifndef MALLOC_FAILURE_ACTION +#define MALLOC_FAILURE_ACTION errno = ENOMEM; +#endif /* MALLOC_FAILURE_ACTION */ +#ifndef HAVE_MORECORE +#if ONLY_MSPACES +#define HAVE_MORECORE 0 +#else /* ONLY_MSPACES */ +#define HAVE_MORECORE 1 +#endif /* ONLY_MSPACES */ +#endif /* HAVE_MORECORE */ +#if !HAVE_MORECORE +#define MORECORE_CONTIGUOUS 0 +#else /* !HAVE_MORECORE */ +#define MORECORE_DEFAULT sbrk +#ifndef MORECORE_CONTIGUOUS +#define MORECORE_CONTIGUOUS 1 +#endif /* MORECORE_CONTIGUOUS */ +#endif /* HAVE_MORECORE */ +#ifndef DEFAULT_GRANULARITY +#if (MORECORE_CONTIGUOUS || defined(WIN32)) +#define DEFAULT_GRANULARITY (0) /* 0 means to compute in init_mparams */ +#else /* MORECORE_CONTIGUOUS */ +#define DEFAULT_GRANULARITY ((size_t)64U * (size_t)1024U) +#endif /* MORECORE_CONTIGUOUS */ +#endif /* DEFAULT_GRANULARITY */ +#ifndef DEFAULT_TRIM_THRESHOLD +#ifndef MORECORE_CANNOT_TRIM +#define DEFAULT_TRIM_THRESHOLD ((size_t)2U * (size_t)1024U * (size_t)1024U) +#else /* MORECORE_CANNOT_TRIM */ +#define DEFAULT_TRIM_THRESHOLD MAX_SIZE_T +#endif /* MORECORE_CANNOT_TRIM */ +#endif /* DEFAULT_TRIM_THRESHOLD */ +#ifndef DEFAULT_MMAP_THRESHOLD +#if HAVE_MMAP +#define DEFAULT_MMAP_THRESHOLD ((size_t)256U * (size_t)1024U) +#else /* HAVE_MMAP */ +#define DEFAULT_MMAP_THRESHOLD MAX_SIZE_T +#endif /* HAVE_MMAP */ +#endif /* DEFAULT_MMAP_THRESHOLD */ +#ifndef MAX_RELEASE_CHECK_RATE +#if HAVE_MMAP +#define MAX_RELEASE_CHECK_RATE 4095 +#else +#define MAX_RELEASE_CHECK_RATE MAX_SIZE_T +#endif /* HAVE_MMAP */ +#endif /* MAX_RELEASE_CHECK_RATE */ +#ifndef USE_BUILTIN_FFS +#define USE_BUILTIN_FFS 0 +#endif /* USE_BUILTIN_FFS */ +#ifndef USE_DEV_RANDOM +#define USE_DEV_RANDOM 0 +#endif /* USE_DEV_RANDOM */ +#ifndef NO_MALLINFO +#define NO_MALLINFO 0 +#endif /* NO_MALLINFO */ +#ifndef MALLINFO_FIELD_TYPE +#define MALLINFO_FIELD_TYPE size_t +#endif /* MALLINFO_FIELD_TYPE */ +#ifndef NO_SEGMENT_TRAVERSAL +#define NO_SEGMENT_TRAVERSAL 0 +#endif /* NO_SEGMENT_TRAVERSAL */ + +/* + mallopt tuning options. SVID/XPG defines four standard parameter + numbers for mallopt, normally defined in malloc.h. None of these + are used in this malloc, so setting them has no effect. But this + malloc does support the following options. +*/ + +#define M_TRIM_THRESHOLD (-1) +#define M_GRANULARITY (-2) +#define M_MMAP_THRESHOLD (-3) + +/* ------------------------ Mallinfo declarations ------------------------ */ + +#if !NO_MALLINFO +/* + This version of malloc supports the standard SVID/XPG mallinfo + routine that returns a struct containing usage properties and + statistics. It should work on any system that has a + /usr/include/malloc.h defining struct mallinfo. The main + declaration needed is the mallinfo struct that is returned (by-copy) + by mallinfo(). The malloinfo struct contains a bunch of fields that + are not even meaningful in this version of malloc. These fields are + are instead filled by mallinfo() with other numbers that might be of + interest. + + HAVE_USR_INCLUDE_MALLOC_H should be set if you have a + /usr/include/malloc.h file that includes a declaration of struct + mallinfo. If so, it is included; else a compliant version is + declared below. These must be precisely the same for mallinfo() to + work. The original SVID version of this struct, defined on most + systems with mallinfo, declares all fields as ints. But some others + define as unsigned long. If your system defines the fields using a + type of different width than listed here, you MUST #include your + system version and #define HAVE_USR_INCLUDE_MALLOC_H. +*/ + +/* #define HAVE_USR_INCLUDE_MALLOC_H */ + +#ifdef HAVE_USR_INCLUDE_MALLOC_H +#include "/usr/include/malloc.h" +#else /* HAVE_USR_INCLUDE_MALLOC_H */ +#ifndef STRUCT_MALLINFO_DECLARED +#define STRUCT_MALLINFO_DECLARED 1 +struct mallinfo { + MALLINFO_FIELD_TYPE arena; /* non-mmapped space allocated from system */ + MALLINFO_FIELD_TYPE ordblks; /* number of free chunks */ + MALLINFO_FIELD_TYPE smblks; /* always 0 */ + MALLINFO_FIELD_TYPE hblks; /* always 0 */ + MALLINFO_FIELD_TYPE hblkhd; /* space in mmapped regions */ + MALLINFO_FIELD_TYPE usmblks; /* maximum total allocated space */ + MALLINFO_FIELD_TYPE fsmblks; /* always 0 */ + MALLINFO_FIELD_TYPE uordblks; /* total allocated space */ + MALLINFO_FIELD_TYPE fordblks; /* total free space */ + MALLINFO_FIELD_TYPE keepcost; /* releasable (via malloc_trim) space */ +}; +#endif /* STRUCT_MALLINFO_DECLARED */ +#endif /* HAVE_USR_INCLUDE_MALLOC_H */ +#endif /* NO_MALLINFO */ + +/* + Try to persuade compilers to inline. The most critical functions for + inlining are defined as macros, so these aren't used for them. +*/ + +#ifndef FORCEINLINE + #if defined(__GNUC__) +#define FORCEINLINE __inline __attribute__ ((always_inline)) + #elif defined(_MSC_VER) + #define FORCEINLINE __forceinline + #endif +#endif +#ifndef NOINLINE + #if defined(__GNUC__) + #define NOINLINE __attribute__ ((noinline)) + #elif defined(_MSC_VER) + #define NOINLINE __declspec(noinline) + #else + #define NOINLINE + #endif +#endif + +#ifdef __cplusplus +extern "C" { +#ifndef FORCEINLINE + #define FORCEINLINE inline +#endif +#endif /* __cplusplus */ +#ifndef FORCEINLINE + #define FORCEINLINE +#endif + +#if !ONLY_MSPACES + +/* ------------------- Declarations of public routines ------------------- */ + +#ifndef USE_DL_PREFIX +#define dlcalloc calloc +#define dlfree free +#define dlmalloc malloc +#define dlmemalign memalign +#define dlrealloc realloc +#define dlvalloc valloc +#define dlpvalloc pvalloc +#define dlmallinfo mallinfo +#define dlmallopt mallopt +#define dlmalloc_trim malloc_trim +#define dlmalloc_stats malloc_stats +#define dlmalloc_usable_size malloc_usable_size +#define dlmalloc_footprint malloc_footprint +#define dlmalloc_max_footprint malloc_max_footprint +#define dlindependent_calloc independent_calloc +#define dlindependent_comalloc independent_comalloc +#endif /* USE_DL_PREFIX */ + + +/* + malloc(size_t n) + Returns a pointer to a newly allocated chunk of at least n bytes, or + null if no space is available, in which case errno is set to ENOMEM + on ANSI C systems. + + If n is zero, malloc returns a minimum-sized chunk. (The minimum + size is 16 bytes on most 32bit systems, and 32 bytes on 64bit + systems.) Note that size_t is an unsigned type, so calls with + arguments that would be negative if signed are interpreted as + requests for huge amounts of space, which will often fail. The + maximum supported value of n differs across systems, but is in all + cases less than the maximum representable value of a size_t. +*/ +void* dlmalloc(size_t); + +/* + free(void* p) + Releases the chunk of memory pointed to by p, that had been previously + allocated using malloc or a related routine such as realloc. + It has no effect if p is null. If p was not malloced or already + freed, free(p) will by default cause the current program to abort. +*/ +void dlfree(void*); + +/* + calloc(size_t n_elements, size_t element_size); + Returns a pointer to n_elements * element_size bytes, with all locations + set to zero. +*/ +void* dlcalloc(size_t, size_t); + +/* + realloc(void* p, size_t n) + Returns a pointer to a chunk of size n that contains the same data + as does chunk p up to the minimum of (n, p's size) bytes, or null + if no space is available. + + The returned pointer may or may not be the same as p. The algorithm + prefers extending p in most cases when possible, otherwise it + employs the equivalent of a malloc-copy-free sequence. + + If p is null, realloc is equivalent to malloc. + + If space is not available, realloc returns null, errno is set (if on + ANSI) and p is NOT freed. + + if n is for fewer bytes than already held by p, the newly unused + space is lopped off and freed if possible. realloc with a size + argument of zero (re)allocates a minimum-sized chunk. + + The old unix realloc convention of allowing the last-free'd chunk + to be used as an argument to realloc is not supported. +*/ + +void* dlrealloc(void*, size_t); + +/* + memalign(size_t alignment, size_t n); + Returns a pointer to a newly allocated chunk of n bytes, aligned + in accord with the alignment argument. + + The alignment argument should be a power of two. If the argument is + not a power of two, the nearest greater power is used. + 8-byte alignment is guaranteed by normal malloc calls, so don't + bother calling memalign with an argument of 8 or less. + + Overreliance on memalign is a sure way to fragment space. +*/ +void* dlmemalign(size_t, size_t); + +/* + valloc(size_t n); + Equivalent to memalign(pagesize, n), where pagesize is the page + size of the system. If the pagesize is unknown, 4096 is used. +*/ +void* dlvalloc(size_t); + +/* + mallopt(int parameter_number, int parameter_value) + Sets tunable parameters The format is to provide a + (parameter-number, parameter-value) pair. mallopt then sets the + corresponding parameter to the argument value if it can (i.e., so + long as the value is meaningful), and returns 1 if successful else + 0. To workaround the fact that mallopt is specified to use int, + not size_t parameters, the value -1 is specially treated as the + maximum unsigned size_t value. + + SVID/XPG/ANSI defines four standard param numbers for mallopt, + normally defined in malloc.h. None of these are use in this malloc, + so setting them has no effect. But this malloc also supports other + options in mallopt. See below for details. Briefly, supported + parameters are as follows (listed defaults are for "typical" + configurations). + + Symbol param # default allowed param values + M_TRIM_THRESHOLD -1 2*1024*1024 any (-1 disables) + M_GRANULARITY -2 page size any power of 2 >= page size + M_MMAP_THRESHOLD -3 256*1024 any (or 0 if no MMAP support) +*/ +int dlmallopt(int, int); + +/* + malloc_footprint(); + Returns the number of bytes obtained from the system. The total + number of bytes allocated by malloc, realloc etc., is less than this + value. Unlike mallinfo, this function returns only a precomputed + result, so can be called frequently to monitor memory consumption. + Even if locks are otherwise defined, this function does not use them, + so results might not be up to date. +*/ +size_t dlmalloc_footprint(void); + +/* + malloc_max_footprint(); + Returns the maximum number of bytes obtained from the system. This + value will be greater than current footprint if deallocated space + has been reclaimed by the system. The peak number of bytes allocated + by malloc, realloc etc., is less than this value. Unlike mallinfo, + this function returns only a precomputed result, so can be called + frequently to monitor memory consumption. Even if locks are + otherwise defined, this function does not use them, so results might + not be up to date. +*/ +size_t dlmalloc_max_footprint(void); + +#if !NO_MALLINFO +/* + mallinfo() + Returns (by copy) a struct containing various summary statistics: + + arena: current total non-mmapped bytes allocated from system + ordblks: the number of free chunks + smblks: always zero. + hblks: current number of mmapped regions + hblkhd: total bytes held in mmapped regions + usmblks: the maximum total allocated space. This will be greater + than current total if trimming has occurred. + fsmblks: always zero + uordblks: current total allocated space (normal or mmapped) + fordblks: total free space + keepcost: the maximum number of bytes that could ideally be released + back to system via malloc_trim. ("ideally" means that + it ignores page restrictions etc.) + + Because these fields are ints, but internal bookkeeping may + be kept as longs, the reported values may wrap around zero and + thus be inaccurate. +*/ +struct mallinfo dlmallinfo(void); +#endif /* NO_MALLINFO */ + +/* + independent_calloc(size_t n_elements, size_t element_size, void* chunks[]); + + independent_calloc is similar to calloc, but instead of returning a + single cleared space, it returns an array of pointers to n_elements + independent elements that can hold contents of size elem_size, each + of which starts out cleared, and can be independently freed, + realloc'ed etc. The elements are guaranteed to be adjacently + allocated (this is not guaranteed to occur with multiple callocs or + mallocs), which may also improve cache locality in some + applications. + + The "chunks" argument is optional (i.e., may be null, which is + probably the most typical usage). If it is null, the returned array + is itself dynamically allocated and should also be freed when it is + no longer needed. Otherwise, the chunks array must be of at least + n_elements in length. It is filled in with the pointers to the + chunks. + + In either case, independent_calloc returns this pointer array, or + null if the allocation failed. If n_elements is zero and "chunks" + is null, it returns a chunk representing an array with zero elements + (which should be freed if not wanted). + + Each element must be individually freed when it is no longer + needed. If you'd like to instead be able to free all at once, you + should instead use regular calloc and assign pointers into this + space to represent elements. (In this case though, you cannot + independently free elements.) + + independent_calloc simplifies and speeds up implementations of many + kinds of pools. It may also be useful when constructing large data + structures that initially have a fixed number of fixed-sized nodes, + but the number is not known at compile time, and some of the nodes + may later need to be freed. For example: + + struct Node { int item; struct Node* next; }; + + struct Node* build_list() { + struct Node** pool; + int n = read_number_of_nodes_needed(); + if (n <= 0) return 0; + pool = (struct Node**)(independent_calloc(n, sizeof(struct Node), 0); + if (pool == 0) die(); + // organize into a linked list... + struct Node* first = pool[0]; + for (i = 0; i < n-1; ++i) + pool[i]->next = pool[i+1]; + free(pool); // Can now free the array (or not, if it is needed later) + return first; + } +*/ +void** dlindependent_calloc(size_t, size_t, void**); + +/* + independent_comalloc(size_t n_elements, size_t sizes[], void* chunks[]); + + independent_comalloc allocates, all at once, a set of n_elements + chunks with sizes indicated in the "sizes" array. It returns + an array of pointers to these elements, each of which can be + independently freed, realloc'ed etc. The elements are guaranteed to + be adjacently allocated (this is not guaranteed to occur with + multiple callocs or mallocs), which may also improve cache locality + in some applications. + + The "chunks" argument is optional (i.e., may be null). If it is null + the returned array is itself dynamically allocated and should also + be freed when it is no longer needed. Otherwise, the chunks array + must be of at least n_elements in length. It is filled in with the + pointers to the chunks. + + In either case, independent_comalloc returns this pointer array, or + null if the allocation failed. If n_elements is zero and chunks is + null, it returns a chunk representing an array with zero elements + (which should be freed if not wanted). + + Each element must be individually freed when it is no longer + needed. If you'd like to instead be able to free all at once, you + should instead use a single regular malloc, and assign pointers at + particular offsets in the aggregate space. (In this case though, you + cannot independently free elements.) + + independent_comallac differs from independent_calloc in that each + element may have a different size, and also that it does not + automatically clear elements. + + independent_comalloc can be used to speed up allocation in cases + where several structs or objects must always be allocated at the + same time. For example: + + struct Head { ... } + struct Foot { ... } + + void send_message(char* msg) { + int msglen = strlen(msg); + size_t sizes[3] = { sizeof(struct Head), msglen, sizeof(struct Foot) }; + void* chunks[3]; + if (independent_comalloc(3, sizes, chunks) == 0) + die(); + struct Head* head = (struct Head*)(chunks[0]); + char* body = (char*)(chunks[1]); + struct Foot* foot = (struct Foot*)(chunks[2]); + // ... + } + + In general though, independent_comalloc is worth using only for + larger values of n_elements. For small values, you probably won't + detect enough difference from series of malloc calls to bother. + + Overuse of independent_comalloc can increase overall memory usage, + since it cannot reuse existing noncontiguous small chunks that + might be available for some of the elements. +*/ +void** dlindependent_comalloc(size_t, size_t*, void**); + + +/* + pvalloc(size_t n); + Equivalent to valloc(minimum-page-that-holds(n)), that is, + round up n to nearest pagesize. + */ +void* dlpvalloc(size_t); + +/* + malloc_trim(size_t pad); + + If possible, gives memory back to the system (via negative arguments + to sbrk) if there is unused memory at the `high' end of the malloc + pool or in unused MMAP segments. You can call this after freeing + large blocks of memory to potentially reduce the system-level memory + requirements of a program. However, it cannot guarantee to reduce + memory. Under some allocation patterns, some large free blocks of + memory will be locked between two used chunks, so they cannot be + given back to the system. + + The `pad' argument to malloc_trim represents the amount of free + trailing space to leave untrimmed. If this argument is zero, only + the minimum amount of memory to maintain internal data structures + will be left. Non-zero arguments can be supplied to maintain enough + trailing space to service future expected allocations without having + to re-obtain memory from the system. + + Malloc_trim returns 1 if it actually released any memory, else 0. +*/ +int dlmalloc_trim(size_t); + +/* + malloc_stats(); + Prints on stderr the amount of space obtained from the system (both + via sbrk and mmap), the maximum amount (which may be more than + current if malloc_trim and/or munmap got called), and the current + number of bytes allocated via malloc (or realloc, etc) but not yet + freed. Note that this is the number of bytes allocated, not the + number requested. It will be larger than the number requested + because of alignment and bookkeeping overhead. Because it includes + alignment wastage as being in use, this figure may be greater than + zero even when no user-level chunks are allocated. + + The reported current and maximum system memory can be inaccurate if + a program makes other calls to system memory allocation functions + (normally sbrk) outside of malloc. + + malloc_stats prints only the most commonly interesting statistics. + More information can be obtained by calling mallinfo. +*/ +void dlmalloc_stats(void); + +#endif /* ONLY_MSPACES */ + +/* + malloc_usable_size(void* p); + + Returns the number of bytes you can actually use in + an allocated chunk, which may be more than you requested (although + often not) due to alignment and minimum size constraints. + You can use this many bytes without worrying about + overwriting other allocated objects. This is not a particularly great + programming practice. malloc_usable_size can be more useful in + debugging and assertions, for example: + + p = malloc(n); + assert(malloc_usable_size(p) >= 256); +*/ +size_t dlmalloc_usable_size(void*); + + +#if MSPACES + +/* + mspace is an opaque type representing an independent + region of space that supports mspace_malloc, etc. +*/ +typedef void* mspace; + +/* + create_mspace creates and returns a new independent space with the + given initial capacity, or, if 0, the default granularity size. It + returns null if there is no system memory available to create the + space. If argument locked is non-zero, the space uses a separate + lock to control access. The capacity of the space will grow + dynamically as needed to service mspace_malloc requests. You can + control the sizes of incremental increases of this space by + compiling with a different DEFAULT_GRANULARITY or dynamically + setting with mallopt(M_GRANULARITY, value). +*/ +mspace create_mspace(size_t capacity, int locked); + +/* + destroy_mspace destroys the given space, and attempts to return all + of its memory back to the system, returning the total number of + bytes freed. After destruction, the results of access to all memory + used by the space become undefined. +*/ +size_t destroy_mspace(mspace msp); + +/* + create_mspace_with_base uses the memory supplied as the initial base + of a new mspace. Part (less than 128*sizeof(size_t) bytes) of this + space is used for bookkeeping, so the capacity must be at least this + large. (Otherwise 0 is returned.) When this initial space is + exhausted, additional memory will be obtained from the system. + Destroying this space will deallocate all additionally allocated + space (if possible) but not the initial base. +*/ +mspace create_mspace_with_base(void* base, size_t capacity, int locked); + +/* + mspace_track_large_chunks controls whether requests for large chunks + are allocated in their own untracked mmapped regions, separate from + others in this mspace. By default large chunks are not tracked, + which reduces fragmentation. However, such chunks are not + necessarily released to the system upon destroy_mspace. Enabling + tracking by setting to true may increase fragmentation, but avoids + leakage when relying on destroy_mspace to release all memory + allocated using this space. The function returns the previous + setting. +*/ +int mspace_track_large_chunks(mspace msp, int enable); + + +/* + mspace_malloc behaves as malloc, but operates within + the given space. +*/ +void* mspace_malloc(mspace msp, size_t bytes); + +/* + mspace_free behaves as free, but operates within + the given space. + + If compiled with FOOTERS==1, mspace_free is not actually needed. + free may be called instead of mspace_free because freed chunks from + any space are handled by their originating spaces. +*/ +void mspace_free(mspace msp, void* mem); + +/* + mspace_realloc behaves as realloc, but operates within + the given space. + + If compiled with FOOTERS==1, mspace_realloc is not actually + needed. realloc may be called instead of mspace_realloc because + realloced chunks from any space are handled by their originating + spaces. +*/ +void* mspace_realloc(mspace msp, void* mem, size_t newsize); + +/* + mspace_calloc behaves as calloc, but operates within + the given space. +*/ +void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size); + +/* + mspace_memalign behaves as memalign, but operates within + the given space. +*/ +void* mspace_memalign(mspace msp, size_t alignment, size_t bytes); + +/* + mspace_independent_calloc behaves as independent_calloc, but + operates within the given space. +*/ +void** mspace_independent_calloc(mspace msp, size_t n_elements, + size_t elem_size, void* chunks[]); + +/* + mspace_independent_comalloc behaves as independent_comalloc, but + operates within the given space. +*/ +void** mspace_independent_comalloc(mspace msp, size_t n_elements, + size_t sizes[], void* chunks[]); + +/* + mspace_footprint() returns the number of bytes obtained from the + system for this space. +*/ +size_t mspace_footprint(mspace msp); + +/* + mspace_max_footprint() returns the peak number of bytes obtained from the + system for this space. +*/ +size_t mspace_max_footprint(mspace msp); + + +#if !NO_MALLINFO +/* + mspace_mallinfo behaves as mallinfo, but reports properties of + the given space. +*/ +struct mallinfo mspace_mallinfo(mspace msp); +#endif /* NO_MALLINFO */ + +/* + malloc_usable_size(void* p) behaves the same as malloc_usable_size; +*/ + size_t mspace_usable_size(void* mem); + +/* + mspace_malloc_stats behaves as malloc_stats, but reports + properties of the given space. +*/ +void mspace_malloc_stats(mspace msp); + +/* + mspace_trim behaves as malloc_trim, but + operates within the given space. +*/ +int mspace_trim(mspace msp, size_t pad); + +/* + An alias for mallopt. +*/ +int mspace_mallopt(int, int); + +#endif /* MSPACES */ + +#ifdef __cplusplus +} /* end of extern "C" */ +#endif /* __cplusplus */ + +/* + ======================================================================== + To make a fully customizable malloc.h header file, cut everything + above this line, put into file malloc.h, edit to suit, and #include it + on the next line, as well as in programs that use this malloc. + ======================================================================== +*/ + +/* #include "malloc.h" */ + +/*------------------------------ internal #includes ---------------------- */ + +#ifdef WIN32 +#pragma warning( disable : 4146 ) /* no "unsigned" warnings */ +#endif /* WIN32 */ + +#include <stdio.h> /* for printing in malloc_stats */ + +#ifndef LACKS_ERRNO_H +#include <errno.h> /* for MALLOC_FAILURE_ACTION */ +#endif /* LACKS_ERRNO_H */ +#if FOOTERS || DEBUG +#include <time.h> /* for magic initialization */ +#endif /* FOOTERS */ +#ifndef LACKS_STDLIB_H +#include <stdlib.h> /* for abort() */ +#endif /* LACKS_STDLIB_H */ +#ifdef DEBUG +#if ABORT_ON_ASSERT_FAILURE +#undef assert +#define assert(x) if(!(x)) ABORT +#else /* ABORT_ON_ASSERT_FAILURE */ +#include <assert.h> +#endif /* ABORT_ON_ASSERT_FAILURE */ +#else /* DEBUG */ +#ifndef assert +#define assert(x) +#endif +#define DEBUG 0 +#endif /* DEBUG */ +#ifndef LACKS_STRING_H +#include <string.h> /* for memset etc */ +#endif /* LACKS_STRING_H */ +#if USE_BUILTIN_FFS +#ifndef LACKS_STRINGS_H +#include <strings.h> /* for ffs */ +#endif /* LACKS_STRINGS_H */ +#endif /* USE_BUILTIN_FFS */ +#if HAVE_MMAP +#ifndef LACKS_SYS_MMAN_H +/* On some versions of linux, mremap decl in mman.h needs __USE_GNU set */ +#if (defined(linux) && !defined(__USE_GNU)) +#define __USE_GNU 1 +#include <sys/mman.h> /* for mmap */ +#undef __USE_GNU +#else +#include <sys/mman.h> /* for mmap */ +#endif /* linux */ +#endif /* LACKS_SYS_MMAN_H */ +#ifndef LACKS_FCNTL_H +#include <fcntl.h> +#endif /* LACKS_FCNTL_H */ +#endif /* HAVE_MMAP */ +#ifndef LACKS_UNISTD_H +#include <unistd.h> /* for sbrk, sysconf */ +#else /* LACKS_UNISTD_H */ +#if !defined(__FreeBSD__) && !defined(__OpenBSD__) && !defined(__NetBSD__) +extern void* sbrk(ptrdiff_t); +#endif /* FreeBSD etc */ +#endif /* LACKS_UNISTD_H */ + +/* Declarations for locking */ +#if USE_LOCKS +#ifndef WIN32 +#include <pthread.h> +#if defined (__SVR4) && defined (__sun) /* solaris */ +#include <thread.h> +#endif /* solaris */ +#else +#ifndef _M_AMD64 +/* These are already defined on AMD64 builds */ +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ +LONG __cdecl _InterlockedCompareExchange(LONG volatile *Dest, LONG Exchange, LONG Comp); +LONG __cdecl _InterlockedExchange(LONG volatile *Target, LONG Value); +#ifdef __cplusplus +} +#endif /* __cplusplus */ +#endif /* _M_AMD64 */ +#pragma intrinsic (_InterlockedCompareExchange) +#pragma intrinsic (_InterlockedExchange) +#define interlockedcompareexchange _InterlockedCompareExchange +#define interlockedexchange _InterlockedExchange +#endif /* Win32 */ +#endif /* USE_LOCKS */ + +/* Declarations for bit scanning on win32 */ +#if defined(_MSC_VER) && _MSC_VER>=1300 +#ifndef BitScanForward /* Try to avoid pulling in WinNT.h */ +#ifdef __cplusplus +extern "C" { +#endif /* __cplusplus */ +unsigned char _BitScanForward(unsigned long *index, unsigned long mask); +unsigned char _BitScanReverse(unsigned long *index, unsigned long mask); +#ifdef __cplusplus +} +#endif /* __cplusplus */ + +#define BitScanForward _BitScanForward +#define BitScanReverse _BitScanReverse +#pragma intrinsic(_BitScanForward) +#pragma intrinsic(_BitScanReverse) +#endif /* BitScanForward */ +#endif /* defined(_MSC_VER) && _MSC_VER>=1300 */ + +#ifndef WIN32 +#ifndef malloc_getpagesize +# ifdef _SC_PAGESIZE /* some SVR4 systems omit an underscore */ +# ifndef _SC_PAGE_SIZE +# define _SC_PAGE_SIZE _SC_PAGESIZE +# endif +# endif +# ifdef _SC_PAGE_SIZE +# define malloc_getpagesize sysconf(_SC_PAGE_SIZE) +# else +# if defined(BSD) || defined(DGUX) || defined(HAVE_GETPAGESIZE) + extern size_t getpagesize(); +# define malloc_getpagesize getpagesize() +# else +# ifdef WIN32 /* use supplied emulation of getpagesize */ +# define malloc_getpagesize getpagesize() +# else +# ifndef LACKS_SYS_PARAM_H +# include <sys/param.h> +# endif +# ifdef EXEC_PAGESIZE +# define malloc_getpagesize EXEC_PAGESIZE +# else +# ifdef NBPG +# ifndef CLSIZE +# define malloc_getpagesize NBPG +# else +# define malloc_getpagesize (NBPG * CLSIZE) +# endif +# else +# ifdef NBPC +# define malloc_getpagesize NBPC +# else +# ifdef PAGESIZE +# define malloc_getpagesize PAGESIZE +# else /* just guess */ +# define malloc_getpagesize ((size_t)4096U) +# endif +# endif +# endif +# endif +# endif +# endif +# endif +#endif +#endif + + + +/* ------------------- size_t and alignment properties -------------------- */ + +/* The byte and bit size of a size_t */ +#define SIZE_T_SIZE (sizeof(size_t)) +#define SIZE_T_BITSIZE (sizeof(size_t) << 3) + +/* Some constants coerced to size_t */ +/* Annoying but necessary to avoid errors on some platforms */ +#define SIZE_T_ZERO ((size_t)0) +#define SIZE_T_ONE ((size_t)1) +#define SIZE_T_TWO ((size_t)2) +#define SIZE_T_FOUR ((size_t)4) +#define TWO_SIZE_T_SIZES (SIZE_T_SIZE<<1) +#define FOUR_SIZE_T_SIZES (SIZE_T_SIZE<<2) +#define SIX_SIZE_T_SIZES (FOUR_SIZE_T_SIZES+TWO_SIZE_T_SIZES) +#define HALF_MAX_SIZE_T (MAX_SIZE_T / 2U) + +/* The bit mask value corresponding to MALLOC_ALIGNMENT */ +#define CHUNK_ALIGN_MASK (MALLOC_ALIGNMENT - SIZE_T_ONE) + +/* True if address a has acceptable alignment */ +#define is_aligned(A) (((size_t)((A)) & (CHUNK_ALIGN_MASK)) == 0) + +/* the number of bytes to offset an address to align it */ +#define align_offset(A)\ + ((((size_t)(A) & CHUNK_ALIGN_MASK) == 0)? 0 :\ + ((MALLOC_ALIGNMENT - ((size_t)(A) & CHUNK_ALIGN_MASK)) & CHUNK_ALIGN_MASK)) + +/* + malloc_params holds global properties, including those that can be + dynamically set using mallopt. There is a single instance, mparams, + initialized in init_mparams. Note that the non-zeroness of "magic" + also serves as an initialization flag. +*/ +typedef unsigned int flag_t; +struct malloc_params { + volatile size_t magic; + size_t page_size; + size_t granularity; + size_t mmap_threshold; + size_t trim_threshold; + flag_t default_mflags; +}; + +static struct malloc_params mparams; + +/* Ensure mparams initialized */ +#define ensure_initialization() (void)(mparams.magic != 0 || init_mparams()) + +/* -------------------------- MMAP preliminaries ------------------------- */ + +/* + If HAVE_MORECORE or HAVE_MMAP are false, we just define calls and + checks to fail so compiler optimizer can delete code rather than + using so many "#if"s. +*/ + + +/* MORECORE and MMAP must return MFAIL on failure */ +#define MFAIL ((void*)(MAX_SIZE_T)) +#define CMFAIL ((char*)(MFAIL)) /* defined for convenience */ + +#if HAVE_MMAP + +#ifndef WIN32 +#if !defined(MAP_ANONYMOUS) && defined(MAP_ANON) +#define MAP_ANONYMOUS MAP_ANON +#endif /* MAP_ANON */ +#ifdef DEFAULT_GRANULARITY_ALIGNED +#define MMAP_IMPL mmap_aligned +static void* lastAlignedmmap; /* Used as a hint */ +static void* mmap_aligned(void *start, size_t length, int prot, int flags, int fd, off_t offset) { + void* baseaddress = 0; + void* ptr = 0; + if(!start) { + baseaddress = lastAlignedmmap; + for(;;) { + if(baseaddress) flags|=MAP_FIXED; + ptr = mmap(baseaddress, length, prot, flags, fd, offset); + if(!ptr) + baseaddress = (void*)((size_t)baseaddress + mparams.granularity); + else if((size_t)ptr & (mparams.granularity - SIZE_T_ONE)) { + munmap(ptr, length); + baseaddress = (void*)(((size_t)ptr + mparams.granularity) & ~(mparams.granularity - SIZE_T_ONE)); + } + else break; + } + } + else ptr = mmap(start, length, prot, flags, fd, offset); + if(ptr) lastAlignedmmap = (void*)((size_t) ptr + mparams.granularity); + return ptr; +} +#else +#define MMAP_IMPL mmap +#endif /* DEFAULT_GRANULARITY_ALIGNED */ +#define MUNMAP_DEFAULT(a, s) munmap((a), (s)) +#define MMAP_PROT (PROT_READ|PROT_WRITE) +#ifdef MAP_ANONYMOUS +#define MMAP_FLAGS (MAP_PRIVATE|MAP_ANONYMOUS) +#define MMAP_DEFAULT(s) MMAP_IMPL(0, (s), MMAP_PROT, MMAP_FLAGS, -1, 0) +#else /* MAP_ANONYMOUS */ +/* + Nearly all versions of mmap support MAP_ANONYMOUS, so the following + is unlikely to be needed, but is supplied just in case. +*/ +#define MMAP_FLAGS (MAP_PRIVATE) +static int dev_zero_fd = -1; /* Cached file descriptor for /dev/zero. */ +#define MMAP_DEFAULT(s) ((dev_zero_fd < 0) ? \ + (dev_zero_fd = open("/dev/zero", O_RDWR), \ + MMAP_IMPL(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) : \ + MMAP_IMPL(0, (s), MMAP_PROT, MMAP_FLAGS, dev_zero_fd, 0)) +#endif /* MAP_ANONYMOUS */ + +#define DIRECT_MMAP_DEFAULT(s) MMAP_DEFAULT(s) + +#else /* WIN32 */ + +/* Win32 MMAP via VirtualAlloc */ +#ifdef DEFAULT_GRANULARITY_ALIGNED +static void* lastWin32mmap; /* Used as a hint */ +#endif /* DEFAULT_GRANULARITY_ALIGNED */ +#ifdef ENABLE_LARGE_PAGES +static int largepagesavailable = 1; +#endif /* ENABLE_LARGE_PAGES */ +static FORCEINLINE void* win32mmap(size_t size) { + void* baseaddress = 0; + void* ptr = 0; +#ifdef ENABLE_LARGE_PAGES + /* Note that large pages are *always* allocated on a large page boundary. + If however granularity is small then don't waste a kernel call if size + isn't around the size of a large page */ + if(largepagesavailable && size >= 1*1024*1024) { + ptr = VirtualAlloc(baseaddress, size, MEM_RESERVE|MEM_COMMIT|MEM_LARGE_PAGES, PAGE_READWRITE); + if(!ptr && ERROR_PRIVILEGE_NOT_HELD==GetLastError()) largepagesavailable=0; + } +#endif + if(!ptr) { +#ifdef DEFAULT_GRANULARITY_ALIGNED + /* We try to avoid overhead by speculatively reserving at aligned + addresses until we succeed */ + baseaddress = lastWin32mmap; + for(;;) { + void* reserveaddr = VirtualAlloc(baseaddress, size, MEM_RESERVE, PAGE_READWRITE); + if(!reserveaddr) + baseaddress = (void*)((size_t)baseaddress + mparams.granularity); + else if((size_t)reserveaddr & (mparams.granularity - SIZE_T_ONE)) { + VirtualFree(reserveaddr, 0, MEM_RELEASE); + baseaddress = (void*)(((size_t)reserveaddr + mparams.granularity) & ~(mparams.granularity - SIZE_T_ONE)); + } + else break; + } +#endif + if(!ptr) ptr = VirtualAlloc(baseaddress, size, baseaddress ? MEM_COMMIT : MEM_RESERVE|MEM_COMMIT, PAGE_READWRITE); +#if DEBUG + if(lastWin32mmap && ptr!=lastWin32mmap) printf("Non-contiguous VirtualAlloc between %p and %p\n", ptr, lastWin32mmap); +#endif +#ifdef DEFAULT_GRANULARITY_ALIGNED + if(ptr) lastWin32mmap = (void*)((size_t) ptr + mparams.granularity); +#endif + } +#if DEBUG +#ifdef ENABLE_LARGE_PAGES + printf("VirtualAlloc returns %p size %u. LargePagesAvailable=%d\n", ptr, size, largepagesavailable); +#else + printf("VirtualAlloc returns %p size %u\n", ptr, size); +#endif +#endif + return (ptr != 0)? ptr: MFAIL; +} + +/* For direct MMAP, use MEM_TOP_DOWN to minimize interference */ +static FORCEINLINE void* win32direct_mmap(size_t size) { + void* ptr = VirtualAlloc(0, size, MEM_RESERVE|MEM_COMMIT|MEM_TOP_DOWN, + PAGE_READWRITE); + return (ptr != 0)? ptr: MFAIL; +} + +/* This function supports releasing coalesed segments */ +static FORCEINLINE int win32munmap(void* ptr, size_t size) { + MEMORY_BASIC_INFORMATION minfo; + char* cptr = (char*)ptr; + while (size) { + if (VirtualQuery(cptr, &minfo, sizeof(minfo)) == 0) + return -1; + if (minfo.BaseAddress != cptr || minfo.AllocationBase != cptr || + minfo.State != MEM_COMMIT || minfo.RegionSize > size) + return -1; + if (VirtualFree(cptr, 0, MEM_RELEASE) == 0) + return -1; + cptr += minfo.RegionSize; + size -= minfo.RegionSize; + } + return 0; +} + +#define MMAP_DEFAULT(s) win32mmap(s) +#define MUNMAP_DEFAULT(a, s) win32munmap((a), (s)) +#define DIRECT_MMAP_DEFAULT(s) win32direct_mmap(s) +#endif /* WIN32 */ +#endif /* HAVE_MMAP */ + +#if HAVE_MREMAP +#ifndef WIN32 +#define MREMAP_DEFAULT(addr, osz, nsz, mv) mremap((addr), (osz), (nsz), (mv)) +#endif /* WIN32 */ +#endif /* HAVE_MREMAP */ + + +/** + * Define CALL_MORECORE + */ +#if HAVE_MORECORE + #ifdef MORECORE + #define CALL_MORECORE(S) MORECORE(S) + #else /* MORECORE */ + #define CALL_MORECORE(S) MORECORE_DEFAULT(S) + #endif /* MORECORE */ +#else /* HAVE_MORECORE */ + #define CALL_MORECORE(S) MFAIL +#endif /* HAVE_MORECORE */ + +/** + * Define CALL_MMAP/CALL_MUNMAP/CALL_DIRECT_MMAP + */ +#if HAVE_MMAP + #define USE_MMAP_BIT (SIZE_T_ONE) + + #ifdef MMAP + #define CALL_MMAP(s) MMAP(s) + #else /* MMAP */ + #define CALL_MMAP(s) MMAP_DEFAULT(s) + #endif /* MMAP */ + #ifdef MUNMAP + #define CALL_MUNMAP(a, s) MUNMAP((a), (s)) + #else /* MUNMAP */ + #define CALL_MUNMAP(a, s) MUNMAP_DEFAULT((a), (s)) + #endif /* MUNMAP */ + #ifdef DIRECT_MMAP + #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s) + #else /* DIRECT_MMAP */ + #define CALL_DIRECT_MMAP(s) DIRECT_MMAP_DEFAULT(s) + #endif /* DIRECT_MMAP */ +#else /* HAVE_MMAP */ + #define USE_MMAP_BIT (SIZE_T_ZERO) + + #define MMAP(s) MFAIL + #define MUNMAP(a, s) (-1) + #define DIRECT_MMAP(s) MFAIL + #define CALL_DIRECT_MMAP(s) DIRECT_MMAP(s) + #define CALL_MMAP(s) MMAP(s) + #define CALL_MUNMAP(a, s) MUNMAP((a), (s)) +#endif /* HAVE_MMAP */ + +/** + * Define CALL_MREMAP + */ +#if HAVE_MMAP && HAVE_MREMAP + #ifdef MREMAP + #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP((addr), (osz), (nsz), (mv)) + #else /* MREMAP */ + #define CALL_MREMAP(addr, osz, nsz, mv) MREMAP_DEFAULT((addr), (osz), (nsz), (mv)) + #endif /* MREMAP */ +#else /* HAVE_MMAP && HAVE_MREMAP */ + #define CALL_MREMAP(addr, osz, nsz, mv) MFAIL +#endif /* HAVE_MMAP && HAVE_MREMAP */ + +/* mstate bit set if continguous morecore disabled or failed */ +#define USE_NONCONTIGUOUS_BIT (4U) + +/* segment bit set in create_mspace_with_base */ +#define EXTERN_BIT (8U) + + +/* --------------------------- Lock preliminaries ------------------------ */ + +/* + When locks are defined, there is one global lock, plus + one per-mspace lock. + + The global lock_ensures that mparams.magic and other unique + mparams values are initialized only once. It also protects + sequences of calls to MORECORE. In many cases sys_alloc requires + two calls, that should not be interleaved with calls by other + threads. This does not protect against direct calls to MORECORE + by other threads not using this lock, so there is still code to + cope the best we can on interference. + + Per-mspace locks surround calls to malloc, free, etc. To enable use + in layered extensions, per-mspace locks are reentrant. + + Because lock-protected regions generally have bounded times, it is + OK to use the supplied simple spinlocks in the custom versions for + x86. Spinlocks are likely to improve performance for lightly + contended applications, but worsen performance under heavy + contention. + + If USE_LOCKS is > 1, the definitions of lock routines here are + bypassed, in which case you will need to define the type MLOCK_T, + and at least INITIAL_LOCK, ACQUIRE_LOCK, RELEASE_LOCK and possibly + TRY_LOCK (which is not used in this malloc, but commonly needed in + extensions.) You must also declare a + static MLOCK_T malloc_global_mutex = { initialization values };. + +*/ + +#if USE_LOCKS == 1 + +#if USE_SPIN_LOCKS && SPIN_LOCKS_AVAILABLE +#ifndef WIN32 + +/* Custom pthread-style spin locks on x86 and x64 for gcc */ +struct pthread_mlock_t { + volatile unsigned int l; + char cachelinepadding[64]; + unsigned int c; + pthread_t threadid; +}; +#define MLOCK_T struct pthread_mlock_t +#define CURRENT_THREAD pthread_self() +#define INITIAL_LOCK(sl) ((sl)->threadid = 0, (sl)->l = (sl)->c = 0, 0) +#define ACQUIRE_LOCK(sl) pthread_acquire_lock(sl) +#define RELEASE_LOCK(sl) pthread_release_lock(sl) +#define TRY_LOCK(sl) pthread_try_lock(sl) +#define SPINS_PER_YIELD 63 + +static MLOCK_T malloc_global_mutex = { 0, "", 0, 0}; + +static FORCEINLINE int pthread_acquire_lock (MLOCK_T *sl) { + int spins = 0; + volatile unsigned int* lp = &sl->l; + for (;;) { + if (*lp != 0) { + if (sl->threadid == CURRENT_THREAD) { + ++sl->c; + return 0; + } + } + else { + /* place args to cmpxchgl in locals to evade oddities in some gccs */ + int cmp = 0; + int val = 1; + int ret; + __asm__ __volatile__ ("lock; cmpxchgl %1, %2" + : "=a" (ret) + : "r" (val), "m" (*(lp)), "0"(cmp) + : "memory", "cc"); + if (!ret) { + assert(!sl->threadid); + sl->threadid = CURRENT_THREAD; + sl->c = 1; + return 0; + } + } + if ((++spins & SPINS_PER_YIELD) == 0) { +#if defined (__SVR4) && defined (__sun) /* solaris */ + thr_yield(); +#else +#if defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) + sched_yield(); +#else /* no-op yield on unknown systems */ + ; +#endif /* __linux__ || __FreeBSD__ || __APPLE__ */ +#endif /* solaris */ + } + } +} + +static FORCEINLINE void pthread_release_lock (MLOCK_T *sl) { + volatile unsigned int* lp = &sl->l; + assert(*lp != 0); + assert(sl->threadid == CURRENT_THREAD); + if (--sl->c == 0) { + sl->threadid = 0; + int prev = 0; + int ret; + __asm__ __volatile__ ("lock; xchgl %0, %1" + : "=r" (ret) + : "m" (*(lp)), "0"(prev) + : "memory"); + } +} + +static FORCEINLINE int pthread_try_lock (MLOCK_T *sl) { + volatile unsigned int* lp = &sl->l; + if (*lp != 0) { + if (sl->threadid == CURRENT_THREAD) { + ++sl->c; + return 1; + } + } + else { + int cmp = 0; + int val = 1; + int ret; + __asm__ __volatile__ ("lock; cmpxchgl %1, %2" + : "=a" (ret) + : "r" (val), "m" (*(lp)), "0"(cmp) + : "memory", "cc"); + if (!ret) { + assert(!sl->threadid); + sl->threadid = CURRENT_THREAD; + sl->c = 1; + return 1; + } + } + return 0; +} + + +#else /* WIN32 */ +/* Custom win32-style spin locks on x86 and x64 for MSC */ +struct win32_mlock_t { + volatile long l; + char cachelinepadding[64]; + unsigned int c; + long threadid; +}; + +#define MLOCK_T struct win32_mlock_t +#define CURRENT_THREAD ((long)GetCurrentThreadId()) +#define INITIAL_LOCK(sl) ((sl)->threadid = 0, (sl)->l = (sl)->c = 0, 0) +#define ACQUIRE_LOCK(sl) win32_acquire_lock(sl) +#define RELEASE_LOCK(sl) win32_release_lock(sl) +#define TRY_LOCK(sl) win32_try_lock(sl) +#define SPINS_PER_YIELD 63 + +static MLOCK_T malloc_global_mutex = { 0, 0, 0}; + +static FORCEINLINE int win32_acquire_lock (MLOCK_T *sl) { + int spins = 0; + for (;;) { + if (sl->l != 0) { + if (sl->threadid == CURRENT_THREAD) { + ++sl->c; + return 0; + } + } + else { + if (!interlockedexchange(&sl->l, 1)) { + assert(!sl->threadid); + sl->threadid = CURRENT_THREAD; + sl->c = 1; + return 0; + } + } + if ((++spins & SPINS_PER_YIELD) == 0) + SleepEx(0, FALSE); + } +} + +static FORCEINLINE void win32_release_lock (MLOCK_T *sl) { + assert(sl->threadid == CURRENT_THREAD); + assert(sl->l != 0); + if (--sl->c == 0) { + sl->threadid = 0; + interlockedexchange (&sl->l, 0); + } +} + +static FORCEINLINE int win32_try_lock (MLOCK_T *sl) { + if (sl->l != 0) { + if (sl->threadid == CURRENT_THREAD) { + ++sl->c; + return 1; + } + } + else { + if (!interlockedexchange(&sl->l, 1)){ + assert(!sl->threadid); + sl->threadid = CURRENT_THREAD; + sl->c = 1; + return 1; + } + } + return 0; +} + +#endif /* WIN32 */ +#else /* USE_SPIN_LOCKS */ + +#ifndef WIN32 +/* pthreads-based locks */ + +#define MLOCK_T pthread_mutex_t +#define CURRENT_THREAD pthread_self() +#define INITIAL_LOCK(sl) pthread_init_lock(sl) +#define ACQUIRE_LOCK(sl) pthread_mutex_lock(sl) +#define RELEASE_LOCK(sl) pthread_mutex_unlock(sl) +#define TRY_LOCK(sl) (!pthread_mutex_trylock(sl)) + +static MLOCK_T malloc_global_mutex = PTHREAD_MUTEX_INITIALIZER; + +/* Cope with old-style linux recursive lock initialization by adding */ +/* skipped internal declaration from pthread.h */ +#ifdef linux +#ifndef PTHREAD_MUTEX_RECURSIVE +extern int pthread_mutexattr_setkind_np __P ((pthread_mutexattr_t *__attr, + int __kind)); +#define PTHREAD_MUTEX_RECURSIVE PTHREAD_MUTEX_RECURSIVE_NP +#define pthread_mutexattr_settype(x,y) pthread_mutexattr_setkind_np(x,y) +#endif +#endif + +static int pthread_init_lock (MLOCK_T *sl) { + pthread_mutexattr_t attr; + if (pthread_mutexattr_init(&attr)) return 1; + if (pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_RECURSIVE)) return 1; + if (pthread_mutex_init(sl, &attr)) return 1; + if (pthread_mutexattr_destroy(&attr)) return 1; + return 0; +} + +#else /* WIN32 */ +/* Win32 critical sections */ +#define MLOCK_T CRITICAL_SECTION +#define CURRENT_THREAD GetCurrentThreadId() +#define INITIAL_LOCK(s) (!InitializeCriticalSectionAndSpinCount((s), 0x80000000|4000)) +#define ACQUIRE_LOCK(s) (EnterCriticalSection(sl), 0) +#define RELEASE_LOCK(s) LeaveCriticalSection(sl) +#define TRY_LOCK(s) TryEnterCriticalSection(sl) +#define NEED_GLOBAL_LOCK_INIT + +static MLOCK_T malloc_global_mutex; +static volatile long malloc_global_mutex_status; + +/* Use spin loop to initialize global lock */ +static void init_malloc_global_mutex() { + for (;;) { + long stat = malloc_global_mutex_status; + if (stat > 0) + return; + /* transition to < 0 while initializing, then to > 0) */ + if (stat == 0 && + interlockedcompareexchange(&malloc_global_mutex_status, -1, 0) == 0) { + InitializeCriticalSection(&malloc_global_mutex); + interlockedexchange(&malloc_global_mutex_status,1); + return; + } + SleepEx(0, FALSE); + } +} + +#endif /* WIN32 */ +#endif /* USE_SPIN_LOCKS */ +#endif /* USE_LOCKS == 1 */ + +/* ----------------------- User-defined locks ------------------------ */ + +#if USE_LOCKS > 1 +/* Define your own lock implementation here */ +/* #define INITIAL_LOCK(sl) ... */ +/* #define ACQUIRE_LOCK(sl) ... */ +/* #define RELEASE_LOCK(sl) ... */ +/* #define TRY_LOCK(sl) ... */ +/* static MLOCK_T malloc_global_mutex = ... */ +#endif /* USE_LOCKS > 1 */ + +/* ----------------------- Lock-based state ------------------------ */ + +#if USE_LOCKS +#define USE_LOCK_BIT (2U) +#else /* USE_LOCKS */ +#define USE_LOCK_BIT (0U) +#define INITIAL_LOCK(l) +#endif /* USE_LOCKS */ + +#if USE_LOCKS +#ifndef ACQUIRE_MALLOC_GLOBAL_LOCK +#define ACQUIRE_MALLOC_GLOBAL_LOCK() ACQUIRE_LOCK(&malloc_global_mutex); +#endif +#ifndef RELEASE_MALLOC_GLOBAL_LOCK +#define RELEASE_MALLOC_GLOBAL_LOCK() RELEASE_LOCK(&malloc_global_mutex); +#endif +#else /* USE_LOCKS */ +#define ACQUIRE_MALLOC_GLOBAL_LOCK() +#define RELEASE_MALLOC_GLOBAL_LOCK() +#endif /* USE_LOCKS */ + + +/* ----------------------- Chunk representations ------------------------ */ + +/* + (The following includes lightly edited explanations by Colin Plumb.) + + The malloc_chunk declaration below is misleading (but accurate and + necessary). It declares a "view" into memory allowing access to + necessary fields at known offsets from a given base. + + Chunks of memory are maintained using a `boundary tag' method as + originally described by Knuth. (See the paper by Paul Wilson + ftp://ftp.cs.utexas.edu/pub/garbage/allocsrv.ps for a survey of such + techniques.) Sizes of free chunks are stored both in the front of + each chunk and at the end. This makes consolidating fragmented + chunks into bigger chunks fast. The head fields also hold bits + representing whether chunks are free or in use. + + Here are some pictures to make it clearer. They are "exploded" to + show that the state of a chunk can be thought of as extending from + the high 31 bits of the head field of its header through the + prev_foot and PINUSE_BIT bit of the following chunk header. + + A chunk that's in use looks like: + + chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Size of previous chunk (if P = 0) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P| + | Size of this chunk 1| +-+ + mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | | + +- -+ + | | + +- -+ + | : + +- size - sizeof(size_t) available payload bytes -+ + : | + chunk-> +- -+ + | | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |1| + | Size of next chunk (may or may not be in use) | +-+ + mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + And if it's free, it looks like this: + + chunk-> +- -+ + | User payload (must be in use, or we would have merged!) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |P| + | Size of this chunk 0| +-+ + mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Next pointer | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Prev pointer | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | : + +- size - sizeof(struct chunk) unused bytes -+ + : | + chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Size of this chunk | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ |0| + | Size of next chunk (must be in use, or we would have merged)| +-+ + mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | : + +- User payload -+ + : | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |0| + +-+ + Note that since we always merge adjacent free chunks, the chunks + adjacent to a free chunk must be in use. + + Given a pointer to a chunk (which can be derived trivially from the + payload pointer) we can, in O(1) time, find out whether the adjacent + chunks are free, and if so, unlink them from the lists that they + are on and merge them with the current chunk. + + Chunks always begin on even word boundaries, so the mem portion + (which is returned to the user) is also on an even word boundary, and + thus at least double-word aligned. + + The P (PINUSE_BIT) bit, stored in the unused low-order bit of the + chunk size (which is always a multiple of two words), is an in-use + bit for the *previous* chunk. If that bit is *clear*, then the + word before the current chunk size contains the previous chunk + size, and can be used to find the front of the previous chunk. + The very first chunk allocated always has this bit set, preventing + access to non-existent (or non-owned) memory. If pinuse is set for + any given chunk, then you CANNOT determine the size of the + previous chunk, and might even get a memory addressing fault when + trying to do so. + + The C (CINUSE_BIT) bit, stored in the unused second-lowest bit of + the chunk size redundantly records whether the current chunk is + inuse (unless the chunk is mmapped). This redundancy enables usage + checks within free and realloc, and reduces indirection when freeing + and consolidating chunks. + + Each freshly allocated chunk must have both cinuse and pinuse set. + That is, each allocated chunk borders either a previously allocated + and still in-use chunk, or the base of its memory arena. This is + ensured by making all allocations from the the `lowest' part of any + found chunk. Further, no free chunk physically borders another one, + so each free chunk is known to be preceded and followed by either + inuse chunks or the ends of memory. + + Note that the `foot' of the current chunk is actually represented + as the prev_foot of the NEXT chunk. This makes it easier to + deal with alignments etc but can be very confusing when trying + to extend or adapt this code. + + The exceptions to all this are + + 1. The special chunk `top' is the top-most available chunk (i.e., + the one bordering the end of available memory). It is treated + specially. Top is never included in any bin, is used only if + no other chunk is available, and is released back to the + system if it is very large (see M_TRIM_THRESHOLD). In effect, + the top chunk is treated as larger (and thus less well + fitting) than any other available chunk. The top chunk + doesn't update its trailing size field since there is no next + contiguous chunk that would have to index off it. However, + space is still allocated for it (TOP_FOOT_SIZE) to enable + separation or merging when space is extended. + + 3. Chunks allocated via mmap, have both cinuse and pinuse bits + cleared in their head fields. Because they are allocated + one-by-one, each must carry its own prev_foot field, which is + also used to hold the offset this chunk has within its mmapped + region, which is needed to preserve alignment. Each mmapped + chunk is trailed by the first two fields of a fake next-chunk + for sake of usage checks. + +*/ + +struct malloc_chunk { + size_t prev_foot; /* Size of previous chunk (if free). */ + size_t head; /* Size and inuse bits. */ + struct malloc_chunk* fd; /* double links -- used only if free. */ + struct malloc_chunk* bk; +}; + +typedef struct malloc_chunk mchunk; +typedef struct malloc_chunk* mchunkptr; +typedef struct malloc_chunk* sbinptr; /* The type of bins of chunks */ +typedef unsigned int bindex_t; /* Described below */ +typedef unsigned int binmap_t; /* Described below */ + +/* ------------------- Chunks sizes and alignments ----------------------- */ + +#define MCHUNK_SIZE (sizeof(mchunk)) + +#if FOOTERS +#define CHUNK_OVERHEAD (TWO_SIZE_T_SIZES) +#else /* FOOTERS */ +#define CHUNK_OVERHEAD (SIZE_T_SIZE) +#endif /* FOOTERS */ + +/* MMapped chunks need a second word of overhead ... */ +#define MMAP_CHUNK_OVERHEAD (TWO_SIZE_T_SIZES) +/* ... and additional padding for fake next-chunk at foot */ +#define MMAP_FOOT_PAD (FOUR_SIZE_T_SIZES) + +/* The smallest size we can malloc is an aligned minimal chunk */ +#define MIN_CHUNK_SIZE\ + ((MCHUNK_SIZE + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK) + +/* conversion from malloc headers to user pointers, and back */ +#define chunk2mem(p) ((void*)((char*)(p) + TWO_SIZE_T_SIZES)) +#define mem2chunk(mem) ((mchunkptr)((char*)(mem) - TWO_SIZE_T_SIZES)) +/* chunk associated with aligned address A */ +#define align_as_chunk(A) (mchunkptr)((A) + align_offset(chunk2mem(A))) + +/* Bounds on request (not chunk) sizes. */ +#define MAX_REQUEST ((-MIN_CHUNK_SIZE) << 2) +#define MIN_REQUEST (MIN_CHUNK_SIZE - CHUNK_OVERHEAD - SIZE_T_ONE) + +/* pad request bytes into a usable size */ +#define pad_request(req) \ + (((req) + CHUNK_OVERHEAD + CHUNK_ALIGN_MASK) & ~CHUNK_ALIGN_MASK) + +/* pad request, checking for minimum (but not maximum) */ +#define request2size(req) \ + (((req) < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(req)) + + +/* ------------------ Operations on head and foot fields ----------------- */ + +/* + The head field of a chunk is or'ed with PINUSE_BIT when previous + adjacent chunk in use, and or'ed with CINUSE_BIT if this chunk is in + use, unless mmapped, in which case both bits are cleared. + + FLAG4_BIT is not used by this malloc, but might be useful in extensions. +*/ + +#define PINUSE_BIT (SIZE_T_ONE) +#define CINUSE_BIT (SIZE_T_TWO) +#define FLAG4_BIT (SIZE_T_FOUR) +#define INUSE_BITS (PINUSE_BIT|CINUSE_BIT) +#define FLAG_BITS (PINUSE_BIT|CINUSE_BIT|FLAG4_BIT) + +/* Head value for fenceposts */ +#define FENCEPOST_HEAD (INUSE_BITS|SIZE_T_SIZE) + +/* extraction of fields from head words */ +#define cinuse(p) ((p)->head & CINUSE_BIT) +#define pinuse(p) ((p)->head & PINUSE_BIT) +#define is_inuse(p) (((p)->head & INUSE_BITS) != PINUSE_BIT) +#define is_mmapped(p) (((p)->head & INUSE_BITS) == 0) + +#define chunksize(p) ((p)->head & ~(FLAG_BITS)) + +#define clear_pinuse(p) ((p)->head &= ~PINUSE_BIT) + +/* Treat space at ptr +/- offset as a chunk */ +#define chunk_plus_offset(p, s) ((mchunkptr)(((char*)(p)) + (s))) +#define chunk_minus_offset(p, s) ((mchunkptr)(((char*)(p)) - (s))) + +/* Ptr to next or previous physical malloc_chunk. */ +#define next_chunk(p) ((mchunkptr)( ((char*)(p)) + ((p)->head & ~FLAG_BITS))) +#define prev_chunk(p) ((mchunkptr)( ((char*)(p)) - ((p)->prev_foot) )) + +/* extract next chunk's pinuse bit */ +#define next_pinuse(p) ((next_chunk(p)->head) & PINUSE_BIT) + +/* Get/set size at footer */ +#define get_foot(p, s) (((mchunkptr)((char*)(p) + (s)))->prev_foot) +#define set_foot(p, s) (((mchunkptr)((char*)(p) + (s)))->prev_foot = (s)) + +/* Set size, pinuse bit, and foot */ +#define set_size_and_pinuse_of_free_chunk(p, s)\ + ((p)->head = (s|PINUSE_BIT), set_foot(p, s)) + +/* Set size, pinuse bit, foot, and clear next pinuse */ +#define set_free_with_pinuse(p, s, n)\ + (clear_pinuse(n), set_size_and_pinuse_of_free_chunk(p, s)) + +/* Get the internal overhead associated with chunk p */ +#define overhead_for(p)\ + (is_mmapped(p)? MMAP_CHUNK_OVERHEAD : CHUNK_OVERHEAD) + +/* Return true if malloced space is not necessarily cleared */ +#if MMAP_CLEARS +#define calloc_must_clear(p) (!is_mmapped(p)) +#else /* MMAP_CLEARS */ +#define calloc_must_clear(p) (1) +#endif /* MMAP_CLEARS */ + +/* ---------------------- Overlaid data structures ----------------------- */ + +/* + When chunks are not in use, they are treated as nodes of either + lists or trees. + + "Small" chunks are stored in circular doubly-linked lists, and look + like this: + + chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Size of previous chunk | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + `head:' | Size of chunk, in bytes |P| + mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Forward pointer to next chunk in list | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Back pointer to previous chunk in list | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Unused space (may be 0 bytes long) . + . . + . | +nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + `foot:' | Size of chunk, in bytes | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + Larger chunks are kept in a form of bitwise digital trees (aka + tries) keyed on chunksizes. Because malloc_tree_chunks are only for + free chunks greater than 256 bytes, their size doesn't impose any + constraints on user chunk sizes. Each node looks like: + + chunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Size of previous chunk | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + `head:' | Size of chunk, in bytes |P| + mem-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Forward pointer to next chunk of same size | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Back pointer to previous chunk of same size | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Pointer to left child (child[0]) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Pointer to right child (child[1]) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Pointer to parent | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | bin index of this chunk | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | Unused space . + . | +nextchunk-> +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + `foot:' | Size of chunk, in bytes | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + + Each tree holding treenodes is a tree of unique chunk sizes. Chunks + of the same size are arranged in a circularly-linked list, with only + the oldest chunk (the next to be used, in our FIFO ordering) + actually in the tree. (Tree members are distinguished by a non-null + parent pointer.) If a chunk with the same size an an existing node + is inserted, it is linked off the existing node using pointers that + work in the same way as fd/bk pointers of small chunks. + + Each tree contains a power of 2 sized range of chunk sizes (the + smallest is 0x100 <= x < 0x180), which is is divided in half at each + tree level, with the chunks in the smaller half of the range (0x100 + <= x < 0x140 for the top nose) in the left subtree and the larger + half (0x140 <= x < 0x180) in the right subtree. This is, of course, + done by inspecting individual bits. + + Using these rules, each node's left subtree contains all smaller + sizes than its right subtree. However, the node at the root of each + subtree has no particular ordering relationship to either. (The + dividing line between the subtree sizes is based on trie relation.) + If we remove the last chunk of a given size from the interior of the + tree, we need to replace it with a leaf node. The tree ordering + rules permit a node to be replaced by any leaf below it. + + The smallest chunk in a tree (a common operation in a best-fit + allocator) can be found by walking a path to the leftmost leaf in + the tree. Unlike a usual binary tree, where we follow left child + pointers until we reach a null, here we follow the right child + pointer any time the left one is null, until we reach a leaf with + both child pointers null. The smallest chunk in the tree will be + somewhere along that path. + + The worst case number of steps to add, find, or remove a node is + bounded by the number of bits differentiating chunks within + bins. Under current bin calculations, this ranges from 6 up to 21 + (for 32 bit sizes) or up to 53 (for 64 bit sizes). The typical case + is of course much better. +*/ + +struct malloc_tree_chunk { + /* The first four fields must be compatible with malloc_chunk */ + size_t prev_foot; + size_t head; + struct malloc_tree_chunk* fd; + struct malloc_tree_chunk* bk; + + struct malloc_tree_chunk* child[2]; + struct malloc_tree_chunk* parent; + bindex_t index; +}; + +typedef struct malloc_tree_chunk tchunk; +typedef struct malloc_tree_chunk* tchunkptr; +typedef struct malloc_tree_chunk* tbinptr; /* The type of bins of trees */ + +/* A little helper macro for trees */ +#define leftmost_child(t) ((t)->child[0] != 0? (t)->child[0] : (t)->child[1]) + +/* ----------------------------- Segments -------------------------------- */ + +/* + Each malloc space may include non-contiguous segments, held in a + list headed by an embedded malloc_segment record representing the + top-most space. Segments also include flags holding properties of + the space. Large chunks that are directly allocated by mmap are not + included in this list. They are instead independently created and + destroyed without otherwise keeping track of them. + + Segment management mainly comes into play for spaces allocated by + MMAP. Any call to MMAP might or might not return memory that is + adjacent to an existing segment. MORECORE normally contiguously + extends the current space, so this space is almost always adjacent, + which is simpler and faster to deal with. (This is why MORECORE is + used preferentially to MMAP when both are available -- see + sys_alloc.) When allocating using MMAP, we don't use any of the + hinting mechanisms (inconsistently) supported in various + implementations of unix mmap, or distinguish reserving from + committing memory. Instead, we just ask for space, and exploit + contiguity when we get it. It is probably possible to do + better than this on some systems, but no general scheme seems + to be significantly better. + + Management entails a simpler variant of the consolidation scheme + used for chunks to reduce fragmentation -- new adjacent memory is + normally prepended or appended to an existing segment. However, + there are limitations compared to chunk consolidation that mostly + reflect the fact that segment processing is relatively infrequent + (occurring only when getting memory from system) and that we + don't expect to have huge numbers of segments: + + * Segments are not indexed, so traversal requires linear scans. (It + would be possible to index these, but is not worth the extra + overhead and complexity for most programs on most platforms.) + * New segments are only appended to old ones when holding top-most + memory; if they cannot be prepended to others, they are held in + different segments. + + Except for the top-most segment of an mstate, each segment record + is kept at the tail of its segment. Segments are added by pushing + segment records onto the list headed by &mstate.seg for the + containing mstate. + + Segment flags control allocation/merge/deallocation policies: + * If EXTERN_BIT set, then we did not allocate this segment, + and so should not try to deallocate or merge with others. + (This currently holds only for the initial segment passed + into create_mspace_with_base.) + * If USE_MMAP_BIT set, the segment may be merged with + other surrounding mmapped segments and trimmed/de-allocated + using munmap. + * If neither bit is set, then the segment was obtained using + MORECORE so can be merged with surrounding MORECORE'd segments + and deallocated/trimmed using MORECORE with negative arguments. +*/ + +struct malloc_segment { + char* base; /* base address */ + size_t size; /* allocated size */ + struct malloc_segment* next; /* ptr to next segment */ + flag_t sflags; /* mmap and extern flag */ +}; + +#define is_mmapped_segment(S) ((S)->sflags & USE_MMAP_BIT) +#define is_extern_segment(S) ((S)->sflags & EXTERN_BIT) + +typedef struct malloc_segment msegment; +typedef struct malloc_segment* msegmentptr; + +/* ---------------------------- malloc_state ----------------------------- */ + +/* + A malloc_state holds all of the bookkeeping for a space. + The main fields are: + + Top + The topmost chunk of the currently active segment. Its size is + cached in topsize. The actual size of topmost space is + topsize+TOP_FOOT_SIZE, which includes space reserved for adding + fenceposts and segment records if necessary when getting more + space from the system. The size at which to autotrim top is + cached from mparams in trim_check, except that it is disabled if + an autotrim fails. + + Designated victim (dv) + This is the preferred chunk for servicing small requests that + don't have exact fits. It is normally the chunk split off most + recently to service another small request. Its size is cached in + dvsize. The link fields of this chunk are not maintained since it + is not kept in a bin. + + SmallBins + An array of bin headers for free chunks. These bins hold chunks + with sizes less than MIN_LARGE_SIZE bytes. Each bin contains + chunks of all the same size, spaced 8 bytes apart. To simplify + use in double-linked lists, each bin header acts as a malloc_chunk + pointing to the real first node, if it exists (else pointing to + itself). This avoids special-casing for headers. But to avoid + waste, we allocate only the fd/bk pointers of bins, and then use + repositioning tricks to treat these as the fields of a chunk. + + TreeBins + Treebins are pointers to the roots of trees holding a range of + sizes. There are 2 equally spaced treebins for each power of two + from TREE_SHIFT to TREE_SHIFT+16. The last bin holds anything + larger. + + Bin maps + There is one bit map for small bins ("smallmap") and one for + treebins ("treemap). Each bin sets its bit when non-empty, and + clears the bit when empty. Bit operations are then used to avoid + bin-by-bin searching -- nearly all "search" is done without ever + looking at bins that won't be selected. The bit maps + conservatively use 32 bits per map word, even if on 64bit system. + For a good description of some of the bit-based techniques used + here, see Henry S. Warren Jr's book "Hacker's Delight" (and + supplement at http://hackersdelight.org/). Many of these are + intended to reduce the branchiness of paths through malloc etc, as + well as to reduce the number of memory locations read or written. + + Segments + A list of segments headed by an embedded malloc_segment record + representing the initial space. + + Address check support + The least_addr field is the least address ever obtained from + MORECORE or MMAP. Attempted frees and reallocs of any address less + than this are trapped (unless INSECURE is defined). + + Magic tag + A cross-check field that should always hold same value as mparams.magic. + + Flags + Bits recording whether to use MMAP, locks, or contiguous MORECORE + + Statistics + Each space keeps track of current and maximum system memory + obtained via MORECORE or MMAP. + + Trim support + Fields holding the amount of unused topmost memory that should trigger + timming, and a counter to force periodic scanning to release unused + non-topmost segments. + + Locking + If USE_LOCKS is defined, the "mutex" lock is acquired and released + around every public call using this mspace. + + Extension support + A void* pointer and a size_t field that can be used to help implement + extensions to this malloc. +*/ + +/* Bin types, widths and sizes */ +#define NSMALLBINS (32U) +#define NTREEBINS (32U) +#define SMALLBIN_SHIFT (3U) +#define SMALLBIN_WIDTH (SIZE_T_ONE << SMALLBIN_SHIFT) +#define TREEBIN_SHIFT (8U) +#define MIN_LARGE_SIZE (SIZE_T_ONE << TREEBIN_SHIFT) +#define MAX_SMALL_SIZE (MIN_LARGE_SIZE - SIZE_T_ONE) +#define MAX_SMALL_REQUEST (MAX_SMALL_SIZE - CHUNK_ALIGN_MASK - CHUNK_OVERHEAD) + +struct malloc_state { + binmap_t smallmap; + binmap_t treemap; + size_t dvsize; + size_t topsize; + char* least_addr; + mchunkptr dv; + mchunkptr top; + size_t trim_check; + size_t release_checks; + size_t magic; + mchunkptr smallbins[(NSMALLBINS+1)*2]; + tbinptr treebins[NTREEBINS]; + size_t footprint; + size_t max_footprint; + flag_t mflags; + msegment seg; +#if USE_LOCKS + MLOCK_T mutex; /* locate lock among fields that rarely change */ +#endif /* USE_LOCKS */ + void* extp; /* Unused but available for extensions */ + size_t exts; +}; + +typedef struct malloc_state* mstate; + +/* ------------- Global malloc_state and malloc_params ------------------- */ + +#if !ONLY_MSPACES + +/* The global malloc_state used for all non-"mspace" calls */ +static struct malloc_state _gm_; +#define gm (&_gm_) +#define is_global(M) ((M) == &_gm_) + +#endif /* !ONLY_MSPACES */ + +#define is_initialized(M) ((M)->top != 0) + +/* -------------------------- system alloc setup ------------------------- */ + +/* Operations on mflags */ + +#define use_lock(M) ((M)->mflags & USE_LOCK_BIT) +#define enable_lock(M) ((M)->mflags |= USE_LOCK_BIT) +#define disable_lock(M) ((M)->mflags &= ~USE_LOCK_BIT) + +#define use_mmap(M) ((M)->mflags & USE_MMAP_BIT) +#define enable_mmap(M) ((M)->mflags |= USE_MMAP_BIT) +#define disable_mmap(M) ((M)->mflags &= ~USE_MMAP_BIT) + +#define use_noncontiguous(M) ((M)->mflags & USE_NONCONTIGUOUS_BIT) +#define disable_contiguous(M) ((M)->mflags |= USE_NONCONTIGUOUS_BIT) + +#define set_lock(M,L)\ + ((M)->mflags = (L)?\ + ((M)->mflags | USE_LOCK_BIT) :\ + ((M)->mflags & ~USE_LOCK_BIT)) + +/* page-align a size */ +#define page_align(S)\ + (((S) + (mparams.page_size - SIZE_T_ONE)) & ~(mparams.page_size - SIZE_T_ONE)) + +/* granularity-align a size */ +#define granularity_align(S)\ + (((S) + (mparams.granularity - SIZE_T_ONE))\ + & ~(mparams.granularity - SIZE_T_ONE)) + + +/* For mmap, use granularity alignment on windows, else page-align */ +#ifdef WIN32 +#define mmap_align(S) granularity_align(S) +#else +#define mmap_align(S) page_align(S) +#endif + +/* For sys_alloc, enough padding to ensure can malloc request on success */ +#define SYS_ALLOC_PADDING (TOP_FOOT_SIZE + MALLOC_ALIGNMENT) + +#define is_page_aligned(S)\ + (((size_t)(S) & (mparams.page_size - SIZE_T_ONE)) == 0) +#define is_granularity_aligned(S)\ + (((size_t)(S) & (mparams.granularity - SIZE_T_ONE)) == 0) + +/* True if segment S holds address A */ +#define segment_holds(S, A)\ + ((char*)(A) >= S->base && (char*)(A) < S->base + S->size) + +/* Return segment holding given address */ +static msegmentptr segment_holding(mstate m, char* addr) { + msegmentptr sp = &m->seg; + for (;;) { + if (addr >= sp->base && addr < sp->base + sp->size) + return sp; + if ((sp = sp->next) == 0) + return 0; + } +} + +/* Return true if segment contains a segment link */ +static int has_segment_link(mstate m, msegmentptr ss) { + msegmentptr sp = &m->seg; + for (;;) { + if ((char*)sp >= ss->base && (char*)sp < ss->base + ss->size) + return 1; + if ((sp = sp->next) == 0) + return 0; + } +} + +#ifndef MORECORE_CANNOT_TRIM +#define should_trim(M,s) ((s) > (M)->trim_check) +#else /* MORECORE_CANNOT_TRIM */ +#define should_trim(M,s) (0) +#endif /* MORECORE_CANNOT_TRIM */ + +/* + TOP_FOOT_SIZE is padding at the end of a segment, including space + that may be needed to place segment records and fenceposts when new + noncontiguous segments are added. +*/ +#define TOP_FOOT_SIZE\ + (align_offset(chunk2mem(0))+pad_request(sizeof(struct malloc_segment))+MIN_CHUNK_SIZE) + + +/* ------------------------------- Hooks -------------------------------- */ + +/* + PREACTION should be defined to return 0 on success, and nonzero on + failure. If you are not using locking, you can redefine these to do + anything you like. +*/ + +#if USE_LOCKS + +#define PREACTION(M) ((use_lock(M))? ACQUIRE_LOCK(&(M)->mutex) : 0) +#define POSTACTION(M) { if (use_lock(M)) RELEASE_LOCK(&(M)->mutex); } +#else /* USE_LOCKS */ + +#ifndef PREACTION +#define PREACTION(M) (0) +#endif /* PREACTION */ + +#ifndef POSTACTION +#define POSTACTION(M) +#endif /* POSTACTION */ + +#endif /* USE_LOCKS */ + +/* + CORRUPTION_ERROR_ACTION is triggered upon detected bad addresses. + USAGE_ERROR_ACTION is triggered on detected bad frees and + reallocs. The argument p is an address that might have triggered the + fault. It is ignored by the two predefined actions, but might be + useful in custom actions that try to help diagnose errors. +*/ + +#if PROCEED_ON_ERROR + +/* A count of the number of corruption errors causing resets */ +int malloc_corruption_error_count; + +/* default corruption action */ +static void reset_on_error(mstate m); + +#define CORRUPTION_ERROR_ACTION(m) reset_on_error(m) +#define USAGE_ERROR_ACTION(m, p) + +#else /* PROCEED_ON_ERROR */ + +#ifndef CORRUPTION_ERROR_ACTION +#define CORRUPTION_ERROR_ACTION(m) ABORT +#endif /* CORRUPTION_ERROR_ACTION */ + +#ifndef USAGE_ERROR_ACTION +#define USAGE_ERROR_ACTION(m,p) ABORT +#endif /* USAGE_ERROR_ACTION */ + +#endif /* PROCEED_ON_ERROR */ + +/* -------------------------- Debugging setup ---------------------------- */ + +#if ! DEBUG + +#define check_free_chunk(M,P) +#define check_inuse_chunk(M,P) +#define check_malloced_chunk(M,P,N) +#define check_mmapped_chunk(M,P) +#define check_malloc_state(M) +#define check_top_chunk(M,P) + +#else /* DEBUG */ +#define check_free_chunk(M,P) do_check_free_chunk(M,P) +#define check_inuse_chunk(M,P) do_check_inuse_chunk(M,P) +#define check_top_chunk(M,P) do_check_top_chunk(M,P) +#define check_malloced_chunk(M,P,N) do_check_malloced_chunk(M,P,N) +#define check_mmapped_chunk(M,P) do_check_mmapped_chunk(M,P) +#define check_malloc_state(M) do_check_malloc_state(M) + +static void do_check_any_chunk(mstate m, mchunkptr p); +static void do_check_top_chunk(mstate m, mchunkptr p); +static void do_check_mmapped_chunk(mstate m, mchunkptr p); +static void do_check_inuse_chunk(mstate m, mchunkptr p); +static void do_check_free_chunk(mstate m, mchunkptr p); +static void do_check_malloced_chunk(mstate m, void* mem, size_t s); +static void do_check_tree(mstate m, tchunkptr t); +static void do_check_treebin(mstate m, bindex_t i); +static void do_check_smallbin(mstate m, bindex_t i); +static void do_check_malloc_state(mstate m); +static int bin_find(mstate m, mchunkptr x); +static size_t traverse_and_check(mstate m); +#endif /* DEBUG */ + +/* ---------------------------- Indexing Bins ---------------------------- */ + +#define is_small(s) (((s) >> SMALLBIN_SHIFT) < NSMALLBINS) +#define small_index(s) (bindex_t)((s) >> SMALLBIN_SHIFT) +#define small_index2size(i) ((i) << SMALLBIN_SHIFT) +#define MIN_SMALL_INDEX (small_index(MIN_CHUNK_SIZE)) + +/* addressing by index. See above about smallbin repositioning */ +#define smallbin_at(M, i) ((sbinptr)((char*)&((M)->smallbins[(i)<<1]))) +#define treebin_at(M,i) (&((M)->treebins[i])) + +/* assign tree index for size S to variable I. Use x86 asm if possible */ +#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#define compute_tree_index(S, I)\ +{\ + unsigned int X = S >> TREEBIN_SHIFT;\ + if (X == 0)\ + I = 0;\ + else if (X > 0xFFFF)\ + I = NTREEBINS-1;\ + else {\ + unsigned int K;\ + __asm__("bsrl\t%1, %0\n\t" : "=r" (K) : "g" (X));\ + I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\ + }\ +} + +#elif defined (__INTEL_COMPILER) +#define compute_tree_index(S, I)\ +{\ + size_t X = S >> TREEBIN_SHIFT;\ + if (X == 0)\ + I = 0;\ + else if (X > 0xFFFF)\ + I = NTREEBINS-1;\ + else {\ + unsigned int K = _bit_scan_reverse (X); \ + I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\ + }\ +} + +#elif defined(_MSC_VER) && _MSC_VER>=1300 +#define compute_tree_index(S, I)\ +{\ + size_t X = S >> TREEBIN_SHIFT;\ + if (X == 0)\ + I = 0;\ + else if (X > 0xFFFF)\ + I = NTREEBINS-1;\ + else {\ + unsigned int K;\ + _BitScanReverse((DWORD *) &K, (DWORD) X);\ + I = (bindex_t)((K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1)));\ + }\ +} + +#else /* GNUC */ +#define compute_tree_index(S, I)\ +{\ + size_t X = S >> TREEBIN_SHIFT;\ + if (X == 0)\ + I = 0;\ + else if (X > 0xFFFF)\ + I = NTREEBINS-1;\ + else {\ + unsigned int Y = (unsigned int)X;\ + unsigned int N = ((Y - 0x100) >> 16) & 8;\ + unsigned int K = (((Y <<= N) - 0x1000) >> 16) & 4;\ + N += K;\ + N += K = (((Y <<= K) - 0x4000) >> 16) & 2;\ + K = 14 - N + ((Y <<= K) >> 15);\ + I = (K << 1) + ((S >> (K + (TREEBIN_SHIFT-1)) & 1));\ + }\ +} +#endif /* GNUC */ + +/* Bit representing maximum resolved size in a treebin at i */ +#define bit_for_tree_index(i) \ + (i == NTREEBINS-1)? (SIZE_T_BITSIZE-1) : (((i) >> 1) + TREEBIN_SHIFT - 2) + +/* Shift placing maximum resolved bit in a treebin at i as sign bit */ +#define leftshift_for_tree_index(i) \ + ((i == NTREEBINS-1)? 0 : \ + ((SIZE_T_BITSIZE-SIZE_T_ONE) - (((i) >> 1) + TREEBIN_SHIFT - 2))) + +/* The size of the smallest chunk held in bin with index i */ +#define minsize_for_tree_index(i) \ + ((SIZE_T_ONE << (((i) >> 1) + TREEBIN_SHIFT)) | \ + (((size_t)((i) & SIZE_T_ONE)) << (((i) >> 1) + TREEBIN_SHIFT - 1))) + + +/* ------------------------ Operations on bin maps ----------------------- */ + +/* bit corresponding to given index */ +#define idx2bit(i) ((binmap_t)(1) << (i)) + +/* Mark/Clear bits with given index */ +#define mark_smallmap(M,i) ((M)->smallmap |= idx2bit(i)) +#define clear_smallmap(M,i) ((M)->smallmap &= ~idx2bit(i)) +#define smallmap_is_marked(M,i) ((M)->smallmap & idx2bit(i)) + +#define mark_treemap(M,i) ((M)->treemap |= idx2bit(i)) +#define clear_treemap(M,i) ((M)->treemap &= ~idx2bit(i)) +#define treemap_is_marked(M,i) ((M)->treemap & idx2bit(i)) + +/* isolate the least set bit of a bitmap */ +#define least_bit(x) ((x) & -(x)) + +/* mask with all bits to left of least bit of x on */ +#define left_bits(x) ((x<<1) | -(x<<1)) + +/* mask with all bits to left of or equal to least bit of x on */ +#define same_or_left_bits(x) ((x) | -(x)) + +/* index corresponding to given bit. Use x86 asm if possible */ + +#if defined(__GNUC__) && (defined(__i386__) || defined(__x86_64__)) +#define compute_bit2idx(X, I)\ +{\ + unsigned int J;\ + __asm__("bsfl\t%1, %0\n\t" : "=r" (J) : "g" (X));\ + I = (bindex_t)J;\ +} + +#elif defined (__INTEL_COMPILER) +#define compute_bit2idx(X, I)\ +{\ + unsigned int J;\ + J = _bit_scan_forward (X); \ + I = (bindex_t)J;\ +} + +#elif defined(_MSC_VER) && _MSC_VER>=1300 +#define compute_bit2idx(X, I)\ +{\ + unsigned int J;\ + _BitScanForward((DWORD *) &J, X);\ + I = (bindex_t)J;\ +} + +#elif USE_BUILTIN_FFS +#define compute_bit2idx(X, I) I = ffs(X)-1 + +#else +#define compute_bit2idx(X, I)\ +{\ + unsigned int Y = X - 1;\ + unsigned int K = Y >> (16-4) & 16;\ + unsigned int N = K; Y >>= K;\ + N += K = Y >> (8-3) & 8; Y >>= K;\ + N += K = Y >> (4-2) & 4; Y >>= K;\ + N += K = Y >> (2-1) & 2; Y >>= K;\ + N += K = Y >> (1-0) & 1; Y >>= K;\ + I = (bindex_t)(N + Y);\ +} +#endif /* GNUC */ + + +/* ----------------------- Runtime Check Support ------------------------- */ + +/* + For security, the main invariant is that malloc/free/etc never + writes to a static address other than malloc_state, unless static + malloc_state itself has been corrupted, which cannot occur via + malloc (because of these checks). In essence this means that we + believe all pointers, sizes, maps etc held in malloc_state, but + check all of those linked or offsetted from other embedded data + structures. These checks are interspersed with main code in a way + that tends to minimize their run-time cost. + + When FOOTERS is defined, in addition to range checking, we also + verify footer fields of inuse chunks, which can be used guarantee + that the mstate controlling malloc/free is intact. This is a + streamlined version of the approach described by William Robertson + et al in "Run-time Detection of Heap-based Overflows" LISA'03 + http://www.usenix.org/events/lisa03/tech/robertson.html The footer + of an inuse chunk holds the xor of its mstate and a random seed, + that is checked upon calls to free() and realloc(). This is + (probablistically) unguessable from outside the program, but can be + computed by any code successfully malloc'ing any chunk, so does not + itself provide protection against code that has already broken + security through some other means. Unlike Robertson et al, we + always dynamically check addresses of all offset chunks (previous, + next, etc). This turns out to be cheaper than relying on hashes. +*/ + +#if !INSECURE +/* Check if address a is at least as high as any from MORECORE or MMAP */ +#define ok_address(M, a) ((char*)(a) >= (M)->least_addr) +/* Check if address of next chunk n is higher than base chunk p */ +#define ok_next(p, n) ((char*)(p) < (char*)(n)) +/* Check if p has inuse status */ +#define ok_inuse(p) is_inuse(p) +/* Check if p has its pinuse bit on */ +#define ok_pinuse(p) pinuse(p) + +#else /* !INSECURE */ +#define ok_address(M, a) (1) +#define ok_next(b, n) (1) +#define ok_inuse(p) (1) +#define ok_pinuse(p) (1) +#endif /* !INSECURE */ + +#if (FOOTERS && !INSECURE) +/* Check if (alleged) mstate m has expected magic field */ +#define ok_magic(M) ((M)->magic == mparams.magic) +#else /* (FOOTERS && !INSECURE) */ +#define ok_magic(M) (1) +#endif /* (FOOTERS && !INSECURE) */ + + +/* In gcc, use __builtin_expect to minimize impact of checks */ +#if !INSECURE +#if defined(__GNUC__) && __GNUC__ >= 3 +#define RTCHECK(e) __builtin_expect(e, 1) +#else /* GNUC */ +#define RTCHECK(e) (e) +#endif /* GNUC */ +#else /* !INSECURE */ +#define RTCHECK(e) (1) +#endif /* !INSECURE */ + +/* macros to set up inuse chunks with or without footers */ + +#if !FOOTERS + +#define mark_inuse_foot(M,p,s) + +/* Macros for setting head/foot of non-mmapped chunks */ + +/* Set cinuse bit and pinuse bit of next chunk */ +#define set_inuse(M,p,s)\ + ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\ + ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT) + +/* Set cinuse and pinuse of this chunk and pinuse of next chunk */ +#define set_inuse_and_pinuse(M,p,s)\ + ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\ + ((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT) + +/* Set size, cinuse and pinuse bit of this chunk */ +#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\ + ((p)->head = (s|PINUSE_BIT|CINUSE_BIT)) + +#else /* FOOTERS */ + +/* Set foot of inuse chunk to be xor of mstate and seed */ +#define mark_inuse_foot(M,p,s)\ + (((mchunkptr)((char*)(p) + (s)))->prev_foot = ((size_t)(M) ^ mparams.magic)) + +#define get_mstate_for(p)\ + ((mstate)(((mchunkptr)((char*)(p) +\ + (chunksize(p))))->prev_foot ^ mparams.magic)) + +#define set_inuse(M,p,s)\ + ((p)->head = (((p)->head & PINUSE_BIT)|s|CINUSE_BIT),\ + (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT), \ + mark_inuse_foot(M,p,s)) + +#define set_inuse_and_pinuse(M,p,s)\ + ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\ + (((mchunkptr)(((char*)(p)) + (s)))->head |= PINUSE_BIT),\ + mark_inuse_foot(M,p,s)) + +#define set_size_and_pinuse_of_inuse_chunk(M, p, s)\ + ((p)->head = (s|PINUSE_BIT|CINUSE_BIT),\ + mark_inuse_foot(M, p, s)) + +#endif /* !FOOTERS */ + +/* ---------------------------- setting mparams -------------------------- */ + +#ifdef ENABLE_LARGE_PAGES +typedef size_t (WINAPI *GetLargePageMinimum_t)(void); +#endif + +/* Initialize mparams */ +static int init_mparams(void) { +#ifdef NEED_GLOBAL_LOCK_INIT + if (malloc_global_mutex_status <= 0) + init_malloc_global_mutex(); +#endif + + ACQUIRE_MALLOC_GLOBAL_LOCK(); + if (mparams.magic == 0) { + size_t magic; + size_t psize; + size_t gsize; + +#ifndef WIN32 + psize = malloc_getpagesize; + gsize = ((DEFAULT_GRANULARITY != 0)? DEFAULT_GRANULARITY : psize); +#else /* WIN32 */ + { + SYSTEM_INFO system_info; + GetSystemInfo(&system_info); + psize = system_info.dwPageSize; + gsize = ((DEFAULT_GRANULARITY != 0)? + DEFAULT_GRANULARITY : system_info.dwAllocationGranularity); +#ifdef ENABLE_LARGE_PAGES + { + GetLargePageMinimum_t GetLargePageMinimum_ = (GetLargePageMinimum_t) GetProcAddress(GetModuleHandle(__T("kernel32.dll")), "GetLargePageMinimum"); + if(GetLargePageMinimum_) { + size_t largepagesize = GetLargePageMinimum_(); + if(largepagesize) { + psize = largepagesize; + gsize = ((DEFAULT_GRANULARITY != 0)? + DEFAULT_GRANULARITY : largepagesize); + if(gsize < largepagesize) gsize = largepagesize; + } + } + } +#endif + } +#endif /* WIN32 */ + + /* Sanity-check configuration: + size_t must be unsigned and as wide as pointer type. + ints must be at least 4 bytes. + alignment must be at least 8. + Alignment, min chunk size, and page size must all be powers of 2. + */ + if ((sizeof(size_t) != sizeof(char*)) || + (MAX_SIZE_T < MIN_CHUNK_SIZE) || + (sizeof(int) < 4) || + (MALLOC_ALIGNMENT < (size_t)8U) || + ((MALLOC_ALIGNMENT & (MALLOC_ALIGNMENT-SIZE_T_ONE)) != 0) || + ((MCHUNK_SIZE & (MCHUNK_SIZE-SIZE_T_ONE)) != 0) || + ((gsize & (gsize-SIZE_T_ONE)) != 0) || + ((psize & (psize-SIZE_T_ONE)) != 0)) + ABORT; + + mparams.granularity = gsize; + mparams.page_size = psize; + mparams.mmap_threshold = DEFAULT_MMAP_THRESHOLD; + mparams.trim_threshold = DEFAULT_TRIM_THRESHOLD; +#if MORECORE_CONTIGUOUS + mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT; +#else /* MORECORE_CONTIGUOUS */ + mparams.default_mflags = USE_LOCK_BIT|USE_MMAP_BIT|USE_NONCONTIGUOUS_BIT; +#endif /* MORECORE_CONTIGUOUS */ + +#if !ONLY_MSPACES + /* Set up lock for main malloc area */ + gm->mflags = mparams.default_mflags; + INITIAL_LOCK(&gm->mutex); +#endif + + { +#if USE_DEV_RANDOM + int fd; + unsigned char buf[sizeof(size_t)]; + /* Try to use /dev/urandom, else fall back on using time */ + if ((fd = open("/dev/urandom", O_RDONLY)) >= 0 && + read(fd, buf, sizeof(buf)) == sizeof(buf)) { + magic = *((size_t *) buf); + close(fd); + } + else +#endif /* USE_DEV_RANDOM */ +#ifdef WIN32 + magic = (size_t)(GetTickCount() ^ (size_t)0x55555555U); +#else + magic = (size_t)(time(0) ^ (size_t)0x55555555U); +#endif + magic |= (size_t)8U; /* ensure nonzero */ + magic &= ~(size_t)7U; /* improve chances of fault for bad values */ + mparams.magic = magic; + } + } + + RELEASE_MALLOC_GLOBAL_LOCK(); + return 1; +} + +/* support for mallopt */ +static int change_mparam(int param_number, int value) { + size_t val; + ensure_initialization(); + val = (value == -1)? MAX_SIZE_T : (size_t)value; + switch(param_number) { + case M_TRIM_THRESHOLD: + mparams.trim_threshold = val; + return 1; + case M_GRANULARITY: + if (val >= mparams.page_size && ((val & (val-1)) == 0)) { + mparams.granularity = val; + return 1; + } + else + return 0; + case M_MMAP_THRESHOLD: + mparams.mmap_threshold = val; + return 1; + default: + return 0; + } +} + +#if DEBUG +/* ------------------------- Debugging Support --------------------------- */ + +/* Check properties of any chunk, whether free, inuse, mmapped etc */ +static void do_check_any_chunk(mstate m, mchunkptr p) { + assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD)); + assert(ok_address(m, p)); +} + +/* Check properties of top chunk */ +static void do_check_top_chunk(mstate m, mchunkptr p) { + msegmentptr sp = segment_holding(m, (char*)p); + size_t sz = p->head & ~INUSE_BITS; /* third-lowest bit can be set! */ + assert(sp != 0); + assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD)); + assert(ok_address(m, p)); + assert(sz == m->topsize); + assert(sz > 0); + assert(sz == ((sp->base + sp->size) - (char*)p) - TOP_FOOT_SIZE); + assert(pinuse(p)); + assert(!pinuse(chunk_plus_offset(p, sz))); +} + +/* Check properties of (inuse) mmapped chunks */ +static void do_check_mmapped_chunk(mstate m, mchunkptr p) { + size_t sz = chunksize(p); + size_t len = (sz + (p->prev_foot) + MMAP_FOOT_PAD); + assert(is_mmapped(p)); + assert(use_mmap(m)); + assert((is_aligned(chunk2mem(p))) || (p->head == FENCEPOST_HEAD)); + assert(ok_address(m, p)); + assert(!is_small(sz)); + assert((len & (mparams.page_size-SIZE_T_ONE)) == 0); + assert(chunk_plus_offset(p, sz)->head == FENCEPOST_HEAD); + assert(chunk_plus_offset(p, sz+SIZE_T_SIZE)->head == 0); +} + +/* Check properties of inuse chunks */ +static void do_check_inuse_chunk(mstate m, mchunkptr p) { + do_check_any_chunk(m, p); + assert(is_inuse(p)); + assert(next_pinuse(p)); + /* If not pinuse and not mmapped, previous chunk has OK offset */ + assert(is_mmapped(p) || pinuse(p) || next_chunk(prev_chunk(p)) == p); + if (is_mmapped(p)) + do_check_mmapped_chunk(m, p); +} + +/* Check properties of free chunks */ +static void do_check_free_chunk(mstate m, mchunkptr p) { + size_t sz = chunksize(p); + mchunkptr next = chunk_plus_offset(p, sz); + do_check_any_chunk(m, p); + assert(!is_inuse(p)); + assert(!next_pinuse(p)); + assert (!is_mmapped(p)); + if (p != m->dv && p != m->top) { + if (sz >= MIN_CHUNK_SIZE) { + assert((sz & CHUNK_ALIGN_MASK) == 0); + assert(is_aligned(chunk2mem(p))); + assert(next->prev_foot == sz); + assert(pinuse(p)); + assert (next == m->top || is_inuse(next)); + assert(p->fd->bk == p); + assert(p->bk->fd == p); + } + else /* markers are always of size SIZE_T_SIZE */ + assert(sz == SIZE_T_SIZE); + } +} + +/* Check properties of malloced chunks at the point they are malloced */ +static void do_check_malloced_chunk(mstate m, void* mem, size_t s) { + if (mem != 0) { + mchunkptr p = mem2chunk(mem); + size_t sz = p->head & ~INUSE_BITS; + do_check_inuse_chunk(m, p); + assert((sz & CHUNK_ALIGN_MASK) == 0); + assert(sz >= MIN_CHUNK_SIZE); + assert(sz >= s); + /* unless mmapped, size is less than MIN_CHUNK_SIZE more than request */ + assert(is_mmapped(p) || sz < (s + MIN_CHUNK_SIZE)); + } +} + +/* Check a tree and its subtrees. */ +static void do_check_tree(mstate m, tchunkptr t) { + tchunkptr head = 0; + tchunkptr u = t; + bindex_t tindex = t->index; + size_t tsize = chunksize(t); + bindex_t idx; + compute_tree_index(tsize, idx); + assert(tindex == idx); + assert(tsize >= MIN_LARGE_SIZE); + assert(tsize >= minsize_for_tree_index(idx)); + assert((idx == NTREEBINS-1) || (tsize < minsize_for_tree_index((idx+1)))); + + do { /* traverse through chain of same-sized nodes */ + do_check_any_chunk(m, ((mchunkptr)u)); + assert(u->index == tindex); + assert(chunksize(u) == tsize); + assert(!is_inuse(u)); + assert(!next_pinuse(u)); + assert(u->fd->bk == u); + assert(u->bk->fd == u); + if (u->parent == 0) { + assert(u->child[0] == 0); + assert(u->child[1] == 0); + } + else { + assert(head == 0); /* only one node on chain has parent */ + head = u; + assert(u->parent != u); + assert (u->parent->child[0] == u || + u->parent->child[1] == u || + *((tbinptr*)(u->parent)) == u); + if (u->child[0] != 0) { + assert(u->child[0]->parent == u); + assert(u->child[0] != u); + do_check_tree(m, u->child[0]); + } + if (u->child[1] != 0) { + assert(u->child[1]->parent == u); + assert(u->child[1] != u); + do_check_tree(m, u->child[1]); + } + if (u->child[0] != 0 && u->child[1] != 0) { + assert(chunksize(u->child[0]) < chunksize(u->child[1])); + } + } + u = u->fd; + } while (u != t); + assert(head != 0); +} + +/* Check all the chunks in a treebin. */ +static void do_check_treebin(mstate m, bindex_t i) { + tbinptr* tb = treebin_at(m, i); + tchunkptr t = *tb; + int empty = (m->treemap & (1U << i)) == 0; + if (t == 0) + assert(empty); + if (!empty) + do_check_tree(m, t); +} + +/* Check all the chunks in a smallbin. */ +static void do_check_smallbin(mstate m, bindex_t i) { + sbinptr b = smallbin_at(m, i); + mchunkptr p = b->bk; + unsigned int empty = (m->smallmap & (1U << i)) == 0; + if (p == b) + assert(empty); + if (!empty) { + for (; p != b; p = p->bk) { + size_t size = chunksize(p); + mchunkptr q; + /* each chunk claims to be free */ + do_check_free_chunk(m, p); + /* chunk belongs in bin */ + assert(small_index(size) == i); + assert(p->bk == b || chunksize(p->bk) == chunksize(p)); + /* chunk is followed by an inuse chunk */ + q = next_chunk(p); + if (q->head != FENCEPOST_HEAD) + do_check_inuse_chunk(m, q); + } + } +} + +/* Find x in a bin. Used in other check functions. */ +static int bin_find(mstate m, mchunkptr x) { + size_t size = chunksize(x); + if (is_small(size)) { + bindex_t sidx = small_index(size); + sbinptr b = smallbin_at(m, sidx); + if (smallmap_is_marked(m, sidx)) { + mchunkptr p = b; + do { + if (p == x) + return 1; + } while ((p = p->fd) != b); + } + } + else { + bindex_t tidx; + compute_tree_index(size, tidx); + if (treemap_is_marked(m, tidx)) { + tchunkptr t = *treebin_at(m, tidx); + size_t sizebits = size << leftshift_for_tree_index(tidx); + while (t != 0 && chunksize(t) != size) { + t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]; + sizebits <<= 1; + } + if (t != 0) { + tchunkptr u = t; + do { + if (u == (tchunkptr)x) + return 1; + } while ((u = u->fd) != t); + } + } + } + return 0; +} + +/* Traverse each chunk and check it; return total */ +static size_t traverse_and_check(mstate m) { + size_t sum = 0; + if (is_initialized(m)) { + msegmentptr s = &m->seg; + sum += m->topsize + TOP_FOOT_SIZE; + while (s != 0) { + mchunkptr q = align_as_chunk(s->base); + mchunkptr lastq = 0; + assert(pinuse(q)); + while (segment_holds(s, q) && + q != m->top && q->head != FENCEPOST_HEAD) { + sum += chunksize(q); + if (is_inuse(q)) { + assert(!bin_find(m, q)); + do_check_inuse_chunk(m, q); + } + else { + assert(q == m->dv || bin_find(m, q)); + assert(lastq == 0 || is_inuse(lastq)); /* Not 2 consecutive free */ + do_check_free_chunk(m, q); + } + lastq = q; + q = next_chunk(q); + } + s = s->next; + } + } + return sum; +} + +/* Check all properties of malloc_state. */ +static void do_check_malloc_state(mstate m) { + bindex_t i; + size_t total; + /* check bins */ + for (i = 0; i < NSMALLBINS; ++i) + do_check_smallbin(m, i); + for (i = 0; i < NTREEBINS; ++i) + do_check_treebin(m, i); + + if (m->dvsize != 0) { /* check dv chunk */ + do_check_any_chunk(m, m->dv); + assert(m->dvsize == chunksize(m->dv)); + assert(m->dvsize >= MIN_CHUNK_SIZE); + assert(bin_find(m, m->dv) == 0); + } + + if (m->top != 0) { /* check top chunk */ + do_check_top_chunk(m, m->top); + /*assert(m->topsize == chunksize(m->top)); redundant */ + assert(m->topsize > 0); + assert(bin_find(m, m->top) == 0); + } + + total = traverse_and_check(m); + assert(total <= m->footprint); + assert(m->footprint <= m->max_footprint); +} +#endif /* DEBUG */ + +/* ----------------------------- statistics ------------------------------ */ + +#if !NO_MALLINFO +static struct mallinfo internal_mallinfo(mstate m) { + struct mallinfo nm = { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 }; + ensure_initialization(); + if (!PREACTION(m)) { + check_malloc_state(m); + if (is_initialized(m)) { + size_t nfree = SIZE_T_ONE; /* top always free */ + size_t mfree = m->topsize + TOP_FOOT_SIZE; + size_t sum = mfree; + msegmentptr s = &m->seg; + while (s != 0) { + mchunkptr q = align_as_chunk(s->base); + while (segment_holds(s, q) && + q != m->top && q->head != FENCEPOST_HEAD) { + size_t sz = chunksize(q); + sum += sz; + if (!is_inuse(q)) { + mfree += sz; + ++nfree; + } + q = next_chunk(q); + } + s = s->next; + } + + nm.arena = sum; + nm.ordblks = nfree; + nm.hblkhd = m->footprint - sum; + nm.usmblks = m->max_footprint; + nm.uordblks = m->footprint - mfree; + nm.fordblks = mfree; + nm.keepcost = m->topsize; + } + + POSTACTION(m); + } + return nm; +} +#endif /* !NO_MALLINFO */ + +static void internal_malloc_stats(mstate m) { + ensure_initialization(); + if (!PREACTION(m)) { + size_t maxfp = 0; + size_t fp = 0; + size_t used = 0; + check_malloc_state(m); + if (is_initialized(m)) { + msegmentptr s = &m->seg; + maxfp = m->max_footprint; + fp = m->footprint; + used = fp - (m->topsize + TOP_FOOT_SIZE); + + while (s != 0) { + mchunkptr q = align_as_chunk(s->base); + while (segment_holds(s, q) && + q != m->top && q->head != FENCEPOST_HEAD) { + if (!is_inuse(q)) + used -= chunksize(q); + q = next_chunk(q); + } + s = s->next; + } + } + + fprintf(stderr, "max system bytes = %10lu\n", (unsigned long)(maxfp)); + fprintf(stderr, "system bytes = %10lu\n", (unsigned long)(fp)); + fprintf(stderr, "in use bytes = %10lu\n", (unsigned long)(used)); + + POSTACTION(m); + } +} + +/* ----------------------- Operations on smallbins ----------------------- */ + +/* + Various forms of linking and unlinking are defined as macros. Even + the ones for trees, which are very long but have very short typical + paths. This is ugly but reduces reliance on inlining support of + compilers. +*/ + +/* Link a free chunk into a smallbin */ +#define insert_small_chunk(M, P, S) {\ + bindex_t I = small_index(S);\ + mchunkptr B = smallbin_at(M, I);\ + mchunkptr F = B;\ + assert(S >= MIN_CHUNK_SIZE);\ + if (!smallmap_is_marked(M, I))\ + mark_smallmap(M, I);\ + else if (RTCHECK(ok_address(M, B->fd)))\ + F = B->fd;\ + else {\ + CORRUPTION_ERROR_ACTION(M);\ + }\ + B->fd = P;\ + F->bk = P;\ + P->fd = F;\ + P->bk = B;\ +} + +/* Unlink a chunk from a smallbin */ +#define unlink_small_chunk(M, P, S) {\ + mchunkptr F = P->fd;\ + mchunkptr B = P->bk;\ + bindex_t I = small_index(S);\ + assert(P != B);\ + assert(P != F);\ + assert(chunksize(P) == small_index2size(I));\ + if (F == B)\ + clear_smallmap(M, I);\ + else if (RTCHECK((F == smallbin_at(M,I) || ok_address(M, F)) &&\ + (B == smallbin_at(M,I) || ok_address(M, B)))) {\ + F->bk = B;\ + B->fd = F;\ + }\ + else {\ + CORRUPTION_ERROR_ACTION(M);\ + }\ +} + +/* Unlink the first chunk from a smallbin */ +#define unlink_first_small_chunk(M, B, P, I) {\ + mchunkptr F = P->fd;\ + assert(P != B);\ + assert(P != F);\ + assert(chunksize(P) == small_index2size(I));\ + if (B == F)\ + clear_smallmap(M, I);\ + else if (RTCHECK(ok_address(M, F))) {\ + B->fd = F;\ + F->bk = B;\ + }\ + else {\ + CORRUPTION_ERROR_ACTION(M);\ + }\ +} + + + +/* Replace dv node, binning the old one */ +/* Used only when dvsize known to be small */ +#define replace_dv(M, P, S) {\ + size_t DVS = M->dvsize;\ + if (DVS != 0) {\ + mchunkptr DV = M->dv;\ + assert(is_small(DVS));\ + insert_small_chunk(M, DV, DVS);\ + }\ + M->dvsize = S;\ + M->dv = P;\ +} + +/* ------------------------- Operations on trees ------------------------- */ + +/* Insert chunk into tree */ +#define insert_large_chunk(M, X, S) {\ + tbinptr* H;\ + bindex_t I;\ + compute_tree_index(S, I);\ + H = treebin_at(M, I);\ + X->index = I;\ + X->child[0] = X->child[1] = 0;\ + if (!treemap_is_marked(M, I)) {\ + mark_treemap(M, I);\ + *H = X;\ + X->parent = (tchunkptr)H;\ + X->fd = X->bk = X;\ + }\ + else {\ + tchunkptr T = *H;\ + size_t K = S << leftshift_for_tree_index(I);\ + for (;;) {\ + if (chunksize(T) != S) {\ + tchunkptr* C = &(T->child[(K >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]);\ + K <<= 1;\ + if (*C != 0)\ + T = *C;\ + else if (RTCHECK(ok_address(M, C))) {\ + *C = X;\ + X->parent = T;\ + X->fd = X->bk = X;\ + break;\ + }\ + else {\ + CORRUPTION_ERROR_ACTION(M);\ + break;\ + }\ + }\ + else {\ + tchunkptr F = T->fd;\ + if (RTCHECK(ok_address(M, T) && ok_address(M, F))) {\ + T->fd = F->bk = X;\ + X->fd = F;\ + X->bk = T;\ + X->parent = 0;\ + break;\ + }\ + else {\ + CORRUPTION_ERROR_ACTION(M);\ + break;\ + }\ + }\ + }\ + }\ +} + +/* + Unlink steps: + + 1. If x is a chained node, unlink it from its same-sized fd/bk links + and choose its bk node as its replacement. + 2. If x was the last node of its size, but not a leaf node, it must + be replaced with a leaf node (not merely one with an open left or + right), to make sure that lefts and rights of descendents + correspond properly to bit masks. We use the rightmost descendent + of x. We could use any other leaf, but this is easy to locate and + tends to counteract removal of leftmosts elsewhere, and so keeps + paths shorter than minimally guaranteed. This doesn't loop much + because on average a node in a tree is near the bottom. + 3. If x is the base of a chain (i.e., has parent links) relink + x's parent and children to x's replacement (or null if none). +*/ + +#define unlink_large_chunk(M, X) {\ + tchunkptr XP = X->parent;\ + tchunkptr R;\ + if (X->bk != X) {\ + tchunkptr F = X->fd;\ + R = X->bk;\ + if (RTCHECK(ok_address(M, F))) {\ + F->bk = R;\ + R->fd = F;\ + }\ + else {\ + CORRUPTION_ERROR_ACTION(M);\ + }\ + }\ + else {\ + tchunkptr* RP;\ + if (((R = *(RP = &(X->child[1]))) != 0) ||\ + ((R = *(RP = &(X->child[0]))) != 0)) {\ + tchunkptr* CP;\ + while ((*(CP = &(R->child[1])) != 0) ||\ + (*(CP = &(R->child[0])) != 0)) {\ + R = *(RP = CP);\ + }\ + if (RTCHECK(ok_address(M, RP)))\ + *RP = 0;\ + else {\ + CORRUPTION_ERROR_ACTION(M);\ + }\ + }\ + }\ + if (XP != 0) {\ + tbinptr* H = treebin_at(M, X->index);\ + if (X == *H) {\ + if ((*H = R) == 0) \ + clear_treemap(M, X->index);\ + }\ + else if (RTCHECK(ok_address(M, XP))) {\ + if (XP->child[0] == X) \ + XP->child[0] = R;\ + else \ + XP->child[1] = R;\ + }\ + else\ + CORRUPTION_ERROR_ACTION(M);\ + if (R != 0) {\ + if (RTCHECK(ok_address(M, R))) {\ + tchunkptr C0, C1;\ + R->parent = XP;\ + if ((C0 = X->child[0]) != 0) {\ + if (RTCHECK(ok_address(M, C0))) {\ + R->child[0] = C0;\ + C0->parent = R;\ + }\ + else\ + CORRUPTION_ERROR_ACTION(M);\ + }\ + if ((C1 = X->child[1]) != 0) {\ + if (RTCHECK(ok_address(M, C1))) {\ + R->child[1] = C1;\ + C1->parent = R;\ + }\ + else\ + CORRUPTION_ERROR_ACTION(M);\ + }\ + }\ + else\ + CORRUPTION_ERROR_ACTION(M);\ + }\ + }\ +} + +/* Relays to large vs small bin operations */ + +#define insert_chunk(M, P, S)\ + if (is_small(S)) insert_small_chunk(M, P, S)\ + else { tchunkptr TP = (tchunkptr)(P); insert_large_chunk(M, TP, S); } + +#define unlink_chunk(M, P, S)\ + if (is_small(S)) unlink_small_chunk(M, P, S)\ + else { tchunkptr TP = (tchunkptr)(P); unlink_large_chunk(M, TP); } + + +/* Relays to internal calls to malloc/free from realloc, memalign etc */ + +#if ONLY_MSPACES +#define internal_malloc(m, b) mspace_malloc(m, b) +#define internal_free(m, mem) mspace_free(m,mem); +#else /* ONLY_MSPACES */ +#if MSPACES +#define internal_malloc(m, b)\ + (m == gm)? dlmalloc(b) : mspace_malloc(m, b) +#define internal_free(m, mem)\ + if (m == gm) dlfree(mem); else mspace_free(m,mem); +#else /* MSPACES */ +#define internal_malloc(m, b) dlmalloc(b) +#define internal_free(m, mem) dlfree(mem) +#endif /* MSPACES */ +#endif /* ONLY_MSPACES */ + +/* ----------------------- Direct-mmapping chunks ----------------------- */ + +/* + Directly mmapped chunks are set up with an offset to the start of + the mmapped region stored in the prev_foot field of the chunk. This + allows reconstruction of the required argument to MUNMAP when freed, + and also allows adjustment of the returned chunk to meet alignment + requirements (especially in memalign). +*/ + +/* Malloc using mmap */ +static void* mmap_alloc(mstate m, size_t nb) { + size_t mmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK); + if (mmsize > nb) { /* Check for wrap around 0 */ + char* mm = (char*)(CALL_DIRECT_MMAP(mmsize)); + if (mm != CMFAIL) { + size_t offset = align_offset(chunk2mem(mm)); + size_t psize = mmsize - offset - MMAP_FOOT_PAD; + mchunkptr p = (mchunkptr)(mm + offset); + p->prev_foot = offset; + p->head = psize; + mark_inuse_foot(m, p, psize); + chunk_plus_offset(p, psize)->head = FENCEPOST_HEAD; + chunk_plus_offset(p, psize+SIZE_T_SIZE)->head = 0; + + if (m->least_addr == 0 || mm < m->least_addr) + m->least_addr = mm; + if ((m->footprint += mmsize) > m->max_footprint) + m->max_footprint = m->footprint; + assert(is_aligned(chunk2mem(p))); + check_mmapped_chunk(m, p); + return chunk2mem(p); + } + } + return 0; +} + +/* Realloc using mmap */ +static mchunkptr mmap_resize(mstate m, mchunkptr oldp, size_t nb) { + size_t oldsize = chunksize(oldp); + if (is_small(nb)) /* Can't shrink mmap regions below small size */ + return 0; + /* Keep old chunk if big enough but not too big */ + if (oldsize >= nb + SIZE_T_SIZE && + (oldsize - nb) <= (mparams.granularity << 1)) + return oldp; + else { + size_t offset = oldp->prev_foot; + size_t oldmmsize = oldsize + offset + MMAP_FOOT_PAD; + size_t newmmsize = mmap_align(nb + SIX_SIZE_T_SIZES + CHUNK_ALIGN_MASK); + char* cp = (char*)CALL_MREMAP((char*)oldp - offset, + oldmmsize, newmmsize, 1); + if (cp != CMFAIL) { + mchunkptr newp = (mchunkptr)(cp + offset); + size_t psize = newmmsize - offset - MMAP_FOOT_PAD; + newp->head = psize; + mark_inuse_foot(m, newp, psize); + chunk_plus_offset(newp, psize)->head = FENCEPOST_HEAD; + chunk_plus_offset(newp, psize+SIZE_T_SIZE)->head = 0; + + if (cp < m->least_addr) + m->least_addr = cp; + if ((m->footprint += newmmsize - oldmmsize) > m->max_footprint) + m->max_footprint = m->footprint; + check_mmapped_chunk(m, newp); + return newp; + } + } + return 0; +} + +/* -------------------------- mspace management -------------------------- */ + +/* Initialize top chunk and its size */ +static void init_top(mstate m, mchunkptr p, size_t psize) { + /* Ensure alignment */ + size_t offset = align_offset(chunk2mem(p)); + p = (mchunkptr)((char*)p + offset); + psize -= offset; + + m->top = p; + m->topsize = psize; + p->head = psize | PINUSE_BIT; + /* set size of fake trailing chunk holding overhead space only once */ + chunk_plus_offset(p, psize)->head = TOP_FOOT_SIZE; + m->trim_check = mparams.trim_threshold; /* reset on each update */ +} + +/* Initialize bins for a new mstate that is otherwise zeroed out */ +static void init_bins(mstate m) { + /* Establish circular links for smallbins */ + bindex_t i; + for (i = 0; i < NSMALLBINS; ++i) { + sbinptr bin = smallbin_at(m,i); + bin->fd = bin->bk = bin; + } +} + +#if PROCEED_ON_ERROR + +/* default corruption action */ +static void reset_on_error(mstate m) { + int i; + ++malloc_corruption_error_count; + /* Reinitialize fields to forget about all memory */ + m->smallbins = m->treebins = 0; + m->dvsize = m->topsize = 0; + m->seg.base = 0; + m->seg.size = 0; + m->seg.next = 0; + m->top = m->dv = 0; + for (i = 0; i < NTREEBINS; ++i) + *treebin_at(m, i) = 0; + init_bins(m); +} +#endif /* PROCEED_ON_ERROR */ + +/* Allocate chunk and prepend remainder with chunk in successor base. */ +static void* prepend_alloc(mstate m, char* newbase, char* oldbase, + size_t nb) { + mchunkptr p = align_as_chunk(newbase); + mchunkptr oldfirst = align_as_chunk(oldbase); + size_t psize = (char*)oldfirst - (char*)p; + mchunkptr q = chunk_plus_offset(p, nb); + size_t qsize = psize - nb; + set_size_and_pinuse_of_inuse_chunk(m, p, nb); + + assert((char*)oldfirst > (char*)q); + assert(pinuse(oldfirst)); + assert(qsize >= MIN_CHUNK_SIZE); + + /* consolidate remainder with first chunk of old base */ + if (oldfirst == m->top) { + size_t tsize = m->topsize += qsize; + m->top = q; + q->head = tsize | PINUSE_BIT; + check_top_chunk(m, q); + } + else if (oldfirst == m->dv) { + size_t dsize = m->dvsize += qsize; + m->dv = q; + set_size_and_pinuse_of_free_chunk(q, dsize); + } + else { + if (!is_inuse(oldfirst)) { + size_t nsize = chunksize(oldfirst); + unlink_chunk(m, oldfirst, nsize); + oldfirst = chunk_plus_offset(oldfirst, nsize); + qsize += nsize; + } + set_free_with_pinuse(q, qsize, oldfirst); + insert_chunk(m, q, qsize); + check_free_chunk(m, q); + } + + check_malloced_chunk(m, chunk2mem(p), nb); + return chunk2mem(p); +} + +/* Add a segment to hold a new noncontiguous region */ +static void add_segment(mstate m, char* tbase, size_t tsize, flag_t mmapped) { + /* Determine locations and sizes of segment, fenceposts, old top */ + char* old_top = (char*)m->top; + msegmentptr oldsp = segment_holding(m, old_top); + char* old_end = oldsp->base + oldsp->size; + size_t ssize = pad_request(sizeof(struct malloc_segment)); + char* rawsp = old_end - (ssize + FOUR_SIZE_T_SIZES + CHUNK_ALIGN_MASK); + size_t offset = align_offset(chunk2mem(rawsp)); + char* asp = rawsp + offset; + char* csp = (asp < (old_top + MIN_CHUNK_SIZE))? old_top : asp; + mchunkptr sp = (mchunkptr)csp; + msegmentptr ss = (msegmentptr)(chunk2mem(sp)); + mchunkptr tnext = chunk_plus_offset(sp, ssize); + mchunkptr p = tnext; + int nfences = 0; + + /* reset top to new space */ + init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE); + + /* Set up segment record */ + assert(is_aligned(ss)); + set_size_and_pinuse_of_inuse_chunk(m, sp, ssize); + *ss = m->seg; /* Push current record */ + m->seg.base = tbase; + m->seg.size = tsize; + m->seg.sflags = mmapped; + m->seg.next = ss; + + /* Insert trailing fenceposts */ + for (;;) { + mchunkptr nextp = chunk_plus_offset(p, SIZE_T_SIZE); + p->head = FENCEPOST_HEAD; + ++nfences; + if ((char*)(&(nextp->head)) < old_end) + p = nextp; + else + break; + } + assert(nfences >= 2); + + /* Insert the rest of old top into a bin as an ordinary free chunk */ + if (csp != old_top) { + mchunkptr q = (mchunkptr)old_top; + size_t psize = csp - old_top; + mchunkptr tn = chunk_plus_offset(q, psize); + set_free_with_pinuse(q, psize, tn); + insert_chunk(m, q, psize); + } + + check_top_chunk(m, m->top); +} + +/* -------------------------- System allocation -------------------------- */ + +/* Get memory from system using MORECORE or MMAP */ +static void* sys_alloc(mstate m, size_t nb) { + char* tbase = CMFAIL; + size_t tsize = 0; + flag_t mmap_flag = 0; + + ensure_initialization(); + + /* Directly map large chunks, but only if already initialized */ + if (use_mmap(m) && nb >= mparams.mmap_threshold && m->topsize != 0) { + void* mem = mmap_alloc(m, nb); + if (mem != 0) + return mem; + } + + /* + Try getting memory in any of three ways (in most-preferred to + least-preferred order): + 1. A call to MORECORE that can normally contiguously extend memory. + (disabled if not MORECORE_CONTIGUOUS or not HAVE_MORECORE or + or main space is mmapped or a previous contiguous call failed) + 2. A call to MMAP new space (disabled if not HAVE_MMAP). + Note that under the default settings, if MORECORE is unable to + fulfill a request, and HAVE_MMAP is true, then mmap is + used as a noncontiguous system allocator. This is a useful backup + strategy for systems with holes in address spaces -- in this case + sbrk cannot contiguously expand the heap, but mmap may be able to + find space. + 3. A call to MORECORE that cannot usually contiguously extend memory. + (disabled if not HAVE_MORECORE) + + In all cases, we need to request enough bytes from system to ensure + we can malloc nb bytes upon success, so pad with enough space for + top_foot, plus alignment-pad to make sure we don't lose bytes if + not on boundary, and round this up to a granularity unit. + */ + + if (MORECORE_CONTIGUOUS && !use_noncontiguous(m)) { + char* br = CMFAIL; + msegmentptr ss = (m->top == 0)? 0 : segment_holding(m, (char*)m->top); + size_t asize = 0; + ACQUIRE_MALLOC_GLOBAL_LOCK(); + + if (ss == 0) { /* First time through or recovery */ + char* base = (char*)CALL_MORECORE(0); + if (base != CMFAIL) { + asize = granularity_align(nb + SYS_ALLOC_PADDING); + /* Adjust to end on a page boundary */ + if (!is_page_aligned(base)) + asize += (page_align((size_t)base) - (size_t)base); + /* Can't call MORECORE if size is negative when treated as signed */ + if (asize < HALF_MAX_SIZE_T && + (br = (char*)(CALL_MORECORE(asize))) == base) { + tbase = base; + tsize = asize; + } + } + } + else { + /* Subtract out existing available top space from MORECORE request. */ + asize = granularity_align(nb - m->topsize + SYS_ALLOC_PADDING); + /* Use mem here only if it did continuously extend old space */ + if (asize < HALF_MAX_SIZE_T && + (br = (char*)(CALL_MORECORE(asize))) == ss->base+ss->size) { + tbase = br; + tsize = asize; + } + } + + if (tbase == CMFAIL) { /* Cope with partial failure */ + if (br != CMFAIL) { /* Try to use/extend the space we did get */ + if (asize < HALF_MAX_SIZE_T && + asize < nb + SYS_ALLOC_PADDING) { + size_t esize = granularity_align(nb + SYS_ALLOC_PADDING - asize); + if (esize < HALF_MAX_SIZE_T) { + char* end = (char*)CALL_MORECORE(esize); + if (end != CMFAIL) + asize += esize; + else { /* Can't use; try to release */ + (void) CALL_MORECORE(-asize); + br = CMFAIL; + } + } + } + } + if (br != CMFAIL) { /* Use the space we did get */ + tbase = br; + tsize = asize; + } + else + disable_contiguous(m); /* Don't try contiguous path in the future */ + } + + RELEASE_MALLOC_GLOBAL_LOCK(); + } + + if (HAVE_MMAP && tbase == CMFAIL) { /* Try MMAP */ + size_t rsize = granularity_align(nb + SYS_ALLOC_PADDING); + if (rsize > nb) { /* Fail if wraps around zero */ + char* mp = (char*)(CALL_MMAP(rsize)); + if (mp != CMFAIL) { + tbase = mp; + tsize = rsize; + mmap_flag = USE_MMAP_BIT; + } + } + } + + if (HAVE_MORECORE && tbase == CMFAIL) { /* Try noncontiguous MORECORE */ + size_t asize = granularity_align(nb + SYS_ALLOC_PADDING); + if (asize < HALF_MAX_SIZE_T) { + char* br = CMFAIL; + char* end = CMFAIL; + ACQUIRE_MALLOC_GLOBAL_LOCK(); + br = (char*)(CALL_MORECORE(asize)); + end = (char*)(CALL_MORECORE(0)); + RELEASE_MALLOC_GLOBAL_LOCK(); + if (br != CMFAIL && end != CMFAIL && br < end) { + size_t ssize = end - br; + if (ssize > nb + TOP_FOOT_SIZE) { + tbase = br; + tsize = ssize; + } + } + } + } + + if (tbase != CMFAIL) { + + if ((m->footprint += tsize) > m->max_footprint) + m->max_footprint = m->footprint; + + if (!is_initialized(m)) { /* first-time initialization */ + if (m->least_addr == 0 || tbase < m->least_addr) + m->least_addr = tbase; + m->seg.base = tbase; + m->seg.size = tsize; + m->seg.sflags = mmap_flag; + m->magic = mparams.magic; + m->release_checks = MAX_RELEASE_CHECK_RATE; + init_bins(m); +#if !ONLY_MSPACES + if (is_global(m)) + init_top(m, (mchunkptr)tbase, tsize - TOP_FOOT_SIZE); + else +#endif + { + /* Offset top by embedded malloc_state */ + mchunkptr mn = next_chunk(mem2chunk(m)); + init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) -TOP_FOOT_SIZE); + } + } + + else { + /* Try to merge with an existing segment */ + msegmentptr sp = &m->seg; + /* Only consider most recent segment if traversal suppressed */ + while (sp != 0 && tbase != sp->base + sp->size) + sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next; + if (sp != 0 && + !is_extern_segment(sp) && + (sp->sflags & USE_MMAP_BIT) == mmap_flag && + segment_holds(sp, m->top)) { /* append */ + sp->size += tsize; + init_top(m, m->top, m->topsize + tsize); + } + else { + if (tbase < m->least_addr) + m->least_addr = tbase; + sp = &m->seg; + while (sp != 0 && sp->base != tbase + tsize) + sp = (NO_SEGMENT_TRAVERSAL) ? 0 : sp->next; + if (sp != 0 && + !is_extern_segment(sp) && + (sp->sflags & USE_MMAP_BIT) == mmap_flag) { + char* oldbase = sp->base; + sp->base = tbase; + sp->size += tsize; + return prepend_alloc(m, tbase, oldbase, nb); + } + else + add_segment(m, tbase, tsize, mmap_flag); + } + } + + if (nb < m->topsize) { /* Allocate from new or extended top space */ + size_t rsize = m->topsize -= nb; + mchunkptr p = m->top; + mchunkptr r = m->top = chunk_plus_offset(p, nb); + r->head = rsize | PINUSE_BIT; + set_size_and_pinuse_of_inuse_chunk(m, p, nb); + check_top_chunk(m, m->top); + check_malloced_chunk(m, chunk2mem(p), nb); + return chunk2mem(p); + } + } + + MALLOC_FAILURE_ACTION; + return 0; +} + +/* ----------------------- system deallocation -------------------------- */ + +/* Unmap and unlink any mmapped segments that don't contain used chunks */ +static size_t release_unused_segments(mstate m) { + size_t released = 0; + int nsegs = 0; + msegmentptr pred = &m->seg; + msegmentptr sp = pred->next; + while (sp != 0) { + char* base = sp->base; + size_t size = sp->size; + msegmentptr next = sp->next; + ++nsegs; + if (is_mmapped_segment(sp) && !is_extern_segment(sp)) { + mchunkptr p = align_as_chunk(base); + size_t psize = chunksize(p); + /* Can unmap if first chunk holds entire segment and not pinned */ + if (!is_inuse(p) && (char*)p + psize >= base + size - TOP_FOOT_SIZE) { + tchunkptr tp = (tchunkptr)p; + assert(segment_holds(sp, (char*)sp)); + if (p == m->dv) { + m->dv = 0; + m->dvsize = 0; + } + else { + unlink_large_chunk(m, tp); + } + if (CALL_MUNMAP(base, size) == 0) { + released += size; + m->footprint -= size; + /* unlink obsoleted record */ + sp = pred; + sp->next = next; + } + else { /* back out if cannot unmap */ + insert_large_chunk(m, tp, psize); + } + } + } + if (NO_SEGMENT_TRAVERSAL) /* scan only first segment */ + break; + pred = sp; + sp = next; + } + /* Reset check counter */ + m->release_checks = ((nsegs > MAX_RELEASE_CHECK_RATE)? + nsegs : MAX_RELEASE_CHECK_RATE); + return released; +} + +static int sys_trim(mstate m, size_t pad) { + size_t released = 0; + ensure_initialization(); + if (pad < MAX_REQUEST && is_initialized(m)) { + pad += TOP_FOOT_SIZE; /* ensure enough room for segment overhead */ + + if (m->topsize > pad) { + /* Shrink top space in granularity-size units, keeping at least one */ + size_t unit = mparams.granularity; + size_t extra = ((m->topsize - pad + (unit - SIZE_T_ONE)) / unit - + SIZE_T_ONE) * unit; + msegmentptr sp = segment_holding(m, (char*)m->top); + + if (!is_extern_segment(sp)) { + if (is_mmapped_segment(sp)) { + if (HAVE_MMAP && + sp->size >= extra && + !has_segment_link(m, sp)) { /* can't shrink if pinned */ + size_t newsize = sp->size - extra; + /* Prefer mremap, fall back to munmap */ + if ((CALL_MREMAP(sp->base, sp->size, newsize, 0) != MFAIL) || + (CALL_MUNMAP(sp->base + newsize, extra) == 0)) { + released = extra; + } + } + } + else if (HAVE_MORECORE) { + if (extra >= HALF_MAX_SIZE_T) /* Avoid wrapping negative */ + extra = (HALF_MAX_SIZE_T) + SIZE_T_ONE - unit; + ACQUIRE_MALLOC_GLOBAL_LOCK(); + { + /* Make sure end of memory is where we last set it. */ + char* old_br = (char*)(CALL_MORECORE(0)); + if (old_br == sp->base + sp->size) { + char* rel_br = (char*)(CALL_MORECORE(-extra)); + char* new_br = (char*)(CALL_MORECORE(0)); + if (rel_br != CMFAIL && new_br < old_br) + released = old_br - new_br; + } + } + RELEASE_MALLOC_GLOBAL_LOCK(); + } + } + + if (released != 0) { + sp->size -= released; + m->footprint -= released; + init_top(m, m->top, m->topsize - released); + check_top_chunk(m, m->top); + } + } + + /* Unmap any unused mmapped segments */ + if (HAVE_MMAP) + released += release_unused_segments(m); + + /* On failure, disable autotrim to avoid repeated failed future calls */ + if (released == 0 && m->topsize > m->trim_check) + m->trim_check = MAX_SIZE_T; + } + + return (released != 0)? 1 : 0; +} + + +/* ---------------------------- malloc support --------------------------- */ + +/* allocate a large request from the best fitting chunk in a treebin */ +static void* tmalloc_large(mstate m, size_t nb) { + tchunkptr v = 0; + size_t rsize = -nb; /* Unsigned negation */ + tchunkptr t; + bindex_t idx; + compute_tree_index(nb, idx); + if ((t = *treebin_at(m, idx)) != 0) { + /* Traverse tree for this bin looking for node with size == nb */ + size_t sizebits = nb << leftshift_for_tree_index(idx); + tchunkptr rst = 0; /* The deepest untaken right subtree */ + for (;;) { + tchunkptr rt; + size_t trem = chunksize(t) - nb; + if (trem < rsize) { + v = t; + if ((rsize = trem) == 0) + break; + } + rt = t->child[1]; + t = t->child[(sizebits >> (SIZE_T_BITSIZE-SIZE_T_ONE)) & 1]; + if (rt != 0 && rt != t) + rst = rt; + if (t == 0) { + t = rst; /* set t to least subtree holding sizes > nb */ + break; + } + sizebits <<= 1; + } + } + if (t == 0 && v == 0) { /* set t to root of next non-empty treebin */ + binmap_t leftbits = left_bits(idx2bit(idx)) & m->treemap; + if (leftbits != 0) { + bindex_t i; + binmap_t leastbit = least_bit(leftbits); + compute_bit2idx(leastbit, i); + t = *treebin_at(m, i); + } + } + + while (t != 0) { /* find smallest of tree or subtree */ + size_t trem = chunksize(t) - nb; + if (trem < rsize) { + rsize = trem; + v = t; + } + t = leftmost_child(t); + } + + /* If dv is a better fit, return 0 so malloc will use it */ + if (v != 0 && rsize < (size_t)(m->dvsize - nb)) { + if (RTCHECK(ok_address(m, v))) { /* split */ + mchunkptr r = chunk_plus_offset(v, nb); + assert(chunksize(v) == rsize + nb); + if (RTCHECK(ok_next(v, r))) { + unlink_large_chunk(m, v); + if (rsize < MIN_CHUNK_SIZE) + set_inuse_and_pinuse(m, v, (rsize + nb)); + else { + set_size_and_pinuse_of_inuse_chunk(m, v, nb); + set_size_and_pinuse_of_free_chunk(r, rsize); + insert_chunk(m, r, rsize); + } + return chunk2mem(v); + } + } + CORRUPTION_ERROR_ACTION(m); + } + return 0; +} + +/* allocate a small request from the best fitting chunk in a treebin */ +static void* tmalloc_small(mstate m, size_t nb) { + tchunkptr t, v; + size_t rsize; + bindex_t i; + binmap_t leastbit = least_bit(m->treemap); + compute_bit2idx(leastbit, i); + v = t = *treebin_at(m, i); + rsize = chunksize(t) - nb; + + while ((t = leftmost_child(t)) != 0) { + size_t trem = chunksize(t) - nb; + if (trem < rsize) { + rsize = trem; + v = t; + } + } + + if (RTCHECK(ok_address(m, v))) { + mchunkptr r = chunk_plus_offset(v, nb); + assert(chunksize(v) == rsize + nb); + if (RTCHECK(ok_next(v, r))) { + unlink_large_chunk(m, v); + if (rsize < MIN_CHUNK_SIZE) + set_inuse_and_pinuse(m, v, (rsize + nb)); + else { + set_size_and_pinuse_of_inuse_chunk(m, v, nb); + set_size_and_pinuse_of_free_chunk(r, rsize); + replace_dv(m, r, rsize); + } + return chunk2mem(v); + } + } + + CORRUPTION_ERROR_ACTION(m); + return 0; +} + +/* --------------------------- realloc support --------------------------- */ + +static void* internal_realloc(mstate m, void* oldmem, size_t bytes) { + if (bytes >= MAX_REQUEST) { + MALLOC_FAILURE_ACTION; + return 0; + } + if (!PREACTION(m)) { + mchunkptr oldp = mem2chunk(oldmem); + size_t oldsize = chunksize(oldp); + mchunkptr next = chunk_plus_offset(oldp, oldsize); + mchunkptr newp = 0; + void* extra = 0; + + /* Try to either shrink or extend into top. Else malloc-copy-free */ + + if (RTCHECK(ok_address(m, oldp) && ok_inuse(oldp) && + ok_next(oldp, next) && ok_pinuse(next))) { + size_t nb = request2size(bytes); + if (is_mmapped(oldp)) + newp = mmap_resize(m, oldp, nb); + else if (oldsize >= nb) { /* already big enough */ + size_t rsize = oldsize - nb; + newp = oldp; + if (rsize >= MIN_CHUNK_SIZE) { + mchunkptr remainder = chunk_plus_offset(newp, nb); + set_inuse(m, newp, nb); + set_inuse_and_pinuse(m, remainder, rsize); + extra = chunk2mem(remainder); + } + } + else if (next == m->top && oldsize + m->topsize > nb) { + /* Expand into top */ + size_t newsize = oldsize + m->topsize; + size_t newtopsize = newsize - nb; + mchunkptr newtop = chunk_plus_offset(oldp, nb); + set_inuse(m, oldp, nb); + newtop->head = newtopsize |PINUSE_BIT; + m->top = newtop; + m->topsize = newtopsize; + newp = oldp; + } + } + else { + USAGE_ERROR_ACTION(m, oldmem); + POSTACTION(m); + return 0; + } +#if DEBUG + if (newp != 0) { + check_inuse_chunk(m, newp); /* Check requires lock */ + } +#endif + + POSTACTION(m); + + if (newp != 0) { + if (extra != 0) { + internal_free(m, extra); + } + return chunk2mem(newp); + } + else { + void* newmem = internal_malloc(m, bytes); + if (newmem != 0) { + size_t oc = oldsize - overhead_for(oldp); + memcpy(newmem, oldmem, (oc < bytes)? oc : bytes); + internal_free(m, oldmem); + } + return newmem; + } + } + return 0; +} + +/* --------------------------- memalign support -------------------------- */ + +static void* internal_memalign(mstate m, size_t alignment, size_t bytes) { + if (alignment <= MALLOC_ALIGNMENT) /* Can just use malloc */ + return internal_malloc(m, bytes); + if (alignment < MIN_CHUNK_SIZE) /* must be at least a minimum chunk size */ + alignment = MIN_CHUNK_SIZE; + if ((alignment & (alignment-SIZE_T_ONE)) != 0) {/* Ensure a power of 2 */ + size_t a = MALLOC_ALIGNMENT << 1; + while (a < alignment) a <<= 1; + alignment = a; + } + + if (bytes >= MAX_REQUEST - alignment) { + if (m != 0) { /* Test isn't needed but avoids compiler warning */ + MALLOC_FAILURE_ACTION; + } + } + else { + size_t nb = request2size(bytes); + size_t req = nb + alignment + MIN_CHUNK_SIZE - CHUNK_OVERHEAD; + char* mem = (char*)internal_malloc(m, req); + if (mem != 0) { + void* leader = 0; + void* trailer = 0; + mchunkptr p = mem2chunk(mem); + + if (PREACTION(m)) return 0; + if ((((size_t)(mem)) % alignment) != 0) { /* misaligned */ + /* + Find an aligned spot inside chunk. Since we need to give + back leading space in a chunk of at least MIN_CHUNK_SIZE, if + the first calculation places us at a spot with less than + MIN_CHUNK_SIZE leader, we can move to the next aligned spot. + We've allocated enough total room so that this is always + possible. + */ + char* br = (char*)mem2chunk((size_t)(((size_t)(mem + + alignment - + SIZE_T_ONE)) & + -alignment)); + char* pos = ((size_t)(br - (char*)(p)) >= MIN_CHUNK_SIZE)? + br : br+alignment; + mchunkptr newp = (mchunkptr)pos; + size_t leadsize = pos - (char*)(p); + size_t newsize = chunksize(p) - leadsize; + + if (is_mmapped(p)) { /* For mmapped chunks, just adjust offset */ + newp->prev_foot = p->prev_foot + leadsize; + newp->head = newsize; + } + else { /* Otherwise, give back leader, use the rest */ + set_inuse(m, newp, newsize); + set_inuse(m, p, leadsize); + leader = chunk2mem(p); + } + p = newp; + } + + /* Give back spare room at the end */ + if (!is_mmapped(p)) { + size_t size = chunksize(p); + if (size > nb + MIN_CHUNK_SIZE) { + size_t remainder_size = size - nb; + mchunkptr remainder = chunk_plus_offset(p, nb); + set_inuse(m, p, nb); + set_inuse(m, remainder, remainder_size); + trailer = chunk2mem(remainder); + } + } + + assert (chunksize(p) >= nb); + assert((((size_t)(chunk2mem(p))) % alignment) == 0); + check_inuse_chunk(m, p); + POSTACTION(m); + if (leader != 0) { + internal_free(m, leader); + } + if (trailer != 0) { + internal_free(m, trailer); + } + return chunk2mem(p); + } + } + return 0; +} + +/* ------------------------ comalloc/coalloc support --------------------- */ + +static void** ialloc(mstate m, + size_t n_elements, + size_t* sizes, + int opts, + void* chunks[]) { + /* + This provides common support for independent_X routines, handling + all of the combinations that can result. + + The opts arg has: + bit 0 set if all elements are same size (using sizes[0]) + bit 1 set if elements should be zeroed + */ + + size_t element_size; /* chunksize of each element, if all same */ + size_t contents_size; /* total size of elements */ + size_t array_size; /* request size of pointer array */ + void* mem; /* malloced aggregate space */ + mchunkptr p; /* corresponding chunk */ + size_t remainder_size; /* remaining bytes while splitting */ + void** marray; /* either "chunks" or malloced ptr array */ + mchunkptr array_chunk; /* chunk for malloced ptr array */ + flag_t was_enabled; /* to disable mmap */ + size_t size; + size_t i; + + ensure_initialization(); + /* compute array length, if needed */ + if (chunks != 0) { + if (n_elements == 0) + return chunks; /* nothing to do */ + marray = chunks; + array_size = 0; + } + else { + /* if empty req, must still return chunk representing empty array */ + if (n_elements == 0) + return (void**)internal_malloc(m, 0); + marray = 0; + array_size = request2size(n_elements * (sizeof(void*))); + } + + /* compute total element size */ + if (opts & 0x1) { /* all-same-size */ + element_size = request2size(*sizes); + contents_size = n_elements * element_size; + } + else { /* add up all the sizes */ + element_size = 0; + contents_size = 0; + for (i = 0; i != n_elements; ++i) + contents_size += request2size(sizes[i]); + } + + size = contents_size + array_size; + + /* + Allocate the aggregate chunk. First disable direct-mmapping so + malloc won't use it, since we would not be able to later + free/realloc space internal to a segregated mmap region. + */ + was_enabled = use_mmap(m); + disable_mmap(m); + mem = internal_malloc(m, size - CHUNK_OVERHEAD); + if (was_enabled) + enable_mmap(m); + if (mem == 0) + return 0; + + if (PREACTION(m)) return 0; + p = mem2chunk(mem); + remainder_size = chunksize(p); + + assert(!is_mmapped(p)); + + if (opts & 0x2) { /* optionally clear the elements */ + memset((size_t*)mem, 0, remainder_size - SIZE_T_SIZE - array_size); + } + + /* If not provided, allocate the pointer array as final part of chunk */ + if (marray == 0) { + size_t array_chunk_size; + array_chunk = chunk_plus_offset(p, contents_size); + array_chunk_size = remainder_size - contents_size; + marray = (void**) (chunk2mem(array_chunk)); + set_size_and_pinuse_of_inuse_chunk(m, array_chunk, array_chunk_size); + remainder_size = contents_size; + } + + /* split out elements */ + for (i = 0; ; ++i) { + marray[i] = chunk2mem(p); + if (i != n_elements-1) { + if (element_size != 0) + size = element_size; + else + size = request2size(sizes[i]); + remainder_size -= size; + set_size_and_pinuse_of_inuse_chunk(m, p, size); + p = chunk_plus_offset(p, size); + } + else { /* the final element absorbs any overallocation slop */ + set_size_and_pinuse_of_inuse_chunk(m, p, remainder_size); + break; + } + } + +#if DEBUG + if (marray != chunks) { + /* final element must have exactly exhausted chunk */ + if (element_size != 0) { + assert(remainder_size == element_size); + } + else { + assert(remainder_size == request2size(sizes[i])); + } + check_inuse_chunk(m, mem2chunk(marray)); + } + for (i = 0; i != n_elements; ++i) + check_inuse_chunk(m, mem2chunk(marray[i])); + +#endif /* DEBUG */ + + POSTACTION(m); + return marray; +} + + +/* -------------------------- public routines ---------------------------- */ + +#if !ONLY_MSPACES + +void* dlmalloc(size_t bytes) { + /* + Basic algorithm: + If a small request (< 256 bytes minus per-chunk overhead): + 1. If one exists, use a remainderless chunk in associated smallbin. + (Remainderless means that there are too few excess bytes to + represent as a chunk.) + 2. If it is big enough, use the dv chunk, which is normally the + chunk adjacent to the one used for the most recent small request. + 3. If one exists, split the smallest available chunk in a bin, + saving remainder in dv. + 4. If it is big enough, use the top chunk. + 5. If available, get memory from system and use it + Otherwise, for a large request: + 1. Find the smallest available binned chunk that fits, and use it + if it is better fitting than dv chunk, splitting if necessary. + 2. If better fitting than any binned chunk, use the dv chunk. + 3. If it is big enough, use the top chunk. + 4. If request size >= mmap threshold, try to directly mmap this chunk. + 5. If available, get memory from system and use it + + The ugly goto's here ensure that postaction occurs along all paths. + */ + +#if USE_LOCKS + ensure_initialization(); /* initialize in sys_alloc if not using locks */ +#endif + + if (!PREACTION(gm)) { + void* mem; + size_t nb; + if (bytes <= MAX_SMALL_REQUEST) { + bindex_t idx; + binmap_t smallbits; + nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes); + idx = small_index(nb); + smallbits = gm->smallmap >> idx; + + if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */ + mchunkptr b, p; + idx += ~smallbits & 1; /* Uses next bin if idx empty */ + b = smallbin_at(gm, idx); + p = b->fd; + assert(chunksize(p) == small_index2size(idx)); + unlink_first_small_chunk(gm, b, p, idx); + set_inuse_and_pinuse(gm, p, small_index2size(idx)); + mem = chunk2mem(p); + check_malloced_chunk(gm, mem, nb); + goto postaction; + } + + else if (nb > gm->dvsize) { + if (smallbits != 0) { /* Use chunk in next nonempty smallbin */ + mchunkptr b, p, r; + size_t rsize; + bindex_t i; + binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx)); + binmap_t leastbit = least_bit(leftbits); + compute_bit2idx(leastbit, i); + b = smallbin_at(gm, i); + p = b->fd; + assert(chunksize(p) == small_index2size(i)); + unlink_first_small_chunk(gm, b, p, i); + rsize = small_index2size(i) - nb; + /* Fit here cannot be remainderless if 4byte sizes */ + if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE) + set_inuse_and_pinuse(gm, p, small_index2size(i)); + else { + set_size_and_pinuse_of_inuse_chunk(gm, p, nb); + r = chunk_plus_offset(p, nb); + set_size_and_pinuse_of_free_chunk(r, rsize); + replace_dv(gm, r, rsize); + } + mem = chunk2mem(p); + check_malloced_chunk(gm, mem, nb); + goto postaction; + } + + else if (gm->treemap != 0 && (mem = tmalloc_small(gm, nb)) != 0) { + check_malloced_chunk(gm, mem, nb); + goto postaction; + } + } + } + else if (bytes >= MAX_REQUEST) + nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */ + else { + nb = pad_request(bytes); + if (gm->treemap != 0 && (mem = tmalloc_large(gm, nb)) != 0) { + check_malloced_chunk(gm, mem, nb); + goto postaction; + } + } + + if (nb <= gm->dvsize) { + size_t rsize = gm->dvsize - nb; + mchunkptr p = gm->dv; + if (rsize >= MIN_CHUNK_SIZE) { /* split dv */ + mchunkptr r = gm->dv = chunk_plus_offset(p, nb); + gm->dvsize = rsize; + set_size_and_pinuse_of_free_chunk(r, rsize); + set_size_and_pinuse_of_inuse_chunk(gm, p, nb); + } + else { /* exhaust dv */ + size_t dvs = gm->dvsize; + gm->dvsize = 0; + gm->dv = 0; + set_inuse_and_pinuse(gm, p, dvs); + } + mem = chunk2mem(p); + check_malloced_chunk(gm, mem, nb); + goto postaction; + } + + else if (nb < gm->topsize) { /* Split top */ + size_t rsize = gm->topsize -= nb; + mchunkptr p = gm->top; + mchunkptr r = gm->top = chunk_plus_offset(p, nb); + r->head = rsize | PINUSE_BIT; + set_size_and_pinuse_of_inuse_chunk(gm, p, nb); + mem = chunk2mem(p); + check_top_chunk(gm, gm->top); + check_malloced_chunk(gm, mem, nb); + goto postaction; + } + + mem = sys_alloc(gm, nb); + + postaction: + POSTACTION(gm); + return mem; + } + + return 0; +} + +void dlfree(void* mem) { + /* + Consolidate freed chunks with preceeding or succeeding bordering + free chunks, if they exist, and then place in a bin. Intermixed + with special cases for top, dv, mmapped chunks, and usage errors. + */ + + if (mem != 0) { + mchunkptr p = mem2chunk(mem); +#if FOOTERS + mstate fm = get_mstate_for(p); + if (!ok_magic(fm)) { + USAGE_ERROR_ACTION(fm, p); + return; + } +#else /* FOOTERS */ +#define fm gm +#endif /* FOOTERS */ + if (!PREACTION(fm)) { + check_inuse_chunk(fm, p); + if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) { + size_t psize = chunksize(p); + mchunkptr next = chunk_plus_offset(p, psize); + if (!pinuse(p)) { + size_t prevsize = p->prev_foot; + if (is_mmapped(p)) { + psize += prevsize + MMAP_FOOT_PAD; + if (CALL_MUNMAP((char*)p - prevsize, psize) == 0) + fm->footprint -= psize; + goto postaction; + } + else { + mchunkptr prev = chunk_minus_offset(p, prevsize); + psize += prevsize; + p = prev; + if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */ + if (p != fm->dv) { + unlink_chunk(fm, p, prevsize); + } + else if ((next->head & INUSE_BITS) == INUSE_BITS) { + fm->dvsize = psize; + set_free_with_pinuse(p, psize, next); + goto postaction; + } + } + else + goto erroraction; + } + } + + if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) { + if (!cinuse(next)) { /* consolidate forward */ + if (next == fm->top) { + size_t tsize = fm->topsize += psize; + fm->top = p; + p->head = tsize | PINUSE_BIT; + if (p == fm->dv) { + fm->dv = 0; + fm->dvsize = 0; + } + if (should_trim(fm, tsize)) + sys_trim(fm, 0); + goto postaction; + } + else if (next == fm->dv) { + size_t dsize = fm->dvsize += psize; + fm->dv = p; + set_size_and_pinuse_of_free_chunk(p, dsize); + goto postaction; + } + else { + size_t nsize = chunksize(next); + psize += nsize; + unlink_chunk(fm, next, nsize); + set_size_and_pinuse_of_free_chunk(p, psize); + if (p == fm->dv) { + fm->dvsize = psize; + goto postaction; + } + } + } + else + set_free_with_pinuse(p, psize, next); + + if (is_small(psize)) { + insert_small_chunk(fm, p, psize); + check_free_chunk(fm, p); + } + else { + tchunkptr tp = (tchunkptr)p; + insert_large_chunk(fm, tp, psize); + check_free_chunk(fm, p); + if (--fm->release_checks == 0) + release_unused_segments(fm); + } + goto postaction; + } + } + erroraction: + USAGE_ERROR_ACTION(fm, p); + postaction: + POSTACTION(fm); + } + } +#if !FOOTERS +#undef fm +#endif /* FOOTERS */ +} + +void* dlcalloc(size_t n_elements, size_t elem_size) { + void* mem; + size_t req = 0; + if (n_elements != 0) { + req = n_elements * elem_size; + if (((n_elements | elem_size) & ~(size_t)0xffff) && + (req / n_elements != elem_size)) + req = MAX_SIZE_T; /* force downstream failure on overflow */ + } + mem = dlmalloc(req); + if (mem != 0 && calloc_must_clear(mem2chunk(mem))) + memset(mem, 0, req); + return mem; +} + +void* dlrealloc(void* oldmem, size_t bytes) { + if (oldmem == 0) + return dlmalloc(bytes); +#ifdef REALLOC_ZERO_BYTES_FREES + if (bytes == 0) { + dlfree(oldmem); + return 0; + } +#endif /* REALLOC_ZERO_BYTES_FREES */ + else { +#if ! FOOTERS + mstate m = gm; +#else /* FOOTERS */ + mstate m = get_mstate_for(mem2chunk(oldmem)); + if (!ok_magic(m)) { + USAGE_ERROR_ACTION(m, oldmem); + return 0; + } +#endif /* FOOTERS */ + return internal_realloc(m, oldmem, bytes); + } +} + +void* dlmemalign(size_t alignment, size_t bytes) { + return internal_memalign(gm, alignment, bytes); +} + +void** dlindependent_calloc(size_t n_elements, size_t elem_size, + void* chunks[]) { + size_t sz = elem_size; /* serves as 1-element array */ + return ialloc(gm, n_elements, &sz, 3, chunks); +} + +void** dlindependent_comalloc(size_t n_elements, size_t sizes[], + void* chunks[]) { + return ialloc(gm, n_elements, sizes, 0, chunks); +} + +void* dlvalloc(size_t bytes) { + size_t pagesz; + ensure_initialization(); + pagesz = mparams.page_size; + return dlmemalign(pagesz, bytes); +} + +void* dlpvalloc(size_t bytes) { + size_t pagesz; + ensure_initialization(); + pagesz = mparams.page_size; + return dlmemalign(pagesz, (bytes + pagesz - SIZE_T_ONE) & ~(pagesz - SIZE_T_ONE)); +} + +int dlmalloc_trim(size_t pad) { + int result = 0; + ensure_initialization(); + if (!PREACTION(gm)) { + result = sys_trim(gm, pad); + POSTACTION(gm); + } + return result; +} + +size_t dlmalloc_footprint(void) { + return gm->footprint; +} + +size_t dlmalloc_max_footprint(void) { + return gm->max_footprint; +} + +#if !NO_MALLINFO +struct mallinfo dlmallinfo(void) { + return internal_mallinfo(gm); +} +#endif /* NO_MALLINFO */ + +void dlmalloc_stats() { + internal_malloc_stats(gm); +} + +int dlmallopt(int param_number, int value) { + return change_mparam(param_number, value); +} + +#endif /* !ONLY_MSPACES */ + +size_t dlmalloc_usable_size(void* mem) { + if (mem != 0) { + mchunkptr p = mem2chunk(mem); + if (is_inuse(p)) + return chunksize(p) - overhead_for(p); + } + return 0; +} + +/* ----------------------------- user mspaces ---------------------------- */ + +#if MSPACES + +static mstate init_user_mstate(char* tbase, size_t tsize) { + size_t msize = pad_request(sizeof(struct malloc_state)); + mchunkptr mn; + mchunkptr msp = align_as_chunk(tbase); + mstate m = (mstate)(chunk2mem(msp)); + memset(m, 0, msize); + INITIAL_LOCK(&m->mutex); + msp->head = (msize|INUSE_BITS); + m->seg.base = m->least_addr = tbase; + m->seg.size = m->footprint = m->max_footprint = tsize; + m->magic = mparams.magic; + m->release_checks = MAX_RELEASE_CHECK_RATE; + m->mflags = mparams.default_mflags; + m->extp = 0; + m->exts = 0; + disable_contiguous(m); + init_bins(m); + mn = next_chunk(mem2chunk(m)); + init_top(m, mn, (size_t)((tbase + tsize) - (char*)mn) - TOP_FOOT_SIZE); + check_top_chunk(m, m->top); + return m; +} + +mspace create_mspace(size_t capacity, int locked) { + mstate m = 0; + size_t msize; + ensure_initialization(); + msize = pad_request(sizeof(struct malloc_state)); + if (capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) { + size_t rs = ((capacity == 0)? mparams.granularity : + (capacity + TOP_FOOT_SIZE + msize)); + size_t tsize = granularity_align(rs); + char* tbase = (char*)(CALL_MMAP(tsize)); + if (tbase != CMFAIL) { + m = init_user_mstate(tbase, tsize); + m->seg.sflags = USE_MMAP_BIT; + set_lock(m, locked); + } + } + return (mspace)m; +} + +mspace create_mspace_with_base(void* base, size_t capacity, int locked) { + mstate m = 0; + size_t msize; + ensure_initialization(); + msize = pad_request(sizeof(struct malloc_state)); + if (capacity > msize + TOP_FOOT_SIZE && + capacity < (size_t) -(msize + TOP_FOOT_SIZE + mparams.page_size)) { + m = init_user_mstate((char*)base, capacity); + m->seg.sflags = EXTERN_BIT; + set_lock(m, locked); + } + return (mspace)m; +} + +int mspace_track_large_chunks(mspace msp, int enable) { + int ret = 0; + mstate ms = (mstate)msp; + if (!PREACTION(ms)) { + if (!use_mmap(ms)) + ret = 1; + if (!enable) + enable_mmap(ms); + else + disable_mmap(ms); + POSTACTION(ms); + } + return ret; +} + +size_t destroy_mspace(mspace msp) { + size_t freed = 0; + mstate ms = (mstate)msp; + if (ok_magic(ms)) { + msegmentptr sp = &ms->seg; + while (sp != 0) { + char* base = sp->base; + size_t size = sp->size; + flag_t flag = sp->sflags; + sp = sp->next; + if ((flag & USE_MMAP_BIT) && !(flag & EXTERN_BIT) && + CALL_MUNMAP(base, size) == 0) + freed += size; + } + } + else { + USAGE_ERROR_ACTION(ms,ms); + } + return freed; +} + +/* + mspace versions of routines are near-clones of the global + versions. This is not so nice but better than the alternatives. +*/ + + +void* mspace_malloc(mspace msp, size_t bytes) { + mstate ms = (mstate)msp; + if (!ok_magic(ms)) { + USAGE_ERROR_ACTION(ms,ms); + return 0; + } + if (!PREACTION(ms)) { + void* mem; + size_t nb; + if (bytes <= MAX_SMALL_REQUEST) { + bindex_t idx; + binmap_t smallbits; + nb = (bytes < MIN_REQUEST)? MIN_CHUNK_SIZE : pad_request(bytes); + idx = small_index(nb); + smallbits = ms->smallmap >> idx; + + if ((smallbits & 0x3U) != 0) { /* Remainderless fit to a smallbin. */ + mchunkptr b, p; + idx += ~smallbits & 1; /* Uses next bin if idx empty */ + b = smallbin_at(ms, idx); + p = b->fd; + assert(chunksize(p) == small_index2size(idx)); + unlink_first_small_chunk(ms, b, p, idx); + set_inuse_and_pinuse(ms, p, small_index2size(idx)); + mem = chunk2mem(p); + check_malloced_chunk(ms, mem, nb); + goto postaction; + } + + else if (nb > ms->dvsize) { + if (smallbits != 0) { /* Use chunk in next nonempty smallbin */ + mchunkptr b, p, r; + size_t rsize; + bindex_t i; + binmap_t leftbits = (smallbits << idx) & left_bits(idx2bit(idx)); + binmap_t leastbit = least_bit(leftbits); + compute_bit2idx(leastbit, i); + b = smallbin_at(ms, i); + p = b->fd; + assert(chunksize(p) == small_index2size(i)); + unlink_first_small_chunk(ms, b, p, i); + rsize = small_index2size(i) - nb; + /* Fit here cannot be remainderless if 4byte sizes */ + if (SIZE_T_SIZE != 4 && rsize < MIN_CHUNK_SIZE) + set_inuse_and_pinuse(ms, p, small_index2size(i)); + else { + set_size_and_pinuse_of_inuse_chunk(ms, p, nb); + r = chunk_plus_offset(p, nb); + set_size_and_pinuse_of_free_chunk(r, rsize); + replace_dv(ms, r, rsize); + } + mem = chunk2mem(p); + check_malloced_chunk(ms, mem, nb); + goto postaction; + } + + else if (ms->treemap != 0 && (mem = tmalloc_small(ms, nb)) != 0) { + check_malloced_chunk(ms, mem, nb); + goto postaction; + } + } + } + else if (bytes >= MAX_REQUEST) + nb = MAX_SIZE_T; /* Too big to allocate. Force failure (in sys alloc) */ + else { + nb = pad_request(bytes); + if (ms->treemap != 0 && (mem = tmalloc_large(ms, nb)) != 0) { + check_malloced_chunk(ms, mem, nb); + goto postaction; + } + } + + if (nb <= ms->dvsize) { + size_t rsize = ms->dvsize - nb; + mchunkptr p = ms->dv; + if (rsize >= MIN_CHUNK_SIZE) { /* split dv */ + mchunkptr r = ms->dv = chunk_plus_offset(p, nb); + ms->dvsize = rsize; + set_size_and_pinuse_of_free_chunk(r, rsize); + set_size_and_pinuse_of_inuse_chunk(ms, p, nb); + } + else { /* exhaust dv */ + size_t dvs = ms->dvsize; + ms->dvsize = 0; + ms->dv = 0; + set_inuse_and_pinuse(ms, p, dvs); + } + mem = chunk2mem(p); + check_malloced_chunk(ms, mem, nb); + goto postaction; + } + + else if (nb < ms->topsize) { /* Split top */ + size_t rsize = ms->topsize -= nb; + mchunkptr p = ms->top; + mchunkptr r = ms->top = chunk_plus_offset(p, nb); + r->head = rsize | PINUSE_BIT; + set_size_and_pinuse_of_inuse_chunk(ms, p, nb); + mem = chunk2mem(p); + check_top_chunk(ms, ms->top); + check_malloced_chunk(ms, mem, nb); + goto postaction; + } + + mem = sys_alloc(ms, nb); + + postaction: + POSTACTION(ms); + return mem; + } + + return 0; +} + +void mspace_free(mspace msp, void* mem) { + if (mem != 0) { + mchunkptr p = mem2chunk(mem); +#if FOOTERS + mstate fm = get_mstate_for(p); + msp = msp; /* placate people compiling -Wunused */ +#else /* FOOTERS */ + mstate fm = (mstate)msp; +#endif /* FOOTERS */ + if (!ok_magic(fm)) { + USAGE_ERROR_ACTION(fm, p); + return; + } + if (!PREACTION(fm)) { + check_inuse_chunk(fm, p); + if (RTCHECK(ok_address(fm, p) && ok_inuse(p))) { + size_t psize = chunksize(p); + mchunkptr next = chunk_plus_offset(p, psize); + if (!pinuse(p)) { + size_t prevsize = p->prev_foot; + if (is_mmapped(p)) { + psize += prevsize + MMAP_FOOT_PAD; + if (CALL_MUNMAP((char*)p - prevsize, psize) == 0) + fm->footprint -= psize; + goto postaction; + } + else { + mchunkptr prev = chunk_minus_offset(p, prevsize); + psize += prevsize; + p = prev; + if (RTCHECK(ok_address(fm, prev))) { /* consolidate backward */ + if (p != fm->dv) { + unlink_chunk(fm, p, prevsize); + } + else if ((next->head & INUSE_BITS) == INUSE_BITS) { + fm->dvsize = psize; + set_free_with_pinuse(p, psize, next); + goto postaction; + } + } + else + goto erroraction; + } + } + + if (RTCHECK(ok_next(p, next) && ok_pinuse(next))) { + if (!cinuse(next)) { /* consolidate forward */ + if (next == fm->top) { + size_t tsize = fm->topsize += psize; + fm->top = p; + p->head = tsize | PINUSE_BIT; + if (p == fm->dv) { + fm->dv = 0; + fm->dvsize = 0; + } + if (should_trim(fm, tsize)) + sys_trim(fm, 0); + goto postaction; + } + else if (next == fm->dv) { + size_t dsize = fm->dvsize += psize; + fm->dv = p; + set_size_and_pinuse_of_free_chunk(p, dsize); + goto postaction; + } + else { + size_t nsize = chunksize(next); + psize += nsize; + unlink_chunk(fm, next, nsize); + set_size_and_pinuse_of_free_chunk(p, psize); + if (p == fm->dv) { + fm->dvsize = psize; + goto postaction; + } + } + } + else + set_free_with_pinuse(p, psize, next); + + if (is_small(psize)) { + insert_small_chunk(fm, p, psize); + check_free_chunk(fm, p); + } + else { + tchunkptr tp = (tchunkptr)p; + insert_large_chunk(fm, tp, psize); + check_free_chunk(fm, p); + if (--fm->release_checks == 0) + release_unused_segments(fm); + } + goto postaction; + } + } + erroraction: + USAGE_ERROR_ACTION(fm, p); + postaction: + POSTACTION(fm); + } + } +} + +void* mspace_calloc(mspace msp, size_t n_elements, size_t elem_size) { + void* mem; + size_t req = 0; + mstate ms = (mstate)msp; + if (!ok_magic(ms)) { + USAGE_ERROR_ACTION(ms,ms); + return 0; + } + if (n_elements != 0) { + req = n_elements * elem_size; + if (((n_elements | elem_size) & ~(size_t)0xffff) && + (req / n_elements != elem_size)) + req = MAX_SIZE_T; /* force downstream failure on overflow */ + } + mem = internal_malloc(ms, req); + if (mem != 0 && calloc_must_clear(mem2chunk(mem))) + memset(mem, 0, req); + return mem; +} + +void* mspace_realloc(mspace msp, void* oldmem, size_t bytes) { + if (oldmem == 0) + return mspace_malloc(msp, bytes); +#ifdef REALLOC_ZERO_BYTES_FREES + if (bytes == 0) { + mspace_free(msp, oldmem); + return 0; + } +#endif /* REALLOC_ZERO_BYTES_FREES */ + else { +#if FOOTERS + mchunkptr p = mem2chunk(oldmem); + mstate ms = get_mstate_for(p); +#else /* FOOTERS */ + mstate ms = (mstate)msp; +#endif /* FOOTERS */ + if (!ok_magic(ms)) { + USAGE_ERROR_ACTION(ms,ms); + return 0; + } + return internal_realloc(ms, oldmem, bytes); + } +} + +void* mspace_memalign(mspace msp, size_t alignment, size_t bytes) { + mstate ms = (mstate)msp; + if (!ok_magic(ms)) { + USAGE_ERROR_ACTION(ms,ms); + return 0; + } + return internal_memalign(ms, alignment, bytes); +} + +void** mspace_independent_calloc(mspace msp, size_t n_elements, + size_t elem_size, void* chunks[]) { + size_t sz = elem_size; /* serves as 1-element array */ + mstate ms = (mstate)msp; + if (!ok_magic(ms)) { + USAGE_ERROR_ACTION(ms,ms); + return 0; + } + return ialloc(ms, n_elements, &sz, 3, chunks); +} + +void** mspace_independent_comalloc(mspace msp, size_t n_elements, + size_t sizes[], void* chunks[]) { + mstate ms = (mstate)msp; + if (!ok_magic(ms)) { + USAGE_ERROR_ACTION(ms,ms); + return 0; + } + return ialloc(ms, n_elements, sizes, 0, chunks); +} + +int mspace_trim(mspace msp, size_t pad) { + int result = 0; + mstate ms = (mstate)msp; + if (ok_magic(ms)) { + if (!PREACTION(ms)) { + result = sys_trim(ms, pad); + POSTACTION(ms); + } + } + else { + USAGE_ERROR_ACTION(ms,ms); + } + return result; +} + +void mspace_malloc_stats(mspace msp) { + mstate ms = (mstate)msp; + if (ok_magic(ms)) { + internal_malloc_stats(ms); + } + else { + USAGE_ERROR_ACTION(ms,ms); + } +} + +size_t mspace_footprint(mspace msp) { + size_t result = 0; + mstate ms = (mstate)msp; + if (ok_magic(ms)) { + result = ms->footprint; + } + else { + USAGE_ERROR_ACTION(ms,ms); + } + return result; +} + + +size_t mspace_max_footprint(mspace msp) { + size_t result = 0; + mstate ms = (mstate)msp; + if (ok_magic(ms)) { + result = ms->max_footprint; + } + else { + USAGE_ERROR_ACTION(ms,ms); + } + return result; +} + + +#if !NO_MALLINFO +struct mallinfo mspace_mallinfo(mspace msp) { + mstate ms = (mstate)msp; + if (!ok_magic(ms)) { + USAGE_ERROR_ACTION(ms,ms); + } + return internal_mallinfo(ms); +} +#endif /* NO_MALLINFO */ + +size_t mspace_usable_size(void* mem) { + if (mem != 0) { + mchunkptr p = mem2chunk(mem); + if (is_inuse(p)) + return chunksize(p) - overhead_for(p); + } + return 0; +} + +int mspace_mallopt(int param_number, int value) { + return change_mparam(param_number, value); +} + +#endif /* MSPACES */ + + +/* -------------------- Alternative MORECORE functions ------------------- */ + +/* + Guidelines for creating a custom version of MORECORE: + + * For best performance, MORECORE should allocate in multiples of pagesize. + * MORECORE may allocate more memory than requested. (Or even less, + but this will usually result in a malloc failure.) + * MORECORE must not allocate memory when given argument zero, but + instead return one past the end address of memory from previous + nonzero call. + * For best performance, consecutive calls to MORECORE with positive + arguments should return increasing addresses, indicating that + space has been contiguously extended. + * Even though consecutive calls to MORECORE need not return contiguous + addresses, it must be OK for malloc'ed chunks to span multiple + regions in those cases where they do happen to be contiguous. + * MORECORE need not handle negative arguments -- it may instead + just return MFAIL when given negative arguments. + Negative arguments are always multiples of pagesize. MORECORE + must not misinterpret negative args as large positive unsigned + args. You can suppress all such calls from even occurring by defining + MORECORE_CANNOT_TRIM, + + As an example alternative MORECORE, here is a custom allocator + kindly contributed for pre-OSX macOS. It uses virtually but not + necessarily physically contiguous non-paged memory (locked in, + present and won't get swapped out). You can use it by uncommenting + this section, adding some #includes, and setting up the appropriate + defines above: + + #define MORECORE osMoreCore + + There is also a shutdown routine that should somehow be called for + cleanup upon program exit. + + #define MAX_POOL_ENTRIES 100 + #define MINIMUM_MORECORE_SIZE (64 * 1024U) + static int next_os_pool; + void *our_os_pools[MAX_POOL_ENTRIES]; + + void *osMoreCore(int size) + { + void *ptr = 0; + static void *sbrk_top = 0; + + if (size > 0) + { + if (size < MINIMUM_MORECORE_SIZE) + size = MINIMUM_MORECORE_SIZE; + if (CurrentExecutionLevel() == kTaskLevel) + ptr = PoolAllocateResident(size + RM_PAGE_SIZE, 0); + if (ptr == 0) + { + return (void *) MFAIL; + } + // save ptrs so they can be freed during cleanup + our_os_pools[next_os_pool] = ptr; + next_os_pool++; + ptr = (void *) ((((size_t) ptr) + RM_PAGE_MASK) & ~RM_PAGE_MASK); + sbrk_top = (char *) ptr + size; + return ptr; + } + else if (size < 0) + { + // we don't currently support shrink behavior + return (void *) MFAIL; + } + else + { + return sbrk_top; + } + } + + // cleanup any allocated memory pools + // called as last thing before shutting down driver + + void osCleanupMem(void) + { + void **ptr; + + for (ptr = our_os_pools; ptr < &our_os_pools[MAX_POOL_ENTRIES]; ptr++) + if (*ptr) + { + PoolDeallocate(*ptr); + *ptr = 0; + } + } + +*/ + + +/* ----------------------------------------------------------------------- +History: + V2.8.4 Wed May 27 09:56:23 2009 Doug Lea (dl at gee) + * Use zeros instead of prev foot for is_mmapped + * Add mspace_track_large_chunks; thanks to Jean Brouwers + * Fix set_inuse in internal_realloc; thanks to Jean Brouwers + * Fix insufficient sys_alloc padding when using 16byte alignment + * Fix bad error check in mspace_footprint + * Adaptations for ptmalloc; thanks to Wolfram Gloger. + * Reentrant spin locks; thanks to Earl Chew and others + * Win32 improvements; thanks to Niall Douglas and Earl Chew + * Add NO_SEGMENT_TRAVERSAL and MAX_RELEASE_CHECK_RATE options + * Extension hook in malloc_state + * Various small adjustments to reduce warnings on some compilers + * Various configuration extensions/changes for more platforms. Thanks + to all who contributed these. + + V2.8.3 Thu Sep 22 11:16:32 2005 Doug Lea (dl at gee) + * Add max_footprint functions + * Ensure all appropriate literals are size_t + * Fix conditional compilation problem for some #define settings + * Avoid concatenating segments with the one provided + in create_mspace_with_base + * Rename some variables to avoid compiler shadowing warnings + * Use explicit lock initialization. + * Better handling of sbrk interference. + * Simplify and fix segment insertion, trimming and mspace_destroy + * Reinstate REALLOC_ZERO_BYTES_FREES option from 2.7.x + * Thanks especially to Dennis Flanagan for help on these. + + V2.8.2 Sun Jun 12 16:01:10 2005 Doug Lea (dl at gee) + * Fix memalign brace error. + + V2.8.1 Wed Jun 8 16:11:46 2005 Doug Lea (dl at gee) + * Fix improper #endif nesting in C++ + * Add explicit casts needed for C++ + + V2.8.0 Mon May 30 14:09:02 2005 Doug Lea (dl at gee) + * Use trees for large bins + * Support mspaces + * Use segments to unify sbrk-based and mmap-based system allocation, + removing need for emulation on most platforms without sbrk. + * Default safety checks + * Optional footer checks. Thanks to William Robertson for the idea. + * Internal code refactoring + * Incorporate suggestions and platform-specific changes. + Thanks to Dennis Flanagan, Colin Plumb, Niall Douglas, + Aaron Bachmann, Emery Berger, and others. + * Speed up non-fastbin processing enough to remove fastbins. + * Remove useless cfree() to avoid conflicts with other apps. + * Remove internal memcpy, memset. Compilers handle builtins better. + * Remove some options that no one ever used and rename others. + + V2.7.2 Sat Aug 17 09:07:30 2002 Doug Lea (dl at gee) + * Fix malloc_state bitmap array misdeclaration + + V2.7.1 Thu Jul 25 10:58:03 2002 Doug Lea (dl at gee) + * Allow tuning of FIRST_SORTED_BIN_SIZE + * Use PTR_UINT as type for all ptr->int casts. Thanks to John Belmonte. + * Better detection and support for non-contiguousness of MORECORE. + Thanks to Andreas Mueller, Conal Walsh, and Wolfram Gloger + * Bypass most of malloc if no frees. Thanks To Emery Berger. + * Fix freeing of old top non-contiguous chunk im sysmalloc. + * Raised default trim and map thresholds to 256K. + * Fix mmap-related #defines. Thanks to Lubos Lunak. + * Fix copy macros; added LACKS_FCNTL_H. Thanks to Neal Walfield. + * Branch-free bin calculation + * Default trim and mmap thresholds now 256K. + + V2.7.0 Sun Mar 11 14:14:06 2001 Doug Lea (dl at gee) + * Introduce independent_comalloc and independent_calloc. + Thanks to Michael Pachos for motivation and help. + * Make optional .h file available + * Allow > 2GB requests on 32bit systems. + * new WIN32 sbrk, mmap, munmap, lock code from <Walter@GeNeSys-e.de>. + Thanks also to Andreas Mueller <a.mueller at paradatec.de>, + and Anonymous. + * Allow override of MALLOC_ALIGNMENT (Thanks to Ruud Waij for + helping test this.) + * memalign: check alignment arg + * realloc: don't try to shift chunks backwards, since this + leads to more fragmentation in some programs and doesn't + seem to help in any others. + * Collect all cases in malloc requiring system memory into sysmalloc + * Use mmap as backup to sbrk + * Place all internal state in malloc_state + * Introduce fastbins (although similar to 2.5.1) + * Many minor tunings and cosmetic improvements + * Introduce USE_PUBLIC_MALLOC_WRAPPERS, USE_MALLOC_LOCK + * Introduce MALLOC_FAILURE_ACTION, MORECORE_CONTIGUOUS + Thanks to Tony E. Bennett <tbennett@nvidia.com> and others. + * Include errno.h to support default failure action. + + V2.6.6 Sun Dec 5 07:42:19 1999 Doug Lea (dl at gee) + * return null for negative arguments + * Added Several WIN32 cleanups from Martin C. Fong <mcfong at yahoo.com> + * Add 'LACKS_SYS_PARAM_H' for those systems without 'sys/param.h' + (e.g. WIN32 platforms) + * Cleanup header file inclusion for WIN32 platforms + * Cleanup code to avoid Microsoft Visual C++ compiler complaints + * Add 'USE_DL_PREFIX' to quickly allow co-existence with existing + memory allocation routines + * Set 'malloc_getpagesize' for WIN32 platforms (needs more work) + * Use 'assert' rather than 'ASSERT' in WIN32 code to conform to + usage of 'assert' in non-WIN32 code + * Improve WIN32 'sbrk()' emulation's 'findRegion()' routine to + avoid infinite loop + * Always call 'fREe()' rather than 'free()' + + V2.6.5 Wed Jun 17 15:57:31 1998 Doug Lea (dl at gee) + * Fixed ordering problem with boundary-stamping + + V2.6.3 Sun May 19 08:17:58 1996 Doug Lea (dl at gee) + * Added pvalloc, as recommended by H.J. Liu + * Added 64bit pointer support mainly from Wolfram Gloger + * Added anonymously donated WIN32 sbrk emulation + * Malloc, calloc, getpagesize: add optimizations from Raymond Nijssen + * malloc_extend_top: fix mask error that caused wastage after + foreign sbrks + * Add linux mremap support code from HJ Liu + + V2.6.2 Tue Dec 5 06:52:55 1995 Doug Lea (dl at gee) + * Integrated most documentation with the code. + * Add support for mmap, with help from + Wolfram Gloger (Gloger@lrz.uni-muenchen.de). + * Use last_remainder in more cases. + * Pack bins using idea from colin@nyx10.cs.du.edu + * Use ordered bins instead of best-fit threshhold + * Eliminate block-local decls to simplify tracing and debugging. + * Support another case of realloc via move into top + * Fix error occuring when initial sbrk_base not word-aligned. + * Rely on page size for units instead of SBRK_UNIT to + avoid surprises about sbrk alignment conventions. + * Add mallinfo, mallopt. Thanks to Raymond Nijssen + (raymond@es.ele.tue.nl) for the suggestion. + * Add `pad' argument to malloc_trim and top_pad mallopt parameter. + * More precautions for cases where other routines call sbrk, + courtesy of Wolfram Gloger (Gloger@lrz.uni-muenchen.de). + * Added macros etc., allowing use in linux libc from + H.J. Lu (hjl@gnu.ai.mit.edu) + * Inverted this history list + + V2.6.1 Sat Dec 2 14:10:57 1995 Doug Lea (dl at gee) + * Re-tuned and fixed to behave more nicely with V2.6.0 changes. + * Removed all preallocation code since under current scheme + the work required to undo bad preallocations exceeds + the work saved in good cases for most test programs. + * No longer use return list or unconsolidated bins since + no scheme using them consistently outperforms those that don't + given above changes. + * Use best fit for very large chunks to prevent some worst-cases. + * Added some support for debugging + + V2.6.0 Sat Nov 4 07:05:23 1995 Doug Lea (dl at gee) + * Removed footers when chunks are in use. Thanks to + Paul Wilson (wilson@cs.texas.edu) for the suggestion. + + V2.5.4 Wed Nov 1 07:54:51 1995 Doug Lea (dl at gee) + * Added malloc_trim, with help from Wolfram Gloger + (wmglo@Dent.MED.Uni-Muenchen.DE). + + V2.5.3 Tue Apr 26 10:16:01 1994 Doug Lea (dl at g) + + V2.5.2 Tue Apr 5 16:20:40 1994 Doug Lea (dl at g) + * realloc: try to expand in both directions + * malloc: swap order of clean-bin strategy; + * realloc: only conditionally expand backwards + * Try not to scavenge used bins + * Use bin counts as a guide to preallocation + * Occasionally bin return list chunks in first scan + * Add a few optimizations from colin@nyx10.cs.du.edu + + V2.5.1 Sat Aug 14 15:40:43 1993 Doug Lea (dl at g) + * faster bin computation & slightly different binning + * merged all consolidations to one part of malloc proper + (eliminating old malloc_find_space & malloc_clean_bin) + * Scan 2 returns chunks (not just 1) + * Propagate failure in realloc if malloc returns 0 + * Add stuff to allow compilation on non-ANSI compilers + from kpv@research.att.com + + V2.5 Sat Aug 7 07:41:59 1993 Doug Lea (dl at g.oswego.edu) + * removed potential for odd address access in prev_chunk + * removed dependency on getpagesize.h + * misc cosmetics and a bit more internal documentation + * anticosmetics: mangled names in macros to evade debugger strangeness + * tested on sparc, hp-700, dec-mips, rs6000 + with gcc & native cc (hp, dec only) allowing + Detlefs & Zorn comparison study (in SIGPLAN Notices.) + + Trial version Fri Aug 28 13:14:29 1992 Doug Lea (dl at g.oswego.edu) + * Based loosely on libg++-1.2X malloc. (It retains some of the overall + structure of old version, but most details differ.) + +*/ + +#endif diff --git a/drivers/nedmalloc/nedmalloc.cpp b/drivers/nedmalloc/nedmalloc.cpp index 8845d96549..9aac277a2a 100644 --- a/drivers/nedmalloc/nedmalloc.cpp +++ b/drivers/nedmalloc/nedmalloc.cpp @@ -1,1467 +1,1467 @@ -#ifdef NEDMALLOC_ENABLED
-/* Alternative malloc implementation for multiple threads without
-lock contention based on dlmalloc. (C) 2005-2009 Niall Douglas
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-#ifdef _MSC_VER
-/* Enable full aliasing on MSVC */
-/*#pragma optimize("a", on)*/
-#pragma warning(push)
-#pragma warning(disable:4100) /* unreferenced formal parameter */
-#pragma warning(disable:4127) /* conditional expression is constant */
-#pragma warning(disable:4706) /* assignment within conditional expression */
-#endif
-
-/*#define ENABLE_TOLERANT_NEDMALLOC 1*/
-/*#define ENABLE_FAST_HEAP_DETECTION 1*/
-/*#define NEDMALLOC_DEBUG 1*/
-
-/*#define FULLSANITYCHECKS*/
-/* If link time code generation is on, don't force or prevent inlining */
-#if defined(_MSC_VER) && defined(NEDMALLOC_DLL_EXPORTS)
-#define FORCEINLINE
-#define NOINLINE
-#endif
-
-
-#include "nedmalloc.h"
-#ifdef WIN32
- #include <malloc.h>
- #include <stddef.h>
-#endif
-#if USE_ALLOCATOR==1
- #define MSPACES 1
- #define ONLY_MSPACES 1
-#endif
-#define USE_DL_PREFIX 1
-#ifndef USE_LOCKS
- #define USE_LOCKS 1
-#endif
-#define FOOTERS 1 /* Need to enable footers so frees lock the right mspace */
-#ifndef NEDMALLOC_DEBUG
- #if defined(DEBUG) || defined(_DEBUG)
- #define NEDMALLOC_DEBUG 1
- #else
- #define NEDMALLOC_DEBUG 0
- #endif
-#endif
-/* We need to consistently define DEBUG=0|1, _DEBUG and NDEBUG for dlmalloc */
-#undef DEBUG
-#undef _DEBUG
-#if NEDMALLOC_DEBUG
- #define _DEBUG
- #define DEBUG 1
-#else
- #define DEBUG 0
-#endif
-#ifdef NDEBUG /* Disable assert checking on release builds */
- #undef DEBUG
- #undef _DEBUG
-#endif
-/* The default of 64Kb means we spend too much time kernel-side */
-#ifndef DEFAULT_GRANULARITY
-#define DEFAULT_GRANULARITY (1*1024*1024)
-#if DEBUG
-#define DEFAULT_GRANULARITY_ALIGNED
-#endif
-#endif
-/*#define USE_SPIN_LOCKS 0*/
-
-
-#include "malloc.c.h"
-#ifdef NDEBUG /* Disable assert checking on release builds */
- #undef DEBUG
-#elif !NEDMALLOC_DEBUG
- #ifdef __GNUC__
- #warning DEBUG is defined so allocator will run with assert checking! Define NDEBUG to run at full speed.
- #elif defined(_MSC_VER)
- #pragma message(__FILE__ ": WARNING: DEBUG is defined so allocator will run with assert checking! Define NDEBUG to run at full speed.")
- #endif
-#endif
-
-/* The maximum concurrent threads in a pool possible */
-#ifndef MAXTHREADSINPOOL
-#define MAXTHREADSINPOOL 16
-#endif
-/* The maximum number of threadcaches which can be allocated */
-#ifndef THREADCACHEMAXCACHES
-#define THREADCACHEMAXCACHES 256
-#endif
-/* The maximum size to be allocated from the thread cache */
-#ifndef THREADCACHEMAX
-#define THREADCACHEMAX 8192
-#endif
-#if 0
-/* The number of cache entries for finer grained bins. This is (topbitpos(THREADCACHEMAX)-4)*2 */
-#define THREADCACHEMAXBINS ((13-4)*2)
-#else
-/* The number of cache entries. This is (topbitpos(THREADCACHEMAX)-4) */
-#define THREADCACHEMAXBINS (13-4)
-#endif
-/* Point at which the free space in a thread cache is garbage collected */
-#ifndef THREADCACHEMAXFREESPACE
-#define THREADCACHEMAXFREESPACE (512*1024)
-#endif
-
-
-#ifdef WIN32
- #define TLSVAR DWORD
- #define TLSALLOC(k) (*(k)=TlsAlloc(), TLS_OUT_OF_INDEXES==*(k))
- #define TLSFREE(k) (!TlsFree(k))
- #define TLSGET(k) TlsGetValue(k)
- #define TLSSET(k, a) (!TlsSetValue(k, a))
- #ifdef DEBUG
-static LPVOID ChkedTlsGetValue(DWORD idx)
-{
- LPVOID ret=TlsGetValue(idx);
- assert(S_OK==GetLastError());
- return ret;
-}
- #undef TLSGET
- #define TLSGET(k) ChkedTlsGetValue(k)
- #endif
-#else
- #define TLSVAR pthread_key_t
- #define TLSALLOC(k) pthread_key_create(k, 0)
- #define TLSFREE(k) pthread_key_delete(k)
- #define TLSGET(k) pthread_getspecific(k)
- #define TLSSET(k, a) pthread_setspecific(k, a)
-#endif
-
-#if defined(__cplusplus)
-#if !defined(NO_NED_NAMESPACE)
-namespace nedalloc {
-#else
-extern "C" {
-#endif
-#endif
-
-#if USE_ALLOCATOR==0
-static void *unsupported_operation(const char *opname) THROWSPEC
-{
- fprintf(stderr, "nedmalloc: The operation %s is not supported under this build configuration\n", opname);
- abort();
- return 0;
-}
-static size_t mspacecounter=(size_t) 0xdeadbeef;
-#endif
-#ifndef ENABLE_FAST_HEAP_DETECTION
-static void *RESTRICT leastusedaddress;
-static size_t largestusedblock;
-#endif
-
-static FORCEINLINE void *CallMalloc(void *RESTRICT mspace, size_t size, size_t alignment) THROWSPEC
-{
- void *RESTRICT ret=0;
- size_t _alignment=alignment;
-#if USE_MAGIC_HEADERS
- size_t *_ret=0;
- size+=alignment+3*sizeof(size_t);
- _alignment=0;
-#endif
-#if USE_ALLOCATOR==0
- ret=_alignment ?
-#ifdef _MSC_VER
- /* This is the MSVCRT equivalent */
- _aligned_malloc(size, _alignment)
-#elif defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__)
- /* This is the glibc/ptmalloc2/dlmalloc/BSD libc equivalent. */
- memalign(_alignment, size)
-#else
-#error Cannot aligned allocate with the memory allocator of an unknown system!
-#endif
- : malloc(size);
-#elif USE_ALLOCATOR==1
- ret=_alignment ? mspace_memalign((mstate) mspace, _alignment, size) : mspace_malloc((mstate) mspace, size);
-#ifndef ENABLE_FAST_HEAP_DETECTION
- if(ret)
- {
- size_t truesize=chunksize(mem2chunk(ret));
- if(!leastusedaddress || (void *)((mstate) mspace)->least_addr<leastusedaddress) leastusedaddress=(void *)((mstate) mspace)->least_addr;
- if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1);
- }
-#endif
-#endif
- if(!ret) return 0;
-#if USE_MAGIC_HEADERS
- _ret=(size_t *) ret;
- ret=(void *)(_ret+3);
- if(alignment) ret=(void *)(((size_t) ret+alignment-1)&~(alignment-1));
- for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *)"NEDMALOC";
- _ret[0]=(size_t) mspace;
- _ret[1]=size-3*sizeof(size_t);
-#endif
- return ret;
-}
-
-static FORCEINLINE void *CallCalloc(void *RESTRICT mspace, size_t size, size_t alignment) THROWSPEC
-{
- void *RESTRICT ret=0;
-#if USE_MAGIC_HEADERS
- size_t *_ret=0;
- size+=alignment+3*sizeof(size_t);
-#endif
-#if USE_ALLOCATOR==0
- ret=calloc(1, size);
-#elif USE_ALLOCATOR==1
- ret=mspace_calloc((mstate) mspace, 1, size);
-#ifndef ENABLE_FAST_HEAP_DETECTION
- if(ret)
- {
- size_t truesize=chunksize(mem2chunk(ret));
- if(!leastusedaddress || (void *)((mstate) mspace)->least_addr<leastusedaddress) leastusedaddress=(void *)((mstate) mspace)->least_addr;
- if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1);
- }
-#endif
-#endif
- if(!ret) return 0;
-#if USE_MAGIC_HEADERS
- _ret=(size_t *) ret;
- ret=(void *)(_ret+3);
- if(alignment) ret=(void *)(((size_t) ret+alignment-1)&~(alignment-1));
- for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *) "NEDMALOC";
- _ret[0]=(size_t) mspace;
- _ret[1]=size-3*sizeof(size_t);
-#endif
- return ret;
-}
-
-static FORCEINLINE void *CallRealloc(void *RESTRICT mspace, void *RESTRICT mem, int isforeign, size_t oldsize, size_t newsize) THROWSPEC
-{
- void *RESTRICT ret=0;
-#if USE_MAGIC_HEADERS
- mstate oldmspace=0;
- size_t *_ret=0, *_mem=(size_t *) mem-3;
-#endif
- if(isforeign)
- { /* Transfer */
-#if USE_MAGIC_HEADERS
- assert(_mem[0]!=*(size_t *) "NEDMALOC");
-#endif
- if((ret=CallMalloc(mspace, newsize, 0)))
- {
-#if defined(DEBUG)
- printf("*** nedmalloc frees system allocated block %p\n", mem);
-#endif
- memcpy(ret, mem, oldsize<newsize ? oldsize : newsize);
- free(mem);
- }
- return ret;
- }
-#if USE_MAGIC_HEADERS
- assert(_mem[0]==*(size_t *) "NEDMALOC");
- newsize+=3*sizeof(size_t);
- oldmspace=(mstate) _mem[1];
- assert(oldsize>=_mem[2]);
- for(; *_mem==*(size_t *) "NEDMALOC"; *_mem--=*(size_t *) "nedmaloc");
- mem=(void *)(++_mem);
-#endif
-#if USE_ALLOCATOR==0
- ret=realloc(mem, newsize);
-#elif USE_ALLOCATOR==1
- ret=mspace_realloc((mstate) mspace, mem, newsize);
-#ifndef ENABLE_FAST_HEAP_DETECTION
- if(ret)
- {
- size_t truesize=chunksize(mem2chunk(ret));
- if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1);
- }
-#endif
-#endif
- if(!ret)
- { /* Put it back the way it was */
-#if USE_MAGIC_HEADERS
- for(; *_mem==0; *_mem++=*(size_t *) "NEDMALOC");
-#endif
- return 0;
- }
-#if USE_MAGIC_HEADERS
- _ret=(size_t *) ret;
- ret=(void *)(_ret+3);
- for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *) "NEDMALOC";
- _ret[0]=(size_t) mspace;
- _ret[1]=newsize-3*sizeof(size_t);
-#endif
- return ret;
-}
-
-static FORCEINLINE void CallFree(void *RESTRICT mspace, void *RESTRICT mem, int isforeign) THROWSPEC
-{
-#if USE_MAGIC_HEADERS
- mstate oldmspace=0;
- size_t *_mem=(size_t *) mem-3, oldsize=0;
-#endif
- if(isforeign)
- {
-#if USE_MAGIC_HEADERS
- assert(_mem[0]!=*(size_t *) "NEDMALOC");
-#endif
-#if defined(DEBUG)
- printf("*** nedmalloc frees system allocated block %p\n", mem);
-#endif
- free(mem);
- return;
- }
-#if USE_MAGIC_HEADERS
- assert(_mem[0]==*(size_t *) "NEDMALOC");
- oldmspace=(mstate) _mem[1];
- oldsize=_mem[2];
- for(; *_mem==*(size_t *) "NEDMALOC"; *_mem--=*(size_t *) "nedmaloc");
- mem=(void *)(++_mem);
-#endif
-#if USE_ALLOCATOR==0
- free(mem);
-#elif USE_ALLOCATOR==1
- mspace_free((mstate) mspace, mem);
-#endif
-}
-
-static NEDMALLOCNOALIASATTR mstate nedblkmstate(void *RESTRICT mem) THROWSPEC
-{
- if(mem)
- {
-#if USE_MAGIC_HEADERS
- size_t *_mem=(size_t *) mem-3;
- if(_mem[0]==*(size_t *) "NEDMALOC")
- {
- return (mstate) _mem[1];
- }
- else return 0;
-#else
-#if USE_ALLOCATOR==0
- /* Fail everything */
- return 0;
-#elif USE_ALLOCATOR==1
-#ifdef ENABLE_FAST_HEAP_DETECTION
-#ifdef WIN32
- /* On Windows for RELEASE both x86 and x64 the NT heap precedes each block with an eight byte header
- which looks like:
- normal: 4 bytes of size, 4 bytes of [char < 64, char < 64, char < 64 bit 0 always set, char random ]
- mmaped: 4 bytes of size 4 bytes of [zero, zero, 0xb, zero ]
-
- On Windows for DEBUG both x86 and x64 the preceding four bytes is always 0xfdfdfdfd (no man's land).
- */
-#pragma pack(push, 1)
- struct _HEAP_ENTRY
- {
- USHORT Size;
- USHORT PreviousSize;
- UCHAR Cookie; /* SegmentIndex */
- UCHAR Flags; /* always bit 0 (HEAP_ENTRY_BUSY). bit 1=(HEAP_ENTRY_EXTRA_PRESENT), bit 2=normal block (HEAP_ENTRY_FILL_PATTERN), bit 3=mmap block (HEAP_ENTRY_VIRTUAL_ALLOC). Bit 4 (HEAP_ENTRY_LAST_ENTRY) could be set */
- UCHAR UnusedBytes;
- UCHAR SmallTagIndex; /* fastbin index. Always one of 0x02, 0x03, 0x04 < 0x80 */
- } *RESTRICT he=((struct _HEAP_ENTRY *) mem)-1;
-#pragma pack(pop)
- unsigned int header=((unsigned int *)mem)[-1], mask1=0x8080E100, result1, mask2=0xFFFFFF06, result2;
- result1=header & mask1; /* Positive testing for NT heap */
- result2=header & mask2; /* Positive testing for dlmalloc */
- if(result1==0x00000100 && result2!=0x00000102)
- { /* This is likely a NT heap block */
- return 0;
- }
-#endif
-#ifdef __linux__
- /* On Linux glibc uses ptmalloc2 (really dlmalloc) just as we do, but prev_foot contains rubbish
- when the preceding block is allocated because ptmalloc2 finds the local mstate by rounding the ptr
- down to the nearest megabyte. It's like dlmalloc with FOOTERS disabled. */
- mchunkptr p=mem2chunk(mem);
- mstate fm=get_mstate_for(p);
- /* If it's a ptmalloc2 block, fm is likely to be some crazy value */
- if(!is_aligned(fm)) return 0;
- if((size_t)mem-(size_t)fm>=(size_t)1<<(SIZE_T_BITSIZE-1)) return 0;
- if(ok_magic(fm))
- return fm;
- else
- return 0;
- if(1) { }
-#endif
- else
- {
- mchunkptr p=mem2chunk(mem);
- mstate fm=get_mstate_for(p);
- assert(ok_magic(fm)); /* If this fails, someone tried to free a block twice */
- if(ok_magic(fm))
- return fm;
- }
-#else
-//#ifdef WIN32
-// __try
-//#endif
- {
- /* We try to return zero here if it isn't one of our own blocks, however
- the current block annotation scheme used by dlmalloc makes it impossible
- to be absolutely sure of avoiding a segfault.
-
- mchunkptr->prev_foot = mem-(2*size_t) = mstate ^ mparams.magic for PRECEDING block;
- mchunkptr->head = mem-(1*size_t) = 8 multiple size of this block with bottom three bits = FLAG_BITS
- FLAG_BITS = bit 0 is CINUSE (currently in use unless is mmap), bit 1 is PINUSE (previous block currently
- in use unless mmap), bit 2 is UNUSED and currently is always zero.
- */
- register void *RESTRICT leastusedaddress_=leastusedaddress; /* Cache these to avoid register reloading */
- register size_t largestusedblock_=largestusedblock;
- if(!is_aligned(mem)) return 0; /* Would fail very rarely as all allocators return aligned blocks */
- if(mem<leastusedaddress_) return 0; /* Simple but effective */
- {
- mchunkptr p=mem2chunk(mem);
- mstate fm=0;
- int ismmapped=is_mmapped(p);
- if((!ismmapped && !is_inuse(p)) || (p->head & FLAG4_BIT)) return 0;
- /* Reduced uncertainty by 0.5^2 = 25.0% */
- /* size should never exceed largestusedblock */
- if(chunksize(p)>largestusedblock_) return 0;
- /* Reduced uncertainty by a minimum of 0.5^3 = 12.5%, maximum 0.5^16 = 0.0015% */
- /* Having sanity checked prev_foot and head, check next block */
- if(!ismmapped && (!next_pinuse(p) || (next_chunk(p)->head & FLAG4_BIT))) return 0;
- /* Reduced uncertainty by 0.5^5 = 3.13% or 0.5^18 = 0.00038% */
- #if 0
- /* If previous block is free, check that its next block pointer equals us */
- if(!ismmapped && !pinuse(p))
- if(next_chunk(prev_chunk(p))!=p) return 0;
- /* We could start comparing prev_foot's for similarity but it starts getting slow. */
- #endif
- fm = get_mstate_for(p);
- if(!is_aligned(fm) || (void *)fm<leastusedaddress_) return 0;
- if((size_t)mem-(size_t)fm>=(size_t)1<<(SIZE_T_BITSIZE-1)) return 0;
- assert(ok_magic(fm)); /* If this fails, someone tried to free a block twice */
- if(ok_magic(fm))
- return fm;
- }
- }
-//#ifdef WIN32
-// __except(1) { }
-//#endif
-#endif
-#endif
-#endif
- }
- return 0;
-}
-NEDMALLOCNOALIASATTR size_t nedblksize(int *RESTRICT isforeign, void *RESTRICT mem) THROWSPEC
-{
- if(mem)
- {
- if(isforeign) *isforeign=1;
-#if USE_MAGIC_HEADERS
- {
- size_t *_mem=(size_t *) mem-3;
- if(_mem[0]==*(size_t *) "NEDMALOC")
- {
- mstate mspace=(mstate) _mem[1];
- size_t size=_mem[2];
- if(isforeign) *isforeign=0;
- return size;
- }
- }
-#elif USE_ALLOCATOR==1
- if(nedblkmstate(mem))
- {
- mchunkptr p=mem2chunk(mem);
- if(isforeign) *isforeign=0;
- return chunksize(p)-overhead_for(p);
- }
-#ifdef DEBUG
- else
- {
- int a=1; /* Set breakpoints here if needed */
- }
-#endif
-#endif
-#if defined(ENABLE_TOLERANT_NEDMALLOC) || USE_ALLOCATOR==0
-#ifdef _MSC_VER
- /* This is the MSVCRT equivalent */
- return _msize(mem);
-#elif defined(__linux__)
- /* This is the glibc/ptmalloc2/dlmalloc equivalent. */
- return malloc_usable_size(mem);
-#elif defined(__FreeBSD__) || defined(__APPLE__)
- /* This is the BSD libc equivalent. */
- return malloc_size(mem);
-#else
-#error Cannot tolerate the memory allocator of an unknown system!
-#endif
-#endif
- }
- return 0;
-}
-
-NEDMALLOCNOALIASATTR void nedsetvalue(void *v) THROWSPEC { nedpsetvalue((nedpool *) 0, v); }
-NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmalloc(size_t size) THROWSPEC { return nedpmalloc((nedpool *) 0, size); }
-NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedcalloc(size_t no, size_t size) THROWSPEC { return nedpcalloc((nedpool *) 0, no, size); }
-NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedrealloc(void *mem, size_t size) THROWSPEC { return nedprealloc((nedpool *) 0, mem, size); }
-NEDMALLOCNOALIASATTR void nedfree(void *mem) THROWSPEC { nedpfree((nedpool *) 0, mem); }
-NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmemalign(size_t alignment, size_t bytes) THROWSPEC { return nedpmemalign((nedpool *) 0, alignment, bytes); }
-NEDMALLOCNOALIASATTR struct nedmallinfo nedmallinfo(void) THROWSPEC { return nedpmallinfo((nedpool *) 0); }
-NEDMALLOCNOALIASATTR int nedmallopt(int parno, int value) THROWSPEC { return nedpmallopt((nedpool *) 0, parno, value); }
-NEDMALLOCNOALIASATTR int nedmalloc_trim(size_t pad) THROWSPEC { return nedpmalloc_trim((nedpool *) 0, pad); }
-void nedmalloc_stats() THROWSPEC { nedpmalloc_stats((nedpool *) 0); }
-NEDMALLOCNOALIASATTR size_t nedmalloc_footprint() THROWSPEC { return nedpmalloc_footprint((nedpool *) 0); }
-NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_calloc(size_t elemsno, size_t elemsize, void **chunks) THROWSPEC { return nedpindependent_calloc((nedpool *) 0, elemsno, elemsize, chunks); }
-NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_comalloc(size_t elems, size_t *sizes, void **chunks) THROWSPEC { return nedpindependent_comalloc((nedpool *) 0, elems, sizes, chunks); }
-
-struct threadcacheblk_t;
-typedef struct threadcacheblk_t threadcacheblk;
-struct threadcacheblk_t
-{ /* Keep less than 16 bytes on 32 bit systems and 32 bytes on 64 bit systems */
-#ifdef FULLSANITYCHECKS
- unsigned int magic;
-#endif
- unsigned int lastUsed, size;
- threadcacheblk *next, *prev;
-};
-typedef struct threadcache_t
-{
-#ifdef FULLSANITYCHECKS
- unsigned int magic1;
-#endif
- int mymspace; /* Last mspace entry this thread used */
- long threadid;
- unsigned int mallocs, frees, successes;
- size_t freeInCache; /* How much free space is stored in this cache */
- threadcacheblk *bins[(THREADCACHEMAXBINS+1)*2];
-#ifdef FULLSANITYCHECKS
- unsigned int magic2;
-#endif
-} threadcache;
-struct nedpool_t
-{
- MLOCK_T mutex;
- void *uservalue;
- int threads; /* Max entries in m to use */
- threadcache *caches[THREADCACHEMAXCACHES];
- TLSVAR mycache; /* Thread cache for this thread. 0 for unset, negative for use mspace-1 directly, otherwise is cache-1 */
- mstate m[MAXTHREADSINPOOL+1]; /* mspace entries for this pool */
-};
-static nedpool syspool;
-
-static FORCEINLINE NEDMALLOCNOALIASATTR unsigned int size2binidx(size_t _size) THROWSPEC
-{ /* 8=1000 16=10000 20=10100 24=11000 32=100000 48=110000 4096=1000000000000 */
- unsigned int topbit, size=(unsigned int)(_size>>4);
- /* 16=1 20=1 24=1 32=10 48=11 64=100 96=110 128=1000 4096=100000000 */
-
-#if defined(__GNUC__)
- topbit = sizeof(size)*__CHAR_BIT__ - 1 - __builtin_clz(size);
-#elif defined(_MSC_VER) && _MSC_VER>=1300
- {
- unsigned long bsrTopBit;
-
- _BitScanReverse(&bsrTopBit, size);
-
- topbit = bsrTopBit;
- }
-#else
-#if 0
- union {
- unsigned asInt[2];
- double asDouble;
- };
- int n;
-
- asDouble = (double)size + 0.5;
- topbit = (asInt[!FOX_BIGENDIAN] >> 20) - 1023;
-#else
- {
- unsigned int x=size;
- x = x | (x >> 1);
- x = x | (x >> 2);
- x = x | (x >> 4);
- x = x | (x >> 8);
- x = x | (x >>16);
- x = ~x;
- x = x - ((x >> 1) & 0x55555555);
- x = (x & 0x33333333) + ((x >> 2) & 0x33333333);
- x = (x + (x >> 4)) & 0x0F0F0F0F;
- x = x + (x << 8);
- x = x + (x << 16);
- topbit=31 - (x >> 24);
- }
-#endif
-#endif
- return topbit;
-}
-
-
-#ifdef FULLSANITYCHECKS
-static void tcsanitycheck(threadcacheblk **ptr) THROWSPEC
-{
- assert((ptr[0] && ptr[1]) || (!ptr[0] && !ptr[1]));
- if(ptr[0] && ptr[1])
- {
- assert(nedblksize(ptr[0])>=sizeof(threadcacheblk));
- assert(nedblksize(ptr[1])>=sizeof(threadcacheblk));
- assert(*(unsigned int *) "NEDN"==ptr[0]->magic);
- assert(*(unsigned int *) "NEDN"==ptr[1]->magic);
- assert(!ptr[0]->prev);
- assert(!ptr[1]->next);
- if(ptr[0]==ptr[1])
- {
- assert(!ptr[0]->next);
- assert(!ptr[1]->prev);
- }
- }
-}
-static void tcfullsanitycheck(threadcache *tc) THROWSPEC
-{
- threadcacheblk **tcbptr=tc->bins;
- int n;
- for(n=0; n<=THREADCACHEMAXBINS; n++, tcbptr+=2)
- {
- threadcacheblk *b, *ob=0;
- tcsanitycheck(tcbptr);
- for(b=tcbptr[0]; b; ob=b, b=b->next)
- {
- assert(*(unsigned int *) "NEDN"==b->magic);
- assert(!ob || ob->next==b);
- assert(!ob || b->prev==ob);
- }
- }
-}
-#endif
-
-static NOINLINE void RemoveCacheEntries(nedpool *RESTRICT p, threadcache *RESTRICT tc, unsigned int age) THROWSPEC
-{
-#ifdef FULLSANITYCHECKS
- tcfullsanitycheck(tc);
-#endif
- if(tc->freeInCache)
- {
- threadcacheblk **tcbptr=tc->bins;
- int n;
- for(n=0; n<=THREADCACHEMAXBINS; n++, tcbptr+=2)
- {
- threadcacheblk **tcb=tcbptr+1; /* come from oldest end of list */
- /*tcsanitycheck(tcbptr);*/
- for(; *tcb && tc->frees-(*tcb)->lastUsed>=age; )
- {
- threadcacheblk *f=*tcb;
- size_t blksize=f->size; /*nedblksize(f);*/
- assert(blksize<=nedblksize(0, f));
- assert(blksize);
-#ifdef FULLSANITYCHECKS
- assert(*(unsigned int *) "NEDN"==(*tcb)->magic);
-#endif
- *tcb=(*tcb)->prev;
- if(*tcb)
- (*tcb)->next=0;
- else
- *tcbptr=0;
- tc->freeInCache-=blksize;
- assert((long) tc->freeInCache>=0);
- CallFree(0, f, 0);
- /*tcsanitycheck(tcbptr);*/
- }
- }
- }
-#ifdef FULLSANITYCHECKS
- tcfullsanitycheck(tc);
-#endif
-}
-static void DestroyCaches(nedpool *RESTRICT p) THROWSPEC
-{
- if(p->caches)
- {
- threadcache *tc;
- int n;
- for(n=0; n<THREADCACHEMAXCACHES; n++)
- {
- if((tc=p->caches[n]))
- {
- tc->frees++;
- RemoveCacheEntries(p, tc, 0);
- assert(!tc->freeInCache);
- tc->mymspace=-1;
- tc->threadid=0;
- CallFree(0, tc, 0);
- p->caches[n]=0;
- }
- }
- }
-}
-
-static NOINLINE threadcache *AllocCache(nedpool *RESTRICT p) THROWSPEC
-{
- threadcache *tc=0;
- int n, end;
- ACQUIRE_LOCK(&p->mutex);
- for(n=0; n<THREADCACHEMAXCACHES && p->caches[n]; n++);
- if(THREADCACHEMAXCACHES==n)
- { /* List exhausted, so disable for this thread */
- RELEASE_LOCK(&p->mutex);
- return 0;
- }
- tc=p->caches[n]=(threadcache *) CallCalloc(p->m[0], sizeof(threadcache), 0);
- if(!tc)
- {
- RELEASE_LOCK(&p->mutex);
- return 0;
- }
-#ifdef FULLSANITYCHECKS
- tc->magic1=*(unsigned int *)"NEDMALC1";
- tc->magic2=*(unsigned int *)"NEDMALC2";
-#endif
- tc->threadid=(long)(size_t)CURRENT_THREAD;
- for(end=0; p->m[end]; end++);
- tc->mymspace=abs(tc->threadid) % end;
- RELEASE_LOCK(&p->mutex);
- if(TLSSET(p->mycache, (void *)(size_t)(n+1))) abort();
- return tc;
-}
-
-static void *threadcache_malloc(nedpool *RESTRICT p, threadcache *RESTRICT tc, size_t *RESTRICT _size) THROWSPEC
-{
- void *RESTRICT ret=0;
- size_t size=*_size, blksize=0;
- unsigned int bestsize;
- unsigned int idx=size2binidx(size);
- threadcacheblk *RESTRICT blk, **RESTRICT binsptr;
-#ifdef FULLSANITYCHECKS
- tcfullsanitycheck(tc);
-#endif
- /* Calculate best fit bin size */
- bestsize=1<<(idx+4);
-#if 0
- /* Finer grained bin fit */
- idx<<=1;
- if(size>bestsize)
- {
- idx++;
- bestsize+=bestsize>>1;
- }
- if(size>bestsize)
- {
- idx++;
- bestsize=1<<(4+(idx>>1));
- }
-#else
- if(size>bestsize)
- {
- idx++;
- bestsize<<=1;
- }
-#endif
- assert(bestsize>=size);
- if(size<bestsize) size=bestsize;
- assert(size<=THREADCACHEMAX);
- assert(idx<=THREADCACHEMAXBINS);
- binsptr=&tc->bins[idx*2];
- /* Try to match close, but move up a bin if necessary */
- blk=*binsptr;
- if(!blk || blk->size<size)
- { /* Bump it up a bin */
- if(idx<THREADCACHEMAXBINS)
- {
- idx++;
- binsptr+=2;
- blk=*binsptr;
- }
- }
- if(blk)
- {
- blksize=blk->size; /*nedblksize(blk);*/
- assert(nedblksize(0, blk)>=blksize);
- assert(blksize>=size);
- if(blk->next)
- blk->next->prev=0;
- *binsptr=blk->next;
- if(!*binsptr)
- binsptr[1]=0;
-#ifdef FULLSANITYCHECKS
- blk->magic=0;
-#endif
- assert(binsptr[0]!=blk && binsptr[1]!=blk);
- assert(nedblksize(0, blk)>=sizeof(threadcacheblk) && nedblksize(0, blk)<=THREADCACHEMAX+CHUNK_OVERHEAD);
- /*printf("malloc: %p, %p, %p, %lu\n", p, tc, blk, (long) _size);*/
- ret=(void *) blk;
- }
- ++tc->mallocs;
- if(ret)
- {
- assert(blksize>=size);
- ++tc->successes;
- tc->freeInCache-=blksize;
- assert((long) tc->freeInCache>=0);
- }
-#if defined(DEBUG) && 0
- if(!(tc->mallocs & 0xfff))
- {
- printf("*** threadcache=%u, mallocs=%u (%f), free=%u (%f), freeInCache=%u\n", (unsigned int) tc->threadid, tc->mallocs,
- (float) tc->successes/tc->mallocs, tc->frees, (float) tc->successes/tc->frees, (unsigned int) tc->freeInCache);
- }
-#endif
-#ifdef FULLSANITYCHECKS
- tcfullsanitycheck(tc);
-#endif
- *_size=size;
- return ret;
-}
-static NOINLINE void ReleaseFreeInCache(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace) THROWSPEC
-{
- unsigned int age=THREADCACHEMAXFREESPACE/8192;
- /*ACQUIRE_LOCK(&p->m[mymspace]->mutex);*/
- while(age && tc->freeInCache>=THREADCACHEMAXFREESPACE)
- {
- RemoveCacheEntries(p, tc, age);
- /*printf("*** Removing cache entries older than %u (%u)\n", age, (unsigned int) tc->freeInCache);*/
- age>>=1;
- }
- /*RELEASE_LOCK(&p->m[mymspace]->mutex);*/
-}
-static void threadcache_free(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace, void *RESTRICT mem, size_t size) THROWSPEC
-{
- unsigned int bestsize;
- unsigned int idx=size2binidx(size);
- threadcacheblk **RESTRICT binsptr, *RESTRICT tck=(threadcacheblk *) mem;
- assert(size>=sizeof(threadcacheblk) && size<=THREADCACHEMAX+CHUNK_OVERHEAD);
-#ifdef DEBUG
- /* Make sure this is a valid memory block */
- assert(nedblksize(0, mem));
-#endif
-#ifdef FULLSANITYCHECKS
- tcfullsanitycheck(tc);
-#endif
- /* Calculate best fit bin size */
- bestsize=1<<(idx+4);
-#if 0
- /* Finer grained bin fit */
- idx<<=1;
- if(size>bestsize)
- {
- unsigned int biggerbestsize=bestsize+bestsize<<1;
- if(size>=biggerbestsize)
- {
- idx++;
- bestsize=biggerbestsize;
- }
- }
-#endif
- if(bestsize!=size) /* dlmalloc can round up, so we round down to preserve indexing */
- size=bestsize;
- binsptr=&tc->bins[idx*2];
- assert(idx<=THREADCACHEMAXBINS);
- if(tck==*binsptr)
- {
- fprintf(stderr, "nedmalloc: Attempt to free already freed memory block %p - aborting!\n", tck);
- abort();
- }
-#ifdef FULLSANITYCHECKS
- tck->magic=*(unsigned int *) "NEDN";
-#endif
- tck->lastUsed=++tc->frees;
- tck->size=(unsigned int) size;
- tck->next=*binsptr;
- tck->prev=0;
- if(tck->next)
- tck->next->prev=tck;
- else
- binsptr[1]=tck;
- assert(!*binsptr || (*binsptr)->size==tck->size);
- *binsptr=tck;
- assert(tck==tc->bins[idx*2]);
- assert(tc->bins[idx*2+1]==tck || binsptr[0]->next->prev==tck);
- /*printf("free: %p, %p, %p, %lu\n", p, tc, mem, (long) size);*/
- tc->freeInCache+=size;
-#ifdef FULLSANITYCHECKS
- tcfullsanitycheck(tc);
-#endif
-#if 1
- if(tc->freeInCache>=THREADCACHEMAXFREESPACE)
- ReleaseFreeInCache(p, tc, mymspace);
-#endif
-}
-
-
-
-
-static NOINLINE int InitPool(nedpool *RESTRICT p, size_t capacity, int threads) THROWSPEC
-{ /* threads is -1 for system pool */
- ensure_initialization();
- ACQUIRE_MALLOC_GLOBAL_LOCK();
- if(p->threads) goto done;
- if(INITIAL_LOCK(&p->mutex)) goto err;
- if(TLSALLOC(&p->mycache)) goto err;
-#if USE_ALLOCATOR==0
- p->m[0]=(mstate) mspacecounter++;
-#elif USE_ALLOCATOR==1
- if(!(p->m[0]=(mstate) create_mspace(capacity, 1))) goto err;
- p->m[0]->extp=p;
-#endif
- p->threads=(threads<1 || threads>MAXTHREADSINPOOL) ? MAXTHREADSINPOOL : threads;
-done:
- RELEASE_MALLOC_GLOBAL_LOCK();
- return 1;
-err:
- if(threads<0)
- abort(); /* If you can't allocate for system pool, we're screwed */
- DestroyCaches(p);
- if(p->m[0])
- {
-#if USE_ALLOCATOR==1
- destroy_mspace(p->m[0]);
-#endif
- p->m[0]=0;
- }
- if(p->mycache)
- {
- if(TLSFREE(p->mycache)) abort();
- p->mycache=0;
- }
- RELEASE_MALLOC_GLOBAL_LOCK();
- return 0;
-}
-static NOINLINE mstate FindMSpace(nedpool *RESTRICT p, threadcache *RESTRICT tc, int *RESTRICT lastUsed, size_t size) THROWSPEC
-{ /* Gets called when thread's last used mspace is in use. The strategy
- is to run through the list of all available mspaces looking for an
- unlocked one and if we fail, we create a new one so long as we don't
- exceed p->threads */
- int n, end;
- for(n=end=*lastUsed+1; p->m[n]; end=++n)
- {
- if(TRY_LOCK(&p->m[n]->mutex)) goto found;
- }
- for(n=0; n<*lastUsed && p->m[n]; n++)
- {
- if(TRY_LOCK(&p->m[n]->mutex)) goto found;
- }
- if(end<p->threads)
- {
- mstate temp;
-#if USE_ALLOCATOR==0
- temp=(mstate) mspacecounter++;
-#elif USE_ALLOCATOR==1
- if(!(temp=(mstate) create_mspace(size, 1)))
- goto badexit;
-#endif
- /* Now we're ready to modify the lists, we lock */
- ACQUIRE_LOCK(&p->mutex);
- while(p->m[end] && end<p->threads)
- end++;
- if(end>=p->threads)
- { /* Drat, must destroy it now */
- RELEASE_LOCK(&p->mutex);
-#if USE_ALLOCATOR==1
- destroy_mspace((mstate) temp);
-#endif
- goto badexit;
- }
- /* We really want to make sure this goes into memory now but we
- have to be careful of breaking aliasing rules, so write it twice */
- *((volatile struct malloc_state **) &p->m[end])=p->m[end]=temp;
- ACQUIRE_LOCK(&p->m[end]->mutex);
- /*printf("Created mspace idx %d\n", end);*/
- RELEASE_LOCK(&p->mutex);
- n=end;
- goto found;
- }
- /* Let it lock on the last one it used */
-badexit:
- ACQUIRE_LOCK(&p->m[*lastUsed]->mutex);
- return p->m[*lastUsed];
-found:
- *lastUsed=n;
- if(tc)
- tc->mymspace=n;
- else
- {
- if(TLSSET(p->mycache, (void *)(size_t)(-(n+1)))) abort();
- }
- return p->m[n];
-}
-
-typedef struct PoolList_t
-{
- size_t size; /* Size of list */
- size_t length; /* Actual entries in list */
-#ifdef DEBUG
- nedpool *list[1]; /* Force testing of list expansion */
-#else
- nedpool *list[16];
-#endif
-} PoolList;
-static MLOCK_T poollistlock;
-static PoolList *poollist;
-NEDMALLOCPTRATTR nedpool *nedcreatepool(size_t capacity, int threads) THROWSPEC
-{
- nedpool *ret=0;
- if(!poollist)
- {
- PoolList *newpoollist=0;
- if(!(newpoollist=(PoolList *) nedpcalloc(0, 1, sizeof(PoolList)+sizeof(nedpool *)))) return 0;
- INITIAL_LOCK(&poollistlock);
- ACQUIRE_LOCK(&poollistlock);
- poollist=newpoollist;
- poollist->size=sizeof(poollist->list)/sizeof(nedpool *);
- }
- else
- ACQUIRE_LOCK(&poollistlock);
- if(poollist->length==poollist->size)
- {
- PoolList *newpoollist=0;
- size_t newsize=0;
- newsize=sizeof(PoolList)+(poollist->size+1)*sizeof(nedpool *);
- if(!(newpoollist=(PoolList *) nedprealloc(0, poollist, newsize))) goto badexit;
- poollist=newpoollist;
- memset(&poollist->list[poollist->size], 0, newsize-((size_t)&poollist->list[poollist->size]-(size_t)&poollist->list[0]));
- poollist->size=((newsize-((char *)&poollist->list[0]-(char *)poollist))/sizeof(nedpool *))-1;
- assert(poollist->size>poollist->length);
- }
- if(!(ret=(nedpool *) nedpcalloc(0, 1, sizeof(nedpool)))) goto badexit;
- if(!InitPool(ret, capacity, threads))
- {
- nedpfree(0, ret);
- goto badexit;
- }
- poollist->list[poollist->length++]=ret;
-badexit:
- RELEASE_LOCK(&poollistlock);
- return ret;
-}
-void neddestroypool(nedpool *p) THROWSPEC
-{
- unsigned int n;
- ACQUIRE_LOCK(&p->mutex);
- DestroyCaches(p);
- for(n=0; p->m[n]; n++)
- {
-#if USE_ALLOCATOR==1
- destroy_mspace(p->m[n]);
-#endif
- p->m[n]=0;
- }
- RELEASE_LOCK(&p->mutex);
- if(TLSFREE(p->mycache)) abort();
- nedpfree(0, p);
- ACQUIRE_LOCK(&poollistlock);
- assert(poollist);
- for(n=0; n<poollist->length && poollist->list[n]!=p; n++);
- assert(n!=poollist->length);
- memmove(&poollist->list[n], &poollist->list[n+1], (size_t)&poollist->list[poollist->length]-(size_t)&poollist->list[n]);
- if(!--poollist->length)
- {
- assert(!poollist->list[0]);
- nedpfree(0, poollist);
- poollist=0;
- }
- RELEASE_LOCK(&poollistlock);
-}
-void neddestroysyspool() THROWSPEC
-{
- nedpool *p=&syspool;
- int n;
- ACQUIRE_LOCK(&p->mutex);
- DestroyCaches(p);
- for(n=0; p->m[n]; n++)
- {
-#if USE_ALLOCATOR==1
- destroy_mspace(p->m[n]);
-#endif
- p->m[n]=0;
- }
- /* Render syspool unusable */
- for(n=0; n<THREADCACHEMAXCACHES; n++)
- p->caches[n]=(threadcache *)(size_t)(sizeof(size_t)>4 ? 0xdeadbeefdeadbeefULL : 0xdeadbeefUL);
- for(n=0; n<MAXTHREADSINPOOL+1; n++)
- p->m[n]=(mstate)(size_t)(sizeof(size_t)>4 ? 0xdeadbeefdeadbeefULL : 0xdeadbeefUL);
- if(TLSFREE(p->mycache)) abort();
- RELEASE_LOCK(&p->mutex);
-}
-nedpool **nedpoollist() THROWSPEC
-{
- nedpool **ret=0;
- if(poollist)
- {
- ACQUIRE_LOCK(&poollistlock);
- if(!(ret=(nedpool **) nedmalloc((poollist->length+1)*sizeof(nedpool *)))) goto badexit;
- memcpy(ret, poollist->list, (poollist->length+1)*sizeof(nedpool *));
-badexit:
- RELEASE_LOCK(&poollistlock);
- }
- return ret;
-}
-
-void nedpsetvalue(nedpool *p, void *v) THROWSPEC
-{
- if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); }
- p->uservalue=v;
-}
-void *nedgetvalue(nedpool **p, void *mem) THROWSPEC
-{
- nedpool *np=0;
- mstate fm=nedblkmstate(mem);
- if(!fm || !fm->extp) return 0;
- np=(nedpool *) fm->extp;
- if(p) *p=np;
- return np->uservalue;
-}
-
-void nedtrimthreadcache(nedpool *p, int disable) THROWSPEC
-{
- int mycache;
- if(!p)
- {
- p=&syspool;
- if(!syspool.threads) InitPool(&syspool, 0, -1);
- }
- mycache=(int)(size_t) TLSGET(p->mycache);
- if(!mycache)
- { /* Set to mspace 0 */
- if(disable && TLSSET(p->mycache, (void *)(size_t)-1)) abort();
- }
- else if(mycache>0)
- { /* Set to last used mspace */
- threadcache *tc=p->caches[mycache-1];
-#if defined(DEBUG)
- printf("Threadcache utilisation: %lf%% in cache with %lf%% lost to other threads\n",
- 100.0*tc->successes/tc->mallocs, 100.0*((double) tc->mallocs-tc->frees)/tc->mallocs);
-#endif
- if(disable && TLSSET(p->mycache, (void *)(size_t)(-tc->mymspace))) abort();
- tc->frees++;
- RemoveCacheEntries(p, tc, 0);
- assert(!tc->freeInCache);
- if(disable)
- {
- tc->mymspace=-1;
- tc->threadid=0;
- CallFree(0, p->caches[mycache-1], 0);
- p->caches[mycache-1]=0;
- }
- }
-}
-void neddisablethreadcache(nedpool *p) THROWSPEC
-{
- nedtrimthreadcache(p, 1);
-}
-
-#define GETMSPACE(m,p,tc,ms,s,action) \
- do \
- { \
- mstate m = GetMSpace((p),(tc),(ms),(s)); \
- action; \
- if(USE_ALLOCATOR==1) { RELEASE_LOCK(&m->mutex); } \
- } while (0)
-
-static FORCEINLINE mstate GetMSpace(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace, size_t size) THROWSPEC
-{ /* Returns a locked and ready for use mspace */
- mstate m=p->m[mymspace];
- assert(m);
-#if USE_ALLOCATOR==1
- if(!TRY_LOCK(&p->m[mymspace]->mutex)) m=FindMSpace(p, tc, &mymspace, size);
- /*assert(IS_LOCKED(&p->m[mymspace]->mutex));*/
-#endif
- return m;
-}
-static NOINLINE void GetThreadCache_cold1(nedpool *RESTRICT *RESTRICT p) THROWSPEC
-{
- *p=&syspool;
- if(!syspool.threads) InitPool(&syspool, 0, -1);
-}
-static NOINLINE void GetThreadCache_cold2(nedpool *RESTRICT *RESTRICT p, threadcache *RESTRICT *RESTRICT tc, int *RESTRICT mymspace, int mycache) THROWSPEC
-{
- if(!mycache)
- { /* Need to allocate a new cache */
- *tc=AllocCache(*p);
- if(!*tc)
- { /* Disable */
- if(TLSSET((*p)->mycache, (void *)(size_t)-1)) abort();
- *mymspace=0;
- }
- else
- *mymspace=(*tc)->mymspace;
- }
- else
- { /* Cache disabled, but we do have an assigned thread pool */
- *tc=0;
- *mymspace=-mycache-1;
- }
-}
-static FORCEINLINE void GetThreadCache(nedpool *RESTRICT *RESTRICT p, threadcache *RESTRICT *RESTRICT tc, int *RESTRICT mymspace, size_t *RESTRICT size) THROWSPEC
-{
- int mycache;
- if(size && *size<sizeof(threadcacheblk)) *size=sizeof(threadcacheblk);
- if(!*p)
- GetThreadCache_cold1(p);
- mycache=(int)(size_t) TLSGET((*p)->mycache);
- if(mycache>0)
- { /* Already have a cache */
- *tc=(*p)->caches[mycache-1];
- *mymspace=(*tc)->mymspace;
- }
- else GetThreadCache_cold2(p, tc, mymspace, mycache);
- assert(*mymspace>=0);
- assert(!(*tc) || (long)(size_t)CURRENT_THREAD==(*tc)->threadid);
-#ifdef FULLSANITYCHECKS
- if(*tc)
- {
- if(*(unsigned int *)"NEDMALC1"!=(*tc)->magic1 || *(unsigned int *)"NEDMALC2"!=(*tc)->magic2)
- {
- abort();
- }
- }
-#endif
-}
-
-NEDMALLOCPTRATTR void * nedpmalloc(nedpool *p, size_t size) THROWSPEC
-{
- void *ret=0;
- threadcache *tc;
- int mymspace;
- GetThreadCache(&p, &tc, &mymspace, &size);
-#if THREADCACHEMAX
- if(tc && size<=THREADCACHEMAX)
- { /* Use the thread cache */
- ret=threadcache_malloc(p, tc, &size);
- }
-#endif
- if(!ret)
- { /* Use this thread's mspace */
- GETMSPACE(m, p, tc, mymspace, size,
- ret=CallMalloc(m, size, 0));
- }
- return ret;
-}
-NEDMALLOCPTRATTR void * nedpcalloc(nedpool *p, size_t no, size_t size) THROWSPEC
-{
- size_t rsize=size*no;
- void *ret=0;
- threadcache *tc;
- int mymspace;
- GetThreadCache(&p, &tc, &mymspace, &rsize);
-#if THREADCACHEMAX
- if(tc && rsize<=THREADCACHEMAX)
- { /* Use the thread cache */
- if((ret=threadcache_malloc(p, tc, &rsize)))
- memset(ret, 0, rsize);
- }
-#endif
- if(!ret)
- { /* Use this thread's mspace */
- GETMSPACE(m, p, tc, mymspace, rsize,
- ret=CallCalloc(m, rsize, 0));
- }
- return ret;
-}
-NEDMALLOCPTRATTR void * nedprealloc(nedpool *p, void *mem, size_t size) THROWSPEC
-{
- void *ret=0;
- threadcache *tc;
- int mymspace, isforeign=1;
- size_t memsize;
- if(!mem) return nedpmalloc(p, size);
- memsize=nedblksize(&isforeign, mem);
- assert(memsize);
- if(!memsize)
- {
- fprintf(stderr, "nedmalloc: nedprealloc() called with a block not created by nedmalloc!\n");
- abort();
- }
- else if(size<=memsize && memsize-size<
-#ifdef DEBUG
- 32
-#else
- 1024
-#endif
- ) /* If realloc size is within 1Kb smaller than existing, noop it */
- return mem;
- GetThreadCache(&p, &tc, &mymspace, &size);
-#if THREADCACHEMAX
- if(tc && size && size<=THREADCACHEMAX)
- { /* Use the thread cache */
- if((ret=threadcache_malloc(p, tc, &size)))
- {
- memcpy(ret, mem, memsize<size ? memsize : size);
- if(memsize>=sizeof(threadcacheblk) && memsize<=(THREADCACHEMAX+CHUNK_OVERHEAD))
- threadcache_free(p, tc, mymspace, mem, memsize);
- else
- CallFree(0, mem, isforeign);
- }
- }
-#endif
- if(!ret)
- { /* Reallocs always happen in the mspace they happened in, so skip
- locking the preferred mspace for this thread */
- ret=CallRealloc(p->m[mymspace], mem, isforeign, memsize, size);
- }
- return ret;
-}
-void nedpfree(nedpool *p, void *mem) THROWSPEC
-{ /* Frees always happen in the mspace they happened in, so skip
- locking the preferred mspace for this thread */
- threadcache *tc;
- int mymspace, isforeign=1;
- size_t memsize;
- if(!mem)
- { /* If you tried this on FreeBSD you'd be sorry! */
-#ifdef DEBUG
- fprintf(stderr, "nedmalloc: WARNING nedpfree() called with zero. This is not portable behaviour!\n");
-#endif
- return;
- }
- memsize=nedblksize(&isforeign, mem);
- assert(memsize);
- if(!memsize)
- {
- fprintf(stderr, "nedmalloc: nedpfree() called with a block not created by nedmalloc!\n");
- abort();
- }
- GetThreadCache(&p, &tc, &mymspace, 0);
-#if THREADCACHEMAX
- if(mem && tc && memsize>=sizeof(threadcacheblk) && memsize<=(THREADCACHEMAX+CHUNK_OVERHEAD))
- threadcache_free(p, tc, mymspace, mem, memsize);
- else
-#endif
- CallFree(0, mem, isforeign);
-}
-NEDMALLOCPTRATTR void * nedpmemalign(nedpool *p, size_t alignment, size_t bytes) THROWSPEC
-{
- void *ret;
- threadcache *tc;
- int mymspace;
- GetThreadCache(&p, &tc, &mymspace, &bytes);
- { /* Use this thread's mspace */
- GETMSPACE(m, p, tc, mymspace, bytes,
- ret=CallMalloc(m, bytes, alignment));
- }
- return ret;
-}
-struct nedmallinfo nedpmallinfo(nedpool *p) THROWSPEC
-{
- int n;
- struct nedmallinfo ret={0};
- if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); }
- for(n=0; p->m[n]; n++)
- {
-#if USE_ALLOCATOR==1 && !NO_MALLINFO
- struct mallinfo t=mspace_mallinfo(p->m[n]);
- ret.arena+=t.arena;
- ret.ordblks+=t.ordblks;
- ret.hblkhd+=t.hblkhd;
- ret.usmblks+=t.usmblks;
- ret.uordblks+=t.uordblks;
- ret.fordblks+=t.fordblks;
- ret.keepcost+=t.keepcost;
-#endif
- }
- return ret;
-}
-int nedpmallopt(nedpool *p, int parno, int value) THROWSPEC
-{
-#if USE_ALLOCATOR==1
- return mspace_mallopt(parno, value);
-#else
- return 0;
-#endif
-}
-NEDMALLOCNOALIASATTR void* nedmalloc_internals(size_t *granularity, size_t *magic) THROWSPEC
-{
-#if USE_ALLOCATOR==1
- if(granularity) *granularity=mparams.granularity;
- if(magic) *magic=mparams.magic;
- return (void *) &syspool;
-#else
- if(granularity) *granularity=0;
- if(magic) *magic=0;
- return 0;
-#endif
-}
-int nedpmalloc_trim(nedpool *p, size_t pad) THROWSPEC
-{
- int n, ret=0;
- if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); }
- for(n=0; p->m[n]; n++)
- {
-#if USE_ALLOCATOR==1
- ret+=mspace_trim(p->m[n], pad);
-#endif
- }
- return ret;
-}
-void nedpmalloc_stats(nedpool *p) THROWSPEC
-{
- int n;
- if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); }
- for(n=0; p->m[n]; n++)
- {
-#if USE_ALLOCATOR==1
- mspace_malloc_stats(p->m[n]);
-#endif
- }
-}
-size_t nedpmalloc_footprint(nedpool *p) THROWSPEC
-{
- size_t ret=0;
- int n;
- if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); }
- for(n=0; p->m[n]; n++)
- {
-#if USE_ALLOCATOR==1
- ret+=mspace_footprint(p->m[n]);
-#endif
- }
- return ret;
-}
-NEDMALLOCPTRATTR void **nedpindependent_calloc(nedpool *p, size_t elemsno, size_t elemsize, void **chunks) THROWSPEC
-{
- void **ret;
- threadcache *tc;
- int mymspace;
- GetThreadCache(&p, &tc, &mymspace, &elemsize);
-#if USE_ALLOCATOR==0
- GETMSPACE(m, p, tc, mymspace, elemsno*elemsize,
- ret=unsupported_operation("independent_calloc"));
-#elif USE_ALLOCATOR==1
- GETMSPACE(m, p, tc, mymspace, elemsno*elemsize,
- ret=mspace_independent_calloc(m, elemsno, elemsize, chunks));
-#endif
- return ret;
-}
-NEDMALLOCPTRATTR void **nedpindependent_comalloc(nedpool *p, size_t elems, size_t *sizes, void **chunks) THROWSPEC
-{
- void **ret;
- threadcache *tc;
- int mymspace;
- size_t i, *adjustedsizes=(size_t *) alloca(elems*sizeof(size_t));
- if(!adjustedsizes) return 0;
- for(i=0; i<elems; i++)
- adjustedsizes[i]=sizes[i]<sizeof(threadcacheblk) ? sizeof(threadcacheblk) : sizes[i];
- GetThreadCache(&p, &tc, &mymspace, 0);
-#if USE_ALLOCATOR==0
- GETMSPACE(m, p, tc, mymspace, 0,
- ret=unsupported_operation("independent_comalloc"));
-#elif USE_ALLOCATOR==1
- GETMSPACE(m, p, tc, mymspace, 0,
- ret=mspace_independent_comalloc(m, elems, adjustedsizes, chunks));
-#endif
- return ret;
-}
-
-#if defined(__cplusplus)
-}
-#endif
-
-#ifdef _MSC_VER
-#pragma warning(pop)
-#endif
-
-#endif
+#ifdef NEDMALLOC_ENABLED +/* Alternative malloc implementation for multiple threads without +lock contention based on dlmalloc. (C) 2005-2009 Niall Douglas + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + +#ifdef _MSC_VER +/* Enable full aliasing on MSVC */ +/*#pragma optimize("a", on)*/ +#pragma warning(push) +#pragma warning(disable:4100) /* unreferenced formal parameter */ +#pragma warning(disable:4127) /* conditional expression is constant */ +#pragma warning(disable:4706) /* assignment within conditional expression */ +#endif + +/*#define ENABLE_TOLERANT_NEDMALLOC 1*/ +/*#define ENABLE_FAST_HEAP_DETECTION 1*/ +/*#define NEDMALLOC_DEBUG 1*/ + +/*#define FULLSANITYCHECKS*/ +/* If link time code generation is on, don't force or prevent inlining */ +#if defined(_MSC_VER) && defined(NEDMALLOC_DLL_EXPORTS) +#define FORCEINLINE +#define NOINLINE +#endif + + +#include "nedmalloc.h" +#ifdef WIN32 + #include <malloc.h> + #include <stddef.h> +#endif +#if USE_ALLOCATOR==1 + #define MSPACES 1 + #define ONLY_MSPACES 1 +#endif +#define USE_DL_PREFIX 1 +#ifndef USE_LOCKS + #define USE_LOCKS 1 +#endif +#define FOOTERS 1 /* Need to enable footers so frees lock the right mspace */ +#ifndef NEDMALLOC_DEBUG + #if defined(DEBUG) || defined(_DEBUG) + #define NEDMALLOC_DEBUG 1 + #else + #define NEDMALLOC_DEBUG 0 + #endif +#endif +/* We need to consistently define DEBUG=0|1, _DEBUG and NDEBUG for dlmalloc */ +#undef DEBUG +#undef _DEBUG +#if NEDMALLOC_DEBUG + #define _DEBUG + #define DEBUG 1 +#else + #define DEBUG 0 +#endif +#ifdef NDEBUG /* Disable assert checking on release builds */ + #undef DEBUG + #undef _DEBUG +#endif +/* The default of 64Kb means we spend too much time kernel-side */ +#ifndef DEFAULT_GRANULARITY +#define DEFAULT_GRANULARITY (1*1024*1024) +#if DEBUG +#define DEFAULT_GRANULARITY_ALIGNED +#endif +#endif +/*#define USE_SPIN_LOCKS 0*/ + + +#include "malloc.c.h" +#ifdef NDEBUG /* Disable assert checking on release builds */ + #undef DEBUG +#elif !NEDMALLOC_DEBUG + #ifdef __GNUC__ + #warning DEBUG is defined so allocator will run with assert checking! Define NDEBUG to run at full speed. + #elif defined(_MSC_VER) + #pragma message(__FILE__ ": WARNING: DEBUG is defined so allocator will run with assert checking! Define NDEBUG to run at full speed.") + #endif +#endif + +/* The maximum concurrent threads in a pool possible */ +#ifndef MAXTHREADSINPOOL +#define MAXTHREADSINPOOL 16 +#endif +/* The maximum number of threadcaches which can be allocated */ +#ifndef THREADCACHEMAXCACHES +#define THREADCACHEMAXCACHES 256 +#endif +/* The maximum size to be allocated from the thread cache */ +#ifndef THREADCACHEMAX +#define THREADCACHEMAX 8192 +#endif +#if 0 +/* The number of cache entries for finer grained bins. This is (topbitpos(THREADCACHEMAX)-4)*2 */ +#define THREADCACHEMAXBINS ((13-4)*2) +#else +/* The number of cache entries. This is (topbitpos(THREADCACHEMAX)-4) */ +#define THREADCACHEMAXBINS (13-4) +#endif +/* Point at which the free space in a thread cache is garbage collected */ +#ifndef THREADCACHEMAXFREESPACE +#define THREADCACHEMAXFREESPACE (512*1024) +#endif + + +#ifdef WIN32 + #define TLSVAR DWORD + #define TLSALLOC(k) (*(k)=TlsAlloc(), TLS_OUT_OF_INDEXES==*(k)) + #define TLSFREE(k) (!TlsFree(k)) + #define TLSGET(k) TlsGetValue(k) + #define TLSSET(k, a) (!TlsSetValue(k, a)) + #ifdef DEBUG +static LPVOID ChkedTlsGetValue(DWORD idx) +{ + LPVOID ret=TlsGetValue(idx); + assert(S_OK==GetLastError()); + return ret; +} + #undef TLSGET + #define TLSGET(k) ChkedTlsGetValue(k) + #endif +#else + #define TLSVAR pthread_key_t + #define TLSALLOC(k) pthread_key_create(k, 0) + #define TLSFREE(k) pthread_key_delete(k) + #define TLSGET(k) pthread_getspecific(k) + #define TLSSET(k, a) pthread_setspecific(k, a) +#endif + +#if defined(__cplusplus) +#if !defined(NO_NED_NAMESPACE) +namespace nedalloc { +#else +extern "C" { +#endif +#endif + +#if USE_ALLOCATOR==0 +static void *unsupported_operation(const char *opname) THROWSPEC +{ + fprintf(stderr, "nedmalloc: The operation %s is not supported under this build configuration\n", opname); + abort(); + return 0; +} +static size_t mspacecounter=(size_t) 0xdeadbeef; +#endif +#ifndef ENABLE_FAST_HEAP_DETECTION +static void *RESTRICT leastusedaddress; +static size_t largestusedblock; +#endif + +static FORCEINLINE void *CallMalloc(void *RESTRICT mspace, size_t size, size_t alignment) THROWSPEC +{ + void *RESTRICT ret=0; + size_t _alignment=alignment; +#if USE_MAGIC_HEADERS + size_t *_ret=0; + size+=alignment+3*sizeof(size_t); + _alignment=0; +#endif +#if USE_ALLOCATOR==0 + ret=_alignment ? +#ifdef _MSC_VER + /* This is the MSVCRT equivalent */ + _aligned_malloc(size, _alignment) +#elif defined(__linux__) || defined(__FreeBSD__) || defined(__APPLE__) + /* This is the glibc/ptmalloc2/dlmalloc/BSD libc equivalent. */ + memalign(_alignment, size) +#else +#error Cannot aligned allocate with the memory allocator of an unknown system! +#endif + : malloc(size); +#elif USE_ALLOCATOR==1 + ret=_alignment ? mspace_memalign((mstate) mspace, _alignment, size) : mspace_malloc((mstate) mspace, size); +#ifndef ENABLE_FAST_HEAP_DETECTION + if(ret) + { + size_t truesize=chunksize(mem2chunk(ret)); + if(!leastusedaddress || (void *)((mstate) mspace)->least_addr<leastusedaddress) leastusedaddress=(void *)((mstate) mspace)->least_addr; + if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1); + } +#endif +#endif + if(!ret) return 0; +#if USE_MAGIC_HEADERS + _ret=(size_t *) ret; + ret=(void *)(_ret+3); + if(alignment) ret=(void *)(((size_t) ret+alignment-1)&~(alignment-1)); + for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *)"NEDMALOC"; + _ret[0]=(size_t) mspace; + _ret[1]=size-3*sizeof(size_t); +#endif + return ret; +} + +static FORCEINLINE void *CallCalloc(void *RESTRICT mspace, size_t size, size_t alignment) THROWSPEC +{ + void *RESTRICT ret=0; +#if USE_MAGIC_HEADERS + size_t *_ret=0; + size+=alignment+3*sizeof(size_t); +#endif +#if USE_ALLOCATOR==0 + ret=calloc(1, size); +#elif USE_ALLOCATOR==1 + ret=mspace_calloc((mstate) mspace, 1, size); +#ifndef ENABLE_FAST_HEAP_DETECTION + if(ret) + { + size_t truesize=chunksize(mem2chunk(ret)); + if(!leastusedaddress || (void *)((mstate) mspace)->least_addr<leastusedaddress) leastusedaddress=(void *)((mstate) mspace)->least_addr; + if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1); + } +#endif +#endif + if(!ret) return 0; +#if USE_MAGIC_HEADERS + _ret=(size_t *) ret; + ret=(void *)(_ret+3); + if(alignment) ret=(void *)(((size_t) ret+alignment-1)&~(alignment-1)); + for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *) "NEDMALOC"; + _ret[0]=(size_t) mspace; + _ret[1]=size-3*sizeof(size_t); +#endif + return ret; +} + +static FORCEINLINE void *CallRealloc(void *RESTRICT mspace, void *RESTRICT mem, int isforeign, size_t oldsize, size_t newsize) THROWSPEC +{ + void *RESTRICT ret=0; +#if USE_MAGIC_HEADERS + mstate oldmspace=0; + size_t *_ret=0, *_mem=(size_t *) mem-3; +#endif + if(isforeign) + { /* Transfer */ +#if USE_MAGIC_HEADERS + assert(_mem[0]!=*(size_t *) "NEDMALOC"); +#endif + if((ret=CallMalloc(mspace, newsize, 0))) + { +#if defined(DEBUG) + printf("*** nedmalloc frees system allocated block %p\n", mem); +#endif + memcpy(ret, mem, oldsize<newsize ? oldsize : newsize); + free(mem); + } + return ret; + } +#if USE_MAGIC_HEADERS + assert(_mem[0]==*(size_t *) "NEDMALOC"); + newsize+=3*sizeof(size_t); + oldmspace=(mstate) _mem[1]; + assert(oldsize>=_mem[2]); + for(; *_mem==*(size_t *) "NEDMALOC"; *_mem--=*(size_t *) "nedmaloc"); + mem=(void *)(++_mem); +#endif +#if USE_ALLOCATOR==0 + ret=realloc(mem, newsize); +#elif USE_ALLOCATOR==1 + ret=mspace_realloc((mstate) mspace, mem, newsize); +#ifndef ENABLE_FAST_HEAP_DETECTION + if(ret) + { + size_t truesize=chunksize(mem2chunk(ret)); + if(!largestusedblock || truesize>largestusedblock) largestusedblock=(truesize+mparams.page_size) & ~(mparams.page_size-1); + } +#endif +#endif + if(!ret) + { /* Put it back the way it was */ +#if USE_MAGIC_HEADERS + for(; *_mem==0; *_mem++=*(size_t *) "NEDMALOC"); +#endif + return 0; + } +#if USE_MAGIC_HEADERS + _ret=(size_t *) ret; + ret=(void *)(_ret+3); + for(; _ret<(size_t *)ret-2; _ret++) *_ret=*(size_t *) "NEDMALOC"; + _ret[0]=(size_t) mspace; + _ret[1]=newsize-3*sizeof(size_t); +#endif + return ret; +} + +static FORCEINLINE void CallFree(void *RESTRICT mspace, void *RESTRICT mem, int isforeign) THROWSPEC +{ +#if USE_MAGIC_HEADERS + mstate oldmspace=0; + size_t *_mem=(size_t *) mem-3, oldsize=0; +#endif + if(isforeign) + { +#if USE_MAGIC_HEADERS + assert(_mem[0]!=*(size_t *) "NEDMALOC"); +#endif +#if defined(DEBUG) + printf("*** nedmalloc frees system allocated block %p\n", mem); +#endif + free(mem); + return; + } +#if USE_MAGIC_HEADERS + assert(_mem[0]==*(size_t *) "NEDMALOC"); + oldmspace=(mstate) _mem[1]; + oldsize=_mem[2]; + for(; *_mem==*(size_t *) "NEDMALOC"; *_mem--=*(size_t *) "nedmaloc"); + mem=(void *)(++_mem); +#endif +#if USE_ALLOCATOR==0 + free(mem); +#elif USE_ALLOCATOR==1 + mspace_free((mstate) mspace, mem); +#endif +} + +static NEDMALLOCNOALIASATTR mstate nedblkmstate(void *RESTRICT mem) THROWSPEC +{ + if(mem) + { +#if USE_MAGIC_HEADERS + size_t *_mem=(size_t *) mem-3; + if(_mem[0]==*(size_t *) "NEDMALOC") + { + return (mstate) _mem[1]; + } + else return 0; +#else +#if USE_ALLOCATOR==0 + /* Fail everything */ + return 0; +#elif USE_ALLOCATOR==1 +#ifdef ENABLE_FAST_HEAP_DETECTION +#ifdef WIN32 + /* On Windows for RELEASE both x86 and x64 the NT heap precedes each block with an eight byte header + which looks like: + normal: 4 bytes of size, 4 bytes of [char < 64, char < 64, char < 64 bit 0 always set, char random ] + mmaped: 4 bytes of size 4 bytes of [zero, zero, 0xb, zero ] + + On Windows for DEBUG both x86 and x64 the preceding four bytes is always 0xfdfdfdfd (no man's land). + */ +#pragma pack(push, 1) + struct _HEAP_ENTRY + { + USHORT Size; + USHORT PreviousSize; + UCHAR Cookie; /* SegmentIndex */ + UCHAR Flags; /* always bit 0 (HEAP_ENTRY_BUSY). bit 1=(HEAP_ENTRY_EXTRA_PRESENT), bit 2=normal block (HEAP_ENTRY_FILL_PATTERN), bit 3=mmap block (HEAP_ENTRY_VIRTUAL_ALLOC). Bit 4 (HEAP_ENTRY_LAST_ENTRY) could be set */ + UCHAR UnusedBytes; + UCHAR SmallTagIndex; /* fastbin index. Always one of 0x02, 0x03, 0x04 < 0x80 */ + } *RESTRICT he=((struct _HEAP_ENTRY *) mem)-1; +#pragma pack(pop) + unsigned int header=((unsigned int *)mem)[-1], mask1=0x8080E100, result1, mask2=0xFFFFFF06, result2; + result1=header & mask1; /* Positive testing for NT heap */ + result2=header & mask2; /* Positive testing for dlmalloc */ + if(result1==0x00000100 && result2!=0x00000102) + { /* This is likely a NT heap block */ + return 0; + } +#endif +#ifdef __linux__ + /* On Linux glibc uses ptmalloc2 (really dlmalloc) just as we do, but prev_foot contains rubbish + when the preceding block is allocated because ptmalloc2 finds the local mstate by rounding the ptr + down to the nearest megabyte. It's like dlmalloc with FOOTERS disabled. */ + mchunkptr p=mem2chunk(mem); + mstate fm=get_mstate_for(p); + /* If it's a ptmalloc2 block, fm is likely to be some crazy value */ + if(!is_aligned(fm)) return 0; + if((size_t)mem-(size_t)fm>=(size_t)1<<(SIZE_T_BITSIZE-1)) return 0; + if(ok_magic(fm)) + return fm; + else + return 0; + if(1) { } +#endif + else + { + mchunkptr p=mem2chunk(mem); + mstate fm=get_mstate_for(p); + assert(ok_magic(fm)); /* If this fails, someone tried to free a block twice */ + if(ok_magic(fm)) + return fm; + } +#else +//#ifdef WIN32 +// __try +//#endif + { + /* We try to return zero here if it isn't one of our own blocks, however + the current block annotation scheme used by dlmalloc makes it impossible + to be absolutely sure of avoiding a segfault. + + mchunkptr->prev_foot = mem-(2*size_t) = mstate ^ mparams.magic for PRECEDING block; + mchunkptr->head = mem-(1*size_t) = 8 multiple size of this block with bottom three bits = FLAG_BITS + FLAG_BITS = bit 0 is CINUSE (currently in use unless is mmap), bit 1 is PINUSE (previous block currently + in use unless mmap), bit 2 is UNUSED and currently is always zero. + */ + register void *RESTRICT leastusedaddress_=leastusedaddress; /* Cache these to avoid register reloading */ + register size_t largestusedblock_=largestusedblock; + if(!is_aligned(mem)) return 0; /* Would fail very rarely as all allocators return aligned blocks */ + if(mem<leastusedaddress_) return 0; /* Simple but effective */ + { + mchunkptr p=mem2chunk(mem); + mstate fm=0; + int ismmapped=is_mmapped(p); + if((!ismmapped && !is_inuse(p)) || (p->head & FLAG4_BIT)) return 0; + /* Reduced uncertainty by 0.5^2 = 25.0% */ + /* size should never exceed largestusedblock */ + if(chunksize(p)>largestusedblock_) return 0; + /* Reduced uncertainty by a minimum of 0.5^3 = 12.5%, maximum 0.5^16 = 0.0015% */ + /* Having sanity checked prev_foot and head, check next block */ + if(!ismmapped && (!next_pinuse(p) || (next_chunk(p)->head & FLAG4_BIT))) return 0; + /* Reduced uncertainty by 0.5^5 = 3.13% or 0.5^18 = 0.00038% */ + #if 0 + /* If previous block is free, check that its next block pointer equals us */ + if(!ismmapped && !pinuse(p)) + if(next_chunk(prev_chunk(p))!=p) return 0; + /* We could start comparing prev_foot's for similarity but it starts getting slow. */ + #endif + fm = get_mstate_for(p); + if(!is_aligned(fm) || (void *)fm<leastusedaddress_) return 0; + if((size_t)mem-(size_t)fm>=(size_t)1<<(SIZE_T_BITSIZE-1)) return 0; + assert(ok_magic(fm)); /* If this fails, someone tried to free a block twice */ + if(ok_magic(fm)) + return fm; + } + } +//#ifdef WIN32 +// __except(1) { } +//#endif +#endif +#endif +#endif + } + return 0; +} +NEDMALLOCNOALIASATTR size_t nedblksize(int *RESTRICT isforeign, void *RESTRICT mem) THROWSPEC +{ + if(mem) + { + if(isforeign) *isforeign=1; +#if USE_MAGIC_HEADERS + { + size_t *_mem=(size_t *) mem-3; + if(_mem[0]==*(size_t *) "NEDMALOC") + { + mstate mspace=(mstate) _mem[1]; + size_t size=_mem[2]; + if(isforeign) *isforeign=0; + return size; + } + } +#elif USE_ALLOCATOR==1 + if(nedblkmstate(mem)) + { + mchunkptr p=mem2chunk(mem); + if(isforeign) *isforeign=0; + return chunksize(p)-overhead_for(p); + } +#ifdef DEBUG + else + { + int a=1; /* Set breakpoints here if needed */ + } +#endif +#endif +#if defined(ENABLE_TOLERANT_NEDMALLOC) || USE_ALLOCATOR==0 +#ifdef _MSC_VER + /* This is the MSVCRT equivalent */ + return _msize(mem); +#elif defined(__linux__) + /* This is the glibc/ptmalloc2/dlmalloc equivalent. */ + return malloc_usable_size(mem); +#elif defined(__FreeBSD__) || defined(__APPLE__) + /* This is the BSD libc equivalent. */ + return malloc_size(mem); +#else +#error Cannot tolerate the memory allocator of an unknown system! +#endif +#endif + } + return 0; +} + +NEDMALLOCNOALIASATTR void nedsetvalue(void *v) THROWSPEC { nedpsetvalue((nedpool *) 0, v); } +NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmalloc(size_t size) THROWSPEC { return nedpmalloc((nedpool *) 0, size); } +NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedcalloc(size_t no, size_t size) THROWSPEC { return nedpcalloc((nedpool *) 0, no, size); } +NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedrealloc(void *mem, size_t size) THROWSPEC { return nedprealloc((nedpool *) 0, mem, size); } +NEDMALLOCNOALIASATTR void nedfree(void *mem) THROWSPEC { nedpfree((nedpool *) 0, mem); } +NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmemalign(size_t alignment, size_t bytes) THROWSPEC { return nedpmemalign((nedpool *) 0, alignment, bytes); } +NEDMALLOCNOALIASATTR struct nedmallinfo nedmallinfo(void) THROWSPEC { return nedpmallinfo((nedpool *) 0); } +NEDMALLOCNOALIASATTR int nedmallopt(int parno, int value) THROWSPEC { return nedpmallopt((nedpool *) 0, parno, value); } +NEDMALLOCNOALIASATTR int nedmalloc_trim(size_t pad) THROWSPEC { return nedpmalloc_trim((nedpool *) 0, pad); } +void nedmalloc_stats() THROWSPEC { nedpmalloc_stats((nedpool *) 0); } +NEDMALLOCNOALIASATTR size_t nedmalloc_footprint() THROWSPEC { return nedpmalloc_footprint((nedpool *) 0); } +NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_calloc(size_t elemsno, size_t elemsize, void **chunks) THROWSPEC { return nedpindependent_calloc((nedpool *) 0, elemsno, elemsize, chunks); } +NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_comalloc(size_t elems, size_t *sizes, void **chunks) THROWSPEC { return nedpindependent_comalloc((nedpool *) 0, elems, sizes, chunks); } + +struct threadcacheblk_t; +typedef struct threadcacheblk_t threadcacheblk; +struct threadcacheblk_t +{ /* Keep less than 16 bytes on 32 bit systems and 32 bytes on 64 bit systems */ +#ifdef FULLSANITYCHECKS + unsigned int magic; +#endif + unsigned int lastUsed, size; + threadcacheblk *next, *prev; +}; +typedef struct threadcache_t +{ +#ifdef FULLSANITYCHECKS + unsigned int magic1; +#endif + int mymspace; /* Last mspace entry this thread used */ + long threadid; + unsigned int mallocs, frees, successes; + size_t freeInCache; /* How much free space is stored in this cache */ + threadcacheblk *bins[(THREADCACHEMAXBINS+1)*2]; +#ifdef FULLSANITYCHECKS + unsigned int magic2; +#endif +} threadcache; +struct nedpool_t +{ + MLOCK_T mutex; + void *uservalue; + int threads; /* Max entries in m to use */ + threadcache *caches[THREADCACHEMAXCACHES]; + TLSVAR mycache; /* Thread cache for this thread. 0 for unset, negative for use mspace-1 directly, otherwise is cache-1 */ + mstate m[MAXTHREADSINPOOL+1]; /* mspace entries for this pool */ +}; +static nedpool syspool; + +static FORCEINLINE NEDMALLOCNOALIASATTR unsigned int size2binidx(size_t _size) THROWSPEC +{ /* 8=1000 16=10000 20=10100 24=11000 32=100000 48=110000 4096=1000000000000 */ + unsigned int topbit, size=(unsigned int)(_size>>4); + /* 16=1 20=1 24=1 32=10 48=11 64=100 96=110 128=1000 4096=100000000 */ + +#if defined(__GNUC__) + topbit = sizeof(size)*__CHAR_BIT__ - 1 - __builtin_clz(size); +#elif defined(_MSC_VER) && _MSC_VER>=1300 + { + unsigned long bsrTopBit; + + _BitScanReverse(&bsrTopBit, size); + + topbit = bsrTopBit; + } +#else +#if 0 + union { + unsigned asInt[2]; + double asDouble; + }; + int n; + + asDouble = (double)size + 0.5; + topbit = (asInt[!FOX_BIGENDIAN] >> 20) - 1023; +#else + { + unsigned int x=size; + x = x | (x >> 1); + x = x | (x >> 2); + x = x | (x >> 4); + x = x | (x >> 8); + x = x | (x >>16); + x = ~x; + x = x - ((x >> 1) & 0x55555555); + x = (x & 0x33333333) + ((x >> 2) & 0x33333333); + x = (x + (x >> 4)) & 0x0F0F0F0F; + x = x + (x << 8); + x = x + (x << 16); + topbit=31 - (x >> 24); + } +#endif +#endif + return topbit; +} + + +#ifdef FULLSANITYCHECKS +static void tcsanitycheck(threadcacheblk **ptr) THROWSPEC +{ + assert((ptr[0] && ptr[1]) || (!ptr[0] && !ptr[1])); + if(ptr[0] && ptr[1]) + { + assert(nedblksize(ptr[0])>=sizeof(threadcacheblk)); + assert(nedblksize(ptr[1])>=sizeof(threadcacheblk)); + assert(*(unsigned int *) "NEDN"==ptr[0]->magic); + assert(*(unsigned int *) "NEDN"==ptr[1]->magic); + assert(!ptr[0]->prev); + assert(!ptr[1]->next); + if(ptr[0]==ptr[1]) + { + assert(!ptr[0]->next); + assert(!ptr[1]->prev); + } + } +} +static void tcfullsanitycheck(threadcache *tc) THROWSPEC +{ + threadcacheblk **tcbptr=tc->bins; + int n; + for(n=0; n<=THREADCACHEMAXBINS; n++, tcbptr+=2) + { + threadcacheblk *b, *ob=0; + tcsanitycheck(tcbptr); + for(b=tcbptr[0]; b; ob=b, b=b->next) + { + assert(*(unsigned int *) "NEDN"==b->magic); + assert(!ob || ob->next==b); + assert(!ob || b->prev==ob); + } + } +} +#endif + +static NOINLINE void RemoveCacheEntries(nedpool *RESTRICT p, threadcache *RESTRICT tc, unsigned int age) THROWSPEC +{ +#ifdef FULLSANITYCHECKS + tcfullsanitycheck(tc); +#endif + if(tc->freeInCache) + { + threadcacheblk **tcbptr=tc->bins; + int n; + for(n=0; n<=THREADCACHEMAXBINS; n++, tcbptr+=2) + { + threadcacheblk **tcb=tcbptr+1; /* come from oldest end of list */ + /*tcsanitycheck(tcbptr);*/ + for(; *tcb && tc->frees-(*tcb)->lastUsed>=age; ) + { + threadcacheblk *f=*tcb; + size_t blksize=f->size; /*nedblksize(f);*/ + assert(blksize<=nedblksize(0, f)); + assert(blksize); +#ifdef FULLSANITYCHECKS + assert(*(unsigned int *) "NEDN"==(*tcb)->magic); +#endif + *tcb=(*tcb)->prev; + if(*tcb) + (*tcb)->next=0; + else + *tcbptr=0; + tc->freeInCache-=blksize; + assert((long) tc->freeInCache>=0); + CallFree(0, f, 0); + /*tcsanitycheck(tcbptr);*/ + } + } + } +#ifdef FULLSANITYCHECKS + tcfullsanitycheck(tc); +#endif +} +static void DestroyCaches(nedpool *RESTRICT p) THROWSPEC +{ + if(p->caches) + { + threadcache *tc; + int n; + for(n=0; n<THREADCACHEMAXCACHES; n++) + { + if((tc=p->caches[n])) + { + tc->frees++; + RemoveCacheEntries(p, tc, 0); + assert(!tc->freeInCache); + tc->mymspace=-1; + tc->threadid=0; + CallFree(0, tc, 0); + p->caches[n]=0; + } + } + } +} + +static NOINLINE threadcache *AllocCache(nedpool *RESTRICT p) THROWSPEC +{ + threadcache *tc=0; + int n, end; + ACQUIRE_LOCK(&p->mutex); + for(n=0; n<THREADCACHEMAXCACHES && p->caches[n]; n++); + if(THREADCACHEMAXCACHES==n) + { /* List exhausted, so disable for this thread */ + RELEASE_LOCK(&p->mutex); + return 0; + } + tc=p->caches[n]=(threadcache *) CallCalloc(p->m[0], sizeof(threadcache), 0); + if(!tc) + { + RELEASE_LOCK(&p->mutex); + return 0; + } +#ifdef FULLSANITYCHECKS + tc->magic1=*(unsigned int *)"NEDMALC1"; + tc->magic2=*(unsigned int *)"NEDMALC2"; +#endif + tc->threadid=(long)(size_t)CURRENT_THREAD; + for(end=0; p->m[end]; end++); + tc->mymspace=abs(tc->threadid) % end; + RELEASE_LOCK(&p->mutex); + if(TLSSET(p->mycache, (void *)(size_t)(n+1))) abort(); + return tc; +} + +static void *threadcache_malloc(nedpool *RESTRICT p, threadcache *RESTRICT tc, size_t *RESTRICT _size) THROWSPEC +{ + void *RESTRICT ret=0; + size_t size=*_size, blksize=0; + unsigned int bestsize; + unsigned int idx=size2binidx(size); + threadcacheblk *RESTRICT blk, **RESTRICT binsptr; +#ifdef FULLSANITYCHECKS + tcfullsanitycheck(tc); +#endif + /* Calculate best fit bin size */ + bestsize=1<<(idx+4); +#if 0 + /* Finer grained bin fit */ + idx<<=1; + if(size>bestsize) + { + idx++; + bestsize+=bestsize>>1; + } + if(size>bestsize) + { + idx++; + bestsize=1<<(4+(idx>>1)); + } +#else + if(size>bestsize) + { + idx++; + bestsize<<=1; + } +#endif + assert(bestsize>=size); + if(size<bestsize) size=bestsize; + assert(size<=THREADCACHEMAX); + assert(idx<=THREADCACHEMAXBINS); + binsptr=&tc->bins[idx*2]; + /* Try to match close, but move up a bin if necessary */ + blk=*binsptr; + if(!blk || blk->size<size) + { /* Bump it up a bin */ + if(idx<THREADCACHEMAXBINS) + { + idx++; + binsptr+=2; + blk=*binsptr; + } + } + if(blk) + { + blksize=blk->size; /*nedblksize(blk);*/ + assert(nedblksize(0, blk)>=blksize); + assert(blksize>=size); + if(blk->next) + blk->next->prev=0; + *binsptr=blk->next; + if(!*binsptr) + binsptr[1]=0; +#ifdef FULLSANITYCHECKS + blk->magic=0; +#endif + assert(binsptr[0]!=blk && binsptr[1]!=blk); + assert(nedblksize(0, blk)>=sizeof(threadcacheblk) && nedblksize(0, blk)<=THREADCACHEMAX+CHUNK_OVERHEAD); + /*printf("malloc: %p, %p, %p, %lu\n", p, tc, blk, (long) _size);*/ + ret=(void *) blk; + } + ++tc->mallocs; + if(ret) + { + assert(blksize>=size); + ++tc->successes; + tc->freeInCache-=blksize; + assert((long) tc->freeInCache>=0); + } +#if defined(DEBUG) && 0 + if(!(tc->mallocs & 0xfff)) + { + printf("*** threadcache=%u, mallocs=%u (%f), free=%u (%f), freeInCache=%u\n", (unsigned int) tc->threadid, tc->mallocs, + (float) tc->successes/tc->mallocs, tc->frees, (float) tc->successes/tc->frees, (unsigned int) tc->freeInCache); + } +#endif +#ifdef FULLSANITYCHECKS + tcfullsanitycheck(tc); +#endif + *_size=size; + return ret; +} +static NOINLINE void ReleaseFreeInCache(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace) THROWSPEC +{ + unsigned int age=THREADCACHEMAXFREESPACE/8192; + /*ACQUIRE_LOCK(&p->m[mymspace]->mutex);*/ + while(age && tc->freeInCache>=THREADCACHEMAXFREESPACE) + { + RemoveCacheEntries(p, tc, age); + /*printf("*** Removing cache entries older than %u (%u)\n", age, (unsigned int) tc->freeInCache);*/ + age>>=1; + } + /*RELEASE_LOCK(&p->m[mymspace]->mutex);*/ +} +static void threadcache_free(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace, void *RESTRICT mem, size_t size) THROWSPEC +{ + unsigned int bestsize; + unsigned int idx=size2binidx(size); + threadcacheblk **RESTRICT binsptr, *RESTRICT tck=(threadcacheblk *) mem; + assert(size>=sizeof(threadcacheblk) && size<=THREADCACHEMAX+CHUNK_OVERHEAD); +#ifdef DEBUG + /* Make sure this is a valid memory block */ + assert(nedblksize(0, mem)); +#endif +#ifdef FULLSANITYCHECKS + tcfullsanitycheck(tc); +#endif + /* Calculate best fit bin size */ + bestsize=1<<(idx+4); +#if 0 + /* Finer grained bin fit */ + idx<<=1; + if(size>bestsize) + { + unsigned int biggerbestsize=bestsize+bestsize<<1; + if(size>=biggerbestsize) + { + idx++; + bestsize=biggerbestsize; + } + } +#endif + if(bestsize!=size) /* dlmalloc can round up, so we round down to preserve indexing */ + size=bestsize; + binsptr=&tc->bins[idx*2]; + assert(idx<=THREADCACHEMAXBINS); + if(tck==*binsptr) + { + fprintf(stderr, "nedmalloc: Attempt to free already freed memory block %p - aborting!\n", tck); + abort(); + } +#ifdef FULLSANITYCHECKS + tck->magic=*(unsigned int *) "NEDN"; +#endif + tck->lastUsed=++tc->frees; + tck->size=(unsigned int) size; + tck->next=*binsptr; + tck->prev=0; + if(tck->next) + tck->next->prev=tck; + else + binsptr[1]=tck; + assert(!*binsptr || (*binsptr)->size==tck->size); + *binsptr=tck; + assert(tck==tc->bins[idx*2]); + assert(tc->bins[idx*2+1]==tck || binsptr[0]->next->prev==tck); + /*printf("free: %p, %p, %p, %lu\n", p, tc, mem, (long) size);*/ + tc->freeInCache+=size; +#ifdef FULLSANITYCHECKS + tcfullsanitycheck(tc); +#endif +#if 1 + if(tc->freeInCache>=THREADCACHEMAXFREESPACE) + ReleaseFreeInCache(p, tc, mymspace); +#endif +} + + + + +static NOINLINE int InitPool(nedpool *RESTRICT p, size_t capacity, int threads) THROWSPEC +{ /* threads is -1 for system pool */ + ensure_initialization(); + ACQUIRE_MALLOC_GLOBAL_LOCK(); + if(p->threads) goto done; + if(INITIAL_LOCK(&p->mutex)) goto err; + if(TLSALLOC(&p->mycache)) goto err; +#if USE_ALLOCATOR==0 + p->m[0]=(mstate) mspacecounter++; +#elif USE_ALLOCATOR==1 + if(!(p->m[0]=(mstate) create_mspace(capacity, 1))) goto err; + p->m[0]->extp=p; +#endif + p->threads=(threads<1 || threads>MAXTHREADSINPOOL) ? MAXTHREADSINPOOL : threads; +done: + RELEASE_MALLOC_GLOBAL_LOCK(); + return 1; +err: + if(threads<0) + abort(); /* If you can't allocate for system pool, we're screwed */ + DestroyCaches(p); + if(p->m[0]) + { +#if USE_ALLOCATOR==1 + destroy_mspace(p->m[0]); +#endif + p->m[0]=0; + } + if(p->mycache) + { + if(TLSFREE(p->mycache)) abort(); + p->mycache=0; + } + RELEASE_MALLOC_GLOBAL_LOCK(); + return 0; +} +static NOINLINE mstate FindMSpace(nedpool *RESTRICT p, threadcache *RESTRICT tc, int *RESTRICT lastUsed, size_t size) THROWSPEC +{ /* Gets called when thread's last used mspace is in use. The strategy + is to run through the list of all available mspaces looking for an + unlocked one and if we fail, we create a new one so long as we don't + exceed p->threads */ + int n, end; + for(n=end=*lastUsed+1; p->m[n]; end=++n) + { + if(TRY_LOCK(&p->m[n]->mutex)) goto found; + } + for(n=0; n<*lastUsed && p->m[n]; n++) + { + if(TRY_LOCK(&p->m[n]->mutex)) goto found; + } + if(end<p->threads) + { + mstate temp; +#if USE_ALLOCATOR==0 + temp=(mstate) mspacecounter++; +#elif USE_ALLOCATOR==1 + if(!(temp=(mstate) create_mspace(size, 1))) + goto badexit; +#endif + /* Now we're ready to modify the lists, we lock */ + ACQUIRE_LOCK(&p->mutex); + while(p->m[end] && end<p->threads) + end++; + if(end>=p->threads) + { /* Drat, must destroy it now */ + RELEASE_LOCK(&p->mutex); +#if USE_ALLOCATOR==1 + destroy_mspace((mstate) temp); +#endif + goto badexit; + } + /* We really want to make sure this goes into memory now but we + have to be careful of breaking aliasing rules, so write it twice */ + *((volatile struct malloc_state **) &p->m[end])=p->m[end]=temp; + ACQUIRE_LOCK(&p->m[end]->mutex); + /*printf("Created mspace idx %d\n", end);*/ + RELEASE_LOCK(&p->mutex); + n=end; + goto found; + } + /* Let it lock on the last one it used */ +badexit: + ACQUIRE_LOCK(&p->m[*lastUsed]->mutex); + return p->m[*lastUsed]; +found: + *lastUsed=n; + if(tc) + tc->mymspace=n; + else + { + if(TLSSET(p->mycache, (void *)(size_t)(-(n+1)))) abort(); + } + return p->m[n]; +} + +typedef struct PoolList_t +{ + size_t size; /* Size of list */ + size_t length; /* Actual entries in list */ +#ifdef DEBUG + nedpool *list[1]; /* Force testing of list expansion */ +#else + nedpool *list[16]; +#endif +} PoolList; +static MLOCK_T poollistlock; +static PoolList *poollist; +NEDMALLOCPTRATTR nedpool *nedcreatepool(size_t capacity, int threads) THROWSPEC +{ + nedpool *ret=0; + if(!poollist) + { + PoolList *newpoollist=0; + if(!(newpoollist=(PoolList *) nedpcalloc(0, 1, sizeof(PoolList)+sizeof(nedpool *)))) return 0; + INITIAL_LOCK(&poollistlock); + ACQUIRE_LOCK(&poollistlock); + poollist=newpoollist; + poollist->size=sizeof(poollist->list)/sizeof(nedpool *); + } + else + ACQUIRE_LOCK(&poollistlock); + if(poollist->length==poollist->size) + { + PoolList *newpoollist=0; + size_t newsize=0; + newsize=sizeof(PoolList)+(poollist->size+1)*sizeof(nedpool *); + if(!(newpoollist=(PoolList *) nedprealloc(0, poollist, newsize))) goto badexit; + poollist=newpoollist; + memset(&poollist->list[poollist->size], 0, newsize-((size_t)&poollist->list[poollist->size]-(size_t)&poollist->list[0])); + poollist->size=((newsize-((char *)&poollist->list[0]-(char *)poollist))/sizeof(nedpool *))-1; + assert(poollist->size>poollist->length); + } + if(!(ret=(nedpool *) nedpcalloc(0, 1, sizeof(nedpool)))) goto badexit; + if(!InitPool(ret, capacity, threads)) + { + nedpfree(0, ret); + goto badexit; + } + poollist->list[poollist->length++]=ret; +badexit: + RELEASE_LOCK(&poollistlock); + return ret; +} +void neddestroypool(nedpool *p) THROWSPEC +{ + unsigned int n; + ACQUIRE_LOCK(&p->mutex); + DestroyCaches(p); + for(n=0; p->m[n]; n++) + { +#if USE_ALLOCATOR==1 + destroy_mspace(p->m[n]); +#endif + p->m[n]=0; + } + RELEASE_LOCK(&p->mutex); + if(TLSFREE(p->mycache)) abort(); + nedpfree(0, p); + ACQUIRE_LOCK(&poollistlock); + assert(poollist); + for(n=0; n<poollist->length && poollist->list[n]!=p; n++); + assert(n!=poollist->length); + memmove(&poollist->list[n], &poollist->list[n+1], (size_t)&poollist->list[poollist->length]-(size_t)&poollist->list[n]); + if(!--poollist->length) + { + assert(!poollist->list[0]); + nedpfree(0, poollist); + poollist=0; + } + RELEASE_LOCK(&poollistlock); +} +void neddestroysyspool() THROWSPEC +{ + nedpool *p=&syspool; + int n; + ACQUIRE_LOCK(&p->mutex); + DestroyCaches(p); + for(n=0; p->m[n]; n++) + { +#if USE_ALLOCATOR==1 + destroy_mspace(p->m[n]); +#endif + p->m[n]=0; + } + /* Render syspool unusable */ + for(n=0; n<THREADCACHEMAXCACHES; n++) + p->caches[n]=(threadcache *)(size_t)(sizeof(size_t)>4 ? 0xdeadbeefdeadbeefULL : 0xdeadbeefUL); + for(n=0; n<MAXTHREADSINPOOL+1; n++) + p->m[n]=(mstate)(size_t)(sizeof(size_t)>4 ? 0xdeadbeefdeadbeefULL : 0xdeadbeefUL); + if(TLSFREE(p->mycache)) abort(); + RELEASE_LOCK(&p->mutex); +} +nedpool **nedpoollist() THROWSPEC +{ + nedpool **ret=0; + if(poollist) + { + ACQUIRE_LOCK(&poollistlock); + if(!(ret=(nedpool **) nedmalloc((poollist->length+1)*sizeof(nedpool *)))) goto badexit; + memcpy(ret, poollist->list, (poollist->length+1)*sizeof(nedpool *)); +badexit: + RELEASE_LOCK(&poollistlock); + } + return ret; +} + +void nedpsetvalue(nedpool *p, void *v) THROWSPEC +{ + if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } + p->uservalue=v; +} +void *nedgetvalue(nedpool **p, void *mem) THROWSPEC +{ + nedpool *np=0; + mstate fm=nedblkmstate(mem); + if(!fm || !fm->extp) return 0; + np=(nedpool *) fm->extp; + if(p) *p=np; + return np->uservalue; +} + +void nedtrimthreadcache(nedpool *p, int disable) THROWSPEC +{ + int mycache; + if(!p) + { + p=&syspool; + if(!syspool.threads) InitPool(&syspool, 0, -1); + } + mycache=(int)(size_t) TLSGET(p->mycache); + if(!mycache) + { /* Set to mspace 0 */ + if(disable && TLSSET(p->mycache, (void *)(size_t)-1)) abort(); + } + else if(mycache>0) + { /* Set to last used mspace */ + threadcache *tc=p->caches[mycache-1]; +#if defined(DEBUG) + printf("Threadcache utilisation: %lf%% in cache with %lf%% lost to other threads\n", + 100.0*tc->successes/tc->mallocs, 100.0*((double) tc->mallocs-tc->frees)/tc->mallocs); +#endif + if(disable && TLSSET(p->mycache, (void *)(size_t)(-tc->mymspace))) abort(); + tc->frees++; + RemoveCacheEntries(p, tc, 0); + assert(!tc->freeInCache); + if(disable) + { + tc->mymspace=-1; + tc->threadid=0; + CallFree(0, p->caches[mycache-1], 0); + p->caches[mycache-1]=0; + } + } +} +void neddisablethreadcache(nedpool *p) THROWSPEC +{ + nedtrimthreadcache(p, 1); +} + +#define GETMSPACE(m,p,tc,ms,s,action) \ + do \ + { \ + mstate m = GetMSpace((p),(tc),(ms),(s)); \ + action; \ + if(USE_ALLOCATOR==1) { RELEASE_LOCK(&m->mutex); } \ + } while (0) + +static FORCEINLINE mstate GetMSpace(nedpool *RESTRICT p, threadcache *RESTRICT tc, int mymspace, size_t size) THROWSPEC +{ /* Returns a locked and ready for use mspace */ + mstate m=p->m[mymspace]; + assert(m); +#if USE_ALLOCATOR==1 + if(!TRY_LOCK(&p->m[mymspace]->mutex)) m=FindMSpace(p, tc, &mymspace, size); + /*assert(IS_LOCKED(&p->m[mymspace]->mutex));*/ +#endif + return m; +} +static NOINLINE void GetThreadCache_cold1(nedpool *RESTRICT *RESTRICT p) THROWSPEC +{ + *p=&syspool; + if(!syspool.threads) InitPool(&syspool, 0, -1); +} +static NOINLINE void GetThreadCache_cold2(nedpool *RESTRICT *RESTRICT p, threadcache *RESTRICT *RESTRICT tc, int *RESTRICT mymspace, int mycache) THROWSPEC +{ + if(!mycache) + { /* Need to allocate a new cache */ + *tc=AllocCache(*p); + if(!*tc) + { /* Disable */ + if(TLSSET((*p)->mycache, (void *)(size_t)-1)) abort(); + *mymspace=0; + } + else + *mymspace=(*tc)->mymspace; + } + else + { /* Cache disabled, but we do have an assigned thread pool */ + *tc=0; + *mymspace=-mycache-1; + } +} +static FORCEINLINE void GetThreadCache(nedpool *RESTRICT *RESTRICT p, threadcache *RESTRICT *RESTRICT tc, int *RESTRICT mymspace, size_t *RESTRICT size) THROWSPEC +{ + int mycache; + if(size && *size<sizeof(threadcacheblk)) *size=sizeof(threadcacheblk); + if(!*p) + GetThreadCache_cold1(p); + mycache=(int)(size_t) TLSGET((*p)->mycache); + if(mycache>0) + { /* Already have a cache */ + *tc=(*p)->caches[mycache-1]; + *mymspace=(*tc)->mymspace; + } + else GetThreadCache_cold2(p, tc, mymspace, mycache); + assert(*mymspace>=0); + assert(!(*tc) || (long)(size_t)CURRENT_THREAD==(*tc)->threadid); +#ifdef FULLSANITYCHECKS + if(*tc) + { + if(*(unsigned int *)"NEDMALC1"!=(*tc)->magic1 || *(unsigned int *)"NEDMALC2"!=(*tc)->magic2) + { + abort(); + } + } +#endif +} + +NEDMALLOCPTRATTR void * nedpmalloc(nedpool *p, size_t size) THROWSPEC +{ + void *ret=0; + threadcache *tc; + int mymspace; + GetThreadCache(&p, &tc, &mymspace, &size); +#if THREADCACHEMAX + if(tc && size<=THREADCACHEMAX) + { /* Use the thread cache */ + ret=threadcache_malloc(p, tc, &size); + } +#endif + if(!ret) + { /* Use this thread's mspace */ + GETMSPACE(m, p, tc, mymspace, size, + ret=CallMalloc(m, size, 0)); + } + return ret; +} +NEDMALLOCPTRATTR void * nedpcalloc(nedpool *p, size_t no, size_t size) THROWSPEC +{ + size_t rsize=size*no; + void *ret=0; + threadcache *tc; + int mymspace; + GetThreadCache(&p, &tc, &mymspace, &rsize); +#if THREADCACHEMAX + if(tc && rsize<=THREADCACHEMAX) + { /* Use the thread cache */ + if((ret=threadcache_malloc(p, tc, &rsize))) + memset(ret, 0, rsize); + } +#endif + if(!ret) + { /* Use this thread's mspace */ + GETMSPACE(m, p, tc, mymspace, rsize, + ret=CallCalloc(m, rsize, 0)); + } + return ret; +} +NEDMALLOCPTRATTR void * nedprealloc(nedpool *p, void *mem, size_t size) THROWSPEC +{ + void *ret=0; + threadcache *tc; + int mymspace, isforeign=1; + size_t memsize; + if(!mem) return nedpmalloc(p, size); + memsize=nedblksize(&isforeign, mem); + assert(memsize); + if(!memsize) + { + fprintf(stderr, "nedmalloc: nedprealloc() called with a block not created by nedmalloc!\n"); + abort(); + } + else if(size<=memsize && memsize-size< +#ifdef DEBUG + 32 +#else + 1024 +#endif + ) /* If realloc size is within 1Kb smaller than existing, noop it */ + return mem; + GetThreadCache(&p, &tc, &mymspace, &size); +#if THREADCACHEMAX + if(tc && size && size<=THREADCACHEMAX) + { /* Use the thread cache */ + if((ret=threadcache_malloc(p, tc, &size))) + { + memcpy(ret, mem, memsize<size ? memsize : size); + if(memsize>=sizeof(threadcacheblk) && memsize<=(THREADCACHEMAX+CHUNK_OVERHEAD)) + threadcache_free(p, tc, mymspace, mem, memsize); + else + CallFree(0, mem, isforeign); + } + } +#endif + if(!ret) + { /* Reallocs always happen in the mspace they happened in, so skip + locking the preferred mspace for this thread */ + ret=CallRealloc(p->m[mymspace], mem, isforeign, memsize, size); + } + return ret; +} +void nedpfree(nedpool *p, void *mem) THROWSPEC +{ /* Frees always happen in the mspace they happened in, so skip + locking the preferred mspace for this thread */ + threadcache *tc; + int mymspace, isforeign=1; + size_t memsize; + if(!mem) + { /* If you tried this on FreeBSD you'd be sorry! */ +#ifdef DEBUG + fprintf(stderr, "nedmalloc: WARNING nedpfree() called with zero. This is not portable behaviour!\n"); +#endif + return; + } + memsize=nedblksize(&isforeign, mem); + assert(memsize); + if(!memsize) + { + fprintf(stderr, "nedmalloc: nedpfree() called with a block not created by nedmalloc!\n"); + abort(); + } + GetThreadCache(&p, &tc, &mymspace, 0); +#if THREADCACHEMAX + if(mem && tc && memsize>=sizeof(threadcacheblk) && memsize<=(THREADCACHEMAX+CHUNK_OVERHEAD)) + threadcache_free(p, tc, mymspace, mem, memsize); + else +#endif + CallFree(0, mem, isforeign); +} +NEDMALLOCPTRATTR void * nedpmemalign(nedpool *p, size_t alignment, size_t bytes) THROWSPEC +{ + void *ret; + threadcache *tc; + int mymspace; + GetThreadCache(&p, &tc, &mymspace, &bytes); + { /* Use this thread's mspace */ + GETMSPACE(m, p, tc, mymspace, bytes, + ret=CallMalloc(m, bytes, alignment)); + } + return ret; +} +struct nedmallinfo nedpmallinfo(nedpool *p) THROWSPEC +{ + int n; + struct nedmallinfo ret={0}; + if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } + for(n=0; p->m[n]; n++) + { +#if USE_ALLOCATOR==1 && !NO_MALLINFO + struct mallinfo t=mspace_mallinfo(p->m[n]); + ret.arena+=t.arena; + ret.ordblks+=t.ordblks; + ret.hblkhd+=t.hblkhd; + ret.usmblks+=t.usmblks; + ret.uordblks+=t.uordblks; + ret.fordblks+=t.fordblks; + ret.keepcost+=t.keepcost; +#endif + } + return ret; +} +int nedpmallopt(nedpool *p, int parno, int value) THROWSPEC +{ +#if USE_ALLOCATOR==1 + return mspace_mallopt(parno, value); +#else + return 0; +#endif +} +NEDMALLOCNOALIASATTR void* nedmalloc_internals(size_t *granularity, size_t *magic) THROWSPEC +{ +#if USE_ALLOCATOR==1 + if(granularity) *granularity=mparams.granularity; + if(magic) *magic=mparams.magic; + return (void *) &syspool; +#else + if(granularity) *granularity=0; + if(magic) *magic=0; + return 0; +#endif +} +int nedpmalloc_trim(nedpool *p, size_t pad) THROWSPEC +{ + int n, ret=0; + if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } + for(n=0; p->m[n]; n++) + { +#if USE_ALLOCATOR==1 + ret+=mspace_trim(p->m[n], pad); +#endif + } + return ret; +} +void nedpmalloc_stats(nedpool *p) THROWSPEC +{ + int n; + if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } + for(n=0; p->m[n]; n++) + { +#if USE_ALLOCATOR==1 + mspace_malloc_stats(p->m[n]); +#endif + } +} +size_t nedpmalloc_footprint(nedpool *p) THROWSPEC +{ + size_t ret=0; + int n; + if(!p) { p=&syspool; if(!syspool.threads) InitPool(&syspool, 0, -1); } + for(n=0; p->m[n]; n++) + { +#if USE_ALLOCATOR==1 + ret+=mspace_footprint(p->m[n]); +#endif + } + return ret; +} +NEDMALLOCPTRATTR void **nedpindependent_calloc(nedpool *p, size_t elemsno, size_t elemsize, void **chunks) THROWSPEC +{ + void **ret; + threadcache *tc; + int mymspace; + GetThreadCache(&p, &tc, &mymspace, &elemsize); +#if USE_ALLOCATOR==0 + GETMSPACE(m, p, tc, mymspace, elemsno*elemsize, + ret=unsupported_operation("independent_calloc")); +#elif USE_ALLOCATOR==1 + GETMSPACE(m, p, tc, mymspace, elemsno*elemsize, + ret=mspace_independent_calloc(m, elemsno, elemsize, chunks)); +#endif + return ret; +} +NEDMALLOCPTRATTR void **nedpindependent_comalloc(nedpool *p, size_t elems, size_t *sizes, void **chunks) THROWSPEC +{ + void **ret; + threadcache *tc; + int mymspace; + size_t i, *adjustedsizes=(size_t *) alloca(elems*sizeof(size_t)); + if(!adjustedsizes) return 0; + for(i=0; i<elems; i++) + adjustedsizes[i]=sizes[i]<sizeof(threadcacheblk) ? sizeof(threadcacheblk) : sizes[i]; + GetThreadCache(&p, &tc, &mymspace, 0); +#if USE_ALLOCATOR==0 + GETMSPACE(m, p, tc, mymspace, 0, + ret=unsupported_operation("independent_comalloc")); +#elif USE_ALLOCATOR==1 + GETMSPACE(m, p, tc, mymspace, 0, + ret=mspace_independent_comalloc(m, elems, adjustedsizes, chunks)); +#endif + return ret; +} + +#if defined(__cplusplus) +} +#endif + +#ifdef _MSC_VER +#pragma warning(pop) +#endif + +#endif diff --git a/drivers/nedmalloc/nedmalloc.h b/drivers/nedmalloc/nedmalloc.h index b9add1683a..7ec65849fc 100644 --- a/drivers/nedmalloc/nedmalloc.h +++ b/drivers/nedmalloc/nedmalloc.h @@ -1,302 +1,302 @@ -#ifdef NEDMALLOC_ENABLED
-
-/* nedalloc, an alternative malloc implementation for multiple threads without
-lock contention based on dlmalloc v2.8.3. (C) 2005-2009 Niall Douglas
-
-Boost Software License - Version 1.0 - August 17th, 2003
-
-Permission is hereby granted, free of charge, to any person or organization
-obtaining a copy of the software and accompanying documentation covered by
-this license (the "Software") to use, reproduce, display, distribute,
-execute, and transmit the Software, and to prepare derivative works of the
-Software, and to permit third-parties to whom the Software is furnished to
-do so, all subject to the following:
-
-The copyright notices in the Software and this entire statement, including
-the above license grant, this restriction and the following disclaimer,
-must be included in all copies of the Software, in whole or in part, and
-all derivative works of the Software, unless such copies or derivative
-works are solely in the form of machine-executable object code generated by
-a source language processor.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
-IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
-FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT
-SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE
-FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
-DEALINGS IN THE SOFTWARE.
-*/
-
-#ifndef NEDMALLOC_H
-#define NEDMALLOC_H
-
-#include "typedefs.h"
-#define MALLOC_ALIGNMENT DEFAULT_ALIGNMENT
-
-#ifdef PSP_ENABLED
-#define USE_LOCKS 0
-#define HAVE_MMAP 0
-#endif
-
-/* See malloc.c.h for what each function does.
-
-REPLACE_SYSTEM_ALLOCATOR on POSIX causes nedalloc's functions to be called
-malloc, free etc. instead of nedmalloc, nedfree etc. You may or may not want
-this. On Windows it causes nedmalloc to patch all loaded DLLs and binaries
-to replace usage of the system allocator.
-
-NO_NED_NAMESPACE prevents the functions from being defined in the nedalloc
-namespace when in C++ (uses the global namespace instead).
-
-NEDMALLOCEXTSPEC can be defined to be __declspec(dllexport) or
-__attribute__ ((visibility("default"))) or whatever you like. It defaults
-to extern unless NEDMALLOC_DLL_EXPORTS is set as it would be when building
-nedmalloc.dll.
-
-USE_LOCKS can be 2 if you want to define your own MLOCK_T, INITIAL_LOCK,
-ACQUIRE_LOCK, RELEASE_LOCK, TRY_LOCK, IS_LOCKED and NULL_LOCK_INITIALIZER.
-
-NEDMALLOC_DEBUG can be defined to cause DEBUG to be set differently for nedmalloc
-than for the rest of the build. Remember to set NDEBUG to disable all assertion
-checking too.
-
-USE_MAGIC_HEADERS causes nedalloc to allocate an extra three sizeof(size_t)
-to each block. nedpfree() and nedprealloc() can then automagically know when
-to free a system allocated block. Enabling this typically adds 20-50% to
-application memory usage.
-
-ENABLE_TOLERANT_NEDMALLOC is automatically turned on if REPLACE_SYSTEM_ALLOCATOR
-is set or the Windows DLL is being built. This causes nedmalloc to detect when a
-system allocator block is passed to it and to handle it appropriately. Note that
-without USE_MAGIC_HEADERS there is a very tiny chance that nedmalloc will segfault
-on non-Windows builds (it uses Win32 SEH to trap segfaults on Windows and there
-is no comparable system on POSIX).
-
-USE_ALLOCATOR can be one of these settings (it defaults to 1):
- 0: System allocator (nedmalloc now simply acts as a threadcache).
- WARNING: Intended for DEBUG USE ONLY - not all functions work correctly.
- 1: dlmalloc
-
-ENABLE_LARGE_PAGES enables support for requesting memory from the system in large
-(typically >=2Mb) pages if the host OS supports this. These occupy just a single
-TLB entry and can significantly improve performance in large working set applications.
-
-ENABLE_FAST_HEAP_DETECTION enables special logic to detect blocks allocated
-by the system heap. This avoids 1.5%-2% overhead when checking for non-nedmalloc
-blocks, but it assumes that the NT and glibc heaps function in a very specific
-fashion which may not hold true across OS upgrades.
-*/
-
-#include <stddef.h> /* for size_t */
-
-#ifndef NEDMALLOCEXTSPEC
- #ifdef NEDMALLOC_DLL_EXPORTS
- #ifdef WIN32
- #define NEDMALLOCEXTSPEC extern __declspec(dllexport)
- #elif defined(__GNUC__)
- #define NEDMALLOCEXTSPEC extern __attribute__ ((visibility("default")))
- #endif
- #ifndef ENABLE_TOLERANT_NEDMALLOC
- #define ENABLE_TOLERANT_NEDMALLOC 1
- #endif
- #else
- #define NEDMALLOCEXTSPEC extern
- #endif
-#endif
-
-#if __STDC_VERSION__ >= 199901L /* C99 or better */
- #define RESTRICT restrict
-#else
- #if defined(_MSC_VER) && _MSC_VER>=1400
- #define RESTRICT __restrict
- #endif
- #ifdef __GNUC__
- #define RESTRICT __restrict
- #endif
-#endif
-#ifndef RESTRICT
- #define RESTRICT
-#endif
-
-#if defined(_MSC_VER) && _MSC_VER>=1400
- #define NEDMALLOCPTRATTR __declspec(restrict)
- #define NEDMALLOCNOALIASATTR __declspec(noalias)
-#endif
-#ifdef __GNUC__
- #define NEDMALLOCPTRATTR __attribute__ ((malloc))
-#endif
-#ifndef NEDMALLOCPTRATTR
- #define NEDMALLOCPTRATTR
-#endif
-#ifndef NEDMALLOCNOALIASATTR
- #define NEDMALLOCNOALIASATTR
-#endif
-
-#ifndef USE_MAGIC_HEADERS
- #define USE_MAGIC_HEADERS 0
-#endif
-
-#ifndef USE_ALLOCATOR
- #define USE_ALLOCATOR 1 /* dlmalloc */
-#endif
-
-#if !USE_ALLOCATOR && !USE_MAGIC_HEADERS
-#error If you are using the system allocator then you MUST use magic headers
-#endif
-
-#ifdef REPLACE_SYSTEM_ALLOCATOR
- #if USE_ALLOCATOR==0
- #error Cannot combine using the system allocator with replacing the system allocator
- #endif
- #ifndef ENABLE_TOLERANT_NEDMALLOC
- #define ENABLE_TOLERANT_NEDMALLOC 1
- #endif
- #ifndef WIN32 /* We have a dedicated patcher for Windows */
- #define nedmalloc malloc
- #define nedcalloc calloc
- #define nedrealloc realloc
- #define nedfree free
- #define nedmemalign memalign
- #define nedmallinfo mallinfo
- #define nedmallopt mallopt
- #define nedmalloc_trim malloc_trim
- #define nedmalloc_stats malloc_stats
- #define nedmalloc_footprint malloc_footprint
- #define nedindependent_calloc independent_calloc
- #define nedindependent_comalloc independent_comalloc
- #ifdef _MSC_VER
- #define nedblksize _msize
- #endif
- #endif
-#endif
-
-#if defined(__cplusplus)
-extern "C" {
-#endif
-struct nedmallinfo {
- size_t arena; /* non-mmapped space allocated from system */
- size_t ordblks; /* number of free chunks */
- size_t smblks; /* always 0 */
- size_t hblks; /* always 0 */
- size_t hblkhd; /* space in mmapped regions */
- size_t usmblks; /* maximum total allocated space */
- size_t fsmblks; /* always 0 */
- size_t uordblks; /* total allocated space */
- size_t fordblks; /* total free space */
- size_t keepcost; /* releasable (via malloc_trim) space */
-};
-#if defined(__cplusplus)
-}
-#endif
-
-#if defined(__cplusplus)
- #if !defined(NO_NED_NAMESPACE)
-namespace nedalloc {
- #else
-extern "C" {
- #endif
- #define THROWSPEC throw()
-#else
- #define THROWSPEC
-#endif
-
-/* These are the global functions */
-
-/* Gets the usable size of an allocated block. Note this will always be bigger than what was
-asked for due to rounding etc. Optionally returns 1 in isforeign if the block came from the
-system allocator - note that there is a small (>0.01%) but real chance of segfault on non-Windows
-systems when passing non-nedmalloc blocks if you don't use USE_MAGIC_HEADERS.
-*/
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR size_t nedblksize(int *RESTRICT isforeign, void *RESTRICT mem) THROWSPEC;
-
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR void nedsetvalue(void *v) THROWSPEC;
-
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmalloc(size_t size) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedcalloc(size_t no, size_t size) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedrealloc(void *mem, size_t size) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR void nedfree(void *mem) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmemalign(size_t alignment, size_t bytes) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR struct nedmallinfo nedmallinfo(void) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR int nedmallopt(int parno, int value) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR void* nedmalloc_internals(size_t *granularity, size_t *magic) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR int nedmalloc_trim(size_t pad) THROWSPEC;
-NEDMALLOCEXTSPEC void nedmalloc_stats(void) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR size_t nedmalloc_footprint(void) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_calloc(size_t elemsno, size_t elemsize, void **chunks) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_comalloc(size_t elems, size_t *sizes, void **chunks) THROWSPEC;
-
-/* Destroys the system memory pool used by the functions above.
-Useful for when you have nedmalloc in a DLL you're about to unload.
-If you call ANY nedmalloc functions after calling this you will
-get a fatal exception!
-*/
-NEDMALLOCEXTSPEC void neddestroysyspool() THROWSPEC;
-
-/* These are the pool functions */
-struct nedpool_t;
-typedef struct nedpool_t nedpool;
-
-/* Creates a memory pool for use with the nedp* functions below.
-Capacity is how much to allocate immediately (if you know you'll be allocating a lot
-of memory very soon) which you can leave at zero. Threads specifies how many threads
-will *normally* be accessing the pool concurrently. Setting this to zero means it
-extends on demand, but be careful of this as it can rapidly consume system resources
-where bursts of concurrent threads use a pool at once.
-*/
-NEDMALLOCEXTSPEC NEDMALLOCPTRATTR nedpool *nedcreatepool(size_t capacity, int threads) THROWSPEC;
-
-/* Destroys a memory pool previously created by nedcreatepool().
-*/
-NEDMALLOCEXTSPEC void neddestroypool(nedpool *p) THROWSPEC;
-
-/* Returns a zero terminated snapshot of threadpools existing at the time of call. Call
-nedfree() on the returned list when you are done. Returns zero if there is only the
-system pool in existence.
-*/
-NEDMALLOCEXTSPEC nedpool **nedpoollist() THROWSPEC;
-
-/* Sets a value to be associated with a pool. You can retrieve this value by passing
-any memory block allocated from that pool.
-*/
-NEDMALLOCEXTSPEC void nedpsetvalue(nedpool *p, void *v) THROWSPEC;
-
-/* Gets a previously set value using nedpsetvalue() or zero if memory is unknown.
-Optionally can also retrieve pool. You can detect an unknown block by the return
-being zero and *p being unmodifed.
-*/
-NEDMALLOCEXTSPEC void *nedgetvalue(nedpool **p, void *mem) THROWSPEC;
-
-/* Trims the thread cache for the calling thread, returning any existing cache
-data to the central pool. Remember to ALWAYS call with zero if you used the
-system pool. Setting disable to non-zero replicates neddisablethreadcache().
-*/
-NEDMALLOCEXTSPEC void nedtrimthreadcache(nedpool *p, int disable) THROWSPEC;
-
-/* Disables the thread cache for the calling thread, returning any existing cache
-data to the central pool. Remember to ALWAYS call with zero if you used the
-system pool.
-*/
-NEDMALLOCEXTSPEC void neddisablethreadcache(nedpool *p) THROWSPEC;
-
-
-NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void * nedpmalloc(nedpool *p, size_t size) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void * nedpcalloc(nedpool *p, size_t no, size_t size) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void * nedprealloc(nedpool *p, void *mem, size_t size) THROWSPEC;
-NEDMALLOCEXTSPEC void nedpfree(nedpool *p, void *mem) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void * nedpmemalign(nedpool *p, size_t alignment, size_t bytes) THROWSPEC;
-NEDMALLOCEXTSPEC struct nedmallinfo nedpmallinfo(nedpool *p) THROWSPEC;
-NEDMALLOCEXTSPEC int nedpmallopt(nedpool *p, int parno, int value) THROWSPEC;
-NEDMALLOCEXTSPEC int nedpmalloc_trim(nedpool *p, size_t pad) THROWSPEC;
-NEDMALLOCEXTSPEC void nedpmalloc_stats(nedpool *p) THROWSPEC;
-NEDMALLOCEXTSPEC size_t nedpmalloc_footprint(nedpool *p) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void **nedpindependent_calloc(nedpool *p, size_t elemsno, size_t elemsize, void **chunks) THROWSPEC;
-NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void **nedpindependent_comalloc(nedpool *p, size_t elems, size_t *sizes, void **chunks) THROWSPEC;
-
-#if defined(__cplusplus)
-}
-#endif
-
-#endif
-
-#endif
+#ifdef NEDMALLOC_ENABLED + +/* nedalloc, an alternative malloc implementation for multiple threads without +lock contention based on dlmalloc v2.8.3. (C) 2005-2009 Niall Douglas + +Boost Software License - Version 1.0 - August 17th, 2003 + +Permission is hereby granted, free of charge, to any person or organization +obtaining a copy of the software and accompanying documentation covered by +this license (the "Software") to use, reproduce, display, distribute, +execute, and transmit the Software, and to prepare derivative works of the +Software, and to permit third-parties to whom the Software is furnished to +do so, all subject to the following: + +The copyright notices in the Software and this entire statement, including +the above license grant, this restriction and the following disclaimer, +must be included in all copies of the Software, in whole or in part, and +all derivative works of the Software, unless such copies or derivative +works are solely in the form of machine-executable object code generated by +a source language processor. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE, TITLE AND NON-INFRINGEMENT. IN NO EVENT +SHALL THE COPYRIGHT HOLDERS OR ANYONE DISTRIBUTING THE SOFTWARE BE LIABLE +FOR ANY DAMAGES OR OTHER LIABILITY, WHETHER IN CONTRACT, TORT OR OTHERWISE, +ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + +#ifndef NEDMALLOC_H +#define NEDMALLOC_H + +#include "typedefs.h" +#define MALLOC_ALIGNMENT DEFAULT_ALIGNMENT + +#ifdef PSP_ENABLED +#define USE_LOCKS 0 +#define HAVE_MMAP 0 +#endif + +/* See malloc.c.h for what each function does. + +REPLACE_SYSTEM_ALLOCATOR on POSIX causes nedalloc's functions to be called +malloc, free etc. instead of nedmalloc, nedfree etc. You may or may not want +this. On Windows it causes nedmalloc to patch all loaded DLLs and binaries +to replace usage of the system allocator. + +NO_NED_NAMESPACE prevents the functions from being defined in the nedalloc +namespace when in C++ (uses the global namespace instead). + +NEDMALLOCEXTSPEC can be defined to be __declspec(dllexport) or +__attribute__ ((visibility("default"))) or whatever you like. It defaults +to extern unless NEDMALLOC_DLL_EXPORTS is set as it would be when building +nedmalloc.dll. + +USE_LOCKS can be 2 if you want to define your own MLOCK_T, INITIAL_LOCK, +ACQUIRE_LOCK, RELEASE_LOCK, TRY_LOCK, IS_LOCKED and NULL_LOCK_INITIALIZER. + +NEDMALLOC_DEBUG can be defined to cause DEBUG to be set differently for nedmalloc +than for the rest of the build. Remember to set NDEBUG to disable all assertion +checking too. + +USE_MAGIC_HEADERS causes nedalloc to allocate an extra three sizeof(size_t) +to each block. nedpfree() and nedprealloc() can then automagically know when +to free a system allocated block. Enabling this typically adds 20-50% to +application memory usage. + +ENABLE_TOLERANT_NEDMALLOC is automatically turned on if REPLACE_SYSTEM_ALLOCATOR +is set or the Windows DLL is being built. This causes nedmalloc to detect when a +system allocator block is passed to it and to handle it appropriately. Note that +without USE_MAGIC_HEADERS there is a very tiny chance that nedmalloc will segfault +on non-Windows builds (it uses Win32 SEH to trap segfaults on Windows and there +is no comparable system on POSIX). + +USE_ALLOCATOR can be one of these settings (it defaults to 1): + 0: System allocator (nedmalloc now simply acts as a threadcache). + WARNING: Intended for DEBUG USE ONLY - not all functions work correctly. + 1: dlmalloc + +ENABLE_LARGE_PAGES enables support for requesting memory from the system in large +(typically >=2Mb) pages if the host OS supports this. These occupy just a single +TLB entry and can significantly improve performance in large working set applications. + +ENABLE_FAST_HEAP_DETECTION enables special logic to detect blocks allocated +by the system heap. This avoids 1.5%-2% overhead when checking for non-nedmalloc +blocks, but it assumes that the NT and glibc heaps function in a very specific +fashion which may not hold true across OS upgrades. +*/ + +#include <stddef.h> /* for size_t */ + +#ifndef NEDMALLOCEXTSPEC + #ifdef NEDMALLOC_DLL_EXPORTS + #ifdef WIN32 + #define NEDMALLOCEXTSPEC extern __declspec(dllexport) + #elif defined(__GNUC__) + #define NEDMALLOCEXTSPEC extern __attribute__ ((visibility("default"))) + #endif + #ifndef ENABLE_TOLERANT_NEDMALLOC + #define ENABLE_TOLERANT_NEDMALLOC 1 + #endif + #else + #define NEDMALLOCEXTSPEC extern + #endif +#endif + +#if __STDC_VERSION__ >= 199901L /* C99 or better */ + #define RESTRICT restrict +#else + #if defined(_MSC_VER) && _MSC_VER>=1400 + #define RESTRICT __restrict + #endif + #ifdef __GNUC__ + #define RESTRICT __restrict + #endif +#endif +#ifndef RESTRICT + #define RESTRICT +#endif + +#if defined(_MSC_VER) && _MSC_VER>=1400 + #define NEDMALLOCPTRATTR __declspec(restrict) + #define NEDMALLOCNOALIASATTR __declspec(noalias) +#endif +#ifdef __GNUC__ + #define NEDMALLOCPTRATTR __attribute__ ((malloc)) +#endif +#ifndef NEDMALLOCPTRATTR + #define NEDMALLOCPTRATTR +#endif +#ifndef NEDMALLOCNOALIASATTR + #define NEDMALLOCNOALIASATTR +#endif + +#ifndef USE_MAGIC_HEADERS + #define USE_MAGIC_HEADERS 0 +#endif + +#ifndef USE_ALLOCATOR + #define USE_ALLOCATOR 1 /* dlmalloc */ +#endif + +#if !USE_ALLOCATOR && !USE_MAGIC_HEADERS +#error If you are using the system allocator then you MUST use magic headers +#endif + +#ifdef REPLACE_SYSTEM_ALLOCATOR + #if USE_ALLOCATOR==0 + #error Cannot combine using the system allocator with replacing the system allocator + #endif + #ifndef ENABLE_TOLERANT_NEDMALLOC + #define ENABLE_TOLERANT_NEDMALLOC 1 + #endif + #ifndef WIN32 /* We have a dedicated patcher for Windows */ + #define nedmalloc malloc + #define nedcalloc calloc + #define nedrealloc realloc + #define nedfree free + #define nedmemalign memalign + #define nedmallinfo mallinfo + #define nedmallopt mallopt + #define nedmalloc_trim malloc_trim + #define nedmalloc_stats malloc_stats + #define nedmalloc_footprint malloc_footprint + #define nedindependent_calloc independent_calloc + #define nedindependent_comalloc independent_comalloc + #ifdef _MSC_VER + #define nedblksize _msize + #endif + #endif +#endif + +#if defined(__cplusplus) +extern "C" { +#endif +struct nedmallinfo { + size_t arena; /* non-mmapped space allocated from system */ + size_t ordblks; /* number of free chunks */ + size_t smblks; /* always 0 */ + size_t hblks; /* always 0 */ + size_t hblkhd; /* space in mmapped regions */ + size_t usmblks; /* maximum total allocated space */ + size_t fsmblks; /* always 0 */ + size_t uordblks; /* total allocated space */ + size_t fordblks; /* total free space */ + size_t keepcost; /* releasable (via malloc_trim) space */ +}; +#if defined(__cplusplus) +} +#endif + +#if defined(__cplusplus) + #if !defined(NO_NED_NAMESPACE) +namespace nedalloc { + #else +extern "C" { + #endif + #define THROWSPEC throw() +#else + #define THROWSPEC +#endif + +/* These are the global functions */ + +/* Gets the usable size of an allocated block. Note this will always be bigger than what was +asked for due to rounding etc. Optionally returns 1 in isforeign if the block came from the +system allocator - note that there is a small (>0.01%) but real chance of segfault on non-Windows +systems when passing non-nedmalloc blocks if you don't use USE_MAGIC_HEADERS. +*/ +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR size_t nedblksize(int *RESTRICT isforeign, void *RESTRICT mem) THROWSPEC; + +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR void nedsetvalue(void *v) THROWSPEC; + +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmalloc(size_t size) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedcalloc(size_t no, size_t size) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedrealloc(void *mem, size_t size) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR void nedfree(void *mem) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void * nedmemalign(size_t alignment, size_t bytes) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR struct nedmallinfo nedmallinfo(void) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR int nedmallopt(int parno, int value) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR void* nedmalloc_internals(size_t *granularity, size_t *magic) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR int nedmalloc_trim(size_t pad) THROWSPEC; +NEDMALLOCEXTSPEC void nedmalloc_stats(void) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR size_t nedmalloc_footprint(void) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_calloc(size_t elemsno, size_t elemsize, void **chunks) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCNOALIASATTR NEDMALLOCPTRATTR void **nedindependent_comalloc(size_t elems, size_t *sizes, void **chunks) THROWSPEC; + +/* Destroys the system memory pool used by the functions above. +Useful for when you have nedmalloc in a DLL you're about to unload. +If you call ANY nedmalloc functions after calling this you will +get a fatal exception! +*/ +NEDMALLOCEXTSPEC void neddestroysyspool() THROWSPEC; + +/* These are the pool functions */ +struct nedpool_t; +typedef struct nedpool_t nedpool; + +/* Creates a memory pool for use with the nedp* functions below. +Capacity is how much to allocate immediately (if you know you'll be allocating a lot +of memory very soon) which you can leave at zero. Threads specifies how many threads +will *normally* be accessing the pool concurrently. Setting this to zero means it +extends on demand, but be careful of this as it can rapidly consume system resources +where bursts of concurrent threads use a pool at once. +*/ +NEDMALLOCEXTSPEC NEDMALLOCPTRATTR nedpool *nedcreatepool(size_t capacity, int threads) THROWSPEC; + +/* Destroys a memory pool previously created by nedcreatepool(). +*/ +NEDMALLOCEXTSPEC void neddestroypool(nedpool *p) THROWSPEC; + +/* Returns a zero terminated snapshot of threadpools existing at the time of call. Call +nedfree() on the returned list when you are done. Returns zero if there is only the +system pool in existence. +*/ +NEDMALLOCEXTSPEC nedpool **nedpoollist() THROWSPEC; + +/* Sets a value to be associated with a pool. You can retrieve this value by passing +any memory block allocated from that pool. +*/ +NEDMALLOCEXTSPEC void nedpsetvalue(nedpool *p, void *v) THROWSPEC; + +/* Gets a previously set value using nedpsetvalue() or zero if memory is unknown. +Optionally can also retrieve pool. You can detect an unknown block by the return +being zero and *p being unmodifed. +*/ +NEDMALLOCEXTSPEC void *nedgetvalue(nedpool **p, void *mem) THROWSPEC; + +/* Trims the thread cache for the calling thread, returning any existing cache +data to the central pool. Remember to ALWAYS call with zero if you used the +system pool. Setting disable to non-zero replicates neddisablethreadcache(). +*/ +NEDMALLOCEXTSPEC void nedtrimthreadcache(nedpool *p, int disable) THROWSPEC; + +/* Disables the thread cache for the calling thread, returning any existing cache +data to the central pool. Remember to ALWAYS call with zero if you used the +system pool. +*/ +NEDMALLOCEXTSPEC void neddisablethreadcache(nedpool *p) THROWSPEC; + + +NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void * nedpmalloc(nedpool *p, size_t size) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void * nedpcalloc(nedpool *p, size_t no, size_t size) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void * nedprealloc(nedpool *p, void *mem, size_t size) THROWSPEC; +NEDMALLOCEXTSPEC void nedpfree(nedpool *p, void *mem) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void * nedpmemalign(nedpool *p, size_t alignment, size_t bytes) THROWSPEC; +NEDMALLOCEXTSPEC struct nedmallinfo nedpmallinfo(nedpool *p) THROWSPEC; +NEDMALLOCEXTSPEC int nedpmallopt(nedpool *p, int parno, int value) THROWSPEC; +NEDMALLOCEXTSPEC int nedpmalloc_trim(nedpool *p, size_t pad) THROWSPEC; +NEDMALLOCEXTSPEC void nedpmalloc_stats(nedpool *p) THROWSPEC; +NEDMALLOCEXTSPEC size_t nedpmalloc_footprint(nedpool *p) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void **nedpindependent_calloc(nedpool *p, size_t elemsno, size_t elemsize, void **chunks) THROWSPEC; +NEDMALLOCEXTSPEC NEDMALLOCPTRATTR void **nedpindependent_comalloc(nedpool *p, size_t elems, size_t *sizes, void **chunks) THROWSPEC; + +#if defined(__cplusplus) +} +#endif + +#endif + +#endif diff --git a/drivers/openssl/register_openssl.cpp b/drivers/openssl/register_openssl.cpp index a4a60813b6..ed2150bef5 100644 --- a/drivers/openssl/register_openssl.cpp +++ b/drivers/openssl/register_openssl.cpp @@ -1,19 +1,19 @@ -#include "register_openssl.h"
-
-#include "stream_peer_openssl.h"
-#ifdef OPENSSL_ENABLED
-
-void register_openssl() {
-
- ObjectTypeDB::register_type<StreamPeerOpenSSL>();
- StreamPeerOpenSSL::initialize_ssl();
-
-}
-
-void unregister_openssl() {
-
- StreamPeerOpenSSL::finalize_ssl();
-
-}
-#endif
-
+#include "register_openssl.h" + +#include "stream_peer_openssl.h" +#ifdef OPENSSL_ENABLED + +void register_openssl() { + + ObjectTypeDB::register_type<StreamPeerOpenSSL>(); + StreamPeerOpenSSL::initialize_ssl(); + +} + +void unregister_openssl() { + + StreamPeerOpenSSL::finalize_ssl(); + +} +#endif + diff --git a/drivers/openssl/register_openssl.h b/drivers/openssl/register_openssl.h index e1c554ca4a..e547a2b750 100644 --- a/drivers/openssl/register_openssl.h +++ b/drivers/openssl/register_openssl.h @@ -1,11 +1,11 @@ -#ifndef REGISTER_OPENSSL_H
-#define REGISTER_OPENSSL_H
-
-#ifdef OPENSSL_ENABLED
-
-void register_openssl();
-void unregister_openssl();
-
-#endif
-
-#endif // REGISTER_OPENSSL_H
+#ifndef REGISTER_OPENSSL_H +#define REGISTER_OPENSSL_H + +#ifdef OPENSSL_ENABLED + +void register_openssl(); +void unregister_openssl(); + +#endif + +#endif // REGISTER_OPENSSL_H diff --git a/drivers/rtaudio/RtAudio.cpp b/drivers/rtaudio/RtAudio.cpp index 8876f72e21..72ca836907 100644 --- a/drivers/rtaudio/RtAudio.cpp +++ b/drivers/rtaudio/RtAudio.cpp @@ -1,10234 +1,10234 @@ -#ifdef RTAUDIO_ENABLED
-/************************************************************************/
-/*! \class RtAudio
- \brief Realtime audio i/o C++ classes.
-
- RtAudio provides a common API (Application Programming Interface)
- for realtime audio input/output across Linux (native ALSA, Jack,
- and OSS), Macintosh OS X (CoreAudio and Jack), and Windows
- (DirectSound, ASIO and WASAPI) operating systems.
-
- RtAudio WWW site: http://www.music.mcgill.ca/~gary/rtaudio/
-
- RtAudio: realtime audio i/o C++ classes
- Copyright (c) 2001-2014 Gary P. Scavone
-
- Permission is hereby granted, free of charge, to any person
- obtaining a copy of this software and associated documentation files
- (the "Software"), to deal in the Software without restriction,
- including without limitation the rights to use, copy, modify, merge,
- publish, distribute, sublicense, and/or sell copies of the Software,
- and to permit persons to whom the Software is furnished to do so,
- subject to the following conditions:
-
- The above copyright notice and this permission notice shall be
- included in all copies or substantial portions of the Software.
-
- Any person wishing to distribute modifications to the Software is
- asked to send the modifications to the original developer so that
- they can be incorporated into the canonical version. This is,
- however, not a binding provision of this license.
-
- THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
- EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
- MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
- IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
- ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF
- CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
- WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-*/
-/************************************************************************/
-
-// RtAudio: Version 4.1.1
-
-#include "RtAudio.h"
-#include <iostream>
-#include <cstdlib>
-#include <cstring>
-#include <climits>
-#include <algorithm>
-
-// Static variable definitions.
-const unsigned int RtApi::MAX_SAMPLE_RATES = 14;
-const unsigned int RtApi::SAMPLE_RATES[] = {
- 4000, 5512, 8000, 9600, 11025, 16000, 22050,
- 32000, 44100, 48000, 88200, 96000, 176400, 192000
-};
-
-#if defined(__WINDOWS_DS__) || defined(__WINDOWS_ASIO__) || defined(__WINDOWS_WASAPI__)
-#ifdef WINRT_ENABLED
- #define MUTEX_INITIALIZE(A) InitializeCriticalSectionEx(A, 0, 0)
-#else
- #define MUTEX_INITIALIZE(A) InitializeCriticalSection(A)
-#endif
- #define MUTEX_DESTROY(A) DeleteCriticalSection(A)
- #define MUTEX_LOCK(A) EnterCriticalSection(A)
- #define MUTEX_UNLOCK(A) LeaveCriticalSection(A)
-
- #include "tchar.h"
-
- static std::string convertCharPointerToStdString(const char *text)
- {
- return std::string(text);
- }
-
- static std::string convertCharPointerToStdString(const wchar_t *text)
- {
- int length = WideCharToMultiByte(CP_UTF8, 0, text, -1, NULL, 0, NULL, NULL);
- std::string s( length-1, '\0' );
- WideCharToMultiByte(CP_UTF8, 0, text, -1, &s[0], length, NULL, NULL);
- return s;
- }
-
-#elif defined(__LINUX_ALSA__) || defined(__LINUX_PULSE__) || defined(__UNIX_JACK__) || defined(__LINUX_OSS__) || defined(__MACOSX_CORE__)
- // pthread API
- #define MUTEX_INITIALIZE(A) pthread_mutex_init(A, NULL)
- #define MUTEX_DESTROY(A) pthread_mutex_destroy(A)
- #define MUTEX_LOCK(A) pthread_mutex_lock(A)
- #define MUTEX_UNLOCK(A) pthread_mutex_unlock(A)
-#else
- #define MUTEX_INITIALIZE(A) abs(*A) // dummy definitions
- #define MUTEX_DESTROY(A) abs(*A) // dummy definitions
-#endif
-
-// *************************************************** //
-//
-// RtAudio definitions.
-//
-// *************************************************** //
-
-std::string RtAudio :: getVersion( void ) throw()
-{
- return RTAUDIO_VERSION;
-}
-
-void RtAudio :: getCompiledApi( std::vector<RtAudio::Api> &apis ) throw()
-{
- apis.clear();
-
- // The order here will control the order of RtAudio's API search in
- // the constructor.
-#if defined(__UNIX_JACK__)
- apis.push_back( UNIX_JACK );
-#endif
-#if defined(__LINUX_ALSA__)
- apis.push_back( LINUX_ALSA );
-#endif
-#if defined(__LINUX_PULSE__)
- apis.push_back( LINUX_PULSE );
-#endif
-#if defined(__LINUX_OSS__)
- apis.push_back( LINUX_OSS );
-#endif
-#if defined(__WINDOWS_ASIO__)
- apis.push_back( WINDOWS_ASIO );
-#endif
-#if defined(__WINDOWS_WASAPI__)
- apis.push_back( WINDOWS_WASAPI );
-#endif
-#if defined(__WINDOWS_DS__)
- apis.push_back( WINDOWS_DS );
-#endif
-#if defined(__MACOSX_CORE__)
- apis.push_back( MACOSX_CORE );
-#endif
-#if defined(__RTAUDIO_DUMMY__)
- apis.push_back( RTAUDIO_DUMMY );
-#endif
-}
-
-void RtAudio :: openRtApi( RtAudio::Api api )
-{
- if ( rtapi_ )
- delete rtapi_;
- rtapi_ = 0;
-
-#if defined(__UNIX_JACK__)
- if ( api == UNIX_JACK )
- rtapi_ = new RtApiJack();
-#endif
-#if defined(__LINUX_ALSA__)
- if ( api == LINUX_ALSA )
- rtapi_ = new RtApiAlsa();
-#endif
-#if defined(__LINUX_PULSE__)
- if ( api == LINUX_PULSE )
- rtapi_ = new RtApiPulse();
-#endif
-#if defined(__LINUX_OSS__)
- if ( api == LINUX_OSS )
- rtapi_ = new RtApiOss();
-#endif
-#if defined(__WINDOWS_ASIO__)
- if ( api == WINDOWS_ASIO )
- rtapi_ = new RtApiAsio();
-#endif
-#if defined(__WINDOWS_WASAPI__)
- if ( api == WINDOWS_WASAPI )
- rtapi_ = new RtApiWasapi();
-#endif
-#if defined(__WINDOWS_DS__)
- if ( api == WINDOWS_DS )
- rtapi_ = new RtApiDs();
-#endif
-#if defined(__MACOSX_CORE__)
- if ( api == MACOSX_CORE )
- rtapi_ = new RtApiCore();
-#endif
-#if defined(__RTAUDIO_DUMMY__)
- if ( api == RTAUDIO_DUMMY )
- rtapi_ = new RtApiDummy();
-#endif
-}
-
-RtAudio :: RtAudio( RtAudio::Api api )
-{
- rtapi_ = 0;
-
- if ( api != UNSPECIFIED ) {
- // Attempt to open the specified API.
- openRtApi( api );
- if ( rtapi_ ) return;
-
- // No compiled support for specified API value. Issue a debug
- // warning and continue as if no API was specified.
- std::cerr << "\nRtAudio: no compiled support for specified API argument!\n" << std::endl;
- }
-
- // Iterate through the compiled APIs and return as soon as we find
- // one with at least one device or we reach the end of the list.
- std::vector< RtAudio::Api > apis;
- getCompiledApi( apis );
- for ( unsigned int i=0; i<apis.size(); i++ ) {
- openRtApi( apis[i] );
- if ( rtapi_ && rtapi_->getDeviceCount() ) break;
- }
-
- if ( rtapi_ ) return;
-
- // It should not be possible to get here because the preprocessor
- // definition __RTAUDIO_DUMMY__ is automatically defined if no
- // API-specific definitions are passed to the compiler. But just in
- // case something weird happens, we'll thow an error.
- std::string errorText = "\nRtAudio: no compiled API support found ... critical error!!\n\n";
- throw( RtAudioError( errorText, RtAudioError::UNSPECIFIED ) );
-}
-
-RtAudio :: ~RtAudio() throw()
-{
- if ( rtapi_ )
- delete rtapi_;
-}
-
-void RtAudio :: openStream( RtAudio::StreamParameters *outputParameters,
- RtAudio::StreamParameters *inputParameters,
- RtAudioFormat format, unsigned int sampleRate,
- unsigned int *bufferFrames,
- RtAudioCallback callback, void *userData,
- RtAudio::StreamOptions *options,
- RtAudioErrorCallback errorCallback )
-{
- return rtapi_->openStream( outputParameters, inputParameters, format,
- sampleRate, bufferFrames, callback,
- userData, options, errorCallback );
-}
-
-// *************************************************** //
-//
-// Public RtApi definitions (see end of file for
-// private or protected utility functions).
-//
-// *************************************************** //
-
-RtApi :: RtApi()
-{
- stream_.state = STREAM_CLOSED;
- stream_.mode = UNINITIALIZED;
- stream_.apiHandle = 0;
- stream_.userBuffer[0] = 0;
- stream_.userBuffer[1] = 0;
- MUTEX_INITIALIZE( &stream_.mutex );
- showWarnings_ = true;
- firstErrorOccurred_ = false;
-}
-
-RtApi :: ~RtApi()
-{
- MUTEX_DESTROY( &stream_.mutex );
-}
-
-void RtApi :: openStream( RtAudio::StreamParameters *oParams,
- RtAudio::StreamParameters *iParams,
- RtAudioFormat format, unsigned int sampleRate,
- unsigned int *bufferFrames,
- RtAudioCallback callback, void *userData,
- RtAudio::StreamOptions *options,
- RtAudioErrorCallback errorCallback )
-{
- if ( stream_.state != STREAM_CLOSED ) {
- errorText_ = "RtApi::openStream: a stream is already open!";
- error( RtAudioError::INVALID_USE );
- return;
- }
-
- // Clear stream information potentially left from a previously open stream.
- clearStreamInfo();
-
- if ( oParams && oParams->nChannels < 1 ) {
- errorText_ = "RtApi::openStream: a non-NULL output StreamParameters structure cannot have an nChannels value less than one.";
- error( RtAudioError::INVALID_USE );
- return;
- }
-
- if ( iParams && iParams->nChannels < 1 ) {
- errorText_ = "RtApi::openStream: a non-NULL input StreamParameters structure cannot have an nChannels value less than one.";
- error( RtAudioError::INVALID_USE );
- return;
- }
-
- if ( oParams == NULL && iParams == NULL ) {
- errorText_ = "RtApi::openStream: input and output StreamParameters structures are both NULL!";
- error( RtAudioError::INVALID_USE );
- return;
- }
-
- if ( formatBytes(format) == 0 ) {
- errorText_ = "RtApi::openStream: 'format' parameter value is undefined.";
- error( RtAudioError::INVALID_USE );
- return;
- }
-
- unsigned int nDevices = getDeviceCount();
- unsigned int oChannels = 0;
- if ( oParams ) {
- oChannels = oParams->nChannels;
- if ( oParams->deviceId >= nDevices ) {
- errorText_ = "RtApi::openStream: output device parameter value is invalid.";
- error( RtAudioError::INVALID_USE );
- return;
- }
- }
-
- unsigned int iChannels = 0;
- if ( iParams ) {
- iChannels = iParams->nChannels;
- if ( iParams->deviceId >= nDevices ) {
- errorText_ = "RtApi::openStream: input device parameter value is invalid.";
- error( RtAudioError::INVALID_USE );
- return;
- }
- }
-
- bool result;
-
- if ( oChannels > 0 ) {
-
- result = probeDeviceOpen( oParams->deviceId, OUTPUT, oChannels, oParams->firstChannel,
- sampleRate, format, bufferFrames, options );
- if ( result == false ) {
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- }
-
- if ( iChannels > 0 ) {
-
- result = probeDeviceOpen( iParams->deviceId, INPUT, iChannels, iParams->firstChannel,
- sampleRate, format, bufferFrames, options );
- if ( result == false ) {
- if ( oChannels > 0 ) closeStream();
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- }
-
- stream_.callbackInfo.callback = (void *) callback;
- stream_.callbackInfo.userData = userData;
- stream_.callbackInfo.errorCallback = (void *) errorCallback;
-
- if ( options ) options->numberOfBuffers = stream_.nBuffers;
- stream_.state = STREAM_STOPPED;
-}
-
-unsigned int RtApi :: getDefaultInputDevice( void )
-{
- // Should be implemented in subclasses if possible.
- return 0;
-}
-
-unsigned int RtApi :: getDefaultOutputDevice( void )
-{
- // Should be implemented in subclasses if possible.
- return 0;
-}
-
-void RtApi :: closeStream( void )
-{
- // MUST be implemented in subclasses!
- return;
-}
-
-bool RtApi :: probeDeviceOpen( unsigned int /*device*/, StreamMode /*mode*/, unsigned int /*channels*/,
- unsigned int /*firstChannel*/, unsigned int /*sampleRate*/,
- RtAudioFormat /*format*/, unsigned int * /*bufferSize*/,
- RtAudio::StreamOptions * /*options*/ )
-{
- // MUST be implemented in subclasses!
- return FAILURE;
-}
-
-void RtApi :: tickStreamTime( void )
-{
- // Subclasses that do not provide their own implementation of
- // getStreamTime should call this function once per buffer I/O to
- // provide basic stream time support.
-
- stream_.streamTime += ( stream_.bufferSize * 1.0 / stream_.sampleRate );
-
-#if defined( HAVE_GETTIMEOFDAY )
- gettimeofday( &stream_.lastTickTimestamp, NULL );
-#endif
-}
-
-long RtApi :: getStreamLatency( void )
-{
- verifyStream();
-
- long totalLatency = 0;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX )
- totalLatency = stream_.latency[0];
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX )
- totalLatency += stream_.latency[1];
-
- return totalLatency;
-}
-
-double RtApi :: getStreamTime( void )
-{
- verifyStream();
-
-#if defined( HAVE_GETTIMEOFDAY )
- // Return a very accurate estimate of the stream time by
- // adding in the elapsed time since the last tick.
- struct timeval then;
- struct timeval now;
-
- if ( stream_.state != STREAM_RUNNING || stream_.streamTime == 0.0 )
- return stream_.streamTime;
-
- gettimeofday( &now, NULL );
- then = stream_.lastTickTimestamp;
- return stream_.streamTime +
- ((now.tv_sec + 0.000001 * now.tv_usec) -
- (then.tv_sec + 0.000001 * then.tv_usec));
-#else
- return stream_.streamTime;
-#endif
-}
-
-void RtApi :: setStreamTime( double time )
-{
- verifyStream();
-
- if ( time >= 0.0 )
- stream_.streamTime = time;
-}
-
-unsigned int RtApi :: getStreamSampleRate( void )
-{
- verifyStream();
-
- return stream_.sampleRate;
-}
-
-
-// *************************************************** //
-//
-// OS/API-specific methods.
-//
-// *************************************************** //
-
-#if defined(__MACOSX_CORE__)
-
-// The OS X CoreAudio API is designed to use a separate callback
-// procedure for each of its audio devices. A single RtAudio duplex
-// stream using two different devices is supported here, though it
-// cannot be guaranteed to always behave correctly because we cannot
-// synchronize these two callbacks.
-//
-// A property listener is installed for over/underrun information.
-// However, no functionality is currently provided to allow property
-// listeners to trigger user handlers because it is unclear what could
-// be done if a critical stream parameter (buffer size, sample rate,
-// device disconnect) notification arrived. The listeners entail
-// quite a bit of extra code and most likely, a user program wouldn't
-// be prepared for the result anyway. However, we do provide a flag
-// to the client callback function to inform of an over/underrun.
-
-// A structure to hold various information related to the CoreAudio API
-// implementation.
-struct CoreHandle {
- AudioDeviceID id[2]; // device ids
-#if defined( MAC_OS_X_VERSION_10_5 ) && ( MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_5 )
- AudioDeviceIOProcID procId[2];
-#endif
- UInt32 iStream[2]; // device stream index (or first if using multiple)
- UInt32 nStreams[2]; // number of streams to use
- bool xrun[2];
- char *deviceBuffer;
- pthread_cond_t condition;
- int drainCounter; // Tracks callback counts when draining
- bool internalDrain; // Indicates if stop is initiated from callback or not.
-
- CoreHandle()
- :deviceBuffer(0), drainCounter(0), internalDrain(false) { nStreams[0] = 1; nStreams[1] = 1; id[0] = 0; id[1] = 0; xrun[0] = false; xrun[1] = false; }
-};
-
-RtApiCore:: RtApiCore()
-{
-#if defined( AVAILABLE_MAC_OS_X_VERSION_10_6_AND_LATER )
- // This is a largely undocumented but absolutely necessary
- // requirement starting with OS-X 10.6. If not called, queries and
- // updates to various audio device properties are not handled
- // correctly.
- CFRunLoopRef theRunLoop = NULL;
- AudioObjectPropertyAddress property = { kAudioHardwarePropertyRunLoop,
- kAudioObjectPropertyScopeGlobal,
- kAudioObjectPropertyElementMaster };
- OSStatus result = AudioObjectSetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, sizeof(CFRunLoopRef), &theRunLoop);
- if ( result != noErr ) {
- errorText_ = "RtApiCore::RtApiCore: error setting run loop property!";
- error( RtAudioError::WARNING );
- }
-#endif
-}
-
-RtApiCore :: ~RtApiCore()
-{
- // The subclass destructor gets called before the base class
- // destructor, so close an existing stream before deallocating
- // apiDeviceId memory.
- if ( stream_.state != STREAM_CLOSED ) closeStream();
-}
-
-unsigned int RtApiCore :: getDeviceCount( void )
-{
- // Find out how many audio devices there are, if any.
- UInt32 dataSize;
- AudioObjectPropertyAddress propertyAddress = { kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster };
- OSStatus result = AudioObjectGetPropertyDataSize( kAudioObjectSystemObject, &propertyAddress, 0, NULL, &dataSize );
- if ( result != noErr ) {
- errorText_ = "RtApiCore::getDeviceCount: OS-X error getting device info!";
- error( RtAudioError::WARNING );
- return 0;
- }
-
- return dataSize / sizeof( AudioDeviceID );
-}
-
-unsigned int RtApiCore :: getDefaultInputDevice( void )
-{
- unsigned int nDevices = getDeviceCount();
- if ( nDevices <= 1 ) return 0;
-
- AudioDeviceID id;
- UInt32 dataSize = sizeof( AudioDeviceID );
- AudioObjectPropertyAddress property = { kAudioHardwarePropertyDefaultInputDevice, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster };
- OSStatus result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, &dataSize, &id );
- if ( result != noErr ) {
- errorText_ = "RtApiCore::getDefaultInputDevice: OS-X system error getting device.";
- error( RtAudioError::WARNING );
- return 0;
- }
-
- dataSize *= nDevices;
- AudioDeviceID deviceList[ nDevices ];
- property.mSelector = kAudioHardwarePropertyDevices;
- result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, &dataSize, (void *) &deviceList );
- if ( result != noErr ) {
- errorText_ = "RtApiCore::getDefaultInputDevice: OS-X system error getting device IDs.";
- error( RtAudioError::WARNING );
- return 0;
- }
-
- for ( unsigned int i=0; i<nDevices; i++ )
- if ( id == deviceList[i] ) return i;
-
- errorText_ = "RtApiCore::getDefaultInputDevice: No default device found!";
- error( RtAudioError::WARNING );
- return 0;
-}
-
-unsigned int RtApiCore :: getDefaultOutputDevice( void )
-{
- unsigned int nDevices = getDeviceCount();
- if ( nDevices <= 1 ) return 0;
-
- AudioDeviceID id;
- UInt32 dataSize = sizeof( AudioDeviceID );
- AudioObjectPropertyAddress property = { kAudioHardwarePropertyDefaultOutputDevice, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster };
- OSStatus result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, &dataSize, &id );
- if ( result != noErr ) {
- errorText_ = "RtApiCore::getDefaultOutputDevice: OS-X system error getting device.";
- error( RtAudioError::WARNING );
- return 0;
- }
-
- dataSize = sizeof( AudioDeviceID ) * nDevices;
- AudioDeviceID deviceList[ nDevices ];
- property.mSelector = kAudioHardwarePropertyDevices;
- result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, &dataSize, (void *) &deviceList );
- if ( result != noErr ) {
- errorText_ = "RtApiCore::getDefaultOutputDevice: OS-X system error getting device IDs.";
- error( RtAudioError::WARNING );
- return 0;
- }
-
- for ( unsigned int i=0; i<nDevices; i++ )
- if ( id == deviceList[i] ) return i;
-
- errorText_ = "RtApiCore::getDefaultOutputDevice: No default device found!";
- error( RtAudioError::WARNING );
- return 0;
-}
-
-RtAudio::DeviceInfo RtApiCore :: getDeviceInfo( unsigned int device )
-{
- RtAudio::DeviceInfo info;
- info.probed = false;
-
- // Get device ID
- unsigned int nDevices = getDeviceCount();
- if ( nDevices == 0 ) {
- errorText_ = "RtApiCore::getDeviceInfo: no devices found!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- if ( device >= nDevices ) {
- errorText_ = "RtApiCore::getDeviceInfo: device ID is invalid!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- AudioDeviceID deviceList[ nDevices ];
- UInt32 dataSize = sizeof( AudioDeviceID ) * nDevices;
- AudioObjectPropertyAddress property = { kAudioHardwarePropertyDevices,
- kAudioObjectPropertyScopeGlobal,
- kAudioObjectPropertyElementMaster };
- OSStatus result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property,
- 0, NULL, &dataSize, (void *) &deviceList );
- if ( result != noErr ) {
- errorText_ = "RtApiCore::getDeviceInfo: OS-X system error getting device IDs.";
- error( RtAudioError::WARNING );
- return info;
- }
-
- AudioDeviceID id = deviceList[ device ];
-
- // Get the device name.
- info.name.erase();
- CFStringRef cfname;
- dataSize = sizeof( CFStringRef );
- property.mSelector = kAudioObjectPropertyManufacturer;
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &cfname );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceInfo: system error (" << getErrorCode( result ) << ") getting device manufacturer.";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- //const char *mname = CFStringGetCStringPtr( cfname, CFStringGetSystemEncoding() );
- int length = CFStringGetLength(cfname);
- char *mname = (char *)malloc(length * 3 + 1);
-#if defined( UNICODE ) || defined( _UNICODE )
- CFStringGetCString(cfname, mname, length * 3 + 1, kCFStringEncodingUTF8);
-#else
- CFStringGetCString(cfname, mname, length * 3 + 1, CFStringGetSystemEncoding());
-#endif
- info.name.append( (const char *)mname, strlen(mname) );
- info.name.append( ": " );
- CFRelease( cfname );
- free(mname);
-
- property.mSelector = kAudioObjectPropertyName;
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &cfname );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceInfo: system error (" << getErrorCode( result ) << ") getting device name.";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- //const char *name = CFStringGetCStringPtr( cfname, CFStringGetSystemEncoding() );
- length = CFStringGetLength(cfname);
- char *name = (char *)malloc(length * 3 + 1);
-#if defined( UNICODE ) || defined( _UNICODE )
- CFStringGetCString(cfname, name, length * 3 + 1, kCFStringEncodingUTF8);
-#else
- CFStringGetCString(cfname, name, length * 3 + 1, CFStringGetSystemEncoding());
-#endif
- info.name.append( (const char *)name, strlen(name) );
- CFRelease( cfname );
- free(name);
-
- // Get the output stream "configuration".
- AudioBufferList *bufferList = nil;
- property.mSelector = kAudioDevicePropertyStreamConfiguration;
- property.mScope = kAudioDevicePropertyScopeOutput;
- // property.mElement = kAudioObjectPropertyElementWildcard;
- dataSize = 0;
- result = AudioObjectGetPropertyDataSize( id, &property, 0, NULL, &dataSize );
- if ( result != noErr || dataSize == 0 ) {
- errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting output stream configuration info for device (" << device << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Allocate the AudioBufferList.
- bufferList = (AudioBufferList *) malloc( dataSize );
- if ( bufferList == NULL ) {
- errorText_ = "RtApiCore::getDeviceInfo: memory error allocating output AudioBufferList.";
- error( RtAudioError::WARNING );
- return info;
- }
-
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, bufferList );
- if ( result != noErr || dataSize == 0 ) {
- free( bufferList );
- errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting output stream configuration for device (" << device << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Get output channel information.
- unsigned int i, nStreams = bufferList->mNumberBuffers;
- for ( i=0; i<nStreams; i++ )
- info.outputChannels += bufferList->mBuffers[i].mNumberChannels;
- free( bufferList );
-
- // Get the input stream "configuration".
- property.mScope = kAudioDevicePropertyScopeInput;
- result = AudioObjectGetPropertyDataSize( id, &property, 0, NULL, &dataSize );
- if ( result != noErr || dataSize == 0 ) {
- errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting input stream configuration info for device (" << device << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Allocate the AudioBufferList.
- bufferList = (AudioBufferList *) malloc( dataSize );
- if ( bufferList == NULL ) {
- errorText_ = "RtApiCore::getDeviceInfo: memory error allocating input AudioBufferList.";
- error( RtAudioError::WARNING );
- return info;
- }
-
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, bufferList );
- if (result != noErr || dataSize == 0) {
- free( bufferList );
- errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting input stream configuration for device (" << device << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Get input channel information.
- nStreams = bufferList->mNumberBuffers;
- for ( i=0; i<nStreams; i++ )
- info.inputChannels += bufferList->mBuffers[i].mNumberChannels;
- free( bufferList );
-
- // If device opens for both playback and capture, we determine the channels.
- if ( info.outputChannels > 0 && info.inputChannels > 0 )
- info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels;
-
- // Probe the device sample rates.
- bool isInput = false;
- if ( info.outputChannels == 0 ) isInput = true;
-
- // Determine the supported sample rates.
- property.mSelector = kAudioDevicePropertyAvailableNominalSampleRates;
- if ( isInput == false ) property.mScope = kAudioDevicePropertyScopeOutput;
- result = AudioObjectGetPropertyDataSize( id, &property, 0, NULL, &dataSize );
- if ( result != kAudioHardwareNoError || dataSize == 0 ) {
- errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting sample rate info.";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- UInt32 nRanges = dataSize / sizeof( AudioValueRange );
- AudioValueRange rangeList[ nRanges ];
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &rangeList );
- if ( result != kAudioHardwareNoError ) {
- errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting sample rates.";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // The sample rate reporting mechanism is a bit of a mystery. It
- // seems that it can either return individual rates or a range of
- // rates. I assume that if the min / max range values are the same,
- // then that represents a single supported rate and if the min / max
- // range values are different, the device supports an arbitrary
- // range of values (though there might be multiple ranges, so we'll
- // use the most conservative range).
- Float64 minimumRate = 1.0, maximumRate = 10000000000.0;
- bool haveValueRange = false;
- info.sampleRates.clear();
- for ( UInt32 i=0; i<nRanges; i++ ) {
- if ( rangeList[i].mMinimum == rangeList[i].mMaximum ) {
- unsigned int tmpSr = (unsigned int) rangeList[i].mMinimum;
- info.sampleRates.push_back( tmpSr );
-
- if ( !info.preferredSampleRate || ( tmpSr <= 48000 && tmpSr > info.preferredSampleRate ) )
- info.preferredSampleRate = tmpSr;
-
- } else {
- haveValueRange = true;
- if ( rangeList[i].mMinimum > minimumRate ) minimumRate = rangeList[i].mMinimum;
- if ( rangeList[i].mMaximum < maximumRate ) maximumRate = rangeList[i].mMaximum;
- }
- }
-
- if ( haveValueRange ) {
- for ( unsigned int k=0; k<MAX_SAMPLE_RATES; k++ ) {
- if ( SAMPLE_RATES[k] >= (unsigned int) minimumRate && SAMPLE_RATES[k] <= (unsigned int) maximumRate ) {
- info.sampleRates.push_back( SAMPLE_RATES[k] );
-
- if ( !info.preferredSampleRate || ( SAMPLE_RATES[k] <= 48000 && SAMPLE_RATES[k] > info.preferredSampleRate ) )
- info.preferredSampleRate = SAMPLE_RATES[k];
- }
- }
- }
-
- // Sort and remove any redundant values
- std::sort( info.sampleRates.begin(), info.sampleRates.end() );
- info.sampleRates.erase( unique( info.sampleRates.begin(), info.sampleRates.end() ), info.sampleRates.end() );
-
- if ( info.sampleRates.size() == 0 ) {
- errorStream_ << "RtApiCore::probeDeviceInfo: No supported sample rates found for device (" << device << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // CoreAudio always uses 32-bit floating point data for PCM streams.
- // Thus, any other "physical" formats supported by the device are of
- // no interest to the client.
- info.nativeFormats = RTAUDIO_FLOAT32;
-
- if ( info.outputChannels > 0 )
- if ( getDefaultOutputDevice() == device ) info.isDefaultOutput = true;
- if ( info.inputChannels > 0 )
- if ( getDefaultInputDevice() == device ) info.isDefaultInput = true;
-
- info.probed = true;
- return info;
-}
-
-static OSStatus callbackHandler( AudioDeviceID inDevice,
- const AudioTimeStamp* /*inNow*/,
- const AudioBufferList* inInputData,
- const AudioTimeStamp* /*inInputTime*/,
- AudioBufferList* outOutputData,
- const AudioTimeStamp* /*inOutputTime*/,
- void* infoPointer )
-{
- CallbackInfo *info = (CallbackInfo *) infoPointer;
-
- RtApiCore *object = (RtApiCore *) info->object;
- if ( object->callbackEvent( inDevice, inInputData, outOutputData ) == false )
- return kAudioHardwareUnspecifiedError;
- else
- return kAudioHardwareNoError;
-}
-
-static OSStatus xrunListener( AudioObjectID /*inDevice*/,
- UInt32 nAddresses,
- const AudioObjectPropertyAddress properties[],
- void* handlePointer )
-{
- CoreHandle *handle = (CoreHandle *) handlePointer;
- for ( UInt32 i=0; i<nAddresses; i++ ) {
- if ( properties[i].mSelector == kAudioDeviceProcessorOverload ) {
- if ( properties[i].mScope == kAudioDevicePropertyScopeInput )
- handle->xrun[1] = true;
- else
- handle->xrun[0] = true;
- }
- }
-
- return kAudioHardwareNoError;
-}
-
-static OSStatus rateListener( AudioObjectID inDevice,
- UInt32 /*nAddresses*/,
- const AudioObjectPropertyAddress /*properties*/[],
- void* ratePointer )
-{
- Float64 *rate = (Float64 *) ratePointer;
- UInt32 dataSize = sizeof( Float64 );
- AudioObjectPropertyAddress property = { kAudioDevicePropertyNominalSampleRate,
- kAudioObjectPropertyScopeGlobal,
- kAudioObjectPropertyElementMaster };
- AudioObjectGetPropertyData( inDevice, &property, 0, NULL, &dataSize, rate );
- return kAudioHardwareNoError;
-}
-
-bool RtApiCore :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels,
- unsigned int firstChannel, unsigned int sampleRate,
- RtAudioFormat format, unsigned int *bufferSize,
- RtAudio::StreamOptions *options )
-{
- // Get device ID
- unsigned int nDevices = getDeviceCount();
- if ( nDevices == 0 ) {
- // This should not happen because a check is made before this function is called.
- errorText_ = "RtApiCore::probeDeviceOpen: no devices found!";
- return FAILURE;
- }
-
- if ( device >= nDevices ) {
- // This should not happen because a check is made before this function is called.
- errorText_ = "RtApiCore::probeDeviceOpen: device ID is invalid!";
- return FAILURE;
- }
-
- AudioDeviceID deviceList[ nDevices ];
- UInt32 dataSize = sizeof( AudioDeviceID ) * nDevices;
- AudioObjectPropertyAddress property = { kAudioHardwarePropertyDevices,
- kAudioObjectPropertyScopeGlobal,
- kAudioObjectPropertyElementMaster };
- OSStatus result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property,
- 0, NULL, &dataSize, (void *) &deviceList );
- if ( result != noErr ) {
- errorText_ = "RtApiCore::probeDeviceOpen: OS-X system error getting device IDs.";
- return FAILURE;
- }
-
- AudioDeviceID id = deviceList[ device ];
-
- // Setup for stream mode.
- bool isInput = false;
- if ( mode == INPUT ) {
- isInput = true;
- property.mScope = kAudioDevicePropertyScopeInput;
- }
- else
- property.mScope = kAudioDevicePropertyScopeOutput;
-
- // Get the stream "configuration".
- AudioBufferList *bufferList = nil;
- dataSize = 0;
- property.mSelector = kAudioDevicePropertyStreamConfiguration;
- result = AudioObjectGetPropertyDataSize( id, &property, 0, NULL, &dataSize );
- if ( result != noErr || dataSize == 0 ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting stream configuration info for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Allocate the AudioBufferList.
- bufferList = (AudioBufferList *) malloc( dataSize );
- if ( bufferList == NULL ) {
- errorText_ = "RtApiCore::probeDeviceOpen: memory error allocating AudioBufferList.";
- return FAILURE;
- }
-
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, bufferList );
- if (result != noErr || dataSize == 0) {
- free( bufferList );
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting stream configuration for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Search for one or more streams that contain the desired number of
- // channels. CoreAudio devices can have an arbitrary number of
- // streams and each stream can have an arbitrary number of channels.
- // For each stream, a single buffer of interleaved samples is
- // provided. RtAudio prefers the use of one stream of interleaved
- // data or multiple consecutive single-channel streams. However, we
- // now support multiple consecutive multi-channel streams of
- // interleaved data as well.
- UInt32 iStream, offsetCounter = firstChannel;
- UInt32 nStreams = bufferList->mNumberBuffers;
- bool monoMode = false;
- bool foundStream = false;
-
- // First check that the device supports the requested number of
- // channels.
- UInt32 deviceChannels = 0;
- for ( iStream=0; iStream<nStreams; iStream++ )
- deviceChannels += bufferList->mBuffers[iStream].mNumberChannels;
-
- if ( deviceChannels < ( channels + firstChannel ) ) {
- free( bufferList );
- errorStream_ << "RtApiCore::probeDeviceOpen: the device (" << device << ") does not support the requested channel count.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Look for a single stream meeting our needs.
- UInt32 firstStream, streamCount = 1, streamChannels = 0, channelOffset = 0;
- for ( iStream=0; iStream<nStreams; iStream++ ) {
- streamChannels = bufferList->mBuffers[iStream].mNumberChannels;
- if ( streamChannels >= channels + offsetCounter ) {
- firstStream = iStream;
- channelOffset = offsetCounter;
- foundStream = true;
- break;
- }
- if ( streamChannels > offsetCounter ) break;
- offsetCounter -= streamChannels;
- }
-
- // If we didn't find a single stream above, then we should be able
- // to meet the channel specification with multiple streams.
- if ( foundStream == false ) {
- monoMode = true;
- offsetCounter = firstChannel;
- for ( iStream=0; iStream<nStreams; iStream++ ) {
- streamChannels = bufferList->mBuffers[iStream].mNumberChannels;
- if ( streamChannels > offsetCounter ) break;
- offsetCounter -= streamChannels;
- }
-
- firstStream = iStream;
- channelOffset = offsetCounter;
- Int32 channelCounter = channels + offsetCounter - streamChannels;
-
- if ( streamChannels > 1 ) monoMode = false;
- while ( channelCounter > 0 ) {
- streamChannels = bufferList->mBuffers[++iStream].mNumberChannels;
- if ( streamChannels > 1 ) monoMode = false;
- channelCounter -= streamChannels;
- streamCount++;
- }
- }
-
- free( bufferList );
-
- // Determine the buffer size.
- AudioValueRange bufferRange;
- dataSize = sizeof( AudioValueRange );
- property.mSelector = kAudioDevicePropertyBufferFrameSizeRange;
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &bufferRange );
-
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting buffer size range for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- if ( bufferRange.mMinimum > *bufferSize ) *bufferSize = (unsigned long) bufferRange.mMinimum;
- else if ( bufferRange.mMaximum < *bufferSize ) *bufferSize = (unsigned long) bufferRange.mMaximum;
- if ( options && options->flags & RTAUDIO_MINIMIZE_LATENCY ) *bufferSize = (unsigned long) bufferRange.mMinimum;
-
- // Set the buffer size. For multiple streams, I'm assuming we only
- // need to make this setting for the master channel.
- UInt32 theSize = (UInt32) *bufferSize;
- dataSize = sizeof( UInt32 );
- property.mSelector = kAudioDevicePropertyBufferFrameSize;
- result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &theSize );
-
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting the buffer size for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // If attempting to setup a duplex stream, the bufferSize parameter
- // MUST be the same in both directions!
- *bufferSize = theSize;
- if ( stream_.mode == OUTPUT && mode == INPUT && *bufferSize != stream_.bufferSize ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error setting buffer size for duplex stream on device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- stream_.bufferSize = *bufferSize;
- stream_.nBuffers = 1;
-
- // Try to set "hog" mode ... it's not clear to me this is working.
- if ( options && options->flags & RTAUDIO_HOG_DEVICE ) {
- pid_t hog_pid;
- dataSize = sizeof( hog_pid );
- property.mSelector = kAudioDevicePropertyHogMode;
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &hog_pid );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting 'hog' state!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- if ( hog_pid != getpid() ) {
- hog_pid = getpid();
- result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &hog_pid );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting 'hog' state!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
- }
-
- // Check and if necessary, change the sample rate for the device.
- Float64 nominalRate;
- dataSize = sizeof( Float64 );
- property.mSelector = kAudioDevicePropertyNominalSampleRate;
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &nominalRate );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting current sample rate.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Only change the sample rate if off by more than 1 Hz.
- if ( fabs( nominalRate - (double)sampleRate ) > 1.0 ) {
-
- // Set a property listener for the sample rate change
- Float64 reportedRate = 0.0;
- AudioObjectPropertyAddress tmp = { kAudioDevicePropertyNominalSampleRate, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster };
- result = AudioObjectAddPropertyListener( id, &tmp, rateListener, (void *) &reportedRate );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting sample rate property listener for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- nominalRate = (Float64) sampleRate;
- result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &nominalRate );
- if ( result != noErr ) {
- AudioObjectRemovePropertyListener( id, &tmp, rateListener, (void *) &reportedRate );
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting sample rate for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Now wait until the reported nominal rate is what we just set.
- UInt32 microCounter = 0;
- while ( reportedRate != nominalRate ) {
- microCounter += 5000;
- if ( microCounter > 5000000 ) break;
- usleep( 5000 );
- }
-
- // Remove the property listener.
- AudioObjectRemovePropertyListener( id, &tmp, rateListener, (void *) &reportedRate );
-
- if ( microCounter > 5000000 ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: timeout waiting for sample rate update for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
-
- // Now set the stream format for all streams. Also, check the
- // physical format of the device and change that if necessary.
- AudioStreamBasicDescription description;
- dataSize = sizeof( AudioStreamBasicDescription );
- property.mSelector = kAudioStreamPropertyVirtualFormat;
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &description );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting stream format for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Set the sample rate and data format id. However, only make the
- // change if the sample rate is not within 1.0 of the desired
- // rate and the format is not linear pcm.
- bool updateFormat = false;
- if ( fabs( description.mSampleRate - (Float64)sampleRate ) > 1.0 ) {
- description.mSampleRate = (Float64) sampleRate;
- updateFormat = true;
- }
-
- if ( description.mFormatID != kAudioFormatLinearPCM ) {
- description.mFormatID = kAudioFormatLinearPCM;
- updateFormat = true;
- }
-
- if ( updateFormat ) {
- result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &description );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting sample rate or data format for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
-
- // Now check the physical format.
- property.mSelector = kAudioStreamPropertyPhysicalFormat;
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &description );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting stream physical format for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- //std::cout << "Current physical stream format:" << std::endl;
- //std::cout << " mBitsPerChan = " << description.mBitsPerChannel << std::endl;
- //std::cout << " aligned high = " << (description.mFormatFlags & kAudioFormatFlagIsAlignedHigh) << ", isPacked = " << (description.mFormatFlags & kAudioFormatFlagIsPacked) << std::endl;
- //std::cout << " bytesPerFrame = " << description.mBytesPerFrame << std::endl;
- //std::cout << " sample rate = " << description.mSampleRate << std::endl;
-
- if ( description.mFormatID != kAudioFormatLinearPCM || description.mBitsPerChannel < 16 ) {
- description.mFormatID = kAudioFormatLinearPCM;
- //description.mSampleRate = (Float64) sampleRate;
- AudioStreamBasicDescription testDescription = description;
- UInt32 formatFlags;
-
- // We'll try higher bit rates first and then work our way down.
- std::vector< std::pair<UInt32, UInt32> > physicalFormats;
- formatFlags = (description.mFormatFlags | kLinearPCMFormatFlagIsFloat) & ~kLinearPCMFormatFlagIsSignedInteger;
- physicalFormats.push_back( std::pair<Float32, UInt32>( 32, formatFlags ) );
- formatFlags = (description.mFormatFlags | kLinearPCMFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked) & ~kLinearPCMFormatFlagIsFloat;
- physicalFormats.push_back( std::pair<Float32, UInt32>( 32, formatFlags ) );
- physicalFormats.push_back( std::pair<Float32, UInt32>( 24, formatFlags ) ); // 24-bit packed
- formatFlags &= ~( kAudioFormatFlagIsPacked | kAudioFormatFlagIsAlignedHigh );
- physicalFormats.push_back( std::pair<Float32, UInt32>( 24.2, formatFlags ) ); // 24-bit in 4 bytes, aligned low
- formatFlags |= kAudioFormatFlagIsAlignedHigh;
- physicalFormats.push_back( std::pair<Float32, UInt32>( 24.4, formatFlags ) ); // 24-bit in 4 bytes, aligned high
- formatFlags = (description.mFormatFlags | kLinearPCMFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked) & ~kLinearPCMFormatFlagIsFloat;
- physicalFormats.push_back( std::pair<Float32, UInt32>( 16, formatFlags ) );
- physicalFormats.push_back( std::pair<Float32, UInt32>( 8, formatFlags ) );
-
- bool setPhysicalFormat = false;
- for( unsigned int i=0; i<physicalFormats.size(); i++ ) {
- testDescription = description;
- testDescription.mBitsPerChannel = (UInt32) physicalFormats[i].first;
- testDescription.mFormatFlags = physicalFormats[i].second;
- if ( (24 == (UInt32)physicalFormats[i].first) && ~( physicalFormats[i].second & kAudioFormatFlagIsPacked ) )
- testDescription.mBytesPerFrame = 4 * testDescription.mChannelsPerFrame;
- else
- testDescription.mBytesPerFrame = testDescription.mBitsPerChannel/8 * testDescription.mChannelsPerFrame;
- testDescription.mBytesPerPacket = testDescription.mBytesPerFrame * testDescription.mFramesPerPacket;
- result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &testDescription );
- if ( result == noErr ) {
- setPhysicalFormat = true;
- //std::cout << "Updated physical stream format:" << std::endl;
- //std::cout << " mBitsPerChan = " << testDescription.mBitsPerChannel << std::endl;
- //std::cout << " aligned high = " << (testDescription.mFormatFlags & kAudioFormatFlagIsAlignedHigh) << ", isPacked = " << (testDescription.mFormatFlags & kAudioFormatFlagIsPacked) << std::endl;
- //std::cout << " bytesPerFrame = " << testDescription.mBytesPerFrame << std::endl;
- //std::cout << " sample rate = " << testDescription.mSampleRate << std::endl;
- break;
- }
- }
-
- if ( !setPhysicalFormat ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting physical data format for device (" << device << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- } // done setting virtual/physical formats.
-
- // Get the stream / device latency.
- UInt32 latency;
- dataSize = sizeof( UInt32 );
- property.mSelector = kAudioDevicePropertyLatency;
- if ( AudioObjectHasProperty( id, &property ) == true ) {
- result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &latency );
- if ( result == kAudioHardwareNoError ) stream_.latency[ mode ] = latency;
- else {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting device latency for device (" << device << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- }
- }
-
- // Byte-swapping: According to AudioHardware.h, the stream data will
- // always be presented in native-endian format, so we should never
- // need to byte swap.
- stream_.doByteSwap[mode] = false;
-
- // From the CoreAudio documentation, PCM data must be supplied as
- // 32-bit floats.
- stream_.userFormat = format;
- stream_.deviceFormat[mode] = RTAUDIO_FLOAT32;
-
- if ( streamCount == 1 )
- stream_.nDeviceChannels[mode] = description.mChannelsPerFrame;
- else // multiple streams
- stream_.nDeviceChannels[mode] = channels;
- stream_.nUserChannels[mode] = channels;
- stream_.channelOffset[mode] = channelOffset; // offset within a CoreAudio stream
- if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false;
- else stream_.userInterleaved = true;
- stream_.deviceInterleaved[mode] = true;
- if ( monoMode == true ) stream_.deviceInterleaved[mode] = false;
-
- // Set flags for buffer conversion.
- stream_.doConvertBuffer[mode] = false;
- if ( stream_.userFormat != stream_.deviceFormat[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.nUserChannels[mode] < stream_.nDeviceChannels[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( streamCount == 1 ) {
- if ( stream_.nUserChannels[mode] > 1 &&
- stream_.userInterleaved != stream_.deviceInterleaved[mode] )
- stream_.doConvertBuffer[mode] = true;
- }
- else if ( monoMode && stream_.userInterleaved )
- stream_.doConvertBuffer[mode] = true;
-
- // Allocate our CoreHandle structure for the stream.
- CoreHandle *handle = 0;
- if ( stream_.apiHandle == 0 ) {
- try {
- handle = new CoreHandle;
- }
- catch ( std::bad_alloc& ) {
- errorText_ = "RtApiCore::probeDeviceOpen: error allocating CoreHandle memory.";
- goto error;
- }
-
- if ( pthread_cond_init( &handle->condition, NULL ) ) {
- errorText_ = "RtApiCore::probeDeviceOpen: error initializing pthread condition variable.";
- goto error;
- }
- stream_.apiHandle = (void *) handle;
- }
- else
- handle = (CoreHandle *) stream_.apiHandle;
- handle->iStream[mode] = firstStream;
- handle->nStreams[mode] = streamCount;
- handle->id[mode] = id;
-
- // Allocate necessary internal buffers.
- unsigned long bufferBytes;
- bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat );
- // stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 );
- stream_.userBuffer[mode] = (char *) malloc( bufferBytes * sizeof(char) );
- memset( stream_.userBuffer[mode], 0, bufferBytes * sizeof(char) );
- if ( stream_.userBuffer[mode] == NULL ) {
- errorText_ = "RtApiCore::probeDeviceOpen: error allocating user buffer memory.";
- goto error;
- }
-
- // If possible, we will make use of the CoreAudio stream buffers as
- // "device buffers". However, we can't do this if using multiple
- // streams.
- if ( stream_.doConvertBuffer[mode] && handle->nStreams[mode] > 1 ) {
-
- bool makeBuffer = true;
- bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] );
- if ( mode == INPUT ) {
- if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) {
- unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] );
- if ( bufferBytes <= bytesOut ) makeBuffer = false;
- }
- }
-
- if ( makeBuffer ) {
- bufferBytes *= *bufferSize;
- if ( stream_.deviceBuffer ) free( stream_.deviceBuffer );
- stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 );
- if ( stream_.deviceBuffer == NULL ) {
- errorText_ = "RtApiCore::probeDeviceOpen: error allocating device buffer memory.";
- goto error;
- }
- }
- }
-
- stream_.sampleRate = sampleRate;
- stream_.device[mode] = device;
- stream_.state = STREAM_STOPPED;
- stream_.callbackInfo.object = (void *) this;
-
- // Setup the buffer conversion information structure.
- if ( stream_.doConvertBuffer[mode] ) {
- if ( streamCount > 1 ) setConvertInfo( mode, 0 );
- else setConvertInfo( mode, channelOffset );
- }
-
- if ( mode == INPUT && stream_.mode == OUTPUT && stream_.device[0] == device )
- // Only one callback procedure per device.
- stream_.mode = DUPLEX;
- else {
-#if defined( MAC_OS_X_VERSION_10_5 ) && ( MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_5 )
- result = AudioDeviceCreateIOProcID( id, callbackHandler, (void *) &stream_.callbackInfo, &handle->procId[mode] );
-#else
- // deprecated in favor of AudioDeviceCreateIOProcID()
- result = AudioDeviceAddIOProc( id, callbackHandler, (void *) &stream_.callbackInfo );
-#endif
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::probeDeviceOpen: system error setting callback for device (" << device << ").";
- errorText_ = errorStream_.str();
- goto error;
- }
- if ( stream_.mode == OUTPUT && mode == INPUT )
- stream_.mode = DUPLEX;
- else
- stream_.mode = mode;
- }
-
- // Setup the device property listener for over/underload.
- property.mSelector = kAudioDeviceProcessorOverload;
- property.mScope = kAudioObjectPropertyScopeGlobal;
- result = AudioObjectAddPropertyListener( id, &property, xrunListener, (void *) handle );
-
- return SUCCESS;
-
- error:
- if ( handle ) {
- pthread_cond_destroy( &handle->condition );
- delete handle;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- stream_.state = STREAM_CLOSED;
- return FAILURE;
-}
-
-void RtApiCore :: closeStream( void )
-{
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiCore::closeStream(): no open stream to close!";
- error( RtAudioError::WARNING );
- return;
- }
-
- CoreHandle *handle = (CoreHandle *) stream_.apiHandle;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- if (handle) {
- AudioObjectPropertyAddress property = { kAudioHardwarePropertyDevices,
- kAudioObjectPropertyScopeGlobal,
- kAudioObjectPropertyElementMaster };
-
- property.mSelector = kAudioDeviceProcessorOverload;
- property.mScope = kAudioObjectPropertyScopeGlobal;
- if (AudioObjectRemovePropertyListener( handle->id[0], &property, xrunListener, (void *) handle ) != noErr) {
- errorText_ = "RtApiCore::closeStream(): error removing property listener!";
- error( RtAudioError::WARNING );
- }
- }
- if ( stream_.state == STREAM_RUNNING )
- AudioDeviceStop( handle->id[0], callbackHandler );
-#if defined( MAC_OS_X_VERSION_10_5 ) && ( MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_5 )
- AudioDeviceDestroyIOProcID( handle->id[0], handle->procId[0] );
-#else
- // deprecated in favor of AudioDeviceDestroyIOProcID()
- AudioDeviceRemoveIOProc( handle->id[0], callbackHandler );
-#endif
- }
-
- if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && stream_.device[0] != stream_.device[1] ) ) {
- if (handle) {
- AudioObjectPropertyAddress property = { kAudioHardwarePropertyDevices,
- kAudioObjectPropertyScopeGlobal,
- kAudioObjectPropertyElementMaster };
-
- property.mSelector = kAudioDeviceProcessorOverload;
- property.mScope = kAudioObjectPropertyScopeGlobal;
- if (AudioObjectRemovePropertyListener( handle->id[1], &property, xrunListener, (void *) handle ) != noErr) {
- errorText_ = "RtApiCore::closeStream(): error removing property listener!";
- error( RtAudioError::WARNING );
- }
- }
- if ( stream_.state == STREAM_RUNNING )
- AudioDeviceStop( handle->id[1], callbackHandler );
-#if defined( MAC_OS_X_VERSION_10_5 ) && ( MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_5 )
- AudioDeviceDestroyIOProcID( handle->id[1], handle->procId[1] );
-#else
- // deprecated in favor of AudioDeviceDestroyIOProcID()
- AudioDeviceRemoveIOProc( handle->id[1], callbackHandler );
-#endif
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- // Destroy pthread condition variable.
- pthread_cond_destroy( &handle->condition );
- delete handle;
- stream_.apiHandle = 0;
-
- stream_.mode = UNINITIALIZED;
- stream_.state = STREAM_CLOSED;
-}
-
-void RtApiCore :: startStream( void )
-{
- verifyStream();
- if ( stream_.state == STREAM_RUNNING ) {
- errorText_ = "RtApiCore::startStream(): the stream is already running!";
- error( RtAudioError::WARNING );
- return;
- }
-
- OSStatus result = noErr;
- CoreHandle *handle = (CoreHandle *) stream_.apiHandle;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- result = AudioDeviceStart( handle->id[0], callbackHandler );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::startStream: system error (" << getErrorCode( result ) << ") starting callback procedure on device (" << stream_.device[0] << ").";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- if ( stream_.mode == INPUT ||
- ( stream_.mode == DUPLEX && stream_.device[0] != stream_.device[1] ) ) {
-
- result = AudioDeviceStart( handle->id[1], callbackHandler );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::startStream: system error starting input callback procedure on device (" << stream_.device[1] << ").";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- handle->drainCounter = 0;
- handle->internalDrain = false;
- stream_.state = STREAM_RUNNING;
-
- unlock:
- if ( result == noErr ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiCore :: stopStream( void )
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiCore::stopStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- OSStatus result = noErr;
- CoreHandle *handle = (CoreHandle *) stream_.apiHandle;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- if ( handle->drainCounter == 0 ) {
- handle->drainCounter = 2;
- pthread_cond_wait( &handle->condition, &stream_.mutex ); // block until signaled
- }
-
- result = AudioDeviceStop( handle->id[0], callbackHandler );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::stopStream: system error (" << getErrorCode( result ) << ") stopping callback procedure on device (" << stream_.device[0] << ").";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && stream_.device[0] != stream_.device[1] ) ) {
-
- result = AudioDeviceStop( handle->id[1], callbackHandler );
- if ( result != noErr ) {
- errorStream_ << "RtApiCore::stopStream: system error (" << getErrorCode( result ) << ") stopping input callback procedure on device (" << stream_.device[1] << ").";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- stream_.state = STREAM_STOPPED;
-
- unlock:
- if ( result == noErr ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiCore :: abortStream( void )
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiCore::abortStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- CoreHandle *handle = (CoreHandle *) stream_.apiHandle;
- handle->drainCounter = 2;
-
- stopStream();
-}
-
-// This function will be called by a spawned thread when the user
-// callback function signals that the stream should be stopped or
-// aborted. It is better to handle it this way because the
-// callbackEvent() function probably should return before the AudioDeviceStop()
-// function is called.
-static void *coreStopStream( void *ptr )
-{
- CallbackInfo *info = (CallbackInfo *) ptr;
- RtApiCore *object = (RtApiCore *) info->object;
-
- object->stopStream();
- pthread_exit( NULL );
-}
-
-bool RtApiCore :: callbackEvent( AudioDeviceID deviceId,
- const AudioBufferList *inBufferList,
- const AudioBufferList *outBufferList )
-{
- if ( stream_.state == STREAM_STOPPED || stream_.state == STREAM_STOPPING ) return SUCCESS;
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiCore::callbackEvent(): the stream is closed ... this shouldn't happen!";
- error( RtAudioError::WARNING );
- return FAILURE;
- }
-
- CallbackInfo *info = (CallbackInfo *) &stream_.callbackInfo;
- CoreHandle *handle = (CoreHandle *) stream_.apiHandle;
-
- // Check if we were draining the stream and signal is finished.
- if ( handle->drainCounter > 3 ) {
- ThreadHandle threadId;
-
- stream_.state = STREAM_STOPPING;
- if ( handle->internalDrain == true )
- pthread_create( &threadId, NULL, coreStopStream, info );
- else // external call to stopStream()
- pthread_cond_signal( &handle->condition );
- return SUCCESS;
- }
-
- AudioDeviceID outputDevice = handle->id[0];
-
- // Invoke user callback to get fresh output data UNLESS we are
- // draining stream or duplex mode AND the input/output devices are
- // different AND this function is called for the input device.
- if ( handle->drainCounter == 0 && ( stream_.mode != DUPLEX || deviceId == outputDevice ) ) {
- RtAudioCallback callback = (RtAudioCallback) info->callback;
- double streamTime = getStreamTime();
- RtAudioStreamStatus status = 0;
- if ( stream_.mode != INPUT && handle->xrun[0] == true ) {
- status |= RTAUDIO_OUTPUT_UNDERFLOW;
- handle->xrun[0] = false;
- }
- if ( stream_.mode != OUTPUT && handle->xrun[1] == true ) {
- status |= RTAUDIO_INPUT_OVERFLOW;
- handle->xrun[1] = false;
- }
-
- int cbReturnValue = callback( stream_.userBuffer[0], stream_.userBuffer[1],
- stream_.bufferSize, streamTime, status, info->userData );
- if ( cbReturnValue == 2 ) {
- stream_.state = STREAM_STOPPING;
- handle->drainCounter = 2;
- abortStream();
- return SUCCESS;
- }
- else if ( cbReturnValue == 1 ) {
- handle->drainCounter = 1;
- handle->internalDrain = true;
- }
- }
-
- if ( stream_.mode == OUTPUT || ( stream_.mode == DUPLEX && deviceId == outputDevice ) ) {
-
- if ( handle->drainCounter > 1 ) { // write zeros to the output stream
-
- if ( handle->nStreams[0] == 1 ) {
- memset( outBufferList->mBuffers[handle->iStream[0]].mData,
- 0,
- outBufferList->mBuffers[handle->iStream[0]].mDataByteSize );
- }
- else { // fill multiple streams with zeros
- for ( unsigned int i=0; i<handle->nStreams[0]; i++ ) {
- memset( outBufferList->mBuffers[handle->iStream[0]+i].mData,
- 0,
- outBufferList->mBuffers[handle->iStream[0]+i].mDataByteSize );
- }
- }
- }
- else if ( handle->nStreams[0] == 1 ) {
- if ( stream_.doConvertBuffer[0] ) { // convert directly to CoreAudio stream buffer
- convertBuffer( (char *) outBufferList->mBuffers[handle->iStream[0]].mData,
- stream_.userBuffer[0], stream_.convertInfo[0] );
- }
- else { // copy from user buffer
- memcpy( outBufferList->mBuffers[handle->iStream[0]].mData,
- stream_.userBuffer[0],
- outBufferList->mBuffers[handle->iStream[0]].mDataByteSize );
- }
- }
- else { // fill multiple streams
- Float32 *inBuffer = (Float32 *) stream_.userBuffer[0];
- if ( stream_.doConvertBuffer[0] ) {
- convertBuffer( stream_.deviceBuffer, stream_.userBuffer[0], stream_.convertInfo[0] );
- inBuffer = (Float32 *) stream_.deviceBuffer;
- }
-
- if ( stream_.deviceInterleaved[0] == false ) { // mono mode
- UInt32 bufferBytes = outBufferList->mBuffers[handle->iStream[0]].mDataByteSize;
- for ( unsigned int i=0; i<stream_.nUserChannels[0]; i++ ) {
- memcpy( outBufferList->mBuffers[handle->iStream[0]+i].mData,
- (void *)&inBuffer[i*stream_.bufferSize], bufferBytes );
- }
- }
- else { // fill multiple multi-channel streams with interleaved data
- UInt32 streamChannels, channelsLeft, inJump, outJump, inOffset;
- Float32 *out, *in;
-
- bool inInterleaved = ( stream_.userInterleaved ) ? true : false;
- UInt32 inChannels = stream_.nUserChannels[0];
- if ( stream_.doConvertBuffer[0] ) {
- inInterleaved = true; // device buffer will always be interleaved for nStreams > 1 and not mono mode
- inChannels = stream_.nDeviceChannels[0];
- }
-
- if ( inInterleaved ) inOffset = 1;
- else inOffset = stream_.bufferSize;
-
- channelsLeft = inChannels;
- for ( unsigned int i=0; i<handle->nStreams[0]; i++ ) {
- in = inBuffer;
- out = (Float32 *) outBufferList->mBuffers[handle->iStream[0]+i].mData;
- streamChannels = outBufferList->mBuffers[handle->iStream[0]+i].mNumberChannels;
-
- outJump = 0;
- // Account for possible channel offset in first stream
- if ( i == 0 && stream_.channelOffset[0] > 0 ) {
- streamChannels -= stream_.channelOffset[0];
- outJump = stream_.channelOffset[0];
- out += outJump;
- }
-
- // Account for possible unfilled channels at end of the last stream
- if ( streamChannels > channelsLeft ) {
- outJump = streamChannels - channelsLeft;
- streamChannels = channelsLeft;
- }
-
- // Determine input buffer offsets and skips
- if ( inInterleaved ) {
- inJump = inChannels;
- in += inChannels - channelsLeft;
- }
- else {
- inJump = 1;
- in += (inChannels - channelsLeft) * inOffset;
- }
-
- for ( unsigned int i=0; i<stream_.bufferSize; i++ ) {
- for ( unsigned int j=0; j<streamChannels; j++ ) {
- *out++ = in[j*inOffset];
- }
- out += outJump;
- in += inJump;
- }
- channelsLeft -= streamChannels;
- }
- }
- }
- }
-
- // Don't bother draining input
- if ( handle->drainCounter ) {
- handle->drainCounter++;
- goto unlock;
- }
-
- AudioDeviceID inputDevice;
- inputDevice = handle->id[1];
- if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && deviceId == inputDevice ) ) {
-
- if ( handle->nStreams[1] == 1 ) {
- if ( stream_.doConvertBuffer[1] ) { // convert directly from CoreAudio stream buffer
- convertBuffer( stream_.userBuffer[1],
- (char *) inBufferList->mBuffers[handle->iStream[1]].mData,
- stream_.convertInfo[1] );
- }
- else { // copy to user buffer
- memcpy( stream_.userBuffer[1],
- inBufferList->mBuffers[handle->iStream[1]].mData,
- inBufferList->mBuffers[handle->iStream[1]].mDataByteSize );
- }
- }
- else { // read from multiple streams
- Float32 *outBuffer = (Float32 *) stream_.userBuffer[1];
- if ( stream_.doConvertBuffer[1] ) outBuffer = (Float32 *) stream_.deviceBuffer;
-
- if ( stream_.deviceInterleaved[1] == false ) { // mono mode
- UInt32 bufferBytes = inBufferList->mBuffers[handle->iStream[1]].mDataByteSize;
- for ( unsigned int i=0; i<stream_.nUserChannels[1]; i++ ) {
- memcpy( (void *)&outBuffer[i*stream_.bufferSize],
- inBufferList->mBuffers[handle->iStream[1]+i].mData, bufferBytes );
- }
- }
- else { // read from multiple multi-channel streams
- UInt32 streamChannels, channelsLeft, inJump, outJump, outOffset;
- Float32 *out, *in;
-
- bool outInterleaved = ( stream_.userInterleaved ) ? true : false;
- UInt32 outChannels = stream_.nUserChannels[1];
- if ( stream_.doConvertBuffer[1] ) {
- outInterleaved = true; // device buffer will always be interleaved for nStreams > 1 and not mono mode
- outChannels = stream_.nDeviceChannels[1];
- }
-
- if ( outInterleaved ) outOffset = 1;
- else outOffset = stream_.bufferSize;
-
- channelsLeft = outChannels;
- for ( unsigned int i=0; i<handle->nStreams[1]; i++ ) {
- out = outBuffer;
- in = (Float32 *) inBufferList->mBuffers[handle->iStream[1]+i].mData;
- streamChannels = inBufferList->mBuffers[handle->iStream[1]+i].mNumberChannels;
-
- inJump = 0;
- // Account for possible channel offset in first stream
- if ( i == 0 && stream_.channelOffset[1] > 0 ) {
- streamChannels -= stream_.channelOffset[1];
- inJump = stream_.channelOffset[1];
- in += inJump;
- }
-
- // Account for possible unread channels at end of the last stream
- if ( streamChannels > channelsLeft ) {
- inJump = streamChannels - channelsLeft;
- streamChannels = channelsLeft;
- }
-
- // Determine output buffer offsets and skips
- if ( outInterleaved ) {
- outJump = outChannels;
- out += outChannels - channelsLeft;
- }
- else {
- outJump = 1;
- out += (outChannels - channelsLeft) * outOffset;
- }
-
- for ( unsigned int i=0; i<stream_.bufferSize; i++ ) {
- for ( unsigned int j=0; j<streamChannels; j++ ) {
- out[j*outOffset] = *in++;
- }
- out += outJump;
- in += inJump;
- }
- channelsLeft -= streamChannels;
- }
- }
-
- if ( stream_.doConvertBuffer[1] ) { // convert from our internal "device" buffer
- convertBuffer( stream_.userBuffer[1],
- stream_.deviceBuffer,
- stream_.convertInfo[1] );
- }
- }
- }
-
- unlock:
- //MUTEX_UNLOCK( &stream_.mutex );
-
- RtApi::tickStreamTime();
- return SUCCESS;
-}
-
-const char* RtApiCore :: getErrorCode( OSStatus code )
-{
- switch( code ) {
-
- case kAudioHardwareNotRunningError:
- return "kAudioHardwareNotRunningError";
-
- case kAudioHardwareUnspecifiedError:
- return "kAudioHardwareUnspecifiedError";
-
- case kAudioHardwareUnknownPropertyError:
- return "kAudioHardwareUnknownPropertyError";
-
- case kAudioHardwareBadPropertySizeError:
- return "kAudioHardwareBadPropertySizeError";
-
- case kAudioHardwareIllegalOperationError:
- return "kAudioHardwareIllegalOperationError";
-
- case kAudioHardwareBadObjectError:
- return "kAudioHardwareBadObjectError";
-
- case kAudioHardwareBadDeviceError:
- return "kAudioHardwareBadDeviceError";
-
- case kAudioHardwareBadStreamError:
- return "kAudioHardwareBadStreamError";
-
- case kAudioHardwareUnsupportedOperationError:
- return "kAudioHardwareUnsupportedOperationError";
-
- case kAudioDeviceUnsupportedFormatError:
- return "kAudioDeviceUnsupportedFormatError";
-
- case kAudioDevicePermissionsError:
- return "kAudioDevicePermissionsError";
-
- default:
- return "CoreAudio unknown error";
- }
-}
-
- //******************** End of __MACOSX_CORE__ *********************//
-#endif
-
-#if defined(__UNIX_JACK__)
-
-// JACK is a low-latency audio server, originally written for the
-// GNU/Linux operating system and now also ported to OS-X. It can
-// connect a number of different applications to an audio device, as
-// well as allowing them to share audio between themselves.
-//
-// When using JACK with RtAudio, "devices" refer to JACK clients that
-// have ports connected to the server. The JACK server is typically
-// started in a terminal as follows:
-//
-// .jackd -d alsa -d hw:0
-//
-// or through an interface program such as qjackctl. Many of the
-// parameters normally set for a stream are fixed by the JACK server
-// and can be specified when the JACK server is started. In
-// particular,
-//
-// .jackd -d alsa -d hw:0 -r 44100 -p 512 -n 4
-//
-// specifies a sample rate of 44100 Hz, a buffer size of 512 sample
-// frames, and number of buffers = 4. Once the server is running, it
-// is not possible to override these values. If the values are not
-// specified in the command-line, the JACK server uses default values.
-//
-// The JACK server does not have to be running when an instance of
-// RtApiJack is created, though the function getDeviceCount() will
-// report 0 devices found until JACK has been started. When no
-// devices are available (i.e., the JACK server is not running), a
-// stream cannot be opened.
-
-#include <jack/jack.h>
-#include <unistd.h>
-#include <cstdio>
-
-// A structure to hold various information related to the Jack API
-// implementation.
-struct JackHandle {
- jack_client_t *client;
- jack_port_t **ports[2];
- std::string deviceName[2];
- bool xrun[2];
- pthread_cond_t condition;
- int drainCounter; // Tracks callback counts when draining
- bool internalDrain; // Indicates if stop is initiated from callback or not.
-
- JackHandle()
- :client(0), drainCounter(0), internalDrain(false) { ports[0] = 0; ports[1] = 0; xrun[0] = false; xrun[1] = false; }
-};
-
-static void jackSilentError( const char * ) {};
-
-RtApiJack :: RtApiJack()
-{
- // Nothing to do here.
-#if !defined(__RTAUDIO_DEBUG__)
- // Turn off Jack's internal error reporting.
- jack_set_error_function( &jackSilentError );
-#endif
-}
-
-RtApiJack :: ~RtApiJack()
-{
- if ( stream_.state != STREAM_CLOSED ) closeStream();
-}
-
-unsigned int RtApiJack :: getDeviceCount( void )
-{
- // See if we can become a jack client.
- jack_options_t options = (jack_options_t) ( JackNoStartServer ); //JackNullOption;
- jack_status_t *status = NULL;
- jack_client_t *client = jack_client_open( "RtApiJackCount", options, status );
- if ( client == 0 ) return 0;
-
- const char **ports;
- std::string port, previousPort;
- unsigned int nChannels = 0, nDevices = 0;
- ports = jack_get_ports( client, NULL, NULL, 0 );
- if ( ports ) {
- // Parse the port names up to the first colon (:).
- size_t iColon = 0;
- do {
- port = (char *) ports[ nChannels ];
- iColon = port.find(":");
- if ( iColon != std::string::npos ) {
- port = port.substr( 0, iColon + 1 );
- if ( port != previousPort ) {
- nDevices++;
- previousPort = port;
- }
- }
- } while ( ports[++nChannels] );
- free( ports );
- }
-
- jack_client_close( client );
- return nDevices;
-}
-
-RtAudio::DeviceInfo RtApiJack :: getDeviceInfo( unsigned int device )
-{
- RtAudio::DeviceInfo info;
- info.probed = false;
-
- jack_options_t options = (jack_options_t) ( JackNoStartServer ); //JackNullOption
- jack_status_t *status = NULL;
- jack_client_t *client = jack_client_open( "RtApiJackInfo", options, status );
- if ( client == 0 ) {
- errorText_ = "RtApiJack::getDeviceInfo: Jack server not found or connection error!";
- error( RtAudioError::WARNING );
- return info;
- }
-
- const char **ports;
- std::string port, previousPort;
- unsigned int nPorts = 0, nDevices = 0;
- ports = jack_get_ports( client, NULL, NULL, 0 );
- if ( ports ) {
- // Parse the port names up to the first colon (:).
- size_t iColon = 0;
- do {
- port = (char *) ports[ nPorts ];
- iColon = port.find(":");
- if ( iColon != std::string::npos ) {
- port = port.substr( 0, iColon );
- if ( port != previousPort ) {
- if ( nDevices == device ) info.name = port;
- nDevices++;
- previousPort = port;
- }
- }
- } while ( ports[++nPorts] );
- free( ports );
- }
-
- if ( device >= nDevices ) {
- jack_client_close( client );
- errorText_ = "RtApiJack::getDeviceInfo: device ID is invalid!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- // Get the current jack server sample rate.
- info.sampleRates.clear();
-
- info.preferredSampleRate = jack_get_sample_rate( client );
- info.sampleRates.push_back( info.preferredSampleRate );
-
- // Count the available ports containing the client name as device
- // channels. Jack "input ports" equal RtAudio output channels.
- unsigned int nChannels = 0;
- ports = jack_get_ports( client, info.name.c_str(), NULL, JackPortIsInput );
- if ( ports ) {
- while ( ports[ nChannels ] ) nChannels++;
- free( ports );
- info.outputChannels = nChannels;
- }
-
- // Jack "output ports" equal RtAudio input channels.
- nChannels = 0;
- ports = jack_get_ports( client, info.name.c_str(), NULL, JackPortIsOutput );
- if ( ports ) {
- while ( ports[ nChannels ] ) nChannels++;
- free( ports );
- info.inputChannels = nChannels;
- }
-
- if ( info.outputChannels == 0 && info.inputChannels == 0 ) {
- jack_client_close(client);
- errorText_ = "RtApiJack::getDeviceInfo: error determining Jack input/output channels!";
- error( RtAudioError::WARNING );
- return info;
- }
-
- // If device opens for both playback and capture, we determine the channels.
- if ( info.outputChannels > 0 && info.inputChannels > 0 )
- info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels;
-
- // Jack always uses 32-bit floats.
- info.nativeFormats = RTAUDIO_FLOAT32;
-
- // Jack doesn't provide default devices so we'll use the first available one.
- if ( device == 0 && info.outputChannels > 0 )
- info.isDefaultOutput = true;
- if ( device == 0 && info.inputChannels > 0 )
- info.isDefaultInput = true;
-
- jack_client_close(client);
- info.probed = true;
- return info;
-}
-
-static int jackCallbackHandler( jack_nframes_t nframes, void *infoPointer )
-{
- CallbackInfo *info = (CallbackInfo *) infoPointer;
-
- RtApiJack *object = (RtApiJack *) info->object;
- if ( object->callbackEvent( (unsigned long) nframes ) == false ) return 1;
-
- return 0;
-}
-
-// This function will be called by a spawned thread when the Jack
-// server signals that it is shutting down. It is necessary to handle
-// it this way because the jackShutdown() function must return before
-// the jack_deactivate() function (in closeStream()) will return.
-static void *jackCloseStream( void *ptr )
-{
- CallbackInfo *info = (CallbackInfo *) ptr;
- RtApiJack *object = (RtApiJack *) info->object;
-
- object->closeStream();
-
- pthread_exit( NULL );
-}
-static void jackShutdown( void *infoPointer )
-{
- CallbackInfo *info = (CallbackInfo *) infoPointer;
- RtApiJack *object = (RtApiJack *) info->object;
-
- // Check current stream state. If stopped, then we'll assume this
- // was called as a result of a call to RtApiJack::stopStream (the
- // deactivation of a client handle causes this function to be called).
- // If not, we'll assume the Jack server is shutting down or some
- // other problem occurred and we should close the stream.
- if ( object->isStreamRunning() == false ) return;
-
- ThreadHandle threadId;
- pthread_create( &threadId, NULL, jackCloseStream, info );
- std::cerr << "\nRtApiJack: the Jack server is shutting down this client ... stream stopped and closed!!\n" << std::endl;
-}
-
-static int jackXrun( void *infoPointer )
-{
- JackHandle *handle = (JackHandle *) infoPointer;
-
- if ( handle->ports[0] ) handle->xrun[0] = true;
- if ( handle->ports[1] ) handle->xrun[1] = true;
-
- return 0;
-}
-
-bool RtApiJack :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels,
- unsigned int firstChannel, unsigned int sampleRate,
- RtAudioFormat format, unsigned int *bufferSize,
- RtAudio::StreamOptions *options )
-{
- JackHandle *handle = (JackHandle *) stream_.apiHandle;
-
- // Look for jack server and try to become a client (only do once per stream).
- jack_client_t *client = 0;
- if ( mode == OUTPUT || ( mode == INPUT && stream_.mode != OUTPUT ) ) {
- jack_options_t jackoptions = (jack_options_t) ( JackNoStartServer ); //JackNullOption;
- jack_status_t *status = NULL;
- if ( options && !options->streamName.empty() )
- client = jack_client_open( options->streamName.c_str(), jackoptions, status );
- else
- client = jack_client_open( "RtApiJack", jackoptions, status );
- if ( client == 0 ) {
- errorText_ = "RtApiJack::probeDeviceOpen: Jack server not found or connection error!";
- error( RtAudioError::WARNING );
- return FAILURE;
- }
- }
- else {
- // The handle must have been created on an earlier pass.
- client = handle->client;
- }
-
- const char **ports;
- std::string port, previousPort, deviceName;
- unsigned int nPorts = 0, nDevices = 0;
- ports = jack_get_ports( client, NULL, NULL, 0 );
- if ( ports ) {
- // Parse the port names up to the first colon (:).
- size_t iColon = 0;
- do {
- port = (char *) ports[ nPorts ];
- iColon = port.find(":");
- if ( iColon != std::string::npos ) {
- port = port.substr( 0, iColon );
- if ( port != previousPort ) {
- if ( nDevices == device ) deviceName = port;
- nDevices++;
- previousPort = port;
- }
- }
- } while ( ports[++nPorts] );
- free( ports );
- }
-
- if ( device >= nDevices ) {
- errorText_ = "RtApiJack::probeDeviceOpen: device ID is invalid!";
- return FAILURE;
- }
-
- // Count the available ports containing the client name as device
- // channels. Jack "input ports" equal RtAudio output channels.
- unsigned int nChannels = 0;
- unsigned long flag = JackPortIsInput;
- if ( mode == INPUT ) flag = JackPortIsOutput;
- ports = jack_get_ports( client, deviceName.c_str(), NULL, flag );
- if ( ports ) {
- while ( ports[ nChannels ] ) nChannels++;
- free( ports );
- }
-
- // Compare the jack ports for specified client to the requested number of channels.
- if ( nChannels < (channels + firstChannel) ) {
- errorStream_ << "RtApiJack::probeDeviceOpen: requested number of channels (" << channels << ") + offset (" << firstChannel << ") not found for specified device (" << device << ":" << deviceName << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Check the jack server sample rate.
- unsigned int jackRate = jack_get_sample_rate( client );
- if ( sampleRate != jackRate ) {
- jack_client_close( client );
- errorStream_ << "RtApiJack::probeDeviceOpen: the requested sample rate (" << sampleRate << ") is different than the JACK server rate (" << jackRate << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- stream_.sampleRate = jackRate;
-
- // Get the latency of the JACK port.
- ports = jack_get_ports( client, deviceName.c_str(), NULL, flag );
- if ( ports[ firstChannel ] ) {
- // Added by Ge Wang
- jack_latency_callback_mode_t cbmode = (mode == INPUT ? JackCaptureLatency : JackPlaybackLatency);
- // the range (usually the min and max are equal)
- jack_latency_range_t latrange; latrange.min = latrange.max = 0;
- // get the latency range
- jack_port_get_latency_range( jack_port_by_name( client, ports[firstChannel] ), cbmode, &latrange );
- // be optimistic, use the min!
- stream_.latency[mode] = latrange.min;
- //stream_.latency[mode] = jack_port_get_latency( jack_port_by_name( client, ports[ firstChannel ] ) );
- }
- free( ports );
-
- // The jack server always uses 32-bit floating-point data.
- stream_.deviceFormat[mode] = RTAUDIO_FLOAT32;
- stream_.userFormat = format;
-
- if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false;
- else stream_.userInterleaved = true;
-
- // Jack always uses non-interleaved buffers.
- stream_.deviceInterleaved[mode] = false;
-
- // Jack always provides host byte-ordered data.
- stream_.doByteSwap[mode] = false;
-
- // Get the buffer size. The buffer size and number of buffers
- // (periods) is set when the jack server is started.
- stream_.bufferSize = (int) jack_get_buffer_size( client );
- *bufferSize = stream_.bufferSize;
-
- stream_.nDeviceChannels[mode] = channels;
- stream_.nUserChannels[mode] = channels;
-
- // Set flags for buffer conversion.
- stream_.doConvertBuffer[mode] = false;
- if ( stream_.userFormat != stream_.deviceFormat[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] &&
- stream_.nUserChannels[mode] > 1 )
- stream_.doConvertBuffer[mode] = true;
-
- // Allocate our JackHandle structure for the stream.
- if ( handle == 0 ) {
- try {
- handle = new JackHandle;
- }
- catch ( std::bad_alloc& ) {
- errorText_ = "RtApiJack::probeDeviceOpen: error allocating JackHandle memory.";
- goto error;
- }
-
- if ( pthread_cond_init(&handle->condition, NULL) ) {
- errorText_ = "RtApiJack::probeDeviceOpen: error initializing pthread condition variable.";
- goto error;
- }
- stream_.apiHandle = (void *) handle;
- handle->client = client;
- }
- handle->deviceName[mode] = deviceName;
-
- // Allocate necessary internal buffers.
- unsigned long bufferBytes;
- bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat );
- stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 );
- if ( stream_.userBuffer[mode] == NULL ) {
- errorText_ = "RtApiJack::probeDeviceOpen: error allocating user buffer memory.";
- goto error;
- }
-
- if ( stream_.doConvertBuffer[mode] ) {
-
- bool makeBuffer = true;
- if ( mode == OUTPUT )
- bufferBytes = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] );
- else { // mode == INPUT
- bufferBytes = stream_.nDeviceChannels[1] * formatBytes( stream_.deviceFormat[1] );
- if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) {
- unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes(stream_.deviceFormat[0]);
- if ( bufferBytes < bytesOut ) makeBuffer = false;
- }
- }
-
- if ( makeBuffer ) {
- bufferBytes *= *bufferSize;
- if ( stream_.deviceBuffer ) free( stream_.deviceBuffer );
- stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 );
- if ( stream_.deviceBuffer == NULL ) {
- errorText_ = "RtApiJack::probeDeviceOpen: error allocating device buffer memory.";
- goto error;
- }
- }
- }
-
- // Allocate memory for the Jack ports (channels) identifiers.
- handle->ports[mode] = (jack_port_t **) malloc ( sizeof (jack_port_t *) * channels );
- if ( handle->ports[mode] == NULL ) {
- errorText_ = "RtApiJack::probeDeviceOpen: error allocating port memory.";
- goto error;
- }
-
- stream_.device[mode] = device;
- stream_.channelOffset[mode] = firstChannel;
- stream_.state = STREAM_STOPPED;
- stream_.callbackInfo.object = (void *) this;
-
- if ( stream_.mode == OUTPUT && mode == INPUT )
- // We had already set up the stream for output.
- stream_.mode = DUPLEX;
- else {
- stream_.mode = mode;
- jack_set_process_callback( handle->client, jackCallbackHandler, (void *) &stream_.callbackInfo );
- jack_set_xrun_callback( handle->client, jackXrun, (void *) &handle );
- jack_on_shutdown( handle->client, jackShutdown, (void *) &stream_.callbackInfo );
- }
-
- // Register our ports.
- char label[64];
- if ( mode == OUTPUT ) {
- for ( unsigned int i=0; i<stream_.nUserChannels[0]; i++ ) {
- snprintf( label, 64, "outport %d", i );
- handle->ports[0][i] = jack_port_register( handle->client, (const char *)label,
- JACK_DEFAULT_AUDIO_TYPE, JackPortIsOutput, 0 );
- }
- }
- else {
- for ( unsigned int i=0; i<stream_.nUserChannels[1]; i++ ) {
- snprintf( label, 64, "inport %d", i );
- handle->ports[1][i] = jack_port_register( handle->client, (const char *)label,
- JACK_DEFAULT_AUDIO_TYPE, JackPortIsInput, 0 );
- }
- }
-
- // Setup the buffer conversion information structure. We don't use
- // buffers to do channel offsets, so we override that parameter
- // here.
- if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, 0 );
-
- return SUCCESS;
-
- error:
- if ( handle ) {
- pthread_cond_destroy( &handle->condition );
- jack_client_close( handle->client );
-
- if ( handle->ports[0] ) free( handle->ports[0] );
- if ( handle->ports[1] ) free( handle->ports[1] );
-
- delete handle;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- return FAILURE;
-}
-
-void RtApiJack :: closeStream( void )
-{
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiJack::closeStream(): no open stream to close!";
- error( RtAudioError::WARNING );
- return;
- }
-
- JackHandle *handle = (JackHandle *) stream_.apiHandle;
- if ( handle ) {
-
- if ( stream_.state == STREAM_RUNNING )
- jack_deactivate( handle->client );
-
- jack_client_close( handle->client );
- }
-
- if ( handle ) {
- if ( handle->ports[0] ) free( handle->ports[0] );
- if ( handle->ports[1] ) free( handle->ports[1] );
- pthread_cond_destroy( &handle->condition );
- delete handle;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- stream_.mode = UNINITIALIZED;
- stream_.state = STREAM_CLOSED;
-}
-
-void RtApiJack :: startStream( void )
-{
- verifyStream();
- if ( stream_.state == STREAM_RUNNING ) {
- errorText_ = "RtApiJack::startStream(): the stream is already running!";
- error( RtAudioError::WARNING );
- return;
- }
-
- JackHandle *handle = (JackHandle *) stream_.apiHandle;
- int result = jack_activate( handle->client );
- if ( result ) {
- errorText_ = "RtApiJack::startStream(): unable to activate JACK client!";
- goto unlock;
- }
-
- const char **ports;
-
- // Get the list of available ports.
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- result = 1;
- ports = jack_get_ports( handle->client, handle->deviceName[0].c_str(), NULL, JackPortIsInput);
- if ( ports == NULL) {
- errorText_ = "RtApiJack::startStream(): error determining available JACK input ports!";
- goto unlock;
- }
-
- // Now make the port connections. Since RtAudio wasn't designed to
- // allow the user to select particular channels of a device, we'll
- // just open the first "nChannels" ports with offset.
- for ( unsigned int i=0; i<stream_.nUserChannels[0]; i++ ) {
- result = 1;
- if ( ports[ stream_.channelOffset[0] + i ] )
- result = jack_connect( handle->client, jack_port_name( handle->ports[0][i] ), ports[ stream_.channelOffset[0] + i ] );
- if ( result ) {
- free( ports );
- errorText_ = "RtApiJack::startStream(): error connecting output ports!";
- goto unlock;
- }
- }
- free(ports);
- }
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) {
- result = 1;
- ports = jack_get_ports( handle->client, handle->deviceName[1].c_str(), NULL, JackPortIsOutput );
- if ( ports == NULL) {
- errorText_ = "RtApiJack::startStream(): error determining available JACK output ports!";
- goto unlock;
- }
-
- // Now make the port connections. See note above.
- for ( unsigned int i=0; i<stream_.nUserChannels[1]; i++ ) {
- result = 1;
- if ( ports[ stream_.channelOffset[1] + i ] )
- result = jack_connect( handle->client, ports[ stream_.channelOffset[1] + i ], jack_port_name( handle->ports[1][i] ) );
- if ( result ) {
- free( ports );
- errorText_ = "RtApiJack::startStream(): error connecting input ports!";
- goto unlock;
- }
- }
- free(ports);
- }
-
- handle->drainCounter = 0;
- handle->internalDrain = false;
- stream_.state = STREAM_RUNNING;
-
- unlock:
- if ( result == 0 ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiJack :: stopStream( void )
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiJack::stopStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- JackHandle *handle = (JackHandle *) stream_.apiHandle;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- if ( handle->drainCounter == 0 ) {
- handle->drainCounter = 2;
- pthread_cond_wait( &handle->condition, &stream_.mutex ); // block until signaled
- }
- }
-
- jack_deactivate( handle->client );
- stream_.state = STREAM_STOPPED;
-}
-
-void RtApiJack :: abortStream( void )
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiJack::abortStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- JackHandle *handle = (JackHandle *) stream_.apiHandle;
- handle->drainCounter = 2;
-
- stopStream();
-}
-
-// This function will be called by a spawned thread when the user
-// callback function signals that the stream should be stopped or
-// aborted. It is necessary to handle it this way because the
-// callbackEvent() function must return before the jack_deactivate()
-// function will return.
-static void *jackStopStream( void *ptr )
-{
- CallbackInfo *info = (CallbackInfo *) ptr;
- RtApiJack *object = (RtApiJack *) info->object;
-
- object->stopStream();
- pthread_exit( NULL );
-}
-
-bool RtApiJack :: callbackEvent( unsigned long nframes )
-{
- if ( stream_.state == STREAM_STOPPED || stream_.state == STREAM_STOPPING ) return SUCCESS;
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiCore::callbackEvent(): the stream is closed ... this shouldn't happen!";
- error( RtAudioError::WARNING );
- return FAILURE;
- }
- if ( stream_.bufferSize != nframes ) {
- errorText_ = "RtApiCore::callbackEvent(): the JACK buffer size has changed ... cannot process!";
- error( RtAudioError::WARNING );
- return FAILURE;
- }
-
- CallbackInfo *info = (CallbackInfo *) &stream_.callbackInfo;
- JackHandle *handle = (JackHandle *) stream_.apiHandle;
-
- // Check if we were draining the stream and signal is finished.
- if ( handle->drainCounter > 3 ) {
- ThreadHandle threadId;
-
- stream_.state = STREAM_STOPPING;
- if ( handle->internalDrain == true )
- pthread_create( &threadId, NULL, jackStopStream, info );
- else
- pthread_cond_signal( &handle->condition );
- return SUCCESS;
- }
-
- // Invoke user callback first, to get fresh output data.
- if ( handle->drainCounter == 0 ) {
- RtAudioCallback callback = (RtAudioCallback) info->callback;
- double streamTime = getStreamTime();
- RtAudioStreamStatus status = 0;
- if ( stream_.mode != INPUT && handle->xrun[0] == true ) {
- status |= RTAUDIO_OUTPUT_UNDERFLOW;
- handle->xrun[0] = false;
- }
- if ( stream_.mode != OUTPUT && handle->xrun[1] == true ) {
- status |= RTAUDIO_INPUT_OVERFLOW;
- handle->xrun[1] = false;
- }
- int cbReturnValue = callback( stream_.userBuffer[0], stream_.userBuffer[1],
- stream_.bufferSize, streamTime, status, info->userData );
- if ( cbReturnValue == 2 ) {
- stream_.state = STREAM_STOPPING;
- handle->drainCounter = 2;
- ThreadHandle id;
- pthread_create( &id, NULL, jackStopStream, info );
- return SUCCESS;
- }
- else if ( cbReturnValue == 1 ) {
- handle->drainCounter = 1;
- handle->internalDrain = true;
- }
- }
-
- jack_default_audio_sample_t *jackbuffer;
- unsigned long bufferBytes = nframes * sizeof( jack_default_audio_sample_t );
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- if ( handle->drainCounter > 1 ) { // write zeros to the output stream
-
- for ( unsigned int i=0; i<stream_.nDeviceChannels[0]; i++ ) {
- jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[0][i], (jack_nframes_t) nframes );
- memset( jackbuffer, 0, bufferBytes );
- }
-
- }
- else if ( stream_.doConvertBuffer[0] ) {
-
- convertBuffer( stream_.deviceBuffer, stream_.userBuffer[0], stream_.convertInfo[0] );
-
- for ( unsigned int i=0; i<stream_.nDeviceChannels[0]; i++ ) {
- jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[0][i], (jack_nframes_t) nframes );
- memcpy( jackbuffer, &stream_.deviceBuffer[i*bufferBytes], bufferBytes );
- }
- }
- else { // no buffer conversion
- for ( unsigned int i=0; i<stream_.nUserChannels[0]; i++ ) {
- jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[0][i], (jack_nframes_t) nframes );
- memcpy( jackbuffer, &stream_.userBuffer[0][i*bufferBytes], bufferBytes );
- }
- }
- }
-
- // Don't bother draining input
- if ( handle->drainCounter ) {
- handle->drainCounter++;
- goto unlock;
- }
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) {
-
- if ( stream_.doConvertBuffer[1] ) {
- for ( unsigned int i=0; i<stream_.nDeviceChannels[1]; i++ ) {
- jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[1][i], (jack_nframes_t) nframes );
- memcpy( &stream_.deviceBuffer[i*bufferBytes], jackbuffer, bufferBytes );
- }
- convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] );
- }
- else { // no buffer conversion
- for ( unsigned int i=0; i<stream_.nUserChannels[1]; i++ ) {
- jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[1][i], (jack_nframes_t) nframes );
- memcpy( &stream_.userBuffer[1][i*bufferBytes], jackbuffer, bufferBytes );
- }
- }
- }
-
- unlock:
- RtApi::tickStreamTime();
- return SUCCESS;
-}
- //******************** End of __UNIX_JACK__ *********************//
-#endif
-
-#if defined(__WINDOWS_ASIO__) // ASIO API on Windows
-
-// The ASIO API is designed around a callback scheme, so this
-// implementation is similar to that used for OS-X CoreAudio and Linux
-// Jack. The primary constraint with ASIO is that it only allows
-// access to a single driver at a time. Thus, it is not possible to
-// have more than one simultaneous RtAudio stream.
-//
-// This implementation also requires a number of external ASIO files
-// and a few global variables. The ASIO callback scheme does not
-// allow for the passing of user data, so we must create a global
-// pointer to our callbackInfo structure.
-//
-// On unix systems, we make use of a pthread condition variable.
-// Since there is no equivalent in Windows, I hacked something based
-// on information found in
-// http://www.cs.wustl.edu/~schmidt/win32-cv-1.html.
-
-#include "asiosys.h"
-#include "asio.h"
-#include "iasiothiscallresolver.h"
-#include "asiodrivers.h"
-#include <cmath>
-
-static AsioDrivers drivers;
-static ASIOCallbacks asioCallbacks;
-static ASIODriverInfo driverInfo;
-static CallbackInfo *asioCallbackInfo;
-static bool asioXRun;
-
-struct AsioHandle {
- int drainCounter; // Tracks callback counts when draining
- bool internalDrain; // Indicates if stop is initiated from callback or not.
- ASIOBufferInfo *bufferInfos;
- HANDLE condition;
-
- AsioHandle()
- :drainCounter(0), internalDrain(false), bufferInfos(0) {}
-};
-
-// Function declarations (definitions at end of section)
-static const char* getAsioErrorString( ASIOError result );
-static void sampleRateChanged( ASIOSampleRate sRate );
-static long asioMessages( long selector, long value, void* message, double* opt );
-
-RtApiAsio :: RtApiAsio()
-{
- // ASIO cannot run on a multi-threaded appartment. You can call
- // CoInitialize beforehand, but it must be for appartment threading
- // (in which case, CoInitilialize will return S_FALSE here).
- coInitialized_ = false;
- HRESULT hr = CoInitialize( NULL );
- if ( FAILED(hr) ) {
- errorText_ = "RtApiAsio::ASIO requires a single-threaded appartment. Call CoInitializeEx(0,COINIT_APARTMENTTHREADED)";
- error( RtAudioError::WARNING );
- }
- coInitialized_ = true;
-
- drivers.removeCurrentDriver();
- driverInfo.asioVersion = 2;
-
- // See note in DirectSound implementation about GetDesktopWindow().
- driverInfo.sysRef = GetForegroundWindow();
-}
-
-RtApiAsio :: ~RtApiAsio()
-{
- if ( stream_.state != STREAM_CLOSED ) closeStream();
- if ( coInitialized_ ) CoUninitialize();
-}
-
-unsigned int RtApiAsio :: getDeviceCount( void )
-{
- return (unsigned int) drivers.asioGetNumDev();
-}
-
-RtAudio::DeviceInfo RtApiAsio :: getDeviceInfo( unsigned int device )
-{
- RtAudio::DeviceInfo info;
- info.probed = false;
-
- // Get device ID
- unsigned int nDevices = getDeviceCount();
- if ( nDevices == 0 ) {
- errorText_ = "RtApiAsio::getDeviceInfo: no devices found!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- if ( device >= nDevices ) {
- errorText_ = "RtApiAsio::getDeviceInfo: device ID is invalid!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- // If a stream is already open, we cannot probe other devices. Thus, use the saved results.
- if ( stream_.state != STREAM_CLOSED ) {
- if ( device >= devices_.size() ) {
- errorText_ = "RtApiAsio::getDeviceInfo: device ID was not present before stream was opened.";
- error( RtAudioError::WARNING );
- return info;
- }
- return devices_[ device ];
- }
-
- char driverName[32];
- ASIOError result = drivers.asioGetDriverName( (int) device, driverName, 32 );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::getDeviceInfo: unable to get driver name (" << getAsioErrorString( result ) << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- info.name = driverName;
-
- if ( !drivers.loadDriver( driverName ) ) {
- errorStream_ << "RtApiAsio::getDeviceInfo: unable to load driver (" << driverName << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- result = ASIOInit( &driverInfo );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::getDeviceInfo: error (" << getAsioErrorString( result ) << ") initializing driver (" << driverName << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Determine the device channel information.
- long inputChannels, outputChannels;
- result = ASIOGetChannels( &inputChannels, &outputChannels );
- if ( result != ASE_OK ) {
- drivers.removeCurrentDriver();
- errorStream_ << "RtApiAsio::getDeviceInfo: error (" << getAsioErrorString( result ) << ") getting channel count (" << driverName << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- info.outputChannels = outputChannels;
- info.inputChannels = inputChannels;
- if ( info.outputChannels > 0 && info.inputChannels > 0 )
- info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels;
-
- // Determine the supported sample rates.
- info.sampleRates.clear();
- for ( unsigned int i=0; i<MAX_SAMPLE_RATES; i++ ) {
- result = ASIOCanSampleRate( (ASIOSampleRate) SAMPLE_RATES[i] );
- if ( result == ASE_OK ) {
- info.sampleRates.push_back( SAMPLE_RATES[i] );
-
- if ( !info.preferredSampleRate || ( SAMPLE_RATES[i] <= 48000 && SAMPLE_RATES[i] > info.preferredSampleRate ) )
- info.preferredSampleRate = SAMPLE_RATES[i];
- }
- }
-
- // Determine supported data types ... just check first channel and assume rest are the same.
- ASIOChannelInfo channelInfo;
- channelInfo.channel = 0;
- channelInfo.isInput = true;
- if ( info.inputChannels <= 0 ) channelInfo.isInput = false;
- result = ASIOGetChannelInfo( &channelInfo );
- if ( result != ASE_OK ) {
- drivers.removeCurrentDriver();
- errorStream_ << "RtApiAsio::getDeviceInfo: error (" << getAsioErrorString( result ) << ") getting driver channel info (" << driverName << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- info.nativeFormats = 0;
- if ( channelInfo.type == ASIOSTInt16MSB || channelInfo.type == ASIOSTInt16LSB )
- info.nativeFormats |= RTAUDIO_SINT16;
- else if ( channelInfo.type == ASIOSTInt32MSB || channelInfo.type == ASIOSTInt32LSB )
- info.nativeFormats |= RTAUDIO_SINT32;
- else if ( channelInfo.type == ASIOSTFloat32MSB || channelInfo.type == ASIOSTFloat32LSB )
- info.nativeFormats |= RTAUDIO_FLOAT32;
- else if ( channelInfo.type == ASIOSTFloat64MSB || channelInfo.type == ASIOSTFloat64LSB )
- info.nativeFormats |= RTAUDIO_FLOAT64;
- else if ( channelInfo.type == ASIOSTInt24MSB || channelInfo.type == ASIOSTInt24LSB )
- info.nativeFormats |= RTAUDIO_SINT24;
-
- if ( info.outputChannels > 0 )
- if ( getDefaultOutputDevice() == device ) info.isDefaultOutput = true;
- if ( info.inputChannels > 0 )
- if ( getDefaultInputDevice() == device ) info.isDefaultInput = true;
-
- info.probed = true;
- drivers.removeCurrentDriver();
- return info;
-}
-
-static void bufferSwitch( long index, ASIOBool /*processNow*/ )
-{
- RtApiAsio *object = (RtApiAsio *) asioCallbackInfo->object;
- object->callbackEvent( index );
-}
-
-void RtApiAsio :: saveDeviceInfo( void )
-{
- devices_.clear();
-
- unsigned int nDevices = getDeviceCount();
- devices_.resize( nDevices );
- for ( unsigned int i=0; i<nDevices; i++ )
- devices_[i] = getDeviceInfo( i );
-}
-
-bool RtApiAsio :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels,
- unsigned int firstChannel, unsigned int sampleRate,
- RtAudioFormat format, unsigned int *bufferSize,
- RtAudio::StreamOptions *options )
-{////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
- bool isDuplexInput = mode == INPUT && stream_.mode == OUTPUT;
-
- // For ASIO, a duplex stream MUST use the same driver.
- if ( isDuplexInput && stream_.device[0] != device ) {
- errorText_ = "RtApiAsio::probeDeviceOpen: an ASIO duplex stream must use the same device for input and output!";
- return FAILURE;
- }
-
- char driverName[32];
- ASIOError result = drivers.asioGetDriverName( (int) device, driverName, 32 );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: unable to get driver name (" << getAsioErrorString( result ) << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Only load the driver once for duplex stream.
- if ( !isDuplexInput ) {
- // The getDeviceInfo() function will not work when a stream is open
- // because ASIO does not allow multiple devices to run at the same
- // time. Thus, we'll probe the system before opening a stream and
- // save the results for use by getDeviceInfo().
- this->saveDeviceInfo();
-
- if ( !drivers.loadDriver( driverName ) ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: unable to load driver (" << driverName << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- result = ASIOInit( &driverInfo );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: error (" << getAsioErrorString( result ) << ") initializing driver (" << driverName << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
-
- // keep them before any "goto error", they are used for error cleanup + goto device boundary checks
- bool buffersAllocated = false;
- AsioHandle *handle = (AsioHandle *) stream_.apiHandle;
- unsigned int nChannels;
-
-
- // Check the device channel count.
- long inputChannels, outputChannels;
- result = ASIOGetChannels( &inputChannels, &outputChannels );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: error (" << getAsioErrorString( result ) << ") getting channel count (" << driverName << ").";
- errorText_ = errorStream_.str();
- goto error;
- }
-
- if ( ( mode == OUTPUT && (channels+firstChannel) > (unsigned int) outputChannels) ||
- ( mode == INPUT && (channels+firstChannel) > (unsigned int) inputChannels) ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") does not support requested channel count (" << channels << ") + offset (" << firstChannel << ").";
- errorText_ = errorStream_.str();
- goto error;
- }
- stream_.nDeviceChannels[mode] = channels;
- stream_.nUserChannels[mode] = channels;
- stream_.channelOffset[mode] = firstChannel;
-
- // Verify the sample rate is supported.
- result = ASIOCanSampleRate( (ASIOSampleRate) sampleRate );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") does not support requested sample rate (" << sampleRate << ").";
- errorText_ = errorStream_.str();
- goto error;
- }
-
- // Get the current sample rate
- ASIOSampleRate currentRate;
- result = ASIOGetSampleRate( ¤tRate );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error getting sample rate.";
- errorText_ = errorStream_.str();
- goto error;
- }
-
- // Set the sample rate only if necessary
- if ( currentRate != sampleRate ) {
- result = ASIOSetSampleRate( (ASIOSampleRate) sampleRate );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error setting sample rate (" << sampleRate << ").";
- errorText_ = errorStream_.str();
- goto error;
- }
- }
-
- // Determine the driver data type.
- ASIOChannelInfo channelInfo;
- channelInfo.channel = 0;
- if ( mode == OUTPUT ) channelInfo.isInput = false;
- else channelInfo.isInput = true;
- result = ASIOGetChannelInfo( &channelInfo );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error (" << getAsioErrorString( result ) << ") getting data format.";
- errorText_ = errorStream_.str();
- goto error;
- }
-
- // Assuming WINDOWS host is always little-endian.
- stream_.doByteSwap[mode] = false;
- stream_.userFormat = format;
- stream_.deviceFormat[mode] = 0;
- if ( channelInfo.type == ASIOSTInt16MSB || channelInfo.type == ASIOSTInt16LSB ) {
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- if ( channelInfo.type == ASIOSTInt16MSB ) stream_.doByteSwap[mode] = true;
- }
- else if ( channelInfo.type == ASIOSTInt32MSB || channelInfo.type == ASIOSTInt32LSB ) {
- stream_.deviceFormat[mode] = RTAUDIO_SINT32;
- if ( channelInfo.type == ASIOSTInt32MSB ) stream_.doByteSwap[mode] = true;
- }
- else if ( channelInfo.type == ASIOSTFloat32MSB || channelInfo.type == ASIOSTFloat32LSB ) {
- stream_.deviceFormat[mode] = RTAUDIO_FLOAT32;
- if ( channelInfo.type == ASIOSTFloat32MSB ) stream_.doByteSwap[mode] = true;
- }
- else if ( channelInfo.type == ASIOSTFloat64MSB || channelInfo.type == ASIOSTFloat64LSB ) {
- stream_.deviceFormat[mode] = RTAUDIO_FLOAT64;
- if ( channelInfo.type == ASIOSTFloat64MSB ) stream_.doByteSwap[mode] = true;
- }
- else if ( channelInfo.type == ASIOSTInt24MSB || channelInfo.type == ASIOSTInt24LSB ) {
- stream_.deviceFormat[mode] = RTAUDIO_SINT24;
- if ( channelInfo.type == ASIOSTInt24MSB ) stream_.doByteSwap[mode] = true;
- }
-
- if ( stream_.deviceFormat[mode] == 0 ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") data format not supported by RtAudio.";
- errorText_ = errorStream_.str();
- goto error;
- }
-
- // Set the buffer size. For a duplex stream, this will end up
- // setting the buffer size based on the input constraints, which
- // should be ok.
- long minSize, maxSize, preferSize, granularity;
- result = ASIOGetBufferSize( &minSize, &maxSize, &preferSize, &granularity );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error (" << getAsioErrorString( result ) << ") getting buffer size.";
- errorText_ = errorStream_.str();
- goto error;
- }
-
- if ( isDuplexInput ) {
- // When this is the duplex input (output was opened before), then we have to use the same
- // buffersize as the output, because it might use the preferred buffer size, which most
- // likely wasn't passed as input to this. The buffer sizes have to be identically anyway,
- // So instead of throwing an error, make them equal. The caller uses the reference
- // to the "bufferSize" param as usual to set up processing buffers.
-
- *bufferSize = stream_.bufferSize;
-
- } else {
- if ( *bufferSize == 0 ) *bufferSize = preferSize;
- else if ( *bufferSize < (unsigned int) minSize ) *bufferSize = (unsigned int) minSize;
- else if ( *bufferSize > (unsigned int) maxSize ) *bufferSize = (unsigned int) maxSize;
- else if ( granularity == -1 ) {
- // Make sure bufferSize is a power of two.
- int log2_of_min_size = 0;
- int log2_of_max_size = 0;
-
- for ( unsigned int i = 0; i < sizeof(long) * 8; i++ ) {
- if ( minSize & ((long)1 << i) ) log2_of_min_size = i;
- if ( maxSize & ((long)1 << i) ) log2_of_max_size = i;
- }
-
- long min_delta = std::abs( (long)*bufferSize - ((long)1 << log2_of_min_size) );
- int min_delta_num = log2_of_min_size;
-
- for (int i = log2_of_min_size + 1; i <= log2_of_max_size; i++) {
- long current_delta = std::abs( (long)*bufferSize - ((long)1 << i) );
- if (current_delta < min_delta) {
- min_delta = current_delta;
- min_delta_num = i;
- }
- }
-
- *bufferSize = ( (unsigned int)1 << min_delta_num );
- if ( *bufferSize < (unsigned int) minSize ) *bufferSize = (unsigned int) minSize;
- else if ( *bufferSize > (unsigned int) maxSize ) *bufferSize = (unsigned int) maxSize;
- }
- else if ( granularity != 0 ) {
- // Set to an even multiple of granularity, rounding up.
- *bufferSize = (*bufferSize + granularity-1) / granularity * granularity;
- }
- }
-
- /*
- // we don't use it anymore, see above!
- // Just left it here for the case...
- if ( isDuplexInput && stream_.bufferSize != *bufferSize ) {
- errorText_ = "RtApiAsio::probeDeviceOpen: input/output buffersize discrepancy!";
- goto error;
- }
- */
-
- stream_.bufferSize = *bufferSize;
- stream_.nBuffers = 2;
-
- if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false;
- else stream_.userInterleaved = true;
-
- // ASIO always uses non-interleaved buffers.
- stream_.deviceInterleaved[mode] = false;
-
- // Allocate, if necessary, our AsioHandle structure for the stream.
- if ( handle == 0 ) {
- try {
- handle = new AsioHandle;
- }
- catch ( std::bad_alloc& ) {
- errorText_ = "RtApiAsio::probeDeviceOpen: error allocating AsioHandle memory.";
- goto error;
- }
- handle->bufferInfos = 0;
-
- // Create a manual-reset event.
- handle->condition = CreateEvent( NULL, // no security
- TRUE, // manual-reset
- FALSE, // non-signaled initially
- NULL ); // unnamed
- stream_.apiHandle = (void *) handle;
- }
-
- // Create the ASIO internal buffers. Since RtAudio sets up input
- // and output separately, we'll have to dispose of previously
- // created output buffers for a duplex stream.
- if ( mode == INPUT && stream_.mode == OUTPUT ) {
- ASIODisposeBuffers();
- if ( handle->bufferInfos ) free( handle->bufferInfos );
- }
-
- // Allocate, initialize, and save the bufferInfos in our stream callbackInfo structure.
- unsigned int i;
- nChannels = stream_.nDeviceChannels[0] + stream_.nDeviceChannels[1];
- handle->bufferInfos = (ASIOBufferInfo *) malloc( nChannels * sizeof(ASIOBufferInfo) );
- if ( handle->bufferInfos == NULL ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: error allocating bufferInfo memory for driver (" << driverName << ").";
- errorText_ = errorStream_.str();
- goto error;
- }
-
- ASIOBufferInfo *infos;
- infos = handle->bufferInfos;
- for ( i=0; i<stream_.nDeviceChannels[0]; i++, infos++ ) {
- infos->isInput = ASIOFalse;
- infos->channelNum = i + stream_.channelOffset[0];
- infos->buffers[0] = infos->buffers[1] = 0;
- }
- for ( i=0; i<stream_.nDeviceChannels[1]; i++, infos++ ) {
- infos->isInput = ASIOTrue;
- infos->channelNum = i + stream_.channelOffset[1];
- infos->buffers[0] = infos->buffers[1] = 0;
- }
-
- // prepare for callbacks
- stream_.sampleRate = sampleRate;
- stream_.device[mode] = device;
- stream_.mode = isDuplexInput ? DUPLEX : mode;
-
- // store this class instance before registering callbacks, that are going to use it
- asioCallbackInfo = &stream_.callbackInfo;
- stream_.callbackInfo.object = (void *) this;
-
- // Set up the ASIO callback structure and create the ASIO data buffers.
- asioCallbacks.bufferSwitch = &bufferSwitch;
- asioCallbacks.sampleRateDidChange = &sampleRateChanged;
- asioCallbacks.asioMessage = &asioMessages;
- asioCallbacks.bufferSwitchTimeInfo = NULL;
- result = ASIOCreateBuffers( handle->bufferInfos, nChannels, stream_.bufferSize, &asioCallbacks );
- if ( result != ASE_OK ) {
- // Standard method failed. This can happen with strict/misbehaving drivers that return valid buffer size ranges
- // but only accept the preferred buffer size as parameter for ASIOCreateBuffers. eg. Creatives ASIO driver
- // in that case, let's be naïve and try that instead
- *bufferSize = preferSize;
- stream_.bufferSize = *bufferSize;
- result = ASIOCreateBuffers( handle->bufferInfos, nChannels, stream_.bufferSize, &asioCallbacks );
- }
-
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error (" << getAsioErrorString( result ) << ") creating buffers.";
- errorText_ = errorStream_.str();
- goto error;
- }
- buffersAllocated = true;
- stream_.state = STREAM_STOPPED;
-
- // Set flags for buffer conversion.
- stream_.doConvertBuffer[mode] = false;
- if ( stream_.userFormat != stream_.deviceFormat[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] &&
- stream_.nUserChannels[mode] > 1 )
- stream_.doConvertBuffer[mode] = true;
-
- // Allocate necessary internal buffers
- unsigned long bufferBytes;
- bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat );
- stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 );
- if ( stream_.userBuffer[mode] == NULL ) {
- errorText_ = "RtApiAsio::probeDeviceOpen: error allocating user buffer memory.";
- goto error;
- }
-
- if ( stream_.doConvertBuffer[mode] ) {
-
- bool makeBuffer = true;
- bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] );
- if ( isDuplexInput && stream_.deviceBuffer ) {
- unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] );
- if ( bufferBytes <= bytesOut ) makeBuffer = false;
- }
-
- if ( makeBuffer ) {
- bufferBytes *= *bufferSize;
- if ( stream_.deviceBuffer ) free( stream_.deviceBuffer );
- stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 );
- if ( stream_.deviceBuffer == NULL ) {
- errorText_ = "RtApiAsio::probeDeviceOpen: error allocating device buffer memory.";
- goto error;
- }
- }
- }
-
- // Determine device latencies
- long inputLatency, outputLatency;
- result = ASIOGetLatencies( &inputLatency, &outputLatency );
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error (" << getAsioErrorString( result ) << ") getting latency.";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING); // warn but don't fail
- }
- else {
- stream_.latency[0] = outputLatency;
- stream_.latency[1] = inputLatency;
- }
-
- // Setup the buffer conversion information structure. We don't use
- // buffers to do channel offsets, so we override that parameter
- // here.
- if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, 0 );
-
- return SUCCESS;
-
- error:
- if ( !isDuplexInput ) {
- // the cleanup for error in the duplex input, is done by RtApi::openStream
- // So we clean up for single channel only
-
- if ( buffersAllocated )
- ASIODisposeBuffers();
-
- drivers.removeCurrentDriver();
-
- if ( handle ) {
- CloseHandle( handle->condition );
- if ( handle->bufferInfos )
- free( handle->bufferInfos );
-
- delete handle;
- stream_.apiHandle = 0;
- }
-
-
- if ( stream_.userBuffer[mode] ) {
- free( stream_.userBuffer[mode] );
- stream_.userBuffer[mode] = 0;
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
- }
-
- return FAILURE;
-}////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-void RtApiAsio :: closeStream()
-{
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiAsio::closeStream(): no open stream to close!";
- error( RtAudioError::WARNING );
- return;
- }
-
- if ( stream_.state == STREAM_RUNNING ) {
- stream_.state = STREAM_STOPPED;
- ASIOStop();
- }
- ASIODisposeBuffers();
- drivers.removeCurrentDriver();
-
- AsioHandle *handle = (AsioHandle *) stream_.apiHandle;
- if ( handle ) {
- CloseHandle( handle->condition );
- if ( handle->bufferInfos )
- free( handle->bufferInfos );
- delete handle;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- stream_.mode = UNINITIALIZED;
- stream_.state = STREAM_CLOSED;
-}
-
-bool stopThreadCalled = false;
-
-void RtApiAsio :: startStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_RUNNING ) {
- errorText_ = "RtApiAsio::startStream(): the stream is already running!";
- error( RtAudioError::WARNING );
- return;
- }
-
- AsioHandle *handle = (AsioHandle *) stream_.apiHandle;
- ASIOError result = ASIOStart();
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::startStream: error (" << getAsioErrorString( result ) << ") starting device.";
- errorText_ = errorStream_.str();
- goto unlock;
- }
-
- handle->drainCounter = 0;
- handle->internalDrain = false;
- ResetEvent( handle->condition );
- stream_.state = STREAM_RUNNING;
- asioXRun = false;
-
- unlock:
- stopThreadCalled = false;
-
- if ( result == ASE_OK ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiAsio :: stopStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiAsio::stopStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- AsioHandle *handle = (AsioHandle *) stream_.apiHandle;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- if ( handle->drainCounter == 0 ) {
- handle->drainCounter = 2;
- WaitForSingleObject( handle->condition, INFINITE ); // block until signaled
- }
- }
-
- stream_.state = STREAM_STOPPED;
-
- ASIOError result = ASIOStop();
- if ( result != ASE_OK ) {
- errorStream_ << "RtApiAsio::stopStream: error (" << getAsioErrorString( result ) << ") stopping device.";
- errorText_ = errorStream_.str();
- }
-
- if ( result == ASE_OK ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiAsio :: abortStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiAsio::abortStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- // The following lines were commented-out because some behavior was
- // noted where the device buffers need to be zeroed to avoid
- // continuing sound, even when the device buffers are completely
- // disposed. So now, calling abort is the same as calling stop.
- // AsioHandle *handle = (AsioHandle *) stream_.apiHandle;
- // handle->drainCounter = 2;
- stopStream();
-}
-
-// This function will be called by a spawned thread when the user
-// callback function signals that the stream should be stopped or
-// aborted. It is necessary to handle it this way because the
-// callbackEvent() function must return before the ASIOStop()
-// function will return.
-static unsigned __stdcall asioStopStream( void *ptr )
-{
- CallbackInfo *info = (CallbackInfo *) ptr;
- RtApiAsio *object = (RtApiAsio *) info->object;
-
- object->stopStream();
- _endthreadex( 0 );
- return 0;
-}
-
-bool RtApiAsio :: callbackEvent( long bufferIndex )
-{
- if ( stream_.state == STREAM_STOPPED || stream_.state == STREAM_STOPPING ) return SUCCESS;
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiAsio::callbackEvent(): the stream is closed ... this shouldn't happen!";
- error( RtAudioError::WARNING );
- return FAILURE;
- }
-
- CallbackInfo *info = (CallbackInfo *) &stream_.callbackInfo;
- AsioHandle *handle = (AsioHandle *) stream_.apiHandle;
-
- // Check if we were draining the stream and signal if finished.
- if ( handle->drainCounter > 3 ) {
-
- stream_.state = STREAM_STOPPING;
- if ( handle->internalDrain == false )
- SetEvent( handle->condition );
- else { // spawn a thread to stop the stream
- unsigned threadId;
- stream_.callbackInfo.thread = _beginthreadex( NULL, 0, &asioStopStream,
- &stream_.callbackInfo, 0, &threadId );
- }
- return SUCCESS;
- }
-
- // Invoke user callback to get fresh output data UNLESS we are
- // draining stream.
- if ( handle->drainCounter == 0 ) {
- RtAudioCallback callback = (RtAudioCallback) info->callback;
- double streamTime = getStreamTime();
- RtAudioStreamStatus status = 0;
- if ( stream_.mode != INPUT && asioXRun == true ) {
- status |= RTAUDIO_OUTPUT_UNDERFLOW;
- asioXRun = false;
- }
- if ( stream_.mode != OUTPUT && asioXRun == true ) {
- status |= RTAUDIO_INPUT_OVERFLOW;
- asioXRun = false;
- }
- int cbReturnValue = callback( stream_.userBuffer[0], stream_.userBuffer[1],
- stream_.bufferSize, streamTime, status, info->userData );
- if ( cbReturnValue == 2 ) {
- stream_.state = STREAM_STOPPING;
- handle->drainCounter = 2;
- unsigned threadId;
- stream_.callbackInfo.thread = _beginthreadex( NULL, 0, &asioStopStream,
- &stream_.callbackInfo, 0, &threadId );
- return SUCCESS;
- }
- else if ( cbReturnValue == 1 ) {
- handle->drainCounter = 1;
- handle->internalDrain = true;
- }
- }
-
- unsigned int nChannels, bufferBytes, i, j;
- nChannels = stream_.nDeviceChannels[0] + stream_.nDeviceChannels[1];
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- bufferBytes = stream_.bufferSize * formatBytes( stream_.deviceFormat[0] );
-
- if ( handle->drainCounter > 1 ) { // write zeros to the output stream
-
- for ( i=0, j=0; i<nChannels; i++ ) {
- if ( handle->bufferInfos[i].isInput != ASIOTrue )
- memset( handle->bufferInfos[i].buffers[bufferIndex], 0, bufferBytes );
- }
-
- }
- else if ( stream_.doConvertBuffer[0] ) {
-
- convertBuffer( stream_.deviceBuffer, stream_.userBuffer[0], stream_.convertInfo[0] );
- if ( stream_.doByteSwap[0] )
- byteSwapBuffer( stream_.deviceBuffer,
- stream_.bufferSize * stream_.nDeviceChannels[0],
- stream_.deviceFormat[0] );
-
- for ( i=0, j=0; i<nChannels; i++ ) {
- if ( handle->bufferInfos[i].isInput != ASIOTrue )
- memcpy( handle->bufferInfos[i].buffers[bufferIndex],
- &stream_.deviceBuffer[j++*bufferBytes], bufferBytes );
- }
-
- }
- else {
-
- if ( stream_.doByteSwap[0] )
- byteSwapBuffer( stream_.userBuffer[0],
- stream_.bufferSize * stream_.nUserChannels[0],
- stream_.userFormat );
-
- for ( i=0, j=0; i<nChannels; i++ ) {
- if ( handle->bufferInfos[i].isInput != ASIOTrue )
- memcpy( handle->bufferInfos[i].buffers[bufferIndex],
- &stream_.userBuffer[0][bufferBytes*j++], bufferBytes );
- }
-
- }
- }
-
- // Don't bother draining input
- if ( handle->drainCounter ) {
- handle->drainCounter++;
- goto unlock;
- }
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) {
-
- bufferBytes = stream_.bufferSize * formatBytes(stream_.deviceFormat[1]);
-
- if (stream_.doConvertBuffer[1]) {
-
- // Always interleave ASIO input data.
- for ( i=0, j=0; i<nChannels; i++ ) {
- if ( handle->bufferInfos[i].isInput == ASIOTrue )
- memcpy( &stream_.deviceBuffer[j++*bufferBytes],
- handle->bufferInfos[i].buffers[bufferIndex],
- bufferBytes );
- }
-
- if ( stream_.doByteSwap[1] )
- byteSwapBuffer( stream_.deviceBuffer,
- stream_.bufferSize * stream_.nDeviceChannels[1],
- stream_.deviceFormat[1] );
- convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] );
-
- }
- else {
- for ( i=0, j=0; i<nChannels; i++ ) {
- if ( handle->bufferInfos[i].isInput == ASIOTrue ) {
- memcpy( &stream_.userBuffer[1][bufferBytes*j++],
- handle->bufferInfos[i].buffers[bufferIndex],
- bufferBytes );
- }
- }
-
- if ( stream_.doByteSwap[1] )
- byteSwapBuffer( stream_.userBuffer[1],
- stream_.bufferSize * stream_.nUserChannels[1],
- stream_.userFormat );
- }
- }
-
- unlock:
- // The following call was suggested by Malte Clasen. While the API
- // documentation indicates it should not be required, some device
- // drivers apparently do not function correctly without it.
- ASIOOutputReady();
-
- RtApi::tickStreamTime();
- return SUCCESS;
-}
-
-static void sampleRateChanged( ASIOSampleRate sRate )
-{
- // The ASIO documentation says that this usually only happens during
- // external sync. Audio processing is not stopped by the driver,
- // actual sample rate might not have even changed, maybe only the
- // sample rate status of an AES/EBU or S/PDIF digital input at the
- // audio device.
-
- RtApi *object = (RtApi *) asioCallbackInfo->object;
- try {
- object->stopStream();
- }
- catch ( RtAudioError &exception ) {
- std::cerr << "\nRtApiAsio: sampleRateChanged() error (" << exception.getMessage() << ")!\n" << std::endl;
- return;
- }
-
- std::cerr << "\nRtApiAsio: driver reports sample rate changed to " << sRate << " ... stream stopped!!!\n" << std::endl;
-}
-
-static long asioMessages( long selector, long value, void* /*message*/, double* /*opt*/ )
-{
- long ret = 0;
-
- switch( selector ) {
- case kAsioSelectorSupported:
- if ( value == kAsioResetRequest
- || value == kAsioEngineVersion
- || value == kAsioResyncRequest
- || value == kAsioLatenciesChanged
- // The following three were added for ASIO 2.0, you don't
- // necessarily have to support them.
- || value == kAsioSupportsTimeInfo
- || value == kAsioSupportsTimeCode
- || value == kAsioSupportsInputMonitor)
- ret = 1L;
- break;
- case kAsioResetRequest:
- // Defer the task and perform the reset of the driver during the
- // next "safe" situation. You cannot reset the driver right now,
- // as this code is called from the driver. Reset the driver is
- // done by completely destruct is. I.e. ASIOStop(),
- // ASIODisposeBuffers(), Destruction Afterwards you initialize the
- // driver again.
- std::cerr << "\nRtApiAsio: driver reset requested!!!" << std::endl;
- ret = 1L;
- break;
- case kAsioResyncRequest:
- // This informs the application that the driver encountered some
- // non-fatal data loss. It is used for synchronization purposes
- // of different media. Added mainly to work around the Win16Mutex
- // problems in Windows 95/98 with the Windows Multimedia system,
- // which could lose data because the Mutex was held too long by
- // another thread. However a driver can issue it in other
- // situations, too.
- // std::cerr << "\nRtApiAsio: driver resync requested!!!" << std::endl;
- asioXRun = true;
- ret = 1L;
- break;
- case kAsioLatenciesChanged:
- // This will inform the host application that the drivers were
- // latencies changed. Beware, it this does not mean that the
- // buffer sizes have changed! You might need to update internal
- // delay data.
- std::cerr << "\nRtApiAsio: driver latency may have changed!!!" << std::endl;
- ret = 1L;
- break;
- case kAsioEngineVersion:
- // Return the supported ASIO version of the host application. If
- // a host application does not implement this selector, ASIO 1.0
- // is assumed by the driver.
- ret = 2L;
- break;
- case kAsioSupportsTimeInfo:
- // Informs the driver whether the
- // asioCallbacks.bufferSwitchTimeInfo() callback is supported.
- // For compatibility with ASIO 1.0 drivers the host application
- // should always support the "old" bufferSwitch method, too.
- ret = 0;
- break;
- case kAsioSupportsTimeCode:
- // Informs the driver whether application is interested in time
- // code info. If an application does not need to know about time
- // code, the driver has less work to do.
- ret = 0;
- break;
- }
- return ret;
-}
-
-static const char* getAsioErrorString( ASIOError result )
-{
- struct Messages
- {
- ASIOError value;
- const char*message;
- };
-
- static const Messages m[] =
- {
- { ASE_NotPresent, "Hardware input or output is not present or available." },
- { ASE_HWMalfunction, "Hardware is malfunctioning." },
- { ASE_InvalidParameter, "Invalid input parameter." },
- { ASE_InvalidMode, "Invalid mode." },
- { ASE_SPNotAdvancing, "Sample position not advancing." },
- { ASE_NoClock, "Sample clock or rate cannot be determined or is not present." },
- { ASE_NoMemory, "Not enough memory to complete the request." }
- };
-
- for ( unsigned int i = 0; i < sizeof(m)/sizeof(m[0]); ++i )
- if ( m[i].value == result ) return m[i].message;
-
- return "Unknown error.";
-}
-
-//******************** End of __WINDOWS_ASIO__ *********************//
-#endif
-
-
-#if defined(__WINDOWS_WASAPI__) // Windows WASAPI API
-
-// Authored by Marcus Tomlinson <themarcustomlinson@gmail.com>, April 2014
-// - Introduces support for the Windows WASAPI API
-// - Aims to deliver bit streams to and from hardware at the lowest possible latency, via the absolute minimum buffer sizes required
-// - Provides flexible stream configuration to an otherwise strict and inflexible WASAPI interface
-// - Includes automatic internal conversion of sample rate and buffer size between hardware and the user
-
-#ifndef INITGUID
- #define INITGUID
-#endif
-#include <audioclient.h>
-#include <avrt.h>
-#include <mmdeviceapi.h>
-#include <functiondiscoverykeys_devpkey.h>
-
-//=============================================================================
-
-#define SAFE_RELEASE( objectPtr )\
-if ( objectPtr )\
-{\
- objectPtr->Release();\
- objectPtr = NULL;\
-}
-
-typedef HANDLE ( __stdcall *TAvSetMmThreadCharacteristicsPtr )( LPCWSTR TaskName, LPDWORD TaskIndex );
-
-//-----------------------------------------------------------------------------
-
-// WASAPI dictates stream sample rate, format, channel count, and in some cases, buffer size.
-// Therefore we must perform all necessary conversions to user buffers in order to satisfy these
-// requirements. WasapiBuffer ring buffers are used between HwIn->UserIn and UserOut->HwOut to
-// provide intermediate storage for read / write synchronization.
-class WasapiBuffer
-{
-public:
- WasapiBuffer()
- : buffer_( NULL ),
- bufferSize_( 0 ),
- inIndex_( 0 ),
- outIndex_( 0 ) {}
-
- ~WasapiBuffer() {
- free( buffer_ );
- }
-
- // sets the length of the internal ring buffer
- void setBufferSize( unsigned int bufferSize, unsigned int formatBytes ) {
- free( buffer_ );
-
- buffer_ = ( char* ) calloc( bufferSize, formatBytes );
-
- bufferSize_ = bufferSize;
- inIndex_ = 0;
- outIndex_ = 0;
- }
-
- // attempt to push a buffer into the ring buffer at the current "in" index
- bool pushBuffer( char* buffer, unsigned int bufferSize, RtAudioFormat format )
- {
- if ( !buffer || // incoming buffer is NULL
- bufferSize == 0 || // incoming buffer has no data
- bufferSize > bufferSize_ ) // incoming buffer too large
- {
- return false;
- }
-
- unsigned int relOutIndex = outIndex_;
- unsigned int inIndexEnd = inIndex_ + bufferSize;
- if ( relOutIndex < inIndex_ && inIndexEnd >= bufferSize_ ) {
- relOutIndex += bufferSize_;
- }
-
- // "in" index can end on the "out" index but cannot begin at it
- if ( inIndex_ <= relOutIndex && inIndexEnd > relOutIndex ) {
- return false; // not enough space between "in" index and "out" index
- }
-
- // copy buffer from external to internal
- int fromZeroSize = inIndex_ + bufferSize - bufferSize_;
- fromZeroSize = fromZeroSize < 0 ? 0 : fromZeroSize;
- int fromInSize = bufferSize - fromZeroSize;
-
- switch( format )
- {
- case RTAUDIO_SINT8:
- memcpy( &( ( char* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( char ) );
- memcpy( buffer_, &( ( char* ) buffer )[fromInSize], fromZeroSize * sizeof( char ) );
- break;
- case RTAUDIO_SINT16:
- memcpy( &( ( short* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( short ) );
- memcpy( buffer_, &( ( short* ) buffer )[fromInSize], fromZeroSize * sizeof( short ) );
- break;
- case RTAUDIO_SINT24:
- memcpy( &( ( S24* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( S24 ) );
- memcpy( buffer_, &( ( S24* ) buffer )[fromInSize], fromZeroSize * sizeof( S24 ) );
- break;
- case RTAUDIO_SINT32:
- memcpy( &( ( int* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( int ) );
- memcpy( buffer_, &( ( int* ) buffer )[fromInSize], fromZeroSize * sizeof( int ) );
- break;
- case RTAUDIO_FLOAT32:
- memcpy( &( ( float* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( float ) );
- memcpy( buffer_, &( ( float* ) buffer )[fromInSize], fromZeroSize * sizeof( float ) );
- break;
- case RTAUDIO_FLOAT64:
- memcpy( &( ( double* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( double ) );
- memcpy( buffer_, &( ( double* ) buffer )[fromInSize], fromZeroSize * sizeof( double ) );
- break;
- }
-
- // update "in" index
- inIndex_ += bufferSize;
- inIndex_ %= bufferSize_;
-
- return true;
- }
-
- // attempt to pull a buffer from the ring buffer from the current "out" index
- bool pullBuffer( char* buffer, unsigned int bufferSize, RtAudioFormat format )
- {
- if ( !buffer || // incoming buffer is NULL
- bufferSize == 0 || // incoming buffer has no data
- bufferSize > bufferSize_ ) // incoming buffer too large
- {
- return false;
- }
-
- unsigned int relInIndex = inIndex_;
- unsigned int outIndexEnd = outIndex_ + bufferSize;
- if ( relInIndex < outIndex_ && outIndexEnd >= bufferSize_ ) {
- relInIndex += bufferSize_;
- }
-
- // "out" index can begin at and end on the "in" index
- if ( outIndex_ < relInIndex && outIndexEnd > relInIndex ) {
- return false; // not enough space between "out" index and "in" index
- }
-
- // copy buffer from internal to external
- int fromZeroSize = outIndex_ + bufferSize - bufferSize_;
- fromZeroSize = fromZeroSize < 0 ? 0 : fromZeroSize;
- int fromOutSize = bufferSize - fromZeroSize;
-
- switch( format )
- {
- case RTAUDIO_SINT8:
- memcpy( buffer, &( ( char* ) buffer_ )[outIndex_], fromOutSize * sizeof( char ) );
- memcpy( &( ( char* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( char ) );
- break;
- case RTAUDIO_SINT16:
- memcpy( buffer, &( ( short* ) buffer_ )[outIndex_], fromOutSize * sizeof( short ) );
- memcpy( &( ( short* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( short ) );
- break;
- case RTAUDIO_SINT24:
- memcpy( buffer, &( ( S24* ) buffer_ )[outIndex_], fromOutSize * sizeof( S24 ) );
- memcpy( &( ( S24* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( S24 ) );
- break;
- case RTAUDIO_SINT32:
- memcpy( buffer, &( ( int* ) buffer_ )[outIndex_], fromOutSize * sizeof( int ) );
- memcpy( &( ( int* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( int ) );
- break;
- case RTAUDIO_FLOAT32:
- memcpy( buffer, &( ( float* ) buffer_ )[outIndex_], fromOutSize * sizeof( float ) );
- memcpy( &( ( float* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( float ) );
- break;
- case RTAUDIO_FLOAT64:
- memcpy( buffer, &( ( double* ) buffer_ )[outIndex_], fromOutSize * sizeof( double ) );
- memcpy( &( ( double* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( double ) );
- break;
- }
-
- // update "out" index
- outIndex_ += bufferSize;
- outIndex_ %= bufferSize_;
-
- return true;
- }
-
-private:
- char* buffer_;
- unsigned int bufferSize_;
- unsigned int inIndex_;
- unsigned int outIndex_;
-};
-
-//-----------------------------------------------------------------------------
-
-// In order to satisfy WASAPI's buffer requirements, we need a means of converting sample rate
-// between HW and the user. The convertBufferWasapi function is used to perform this conversion
-// between HwIn->UserIn and UserOut->HwOut during the stream callback loop.
-// This sample rate converter favors speed over quality, and works best with conversions between
-// one rate and its multiple.
-void convertBufferWasapi( char* outBuffer,
- const char* inBuffer,
- const unsigned int& channelCount,
- const unsigned int& inSampleRate,
- const unsigned int& outSampleRate,
- const unsigned int& inSampleCount,
- unsigned int& outSampleCount,
- const RtAudioFormat& format )
-{
- // calculate the new outSampleCount and relative sampleStep
- float sampleRatio = ( float ) outSampleRate / inSampleRate;
- float sampleStep = 1.0f / sampleRatio;
- float inSampleFraction = 0.0f;
-
- outSampleCount = ( unsigned int ) roundf( inSampleCount * sampleRatio );
-
- // frame-by-frame, copy each relative input sample into it's corresponding output sample
- for ( unsigned int outSample = 0; outSample < outSampleCount; outSample++ )
- {
- unsigned int inSample = ( unsigned int ) inSampleFraction;
-
- switch ( format )
- {
- case RTAUDIO_SINT8:
- memcpy( &( ( char* ) outBuffer )[ outSample * channelCount ], &( ( char* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( char ) );
- break;
- case RTAUDIO_SINT16:
- memcpy( &( ( short* ) outBuffer )[ outSample * channelCount ], &( ( short* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( short ) );
- break;
- case RTAUDIO_SINT24:
- memcpy( &( ( S24* ) outBuffer )[ outSample * channelCount ], &( ( S24* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( S24 ) );
- break;
- case RTAUDIO_SINT32:
- memcpy( &( ( int* ) outBuffer )[ outSample * channelCount ], &( ( int* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( int ) );
- break;
- case RTAUDIO_FLOAT32:
- memcpy( &( ( float* ) outBuffer )[ outSample * channelCount ], &( ( float* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( float ) );
- break;
- case RTAUDIO_FLOAT64:
- memcpy( &( ( double* ) outBuffer )[ outSample * channelCount ], &( ( double* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( double ) );
- break;
- }
-
- // jump to next in sample
- inSampleFraction += sampleStep;
- }
-}
-
-//-----------------------------------------------------------------------------
-
-// A structure to hold various information related to the WASAPI implementation.
-struct WasapiHandle
-{
- IAudioClient* captureAudioClient;
- IAudioClient* renderAudioClient;
- IAudioCaptureClient* captureClient;
- IAudioRenderClient* renderClient;
- HANDLE captureEvent;
- HANDLE renderEvent;
-
- WasapiHandle()
- : captureAudioClient( NULL ),
- renderAudioClient( NULL ),
- captureClient( NULL ),
- renderClient( NULL ),
- captureEvent( NULL ),
- renderEvent( NULL ) {}
-};
-
-//=============================================================================
-
-RtApiWasapi::RtApiWasapi()
- : coInitialized_( false ), deviceEnumerator_( NULL )
-{
- // WASAPI can run either apartment or multi-threaded
- HRESULT hr = CoInitialize( NULL );
- if ( !FAILED( hr ) )
- coInitialized_ = true;
-
- // Instantiate device enumerator
- hr = CoCreateInstance( __uuidof( MMDeviceEnumerator ), NULL,
- CLSCTX_ALL, __uuidof( IMMDeviceEnumerator ),
- ( void** ) &deviceEnumerator_ );
-
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::RtApiWasapi: Unable to instantiate device enumerator";
- error( RtAudioError::DRIVER_ERROR );
- }
-}
-
-//-----------------------------------------------------------------------------
-
-RtApiWasapi::~RtApiWasapi()
-{
- if ( stream_.state != STREAM_CLOSED )
- closeStream();
-
- SAFE_RELEASE( deviceEnumerator_ );
-
- // If this object previously called CoInitialize()
- if ( coInitialized_ )
- CoUninitialize();
-}
-
-//=============================================================================
-
-unsigned int RtApiWasapi::getDeviceCount( void )
-{
- unsigned int captureDeviceCount = 0;
- unsigned int renderDeviceCount = 0;
-
- IMMDeviceCollection* captureDevices = NULL;
- IMMDeviceCollection* renderDevices = NULL;
-
- // Count capture devices
- errorText_.clear();
- HRESULT hr = deviceEnumerator_->EnumAudioEndpoints( eCapture, DEVICE_STATE_ACTIVE, &captureDevices );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceCount: Unable to retrieve capture device collection.";
- goto Exit;
- }
-
- hr = captureDevices->GetCount( &captureDeviceCount );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceCount: Unable to retrieve capture device count.";
- goto Exit;
- }
-
- // Count render devices
- hr = deviceEnumerator_->EnumAudioEndpoints( eRender, DEVICE_STATE_ACTIVE, &renderDevices );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceCount: Unable to retrieve render device collection.";
- goto Exit;
- }
-
- hr = renderDevices->GetCount( &renderDeviceCount );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceCount: Unable to retrieve render device count.";
- goto Exit;
- }
-
-Exit:
- // release all references
- SAFE_RELEASE( captureDevices );
- SAFE_RELEASE( renderDevices );
-
- if ( errorText_.empty() )
- return captureDeviceCount + renderDeviceCount;
-
- error( RtAudioError::DRIVER_ERROR );
- return 0;
-}
-
-//-----------------------------------------------------------------------------
-
-RtAudio::DeviceInfo RtApiWasapi::getDeviceInfo( unsigned int device )
-{
- RtAudio::DeviceInfo info;
- unsigned int captureDeviceCount = 0;
- unsigned int renderDeviceCount = 0;
- std::string defaultDeviceName;
- bool isCaptureDevice = false;
-
- PROPVARIANT deviceNameProp;
- PROPVARIANT defaultDeviceNameProp;
-
- IMMDeviceCollection* captureDevices = NULL;
- IMMDeviceCollection* renderDevices = NULL;
- IMMDevice* devicePtr = NULL;
- IMMDevice* defaultDevicePtr = NULL;
- IAudioClient* audioClient = NULL;
- IPropertyStore* devicePropStore = NULL;
- IPropertyStore* defaultDevicePropStore = NULL;
-
- WAVEFORMATEX* deviceFormat = NULL;
- WAVEFORMATEX* closestMatchFormat = NULL;
-
- // probed
- info.probed = false;
-
- // Count capture devices
- errorText_.clear();
- RtAudioError::Type errorType = RtAudioError::DRIVER_ERROR;
- HRESULT hr = deviceEnumerator_->EnumAudioEndpoints( eCapture, DEVICE_STATE_ACTIVE, &captureDevices );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve capture device collection.";
- goto Exit;
- }
-
- hr = captureDevices->GetCount( &captureDeviceCount );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve capture device count.";
- goto Exit;
- }
-
- // Count render devices
- hr = deviceEnumerator_->EnumAudioEndpoints( eRender, DEVICE_STATE_ACTIVE, &renderDevices );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve render device collection.";
- goto Exit;
- }
-
- hr = renderDevices->GetCount( &renderDeviceCount );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve render device count.";
- goto Exit;
- }
-
- // validate device index
- if ( device >= captureDeviceCount + renderDeviceCount ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Invalid device index.";
- errorType = RtAudioError::INVALID_USE;
- goto Exit;
- }
-
- // determine whether index falls within capture or render devices
- if ( device >= renderDeviceCount ) {
- hr = captureDevices->Item( device - renderDeviceCount, &devicePtr );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve capture device handle.";
- goto Exit;
- }
- isCaptureDevice = true;
- }
- else {
- hr = renderDevices->Item( device, &devicePtr );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve render device handle.";
- goto Exit;
- }
- isCaptureDevice = false;
- }
-
- // get default device name
- if ( isCaptureDevice ) {
- hr = deviceEnumerator_->GetDefaultAudioEndpoint( eCapture, eConsole, &defaultDevicePtr );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve default capture device handle.";
- goto Exit;
- }
- }
- else {
- hr = deviceEnumerator_->GetDefaultAudioEndpoint( eRender, eConsole, &defaultDevicePtr );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve default render device handle.";
- goto Exit;
- }
- }
-
- hr = defaultDevicePtr->OpenPropertyStore( STGM_READ, &defaultDevicePropStore );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to open default device property store.";
- goto Exit;
- }
- PropVariantInit( &defaultDeviceNameProp );
-
- hr = defaultDevicePropStore->GetValue( PKEY_Device_FriendlyName, &defaultDeviceNameProp );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve default device property: PKEY_Device_FriendlyName.";
- goto Exit;
- }
-
- defaultDeviceName = convertCharPointerToStdString(defaultDeviceNameProp.pwszVal);
-
- // name
- hr = devicePtr->OpenPropertyStore( STGM_READ, &devicePropStore );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to open device property store.";
- goto Exit;
- }
-
- PropVariantInit( &deviceNameProp );
-
- hr = devicePropStore->GetValue( PKEY_Device_FriendlyName, &deviceNameProp );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve device property: PKEY_Device_FriendlyName.";
- goto Exit;
- }
-
- info.name =convertCharPointerToStdString(deviceNameProp.pwszVal);
-
- // is default
- if ( isCaptureDevice ) {
- info.isDefaultInput = info.name == defaultDeviceName;
- info.isDefaultOutput = false;
- }
- else {
- info.isDefaultInput = false;
- info.isDefaultOutput = info.name == defaultDeviceName;
- }
-
- // channel count
- hr = devicePtr->Activate( __uuidof( IAudioClient ), CLSCTX_ALL, NULL, ( void** ) &audioClient );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve device audio client.";
- goto Exit;
- }
-
- hr = audioClient->GetMixFormat( &deviceFormat );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve device mix format.";
- goto Exit;
- }
-
- if ( isCaptureDevice ) {
- info.inputChannels = deviceFormat->nChannels;
- info.outputChannels = 0;
- info.duplexChannels = 0;
- }
- else {
- info.inputChannels = 0;
- info.outputChannels = deviceFormat->nChannels;
- info.duplexChannels = 0;
- }
-
- // sample rates
- info.sampleRates.clear();
-
- // allow support for all sample rates as we have a built-in sample rate converter
- for ( unsigned int i = 0; i < MAX_SAMPLE_RATES; i++ ) {
- info.sampleRates.push_back( SAMPLE_RATES[i] );
- }
- info.preferredSampleRate = deviceFormat->nSamplesPerSec;
-
- // native format
- info.nativeFormats = 0;
-
- if ( deviceFormat->wFormatTag == WAVE_FORMAT_IEEE_FLOAT ||
- ( deviceFormat->wFormatTag == WAVE_FORMAT_EXTENSIBLE &&
- ( ( WAVEFORMATEXTENSIBLE* ) deviceFormat )->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT ) )
- {
- if ( deviceFormat->wBitsPerSample == 32 ) {
- info.nativeFormats |= RTAUDIO_FLOAT32;
- }
- else if ( deviceFormat->wBitsPerSample == 64 ) {
- info.nativeFormats |= RTAUDIO_FLOAT64;
- }
- }
- else if ( deviceFormat->wFormatTag == WAVE_FORMAT_PCM ||
- ( deviceFormat->wFormatTag == WAVE_FORMAT_EXTENSIBLE &&
- ( ( WAVEFORMATEXTENSIBLE* ) deviceFormat )->SubFormat == KSDATAFORMAT_SUBTYPE_PCM ) )
- {
- if ( deviceFormat->wBitsPerSample == 8 ) {
- info.nativeFormats |= RTAUDIO_SINT8;
- }
- else if ( deviceFormat->wBitsPerSample == 16 ) {
- info.nativeFormats |= RTAUDIO_SINT16;
- }
- else if ( deviceFormat->wBitsPerSample == 24 ) {
- info.nativeFormats |= RTAUDIO_SINT24;
- }
- else if ( deviceFormat->wBitsPerSample == 32 ) {
- info.nativeFormats |= RTAUDIO_SINT32;
- }
- }
-
- // probed
- info.probed = true;
-
-Exit:
- // release all references
- PropVariantClear( &deviceNameProp );
- PropVariantClear( &defaultDeviceNameProp );
-
- SAFE_RELEASE( captureDevices );
- SAFE_RELEASE( renderDevices );
- SAFE_RELEASE( devicePtr );
- SAFE_RELEASE( defaultDevicePtr );
- SAFE_RELEASE( audioClient );
- SAFE_RELEASE( devicePropStore );
- SAFE_RELEASE( defaultDevicePropStore );
-
- CoTaskMemFree( deviceFormat );
- CoTaskMemFree( closestMatchFormat );
-
- if ( !errorText_.empty() )
- error( errorType );
- return info;
-}
-
-//-----------------------------------------------------------------------------
-
-unsigned int RtApiWasapi::getDefaultOutputDevice( void )
-{
- for ( unsigned int i = 0; i < getDeviceCount(); i++ ) {
- if ( getDeviceInfo( i ).isDefaultOutput ) {
- return i;
- }
- }
-
- return 0;
-}
-
-//-----------------------------------------------------------------------------
-
-unsigned int RtApiWasapi::getDefaultInputDevice( void )
-{
- for ( unsigned int i = 0; i < getDeviceCount(); i++ ) {
- if ( getDeviceInfo( i ).isDefaultInput ) {
- return i;
- }
- }
-
- return 0;
-}
-
-//-----------------------------------------------------------------------------
-
-void RtApiWasapi::closeStream( void )
-{
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiWasapi::closeStream: No open stream to close.";
- error( RtAudioError::WARNING );
- return;
- }
-
- if ( stream_.state != STREAM_STOPPED )
- stopStream();
-
- // clean up stream memory
- SAFE_RELEASE( ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient )
- SAFE_RELEASE( ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient )
-
- SAFE_RELEASE( ( ( WasapiHandle* ) stream_.apiHandle )->captureClient )
- SAFE_RELEASE( ( ( WasapiHandle* ) stream_.apiHandle )->renderClient )
-
- if ( ( ( WasapiHandle* ) stream_.apiHandle )->captureEvent )
- CloseHandle( ( ( WasapiHandle* ) stream_.apiHandle )->captureEvent );
-
- if ( ( ( WasapiHandle* ) stream_.apiHandle )->renderEvent )
- CloseHandle( ( ( WasapiHandle* ) stream_.apiHandle )->renderEvent );
-
- delete ( WasapiHandle* ) stream_.apiHandle;
- stream_.apiHandle = NULL;
-
- for ( int i = 0; i < 2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- // update stream state
- stream_.state = STREAM_CLOSED;
-}
-
-//-----------------------------------------------------------------------------
-
-void RtApiWasapi::startStream( void )
-{
- verifyStream();
-
- if ( stream_.state == STREAM_RUNNING ) {
- errorText_ = "RtApiWasapi::startStream: The stream is already running.";
- error( RtAudioError::WARNING );
- return;
- }
-
- // update stream state
- stream_.state = STREAM_RUNNING;
-
- // create WASAPI stream thread
- stream_.callbackInfo.thread = ( ThreadHandle ) CreateThread( NULL, 0, runWasapiThread, this, CREATE_SUSPENDED, NULL );
-
- if ( !stream_.callbackInfo.thread ) {
- errorText_ = "RtApiWasapi::startStream: Unable to instantiate callback thread.";
- error( RtAudioError::THREAD_ERROR );
- }
- else {
- SetThreadPriority( ( void* ) stream_.callbackInfo.thread, stream_.callbackInfo.priority );
- ResumeThread( ( void* ) stream_.callbackInfo.thread );
- }
-}
-
-//-----------------------------------------------------------------------------
-
-void RtApiWasapi::stopStream( void )
-{
- verifyStream();
-
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiWasapi::stopStream: The stream is already stopped.";
- error( RtAudioError::WARNING );
- return;
- }
-
- // inform stream thread by setting stream state to STREAM_STOPPING
- stream_.state = STREAM_STOPPING;
-
- // wait until stream thread is stopped
- while( stream_.state != STREAM_STOPPED ) {
- Sleep( 1 );
- }
-
- // Wait for the last buffer to play before stopping.
- Sleep( 1000 * stream_.bufferSize / stream_.sampleRate );
-
- // stop capture client if applicable
- if ( ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient ) {
- HRESULT hr = ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient->Stop();
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::stopStream: Unable to stop capture stream.";
- error( RtAudioError::DRIVER_ERROR );
- return;
- }
- }
-
- // stop render client if applicable
- if ( ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient ) {
- HRESULT hr = ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient->Stop();
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::stopStream: Unable to stop render stream.";
- error( RtAudioError::DRIVER_ERROR );
- return;
- }
- }
-
- // close thread handle
- if ( stream_.callbackInfo.thread && !CloseHandle( ( void* ) stream_.callbackInfo.thread ) ) {
- errorText_ = "RtApiWasapi::stopStream: Unable to close callback thread.";
- error( RtAudioError::THREAD_ERROR );
- return;
- }
-
- stream_.callbackInfo.thread = (ThreadHandle) NULL;
-}
-
-//-----------------------------------------------------------------------------
-
-void RtApiWasapi::abortStream( void )
-{
- verifyStream();
-
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiWasapi::abortStream: The stream is already stopped.";
- error( RtAudioError::WARNING );
- return;
- }
-
- // inform stream thread by setting stream state to STREAM_STOPPING
- stream_.state = STREAM_STOPPING;
-
- // wait until stream thread is stopped
- while ( stream_.state != STREAM_STOPPED ) {
- Sleep( 1 );
- }
-
- // stop capture client if applicable
- if ( ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient ) {
- HRESULT hr = ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient->Stop();
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::abortStream: Unable to stop capture stream.";
- error( RtAudioError::DRIVER_ERROR );
- return;
- }
- }
-
- // stop render client if applicable
- if ( ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient ) {
- HRESULT hr = ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient->Stop();
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::abortStream: Unable to stop render stream.";
- error( RtAudioError::DRIVER_ERROR );
- return;
- }
- }
-
- // close thread handle
- if ( stream_.callbackInfo.thread && !CloseHandle( ( void* ) stream_.callbackInfo.thread ) ) {
- errorText_ = "RtApiWasapi::abortStream: Unable to close callback thread.";
- error( RtAudioError::THREAD_ERROR );
- return;
- }
-
- stream_.callbackInfo.thread = (ThreadHandle) NULL;
-}
-
-//-----------------------------------------------------------------------------
-
-bool RtApiWasapi::probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels,
- unsigned int firstChannel, unsigned int sampleRate,
- RtAudioFormat format, unsigned int* bufferSize,
- RtAudio::StreamOptions* options )
-{
- bool methodResult = FAILURE;
- unsigned int captureDeviceCount = 0;
- unsigned int renderDeviceCount = 0;
-
- IMMDeviceCollection* captureDevices = NULL;
- IMMDeviceCollection* renderDevices = NULL;
- IMMDevice* devicePtr = NULL;
- WAVEFORMATEX* deviceFormat = NULL;
- unsigned int bufferBytes;
- stream_.state = STREAM_STOPPED;
-
- // create API Handle if not already created
- if ( !stream_.apiHandle )
- stream_.apiHandle = ( void* ) new WasapiHandle();
-
- // Count capture devices
- errorText_.clear();
- RtAudioError::Type errorType = RtAudioError::DRIVER_ERROR;
- HRESULT hr = deviceEnumerator_->EnumAudioEndpoints( eCapture, DEVICE_STATE_ACTIVE, &captureDevices );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve capture device collection.";
- goto Exit;
- }
-
- hr = captureDevices->GetCount( &captureDeviceCount );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve capture device count.";
- goto Exit;
- }
-
- // Count render devices
- hr = deviceEnumerator_->EnumAudioEndpoints( eRender, DEVICE_STATE_ACTIVE, &renderDevices );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve render device collection.";
- goto Exit;
- }
-
- hr = renderDevices->GetCount( &renderDeviceCount );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve render device count.";
- goto Exit;
- }
-
- // validate device index
- if ( device >= captureDeviceCount + renderDeviceCount ) {
- errorType = RtAudioError::INVALID_USE;
- errorText_ = "RtApiWasapi::probeDeviceOpen: Invalid device index.";
- goto Exit;
- }
-
- // determine whether index falls within capture or render devices
- if ( device >= renderDeviceCount ) {
- if ( mode != INPUT ) {
- errorType = RtAudioError::INVALID_USE;
- errorText_ = "RtApiWasapi::probeDeviceOpen: Capture device selected as output device.";
- goto Exit;
- }
-
- // retrieve captureAudioClient from devicePtr
- IAudioClient*& captureAudioClient = ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient;
-
- hr = captureDevices->Item( device - renderDeviceCount, &devicePtr );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve capture device handle.";
- goto Exit;
- }
-
- hr = devicePtr->Activate( __uuidof( IAudioClient ), CLSCTX_ALL,
- NULL, ( void** ) &captureAudioClient );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve device audio client.";
- goto Exit;
- }
-
- hr = captureAudioClient->GetMixFormat( &deviceFormat );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve device mix format.";
- goto Exit;
- }
-
- stream_.nDeviceChannels[mode] = deviceFormat->nChannels;
- captureAudioClient->GetStreamLatency( ( long long* ) &stream_.latency[mode] );
- }
- else {
- if ( mode != OUTPUT ) {
- errorType = RtAudioError::INVALID_USE;
- errorText_ = "RtApiWasapi::probeDeviceOpen: Render device selected as input device.";
- goto Exit;
- }
-
- // retrieve renderAudioClient from devicePtr
- IAudioClient*& renderAudioClient = ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient;
-
- hr = renderDevices->Item( device, &devicePtr );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve render device handle.";
- goto Exit;
- }
-
- hr = devicePtr->Activate( __uuidof( IAudioClient ), CLSCTX_ALL,
- NULL, ( void** ) &renderAudioClient );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve device audio client.";
- goto Exit;
- }
-
- hr = renderAudioClient->GetMixFormat( &deviceFormat );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve device mix format.";
- goto Exit;
- }
-
- stream_.nDeviceChannels[mode] = deviceFormat->nChannels;
- renderAudioClient->GetStreamLatency( ( long long* ) &stream_.latency[mode] );
- }
-
- // fill stream data
- if ( ( stream_.mode == OUTPUT && mode == INPUT ) ||
- ( stream_.mode == INPUT && mode == OUTPUT ) ) {
- stream_.mode = DUPLEX;
- }
- else {
- stream_.mode = mode;
- }
-
- stream_.device[mode] = device;
- stream_.doByteSwap[mode] = false;
- stream_.sampleRate = sampleRate;
- stream_.bufferSize = *bufferSize;
- stream_.nBuffers = 1;
- stream_.nUserChannels[mode] = channels;
- stream_.channelOffset[mode] = firstChannel;
- stream_.userFormat = format;
- stream_.deviceFormat[mode] = getDeviceInfo( device ).nativeFormats;
-
- if ( options && options->flags & RTAUDIO_NONINTERLEAVED )
- stream_.userInterleaved = false;
- else
- stream_.userInterleaved = true;
- stream_.deviceInterleaved[mode] = true;
-
- // Set flags for buffer conversion.
- stream_.doConvertBuffer[mode] = false;
- if ( stream_.userFormat != stream_.deviceFormat[mode] ||
- stream_.nUserChannels != stream_.nDeviceChannels )
- stream_.doConvertBuffer[mode] = true;
- else if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] &&
- stream_.nUserChannels[mode] > 1 )
- stream_.doConvertBuffer[mode] = true;
-
- if ( stream_.doConvertBuffer[mode] )
- setConvertInfo( mode, 0 );
-
- // Allocate necessary internal buffers
- bufferBytes = stream_.nUserChannels[mode] * stream_.bufferSize * formatBytes( stream_.userFormat );
-
- stream_.userBuffer[mode] = ( char* ) calloc( bufferBytes, 1 );
- if ( !stream_.userBuffer[mode] ) {
- errorType = RtAudioError::MEMORY_ERROR;
- errorText_ = "RtApiWasapi::probeDeviceOpen: Error allocating user buffer memory.";
- goto Exit;
- }
-
- if ( options && options->flags & RTAUDIO_SCHEDULE_REALTIME )
- stream_.callbackInfo.priority = 15;
- else
- stream_.callbackInfo.priority = 0;
-
- ///! TODO: RTAUDIO_MINIMIZE_LATENCY // Provide stream buffers directly to callback
- ///! TODO: RTAUDIO_HOG_DEVICE // Exclusive mode
-
- methodResult = SUCCESS;
-
-Exit:
- //clean up
- SAFE_RELEASE( captureDevices );
- SAFE_RELEASE( renderDevices );
- SAFE_RELEASE( devicePtr );
- CoTaskMemFree( deviceFormat );
-
- // if method failed, close the stream
- if ( methodResult == FAILURE )
- closeStream();
-
- if ( !errorText_.empty() )
- error( errorType );
- return methodResult;
-}
-
-//=============================================================================
-
-DWORD WINAPI RtApiWasapi::runWasapiThread( void* wasapiPtr )
-{
- if ( wasapiPtr )
- ( ( RtApiWasapi* ) wasapiPtr )->wasapiThread();
-
- return 0;
-}
-
-DWORD WINAPI RtApiWasapi::stopWasapiThread( void* wasapiPtr )
-{
- if ( wasapiPtr )
- ( ( RtApiWasapi* ) wasapiPtr )->stopStream();
-
- return 0;
-}
-
-DWORD WINAPI RtApiWasapi::abortWasapiThread( void* wasapiPtr )
-{
- if ( wasapiPtr )
- ( ( RtApiWasapi* ) wasapiPtr )->abortStream();
-
- return 0;
-}
-
-//-----------------------------------------------------------------------------
-
-void RtApiWasapi::wasapiThread()
-{
- // as this is a new thread, we must CoInitialize it
- CoInitialize( NULL );
-
- HRESULT hr;
-
- IAudioClient* captureAudioClient = ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient;
- IAudioClient* renderAudioClient = ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient;
- IAudioCaptureClient* captureClient = ( ( WasapiHandle* ) stream_.apiHandle )->captureClient;
- IAudioRenderClient* renderClient = ( ( WasapiHandle* ) stream_.apiHandle )->renderClient;
- HANDLE captureEvent = ( ( WasapiHandle* ) stream_.apiHandle )->captureEvent;
- HANDLE renderEvent = ( ( WasapiHandle* ) stream_.apiHandle )->renderEvent;
-
- WAVEFORMATEX* captureFormat = NULL;
- WAVEFORMATEX* renderFormat = NULL;
- float captureSrRatio = 0.0f;
- float renderSrRatio = 0.0f;
- WasapiBuffer captureBuffer;
- WasapiBuffer renderBuffer;
-
- // declare local stream variables
- RtAudioCallback callback = ( RtAudioCallback ) stream_.callbackInfo.callback;
- BYTE* streamBuffer = NULL;
- unsigned long captureFlags = 0;
- unsigned int bufferFrameCount = 0;
- unsigned int numFramesPadding = 0;
- unsigned int convBufferSize = 0;
- bool callbackPushed = false;
- bool callbackPulled = false;
- bool callbackStopped = false;
- int callbackResult = 0;
-
- // convBuffer is used to store converted buffers between WASAPI and the user
- char* convBuffer = NULL;
- unsigned int convBuffSize = 0;
- unsigned int deviceBuffSize = 0;
-
- errorText_.clear();
- RtAudioError::Type errorType = RtAudioError::DRIVER_ERROR;
-
- // Attempt to assign "Pro Audio" characteristic to thread
- HMODULE AvrtDll = LoadLibrary( (LPCTSTR) "AVRT.dll" );
- if ( AvrtDll ) {
- DWORD taskIndex = 0;
- TAvSetMmThreadCharacteristicsPtr AvSetMmThreadCharacteristicsPtr = ( TAvSetMmThreadCharacteristicsPtr ) GetProcAddress( AvrtDll, "AvSetMmThreadCharacteristicsW" );
- AvSetMmThreadCharacteristicsPtr( L"Pro Audio", &taskIndex );
- FreeLibrary( AvrtDll );
- }
-
- // start capture stream if applicable
- if ( captureAudioClient ) {
- hr = captureAudioClient->GetMixFormat( &captureFormat );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve device mix format.";
- goto Exit;
- }
-
- captureSrRatio = ( ( float ) captureFormat->nSamplesPerSec / stream_.sampleRate );
-
- // initialize capture stream according to desire buffer size
- float desiredBufferSize = stream_.bufferSize * captureSrRatio;
- REFERENCE_TIME desiredBufferPeriod = ( REFERENCE_TIME ) ( ( float ) desiredBufferSize * 10000000 / captureFormat->nSamplesPerSec );
-
- if ( !captureClient ) {
- hr = captureAudioClient->Initialize( AUDCLNT_SHAREMODE_SHARED,
- AUDCLNT_STREAMFLAGS_EVENTCALLBACK,
- desiredBufferPeriod,
- desiredBufferPeriod,
- captureFormat,
- NULL );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to initialize capture audio client.";
- goto Exit;
- }
-
- hr = captureAudioClient->GetService( __uuidof( IAudioCaptureClient ),
- ( void** ) &captureClient );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve capture client handle.";
- goto Exit;
- }
-
- // configure captureEvent to trigger on every available capture buffer
- captureEvent = CreateEvent( NULL, FALSE, FALSE, NULL );
- if ( !captureEvent ) {
- errorType = RtAudioError::SYSTEM_ERROR;
- errorText_ = "RtApiWasapi::wasapiThread: Unable to create capture event.";
- goto Exit;
- }
-
- hr = captureAudioClient->SetEventHandle( captureEvent );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to set capture event handle.";
- goto Exit;
- }
-
- ( ( WasapiHandle* ) stream_.apiHandle )->captureClient = captureClient;
- ( ( WasapiHandle* ) stream_.apiHandle )->captureEvent = captureEvent;
- }
-
- unsigned int inBufferSize = 0;
- hr = captureAudioClient->GetBufferSize( &inBufferSize );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to get capture buffer size.";
- goto Exit;
- }
-
- // scale outBufferSize according to stream->user sample rate ratio
- unsigned int outBufferSize = ( unsigned int ) ( stream_.bufferSize * captureSrRatio ) * stream_.nDeviceChannels[INPUT];
- inBufferSize *= stream_.nDeviceChannels[INPUT];
-
- // set captureBuffer size
- captureBuffer.setBufferSize( inBufferSize + outBufferSize, formatBytes( stream_.deviceFormat[INPUT] ) );
-
- // reset the capture stream
- hr = captureAudioClient->Reset();
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to reset capture stream.";
- goto Exit;
- }
-
- // start the capture stream
- hr = captureAudioClient->Start();
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to start capture stream.";
- goto Exit;
- }
- }
-
- // start render stream if applicable
- if ( renderAudioClient ) {
- hr = renderAudioClient->GetMixFormat( &renderFormat );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve device mix format.";
- goto Exit;
- }
-
- renderSrRatio = ( ( float ) renderFormat->nSamplesPerSec / stream_.sampleRate );
-
- // initialize render stream according to desire buffer size
- float desiredBufferSize = stream_.bufferSize * renderSrRatio;
- REFERENCE_TIME desiredBufferPeriod = ( REFERENCE_TIME ) ( ( float ) desiredBufferSize * 10000000 / renderFormat->nSamplesPerSec );
-
- if ( !renderClient ) {
- hr = renderAudioClient->Initialize( AUDCLNT_SHAREMODE_SHARED,
- AUDCLNT_STREAMFLAGS_EVENTCALLBACK,
- desiredBufferPeriod,
- desiredBufferPeriod,
- renderFormat,
- NULL );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to initialize render audio client.";
- goto Exit;
- }
-
- hr = renderAudioClient->GetService( __uuidof( IAudioRenderClient ),
- ( void** ) &renderClient );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve render client handle.";
- goto Exit;
- }
-
- // configure renderEvent to trigger on every available render buffer
- renderEvent = CreateEvent( NULL, FALSE, FALSE, NULL );
- if ( !renderEvent ) {
- errorType = RtAudioError::SYSTEM_ERROR;
- errorText_ = "RtApiWasapi::wasapiThread: Unable to create render event.";
- goto Exit;
- }
-
- hr = renderAudioClient->SetEventHandle( renderEvent );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to set render event handle.";
- goto Exit;
- }
-
- ( ( WasapiHandle* ) stream_.apiHandle )->renderClient = renderClient;
- ( ( WasapiHandle* ) stream_.apiHandle )->renderEvent = renderEvent;
- }
-
- unsigned int outBufferSize = 0;
- hr = renderAudioClient->GetBufferSize( &outBufferSize );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to get render buffer size.";
- goto Exit;
- }
-
- // scale inBufferSize according to user->stream sample rate ratio
- unsigned int inBufferSize = ( unsigned int ) ( stream_.bufferSize * renderSrRatio ) * stream_.nDeviceChannels[OUTPUT];
- outBufferSize *= stream_.nDeviceChannels[OUTPUT];
-
- // set renderBuffer size
- renderBuffer.setBufferSize( inBufferSize + outBufferSize, formatBytes( stream_.deviceFormat[OUTPUT] ) );
-
- // reset the render stream
- hr = renderAudioClient->Reset();
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to reset render stream.";
- goto Exit;
- }
-
- // start the render stream
- hr = renderAudioClient->Start();
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to start render stream.";
- goto Exit;
- }
- }
-
- if ( stream_.mode == INPUT ) {
- convBuffSize = ( size_t ) ( stream_.bufferSize * captureSrRatio ) * stream_.nDeviceChannels[INPUT] * formatBytes( stream_.deviceFormat[INPUT] );
- deviceBuffSize = stream_.bufferSize * stream_.nDeviceChannels[INPUT] * formatBytes( stream_.deviceFormat[INPUT] );
- }
- else if ( stream_.mode == OUTPUT ) {
- convBuffSize = ( size_t ) ( stream_.bufferSize * renderSrRatio ) * stream_.nDeviceChannels[OUTPUT] * formatBytes( stream_.deviceFormat[OUTPUT] );
- deviceBuffSize = stream_.bufferSize * stream_.nDeviceChannels[OUTPUT] * formatBytes( stream_.deviceFormat[OUTPUT] );
- }
- else if ( stream_.mode == DUPLEX ) {
- convBuffSize = std::max( ( size_t ) ( stream_.bufferSize * captureSrRatio ) * stream_.nDeviceChannels[INPUT] * formatBytes( stream_.deviceFormat[INPUT] ),
- ( size_t ) ( stream_.bufferSize * renderSrRatio ) * stream_.nDeviceChannels[OUTPUT] * formatBytes( stream_.deviceFormat[OUTPUT] ) );
- deviceBuffSize = std::max( stream_.bufferSize * stream_.nDeviceChannels[INPUT] * formatBytes( stream_.deviceFormat[INPUT] ),
- stream_.bufferSize * stream_.nDeviceChannels[OUTPUT] * formatBytes( stream_.deviceFormat[OUTPUT] ) );
- }
-
- convBuffer = ( char* ) malloc( convBuffSize );
- stream_.deviceBuffer = ( char* ) malloc( deviceBuffSize );
- if ( !convBuffer || !stream_.deviceBuffer ) {
- errorType = RtAudioError::MEMORY_ERROR;
- errorText_ = "RtApiWasapi::wasapiThread: Error allocating device buffer memory.";
- goto Exit;
- }
-
- // stream process loop
- while ( stream_.state != STREAM_STOPPING ) {
- if ( !callbackPulled ) {
- // Callback Input
- // ==============
- // 1. Pull callback buffer from inputBuffer
- // 2. If 1. was successful: Convert callback buffer to user sample rate and channel count
- // Convert callback buffer to user format
-
- if ( captureAudioClient ) {
- // Pull callback buffer from inputBuffer
- callbackPulled = captureBuffer.pullBuffer( convBuffer,
- ( unsigned int ) ( stream_.bufferSize * captureSrRatio ) * stream_.nDeviceChannels[INPUT],
- stream_.deviceFormat[INPUT] );
-
- if ( callbackPulled ) {
- // Convert callback buffer to user sample rate
- convertBufferWasapi( stream_.deviceBuffer,
- convBuffer,
- stream_.nDeviceChannels[INPUT],
- captureFormat->nSamplesPerSec,
- stream_.sampleRate,
- ( unsigned int ) ( stream_.bufferSize * captureSrRatio ),
- convBufferSize,
- stream_.deviceFormat[INPUT] );
-
- if ( stream_.doConvertBuffer[INPUT] ) {
- // Convert callback buffer to user format
- convertBuffer( stream_.userBuffer[INPUT],
- stream_.deviceBuffer,
- stream_.convertInfo[INPUT] );
- }
- else {
- // no further conversion, simple copy deviceBuffer to userBuffer
- memcpy( stream_.userBuffer[INPUT],
- stream_.deviceBuffer,
- stream_.bufferSize * stream_.nUserChannels[INPUT] * formatBytes( stream_.userFormat ) );
- }
- }
- }
- else {
- // if there is no capture stream, set callbackPulled flag
- callbackPulled = true;
- }
-
- // Execute Callback
- // ================
- // 1. Execute user callback method
- // 2. Handle return value from callback
-
- // if callback has not requested the stream to stop
- if ( callbackPulled && !callbackStopped ) {
- // Execute user callback method
- callbackResult = callback( stream_.userBuffer[OUTPUT],
- stream_.userBuffer[INPUT],
- stream_.bufferSize,
- getStreamTime(),
- captureFlags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY ? RTAUDIO_INPUT_OVERFLOW : 0,
- stream_.callbackInfo.userData );
-
- // Handle return value from callback
- if ( callbackResult == 1 ) {
- // instantiate a thread to stop this thread
- HANDLE threadHandle = CreateThread( NULL, 0, stopWasapiThread, this, 0, NULL );
- if ( !threadHandle ) {
- errorType = RtAudioError::THREAD_ERROR;
- errorText_ = "RtApiWasapi::wasapiThread: Unable to instantiate stream stop thread.";
- goto Exit;
- }
- else if ( !CloseHandle( threadHandle ) ) {
- errorType = RtAudioError::THREAD_ERROR;
- errorText_ = "RtApiWasapi::wasapiThread: Unable to close stream stop thread handle.";
- goto Exit;
- }
-
- callbackStopped = true;
- }
- else if ( callbackResult == 2 ) {
- // instantiate a thread to stop this thread
- HANDLE threadHandle = CreateThread( NULL, 0, abortWasapiThread, this, 0, NULL );
- if ( !threadHandle ) {
- errorType = RtAudioError::THREAD_ERROR;
- errorText_ = "RtApiWasapi::wasapiThread: Unable to instantiate stream abort thread.";
- goto Exit;
- }
- else if ( !CloseHandle( threadHandle ) ) {
- errorType = RtAudioError::THREAD_ERROR;
- errorText_ = "RtApiWasapi::wasapiThread: Unable to close stream abort thread handle.";
- goto Exit;
- }
-
- callbackStopped = true;
- }
- }
- }
-
- // Callback Output
- // ===============
- // 1. Convert callback buffer to stream format
- // 2. Convert callback buffer to stream sample rate and channel count
- // 3. Push callback buffer into outputBuffer
-
- if ( renderAudioClient && callbackPulled ) {
- if ( stream_.doConvertBuffer[OUTPUT] ) {
- // Convert callback buffer to stream format
- convertBuffer( stream_.deviceBuffer,
- stream_.userBuffer[OUTPUT],
- stream_.convertInfo[OUTPUT] );
-
- }
-
- // Convert callback buffer to stream sample rate
- convertBufferWasapi( convBuffer,
- stream_.deviceBuffer,
- stream_.nDeviceChannels[OUTPUT],
- stream_.sampleRate,
- renderFormat->nSamplesPerSec,
- stream_.bufferSize,
- convBufferSize,
- stream_.deviceFormat[OUTPUT] );
-
- // Push callback buffer into outputBuffer
- callbackPushed = renderBuffer.pushBuffer( convBuffer,
- convBufferSize * stream_.nDeviceChannels[OUTPUT],
- stream_.deviceFormat[OUTPUT] );
- }
- else {
- // if there is no render stream, set callbackPushed flag
- callbackPushed = true;
- }
-
- // Stream Capture
- // ==============
- // 1. Get capture buffer from stream
- // 2. Push capture buffer into inputBuffer
- // 3. If 2. was successful: Release capture buffer
-
- if ( captureAudioClient ) {
- // if the callback input buffer was not pulled from captureBuffer, wait for next capture event
- if ( !callbackPulled ) {
- WaitForSingleObject( captureEvent, INFINITE );
- }
-
- // Get capture buffer from stream
- hr = captureClient->GetBuffer( &streamBuffer,
- &bufferFrameCount,
- &captureFlags, NULL, NULL );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve capture buffer.";
- goto Exit;
- }
-
- if ( bufferFrameCount != 0 ) {
- // Push capture buffer into inputBuffer
- if ( captureBuffer.pushBuffer( ( char* ) streamBuffer,
- bufferFrameCount * stream_.nDeviceChannels[INPUT],
- stream_.deviceFormat[INPUT] ) )
- {
- // Release capture buffer
- hr = captureClient->ReleaseBuffer( bufferFrameCount );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to release capture buffer.";
- goto Exit;
- }
- }
- else
- {
- // Inform WASAPI that capture was unsuccessful
- hr = captureClient->ReleaseBuffer( 0 );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to release capture buffer.";
- goto Exit;
- }
- }
- }
- else
- {
- // Inform WASAPI that capture was unsuccessful
- hr = captureClient->ReleaseBuffer( 0 );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to release capture buffer.";
- goto Exit;
- }
- }
- }
-
- // Stream Render
- // =============
- // 1. Get render buffer from stream
- // 2. Pull next buffer from outputBuffer
- // 3. If 2. was successful: Fill render buffer with next buffer
- // Release render buffer
-
- if ( renderAudioClient ) {
- // if the callback output buffer was not pushed to renderBuffer, wait for next render event
- if ( callbackPulled && !callbackPushed ) {
- WaitForSingleObject( renderEvent, INFINITE );
- }
-
- // Get render buffer from stream
- hr = renderAudioClient->GetBufferSize( &bufferFrameCount );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve render buffer size.";
- goto Exit;
- }
-
- hr = renderAudioClient->GetCurrentPadding( &numFramesPadding );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve render buffer padding.";
- goto Exit;
- }
-
- bufferFrameCount -= numFramesPadding;
-
- if ( bufferFrameCount != 0 ) {
- hr = renderClient->GetBuffer( bufferFrameCount, &streamBuffer );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve render buffer.";
- goto Exit;
- }
-
- // Pull next buffer from outputBuffer
- // Fill render buffer with next buffer
- if ( renderBuffer.pullBuffer( ( char* ) streamBuffer,
- bufferFrameCount * stream_.nDeviceChannels[OUTPUT],
- stream_.deviceFormat[OUTPUT] ) )
- {
- // Release render buffer
- hr = renderClient->ReleaseBuffer( bufferFrameCount, 0 );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to release render buffer.";
- goto Exit;
- }
- }
- else
- {
- // Inform WASAPI that render was unsuccessful
- hr = renderClient->ReleaseBuffer( 0, 0 );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to release render buffer.";
- goto Exit;
- }
- }
- }
- else
- {
- // Inform WASAPI that render was unsuccessful
- hr = renderClient->ReleaseBuffer( 0, 0 );
- if ( FAILED( hr ) ) {
- errorText_ = "RtApiWasapi::wasapiThread: Unable to release render buffer.";
- goto Exit;
- }
- }
- }
-
- // if the callback buffer was pushed renderBuffer reset callbackPulled flag
- if ( callbackPushed ) {
- callbackPulled = false;
- }
-
- // tick stream time
- RtApi::tickStreamTime();
- }
-
-Exit:
- // clean up
- CoTaskMemFree( captureFormat );
- CoTaskMemFree( renderFormat );
-
- free ( convBuffer );
-
- CoUninitialize();
-
- // update stream state
- stream_.state = STREAM_STOPPED;
-
- if ( errorText_.empty() )
- return;
- else
- error( errorType );
-}
-
-//******************** End of __WINDOWS_WASAPI__ *********************//
-#endif
-
-
-#if defined(__WINDOWS_DS__) // Windows DirectSound API
-
-// Modified by Robin Davies, October 2005
-// - Improvements to DirectX pointer chasing.
-// - Bug fix for non-power-of-two Asio granularity used by Edirol PCR-A30.
-// - Auto-call CoInitialize for DSOUND and ASIO platforms.
-// Various revisions for RtAudio 4.0 by Gary Scavone, April 2007
-// Changed device query structure for RtAudio 4.0.7, January 2010
-
-#include <dsound.h>
-#include <assert.h>
-#include <algorithm>
-
-#if defined(__MINGW32__)
- // missing from latest mingw winapi
-#define WAVE_FORMAT_96M08 0x00010000 /* 96 kHz, Mono, 8-bit */
-#define WAVE_FORMAT_96S08 0x00020000 /* 96 kHz, Stereo, 8-bit */
-#define WAVE_FORMAT_96M16 0x00040000 /* 96 kHz, Mono, 16-bit */
-#define WAVE_FORMAT_96S16 0x00080000 /* 96 kHz, Stereo, 16-bit */
-#endif
-
-#define MINIMUM_DEVICE_BUFFER_SIZE 32768
-
-#ifdef _MSC_VER // if Microsoft Visual C++
-#pragma comment( lib, "winmm.lib" ) // then, auto-link winmm.lib. Otherwise, it has to be added manually.
-#endif
-
-static inline DWORD dsPointerBetween( DWORD pointer, DWORD laterPointer, DWORD earlierPointer, DWORD bufferSize )
-{
- if ( pointer > bufferSize ) pointer -= bufferSize;
- if ( laterPointer < earlierPointer ) laterPointer += bufferSize;
- if ( pointer < earlierPointer ) pointer += bufferSize;
- return pointer >= earlierPointer && pointer < laterPointer;
-}
-
-// A structure to hold various information related to the DirectSound
-// API implementation.
-struct DsHandle {
- unsigned int drainCounter; // Tracks callback counts when draining
- bool internalDrain; // Indicates if stop is initiated from callback or not.
- void *id[2];
- void *buffer[2];
- bool xrun[2];
- UINT bufferPointer[2];
- DWORD dsBufferSize[2];
- DWORD dsPointerLeadTime[2]; // the number of bytes ahead of the safe pointer to lead by.
- HANDLE condition;
-
- DsHandle()
- :drainCounter(0), internalDrain(false) { id[0] = 0; id[1] = 0; buffer[0] = 0; buffer[1] = 0; xrun[0] = false; xrun[1] = false; bufferPointer[0] = 0; bufferPointer[1] = 0; }
-};
-
-// Declarations for utility functions, callbacks, and structures
-// specific to the DirectSound implementation.
-static BOOL CALLBACK deviceQueryCallback( LPGUID lpguid,
- LPCTSTR description,
- LPCTSTR module,
- LPVOID lpContext );
-
-static const char* getErrorString( int code );
-
-static unsigned __stdcall callbackHandler( void *ptr );
-
-struct DsDevice {
- LPGUID id[2];
- bool validId[2];
- bool found;
- std::string name;
-
- DsDevice()
- : found(false) { validId[0] = false; validId[1] = false; }
-};
-
-struct DsProbeData {
- bool isInput;
- std::vector<struct DsDevice>* dsDevices;
-};
-
-RtApiDs :: RtApiDs()
-{
- // Dsound will run both-threaded. If CoInitialize fails, then just
- // accept whatever the mainline chose for a threading model.
- coInitialized_ = false;
- HRESULT hr = CoInitialize( NULL );
- if ( !FAILED( hr ) ) coInitialized_ = true;
-}
-
-RtApiDs :: ~RtApiDs()
-{
- if ( coInitialized_ ) CoUninitialize(); // balanced call.
- if ( stream_.state != STREAM_CLOSED ) closeStream();
-}
-
-// The DirectSound default output is always the first device.
-unsigned int RtApiDs :: getDefaultOutputDevice( void )
-{
- return 0;
-}
-
-// The DirectSound default input is always the first input device,
-// which is the first capture device enumerated.
-unsigned int RtApiDs :: getDefaultInputDevice( void )
-{
- return 0;
-}
-
-unsigned int RtApiDs :: getDeviceCount( void )
-{
- // Set query flag for previously found devices to false, so that we
- // can check for any devices that have disappeared.
- for ( unsigned int i=0; i<dsDevices.size(); i++ )
- dsDevices[i].found = false;
-
- // Query DirectSound devices.
- struct DsProbeData probeInfo;
- probeInfo.isInput = false;
- probeInfo.dsDevices = &dsDevices;
- HRESULT result = DirectSoundEnumerate( (LPDSENUMCALLBACK) deviceQueryCallback, &probeInfo );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::getDeviceCount: error (" << getErrorString( result ) << ") enumerating output devices!";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- }
-
- // Query DirectSoundCapture devices.
- probeInfo.isInput = true;
- result = DirectSoundCaptureEnumerate( (LPDSENUMCALLBACK) deviceQueryCallback, &probeInfo );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::getDeviceCount: error (" << getErrorString( result ) << ") enumerating input devices!";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- }
-
- // Clean out any devices that may have disappeared (code update submitted by Eli Zehngut).
- for ( unsigned int i=0; i<dsDevices.size(); ) {
- if ( dsDevices[i].found == false ) dsDevices.erase( dsDevices.begin() + i );
- else i++;
- }
-
- return static_cast<unsigned int>(dsDevices.size());
-}
-
-RtAudio::DeviceInfo RtApiDs :: getDeviceInfo( unsigned int device )
-{
- RtAudio::DeviceInfo info;
- info.probed = false;
-
- if ( dsDevices.size() == 0 ) {
- // Force a query of all devices
- getDeviceCount();
- if ( dsDevices.size() == 0 ) {
- errorText_ = "RtApiDs::getDeviceInfo: no devices found!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
- }
-
- if ( device >= dsDevices.size() ) {
- errorText_ = "RtApiDs::getDeviceInfo: device ID is invalid!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- HRESULT result;
- if ( dsDevices[ device ].validId[0] == false ) goto probeInput;
-
- LPDIRECTSOUND output;
- DSCAPS outCaps;
- result = DirectSoundCreate( dsDevices[ device ].id[0], &output, NULL );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::getDeviceInfo: error (" << getErrorString( result ) << ") opening output device (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- goto probeInput;
- }
-
- outCaps.dwSize = sizeof( outCaps );
- result = output->GetCaps( &outCaps );
- if ( FAILED( result ) ) {
- output->Release();
- errorStream_ << "RtApiDs::getDeviceInfo: error (" << getErrorString( result ) << ") getting capabilities!";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- goto probeInput;
- }
-
- // Get output channel information.
- info.outputChannels = ( outCaps.dwFlags & DSCAPS_PRIMARYSTEREO ) ? 2 : 1;
-
- // Get sample rate information.
- info.sampleRates.clear();
- for ( unsigned int k=0; k<MAX_SAMPLE_RATES; k++ ) {
- if ( SAMPLE_RATES[k] >= (unsigned int) outCaps.dwMinSecondarySampleRate &&
- SAMPLE_RATES[k] <= (unsigned int) outCaps.dwMaxSecondarySampleRate ) {
- info.sampleRates.push_back( SAMPLE_RATES[k] );
-
- if ( !info.preferredSampleRate || ( SAMPLE_RATES[k] <= 48000 && SAMPLE_RATES[k] > info.preferredSampleRate ) )
- info.preferredSampleRate = SAMPLE_RATES[k];
- }
- }
-
- // Get format information.
- if ( outCaps.dwFlags & DSCAPS_PRIMARY16BIT ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( outCaps.dwFlags & DSCAPS_PRIMARY8BIT ) info.nativeFormats |= RTAUDIO_SINT8;
-
- output->Release();
-
- if ( getDefaultOutputDevice() == device )
- info.isDefaultOutput = true;
-
- if ( dsDevices[ device ].validId[1] == false ) {
- info.name = dsDevices[ device ].name;
- info.probed = true;
- return info;
- }
-
- probeInput:
-
- LPDIRECTSOUNDCAPTURE input;
- result = DirectSoundCaptureCreate( dsDevices[ device ].id[1], &input, NULL );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::getDeviceInfo: error (" << getErrorString( result ) << ") opening input device (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- DSCCAPS inCaps;
- inCaps.dwSize = sizeof( inCaps );
- result = input->GetCaps( &inCaps );
- if ( FAILED( result ) ) {
- input->Release();
- errorStream_ << "RtApiDs::getDeviceInfo: error (" << getErrorString( result ) << ") getting object capabilities (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Get input channel information.
- info.inputChannels = inCaps.dwChannels;
-
- // Get sample rate and format information.
- std::vector<unsigned int> rates;
- if ( inCaps.dwChannels >= 2 ) {
- if ( inCaps.dwFormats & WAVE_FORMAT_1S16 ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( inCaps.dwFormats & WAVE_FORMAT_2S16 ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( inCaps.dwFormats & WAVE_FORMAT_4S16 ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( inCaps.dwFormats & WAVE_FORMAT_96S16 ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( inCaps.dwFormats & WAVE_FORMAT_1S08 ) info.nativeFormats |= RTAUDIO_SINT8;
- if ( inCaps.dwFormats & WAVE_FORMAT_2S08 ) info.nativeFormats |= RTAUDIO_SINT8;
- if ( inCaps.dwFormats & WAVE_FORMAT_4S08 ) info.nativeFormats |= RTAUDIO_SINT8;
- if ( inCaps.dwFormats & WAVE_FORMAT_96S08 ) info.nativeFormats |= RTAUDIO_SINT8;
-
- if ( info.nativeFormats & RTAUDIO_SINT16 ) {
- if ( inCaps.dwFormats & WAVE_FORMAT_1S16 ) rates.push_back( 11025 );
- if ( inCaps.dwFormats & WAVE_FORMAT_2S16 ) rates.push_back( 22050 );
- if ( inCaps.dwFormats & WAVE_FORMAT_4S16 ) rates.push_back( 44100 );
- if ( inCaps.dwFormats & WAVE_FORMAT_96S16 ) rates.push_back( 96000 );
- }
- else if ( info.nativeFormats & RTAUDIO_SINT8 ) {
- if ( inCaps.dwFormats & WAVE_FORMAT_1S08 ) rates.push_back( 11025 );
- if ( inCaps.dwFormats & WAVE_FORMAT_2S08 ) rates.push_back( 22050 );
- if ( inCaps.dwFormats & WAVE_FORMAT_4S08 ) rates.push_back( 44100 );
- if ( inCaps.dwFormats & WAVE_FORMAT_96S08 ) rates.push_back( 96000 );
- }
- }
- else if ( inCaps.dwChannels == 1 ) {
- if ( inCaps.dwFormats & WAVE_FORMAT_1M16 ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( inCaps.dwFormats & WAVE_FORMAT_2M16 ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( inCaps.dwFormats & WAVE_FORMAT_4M16 ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( inCaps.dwFormats & WAVE_FORMAT_96M16 ) info.nativeFormats |= RTAUDIO_SINT16;
- if ( inCaps.dwFormats & WAVE_FORMAT_1M08 ) info.nativeFormats |= RTAUDIO_SINT8;
- if ( inCaps.dwFormats & WAVE_FORMAT_2M08 ) info.nativeFormats |= RTAUDIO_SINT8;
- if ( inCaps.dwFormats & WAVE_FORMAT_4M08 ) info.nativeFormats |= RTAUDIO_SINT8;
- if ( inCaps.dwFormats & WAVE_FORMAT_96M08 ) info.nativeFormats |= RTAUDIO_SINT8;
-
- if ( info.nativeFormats & RTAUDIO_SINT16 ) {
- if ( inCaps.dwFormats & WAVE_FORMAT_1M16 ) rates.push_back( 11025 );
- if ( inCaps.dwFormats & WAVE_FORMAT_2M16 ) rates.push_back( 22050 );
- if ( inCaps.dwFormats & WAVE_FORMAT_4M16 ) rates.push_back( 44100 );
- if ( inCaps.dwFormats & WAVE_FORMAT_96M16 ) rates.push_back( 96000 );
- }
- else if ( info.nativeFormats & RTAUDIO_SINT8 ) {
- if ( inCaps.dwFormats & WAVE_FORMAT_1M08 ) rates.push_back( 11025 );
- if ( inCaps.dwFormats & WAVE_FORMAT_2M08 ) rates.push_back( 22050 );
- if ( inCaps.dwFormats & WAVE_FORMAT_4M08 ) rates.push_back( 44100 );
- if ( inCaps.dwFormats & WAVE_FORMAT_96M08 ) rates.push_back( 96000 );
- }
- }
- else info.inputChannels = 0; // technically, this would be an error
-
- input->Release();
-
- if ( info.inputChannels == 0 ) return info;
-
- // Copy the supported rates to the info structure but avoid duplication.
- bool found;
- for ( unsigned int i=0; i<rates.size(); i++ ) {
- found = false;
- for ( unsigned int j=0; j<info.sampleRates.size(); j++ ) {
- if ( rates[i] == info.sampleRates[j] ) {
- found = true;
- break;
- }
- }
- if ( found == false ) info.sampleRates.push_back( rates[i] );
- }
- std::sort( info.sampleRates.begin(), info.sampleRates.end() );
-
- // If device opens for both playback and capture, we determine the channels.
- if ( info.outputChannels > 0 && info.inputChannels > 0 )
- info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels;
-
- if ( device == 0 ) info.isDefaultInput = true;
-
- // Copy name and return.
- info.name = dsDevices[ device ].name;
- info.probed = true;
- return info;
-}
-
-bool RtApiDs :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels,
- unsigned int firstChannel, unsigned int sampleRate,
- RtAudioFormat format, unsigned int *bufferSize,
- RtAudio::StreamOptions *options )
-{
- if ( channels + firstChannel > 2 ) {
- errorText_ = "RtApiDs::probeDeviceOpen: DirectSound does not support more than 2 channels per device.";
- return FAILURE;
- }
-
- size_t nDevices = dsDevices.size();
- if ( nDevices == 0 ) {
- // This should not happen because a check is made before this function is called.
- errorText_ = "RtApiDs::probeDeviceOpen: no devices found!";
- return FAILURE;
- }
-
- if ( device >= nDevices ) {
- // This should not happen because a check is made before this function is called.
- errorText_ = "RtApiDs::probeDeviceOpen: device ID is invalid!";
- return FAILURE;
- }
-
- if ( mode == OUTPUT ) {
- if ( dsDevices[ device ].validId[0] == false ) {
- errorStream_ << "RtApiDs::probeDeviceOpen: device (" << device << ") does not support output!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
- else { // mode == INPUT
- if ( dsDevices[ device ].validId[1] == false ) {
- errorStream_ << "RtApiDs::probeDeviceOpen: device (" << device << ") does not support input!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
-
- // According to a note in PortAudio, using GetDesktopWindow()
- // instead of GetForegroundWindow() is supposed to avoid problems
- // that occur when the application's window is not the foreground
- // window. Also, if the application window closes before the
- // DirectSound buffer, DirectSound can crash. In the past, I had
- // problems when using GetDesktopWindow() but it seems fine now
- // (January 2010). I'll leave it commented here.
- // HWND hWnd = GetForegroundWindow();
- HWND hWnd = GetDesktopWindow();
-
- // Check the numberOfBuffers parameter and limit the lowest value to
- // two. This is a judgement call and a value of two is probably too
- // low for capture, but it should work for playback.
- int nBuffers = 0;
- if ( options ) nBuffers = options->numberOfBuffers;
- if ( options && options->flags & RTAUDIO_MINIMIZE_LATENCY ) nBuffers = 2;
- if ( nBuffers < 2 ) nBuffers = 3;
-
- // Check the lower range of the user-specified buffer size and set
- // (arbitrarily) to a lower bound of 32.
- if ( *bufferSize < 32 ) *bufferSize = 32;
-
- // Create the wave format structure. The data format setting will
- // be determined later.
- WAVEFORMATEX waveFormat;
- ZeroMemory( &waveFormat, sizeof(WAVEFORMATEX) );
- waveFormat.wFormatTag = WAVE_FORMAT_PCM;
- waveFormat.nChannels = channels + firstChannel;
- waveFormat.nSamplesPerSec = (unsigned long) sampleRate;
-
- // Determine the device buffer size. By default, we'll use the value
- // defined above (32K), but we will grow it to make allowances for
- // very large software buffer sizes.
- DWORD dsBufferSize = MINIMUM_DEVICE_BUFFER_SIZE;
- DWORD dsPointerLeadTime = 0;
-
- void *ohandle = 0, *bhandle = 0;
- HRESULT result;
- if ( mode == OUTPUT ) {
-
- LPDIRECTSOUND output;
- result = DirectSoundCreate( dsDevices[ device ].id[0], &output, NULL );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") opening output device (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- DSCAPS outCaps;
- outCaps.dwSize = sizeof( outCaps );
- result = output->GetCaps( &outCaps );
- if ( FAILED( result ) ) {
- output->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") getting capabilities (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Check channel information.
- if ( channels + firstChannel == 2 && !( outCaps.dwFlags & DSCAPS_PRIMARYSTEREO ) ) {
- errorStream_ << "RtApiDs::getDeviceInfo: the output device (" << dsDevices[ device ].name << ") does not support stereo playback.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Check format information. Use 16-bit format unless not
- // supported or user requests 8-bit.
- if ( outCaps.dwFlags & DSCAPS_PRIMARY16BIT &&
- !( format == RTAUDIO_SINT8 && outCaps.dwFlags & DSCAPS_PRIMARY8BIT ) ) {
- waveFormat.wBitsPerSample = 16;
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- }
- else {
- waveFormat.wBitsPerSample = 8;
- stream_.deviceFormat[mode] = RTAUDIO_SINT8;
- }
- stream_.userFormat = format;
-
- // Update wave format structure and buffer information.
- waveFormat.nBlockAlign = waveFormat.nChannels * waveFormat.wBitsPerSample / 8;
- waveFormat.nAvgBytesPerSec = waveFormat.nSamplesPerSec * waveFormat.nBlockAlign;
- dsPointerLeadTime = nBuffers * (*bufferSize) * (waveFormat.wBitsPerSample / 8) * channels;
-
- // If the user wants an even bigger buffer, increase the device buffer size accordingly.
- while ( dsPointerLeadTime * 2U > dsBufferSize )
- dsBufferSize *= 2;
-
- // Set cooperative level to DSSCL_EXCLUSIVE ... sound stops when window focus changes.
- // result = output->SetCooperativeLevel( hWnd, DSSCL_EXCLUSIVE );
- // Set cooperative level to DSSCL_PRIORITY ... sound remains when window focus changes.
- result = output->SetCooperativeLevel( hWnd, DSSCL_PRIORITY );
- if ( FAILED( result ) ) {
- output->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") setting cooperative level (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Even though we will write to the secondary buffer, we need to
- // access the primary buffer to set the correct output format
- // (since the default is 8-bit, 22 kHz!). Setup the DS primary
- // buffer description.
- DSBUFFERDESC bufferDescription;
- ZeroMemory( &bufferDescription, sizeof( DSBUFFERDESC ) );
- bufferDescription.dwSize = sizeof( DSBUFFERDESC );
- bufferDescription.dwFlags = DSBCAPS_PRIMARYBUFFER;
-
- // Obtain the primary buffer
- LPDIRECTSOUNDBUFFER buffer;
- result = output->CreateSoundBuffer( &bufferDescription, &buffer, NULL );
- if ( FAILED( result ) ) {
- output->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") accessing primary buffer (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Set the primary DS buffer sound format.
- result = buffer->SetFormat( &waveFormat );
- if ( FAILED( result ) ) {
- output->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") setting primary buffer format (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Setup the secondary DS buffer description.
- ZeroMemory( &bufferDescription, sizeof( DSBUFFERDESC ) );
- bufferDescription.dwSize = sizeof( DSBUFFERDESC );
- bufferDescription.dwFlags = ( DSBCAPS_STICKYFOCUS |
- DSBCAPS_GLOBALFOCUS |
- DSBCAPS_GETCURRENTPOSITION2 |
- DSBCAPS_LOCHARDWARE ); // Force hardware mixing
- bufferDescription.dwBufferBytes = dsBufferSize;
- bufferDescription.lpwfxFormat = &waveFormat;
-
- // Try to create the secondary DS buffer. If that doesn't work,
- // try to use software mixing. Otherwise, there's a problem.
- result = output->CreateSoundBuffer( &bufferDescription, &buffer, NULL );
- if ( FAILED( result ) ) {
- bufferDescription.dwFlags = ( DSBCAPS_STICKYFOCUS |
- DSBCAPS_GLOBALFOCUS |
- DSBCAPS_GETCURRENTPOSITION2 |
- DSBCAPS_LOCSOFTWARE ); // Force software mixing
- result = output->CreateSoundBuffer( &bufferDescription, &buffer, NULL );
- if ( FAILED( result ) ) {
- output->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") creating secondary buffer (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
-
- // Get the buffer size ... might be different from what we specified.
- DSBCAPS dsbcaps;
- dsbcaps.dwSize = sizeof( DSBCAPS );
- result = buffer->GetCaps( &dsbcaps );
- if ( FAILED( result ) ) {
- output->Release();
- buffer->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") getting buffer settings (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- dsBufferSize = dsbcaps.dwBufferBytes;
-
- // Lock the DS buffer
- LPVOID audioPtr;
- DWORD dataLen;
- result = buffer->Lock( 0, dsBufferSize, &audioPtr, &dataLen, NULL, NULL, 0 );
- if ( FAILED( result ) ) {
- output->Release();
- buffer->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") locking buffer (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Zero the DS buffer
- ZeroMemory( audioPtr, dataLen );
-
- // Unlock the DS buffer
- result = buffer->Unlock( audioPtr, dataLen, NULL, 0 );
- if ( FAILED( result ) ) {
- output->Release();
- buffer->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") unlocking buffer (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- ohandle = (void *) output;
- bhandle = (void *) buffer;
- }
-
- if ( mode == INPUT ) {
-
- LPDIRECTSOUNDCAPTURE input;
- result = DirectSoundCaptureCreate( dsDevices[ device ].id[1], &input, NULL );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") opening input device (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- DSCCAPS inCaps;
- inCaps.dwSize = sizeof( inCaps );
- result = input->GetCaps( &inCaps );
- if ( FAILED( result ) ) {
- input->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") getting input capabilities (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Check channel information.
- if ( inCaps.dwChannels < channels + firstChannel ) {
- errorText_ = "RtApiDs::getDeviceInfo: the input device does not support requested input channels.";
- return FAILURE;
- }
-
- // Check format information. Use 16-bit format unless user
- // requests 8-bit.
- DWORD deviceFormats;
- if ( channels + firstChannel == 2 ) {
- deviceFormats = WAVE_FORMAT_1S08 | WAVE_FORMAT_2S08 | WAVE_FORMAT_4S08 | WAVE_FORMAT_96S08;
- if ( format == RTAUDIO_SINT8 && inCaps.dwFormats & deviceFormats ) {
- waveFormat.wBitsPerSample = 8;
- stream_.deviceFormat[mode] = RTAUDIO_SINT8;
- }
- else { // assume 16-bit is supported
- waveFormat.wBitsPerSample = 16;
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- }
- }
- else { // channel == 1
- deviceFormats = WAVE_FORMAT_1M08 | WAVE_FORMAT_2M08 | WAVE_FORMAT_4M08 | WAVE_FORMAT_96M08;
- if ( format == RTAUDIO_SINT8 && inCaps.dwFormats & deviceFormats ) {
- waveFormat.wBitsPerSample = 8;
- stream_.deviceFormat[mode] = RTAUDIO_SINT8;
- }
- else { // assume 16-bit is supported
- waveFormat.wBitsPerSample = 16;
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- }
- }
- stream_.userFormat = format;
-
- // Update wave format structure and buffer information.
- waveFormat.nBlockAlign = waveFormat.nChannels * waveFormat.wBitsPerSample / 8;
- waveFormat.nAvgBytesPerSec = waveFormat.nSamplesPerSec * waveFormat.nBlockAlign;
- dsPointerLeadTime = nBuffers * (*bufferSize) * (waveFormat.wBitsPerSample / 8) * channels;
-
- // If the user wants an even bigger buffer, increase the device buffer size accordingly.
- while ( dsPointerLeadTime * 2U > dsBufferSize )
- dsBufferSize *= 2;
-
- // Setup the secondary DS buffer description.
- DSCBUFFERDESC bufferDescription;
- ZeroMemory( &bufferDescription, sizeof( DSCBUFFERDESC ) );
- bufferDescription.dwSize = sizeof( DSCBUFFERDESC );
- bufferDescription.dwFlags = 0;
- bufferDescription.dwReserved = 0;
- bufferDescription.dwBufferBytes = dsBufferSize;
- bufferDescription.lpwfxFormat = &waveFormat;
-
- // Create the capture buffer.
- LPDIRECTSOUNDCAPTUREBUFFER buffer;
- result = input->CreateCaptureBuffer( &bufferDescription, &buffer, NULL );
- if ( FAILED( result ) ) {
- input->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") creating input buffer (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Get the buffer size ... might be different from what we specified.
- DSCBCAPS dscbcaps;
- dscbcaps.dwSize = sizeof( DSCBCAPS );
- result = buffer->GetCaps( &dscbcaps );
- if ( FAILED( result ) ) {
- input->Release();
- buffer->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") getting buffer settings (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- dsBufferSize = dscbcaps.dwBufferBytes;
-
- // NOTE: We could have a problem here if this is a duplex stream
- // and the play and capture hardware buffer sizes are different
- // (I'm actually not sure if that is a problem or not).
- // Currently, we are not verifying that.
-
- // Lock the capture buffer
- LPVOID audioPtr;
- DWORD dataLen;
- result = buffer->Lock( 0, dsBufferSize, &audioPtr, &dataLen, NULL, NULL, 0 );
- if ( FAILED( result ) ) {
- input->Release();
- buffer->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") locking input buffer (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Zero the buffer
- ZeroMemory( audioPtr, dataLen );
-
- // Unlock the buffer
- result = buffer->Unlock( audioPtr, dataLen, NULL, 0 );
- if ( FAILED( result ) ) {
- input->Release();
- buffer->Release();
- errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") unlocking input buffer (" << dsDevices[ device ].name << ")!";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- ohandle = (void *) input;
- bhandle = (void *) buffer;
- }
-
- // Set various stream parameters
- DsHandle *handle = 0;
- stream_.nDeviceChannels[mode] = channels + firstChannel;
- stream_.nUserChannels[mode] = channels;
- stream_.bufferSize = *bufferSize;
- stream_.channelOffset[mode] = firstChannel;
- stream_.deviceInterleaved[mode] = true;
- if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false;
- else stream_.userInterleaved = true;
-
- // Set flag for buffer conversion
- stream_.doConvertBuffer[mode] = false;
- if (stream_.nUserChannels[mode] != stream_.nDeviceChannels[mode])
- stream_.doConvertBuffer[mode] = true;
- if (stream_.userFormat != stream_.deviceFormat[mode])
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] &&
- stream_.nUserChannels[mode] > 1 )
- stream_.doConvertBuffer[mode] = true;
-
- // Allocate necessary internal buffers
- long bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat );
- stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 );
- if ( stream_.userBuffer[mode] == NULL ) {
- errorText_ = "RtApiDs::probeDeviceOpen: error allocating user buffer memory.";
- goto error;
- }
-
- if ( stream_.doConvertBuffer[mode] ) {
-
- bool makeBuffer = true;
- bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] );
- if ( mode == INPUT ) {
- if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) {
- unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] );
- if ( bufferBytes <= (long) bytesOut ) makeBuffer = false;
- }
- }
-
- if ( makeBuffer ) {
- bufferBytes *= *bufferSize;
- if ( stream_.deviceBuffer ) free( stream_.deviceBuffer );
- stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 );
- if ( stream_.deviceBuffer == NULL ) {
- errorText_ = "RtApiDs::probeDeviceOpen: error allocating device buffer memory.";
- goto error;
- }
- }
- }
-
- // Allocate our DsHandle structures for the stream.
- if ( stream_.apiHandle == 0 ) {
- try {
- handle = new DsHandle;
- }
- catch ( std::bad_alloc& ) {
- errorText_ = "RtApiDs::probeDeviceOpen: error allocating AsioHandle memory.";
- goto error;
- }
-
- // Create a manual-reset event.
- handle->condition = CreateEvent( NULL, // no security
- TRUE, // manual-reset
- FALSE, // non-signaled initially
- NULL ); // unnamed
- stream_.apiHandle = (void *) handle;
- }
- else
- handle = (DsHandle *) stream_.apiHandle;
- handle->id[mode] = ohandle;
- handle->buffer[mode] = bhandle;
- handle->dsBufferSize[mode] = dsBufferSize;
- handle->dsPointerLeadTime[mode] = dsPointerLeadTime;
-
- stream_.device[mode] = device;
- stream_.state = STREAM_STOPPED;
- if ( stream_.mode == OUTPUT && mode == INPUT )
- // We had already set up an output stream.
- stream_.mode = DUPLEX;
- else
- stream_.mode = mode;
- stream_.nBuffers = nBuffers;
- stream_.sampleRate = sampleRate;
-
- // Setup the buffer conversion information structure.
- if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, firstChannel );
-
- // Setup the callback thread.
- if ( stream_.callbackInfo.isRunning == false ) {
- unsigned threadId;
- stream_.callbackInfo.isRunning = true;
- stream_.callbackInfo.object = (void *) this;
- stream_.callbackInfo.thread = _beginthreadex( NULL, 0, &callbackHandler,
- &stream_.callbackInfo, 0, &threadId );
- if ( stream_.callbackInfo.thread == 0 ) {
- errorText_ = "RtApiDs::probeDeviceOpen: error creating callback thread!";
- goto error;
- }
-
- // Boost DS thread priority
- SetThreadPriority( (HANDLE) stream_.callbackInfo.thread, THREAD_PRIORITY_HIGHEST );
- }
- return SUCCESS;
-
- error:
- if ( handle ) {
- if ( handle->buffer[0] ) { // the object pointer can be NULL and valid
- LPDIRECTSOUND object = (LPDIRECTSOUND) handle->id[0];
- LPDIRECTSOUNDBUFFER buffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0];
- if ( buffer ) buffer->Release();
- object->Release();
- }
- if ( handle->buffer[1] ) {
- LPDIRECTSOUNDCAPTURE object = (LPDIRECTSOUNDCAPTURE) handle->id[1];
- LPDIRECTSOUNDCAPTUREBUFFER buffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1];
- if ( buffer ) buffer->Release();
- object->Release();
- }
- CloseHandle( handle->condition );
- delete handle;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- stream_.state = STREAM_CLOSED;
- return FAILURE;
-}
-
-void RtApiDs :: closeStream()
-{
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiDs::closeStream(): no open stream to close!";
- error( RtAudioError::WARNING );
- return;
- }
-
- // Stop the callback thread.
- stream_.callbackInfo.isRunning = false;
- WaitForSingleObject( (HANDLE) stream_.callbackInfo.thread, INFINITE );
- CloseHandle( (HANDLE) stream_.callbackInfo.thread );
-
- DsHandle *handle = (DsHandle *) stream_.apiHandle;
- if ( handle ) {
- if ( handle->buffer[0] ) { // the object pointer can be NULL and valid
- LPDIRECTSOUND object = (LPDIRECTSOUND) handle->id[0];
- LPDIRECTSOUNDBUFFER buffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0];
- if ( buffer ) {
- buffer->Stop();
- buffer->Release();
- }
- object->Release();
- }
- if ( handle->buffer[1] ) {
- LPDIRECTSOUNDCAPTURE object = (LPDIRECTSOUNDCAPTURE) handle->id[1];
- LPDIRECTSOUNDCAPTUREBUFFER buffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1];
- if ( buffer ) {
- buffer->Stop();
- buffer->Release();
- }
- object->Release();
- }
- CloseHandle( handle->condition );
- delete handle;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- stream_.mode = UNINITIALIZED;
- stream_.state = STREAM_CLOSED;
-}
-
-void RtApiDs :: startStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_RUNNING ) {
- errorText_ = "RtApiDs::startStream(): the stream is already running!";
- error( RtAudioError::WARNING );
- return;
- }
-
- DsHandle *handle = (DsHandle *) stream_.apiHandle;
-
- // Increase scheduler frequency on lesser windows (a side-effect of
- // increasing timer accuracy). On greater windows (Win2K or later),
- // this is already in effect.
- timeBeginPeriod( 1 );
-
- buffersRolling = false;
- duplexPrerollBytes = 0;
-
- if ( stream_.mode == DUPLEX ) {
- // 0.5 seconds of silence in DUPLEX mode while the devices spin up and synchronize.
- duplexPrerollBytes = (int) ( 0.5 * stream_.sampleRate * formatBytes( stream_.deviceFormat[1] ) * stream_.nDeviceChannels[1] );
- }
-
- HRESULT result = 0;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- LPDIRECTSOUNDBUFFER buffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0];
- result = buffer->Play( 0, 0, DSBPLAY_LOOPING );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::startStream: error (" << getErrorString( result ) << ") starting output buffer!";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) {
-
- LPDIRECTSOUNDCAPTUREBUFFER buffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1];
- result = buffer->Start( DSCBSTART_LOOPING );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::startStream: error (" << getErrorString( result ) << ") starting input buffer!";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- handle->drainCounter = 0;
- handle->internalDrain = false;
- ResetEvent( handle->condition );
- stream_.state = STREAM_RUNNING;
-
- unlock:
- if ( FAILED( result ) ) error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiDs :: stopStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiDs::stopStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- HRESULT result = 0;
- LPVOID audioPtr;
- DWORD dataLen;
- DsHandle *handle = (DsHandle *) stream_.apiHandle;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- if ( handle->drainCounter == 0 ) {
- handle->drainCounter = 2;
- WaitForSingleObject( handle->condition, INFINITE ); // block until signaled
- }
-
- stream_.state = STREAM_STOPPED;
-
- MUTEX_LOCK( &stream_.mutex );
-
- // Stop the buffer and clear memory
- LPDIRECTSOUNDBUFFER buffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0];
- result = buffer->Stop();
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") stopping output buffer!";
- errorText_ = errorStream_.str();
- goto unlock;
- }
-
- // Lock the buffer and clear it so that if we start to play again,
- // we won't have old data playing.
- result = buffer->Lock( 0, handle->dsBufferSize[0], &audioPtr, &dataLen, NULL, NULL, 0 );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") locking output buffer!";
- errorText_ = errorStream_.str();
- goto unlock;
- }
-
- // Zero the DS buffer
- ZeroMemory( audioPtr, dataLen );
-
- // Unlock the DS buffer
- result = buffer->Unlock( audioPtr, dataLen, NULL, 0 );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") unlocking output buffer!";
- errorText_ = errorStream_.str();
- goto unlock;
- }
-
- // If we start playing again, we must begin at beginning of buffer.
- handle->bufferPointer[0] = 0;
- }
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) {
- LPDIRECTSOUNDCAPTUREBUFFER buffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1];
- audioPtr = NULL;
- dataLen = 0;
-
- stream_.state = STREAM_STOPPED;
-
- if ( stream_.mode != DUPLEX )
- MUTEX_LOCK( &stream_.mutex );
-
- result = buffer->Stop();
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") stopping input buffer!";
- errorText_ = errorStream_.str();
- goto unlock;
- }
-
- // Lock the buffer and clear it so that if we start to play again,
- // we won't have old data playing.
- result = buffer->Lock( 0, handle->dsBufferSize[1], &audioPtr, &dataLen, NULL, NULL, 0 );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") locking input buffer!";
- errorText_ = errorStream_.str();
- goto unlock;
- }
-
- // Zero the DS buffer
- ZeroMemory( audioPtr, dataLen );
-
- // Unlock the DS buffer
- result = buffer->Unlock( audioPtr, dataLen, NULL, 0 );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") unlocking input buffer!";
- errorText_ = errorStream_.str();
- goto unlock;
- }
-
- // If we start recording again, we must begin at beginning of buffer.
- handle->bufferPointer[1] = 0;
- }
-
- unlock:
- timeEndPeriod( 1 ); // revert to normal scheduler frequency on lesser windows.
- MUTEX_UNLOCK( &stream_.mutex );
-
- if ( FAILED( result ) ) error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiDs :: abortStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiDs::abortStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- DsHandle *handle = (DsHandle *) stream_.apiHandle;
- handle->drainCounter = 2;
-
- stopStream();
-}
-
-void RtApiDs :: callbackEvent()
-{
- if ( stream_.state == STREAM_STOPPED || stream_.state == STREAM_STOPPING ) {
- Sleep( 50 ); // sleep 50 milliseconds
- return;
- }
-
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiDs::callbackEvent(): the stream is closed ... this shouldn't happen!";
- error( RtAudioError::WARNING );
- return;
- }
-
- CallbackInfo *info = (CallbackInfo *) &stream_.callbackInfo;
- DsHandle *handle = (DsHandle *) stream_.apiHandle;
-
- // Check if we were draining the stream and signal is finished.
- if ( handle->drainCounter > stream_.nBuffers + 2 ) {
-
- stream_.state = STREAM_STOPPING;
- if ( handle->internalDrain == false )
- SetEvent( handle->condition );
- else
- stopStream();
- return;
- }
-
- // Invoke user callback to get fresh output data UNLESS we are
- // draining stream.
- if ( handle->drainCounter == 0 ) {
- RtAudioCallback callback = (RtAudioCallback) info->callback;
- double streamTime = getStreamTime();
- RtAudioStreamStatus status = 0;
- if ( stream_.mode != INPUT && handle->xrun[0] == true ) {
- status |= RTAUDIO_OUTPUT_UNDERFLOW;
- handle->xrun[0] = false;
- }
- if ( stream_.mode != OUTPUT && handle->xrun[1] == true ) {
- status |= RTAUDIO_INPUT_OVERFLOW;
- handle->xrun[1] = false;
- }
- int cbReturnValue = callback( stream_.userBuffer[0], stream_.userBuffer[1],
- stream_.bufferSize, streamTime, status, info->userData );
- if ( cbReturnValue == 2 ) {
- stream_.state = STREAM_STOPPING;
- handle->drainCounter = 2;
- abortStream();
- return;
- }
- else if ( cbReturnValue == 1 ) {
- handle->drainCounter = 1;
- handle->internalDrain = true;
- }
- }
-
- HRESULT result;
- DWORD currentWritePointer, safeWritePointer;
- DWORD currentReadPointer, safeReadPointer;
- UINT nextWritePointer;
-
- LPVOID buffer1 = NULL;
- LPVOID buffer2 = NULL;
- DWORD bufferSize1 = 0;
- DWORD bufferSize2 = 0;
-
- char *buffer;
- long bufferBytes;
-
- MUTEX_LOCK( &stream_.mutex );
- if ( stream_.state == STREAM_STOPPED ) {
- MUTEX_UNLOCK( &stream_.mutex );
- return;
- }
-
- if ( buffersRolling == false ) {
- if ( stream_.mode == DUPLEX ) {
- //assert( handle->dsBufferSize[0] == handle->dsBufferSize[1] );
-
- // It takes a while for the devices to get rolling. As a result,
- // there's no guarantee that the capture and write device pointers
- // will move in lockstep. Wait here for both devices to start
- // rolling, and then set our buffer pointers accordingly.
- // e.g. Crystal Drivers: the capture buffer starts up 5700 to 9600
- // bytes later than the write buffer.
-
- // Stub: a serious risk of having a pre-emptive scheduling round
- // take place between the two GetCurrentPosition calls... but I'm
- // really not sure how to solve the problem. Temporarily boost to
- // Realtime priority, maybe; but I'm not sure what priority the
- // DirectSound service threads run at. We *should* be roughly
- // within a ms or so of correct.
-
- LPDIRECTSOUNDBUFFER dsWriteBuffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0];
- LPDIRECTSOUNDCAPTUREBUFFER dsCaptureBuffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1];
-
- DWORD startSafeWritePointer, startSafeReadPointer;
-
- result = dsWriteBuffer->GetCurrentPosition( NULL, &startSafeWritePointer );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current write position!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- result = dsCaptureBuffer->GetCurrentPosition( NULL, &startSafeReadPointer );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current read position!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- while ( true ) {
- result = dsWriteBuffer->GetCurrentPosition( NULL, &safeWritePointer );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current write position!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- result = dsCaptureBuffer->GetCurrentPosition( NULL, &safeReadPointer );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current read position!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- if ( safeWritePointer != startSafeWritePointer && safeReadPointer != startSafeReadPointer ) break;
- Sleep( 1 );
- }
-
- //assert( handle->dsBufferSize[0] == handle->dsBufferSize[1] );
-
- handle->bufferPointer[0] = safeWritePointer + handle->dsPointerLeadTime[0];
- if ( handle->bufferPointer[0] >= handle->dsBufferSize[0] ) handle->bufferPointer[0] -= handle->dsBufferSize[0];
- handle->bufferPointer[1] = safeReadPointer;
- }
- else if ( stream_.mode == OUTPUT ) {
-
- // Set the proper nextWritePosition after initial startup.
- LPDIRECTSOUNDBUFFER dsWriteBuffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0];
- result = dsWriteBuffer->GetCurrentPosition( ¤tWritePointer, &safeWritePointer );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current write position!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- handle->bufferPointer[0] = safeWritePointer + handle->dsPointerLeadTime[0];
- if ( handle->bufferPointer[0] >= handle->dsBufferSize[0] ) handle->bufferPointer[0] -= handle->dsBufferSize[0];
- }
-
- buffersRolling = true;
- }
-
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- LPDIRECTSOUNDBUFFER dsBuffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0];
-
- if ( handle->drainCounter > 1 ) { // write zeros to the output stream
- bufferBytes = stream_.bufferSize * stream_.nUserChannels[0];
- bufferBytes *= formatBytes( stream_.userFormat );
- memset( stream_.userBuffer[0], 0, bufferBytes );
- }
-
- // Setup parameters and do buffer conversion if necessary.
- if ( stream_.doConvertBuffer[0] ) {
- buffer = stream_.deviceBuffer;
- convertBuffer( buffer, stream_.userBuffer[0], stream_.convertInfo[0] );
- bufferBytes = stream_.bufferSize * stream_.nDeviceChannels[0];
- bufferBytes *= formatBytes( stream_.deviceFormat[0] );
- }
- else {
- buffer = stream_.userBuffer[0];
- bufferBytes = stream_.bufferSize * stream_.nUserChannels[0];
- bufferBytes *= formatBytes( stream_.userFormat );
- }
-
- // No byte swapping necessary in DirectSound implementation.
-
- // Ahhh ... windoze. 16-bit data is signed but 8-bit data is
- // unsigned. So, we need to convert our signed 8-bit data here to
- // unsigned.
- if ( stream_.deviceFormat[0] == RTAUDIO_SINT8 )
- for ( int i=0; i<bufferBytes; i++ ) buffer[i] = (unsigned char) ( buffer[i] + 128 );
-
- DWORD dsBufferSize = handle->dsBufferSize[0];
- nextWritePointer = handle->bufferPointer[0];
-
- DWORD endWrite, leadPointer;
- while ( true ) {
- // Find out where the read and "safe write" pointers are.
- result = dsBuffer->GetCurrentPosition( ¤tWritePointer, &safeWritePointer );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current write position!";
- errorText_ = errorStream_.str();
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
-
- // We will copy our output buffer into the region between
- // safeWritePointer and leadPointer. If leadPointer is not
- // beyond the next endWrite position, wait until it is.
- leadPointer = safeWritePointer + handle->dsPointerLeadTime[0];
- //std::cout << "safeWritePointer = " << safeWritePointer << ", leadPointer = " << leadPointer << ", nextWritePointer = " << nextWritePointer << std::endl;
- if ( leadPointer > dsBufferSize ) leadPointer -= dsBufferSize;
- if ( leadPointer < nextWritePointer ) leadPointer += dsBufferSize; // unwrap offset
- endWrite = nextWritePointer + bufferBytes;
-
- // Check whether the entire write region is behind the play pointer.
- if ( leadPointer >= endWrite ) break;
-
- // If we are here, then we must wait until the leadPointer advances
- // beyond the end of our next write region. We use the
- // Sleep() function to suspend operation until that happens.
- double millis = ( endWrite - leadPointer ) * 1000.0;
- millis /= ( formatBytes( stream_.deviceFormat[0]) * stream_.nDeviceChannels[0] * stream_.sampleRate);
- if ( millis < 1.0 ) millis = 1.0;
- Sleep( (DWORD) millis );
- }
-
- if ( dsPointerBetween( nextWritePointer, safeWritePointer, currentWritePointer, dsBufferSize )
- || dsPointerBetween( endWrite, safeWritePointer, currentWritePointer, dsBufferSize ) ) {
- // We've strayed into the forbidden zone ... resync the read pointer.
- handle->xrun[0] = true;
- nextWritePointer = safeWritePointer + handle->dsPointerLeadTime[0] - bufferBytes;
- if ( nextWritePointer >= dsBufferSize ) nextWritePointer -= dsBufferSize;
- handle->bufferPointer[0] = nextWritePointer;
- endWrite = nextWritePointer + bufferBytes;
- }
-
- // Lock free space in the buffer
- result = dsBuffer->Lock( nextWritePointer, bufferBytes, &buffer1,
- &bufferSize1, &buffer2, &bufferSize2, 0 );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") locking buffer during playback!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
-
- // Copy our buffer into the DS buffer
- CopyMemory( buffer1, buffer, bufferSize1 );
- if ( buffer2 != NULL ) CopyMemory( buffer2, buffer+bufferSize1, bufferSize2 );
-
- // Update our buffer offset and unlock sound buffer
- dsBuffer->Unlock( buffer1, bufferSize1, buffer2, bufferSize2 );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") unlocking buffer during playback!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- nextWritePointer = ( nextWritePointer + bufferSize1 + bufferSize2 ) % dsBufferSize;
- handle->bufferPointer[0] = nextWritePointer;
- }
-
- // Don't bother draining input
- if ( handle->drainCounter ) {
- handle->drainCounter++;
- goto unlock;
- }
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) {
-
- // Setup parameters.
- if ( stream_.doConvertBuffer[1] ) {
- buffer = stream_.deviceBuffer;
- bufferBytes = stream_.bufferSize * stream_.nDeviceChannels[1];
- bufferBytes *= formatBytes( stream_.deviceFormat[1] );
- }
- else {
- buffer = stream_.userBuffer[1];
- bufferBytes = stream_.bufferSize * stream_.nUserChannels[1];
- bufferBytes *= formatBytes( stream_.userFormat );
- }
-
- LPDIRECTSOUNDCAPTUREBUFFER dsBuffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1];
- long nextReadPointer = handle->bufferPointer[1];
- DWORD dsBufferSize = handle->dsBufferSize[1];
-
- // Find out where the write and "safe read" pointers are.
- result = dsBuffer->GetCurrentPosition( ¤tReadPointer, &safeReadPointer );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current read position!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
-
- if ( safeReadPointer < (DWORD)nextReadPointer ) safeReadPointer += dsBufferSize; // unwrap offset
- DWORD endRead = nextReadPointer + bufferBytes;
-
- // Handling depends on whether we are INPUT or DUPLEX.
- // If we're in INPUT mode then waiting is a good thing. If we're in DUPLEX mode,
- // then a wait here will drag the write pointers into the forbidden zone.
- //
- // In DUPLEX mode, rather than wait, we will back off the read pointer until
- // it's in a safe position. This causes dropouts, but it seems to be the only
- // practical way to sync up the read and write pointers reliably, given the
- // the very complex relationship between phase and increment of the read and write
- // pointers.
- //
- // In order to minimize audible dropouts in DUPLEX mode, we will
- // provide a pre-roll period of 0.5 seconds in which we return
- // zeros from the read buffer while the pointers sync up.
-
- if ( stream_.mode == DUPLEX ) {
- if ( safeReadPointer < endRead ) {
- if ( duplexPrerollBytes <= 0 ) {
- // Pre-roll time over. Be more agressive.
- int adjustment = endRead-safeReadPointer;
-
- handle->xrun[1] = true;
- // Two cases:
- // - large adjustments: we've probably run out of CPU cycles, so just resync exactly,
- // and perform fine adjustments later.
- // - small adjustments: back off by twice as much.
- if ( adjustment >= 2*bufferBytes )
- nextReadPointer = safeReadPointer-2*bufferBytes;
- else
- nextReadPointer = safeReadPointer-bufferBytes-adjustment;
-
- if ( nextReadPointer < 0 ) nextReadPointer += dsBufferSize;
-
- }
- else {
- // In pre=roll time. Just do it.
- nextReadPointer = safeReadPointer - bufferBytes;
- while ( nextReadPointer < 0 ) nextReadPointer += dsBufferSize;
- }
- endRead = nextReadPointer + bufferBytes;
- }
- }
- else { // mode == INPUT
- while ( safeReadPointer < endRead && stream_.callbackInfo.isRunning ) {
- // See comments for playback.
- double millis = (endRead - safeReadPointer) * 1000.0;
- millis /= ( formatBytes(stream_.deviceFormat[1]) * stream_.nDeviceChannels[1] * stream_.sampleRate);
- if ( millis < 1.0 ) millis = 1.0;
- Sleep( (DWORD) millis );
-
- // Wake up and find out where we are now.
- result = dsBuffer->GetCurrentPosition( ¤tReadPointer, &safeReadPointer );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current read position!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
-
- if ( safeReadPointer < (DWORD)nextReadPointer ) safeReadPointer += dsBufferSize; // unwrap offset
- }
- }
-
- // Lock free space in the buffer
- result = dsBuffer->Lock( nextReadPointer, bufferBytes, &buffer1,
- &bufferSize1, &buffer2, &bufferSize2, 0 );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") locking capture buffer!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
-
- if ( duplexPrerollBytes <= 0 ) {
- // Copy our buffer into the DS buffer
- CopyMemory( buffer, buffer1, bufferSize1 );
- if ( buffer2 != NULL ) CopyMemory( buffer+bufferSize1, buffer2, bufferSize2 );
- }
- else {
- memset( buffer, 0, bufferSize1 );
- if ( buffer2 != NULL ) memset( buffer + bufferSize1, 0, bufferSize2 );
- duplexPrerollBytes -= bufferSize1 + bufferSize2;
- }
-
- // Update our buffer offset and unlock sound buffer
- nextReadPointer = ( nextReadPointer + bufferSize1 + bufferSize2 ) % dsBufferSize;
- dsBuffer->Unlock( buffer1, bufferSize1, buffer2, bufferSize2 );
- if ( FAILED( result ) ) {
- errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") unlocking capture buffer!";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- handle->bufferPointer[1] = nextReadPointer;
-
- // No byte swapping necessary in DirectSound implementation.
-
- // If necessary, convert 8-bit data from unsigned to signed.
- if ( stream_.deviceFormat[1] == RTAUDIO_SINT8 )
- for ( int j=0; j<bufferBytes; j++ ) buffer[j] = (signed char) ( buffer[j] - 128 );
-
- // Do buffer conversion if necessary.
- if ( stream_.doConvertBuffer[1] )
- convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] );
- }
-
- unlock:
- MUTEX_UNLOCK( &stream_.mutex );
- RtApi::tickStreamTime();
-}
-
-// Definitions for utility functions and callbacks
-// specific to the DirectSound implementation.
-
-static unsigned __stdcall callbackHandler( void *ptr )
-{
- CallbackInfo *info = (CallbackInfo *) ptr;
- RtApiDs *object = (RtApiDs *) info->object;
- bool* isRunning = &info->isRunning;
-
- while ( *isRunning == true ) {
- object->callbackEvent();
- }
-
- _endthreadex( 0 );
- return 0;
-}
-
-static BOOL CALLBACK deviceQueryCallback( LPGUID lpguid,
- LPCTSTR description,
- LPCTSTR /*module*/,
- LPVOID lpContext )
-{
- struct DsProbeData& probeInfo = *(struct DsProbeData*) lpContext;
- std::vector<struct DsDevice>& dsDevices = *probeInfo.dsDevices;
-
- HRESULT hr;
- bool validDevice = false;
- if ( probeInfo.isInput == true ) {
- DSCCAPS caps;
- LPDIRECTSOUNDCAPTURE object;
-
- hr = DirectSoundCaptureCreate( lpguid, &object, NULL );
- if ( hr != DS_OK ) return TRUE;
-
- caps.dwSize = sizeof(caps);
- hr = object->GetCaps( &caps );
- if ( hr == DS_OK ) {
- if ( caps.dwChannels > 0 && caps.dwFormats > 0 )
- validDevice = true;
- }
- object->Release();
- }
- else {
- DSCAPS caps;
- LPDIRECTSOUND object;
- hr = DirectSoundCreate( lpguid, &object, NULL );
- if ( hr != DS_OK ) return TRUE;
-
- caps.dwSize = sizeof(caps);
- hr = object->GetCaps( &caps );
- if ( hr == DS_OK ) {
- if ( caps.dwFlags & DSCAPS_PRIMARYMONO || caps.dwFlags & DSCAPS_PRIMARYSTEREO )
- validDevice = true;
- }
- object->Release();
- }
-
- // If good device, then save its name and guid.
- std::string name = convertCharPointerToStdString( description );
- //if ( name == "Primary Sound Driver" || name == "Primary Sound Capture Driver" )
- if ( lpguid == NULL )
- name = "Default Device";
- if ( validDevice ) {
- for ( unsigned int i=0; i<dsDevices.size(); i++ ) {
- if ( dsDevices[i].name == name ) {
- dsDevices[i].found = true;
- if ( probeInfo.isInput ) {
- dsDevices[i].id[1] = lpguid;
- dsDevices[i].validId[1] = true;
- }
- else {
- dsDevices[i].id[0] = lpguid;
- dsDevices[i].validId[0] = true;
- }
- return TRUE;
- }
- }
-
- DsDevice device;
- device.name = name;
- device.found = true;
- if ( probeInfo.isInput ) {
- device.id[1] = lpguid;
- device.validId[1] = true;
- }
- else {
- device.id[0] = lpguid;
- device.validId[0] = true;
- }
- dsDevices.push_back( device );
- }
-
- return TRUE;
-}
-
-static const char* getErrorString( int code )
-{
- switch ( code ) {
-
- case DSERR_ALLOCATED:
- return "Already allocated";
-
- case DSERR_CONTROLUNAVAIL:
- return "Control unavailable";
-
- case DSERR_INVALIDPARAM:
- return "Invalid parameter";
-
- case DSERR_INVALIDCALL:
- return "Invalid call";
-
- case DSERR_GENERIC:
- return "Generic error";
-
- case DSERR_PRIOLEVELNEEDED:
- return "Priority level needed";
-
- case DSERR_OUTOFMEMORY:
- return "Out of memory";
-
- case DSERR_BADFORMAT:
- return "The sample rate or the channel format is not supported";
-
- case DSERR_UNSUPPORTED:
- return "Not supported";
-
- case DSERR_NODRIVER:
- return "No driver";
-
- case DSERR_ALREADYINITIALIZED:
- return "Already initialized";
-
- case DSERR_NOAGGREGATION:
- return "No aggregation";
-
- case DSERR_BUFFERLOST:
- return "Buffer lost";
-
- case DSERR_OTHERAPPHASPRIO:
- return "Another application already has priority";
-
- case DSERR_UNINITIALIZED:
- return "Uninitialized";
-
- default:
- return "DirectSound unknown error";
- }
-}
-//******************** End of __WINDOWS_DS__ *********************//
-#endif
-
-
-#if defined(__LINUX_ALSA__)
-
-#include <alsa/asoundlib.h>
-#include <unistd.h>
-
- // A structure to hold various information related to the ALSA API
- // implementation.
-struct AlsaHandle {
- snd_pcm_t *handles[2];
- bool synchronized;
- bool xrun[2];
- pthread_cond_t runnable_cv;
- bool runnable;
-
- AlsaHandle()
- :synchronized(false), runnable(false) { xrun[0] = false; xrun[1] = false; }
-};
-
-static void *alsaCallbackHandler( void * ptr );
-
-RtApiAlsa :: RtApiAlsa()
-{
- // Nothing to do here.
-}
-
-RtApiAlsa :: ~RtApiAlsa()
-{
- if ( stream_.state != STREAM_CLOSED ) closeStream();
-}
-
-unsigned int RtApiAlsa :: getDeviceCount( void )
-{
- unsigned nDevices = 0;
- int result, subdevice, card;
- char name[64];
- snd_ctl_t *handle;
-
- // Count cards and devices
- card = -1;
- snd_card_next( &card );
- while ( card >= 0 ) {
- sprintf( name, "hw:%d", card );
- result = snd_ctl_open( &handle, name, 0 );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::getDeviceCount: control open, card = " << card << ", " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- goto nextcard;
- }
- subdevice = -1;
- while( 1 ) {
- result = snd_ctl_pcm_next_device( handle, &subdevice );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::getDeviceCount: control next device, card = " << card << ", " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- break;
- }
- if ( subdevice < 0 )
- break;
- nDevices++;
- }
- nextcard:
- snd_ctl_close( handle );
- snd_card_next( &card );
- }
-
- result = snd_ctl_open( &handle, "default", 0 );
- if (result == 0) {
- nDevices++;
- snd_ctl_close( handle );
- }
-
- return nDevices;
-}
-
-RtAudio::DeviceInfo RtApiAlsa :: getDeviceInfo( unsigned int device )
-{
- RtAudio::DeviceInfo info;
- info.probed = false;
-
- unsigned nDevices = 0;
- int result, subdevice, card;
- char name[64];
- snd_ctl_t *chandle;
-
- // Count cards and devices
- card = -1;
- subdevice = -1;
- snd_card_next( &card );
- while ( card >= 0 ) {
- sprintf( name, "hw:%d", card );
- result = snd_ctl_open( &chandle, name, SND_CTL_NONBLOCK );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::getDeviceInfo: control open, card = " << card << ", " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- goto nextcard;
- }
- subdevice = -1;
- while( 1 ) {
- result = snd_ctl_pcm_next_device( chandle, &subdevice );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::getDeviceInfo: control next device, card = " << card << ", " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- break;
- }
- if ( subdevice < 0 ) break;
- if ( nDevices == device ) {
- sprintf( name, "hw:%d,%d", card, subdevice );
- goto foundDevice;
- }
- nDevices++;
- }
- nextcard:
- snd_ctl_close( chandle );
- snd_card_next( &card );
- }
-
- result = snd_ctl_open( &chandle, "default", SND_CTL_NONBLOCK );
- if ( result == 0 ) {
- if ( nDevices == device ) {
- strcpy( name, "default" );
- goto foundDevice;
- }
- nDevices++;
- }
-
- if ( nDevices == 0 ) {
- errorText_ = "RtApiAlsa::getDeviceInfo: no devices found!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- if ( device >= nDevices ) {
- errorText_ = "RtApiAlsa::getDeviceInfo: device ID is invalid!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- foundDevice:
-
- // If a stream is already open, we cannot probe the stream devices.
- // Thus, use the saved results.
- if ( stream_.state != STREAM_CLOSED &&
- ( stream_.device[0] == device || stream_.device[1] == device ) ) {
- snd_ctl_close( chandle );
- if ( device >= devices_.size() ) {
- errorText_ = "RtApiAlsa::getDeviceInfo: device ID was not present before stream was opened.";
- error( RtAudioError::WARNING );
- return info;
- }
- return devices_[ device ];
- }
-
- int openMode = SND_PCM_ASYNC;
- snd_pcm_stream_t stream;
- snd_pcm_info_t *pcminfo;
- snd_pcm_info_alloca( &pcminfo );
- snd_pcm_t *phandle;
- snd_pcm_hw_params_t *params;
- snd_pcm_hw_params_alloca( ¶ms );
-
- // First try for playback unless default device (which has subdev -1)
- stream = SND_PCM_STREAM_PLAYBACK;
- snd_pcm_info_set_stream( pcminfo, stream );
- if ( subdevice != -1 ) {
- snd_pcm_info_set_device( pcminfo, subdevice );
- snd_pcm_info_set_subdevice( pcminfo, 0 );
-
- result = snd_ctl_pcm_info( chandle, pcminfo );
- if ( result < 0 ) {
- // Device probably doesn't support playback.
- goto captureProbe;
- }
- }
-
- result = snd_pcm_open( &phandle, name, stream, openMode | SND_PCM_NONBLOCK );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_open error for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- goto captureProbe;
- }
-
- // The device is open ... fill the parameter structure.
- result = snd_pcm_hw_params_any( phandle, params );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_hw_params error for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- goto captureProbe;
- }
-
- // Get output channel information.
- unsigned int value;
- result = snd_pcm_hw_params_get_channels_max( params, &value );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::getDeviceInfo: error getting device (" << name << ") output channels, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- goto captureProbe;
- }
- info.outputChannels = value;
- snd_pcm_close( phandle );
-
- captureProbe:
- stream = SND_PCM_STREAM_CAPTURE;
- snd_pcm_info_set_stream( pcminfo, stream );
-
- // Now try for capture unless default device (with subdev = -1)
- if ( subdevice != -1 ) {
- result = snd_ctl_pcm_info( chandle, pcminfo );
- snd_ctl_close( chandle );
- if ( result < 0 ) {
- // Device probably doesn't support capture.
- if ( info.outputChannels == 0 ) return info;
- goto probeParameters;
- }
- }
- else
- snd_ctl_close( chandle );
-
- result = snd_pcm_open( &phandle, name, stream, openMode | SND_PCM_NONBLOCK);
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_open error for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- if ( info.outputChannels == 0 ) return info;
- goto probeParameters;
- }
-
- // The device is open ... fill the parameter structure.
- result = snd_pcm_hw_params_any( phandle, params );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_hw_params error for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- if ( info.outputChannels == 0 ) return info;
- goto probeParameters;
- }
-
- result = snd_pcm_hw_params_get_channels_max( params, &value );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::getDeviceInfo: error getting device (" << name << ") input channels, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- if ( info.outputChannels == 0 ) return info;
- goto probeParameters;
- }
- info.inputChannels = value;
- snd_pcm_close( phandle );
-
- // If device opens for both playback and capture, we determine the channels.
- if ( info.outputChannels > 0 && info.inputChannels > 0 )
- info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels;
-
- // ALSA doesn't provide default devices so we'll use the first available one.
- if ( device == 0 && info.outputChannels > 0 )
- info.isDefaultOutput = true;
- if ( device == 0 && info.inputChannels > 0 )
- info.isDefaultInput = true;
-
- probeParameters:
- // At this point, we just need to figure out the supported data
- // formats and sample rates. We'll proceed by opening the device in
- // the direction with the maximum number of channels, or playback if
- // they are equal. This might limit our sample rate options, but so
- // be it.
-
- if ( info.outputChannels >= info.inputChannels )
- stream = SND_PCM_STREAM_PLAYBACK;
- else
- stream = SND_PCM_STREAM_CAPTURE;
- snd_pcm_info_set_stream( pcminfo, stream );
-
- result = snd_pcm_open( &phandle, name, stream, openMode | SND_PCM_NONBLOCK);
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_open error for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // The device is open ... fill the parameter structure.
- result = snd_pcm_hw_params_any( phandle, params );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_hw_params error for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Test our discrete set of sample rate values.
- info.sampleRates.clear();
- for ( unsigned int i=0; i<MAX_SAMPLE_RATES; i++ ) {
- if ( snd_pcm_hw_params_test_rate( phandle, params, SAMPLE_RATES[i], 0 ) == 0 ) {
- info.sampleRates.push_back( SAMPLE_RATES[i] );
-
- if ( !info.preferredSampleRate || ( SAMPLE_RATES[i] <= 48000 && SAMPLE_RATES[i] > info.preferredSampleRate ) )
- info.preferredSampleRate = SAMPLE_RATES[i];
- }
- }
- if ( info.sampleRates.size() == 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::getDeviceInfo: no supported sample rates found for device (" << name << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Probe the supported data formats ... we don't care about endian-ness just yet
- snd_pcm_format_t format;
- info.nativeFormats = 0;
- format = SND_PCM_FORMAT_S8;
- if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 )
- info.nativeFormats |= RTAUDIO_SINT8;
- format = SND_PCM_FORMAT_S16;
- if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 )
- info.nativeFormats |= RTAUDIO_SINT16;
- format = SND_PCM_FORMAT_S24;
- if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 )
- info.nativeFormats |= RTAUDIO_SINT24;
- format = SND_PCM_FORMAT_S32;
- if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 )
- info.nativeFormats |= RTAUDIO_SINT32;
- format = SND_PCM_FORMAT_FLOAT;
- if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 )
- info.nativeFormats |= RTAUDIO_FLOAT32;
- format = SND_PCM_FORMAT_FLOAT64;
- if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 )
- info.nativeFormats |= RTAUDIO_FLOAT64;
-
- // Check that we have at least one supported format
- if ( info.nativeFormats == 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::getDeviceInfo: pcm device (" << name << ") data format not supported by RtAudio.";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Get the device name
- char *cardname;
- result = snd_card_get_name( card, &cardname );
- if ( result >= 0 ) {
- sprintf( name, "hw:%s,%d", cardname, subdevice );
- free( cardname );
- }
- info.name = name;
-
- // That's all ... close the device and return
- snd_pcm_close( phandle );
- info.probed = true;
- return info;
-}
-
-void RtApiAlsa :: saveDeviceInfo( void )
-{
- devices_.clear();
-
- unsigned int nDevices = getDeviceCount();
- devices_.resize( nDevices );
- for ( unsigned int i=0; i<nDevices; i++ )
- devices_[i] = getDeviceInfo( i );
-}
-
-bool RtApiAlsa :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels,
- unsigned int firstChannel, unsigned int sampleRate,
- RtAudioFormat format, unsigned int *bufferSize,
- RtAudio::StreamOptions *options )
-
-{
-#if defined(__RTAUDIO_DEBUG__)
- snd_output_t *out;
- snd_output_stdio_attach(&out, stderr, 0);
-#endif
-
- // I'm not using the "plug" interface ... too much inconsistent behavior.
-
- unsigned nDevices = 0;
- int result, subdevice, card;
- char name[64];
- snd_ctl_t *chandle;
-
- if ( options && options->flags & RTAUDIO_ALSA_USE_DEFAULT )
- snprintf(name, sizeof(name), "%s", "default");
- else {
- // Count cards and devices
- card = -1;
- snd_card_next( &card );
- while ( card >= 0 ) {
- sprintf( name, "hw:%d", card );
- result = snd_ctl_open( &chandle, name, SND_CTL_NONBLOCK );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::probeDeviceOpen: control open, card = " << card << ", " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- subdevice = -1;
- while( 1 ) {
- result = snd_ctl_pcm_next_device( chandle, &subdevice );
- if ( result < 0 ) break;
- if ( subdevice < 0 ) break;
- if ( nDevices == device ) {
- sprintf( name, "hw:%d,%d", card, subdevice );
- snd_ctl_close( chandle );
- goto foundDevice;
- }
- nDevices++;
- }
- snd_ctl_close( chandle );
- snd_card_next( &card );
- }
-
- result = snd_ctl_open( &chandle, "default", SND_CTL_NONBLOCK );
- if ( result == 0 ) {
- if ( nDevices == device ) {
- strcpy( name, "default" );
- goto foundDevice;
- }
- nDevices++;
- }
-
- if ( nDevices == 0 ) {
- // This should not happen because a check is made before this function is called.
- errorText_ = "RtApiAlsa::probeDeviceOpen: no devices found!";
- return FAILURE;
- }
-
- if ( device >= nDevices ) {
- // This should not happen because a check is made before this function is called.
- errorText_ = "RtApiAlsa::probeDeviceOpen: device ID is invalid!";
- return FAILURE;
- }
- }
-
- foundDevice:
-
- // The getDeviceInfo() function will not work for a device that is
- // already open. Thus, we'll probe the system before opening a
- // stream and save the results for use by getDeviceInfo().
- if ( mode == OUTPUT || ( mode == INPUT && stream_.mode != OUTPUT ) ) // only do once
- this->saveDeviceInfo();
-
- snd_pcm_stream_t stream;
- if ( mode == OUTPUT )
- stream = SND_PCM_STREAM_PLAYBACK;
- else
- stream = SND_PCM_STREAM_CAPTURE;
-
- snd_pcm_t *phandle;
- int openMode = SND_PCM_ASYNC;
- result = snd_pcm_open( &phandle, name, stream, openMode );
- if ( result < 0 ) {
- if ( mode == OUTPUT )
- errorStream_ << "RtApiAlsa::probeDeviceOpen: pcm device (" << name << ") won't open for output.";
- else
- errorStream_ << "RtApiAlsa::probeDeviceOpen: pcm device (" << name << ") won't open for input.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Fill the parameter structure.
- snd_pcm_hw_params_t *hw_params;
- snd_pcm_hw_params_alloca( &hw_params );
- result = snd_pcm_hw_params_any( phandle, hw_params );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error getting pcm device (" << name << ") parameters, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
-#if defined(__RTAUDIO_DEBUG__)
- fprintf( stderr, "\nRtApiAlsa: dump hardware params just after device open:\n\n" );
- snd_pcm_hw_params_dump( hw_params, out );
-#endif
-
- // Set access ... check user preference.
- if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) {
- stream_.userInterleaved = false;
- result = snd_pcm_hw_params_set_access( phandle, hw_params, SND_PCM_ACCESS_RW_NONINTERLEAVED );
- if ( result < 0 ) {
- result = snd_pcm_hw_params_set_access( phandle, hw_params, SND_PCM_ACCESS_RW_INTERLEAVED );
- stream_.deviceInterleaved[mode] = true;
- }
- else
- stream_.deviceInterleaved[mode] = false;
- }
- else {
- stream_.userInterleaved = true;
- result = snd_pcm_hw_params_set_access( phandle, hw_params, SND_PCM_ACCESS_RW_INTERLEAVED );
- if ( result < 0 ) {
- result = snd_pcm_hw_params_set_access( phandle, hw_params, SND_PCM_ACCESS_RW_NONINTERLEAVED );
- stream_.deviceInterleaved[mode] = false;
- }
- else
- stream_.deviceInterleaved[mode] = true;
- }
-
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting pcm device (" << name << ") access, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Determine how to set the device format.
- stream_.userFormat = format;
- snd_pcm_format_t deviceFormat = SND_PCM_FORMAT_UNKNOWN;
-
- if ( format == RTAUDIO_SINT8 )
- deviceFormat = SND_PCM_FORMAT_S8;
- else if ( format == RTAUDIO_SINT16 )
- deviceFormat = SND_PCM_FORMAT_S16;
- else if ( format == RTAUDIO_SINT24 )
- deviceFormat = SND_PCM_FORMAT_S24;
- else if ( format == RTAUDIO_SINT32 )
- deviceFormat = SND_PCM_FORMAT_S32;
- else if ( format == RTAUDIO_FLOAT32 )
- deviceFormat = SND_PCM_FORMAT_FLOAT;
- else if ( format == RTAUDIO_FLOAT64 )
- deviceFormat = SND_PCM_FORMAT_FLOAT64;
-
- if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat) == 0) {
- stream_.deviceFormat[mode] = format;
- goto setFormat;
- }
-
- // The user requested format is not natively supported by the device.
- deviceFormat = SND_PCM_FORMAT_FLOAT64;
- if ( snd_pcm_hw_params_test_format( phandle, hw_params, deviceFormat ) == 0 ) {
- stream_.deviceFormat[mode] = RTAUDIO_FLOAT64;
- goto setFormat;
- }
-
- deviceFormat = SND_PCM_FORMAT_FLOAT;
- if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) {
- stream_.deviceFormat[mode] = RTAUDIO_FLOAT32;
- goto setFormat;
- }
-
- deviceFormat = SND_PCM_FORMAT_S32;
- if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) {
- stream_.deviceFormat[mode] = RTAUDIO_SINT32;
- goto setFormat;
- }
-
- deviceFormat = SND_PCM_FORMAT_S24;
- if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) {
- stream_.deviceFormat[mode] = RTAUDIO_SINT24;
- goto setFormat;
- }
-
- deviceFormat = SND_PCM_FORMAT_S16;
- if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) {
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- goto setFormat;
- }
-
- deviceFormat = SND_PCM_FORMAT_S8;
- if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) {
- stream_.deviceFormat[mode] = RTAUDIO_SINT8;
- goto setFormat;
- }
-
- // If we get here, no supported format was found.
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: pcm device " << device << " data format not supported by RtAudio.";
- errorText_ = errorStream_.str();
- return FAILURE;
-
- setFormat:
- result = snd_pcm_hw_params_set_format( phandle, hw_params, deviceFormat );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting pcm device (" << name << ") data format, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Determine whether byte-swaping is necessary.
- stream_.doByteSwap[mode] = false;
- if ( deviceFormat != SND_PCM_FORMAT_S8 ) {
- result = snd_pcm_format_cpu_endian( deviceFormat );
- if ( result == 0 )
- stream_.doByteSwap[mode] = true;
- else if (result < 0) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error getting pcm device (" << name << ") endian-ness, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
-
- // Set the sample rate.
- result = snd_pcm_hw_params_set_rate_near( phandle, hw_params, (unsigned int*) &sampleRate, 0 );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting sample rate on device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Determine the number of channels for this device. We support a possible
- // minimum device channel number > than the value requested by the user.
- stream_.nUserChannels[mode] = channels;
- unsigned int value;
- result = snd_pcm_hw_params_get_channels_max( hw_params, &value );
- unsigned int deviceChannels = value;
- if ( result < 0 || deviceChannels < channels + firstChannel ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: requested channel parameters not supported by device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- result = snd_pcm_hw_params_get_channels_min( hw_params, &value );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error getting minimum channels for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- deviceChannels = value;
- if ( deviceChannels < channels + firstChannel ) deviceChannels = channels + firstChannel;
- stream_.nDeviceChannels[mode] = deviceChannels;
-
- // Set the device channels.
- result = snd_pcm_hw_params_set_channels( phandle, hw_params, deviceChannels );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting channels for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Set the buffer (or period) size.
- int dir = 0;
- snd_pcm_uframes_t periodSize = *bufferSize;
- result = snd_pcm_hw_params_set_period_size_near( phandle, hw_params, &periodSize, &dir );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting period size for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- *bufferSize = periodSize;
-
- // Set the buffer number, which in ALSA is referred to as the "period".
- unsigned int periods = 0;
- if ( options && options->flags & RTAUDIO_MINIMIZE_LATENCY ) periods = 2;
- if ( options && options->numberOfBuffers > 0 ) periods = options->numberOfBuffers;
- if ( periods < 2 ) periods = 4; // a fairly safe default value
- result = snd_pcm_hw_params_set_periods_near( phandle, hw_params, &periods, &dir );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting periods for device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // If attempting to setup a duplex stream, the bufferSize parameter
- // MUST be the same in both directions!
- if ( stream_.mode == OUTPUT && mode == INPUT && *bufferSize != stream_.bufferSize ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: system error setting buffer size for duplex stream on device (" << name << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- stream_.bufferSize = *bufferSize;
-
- // Install the hardware configuration
- result = snd_pcm_hw_params( phandle, hw_params );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error installing hardware configuration on device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
-#if defined(__RTAUDIO_DEBUG__)
- fprintf(stderr, "\nRtApiAlsa: dump hardware params after installation:\n\n");
- snd_pcm_hw_params_dump( hw_params, out );
-#endif
-
- // Set the software configuration to fill buffers with zeros and prevent device stopping on xruns.
- snd_pcm_sw_params_t *sw_params = NULL;
- snd_pcm_sw_params_alloca( &sw_params );
- snd_pcm_sw_params_current( phandle, sw_params );
- snd_pcm_sw_params_set_start_threshold( phandle, sw_params, *bufferSize );
- snd_pcm_sw_params_set_stop_threshold( phandle, sw_params, ULONG_MAX );
- snd_pcm_sw_params_set_silence_threshold( phandle, sw_params, 0 );
-
- // The following two settings were suggested by Theo Veenker
- //snd_pcm_sw_params_set_avail_min( phandle, sw_params, *bufferSize );
- //snd_pcm_sw_params_set_xfer_align( phandle, sw_params, 1 );
-
- // here are two options for a fix
- //snd_pcm_sw_params_set_silence_size( phandle, sw_params, ULONG_MAX );
- snd_pcm_uframes_t val;
- snd_pcm_sw_params_get_boundary( sw_params, &val );
- snd_pcm_sw_params_set_silence_size( phandle, sw_params, val );
-
- result = snd_pcm_sw_params( phandle, sw_params );
- if ( result < 0 ) {
- snd_pcm_close( phandle );
- errorStream_ << "RtApiAlsa::probeDeviceOpen: error installing software configuration on device (" << name << "), " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
-#if defined(__RTAUDIO_DEBUG__)
- fprintf(stderr, "\nRtApiAlsa: dump software params after installation:\n\n");
- snd_pcm_sw_params_dump( sw_params, out );
-#endif
-
- // Set flags for buffer conversion
- stream_.doConvertBuffer[mode] = false;
- if ( stream_.userFormat != stream_.deviceFormat[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.nUserChannels[mode] < stream_.nDeviceChannels[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] &&
- stream_.nUserChannels[mode] > 1 )
- stream_.doConvertBuffer[mode] = true;
-
- // Allocate the ApiHandle if necessary and then save.
- AlsaHandle *apiInfo = 0;
- if ( stream_.apiHandle == 0 ) {
- try {
- apiInfo = (AlsaHandle *) new AlsaHandle;
- }
- catch ( std::bad_alloc& ) {
- errorText_ = "RtApiAlsa::probeDeviceOpen: error allocating AlsaHandle memory.";
- goto error;
- }
-
- if ( pthread_cond_init( &apiInfo->runnable_cv, NULL ) ) {
- errorText_ = "RtApiAlsa::probeDeviceOpen: error initializing pthread condition variable.";
- goto error;
- }
-
- stream_.apiHandle = (void *) apiInfo;
- apiInfo->handles[0] = 0;
- apiInfo->handles[1] = 0;
- }
- else {
- apiInfo = (AlsaHandle *) stream_.apiHandle;
- }
- apiInfo->handles[mode] = phandle;
- phandle = 0;
-
- // Allocate necessary internal buffers.
- unsigned long bufferBytes;
- bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat );
- stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 );
- if ( stream_.userBuffer[mode] == NULL ) {
- errorText_ = "RtApiAlsa::probeDeviceOpen: error allocating user buffer memory.";
- goto error;
- }
-
- if ( stream_.doConvertBuffer[mode] ) {
-
- bool makeBuffer = true;
- bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] );
- if ( mode == INPUT ) {
- if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) {
- unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] );
- if ( bufferBytes <= bytesOut ) makeBuffer = false;
- }
- }
-
- if ( makeBuffer ) {
- bufferBytes *= *bufferSize;
- if ( stream_.deviceBuffer ) free( stream_.deviceBuffer );
- stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 );
- if ( stream_.deviceBuffer == NULL ) {
- errorText_ = "RtApiAlsa::probeDeviceOpen: error allocating device buffer memory.";
- goto error;
- }
- }
- }
-
- stream_.sampleRate = sampleRate;
- stream_.nBuffers = periods;
- stream_.device[mode] = device;
- stream_.state = STREAM_STOPPED;
-
- // Setup the buffer conversion information structure.
- if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, firstChannel );
-
- // Setup thread if necessary.
- if ( stream_.mode == OUTPUT && mode == INPUT ) {
- // We had already set up an output stream.
- stream_.mode = DUPLEX;
- // Link the streams if possible.
- apiInfo->synchronized = false;
- if ( snd_pcm_link( apiInfo->handles[0], apiInfo->handles[1] ) == 0 )
- apiInfo->synchronized = true;
- else {
- errorText_ = "RtApiAlsa::probeDeviceOpen: unable to synchronize input and output devices.";
- error( RtAudioError::WARNING );
- }
- }
- else {
- stream_.mode = mode;
-
- // Setup callback thread.
- stream_.callbackInfo.object = (void *) this;
-
- // Set the thread attributes for joinable and realtime scheduling
- // priority (optional). The higher priority will only take affect
- // if the program is run as root or suid. Note, under Linux
- // processes with CAP_SYS_NICE privilege, a user can change
- // scheduling policy and priority (thus need not be root). See
- // POSIX "capabilities".
- pthread_attr_t attr;
- pthread_attr_init( &attr );
- pthread_attr_setdetachstate( &attr, PTHREAD_CREATE_JOINABLE );
-
-#ifdef SCHED_RR // Undefined with some OSes (eg: NetBSD 1.6.x with GNU Pthread)
- if ( options && options->flags & RTAUDIO_SCHEDULE_REALTIME ) {
- // We previously attempted to increase the audio callback priority
- // to SCHED_RR here via the attributes. However, while no errors
- // were reported in doing so, it did not work. So, now this is
- // done in the alsaCallbackHandler function.
- stream_.callbackInfo.doRealtime = true;
- int priority = options->priority;
- int min = sched_get_priority_min( SCHED_RR );
- int max = sched_get_priority_max( SCHED_RR );
- if ( priority < min ) priority = min;
- else if ( priority > max ) priority = max;
- stream_.callbackInfo.priority = priority;
- }
-#endif
-
- stream_.callbackInfo.isRunning = true;
- result = pthread_create( &stream_.callbackInfo.thread, &attr, alsaCallbackHandler, &stream_.callbackInfo );
- pthread_attr_destroy( &attr );
- if ( result ) {
- stream_.callbackInfo.isRunning = false;
- errorText_ = "RtApiAlsa::error creating callback thread!";
- goto error;
- }
- }
-
- return SUCCESS;
-
- error:
- if ( apiInfo ) {
- pthread_cond_destroy( &apiInfo->runnable_cv );
- if ( apiInfo->handles[0] ) snd_pcm_close( apiInfo->handles[0] );
- if ( apiInfo->handles[1] ) snd_pcm_close( apiInfo->handles[1] );
- delete apiInfo;
- stream_.apiHandle = 0;
- }
-
- if ( phandle) snd_pcm_close( phandle );
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- stream_.state = STREAM_CLOSED;
- return FAILURE;
-}
-
-void RtApiAlsa :: closeStream()
-{
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiAlsa::closeStream(): no open stream to close!";
- error( RtAudioError::WARNING );
- return;
- }
-
- AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle;
- stream_.callbackInfo.isRunning = false;
- MUTEX_LOCK( &stream_.mutex );
- if ( stream_.state == STREAM_STOPPED ) {
- apiInfo->runnable = true;
- pthread_cond_signal( &apiInfo->runnable_cv );
- }
- MUTEX_UNLOCK( &stream_.mutex );
- pthread_join( stream_.callbackInfo.thread, NULL );
-
- if ( stream_.state == STREAM_RUNNING ) {
- stream_.state = STREAM_STOPPED;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX )
- snd_pcm_drop( apiInfo->handles[0] );
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX )
- snd_pcm_drop( apiInfo->handles[1] );
- }
-
- if ( apiInfo ) {
- pthread_cond_destroy( &apiInfo->runnable_cv );
- if ( apiInfo->handles[0] ) snd_pcm_close( apiInfo->handles[0] );
- if ( apiInfo->handles[1] ) snd_pcm_close( apiInfo->handles[1] );
- delete apiInfo;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- stream_.mode = UNINITIALIZED;
- stream_.state = STREAM_CLOSED;
-}
-
-void RtApiAlsa :: startStream()
-{
- // This method calls snd_pcm_prepare if the device isn't already in that state.
-
- verifyStream();
- if ( stream_.state == STREAM_RUNNING ) {
- errorText_ = "RtApiAlsa::startStream(): the stream is already running!";
- error( RtAudioError::WARNING );
- return;
- }
-
- MUTEX_LOCK( &stream_.mutex );
-
- int result = 0;
- snd_pcm_state_t state;
- AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle;
- snd_pcm_t **handle = (snd_pcm_t **) apiInfo->handles;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- state = snd_pcm_state( handle[0] );
- if ( state != SND_PCM_STATE_PREPARED ) {
- result = snd_pcm_prepare( handle[0] );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::startStream: error preparing output pcm device, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
- }
-
- if ( ( stream_.mode == INPUT || stream_.mode == DUPLEX ) && !apiInfo->synchronized ) {
- result = snd_pcm_drop(handle[1]); // fix to remove stale data received since device has been open
- state = snd_pcm_state( handle[1] );
- if ( state != SND_PCM_STATE_PREPARED ) {
- result = snd_pcm_prepare( handle[1] );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::startStream: error preparing input pcm device, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
- }
-
- stream_.state = STREAM_RUNNING;
-
- unlock:
- apiInfo->runnable = true;
- pthread_cond_signal( &apiInfo->runnable_cv );
- MUTEX_UNLOCK( &stream_.mutex );
-
- if ( result >= 0 ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiAlsa :: stopStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiAlsa::stopStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- stream_.state = STREAM_STOPPED;
- MUTEX_LOCK( &stream_.mutex );
-
- int result = 0;
- AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle;
- snd_pcm_t **handle = (snd_pcm_t **) apiInfo->handles;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- if ( apiInfo->synchronized )
- result = snd_pcm_drop( handle[0] );
- else
- result = snd_pcm_drain( handle[0] );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::stopStream: error draining output pcm device, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- if ( ( stream_.mode == INPUT || stream_.mode == DUPLEX ) && !apiInfo->synchronized ) {
- result = snd_pcm_drop( handle[1] );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::stopStream: error stopping input pcm device, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- unlock:
- apiInfo->runnable = false; // fixes high CPU usage when stopped
- MUTEX_UNLOCK( &stream_.mutex );
-
- if ( result >= 0 ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiAlsa :: abortStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiAlsa::abortStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- stream_.state = STREAM_STOPPED;
- MUTEX_LOCK( &stream_.mutex );
-
- int result = 0;
- AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle;
- snd_pcm_t **handle = (snd_pcm_t **) apiInfo->handles;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- result = snd_pcm_drop( handle[0] );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::abortStream: error aborting output pcm device, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- if ( ( stream_.mode == INPUT || stream_.mode == DUPLEX ) && !apiInfo->synchronized ) {
- result = snd_pcm_drop( handle[1] );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::abortStream: error aborting input pcm device, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- unlock:
- apiInfo->runnable = false; // fixes high CPU usage when stopped
- MUTEX_UNLOCK( &stream_.mutex );
-
- if ( result >= 0 ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiAlsa :: callbackEvent()
-{
- AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle;
- if ( stream_.state == STREAM_STOPPED ) {
- MUTEX_LOCK( &stream_.mutex );
- while ( !apiInfo->runnable )
- pthread_cond_wait( &apiInfo->runnable_cv, &stream_.mutex );
-
- if ( stream_.state != STREAM_RUNNING ) {
- MUTEX_UNLOCK( &stream_.mutex );
- return;
- }
- MUTEX_UNLOCK( &stream_.mutex );
- }
-
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiAlsa::callbackEvent(): the stream is closed ... this shouldn't happen!";
- error( RtAudioError::WARNING );
- return;
- }
-
- int doStopStream = 0;
- RtAudioCallback callback = (RtAudioCallback) stream_.callbackInfo.callback;
- double streamTime = getStreamTime();
- RtAudioStreamStatus status = 0;
- if ( stream_.mode != INPUT && apiInfo->xrun[0] == true ) {
- status |= RTAUDIO_OUTPUT_UNDERFLOW;
- apiInfo->xrun[0] = false;
- }
- if ( stream_.mode != OUTPUT && apiInfo->xrun[1] == true ) {
- status |= RTAUDIO_INPUT_OVERFLOW;
- apiInfo->xrun[1] = false;
- }
- doStopStream = callback( stream_.userBuffer[0], stream_.userBuffer[1],
- stream_.bufferSize, streamTime, status, stream_.callbackInfo.userData );
-
- if ( doStopStream == 2 ) {
- abortStream();
- return;
- }
-
- MUTEX_LOCK( &stream_.mutex );
-
- // The state might change while waiting on a mutex.
- if ( stream_.state == STREAM_STOPPED ) goto unlock;
-
- int result;
- char *buffer;
- int channels;
- snd_pcm_t **handle;
- snd_pcm_sframes_t frames;
- RtAudioFormat format;
- handle = (snd_pcm_t **) apiInfo->handles;
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) {
-
- // Setup parameters.
- if ( stream_.doConvertBuffer[1] ) {
- buffer = stream_.deviceBuffer;
- channels = stream_.nDeviceChannels[1];
- format = stream_.deviceFormat[1];
- }
- else {
- buffer = stream_.userBuffer[1];
- channels = stream_.nUserChannels[1];
- format = stream_.userFormat;
- }
-
- // Read samples from device in interleaved/non-interleaved format.
- if ( stream_.deviceInterleaved[1] )
- result = snd_pcm_readi( handle[1], buffer, stream_.bufferSize );
- else {
- void *bufs[channels];
- size_t offset = stream_.bufferSize * formatBytes( format );
- for ( int i=0; i<channels; i++ )
- bufs[i] = (void *) (buffer + (i * offset));
- result = snd_pcm_readn( handle[1], bufs, stream_.bufferSize );
- }
-
- if ( result < (int) stream_.bufferSize ) {
- // Either an error or overrun occured.
- if ( result == -EPIPE ) {
- snd_pcm_state_t state = snd_pcm_state( handle[1] );
- if ( state == SND_PCM_STATE_XRUN ) {
- apiInfo->xrun[1] = true;
- result = snd_pcm_prepare( handle[1] );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::callbackEvent: error preparing device after overrun, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- }
- }
- else {
- errorStream_ << "RtApiAlsa::callbackEvent: error, current state is " << snd_pcm_state_name( state ) << ", " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- }
- }
- else {
- errorStream_ << "RtApiAlsa::callbackEvent: audio read error, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- }
- error( RtAudioError::WARNING );
- goto tryOutput;
- }
-
- // Do byte swapping if necessary.
- if ( stream_.doByteSwap[1] )
- byteSwapBuffer( buffer, stream_.bufferSize * channels, format );
-
- // Do buffer conversion if necessary.
- if ( stream_.doConvertBuffer[1] )
- convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] );
-
- // Check stream latency
- result = snd_pcm_delay( handle[1], &frames );
- if ( result == 0 && frames > 0 ) stream_.latency[1] = frames;
- }
-
- tryOutput:
-
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- // Setup parameters and do buffer conversion if necessary.
- if ( stream_.doConvertBuffer[0] ) {
- buffer = stream_.deviceBuffer;
- convertBuffer( buffer, stream_.userBuffer[0], stream_.convertInfo[0] );
- channels = stream_.nDeviceChannels[0];
- format = stream_.deviceFormat[0];
- }
- else {
- buffer = stream_.userBuffer[0];
- channels = stream_.nUserChannels[0];
- format = stream_.userFormat;
- }
-
- // Do byte swapping if necessary.
- if ( stream_.doByteSwap[0] )
- byteSwapBuffer(buffer, stream_.bufferSize * channels, format);
-
- // Write samples to device in interleaved/non-interleaved format.
- if ( stream_.deviceInterleaved[0] )
- result = snd_pcm_writei( handle[0], buffer, stream_.bufferSize );
- else {
- void *bufs[channels];
- size_t offset = stream_.bufferSize * formatBytes( format );
- for ( int i=0; i<channels; i++ )
- bufs[i] = (void *) (buffer + (i * offset));
- result = snd_pcm_writen( handle[0], bufs, stream_.bufferSize );
- }
-
- if ( result < (int) stream_.bufferSize ) {
- // Either an error or underrun occured.
- if ( result == -EPIPE ) {
- snd_pcm_state_t state = snd_pcm_state( handle[0] );
- if ( state == SND_PCM_STATE_XRUN ) {
- apiInfo->xrun[0] = true;
- result = snd_pcm_prepare( handle[0] );
- if ( result < 0 ) {
- errorStream_ << "RtApiAlsa::callbackEvent: error preparing device after underrun, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- }
- else
- errorText_ = "RtApiAlsa::callbackEvent: audio write error, underrun.";
- }
- else {
- errorStream_ << "RtApiAlsa::callbackEvent: error, current state is " << snd_pcm_state_name( state ) << ", " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- }
- }
- else {
- errorStream_ << "RtApiAlsa::callbackEvent: audio write error, " << snd_strerror( result ) << ".";
- errorText_ = errorStream_.str();
- }
- error( RtAudioError::WARNING );
- goto unlock;
- }
-
- // Check stream latency
- result = snd_pcm_delay( handle[0], &frames );
- if ( result == 0 && frames > 0 ) stream_.latency[0] = frames;
- }
-
- unlock:
- MUTEX_UNLOCK( &stream_.mutex );
-
- RtApi::tickStreamTime();
- if ( doStopStream == 1 ) this->stopStream();
-}
-
-static void *alsaCallbackHandler( void *ptr )
-{
- CallbackInfo *info = (CallbackInfo *) ptr;
- RtApiAlsa *object = (RtApiAlsa *) info->object;
- bool *isRunning = &info->isRunning;
-
-#ifdef SCHED_RR // Undefined with some OSes (eg: NetBSD 1.6.x with GNU Pthread)
- if ( &info->doRealtime ) {
- pthread_t tID = pthread_self(); // ID of this thread
- sched_param prio = { info->priority }; // scheduling priority of thread
- pthread_setschedparam( tID, SCHED_RR, &prio );
- }
-#endif
-
- while ( *isRunning == true ) {
- pthread_testcancel();
- object->callbackEvent();
- }
-
- pthread_exit( NULL );
-}
-
-//******************** End of __LINUX_ALSA__ *********************//
-#endif
-
-#if defined(__LINUX_PULSE__)
-
-// Code written by Peter Meerwald, pmeerw@pmeerw.net
-// and Tristan Matthews.
-
-#include <pulse/error.h>
-#include <pulse/simple.h>
-#include <cstdio>
-
-static const unsigned int SUPPORTED_SAMPLERATES[] = { 8000, 16000, 22050, 32000,
- 44100, 48000, 96000, 0};
-
-struct rtaudio_pa_format_mapping_t {
- RtAudioFormat rtaudio_format;
- pa_sample_format_t pa_format;
-};
-
-static const rtaudio_pa_format_mapping_t supported_sampleformats[] = {
- {RTAUDIO_SINT16, PA_SAMPLE_S16LE},
- {RTAUDIO_SINT32, PA_SAMPLE_S32LE},
- {RTAUDIO_FLOAT32, PA_SAMPLE_FLOAT32LE},
- {0, PA_SAMPLE_INVALID}};
-
-struct PulseAudioHandle {
- pa_simple *s_play;
- pa_simple *s_rec;
- pthread_t thread;
- pthread_cond_t runnable_cv;
- bool runnable;
- PulseAudioHandle() : s_play(0), s_rec(0), runnable(false) { }
-};
-
-RtApiPulse::~RtApiPulse()
-{
- if ( stream_.state != STREAM_CLOSED )
- closeStream();
-}
-
-unsigned int RtApiPulse::getDeviceCount( void )
-{
- return 1;
-}
-
-RtAudio::DeviceInfo RtApiPulse::getDeviceInfo( unsigned int /*device*/ )
-{
- RtAudio::DeviceInfo info;
- info.probed = true;
- info.name = "PulseAudio";
- info.outputChannels = 2;
- info.inputChannels = 2;
- info.duplexChannels = 2;
- info.isDefaultOutput = true;
- info.isDefaultInput = true;
-
- for ( const unsigned int *sr = SUPPORTED_SAMPLERATES; *sr; ++sr )
- info.sampleRates.push_back( *sr );
-
- info.preferredSampleRate = 48000;
- info.nativeFormats = RTAUDIO_SINT16 | RTAUDIO_SINT32 | RTAUDIO_FLOAT32;
-
- return info;
-}
-
-static void *pulseaudio_callback( void * user )
-{
- CallbackInfo *cbi = static_cast<CallbackInfo *>( user );
- RtApiPulse *context = static_cast<RtApiPulse *>( cbi->object );
- volatile bool *isRunning = &cbi->isRunning;
-
- while ( *isRunning ) {
- pthread_testcancel();
- context->callbackEvent();
- }
-
- pthread_exit( NULL );
-}
-
-void RtApiPulse::closeStream( void )
-{
- PulseAudioHandle *pah = static_cast<PulseAudioHandle *>( stream_.apiHandle );
-
- stream_.callbackInfo.isRunning = false;
- if ( pah ) {
- MUTEX_LOCK( &stream_.mutex );
- if ( stream_.state == STREAM_STOPPED ) {
- pah->runnable = true;
- pthread_cond_signal( &pah->runnable_cv );
- }
- MUTEX_UNLOCK( &stream_.mutex );
-
- pthread_join( pah->thread, 0 );
- if ( pah->s_play ) {
- pa_simple_flush( pah->s_play, NULL );
- pa_simple_free( pah->s_play );
- }
- if ( pah->s_rec )
- pa_simple_free( pah->s_rec );
-
- pthread_cond_destroy( &pah->runnable_cv );
- delete pah;
- stream_.apiHandle = 0;
- }
-
- if ( stream_.userBuffer[0] ) {
- free( stream_.userBuffer[0] );
- stream_.userBuffer[0] = 0;
- }
- if ( stream_.userBuffer[1] ) {
- free( stream_.userBuffer[1] );
- stream_.userBuffer[1] = 0;
- }
-
- stream_.state = STREAM_CLOSED;
- stream_.mode = UNINITIALIZED;
-}
-
-void RtApiPulse::callbackEvent( void )
-{
- PulseAudioHandle *pah = static_cast<PulseAudioHandle *>( stream_.apiHandle );
-
- if ( stream_.state == STREAM_STOPPED ) {
- MUTEX_LOCK( &stream_.mutex );
- while ( !pah->runnable )
- pthread_cond_wait( &pah->runnable_cv, &stream_.mutex );
-
- if ( stream_.state != STREAM_RUNNING ) {
- MUTEX_UNLOCK( &stream_.mutex );
- return;
- }
- MUTEX_UNLOCK( &stream_.mutex );
- }
-
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiPulse::callbackEvent(): the stream is closed ... "
- "this shouldn't happen!";
- error( RtAudioError::WARNING );
- return;
- }
-
- RtAudioCallback callback = (RtAudioCallback) stream_.callbackInfo.callback;
- double streamTime = getStreamTime();
- RtAudioStreamStatus status = 0;
- int doStopStream = callback( stream_.userBuffer[OUTPUT], stream_.userBuffer[INPUT],
- stream_.bufferSize, streamTime, status,
- stream_.callbackInfo.userData );
-
- if ( doStopStream == 2 ) {
- abortStream();
- return;
- }
-
- MUTEX_LOCK( &stream_.mutex );
- void *pulse_in = stream_.doConvertBuffer[INPUT] ? stream_.deviceBuffer : stream_.userBuffer[INPUT];
- void *pulse_out = stream_.doConvertBuffer[OUTPUT] ? stream_.deviceBuffer : stream_.userBuffer[OUTPUT];
-
- if ( stream_.state != STREAM_RUNNING )
- goto unlock;
-
- int pa_error;
- size_t bytes;
- if (stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- if ( stream_.doConvertBuffer[OUTPUT] ) {
- convertBuffer( stream_.deviceBuffer,
- stream_.userBuffer[OUTPUT],
- stream_.convertInfo[OUTPUT] );
- bytes = stream_.nDeviceChannels[OUTPUT] * stream_.bufferSize *
- formatBytes( stream_.deviceFormat[OUTPUT] );
- } else
- bytes = stream_.nUserChannels[OUTPUT] * stream_.bufferSize *
- formatBytes( stream_.userFormat );
-
- if ( pa_simple_write( pah->s_play, pulse_out, bytes, &pa_error ) < 0 ) {
- errorStream_ << "RtApiPulse::callbackEvent: audio write error, " <<
- pa_strerror( pa_error ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- }
- }
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX) {
- if ( stream_.doConvertBuffer[INPUT] )
- bytes = stream_.nDeviceChannels[INPUT] * stream_.bufferSize *
- formatBytes( stream_.deviceFormat[INPUT] );
- else
- bytes = stream_.nUserChannels[INPUT] * stream_.bufferSize *
- formatBytes( stream_.userFormat );
-
- if ( pa_simple_read( pah->s_rec, pulse_in, bytes, &pa_error ) < 0 ) {
- errorStream_ << "RtApiPulse::callbackEvent: audio read error, " <<
- pa_strerror( pa_error ) << ".";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- }
- if ( stream_.doConvertBuffer[INPUT] ) {
- convertBuffer( stream_.userBuffer[INPUT],
- stream_.deviceBuffer,
- stream_.convertInfo[INPUT] );
- }
- }
-
- unlock:
- MUTEX_UNLOCK( &stream_.mutex );
- RtApi::tickStreamTime();
-
- if ( doStopStream == 1 )
- stopStream();
-}
-
-void RtApiPulse::startStream( void )
-{
- PulseAudioHandle *pah = static_cast<PulseAudioHandle *>( stream_.apiHandle );
-
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiPulse::startStream(): the stream is not open!";
- error( RtAudioError::INVALID_USE );
- return;
- }
- if ( stream_.state == STREAM_RUNNING ) {
- errorText_ = "RtApiPulse::startStream(): the stream is already running!";
- error( RtAudioError::WARNING );
- return;
- }
-
- MUTEX_LOCK( &stream_.mutex );
-
- stream_.state = STREAM_RUNNING;
-
- pah->runnable = true;
- pthread_cond_signal( &pah->runnable_cv );
- MUTEX_UNLOCK( &stream_.mutex );
-}
-
-void RtApiPulse::stopStream( void )
-{
- PulseAudioHandle *pah = static_cast<PulseAudioHandle *>( stream_.apiHandle );
-
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiPulse::stopStream(): the stream is not open!";
- error( RtAudioError::INVALID_USE );
- return;
- }
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiPulse::stopStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- stream_.state = STREAM_STOPPED;
- MUTEX_LOCK( &stream_.mutex );
-
- if ( pah && pah->s_play ) {
- int pa_error;
- if ( pa_simple_drain( pah->s_play, &pa_error ) < 0 ) {
- errorStream_ << "RtApiPulse::stopStream: error draining output device, " <<
- pa_strerror( pa_error ) << ".";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- }
-
- stream_.state = STREAM_STOPPED;
- MUTEX_UNLOCK( &stream_.mutex );
-}
-
-void RtApiPulse::abortStream( void )
-{
- PulseAudioHandle *pah = static_cast<PulseAudioHandle*>( stream_.apiHandle );
-
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiPulse::abortStream(): the stream is not open!";
- error( RtAudioError::INVALID_USE );
- return;
- }
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiPulse::abortStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- stream_.state = STREAM_STOPPED;
- MUTEX_LOCK( &stream_.mutex );
-
- if ( pah && pah->s_play ) {
- int pa_error;
- if ( pa_simple_flush( pah->s_play, &pa_error ) < 0 ) {
- errorStream_ << "RtApiPulse::abortStream: error flushing output device, " <<
- pa_strerror( pa_error ) << ".";
- errorText_ = errorStream_.str();
- MUTEX_UNLOCK( &stream_.mutex );
- error( RtAudioError::SYSTEM_ERROR );
- return;
- }
- }
-
- stream_.state = STREAM_STOPPED;
- MUTEX_UNLOCK( &stream_.mutex );
-}
-
-bool RtApiPulse::probeDeviceOpen( unsigned int device, StreamMode mode,
- unsigned int channels, unsigned int firstChannel,
- unsigned int sampleRate, RtAudioFormat format,
- unsigned int *bufferSize, RtAudio::StreamOptions *options )
-{
- PulseAudioHandle *pah = 0;
- unsigned long bufferBytes = 0;
- pa_sample_spec ss;
-
- if ( device != 0 ) return false;
- if ( mode != INPUT && mode != OUTPUT ) return false;
- if ( channels != 1 && channels != 2 ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: unsupported number of channels.";
- return false;
- }
- ss.channels = channels;
-
- if ( firstChannel != 0 ) return false;
-
- bool sr_found = false;
- for ( const unsigned int *sr = SUPPORTED_SAMPLERATES; *sr; ++sr ) {
- if ( sampleRate == *sr ) {
- sr_found = true;
- stream_.sampleRate = sampleRate;
- ss.rate = sampleRate;
- break;
- }
- }
- if ( !sr_found ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: unsupported sample rate.";
- return false;
- }
-
- bool sf_found = 0;
- for ( const rtaudio_pa_format_mapping_t *sf = supported_sampleformats;
- sf->rtaudio_format && sf->pa_format != PA_SAMPLE_INVALID; ++sf ) {
- if ( format == sf->rtaudio_format ) {
- sf_found = true;
- stream_.userFormat = sf->rtaudio_format;
- stream_.deviceFormat[mode] = stream_.userFormat;
- ss.format = sf->pa_format;
- break;
- }
- }
- if ( !sf_found ) { // Use internal data format conversion.
- stream_.userFormat = format;
- stream_.deviceFormat[mode] = RTAUDIO_FLOAT32;
- ss.format = PA_SAMPLE_FLOAT32LE;
- }
-
- // Set other stream parameters.
- if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false;
- else stream_.userInterleaved = true;
- stream_.deviceInterleaved[mode] = true;
- stream_.nBuffers = 1;
- stream_.doByteSwap[mode] = false;
- stream_.nUserChannels[mode] = channels;
- stream_.nDeviceChannels[mode] = channels + firstChannel;
- stream_.channelOffset[mode] = 0;
- std::string streamName = "RtAudio";
-
- // Set flags for buffer conversion.
- stream_.doConvertBuffer[mode] = false;
- if ( stream_.userFormat != stream_.deviceFormat[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.nUserChannels[mode] < stream_.nDeviceChannels[mode] )
- stream_.doConvertBuffer[mode] = true;
-
- // Allocate necessary internal buffers.
- bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat );
- stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 );
- if ( stream_.userBuffer[mode] == NULL ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: error allocating user buffer memory.";
- goto error;
- }
- stream_.bufferSize = *bufferSize;
-
- if ( stream_.doConvertBuffer[mode] ) {
-
- bool makeBuffer = true;
- bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] );
- if ( mode == INPUT ) {
- if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) {
- unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] );
- if ( bufferBytes <= bytesOut ) makeBuffer = false;
- }
- }
-
- if ( makeBuffer ) {
- bufferBytes *= *bufferSize;
- if ( stream_.deviceBuffer ) free( stream_.deviceBuffer );
- stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 );
- if ( stream_.deviceBuffer == NULL ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: error allocating device buffer memory.";
- goto error;
- }
- }
- }
-
- stream_.device[mode] = device;
-
- // Setup the buffer conversion information structure.
- if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, firstChannel );
-
- if ( !stream_.apiHandle ) {
- PulseAudioHandle *pah = new PulseAudioHandle;
- if ( !pah ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: error allocating memory for handle.";
- goto error;
- }
-
- stream_.apiHandle = pah;
- if ( pthread_cond_init( &pah->runnable_cv, NULL ) != 0 ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: error creating condition variable.";
- goto error;
- }
- }
- pah = static_cast<PulseAudioHandle *>( stream_.apiHandle );
-
- int error;
- if ( options && !options->streamName.empty() ) streamName = options->streamName;
- switch ( mode ) {
- case INPUT:
- pa_buffer_attr buffer_attr;
- buffer_attr.fragsize = bufferBytes;
- buffer_attr.maxlength = -1;
-
- pah->s_rec = pa_simple_new( NULL, streamName.c_str(), PA_STREAM_RECORD, NULL, "Record", &ss, NULL, &buffer_attr, &error );
- if ( !pah->s_rec ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: error connecting input to PulseAudio server.";
- goto error;
- }
- break;
- case OUTPUT:
- pah->s_play = pa_simple_new( NULL, "RtAudio", PA_STREAM_PLAYBACK, NULL, "Playback", &ss, NULL, NULL, &error );
- if ( !pah->s_play ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: error connecting output to PulseAudio server.";
- goto error;
- }
- break;
- default:
- goto error;
- }
-
- if ( stream_.mode == UNINITIALIZED )
- stream_.mode = mode;
- else if ( stream_.mode == mode )
- goto error;
- else
- stream_.mode = DUPLEX;
-
- if ( !stream_.callbackInfo.isRunning ) {
- stream_.callbackInfo.object = this;
- stream_.callbackInfo.isRunning = true;
- if ( pthread_create( &pah->thread, NULL, pulseaudio_callback, (void *)&stream_.callbackInfo) != 0 ) {
- errorText_ = "RtApiPulse::probeDeviceOpen: error creating thread.";
- goto error;
- }
- }
-
- stream_.state = STREAM_STOPPED;
- return true;
-
- error:
- if ( pah && stream_.callbackInfo.isRunning ) {
- pthread_cond_destroy( &pah->runnable_cv );
- delete pah;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- return FAILURE;
-}
-
-//******************** End of __LINUX_PULSE__ *********************//
-#endif
-
-#if defined(__LINUX_OSS__)
-
-#include <unistd.h>
-#include <sys/ioctl.h>
-#include <unistd.h>
-#include <fcntl.h>
-#include <sys/soundcard.h>
-#include <errno.h>
-#include <math.h>
-
-static void *ossCallbackHandler(void * ptr);
-
-// A structure to hold various information related to the OSS API
-// implementation.
-struct OssHandle {
- int id[2]; // device ids
- bool xrun[2];
- bool triggered;
- pthread_cond_t runnable;
-
- OssHandle()
- :triggered(false) { id[0] = 0; id[1] = 0; xrun[0] = false; xrun[1] = false; }
-};
-
-RtApiOss :: RtApiOss()
-{
- // Nothing to do here.
-}
-
-RtApiOss :: ~RtApiOss()
-{
- if ( stream_.state != STREAM_CLOSED ) closeStream();
-}
-
-unsigned int RtApiOss :: getDeviceCount( void )
-{
- int mixerfd = open( "/dev/mixer", O_RDWR, 0 );
- if ( mixerfd == -1 ) {
- errorText_ = "RtApiOss::getDeviceCount: error opening '/dev/mixer'.";
- error( RtAudioError::WARNING );
- return 0;
- }
-
- oss_sysinfo sysinfo;
- if ( ioctl( mixerfd, SNDCTL_SYSINFO, &sysinfo ) == -1 ) {
- close( mixerfd );
- errorText_ = "RtApiOss::getDeviceCount: error getting sysinfo, OSS version >= 4.0 is required.";
- error( RtAudioError::WARNING );
- return 0;
- }
-
- close( mixerfd );
- return sysinfo.numaudios;
-}
-
-RtAudio::DeviceInfo RtApiOss :: getDeviceInfo( unsigned int device )
-{
- RtAudio::DeviceInfo info;
- info.probed = false;
-
- int mixerfd = open( "/dev/mixer", O_RDWR, 0 );
- if ( mixerfd == -1 ) {
- errorText_ = "RtApiOss::getDeviceInfo: error opening '/dev/mixer'.";
- error( RtAudioError::WARNING );
- return info;
- }
-
- oss_sysinfo sysinfo;
- int result = ioctl( mixerfd, SNDCTL_SYSINFO, &sysinfo );
- if ( result == -1 ) {
- close( mixerfd );
- errorText_ = "RtApiOss::getDeviceInfo: error getting sysinfo, OSS version >= 4.0 is required.";
- error( RtAudioError::WARNING );
- return info;
- }
-
- unsigned nDevices = sysinfo.numaudios;
- if ( nDevices == 0 ) {
- close( mixerfd );
- errorText_ = "RtApiOss::getDeviceInfo: no devices found!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- if ( device >= nDevices ) {
- close( mixerfd );
- errorText_ = "RtApiOss::getDeviceInfo: device ID is invalid!";
- error( RtAudioError::INVALID_USE );
- return info;
- }
-
- oss_audioinfo ainfo;
- ainfo.dev = device;
- result = ioctl( mixerfd, SNDCTL_AUDIOINFO, &ainfo );
- close( mixerfd );
- if ( result == -1 ) {
- errorStream_ << "RtApiOss::getDeviceInfo: error getting device (" << ainfo.name << ") info.";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Probe channels
- if ( ainfo.caps & PCM_CAP_OUTPUT ) info.outputChannels = ainfo.max_channels;
- if ( ainfo.caps & PCM_CAP_INPUT ) info.inputChannels = ainfo.max_channels;
- if ( ainfo.caps & PCM_CAP_DUPLEX ) {
- if ( info.outputChannels > 0 && info.inputChannels > 0 && ainfo.caps & PCM_CAP_DUPLEX )
- info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels;
- }
-
- // Probe data formats ... do for input
- unsigned long mask = ainfo.iformats;
- if ( mask & AFMT_S16_LE || mask & AFMT_S16_BE )
- info.nativeFormats |= RTAUDIO_SINT16;
- if ( mask & AFMT_S8 )
- info.nativeFormats |= RTAUDIO_SINT8;
- if ( mask & AFMT_S32_LE || mask & AFMT_S32_BE )
- info.nativeFormats |= RTAUDIO_SINT32;
- if ( mask & AFMT_FLOAT )
- info.nativeFormats |= RTAUDIO_FLOAT32;
- if ( mask & AFMT_S24_LE || mask & AFMT_S24_BE )
- info.nativeFormats |= RTAUDIO_SINT24;
-
- // Check that we have at least one supported format
- if ( info.nativeFormats == 0 ) {
- errorStream_ << "RtApiOss::getDeviceInfo: device (" << ainfo.name << ") data format not supported by RtAudio.";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- return info;
- }
-
- // Probe the supported sample rates.
- info.sampleRates.clear();
- if ( ainfo.nrates ) {
- for ( unsigned int i=0; i<ainfo.nrates; i++ ) {
- for ( unsigned int k=0; k<MAX_SAMPLE_RATES; k++ ) {
- if ( ainfo.rates[i] == SAMPLE_RATES[k] ) {
- info.sampleRates.push_back( SAMPLE_RATES[k] );
-
- if ( !info.preferredSampleRate || ( SAMPLE_RATES[k] <= 48000 && SAMPLE_RATES[k] > info.preferredSampleRate ) )
- info.preferredSampleRate = SAMPLE_RATES[k];
-
- break;
- }
- }
- }
- }
- else {
- // Check min and max rate values;
- for ( unsigned int k=0; k<MAX_SAMPLE_RATES; k++ ) {
- if ( ainfo.min_rate <= (int) SAMPLE_RATES[k] && ainfo.max_rate >= (int) SAMPLE_RATES[k] ) {
- info.sampleRates.push_back( SAMPLE_RATES[k] );
-
- if ( !info.preferredSampleRate || ( SAMPLE_RATES[k] <= 48000 && SAMPLE_RATES[k] > info.preferredSampleRate ) )
- info.preferredSampleRate = SAMPLE_RATES[k];
- }
- }
- }
-
- if ( info.sampleRates.size() == 0 ) {
- errorStream_ << "RtApiOss::getDeviceInfo: no supported sample rates found for device (" << ainfo.name << ").";
- errorText_ = errorStream_.str();
- error( RtAudioError::WARNING );
- }
- else {
- info.probed = true;
- info.name = ainfo.name;
- }
-
- return info;
-}
-
-
-bool RtApiOss :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels,
- unsigned int firstChannel, unsigned int sampleRate,
- RtAudioFormat format, unsigned int *bufferSize,
- RtAudio::StreamOptions *options )
-{
- int mixerfd = open( "/dev/mixer", O_RDWR, 0 );
- if ( mixerfd == -1 ) {
- errorText_ = "RtApiOss::probeDeviceOpen: error opening '/dev/mixer'.";
- return FAILURE;
- }
-
- oss_sysinfo sysinfo;
- int result = ioctl( mixerfd, SNDCTL_SYSINFO, &sysinfo );
- if ( result == -1 ) {
- close( mixerfd );
- errorText_ = "RtApiOss::probeDeviceOpen: error getting sysinfo, OSS version >= 4.0 is required.";
- return FAILURE;
- }
-
- unsigned nDevices = sysinfo.numaudios;
- if ( nDevices == 0 ) {
- // This should not happen because a check is made before this function is called.
- close( mixerfd );
- errorText_ = "RtApiOss::probeDeviceOpen: no devices found!";
- return FAILURE;
- }
-
- if ( device >= nDevices ) {
- // This should not happen because a check is made before this function is called.
- close( mixerfd );
- errorText_ = "RtApiOss::probeDeviceOpen: device ID is invalid!";
- return FAILURE;
- }
-
- oss_audioinfo ainfo;
- ainfo.dev = device;
- result = ioctl( mixerfd, SNDCTL_AUDIOINFO, &ainfo );
- close( mixerfd );
- if ( result == -1 ) {
- errorStream_ << "RtApiOss::getDeviceInfo: error getting device (" << ainfo.name << ") info.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Check if device supports input or output
- if ( ( mode == OUTPUT && !( ainfo.caps & PCM_CAP_OUTPUT ) ) ||
- ( mode == INPUT && !( ainfo.caps & PCM_CAP_INPUT ) ) ) {
- if ( mode == OUTPUT )
- errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") does not support output.";
- else
- errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") does not support input.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- int flags = 0;
- OssHandle *handle = (OssHandle *) stream_.apiHandle;
- if ( mode == OUTPUT )
- flags |= O_WRONLY;
- else { // mode == INPUT
- if (stream_.mode == OUTPUT && stream_.device[0] == device) {
- // We just set the same device for playback ... close and reopen for duplex (OSS only).
- close( handle->id[0] );
- handle->id[0] = 0;
- if ( !( ainfo.caps & PCM_CAP_DUPLEX ) ) {
- errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") does not support duplex mode.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- // Check that the number previously set channels is the same.
- if ( stream_.nUserChannels[0] != channels ) {
- errorStream_ << "RtApiOss::probeDeviceOpen: input/output channels must be equal for OSS duplex device (" << ainfo.name << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- flags |= O_RDWR;
- }
- else
- flags |= O_RDONLY;
- }
-
- // Set exclusive access if specified.
- if ( options && options->flags & RTAUDIO_HOG_DEVICE ) flags |= O_EXCL;
-
- // Try to open the device.
- int fd;
- fd = open( ainfo.devnode, flags, 0 );
- if ( fd == -1 ) {
- if ( errno == EBUSY )
- errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") is busy.";
- else
- errorStream_ << "RtApiOss::probeDeviceOpen: error opening device (" << ainfo.name << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // For duplex operation, specifically set this mode (this doesn't seem to work).
- /*
- if ( flags | O_RDWR ) {
- result = ioctl( fd, SNDCTL_DSP_SETDUPLEX, NULL );
- if ( result == -1) {
- errorStream_ << "RtApiOss::probeDeviceOpen: error setting duplex mode for device (" << ainfo.name << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- }
- */
-
- // Check the device channel support.
- stream_.nUserChannels[mode] = channels;
- if ( ainfo.max_channels < (int)(channels + firstChannel) ) {
- close( fd );
- errorStream_ << "RtApiOss::probeDeviceOpen: the device (" << ainfo.name << ") does not support requested channel parameters.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Set the number of channels.
- int deviceChannels = channels + firstChannel;
- result = ioctl( fd, SNDCTL_DSP_CHANNELS, &deviceChannels );
- if ( result == -1 || deviceChannels < (int)(channels + firstChannel) ) {
- close( fd );
- errorStream_ << "RtApiOss::probeDeviceOpen: error setting channel parameters on device (" << ainfo.name << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- stream_.nDeviceChannels[mode] = deviceChannels;
-
- // Get the data format mask
- int mask;
- result = ioctl( fd, SNDCTL_DSP_GETFMTS, &mask );
- if ( result == -1 ) {
- close( fd );
- errorStream_ << "RtApiOss::probeDeviceOpen: error getting device (" << ainfo.name << ") data formats.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Determine how to set the device format.
- stream_.userFormat = format;
- int deviceFormat = -1;
- stream_.doByteSwap[mode] = false;
- if ( format == RTAUDIO_SINT8 ) {
- if ( mask & AFMT_S8 ) {
- deviceFormat = AFMT_S8;
- stream_.deviceFormat[mode] = RTAUDIO_SINT8;
- }
- }
- else if ( format == RTAUDIO_SINT16 ) {
- if ( mask & AFMT_S16_NE ) {
- deviceFormat = AFMT_S16_NE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- }
- else if ( mask & AFMT_S16_OE ) {
- deviceFormat = AFMT_S16_OE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- stream_.doByteSwap[mode] = true;
- }
- }
- else if ( format == RTAUDIO_SINT24 ) {
- if ( mask & AFMT_S24_NE ) {
- deviceFormat = AFMT_S24_NE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT24;
- }
- else if ( mask & AFMT_S24_OE ) {
- deviceFormat = AFMT_S24_OE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT24;
- stream_.doByteSwap[mode] = true;
- }
- }
- else if ( format == RTAUDIO_SINT32 ) {
- if ( mask & AFMT_S32_NE ) {
- deviceFormat = AFMT_S32_NE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT32;
- }
- else if ( mask & AFMT_S32_OE ) {
- deviceFormat = AFMT_S32_OE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT32;
- stream_.doByteSwap[mode] = true;
- }
- }
-
- if ( deviceFormat == -1 ) {
- // The user requested format is not natively supported by the device.
- if ( mask & AFMT_S16_NE ) {
- deviceFormat = AFMT_S16_NE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- }
- else if ( mask & AFMT_S32_NE ) {
- deviceFormat = AFMT_S32_NE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT32;
- }
- else if ( mask & AFMT_S24_NE ) {
- deviceFormat = AFMT_S24_NE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT24;
- }
- else if ( mask & AFMT_S16_OE ) {
- deviceFormat = AFMT_S16_OE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT16;
- stream_.doByteSwap[mode] = true;
- }
- else if ( mask & AFMT_S32_OE ) {
- deviceFormat = AFMT_S32_OE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT32;
- stream_.doByteSwap[mode] = true;
- }
- else if ( mask & AFMT_S24_OE ) {
- deviceFormat = AFMT_S24_OE;
- stream_.deviceFormat[mode] = RTAUDIO_SINT24;
- stream_.doByteSwap[mode] = true;
- }
- else if ( mask & AFMT_S8) {
- deviceFormat = AFMT_S8;
- stream_.deviceFormat[mode] = RTAUDIO_SINT8;
- }
- }
-
- if ( stream_.deviceFormat[mode] == 0 ) {
- // This really shouldn't happen ...
- close( fd );
- errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") data format not supported by RtAudio.";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Set the data format.
- int temp = deviceFormat;
- result = ioctl( fd, SNDCTL_DSP_SETFMT, &deviceFormat );
- if ( result == -1 || deviceFormat != temp ) {
- close( fd );
- errorStream_ << "RtApiOss::probeDeviceOpen: error setting data format on device (" << ainfo.name << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Attempt to set the buffer size. According to OSS, the minimum
- // number of buffers is two. The supposed minimum buffer size is 16
- // bytes, so that will be our lower bound. The argument to this
- // call is in the form 0xMMMMSSSS (hex), where the buffer size (in
- // bytes) is given as 2^SSSS and the number of buffers as 2^MMMM.
- // We'll check the actual value used near the end of the setup
- // procedure.
- int ossBufferBytes = *bufferSize * formatBytes( stream_.deviceFormat[mode] ) * deviceChannels;
- if ( ossBufferBytes < 16 ) ossBufferBytes = 16;
- int buffers = 0;
- if ( options ) buffers = options->numberOfBuffers;
- if ( options && options->flags & RTAUDIO_MINIMIZE_LATENCY ) buffers = 2;
- if ( buffers < 2 ) buffers = 3;
- temp = ((int) buffers << 16) + (int)( log10( (double)ossBufferBytes ) / log10( 2.0 ) );
- result = ioctl( fd, SNDCTL_DSP_SETFRAGMENT, &temp );
- if ( result == -1 ) {
- close( fd );
- errorStream_ << "RtApiOss::probeDeviceOpen: error setting buffer size on device (" << ainfo.name << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- stream_.nBuffers = buffers;
-
- // Save buffer size (in sample frames).
- *bufferSize = ossBufferBytes / ( formatBytes(stream_.deviceFormat[mode]) * deviceChannels );
- stream_.bufferSize = *bufferSize;
-
- // Set the sample rate.
- int srate = sampleRate;
- result = ioctl( fd, SNDCTL_DSP_SPEED, &srate );
- if ( result == -1 ) {
- close( fd );
- errorStream_ << "RtApiOss::probeDeviceOpen: error setting sample rate (" << sampleRate << ") on device (" << ainfo.name << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
-
- // Verify the sample rate setup worked.
- if ( abs( srate - sampleRate ) > 100 ) {
- close( fd );
- errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") does not support sample rate (" << sampleRate << ").";
- errorText_ = errorStream_.str();
- return FAILURE;
- }
- stream_.sampleRate = sampleRate;
-
- if ( mode == INPUT && stream_.mode == OUTPUT && stream_.device[0] == device) {
- // We're doing duplex setup here.
- stream_.deviceFormat[0] = stream_.deviceFormat[1];
- stream_.nDeviceChannels[0] = deviceChannels;
- }
-
- // Set interleaving parameters.
- stream_.userInterleaved = true;
- stream_.deviceInterleaved[mode] = true;
- if ( options && options->flags & RTAUDIO_NONINTERLEAVED )
- stream_.userInterleaved = false;
-
- // Set flags for buffer conversion
- stream_.doConvertBuffer[mode] = false;
- if ( stream_.userFormat != stream_.deviceFormat[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.nUserChannels[mode] < stream_.nDeviceChannels[mode] )
- stream_.doConvertBuffer[mode] = true;
- if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] &&
- stream_.nUserChannels[mode] > 1 )
- stream_.doConvertBuffer[mode] = true;
-
- // Allocate the stream handles if necessary and then save.
- if ( stream_.apiHandle == 0 ) {
- try {
- handle = new OssHandle;
- }
- catch ( std::bad_alloc& ) {
- errorText_ = "RtApiOss::probeDeviceOpen: error allocating OssHandle memory.";
- goto error;
- }
-
- if ( pthread_cond_init( &handle->runnable, NULL ) ) {
- errorText_ = "RtApiOss::probeDeviceOpen: error initializing pthread condition variable.";
- goto error;
- }
-
- stream_.apiHandle = (void *) handle;
- }
- else {
- handle = (OssHandle *) stream_.apiHandle;
- }
- handle->id[mode] = fd;
-
- // Allocate necessary internal buffers.
- unsigned long bufferBytes;
- bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat );
- stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 );
- if ( stream_.userBuffer[mode] == NULL ) {
- errorText_ = "RtApiOss::probeDeviceOpen: error allocating user buffer memory.";
- goto error;
- }
-
- if ( stream_.doConvertBuffer[mode] ) {
-
- bool makeBuffer = true;
- bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] );
- if ( mode == INPUT ) {
- if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) {
- unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] );
- if ( bufferBytes <= bytesOut ) makeBuffer = false;
- }
- }
-
- if ( makeBuffer ) {
- bufferBytes *= *bufferSize;
- if ( stream_.deviceBuffer ) free( stream_.deviceBuffer );
- stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 );
- if ( stream_.deviceBuffer == NULL ) {
- errorText_ = "RtApiOss::probeDeviceOpen: error allocating device buffer memory.";
- goto error;
- }
- }
- }
-
- stream_.device[mode] = device;
- stream_.state = STREAM_STOPPED;
-
- // Setup the buffer conversion information structure.
- if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, firstChannel );
-
- // Setup thread if necessary.
- if ( stream_.mode == OUTPUT && mode == INPUT ) {
- // We had already set up an output stream.
- stream_.mode = DUPLEX;
- if ( stream_.device[0] == device ) handle->id[0] = fd;
- }
- else {
- stream_.mode = mode;
-
- // Setup callback thread.
- stream_.callbackInfo.object = (void *) this;
-
- // Set the thread attributes for joinable and realtime scheduling
- // priority. The higher priority will only take affect if the
- // program is run as root or suid.
- pthread_attr_t attr;
- pthread_attr_init( &attr );
- pthread_attr_setdetachstate( &attr, PTHREAD_CREATE_JOINABLE );
-#ifdef SCHED_RR // Undefined with some OSes (eg: NetBSD 1.6.x with GNU Pthread)
- if ( options && options->flags & RTAUDIO_SCHEDULE_REALTIME ) {
- struct sched_param param;
- int priority = options->priority;
- int min = sched_get_priority_min( SCHED_RR );
- int max = sched_get_priority_max( SCHED_RR );
- if ( priority < min ) priority = min;
- else if ( priority > max ) priority = max;
- param.sched_priority = priority;
- pthread_attr_setschedparam( &attr, ¶m );
- pthread_attr_setschedpolicy( &attr, SCHED_RR );
- }
- else
- pthread_attr_setschedpolicy( &attr, SCHED_OTHER );
-#else
- pthread_attr_setschedpolicy( &attr, SCHED_OTHER );
-#endif
-
- stream_.callbackInfo.isRunning = true;
- result = pthread_create( &stream_.callbackInfo.thread, &attr, ossCallbackHandler, &stream_.callbackInfo );
- pthread_attr_destroy( &attr );
- if ( result ) {
- stream_.callbackInfo.isRunning = false;
- errorText_ = "RtApiOss::error creating callback thread!";
- goto error;
- }
- }
-
- return SUCCESS;
-
- error:
- if ( handle ) {
- pthread_cond_destroy( &handle->runnable );
- if ( handle->id[0] ) close( handle->id[0] );
- if ( handle->id[1] ) close( handle->id[1] );
- delete handle;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- return FAILURE;
-}
-
-void RtApiOss :: closeStream()
-{
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiOss::closeStream(): no open stream to close!";
- error( RtAudioError::WARNING );
- return;
- }
-
- OssHandle *handle = (OssHandle *) stream_.apiHandle;
- stream_.callbackInfo.isRunning = false;
- MUTEX_LOCK( &stream_.mutex );
- if ( stream_.state == STREAM_STOPPED )
- pthread_cond_signal( &handle->runnable );
- MUTEX_UNLOCK( &stream_.mutex );
- pthread_join( stream_.callbackInfo.thread, NULL );
-
- if ( stream_.state == STREAM_RUNNING ) {
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX )
- ioctl( handle->id[0], SNDCTL_DSP_HALT, 0 );
- else
- ioctl( handle->id[1], SNDCTL_DSP_HALT, 0 );
- stream_.state = STREAM_STOPPED;
- }
-
- if ( handle ) {
- pthread_cond_destroy( &handle->runnable );
- if ( handle->id[0] ) close( handle->id[0] );
- if ( handle->id[1] ) close( handle->id[1] );
- delete handle;
- stream_.apiHandle = 0;
- }
-
- for ( int i=0; i<2; i++ ) {
- if ( stream_.userBuffer[i] ) {
- free( stream_.userBuffer[i] );
- stream_.userBuffer[i] = 0;
- }
- }
-
- if ( stream_.deviceBuffer ) {
- free( stream_.deviceBuffer );
- stream_.deviceBuffer = 0;
- }
-
- stream_.mode = UNINITIALIZED;
- stream_.state = STREAM_CLOSED;
-}
-
-void RtApiOss :: startStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_RUNNING ) {
- errorText_ = "RtApiOss::startStream(): the stream is already running!";
- error( RtAudioError::WARNING );
- return;
- }
-
- MUTEX_LOCK( &stream_.mutex );
-
- stream_.state = STREAM_RUNNING;
-
- // No need to do anything else here ... OSS automatically starts
- // when fed samples.
-
- MUTEX_UNLOCK( &stream_.mutex );
-
- OssHandle *handle = (OssHandle *) stream_.apiHandle;
- pthread_cond_signal( &handle->runnable );
-}
-
-void RtApiOss :: stopStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiOss::stopStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- MUTEX_LOCK( &stream_.mutex );
-
- // The state might change while waiting on a mutex.
- if ( stream_.state == STREAM_STOPPED ) {
- MUTEX_UNLOCK( &stream_.mutex );
- return;
- }
-
- int result = 0;
- OssHandle *handle = (OssHandle *) stream_.apiHandle;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- // Flush the output with zeros a few times.
- char *buffer;
- int samples;
- RtAudioFormat format;
-
- if ( stream_.doConvertBuffer[0] ) {
- buffer = stream_.deviceBuffer;
- samples = stream_.bufferSize * stream_.nDeviceChannels[0];
- format = stream_.deviceFormat[0];
- }
- else {
- buffer = stream_.userBuffer[0];
- samples = stream_.bufferSize * stream_.nUserChannels[0];
- format = stream_.userFormat;
- }
-
- memset( buffer, 0, samples * formatBytes(format) );
- for ( unsigned int i=0; i<stream_.nBuffers+1; i++ ) {
- result = write( handle->id[0], buffer, samples * formatBytes(format) );
- if ( result == -1 ) {
- errorText_ = "RtApiOss::stopStream: audio write error.";
- error( RtAudioError::WARNING );
- }
- }
-
- result = ioctl( handle->id[0], SNDCTL_DSP_HALT, 0 );
- if ( result == -1 ) {
- errorStream_ << "RtApiOss::stopStream: system error stopping callback procedure on device (" << stream_.device[0] << ").";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- handle->triggered = false;
- }
-
- if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && handle->id[0] != handle->id[1] ) ) {
- result = ioctl( handle->id[1], SNDCTL_DSP_HALT, 0 );
- if ( result == -1 ) {
- errorStream_ << "RtApiOss::stopStream: system error stopping input callback procedure on device (" << stream_.device[0] << ").";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- unlock:
- stream_.state = STREAM_STOPPED;
- MUTEX_UNLOCK( &stream_.mutex );
-
- if ( result != -1 ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiOss :: abortStream()
-{
- verifyStream();
- if ( stream_.state == STREAM_STOPPED ) {
- errorText_ = "RtApiOss::abortStream(): the stream is already stopped!";
- error( RtAudioError::WARNING );
- return;
- }
-
- MUTEX_LOCK( &stream_.mutex );
-
- // The state might change while waiting on a mutex.
- if ( stream_.state == STREAM_STOPPED ) {
- MUTEX_UNLOCK( &stream_.mutex );
- return;
- }
-
- int result = 0;
- OssHandle *handle = (OssHandle *) stream_.apiHandle;
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
- result = ioctl( handle->id[0], SNDCTL_DSP_HALT, 0 );
- if ( result == -1 ) {
- errorStream_ << "RtApiOss::abortStream: system error stopping callback procedure on device (" << stream_.device[0] << ").";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- handle->triggered = false;
- }
-
- if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && handle->id[0] != handle->id[1] ) ) {
- result = ioctl( handle->id[1], SNDCTL_DSP_HALT, 0 );
- if ( result == -1 ) {
- errorStream_ << "RtApiOss::abortStream: system error stopping input callback procedure on device (" << stream_.device[0] << ").";
- errorText_ = errorStream_.str();
- goto unlock;
- }
- }
-
- unlock:
- stream_.state = STREAM_STOPPED;
- MUTEX_UNLOCK( &stream_.mutex );
-
- if ( result != -1 ) return;
- error( RtAudioError::SYSTEM_ERROR );
-}
-
-void RtApiOss :: callbackEvent()
-{
- OssHandle *handle = (OssHandle *) stream_.apiHandle;
- if ( stream_.state == STREAM_STOPPED ) {
- MUTEX_LOCK( &stream_.mutex );
- pthread_cond_wait( &handle->runnable, &stream_.mutex );
- if ( stream_.state != STREAM_RUNNING ) {
- MUTEX_UNLOCK( &stream_.mutex );
- return;
- }
- MUTEX_UNLOCK( &stream_.mutex );
- }
-
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApiOss::callbackEvent(): the stream is closed ... this shouldn't happen!";
- error( RtAudioError::WARNING );
- return;
- }
-
- // Invoke user callback to get fresh output data.
- int doStopStream = 0;
- RtAudioCallback callback = (RtAudioCallback) stream_.callbackInfo.callback;
- double streamTime = getStreamTime();
- RtAudioStreamStatus status = 0;
- if ( stream_.mode != INPUT && handle->xrun[0] == true ) {
- status |= RTAUDIO_OUTPUT_UNDERFLOW;
- handle->xrun[0] = false;
- }
- if ( stream_.mode != OUTPUT && handle->xrun[1] == true ) {
- status |= RTAUDIO_INPUT_OVERFLOW;
- handle->xrun[1] = false;
- }
- doStopStream = callback( stream_.userBuffer[0], stream_.userBuffer[1],
- stream_.bufferSize, streamTime, status, stream_.callbackInfo.userData );
- if ( doStopStream == 2 ) {
- this->abortStream();
- return;
- }
-
- MUTEX_LOCK( &stream_.mutex );
-
- // The state might change while waiting on a mutex.
- if ( stream_.state == STREAM_STOPPED ) goto unlock;
-
- int result;
- char *buffer;
- int samples;
- RtAudioFormat format;
-
- if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) {
-
- // Setup parameters and do buffer conversion if necessary.
- if ( stream_.doConvertBuffer[0] ) {
- buffer = stream_.deviceBuffer;
- convertBuffer( buffer, stream_.userBuffer[0], stream_.convertInfo[0] );
- samples = stream_.bufferSize * stream_.nDeviceChannels[0];
- format = stream_.deviceFormat[0];
- }
- else {
- buffer = stream_.userBuffer[0];
- samples = stream_.bufferSize * stream_.nUserChannels[0];
- format = stream_.userFormat;
- }
-
- // Do byte swapping if necessary.
- if ( stream_.doByteSwap[0] )
- byteSwapBuffer( buffer, samples, format );
-
- if ( stream_.mode == DUPLEX && handle->triggered == false ) {
- int trig = 0;
- ioctl( handle->id[0], SNDCTL_DSP_SETTRIGGER, &trig );
- result = write( handle->id[0], buffer, samples * formatBytes(format) );
- trig = PCM_ENABLE_INPUT|PCM_ENABLE_OUTPUT;
- ioctl( handle->id[0], SNDCTL_DSP_SETTRIGGER, &trig );
- handle->triggered = true;
- }
- else
- // Write samples to device.
- result = write( handle->id[0], buffer, samples * formatBytes(format) );
-
- if ( result == -1 ) {
- // We'll assume this is an underrun, though there isn't a
- // specific means for determining that.
- handle->xrun[0] = true;
- errorText_ = "RtApiOss::callbackEvent: audio write error.";
- error( RtAudioError::WARNING );
- // Continue on to input section.
- }
- }
-
- if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) {
-
- // Setup parameters.
- if ( stream_.doConvertBuffer[1] ) {
- buffer = stream_.deviceBuffer;
- samples = stream_.bufferSize * stream_.nDeviceChannels[1];
- format = stream_.deviceFormat[1];
- }
- else {
- buffer = stream_.userBuffer[1];
- samples = stream_.bufferSize * stream_.nUserChannels[1];
- format = stream_.userFormat;
- }
-
- // Read samples from device.
- result = read( handle->id[1], buffer, samples * formatBytes(format) );
-
- if ( result == -1 ) {
- // We'll assume this is an overrun, though there isn't a
- // specific means for determining that.
- handle->xrun[1] = true;
- errorText_ = "RtApiOss::callbackEvent: audio read error.";
- error( RtAudioError::WARNING );
- goto unlock;
- }
-
- // Do byte swapping if necessary.
- if ( stream_.doByteSwap[1] )
- byteSwapBuffer( buffer, samples, format );
-
- // Do buffer conversion if necessary.
- if ( stream_.doConvertBuffer[1] )
- convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] );
- }
-
- unlock:
- MUTEX_UNLOCK( &stream_.mutex );
-
- RtApi::tickStreamTime();
- if ( doStopStream == 1 ) this->stopStream();
-}
-
-static void *ossCallbackHandler( void *ptr )
-{
- CallbackInfo *info = (CallbackInfo *) ptr;
- RtApiOss *object = (RtApiOss *) info->object;
- bool *isRunning = &info->isRunning;
-
- while ( *isRunning == true ) {
- pthread_testcancel();
- object->callbackEvent();
- }
-
- pthread_exit( NULL );
-}
-
-//******************** End of __LINUX_OSS__ *********************//
-#endif
-
-
-// *************************************************** //
-//
-// Protected common (OS-independent) RtAudio methods.
-//
-// *************************************************** //
-
-// This method can be modified to control the behavior of error
-// message printing.
-void RtApi :: error( RtAudioError::Type type )
-{
- errorStream_.str(""); // clear the ostringstream
-
- RtAudioErrorCallback errorCallback = (RtAudioErrorCallback) stream_.callbackInfo.errorCallback;
- if ( errorCallback ) {
- // abortStream() can generate new error messages. Ignore them. Just keep original one.
-
- if ( firstErrorOccurred_ )
- return;
-
- firstErrorOccurred_ = true;
- const std::string errorMessage = errorText_;
-
- if ( type != RtAudioError::WARNING && stream_.state != STREAM_STOPPED) {
- stream_.callbackInfo.isRunning = false; // exit from the thread
- abortStream();
- }
-
- errorCallback( type, errorMessage );
- firstErrorOccurred_ = false;
- return;
- }
-
- if ( type == RtAudioError::WARNING && showWarnings_ == true )
- std::cerr << '\n' << errorText_ << "\n\n";
- else if ( type != RtAudioError::WARNING )
- throw( RtAudioError( errorText_, type ) );
-}
-
-void RtApi :: verifyStream()
-{
- if ( stream_.state == STREAM_CLOSED ) {
- errorText_ = "RtApi:: a stream is not open!";
- error( RtAudioError::INVALID_USE );
- }
-}
-
-void RtApi :: clearStreamInfo()
-{
- stream_.mode = UNINITIALIZED;
- stream_.state = STREAM_CLOSED;
- stream_.sampleRate = 0;
- stream_.bufferSize = 0;
- stream_.nBuffers = 0;
- stream_.userFormat = 0;
- stream_.userInterleaved = true;
- stream_.streamTime = 0.0;
- stream_.apiHandle = 0;
- stream_.deviceBuffer = 0;
- stream_.callbackInfo.callback = 0;
- stream_.callbackInfo.userData = 0;
- stream_.callbackInfo.isRunning = false;
- stream_.callbackInfo.errorCallback = 0;
- for ( int i=0; i<2; i++ ) {
- stream_.device[i] = 11111;
- stream_.doConvertBuffer[i] = false;
- stream_.deviceInterleaved[i] = true;
- stream_.doByteSwap[i] = false;
- stream_.nUserChannels[i] = 0;
- stream_.nDeviceChannels[i] = 0;
- stream_.channelOffset[i] = 0;
- stream_.deviceFormat[i] = 0;
- stream_.latency[i] = 0;
- stream_.userBuffer[i] = 0;
- stream_.convertInfo[i].channels = 0;
- stream_.convertInfo[i].inJump = 0;
- stream_.convertInfo[i].outJump = 0;
- stream_.convertInfo[i].inFormat = 0;
- stream_.convertInfo[i].outFormat = 0;
- stream_.convertInfo[i].inOffset.clear();
- stream_.convertInfo[i].outOffset.clear();
- }
-}
-
-unsigned int RtApi :: formatBytes( RtAudioFormat format )
-{
- if ( format == RTAUDIO_SINT16 )
- return 2;
- else if ( format == RTAUDIO_SINT32 || format == RTAUDIO_FLOAT32 )
- return 4;
- else if ( format == RTAUDIO_FLOAT64 )
- return 8;
- else if ( format == RTAUDIO_SINT24 )
- return 3;
- else if ( format == RTAUDIO_SINT8 )
- return 1;
-
- errorText_ = "RtApi::formatBytes: undefined format.";
- error( RtAudioError::WARNING );
-
- return 0;
-}
-
-void RtApi :: setConvertInfo( StreamMode mode, unsigned int firstChannel )
-{
- if ( mode == INPUT ) { // convert device to user buffer
- stream_.convertInfo[mode].inJump = stream_.nDeviceChannels[1];
- stream_.convertInfo[mode].outJump = stream_.nUserChannels[1];
- stream_.convertInfo[mode].inFormat = stream_.deviceFormat[1];
- stream_.convertInfo[mode].outFormat = stream_.userFormat;
- }
- else { // convert user to device buffer
- stream_.convertInfo[mode].inJump = stream_.nUserChannels[0];
- stream_.convertInfo[mode].outJump = stream_.nDeviceChannels[0];
- stream_.convertInfo[mode].inFormat = stream_.userFormat;
- stream_.convertInfo[mode].outFormat = stream_.deviceFormat[0];
- }
-
- if ( stream_.convertInfo[mode].inJump < stream_.convertInfo[mode].outJump )
- stream_.convertInfo[mode].channels = stream_.convertInfo[mode].inJump;
- else
- stream_.convertInfo[mode].channels = stream_.convertInfo[mode].outJump;
-
- // Set up the interleave/deinterleave offsets.
- if ( stream_.deviceInterleaved[mode] != stream_.userInterleaved ) {
- if ( ( mode == OUTPUT && stream_.deviceInterleaved[mode] ) ||
- ( mode == INPUT && stream_.userInterleaved ) ) {
- for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) {
- stream_.convertInfo[mode].inOffset.push_back( k * stream_.bufferSize );
- stream_.convertInfo[mode].outOffset.push_back( k );
- stream_.convertInfo[mode].inJump = 1;
- }
- }
- else {
- for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) {
- stream_.convertInfo[mode].inOffset.push_back( k );
- stream_.convertInfo[mode].outOffset.push_back( k * stream_.bufferSize );
- stream_.convertInfo[mode].outJump = 1;
- }
- }
- }
- else { // no (de)interleaving
- if ( stream_.userInterleaved ) {
- for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) {
- stream_.convertInfo[mode].inOffset.push_back( k );
- stream_.convertInfo[mode].outOffset.push_back( k );
- }
- }
- else {
- for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) {
- stream_.convertInfo[mode].inOffset.push_back( k * stream_.bufferSize );
- stream_.convertInfo[mode].outOffset.push_back( k * stream_.bufferSize );
- stream_.convertInfo[mode].inJump = 1;
- stream_.convertInfo[mode].outJump = 1;
- }
- }
- }
-
- // Add channel offset.
- if ( firstChannel > 0 ) {
- if ( stream_.deviceInterleaved[mode] ) {
- if ( mode == OUTPUT ) {
- for ( int k=0; k<stream_.convertInfo[mode].channels; k++ )
- stream_.convertInfo[mode].outOffset[k] += firstChannel;
- }
- else {
- for ( int k=0; k<stream_.convertInfo[mode].channels; k++ )
- stream_.convertInfo[mode].inOffset[k] += firstChannel;
- }
- }
- else {
- if ( mode == OUTPUT ) {
- for ( int k=0; k<stream_.convertInfo[mode].channels; k++ )
- stream_.convertInfo[mode].outOffset[k] += ( firstChannel * stream_.bufferSize );
- }
- else {
- for ( int k=0; k<stream_.convertInfo[mode].channels; k++ )
- stream_.convertInfo[mode].inOffset[k] += ( firstChannel * stream_.bufferSize );
- }
- }
- }
-}
-
-void RtApi :: convertBuffer( char *outBuffer, char *inBuffer, ConvertInfo &info )
-{
- // This function does format conversion, input/output channel compensation, and
- // data interleaving/deinterleaving. 24-bit integers are assumed to occupy
- // the lower three bytes of a 32-bit integer.
-
- // Clear our device buffer when in/out duplex device channels are different
- if ( outBuffer == stream_.deviceBuffer && stream_.mode == DUPLEX &&
- ( stream_.nDeviceChannels[0] < stream_.nDeviceChannels[1] ) )
- memset( outBuffer, 0, stream_.bufferSize * info.outJump * formatBytes( info.outFormat ) );
-
- int j;
- if (info.outFormat == RTAUDIO_FLOAT64) {
- Float64 scale;
- Float64 *out = (Float64 *)outBuffer;
-
- if (info.inFormat == RTAUDIO_SINT8) {
- signed char *in = (signed char *)inBuffer;
- scale = 1.0 / 127.5;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float64) in[info.inOffset[j]];
- out[info.outOffset[j]] += 0.5;
- out[info.outOffset[j]] *= scale;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT16) {
- Int16 *in = (Int16 *)inBuffer;
- scale = 1.0 / 32767.5;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float64) in[info.inOffset[j]];
- out[info.outOffset[j]] += 0.5;
- out[info.outOffset[j]] *= scale;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT24) {
- Int24 *in = (Int24 *)inBuffer;
- scale = 1.0 / 8388607.5;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float64) (in[info.inOffset[j]].asInt());
- out[info.outOffset[j]] += 0.5;
- out[info.outOffset[j]] *= scale;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT32) {
- Int32 *in = (Int32 *)inBuffer;
- scale = 1.0 / 2147483647.5;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float64) in[info.inOffset[j]];
- out[info.outOffset[j]] += 0.5;
- out[info.outOffset[j]] *= scale;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT32) {
- Float32 *in = (Float32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float64) in[info.inOffset[j]];
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT64) {
- // Channel compensation and/or (de)interleaving only.
- Float64 *in = (Float64 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = in[info.inOffset[j]];
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- }
- else if (info.outFormat == RTAUDIO_FLOAT32) {
- Float32 scale;
- Float32 *out = (Float32 *)outBuffer;
-
- if (info.inFormat == RTAUDIO_SINT8) {
- signed char *in = (signed char *)inBuffer;
- scale = (Float32) ( 1.0 / 127.5 );
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float32) in[info.inOffset[j]];
- out[info.outOffset[j]] += 0.5;
- out[info.outOffset[j]] *= scale;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT16) {
- Int16 *in = (Int16 *)inBuffer;
- scale = (Float32) ( 1.0 / 32767.5 );
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float32) in[info.inOffset[j]];
- out[info.outOffset[j]] += 0.5;
- out[info.outOffset[j]] *= scale;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT24) {
- Int24 *in = (Int24 *)inBuffer;
- scale = (Float32) ( 1.0 / 8388607.5 );
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float32) (in[info.inOffset[j]].asInt());
- out[info.outOffset[j]] += 0.5;
- out[info.outOffset[j]] *= scale;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT32) {
- Int32 *in = (Int32 *)inBuffer;
- scale = (Float32) ( 1.0 / 2147483647.5 );
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float32) in[info.inOffset[j]];
- out[info.outOffset[j]] += 0.5;
- out[info.outOffset[j]] *= scale;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT32) {
- // Channel compensation and/or (de)interleaving only.
- Float32 *in = (Float32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = in[info.inOffset[j]];
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT64) {
- Float64 *in = (Float64 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Float32) in[info.inOffset[j]];
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- }
- else if (info.outFormat == RTAUDIO_SINT32) {
- Int32 *out = (Int32 *)outBuffer;
- if (info.inFormat == RTAUDIO_SINT8) {
- signed char *in = (signed char *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) in[info.inOffset[j]];
- out[info.outOffset[j]] <<= 24;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT16) {
- Int16 *in = (Int16 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) in[info.inOffset[j]];
- out[info.outOffset[j]] <<= 16;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT24) {
- Int24 *in = (Int24 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) in[info.inOffset[j]].asInt();
- out[info.outOffset[j]] <<= 8;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT32) {
- // Channel compensation and/or (de)interleaving only.
- Int32 *in = (Int32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = in[info.inOffset[j]];
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT32) {
- Float32 *in = (Float32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] * 2147483647.5 - 0.5);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT64) {
- Float64 *in = (Float64 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] * 2147483647.5 - 0.5);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- }
- else if (info.outFormat == RTAUDIO_SINT24) {
- Int24 *out = (Int24 *)outBuffer;
- if (info.inFormat == RTAUDIO_SINT8) {
- signed char *in = (signed char *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] << 16);
- //out[info.outOffset[j]] <<= 16;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT16) {
- Int16 *in = (Int16 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] << 8);
- //out[info.outOffset[j]] <<= 8;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT24) {
- // Channel compensation and/or (de)interleaving only.
- Int24 *in = (Int24 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = in[info.inOffset[j]];
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT32) {
- Int32 *in = (Int32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] >> 8);
- //out[info.outOffset[j]] >>= 8;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT32) {
- Float32 *in = (Float32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] * 8388607.5 - 0.5);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT64) {
- Float64 *in = (Float64 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] * 8388607.5 - 0.5);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- }
- else if (info.outFormat == RTAUDIO_SINT16) {
- Int16 *out = (Int16 *)outBuffer;
- if (info.inFormat == RTAUDIO_SINT8) {
- signed char *in = (signed char *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int16) in[info.inOffset[j]];
- out[info.outOffset[j]] <<= 8;
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT16) {
- // Channel compensation and/or (de)interleaving only.
- Int16 *in = (Int16 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = in[info.inOffset[j]];
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT24) {
- Int24 *in = (Int24 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int16) (in[info.inOffset[j]].asInt() >> 8);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT32) {
- Int32 *in = (Int32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int16) ((in[info.inOffset[j]] >> 16) & 0x0000ffff);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT32) {
- Float32 *in = (Float32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int16) (in[info.inOffset[j]] * 32767.5 - 0.5);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT64) {
- Float64 *in = (Float64 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (Int16) (in[info.inOffset[j]] * 32767.5 - 0.5);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- }
- else if (info.outFormat == RTAUDIO_SINT8) {
- signed char *out = (signed char *)outBuffer;
- if (info.inFormat == RTAUDIO_SINT8) {
- // Channel compensation and/or (de)interleaving only.
- signed char *in = (signed char *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = in[info.inOffset[j]];
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- if (info.inFormat == RTAUDIO_SINT16) {
- Int16 *in = (Int16 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (signed char) ((in[info.inOffset[j]] >> 8) & 0x00ff);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT24) {
- Int24 *in = (Int24 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (signed char) (in[info.inOffset[j]].asInt() >> 16);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_SINT32) {
- Int32 *in = (Int32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (signed char) ((in[info.inOffset[j]] >> 24) & 0x000000ff);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT32) {
- Float32 *in = (Float32 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (signed char) (in[info.inOffset[j]] * 127.5 - 0.5);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- else if (info.inFormat == RTAUDIO_FLOAT64) {
- Float64 *in = (Float64 *)inBuffer;
- for (unsigned int i=0; i<stream_.bufferSize; i++) {
- for (j=0; j<info.channels; j++) {
- out[info.outOffset[j]] = (signed char) (in[info.inOffset[j]] * 127.5 - 0.5);
- }
- in += info.inJump;
- out += info.outJump;
- }
- }
- }
-}
-
-//static inline uint16_t bswap_16(uint16_t x) { return (x>>8) | (x<<8); }
-//static inline uint32_t bswap_32(uint32_t x) { return (bswap_16(x&0xffff)<<16) | (bswap_16(x>>16)); }
-//static inline uint64_t bswap_64(uint64_t x) { return (((unsigned long long)bswap_32(x&0xffffffffull))<<32) | (bswap_32(x>>32)); }
-
-void RtApi :: byteSwapBuffer( char *buffer, unsigned int samples, RtAudioFormat format )
-{
- register char val;
- register char *ptr;
-
- ptr = buffer;
- if ( format == RTAUDIO_SINT16 ) {
- for ( unsigned int i=0; i<samples; i++ ) {
- // Swap 1st and 2nd bytes.
- val = *(ptr);
- *(ptr) = *(ptr+1);
- *(ptr+1) = val;
-
- // Increment 2 bytes.
- ptr += 2;
- }
- }
- else if ( format == RTAUDIO_SINT32 ||
- format == RTAUDIO_FLOAT32 ) {
- for ( unsigned int i=0; i<samples; i++ ) {
- // Swap 1st and 4th bytes.
- val = *(ptr);
- *(ptr) = *(ptr+3);
- *(ptr+3) = val;
-
- // Swap 2nd and 3rd bytes.
- ptr += 1;
- val = *(ptr);
- *(ptr) = *(ptr+1);
- *(ptr+1) = val;
-
- // Increment 3 more bytes.
- ptr += 3;
- }
- }
- else if ( format == RTAUDIO_SINT24 ) {
- for ( unsigned int i=0; i<samples; i++ ) {
- // Swap 1st and 3rd bytes.
- val = *(ptr);
- *(ptr) = *(ptr+2);
- *(ptr+2) = val;
-
- // Increment 2 more bytes.
- ptr += 2;
- }
- }
- else if ( format == RTAUDIO_FLOAT64 ) {
- for ( unsigned int i=0; i<samples; i++ ) {
- // Swap 1st and 8th bytes
- val = *(ptr);
- *(ptr) = *(ptr+7);
- *(ptr+7) = val;
-
- // Swap 2nd and 7th bytes
- ptr += 1;
- val = *(ptr);
- *(ptr) = *(ptr+5);
- *(ptr+5) = val;
-
- // Swap 3rd and 6th bytes
- ptr += 1;
- val = *(ptr);
- *(ptr) = *(ptr+3);
- *(ptr+3) = val;
-
- // Swap 4th and 5th bytes
- ptr += 1;
- val = *(ptr);
- *(ptr) = *(ptr+1);
- *(ptr+1) = val;
-
- // Increment 5 more bytes.
- ptr += 5;
- }
- }
-}
-
- // Indentation settings for Vim and Emacs
- //
- // Local Variables:
- // c-basic-offset: 2
- // indent-tabs-mode: nil
- // End:
- //
- // vim: et sts=2 sw=2
-
-#endif
+#ifdef RTAUDIO_ENABLED +/************************************************************************/ +/*! \class RtAudio + \brief Realtime audio i/o C++ classes. + + RtAudio provides a common API (Application Programming Interface) + for realtime audio input/output across Linux (native ALSA, Jack, + and OSS), Macintosh OS X (CoreAudio and Jack), and Windows + (DirectSound, ASIO and WASAPI) operating systems. + + RtAudio WWW site: http://www.music.mcgill.ca/~gary/rtaudio/ + + RtAudio: realtime audio i/o C++ classes + Copyright (c) 2001-2014 Gary P. Scavone + + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, + including without limitation the rights to use, copy, modify, merge, + publish, distribute, sublicense, and/or sell copies of the Software, + and to permit persons to whom the Software is furnished to do so, + subject to the following conditions: + + The above copyright notice and this permission notice shall be + included in all copies or substantial portions of the Software. + + Any person wishing to distribute modifications to the Software is + asked to send the modifications to the original developer so that + they can be incorporated into the canonical version. This is, + however, not a binding provision of this license. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. + IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR + ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF + CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION + WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ +/************************************************************************/ + +// RtAudio: Version 4.1.1 + +#include "RtAudio.h" +#include <iostream> +#include <cstdlib> +#include <cstring> +#include <climits> +#include <algorithm> + +// Static variable definitions. +const unsigned int RtApi::MAX_SAMPLE_RATES = 14; +const unsigned int RtApi::SAMPLE_RATES[] = { + 4000, 5512, 8000, 9600, 11025, 16000, 22050, + 32000, 44100, 48000, 88200, 96000, 176400, 192000 +}; + +#if defined(__WINDOWS_DS__) || defined(__WINDOWS_ASIO__) || defined(__WINDOWS_WASAPI__) +#ifdef WINRT_ENABLED + #define MUTEX_INITIALIZE(A) InitializeCriticalSectionEx(A, 0, 0) +#else + #define MUTEX_INITIALIZE(A) InitializeCriticalSection(A) +#endif + #define MUTEX_DESTROY(A) DeleteCriticalSection(A) + #define MUTEX_LOCK(A) EnterCriticalSection(A) + #define MUTEX_UNLOCK(A) LeaveCriticalSection(A) + + #include "tchar.h" + + static std::string convertCharPointerToStdString(const char *text) + { + return std::string(text); + } + + static std::string convertCharPointerToStdString(const wchar_t *text) + { + int length = WideCharToMultiByte(CP_UTF8, 0, text, -1, NULL, 0, NULL, NULL); + std::string s( length-1, '\0' ); + WideCharToMultiByte(CP_UTF8, 0, text, -1, &s[0], length, NULL, NULL); + return s; + } + +#elif defined(__LINUX_ALSA__) || defined(__LINUX_PULSE__) || defined(__UNIX_JACK__) || defined(__LINUX_OSS__) || defined(__MACOSX_CORE__) + // pthread API + #define MUTEX_INITIALIZE(A) pthread_mutex_init(A, NULL) + #define MUTEX_DESTROY(A) pthread_mutex_destroy(A) + #define MUTEX_LOCK(A) pthread_mutex_lock(A) + #define MUTEX_UNLOCK(A) pthread_mutex_unlock(A) +#else + #define MUTEX_INITIALIZE(A) abs(*A) // dummy definitions + #define MUTEX_DESTROY(A) abs(*A) // dummy definitions +#endif + +// *************************************************** // +// +// RtAudio definitions. +// +// *************************************************** // + +std::string RtAudio :: getVersion( void ) throw() +{ + return RTAUDIO_VERSION; +} + +void RtAudio :: getCompiledApi( std::vector<RtAudio::Api> &apis ) throw() +{ + apis.clear(); + + // The order here will control the order of RtAudio's API search in + // the constructor. +#if defined(__UNIX_JACK__) + apis.push_back( UNIX_JACK ); +#endif +#if defined(__LINUX_ALSA__) + apis.push_back( LINUX_ALSA ); +#endif +#if defined(__LINUX_PULSE__) + apis.push_back( LINUX_PULSE ); +#endif +#if defined(__LINUX_OSS__) + apis.push_back( LINUX_OSS ); +#endif +#if defined(__WINDOWS_ASIO__) + apis.push_back( WINDOWS_ASIO ); +#endif +#if defined(__WINDOWS_WASAPI__) + apis.push_back( WINDOWS_WASAPI ); +#endif +#if defined(__WINDOWS_DS__) + apis.push_back( WINDOWS_DS ); +#endif +#if defined(__MACOSX_CORE__) + apis.push_back( MACOSX_CORE ); +#endif +#if defined(__RTAUDIO_DUMMY__) + apis.push_back( RTAUDIO_DUMMY ); +#endif +} + +void RtAudio :: openRtApi( RtAudio::Api api ) +{ + if ( rtapi_ ) + delete rtapi_; + rtapi_ = 0; + +#if defined(__UNIX_JACK__) + if ( api == UNIX_JACK ) + rtapi_ = new RtApiJack(); +#endif +#if defined(__LINUX_ALSA__) + if ( api == LINUX_ALSA ) + rtapi_ = new RtApiAlsa(); +#endif +#if defined(__LINUX_PULSE__) + if ( api == LINUX_PULSE ) + rtapi_ = new RtApiPulse(); +#endif +#if defined(__LINUX_OSS__) + if ( api == LINUX_OSS ) + rtapi_ = new RtApiOss(); +#endif +#if defined(__WINDOWS_ASIO__) + if ( api == WINDOWS_ASIO ) + rtapi_ = new RtApiAsio(); +#endif +#if defined(__WINDOWS_WASAPI__) + if ( api == WINDOWS_WASAPI ) + rtapi_ = new RtApiWasapi(); +#endif +#if defined(__WINDOWS_DS__) + if ( api == WINDOWS_DS ) + rtapi_ = new RtApiDs(); +#endif +#if defined(__MACOSX_CORE__) + if ( api == MACOSX_CORE ) + rtapi_ = new RtApiCore(); +#endif +#if defined(__RTAUDIO_DUMMY__) + if ( api == RTAUDIO_DUMMY ) + rtapi_ = new RtApiDummy(); +#endif +} + +RtAudio :: RtAudio( RtAudio::Api api ) +{ + rtapi_ = 0; + + if ( api != UNSPECIFIED ) { + // Attempt to open the specified API. + openRtApi( api ); + if ( rtapi_ ) return; + + // No compiled support for specified API value. Issue a debug + // warning and continue as if no API was specified. + std::cerr << "\nRtAudio: no compiled support for specified API argument!\n" << std::endl; + } + + // Iterate through the compiled APIs and return as soon as we find + // one with at least one device or we reach the end of the list. + std::vector< RtAudio::Api > apis; + getCompiledApi( apis ); + for ( unsigned int i=0; i<apis.size(); i++ ) { + openRtApi( apis[i] ); + if ( rtapi_ && rtapi_->getDeviceCount() ) break; + } + + if ( rtapi_ ) return; + + // It should not be possible to get here because the preprocessor + // definition __RTAUDIO_DUMMY__ is automatically defined if no + // API-specific definitions are passed to the compiler. But just in + // case something weird happens, we'll thow an error. + std::string errorText = "\nRtAudio: no compiled API support found ... critical error!!\n\n"; + throw( RtAudioError( errorText, RtAudioError::UNSPECIFIED ) ); +} + +RtAudio :: ~RtAudio() throw() +{ + if ( rtapi_ ) + delete rtapi_; +} + +void RtAudio :: openStream( RtAudio::StreamParameters *outputParameters, + RtAudio::StreamParameters *inputParameters, + RtAudioFormat format, unsigned int sampleRate, + unsigned int *bufferFrames, + RtAudioCallback callback, void *userData, + RtAudio::StreamOptions *options, + RtAudioErrorCallback errorCallback ) +{ + return rtapi_->openStream( outputParameters, inputParameters, format, + sampleRate, bufferFrames, callback, + userData, options, errorCallback ); +} + +// *************************************************** // +// +// Public RtApi definitions (see end of file for +// private or protected utility functions). +// +// *************************************************** // + +RtApi :: RtApi() +{ + stream_.state = STREAM_CLOSED; + stream_.mode = UNINITIALIZED; + stream_.apiHandle = 0; + stream_.userBuffer[0] = 0; + stream_.userBuffer[1] = 0; + MUTEX_INITIALIZE( &stream_.mutex ); + showWarnings_ = true; + firstErrorOccurred_ = false; +} + +RtApi :: ~RtApi() +{ + MUTEX_DESTROY( &stream_.mutex ); +} + +void RtApi :: openStream( RtAudio::StreamParameters *oParams, + RtAudio::StreamParameters *iParams, + RtAudioFormat format, unsigned int sampleRate, + unsigned int *bufferFrames, + RtAudioCallback callback, void *userData, + RtAudio::StreamOptions *options, + RtAudioErrorCallback errorCallback ) +{ + if ( stream_.state != STREAM_CLOSED ) { + errorText_ = "RtApi::openStream: a stream is already open!"; + error( RtAudioError::INVALID_USE ); + return; + } + + // Clear stream information potentially left from a previously open stream. + clearStreamInfo(); + + if ( oParams && oParams->nChannels < 1 ) { + errorText_ = "RtApi::openStream: a non-NULL output StreamParameters structure cannot have an nChannels value less than one."; + error( RtAudioError::INVALID_USE ); + return; + } + + if ( iParams && iParams->nChannels < 1 ) { + errorText_ = "RtApi::openStream: a non-NULL input StreamParameters structure cannot have an nChannels value less than one."; + error( RtAudioError::INVALID_USE ); + return; + } + + if ( oParams == NULL && iParams == NULL ) { + errorText_ = "RtApi::openStream: input and output StreamParameters structures are both NULL!"; + error( RtAudioError::INVALID_USE ); + return; + } + + if ( formatBytes(format) == 0 ) { + errorText_ = "RtApi::openStream: 'format' parameter value is undefined."; + error( RtAudioError::INVALID_USE ); + return; + } + + unsigned int nDevices = getDeviceCount(); + unsigned int oChannels = 0; + if ( oParams ) { + oChannels = oParams->nChannels; + if ( oParams->deviceId >= nDevices ) { + errorText_ = "RtApi::openStream: output device parameter value is invalid."; + error( RtAudioError::INVALID_USE ); + return; + } + } + + unsigned int iChannels = 0; + if ( iParams ) { + iChannels = iParams->nChannels; + if ( iParams->deviceId >= nDevices ) { + errorText_ = "RtApi::openStream: input device parameter value is invalid."; + error( RtAudioError::INVALID_USE ); + return; + } + } + + bool result; + + if ( oChannels > 0 ) { + + result = probeDeviceOpen( oParams->deviceId, OUTPUT, oChannels, oParams->firstChannel, + sampleRate, format, bufferFrames, options ); + if ( result == false ) { + error( RtAudioError::SYSTEM_ERROR ); + return; + } + } + + if ( iChannels > 0 ) { + + result = probeDeviceOpen( iParams->deviceId, INPUT, iChannels, iParams->firstChannel, + sampleRate, format, bufferFrames, options ); + if ( result == false ) { + if ( oChannels > 0 ) closeStream(); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + } + + stream_.callbackInfo.callback = (void *) callback; + stream_.callbackInfo.userData = userData; + stream_.callbackInfo.errorCallback = (void *) errorCallback; + + if ( options ) options->numberOfBuffers = stream_.nBuffers; + stream_.state = STREAM_STOPPED; +} + +unsigned int RtApi :: getDefaultInputDevice( void ) +{ + // Should be implemented in subclasses if possible. + return 0; +} + +unsigned int RtApi :: getDefaultOutputDevice( void ) +{ + // Should be implemented in subclasses if possible. + return 0; +} + +void RtApi :: closeStream( void ) +{ + // MUST be implemented in subclasses! + return; +} + +bool RtApi :: probeDeviceOpen( unsigned int /*device*/, StreamMode /*mode*/, unsigned int /*channels*/, + unsigned int /*firstChannel*/, unsigned int /*sampleRate*/, + RtAudioFormat /*format*/, unsigned int * /*bufferSize*/, + RtAudio::StreamOptions * /*options*/ ) +{ + // MUST be implemented in subclasses! + return FAILURE; +} + +void RtApi :: tickStreamTime( void ) +{ + // Subclasses that do not provide their own implementation of + // getStreamTime should call this function once per buffer I/O to + // provide basic stream time support. + + stream_.streamTime += ( stream_.bufferSize * 1.0 / stream_.sampleRate ); + +#if defined( HAVE_GETTIMEOFDAY ) + gettimeofday( &stream_.lastTickTimestamp, NULL ); +#endif +} + +long RtApi :: getStreamLatency( void ) +{ + verifyStream(); + + long totalLatency = 0; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) + totalLatency = stream_.latency[0]; + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) + totalLatency += stream_.latency[1]; + + return totalLatency; +} + +double RtApi :: getStreamTime( void ) +{ + verifyStream(); + +#if defined( HAVE_GETTIMEOFDAY ) + // Return a very accurate estimate of the stream time by + // adding in the elapsed time since the last tick. + struct timeval then; + struct timeval now; + + if ( stream_.state != STREAM_RUNNING || stream_.streamTime == 0.0 ) + return stream_.streamTime; + + gettimeofday( &now, NULL ); + then = stream_.lastTickTimestamp; + return stream_.streamTime + + ((now.tv_sec + 0.000001 * now.tv_usec) - + (then.tv_sec + 0.000001 * then.tv_usec)); +#else + return stream_.streamTime; +#endif +} + +void RtApi :: setStreamTime( double time ) +{ + verifyStream(); + + if ( time >= 0.0 ) + stream_.streamTime = time; +} + +unsigned int RtApi :: getStreamSampleRate( void ) +{ + verifyStream(); + + return stream_.sampleRate; +} + + +// *************************************************** // +// +// OS/API-specific methods. +// +// *************************************************** // + +#if defined(__MACOSX_CORE__) + +// The OS X CoreAudio API is designed to use a separate callback +// procedure for each of its audio devices. A single RtAudio duplex +// stream using two different devices is supported here, though it +// cannot be guaranteed to always behave correctly because we cannot +// synchronize these two callbacks. +// +// A property listener is installed for over/underrun information. +// However, no functionality is currently provided to allow property +// listeners to trigger user handlers because it is unclear what could +// be done if a critical stream parameter (buffer size, sample rate, +// device disconnect) notification arrived. The listeners entail +// quite a bit of extra code and most likely, a user program wouldn't +// be prepared for the result anyway. However, we do provide a flag +// to the client callback function to inform of an over/underrun. + +// A structure to hold various information related to the CoreAudio API +// implementation. +struct CoreHandle { + AudioDeviceID id[2]; // device ids +#if defined( MAC_OS_X_VERSION_10_5 ) && ( MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_5 ) + AudioDeviceIOProcID procId[2]; +#endif + UInt32 iStream[2]; // device stream index (or first if using multiple) + UInt32 nStreams[2]; // number of streams to use + bool xrun[2]; + char *deviceBuffer; + pthread_cond_t condition; + int drainCounter; // Tracks callback counts when draining + bool internalDrain; // Indicates if stop is initiated from callback or not. + + CoreHandle() + :deviceBuffer(0), drainCounter(0), internalDrain(false) { nStreams[0] = 1; nStreams[1] = 1; id[0] = 0; id[1] = 0; xrun[0] = false; xrun[1] = false; } +}; + +RtApiCore:: RtApiCore() +{ +#if defined( AVAILABLE_MAC_OS_X_VERSION_10_6_AND_LATER ) + // This is a largely undocumented but absolutely necessary + // requirement starting with OS-X 10.6. If not called, queries and + // updates to various audio device properties are not handled + // correctly. + CFRunLoopRef theRunLoop = NULL; + AudioObjectPropertyAddress property = { kAudioHardwarePropertyRunLoop, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster }; + OSStatus result = AudioObjectSetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, sizeof(CFRunLoopRef), &theRunLoop); + if ( result != noErr ) { + errorText_ = "RtApiCore::RtApiCore: error setting run loop property!"; + error( RtAudioError::WARNING ); + } +#endif +} + +RtApiCore :: ~RtApiCore() +{ + // The subclass destructor gets called before the base class + // destructor, so close an existing stream before deallocating + // apiDeviceId memory. + if ( stream_.state != STREAM_CLOSED ) closeStream(); +} + +unsigned int RtApiCore :: getDeviceCount( void ) +{ + // Find out how many audio devices there are, if any. + UInt32 dataSize; + AudioObjectPropertyAddress propertyAddress = { kAudioHardwarePropertyDevices, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; + OSStatus result = AudioObjectGetPropertyDataSize( kAudioObjectSystemObject, &propertyAddress, 0, NULL, &dataSize ); + if ( result != noErr ) { + errorText_ = "RtApiCore::getDeviceCount: OS-X error getting device info!"; + error( RtAudioError::WARNING ); + return 0; + } + + return dataSize / sizeof( AudioDeviceID ); +} + +unsigned int RtApiCore :: getDefaultInputDevice( void ) +{ + unsigned int nDevices = getDeviceCount(); + if ( nDevices <= 1 ) return 0; + + AudioDeviceID id; + UInt32 dataSize = sizeof( AudioDeviceID ); + AudioObjectPropertyAddress property = { kAudioHardwarePropertyDefaultInputDevice, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; + OSStatus result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, &dataSize, &id ); + if ( result != noErr ) { + errorText_ = "RtApiCore::getDefaultInputDevice: OS-X system error getting device."; + error( RtAudioError::WARNING ); + return 0; + } + + dataSize *= nDevices; + AudioDeviceID deviceList[ nDevices ]; + property.mSelector = kAudioHardwarePropertyDevices; + result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, &dataSize, (void *) &deviceList ); + if ( result != noErr ) { + errorText_ = "RtApiCore::getDefaultInputDevice: OS-X system error getting device IDs."; + error( RtAudioError::WARNING ); + return 0; + } + + for ( unsigned int i=0; i<nDevices; i++ ) + if ( id == deviceList[i] ) return i; + + errorText_ = "RtApiCore::getDefaultInputDevice: No default device found!"; + error( RtAudioError::WARNING ); + return 0; +} + +unsigned int RtApiCore :: getDefaultOutputDevice( void ) +{ + unsigned int nDevices = getDeviceCount(); + if ( nDevices <= 1 ) return 0; + + AudioDeviceID id; + UInt32 dataSize = sizeof( AudioDeviceID ); + AudioObjectPropertyAddress property = { kAudioHardwarePropertyDefaultOutputDevice, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; + OSStatus result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, &dataSize, &id ); + if ( result != noErr ) { + errorText_ = "RtApiCore::getDefaultOutputDevice: OS-X system error getting device."; + error( RtAudioError::WARNING ); + return 0; + } + + dataSize = sizeof( AudioDeviceID ) * nDevices; + AudioDeviceID deviceList[ nDevices ]; + property.mSelector = kAudioHardwarePropertyDevices; + result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, 0, NULL, &dataSize, (void *) &deviceList ); + if ( result != noErr ) { + errorText_ = "RtApiCore::getDefaultOutputDevice: OS-X system error getting device IDs."; + error( RtAudioError::WARNING ); + return 0; + } + + for ( unsigned int i=0; i<nDevices; i++ ) + if ( id == deviceList[i] ) return i; + + errorText_ = "RtApiCore::getDefaultOutputDevice: No default device found!"; + error( RtAudioError::WARNING ); + return 0; +} + +RtAudio::DeviceInfo RtApiCore :: getDeviceInfo( unsigned int device ) +{ + RtAudio::DeviceInfo info; + info.probed = false; + + // Get device ID + unsigned int nDevices = getDeviceCount(); + if ( nDevices == 0 ) { + errorText_ = "RtApiCore::getDeviceInfo: no devices found!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + if ( device >= nDevices ) { + errorText_ = "RtApiCore::getDeviceInfo: device ID is invalid!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + AudioDeviceID deviceList[ nDevices ]; + UInt32 dataSize = sizeof( AudioDeviceID ) * nDevices; + AudioObjectPropertyAddress property = { kAudioHardwarePropertyDevices, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster }; + OSStatus result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, + 0, NULL, &dataSize, (void *) &deviceList ); + if ( result != noErr ) { + errorText_ = "RtApiCore::getDeviceInfo: OS-X system error getting device IDs."; + error( RtAudioError::WARNING ); + return info; + } + + AudioDeviceID id = deviceList[ device ]; + + // Get the device name. + info.name.erase(); + CFStringRef cfname; + dataSize = sizeof( CFStringRef ); + property.mSelector = kAudioObjectPropertyManufacturer; + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &cfname ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceInfo: system error (" << getErrorCode( result ) << ") getting device manufacturer."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + //const char *mname = CFStringGetCStringPtr( cfname, CFStringGetSystemEncoding() ); + int length = CFStringGetLength(cfname); + char *mname = (char *)malloc(length * 3 + 1); +#if defined( UNICODE ) || defined( _UNICODE ) + CFStringGetCString(cfname, mname, length * 3 + 1, kCFStringEncodingUTF8); +#else + CFStringGetCString(cfname, mname, length * 3 + 1, CFStringGetSystemEncoding()); +#endif + info.name.append( (const char *)mname, strlen(mname) ); + info.name.append( ": " ); + CFRelease( cfname ); + free(mname); + + property.mSelector = kAudioObjectPropertyName; + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &cfname ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceInfo: system error (" << getErrorCode( result ) << ") getting device name."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + //const char *name = CFStringGetCStringPtr( cfname, CFStringGetSystemEncoding() ); + length = CFStringGetLength(cfname); + char *name = (char *)malloc(length * 3 + 1); +#if defined( UNICODE ) || defined( _UNICODE ) + CFStringGetCString(cfname, name, length * 3 + 1, kCFStringEncodingUTF8); +#else + CFStringGetCString(cfname, name, length * 3 + 1, CFStringGetSystemEncoding()); +#endif + info.name.append( (const char *)name, strlen(name) ); + CFRelease( cfname ); + free(name); + + // Get the output stream "configuration". + AudioBufferList *bufferList = nil; + property.mSelector = kAudioDevicePropertyStreamConfiguration; + property.mScope = kAudioDevicePropertyScopeOutput; + // property.mElement = kAudioObjectPropertyElementWildcard; + dataSize = 0; + result = AudioObjectGetPropertyDataSize( id, &property, 0, NULL, &dataSize ); + if ( result != noErr || dataSize == 0 ) { + errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting output stream configuration info for device (" << device << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Allocate the AudioBufferList. + bufferList = (AudioBufferList *) malloc( dataSize ); + if ( bufferList == NULL ) { + errorText_ = "RtApiCore::getDeviceInfo: memory error allocating output AudioBufferList."; + error( RtAudioError::WARNING ); + return info; + } + + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, bufferList ); + if ( result != noErr || dataSize == 0 ) { + free( bufferList ); + errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting output stream configuration for device (" << device << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Get output channel information. + unsigned int i, nStreams = bufferList->mNumberBuffers; + for ( i=0; i<nStreams; i++ ) + info.outputChannels += bufferList->mBuffers[i].mNumberChannels; + free( bufferList ); + + // Get the input stream "configuration". + property.mScope = kAudioDevicePropertyScopeInput; + result = AudioObjectGetPropertyDataSize( id, &property, 0, NULL, &dataSize ); + if ( result != noErr || dataSize == 0 ) { + errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting input stream configuration info for device (" << device << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Allocate the AudioBufferList. + bufferList = (AudioBufferList *) malloc( dataSize ); + if ( bufferList == NULL ) { + errorText_ = "RtApiCore::getDeviceInfo: memory error allocating input AudioBufferList."; + error( RtAudioError::WARNING ); + return info; + } + + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, bufferList ); + if (result != noErr || dataSize == 0) { + free( bufferList ); + errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting input stream configuration for device (" << device << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Get input channel information. + nStreams = bufferList->mNumberBuffers; + for ( i=0; i<nStreams; i++ ) + info.inputChannels += bufferList->mBuffers[i].mNumberChannels; + free( bufferList ); + + // If device opens for both playback and capture, we determine the channels. + if ( info.outputChannels > 0 && info.inputChannels > 0 ) + info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels; + + // Probe the device sample rates. + bool isInput = false; + if ( info.outputChannels == 0 ) isInput = true; + + // Determine the supported sample rates. + property.mSelector = kAudioDevicePropertyAvailableNominalSampleRates; + if ( isInput == false ) property.mScope = kAudioDevicePropertyScopeOutput; + result = AudioObjectGetPropertyDataSize( id, &property, 0, NULL, &dataSize ); + if ( result != kAudioHardwareNoError || dataSize == 0 ) { + errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting sample rate info."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + UInt32 nRanges = dataSize / sizeof( AudioValueRange ); + AudioValueRange rangeList[ nRanges ]; + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &rangeList ); + if ( result != kAudioHardwareNoError ) { + errorStream_ << "RtApiCore::getDeviceInfo: system error (" << getErrorCode( result ) << ") getting sample rates."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // The sample rate reporting mechanism is a bit of a mystery. It + // seems that it can either return individual rates or a range of + // rates. I assume that if the min / max range values are the same, + // then that represents a single supported rate and if the min / max + // range values are different, the device supports an arbitrary + // range of values (though there might be multiple ranges, so we'll + // use the most conservative range). + Float64 minimumRate = 1.0, maximumRate = 10000000000.0; + bool haveValueRange = false; + info.sampleRates.clear(); + for ( UInt32 i=0; i<nRanges; i++ ) { + if ( rangeList[i].mMinimum == rangeList[i].mMaximum ) { + unsigned int tmpSr = (unsigned int) rangeList[i].mMinimum; + info.sampleRates.push_back( tmpSr ); + + if ( !info.preferredSampleRate || ( tmpSr <= 48000 && tmpSr > info.preferredSampleRate ) ) + info.preferredSampleRate = tmpSr; + + } else { + haveValueRange = true; + if ( rangeList[i].mMinimum > minimumRate ) minimumRate = rangeList[i].mMinimum; + if ( rangeList[i].mMaximum < maximumRate ) maximumRate = rangeList[i].mMaximum; + } + } + + if ( haveValueRange ) { + for ( unsigned int k=0; k<MAX_SAMPLE_RATES; k++ ) { + if ( SAMPLE_RATES[k] >= (unsigned int) minimumRate && SAMPLE_RATES[k] <= (unsigned int) maximumRate ) { + info.sampleRates.push_back( SAMPLE_RATES[k] ); + + if ( !info.preferredSampleRate || ( SAMPLE_RATES[k] <= 48000 && SAMPLE_RATES[k] > info.preferredSampleRate ) ) + info.preferredSampleRate = SAMPLE_RATES[k]; + } + } + } + + // Sort and remove any redundant values + std::sort( info.sampleRates.begin(), info.sampleRates.end() ); + info.sampleRates.erase( unique( info.sampleRates.begin(), info.sampleRates.end() ), info.sampleRates.end() ); + + if ( info.sampleRates.size() == 0 ) { + errorStream_ << "RtApiCore::probeDeviceInfo: No supported sample rates found for device (" << device << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // CoreAudio always uses 32-bit floating point data for PCM streams. + // Thus, any other "physical" formats supported by the device are of + // no interest to the client. + info.nativeFormats = RTAUDIO_FLOAT32; + + if ( info.outputChannels > 0 ) + if ( getDefaultOutputDevice() == device ) info.isDefaultOutput = true; + if ( info.inputChannels > 0 ) + if ( getDefaultInputDevice() == device ) info.isDefaultInput = true; + + info.probed = true; + return info; +} + +static OSStatus callbackHandler( AudioDeviceID inDevice, + const AudioTimeStamp* /*inNow*/, + const AudioBufferList* inInputData, + const AudioTimeStamp* /*inInputTime*/, + AudioBufferList* outOutputData, + const AudioTimeStamp* /*inOutputTime*/, + void* infoPointer ) +{ + CallbackInfo *info = (CallbackInfo *) infoPointer; + + RtApiCore *object = (RtApiCore *) info->object; + if ( object->callbackEvent( inDevice, inInputData, outOutputData ) == false ) + return kAudioHardwareUnspecifiedError; + else + return kAudioHardwareNoError; +} + +static OSStatus xrunListener( AudioObjectID /*inDevice*/, + UInt32 nAddresses, + const AudioObjectPropertyAddress properties[], + void* handlePointer ) +{ + CoreHandle *handle = (CoreHandle *) handlePointer; + for ( UInt32 i=0; i<nAddresses; i++ ) { + if ( properties[i].mSelector == kAudioDeviceProcessorOverload ) { + if ( properties[i].mScope == kAudioDevicePropertyScopeInput ) + handle->xrun[1] = true; + else + handle->xrun[0] = true; + } + } + + return kAudioHardwareNoError; +} + +static OSStatus rateListener( AudioObjectID inDevice, + UInt32 /*nAddresses*/, + const AudioObjectPropertyAddress /*properties*/[], + void* ratePointer ) +{ + Float64 *rate = (Float64 *) ratePointer; + UInt32 dataSize = sizeof( Float64 ); + AudioObjectPropertyAddress property = { kAudioDevicePropertyNominalSampleRate, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster }; + AudioObjectGetPropertyData( inDevice, &property, 0, NULL, &dataSize, rate ); + return kAudioHardwareNoError; +} + +bool RtApiCore :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels, + unsigned int firstChannel, unsigned int sampleRate, + RtAudioFormat format, unsigned int *bufferSize, + RtAudio::StreamOptions *options ) +{ + // Get device ID + unsigned int nDevices = getDeviceCount(); + if ( nDevices == 0 ) { + // This should not happen because a check is made before this function is called. + errorText_ = "RtApiCore::probeDeviceOpen: no devices found!"; + return FAILURE; + } + + if ( device >= nDevices ) { + // This should not happen because a check is made before this function is called. + errorText_ = "RtApiCore::probeDeviceOpen: device ID is invalid!"; + return FAILURE; + } + + AudioDeviceID deviceList[ nDevices ]; + UInt32 dataSize = sizeof( AudioDeviceID ) * nDevices; + AudioObjectPropertyAddress property = { kAudioHardwarePropertyDevices, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster }; + OSStatus result = AudioObjectGetPropertyData( kAudioObjectSystemObject, &property, + 0, NULL, &dataSize, (void *) &deviceList ); + if ( result != noErr ) { + errorText_ = "RtApiCore::probeDeviceOpen: OS-X system error getting device IDs."; + return FAILURE; + } + + AudioDeviceID id = deviceList[ device ]; + + // Setup for stream mode. + bool isInput = false; + if ( mode == INPUT ) { + isInput = true; + property.mScope = kAudioDevicePropertyScopeInput; + } + else + property.mScope = kAudioDevicePropertyScopeOutput; + + // Get the stream "configuration". + AudioBufferList *bufferList = nil; + dataSize = 0; + property.mSelector = kAudioDevicePropertyStreamConfiguration; + result = AudioObjectGetPropertyDataSize( id, &property, 0, NULL, &dataSize ); + if ( result != noErr || dataSize == 0 ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting stream configuration info for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Allocate the AudioBufferList. + bufferList = (AudioBufferList *) malloc( dataSize ); + if ( bufferList == NULL ) { + errorText_ = "RtApiCore::probeDeviceOpen: memory error allocating AudioBufferList."; + return FAILURE; + } + + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, bufferList ); + if (result != noErr || dataSize == 0) { + free( bufferList ); + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting stream configuration for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Search for one or more streams that contain the desired number of + // channels. CoreAudio devices can have an arbitrary number of + // streams and each stream can have an arbitrary number of channels. + // For each stream, a single buffer of interleaved samples is + // provided. RtAudio prefers the use of one stream of interleaved + // data or multiple consecutive single-channel streams. However, we + // now support multiple consecutive multi-channel streams of + // interleaved data as well. + UInt32 iStream, offsetCounter = firstChannel; + UInt32 nStreams = bufferList->mNumberBuffers; + bool monoMode = false; + bool foundStream = false; + + // First check that the device supports the requested number of + // channels. + UInt32 deviceChannels = 0; + for ( iStream=0; iStream<nStreams; iStream++ ) + deviceChannels += bufferList->mBuffers[iStream].mNumberChannels; + + if ( deviceChannels < ( channels + firstChannel ) ) { + free( bufferList ); + errorStream_ << "RtApiCore::probeDeviceOpen: the device (" << device << ") does not support the requested channel count."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Look for a single stream meeting our needs. + UInt32 firstStream, streamCount = 1, streamChannels = 0, channelOffset = 0; + for ( iStream=0; iStream<nStreams; iStream++ ) { + streamChannels = bufferList->mBuffers[iStream].mNumberChannels; + if ( streamChannels >= channels + offsetCounter ) { + firstStream = iStream; + channelOffset = offsetCounter; + foundStream = true; + break; + } + if ( streamChannels > offsetCounter ) break; + offsetCounter -= streamChannels; + } + + // If we didn't find a single stream above, then we should be able + // to meet the channel specification with multiple streams. + if ( foundStream == false ) { + monoMode = true; + offsetCounter = firstChannel; + for ( iStream=0; iStream<nStreams; iStream++ ) { + streamChannels = bufferList->mBuffers[iStream].mNumberChannels; + if ( streamChannels > offsetCounter ) break; + offsetCounter -= streamChannels; + } + + firstStream = iStream; + channelOffset = offsetCounter; + Int32 channelCounter = channels + offsetCounter - streamChannels; + + if ( streamChannels > 1 ) monoMode = false; + while ( channelCounter > 0 ) { + streamChannels = bufferList->mBuffers[++iStream].mNumberChannels; + if ( streamChannels > 1 ) monoMode = false; + channelCounter -= streamChannels; + streamCount++; + } + } + + free( bufferList ); + + // Determine the buffer size. + AudioValueRange bufferRange; + dataSize = sizeof( AudioValueRange ); + property.mSelector = kAudioDevicePropertyBufferFrameSizeRange; + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &bufferRange ); + + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting buffer size range for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + if ( bufferRange.mMinimum > *bufferSize ) *bufferSize = (unsigned long) bufferRange.mMinimum; + else if ( bufferRange.mMaximum < *bufferSize ) *bufferSize = (unsigned long) bufferRange.mMaximum; + if ( options && options->flags & RTAUDIO_MINIMIZE_LATENCY ) *bufferSize = (unsigned long) bufferRange.mMinimum; + + // Set the buffer size. For multiple streams, I'm assuming we only + // need to make this setting for the master channel. + UInt32 theSize = (UInt32) *bufferSize; + dataSize = sizeof( UInt32 ); + property.mSelector = kAudioDevicePropertyBufferFrameSize; + result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &theSize ); + + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting the buffer size for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // If attempting to setup a duplex stream, the bufferSize parameter + // MUST be the same in both directions! + *bufferSize = theSize; + if ( stream_.mode == OUTPUT && mode == INPUT && *bufferSize != stream_.bufferSize ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error setting buffer size for duplex stream on device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + stream_.bufferSize = *bufferSize; + stream_.nBuffers = 1; + + // Try to set "hog" mode ... it's not clear to me this is working. + if ( options && options->flags & RTAUDIO_HOG_DEVICE ) { + pid_t hog_pid; + dataSize = sizeof( hog_pid ); + property.mSelector = kAudioDevicePropertyHogMode; + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &hog_pid ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting 'hog' state!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + if ( hog_pid != getpid() ) { + hog_pid = getpid(); + result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &hog_pid ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting 'hog' state!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + } + + // Check and if necessary, change the sample rate for the device. + Float64 nominalRate; + dataSize = sizeof( Float64 ); + property.mSelector = kAudioDevicePropertyNominalSampleRate; + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &nominalRate ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting current sample rate."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Only change the sample rate if off by more than 1 Hz. + if ( fabs( nominalRate - (double)sampleRate ) > 1.0 ) { + + // Set a property listener for the sample rate change + Float64 reportedRate = 0.0; + AudioObjectPropertyAddress tmp = { kAudioDevicePropertyNominalSampleRate, kAudioObjectPropertyScopeGlobal, kAudioObjectPropertyElementMaster }; + result = AudioObjectAddPropertyListener( id, &tmp, rateListener, (void *) &reportedRate ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting sample rate property listener for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + nominalRate = (Float64) sampleRate; + result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &nominalRate ); + if ( result != noErr ) { + AudioObjectRemovePropertyListener( id, &tmp, rateListener, (void *) &reportedRate ); + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting sample rate for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Now wait until the reported nominal rate is what we just set. + UInt32 microCounter = 0; + while ( reportedRate != nominalRate ) { + microCounter += 5000; + if ( microCounter > 5000000 ) break; + usleep( 5000 ); + } + + // Remove the property listener. + AudioObjectRemovePropertyListener( id, &tmp, rateListener, (void *) &reportedRate ); + + if ( microCounter > 5000000 ) { + errorStream_ << "RtApiCore::probeDeviceOpen: timeout waiting for sample rate update for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + + // Now set the stream format for all streams. Also, check the + // physical format of the device and change that if necessary. + AudioStreamBasicDescription description; + dataSize = sizeof( AudioStreamBasicDescription ); + property.mSelector = kAudioStreamPropertyVirtualFormat; + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &description ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting stream format for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Set the sample rate and data format id. However, only make the + // change if the sample rate is not within 1.0 of the desired + // rate and the format is not linear pcm. + bool updateFormat = false; + if ( fabs( description.mSampleRate - (Float64)sampleRate ) > 1.0 ) { + description.mSampleRate = (Float64) sampleRate; + updateFormat = true; + } + + if ( description.mFormatID != kAudioFormatLinearPCM ) { + description.mFormatID = kAudioFormatLinearPCM; + updateFormat = true; + } + + if ( updateFormat ) { + result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &description ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting sample rate or data format for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + + // Now check the physical format. + property.mSelector = kAudioStreamPropertyPhysicalFormat; + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &description ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting stream physical format for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + //std::cout << "Current physical stream format:" << std::endl; + //std::cout << " mBitsPerChan = " << description.mBitsPerChannel << std::endl; + //std::cout << " aligned high = " << (description.mFormatFlags & kAudioFormatFlagIsAlignedHigh) << ", isPacked = " << (description.mFormatFlags & kAudioFormatFlagIsPacked) << std::endl; + //std::cout << " bytesPerFrame = " << description.mBytesPerFrame << std::endl; + //std::cout << " sample rate = " << description.mSampleRate << std::endl; + + if ( description.mFormatID != kAudioFormatLinearPCM || description.mBitsPerChannel < 16 ) { + description.mFormatID = kAudioFormatLinearPCM; + //description.mSampleRate = (Float64) sampleRate; + AudioStreamBasicDescription testDescription = description; + UInt32 formatFlags; + + // We'll try higher bit rates first and then work our way down. + std::vector< std::pair<UInt32, UInt32> > physicalFormats; + formatFlags = (description.mFormatFlags | kLinearPCMFormatFlagIsFloat) & ~kLinearPCMFormatFlagIsSignedInteger; + physicalFormats.push_back( std::pair<Float32, UInt32>( 32, formatFlags ) ); + formatFlags = (description.mFormatFlags | kLinearPCMFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked) & ~kLinearPCMFormatFlagIsFloat; + physicalFormats.push_back( std::pair<Float32, UInt32>( 32, formatFlags ) ); + physicalFormats.push_back( std::pair<Float32, UInt32>( 24, formatFlags ) ); // 24-bit packed + formatFlags &= ~( kAudioFormatFlagIsPacked | kAudioFormatFlagIsAlignedHigh ); + physicalFormats.push_back( std::pair<Float32, UInt32>( 24.2, formatFlags ) ); // 24-bit in 4 bytes, aligned low + formatFlags |= kAudioFormatFlagIsAlignedHigh; + physicalFormats.push_back( std::pair<Float32, UInt32>( 24.4, formatFlags ) ); // 24-bit in 4 bytes, aligned high + formatFlags = (description.mFormatFlags | kLinearPCMFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked) & ~kLinearPCMFormatFlagIsFloat; + physicalFormats.push_back( std::pair<Float32, UInt32>( 16, formatFlags ) ); + physicalFormats.push_back( std::pair<Float32, UInt32>( 8, formatFlags ) ); + + bool setPhysicalFormat = false; + for( unsigned int i=0; i<physicalFormats.size(); i++ ) { + testDescription = description; + testDescription.mBitsPerChannel = (UInt32) physicalFormats[i].first; + testDescription.mFormatFlags = physicalFormats[i].second; + if ( (24 == (UInt32)physicalFormats[i].first) && ~( physicalFormats[i].second & kAudioFormatFlagIsPacked ) ) + testDescription.mBytesPerFrame = 4 * testDescription.mChannelsPerFrame; + else + testDescription.mBytesPerFrame = testDescription.mBitsPerChannel/8 * testDescription.mChannelsPerFrame; + testDescription.mBytesPerPacket = testDescription.mBytesPerFrame * testDescription.mFramesPerPacket; + result = AudioObjectSetPropertyData( id, &property, 0, NULL, dataSize, &testDescription ); + if ( result == noErr ) { + setPhysicalFormat = true; + //std::cout << "Updated physical stream format:" << std::endl; + //std::cout << " mBitsPerChan = " << testDescription.mBitsPerChannel << std::endl; + //std::cout << " aligned high = " << (testDescription.mFormatFlags & kAudioFormatFlagIsAlignedHigh) << ", isPacked = " << (testDescription.mFormatFlags & kAudioFormatFlagIsPacked) << std::endl; + //std::cout << " bytesPerFrame = " << testDescription.mBytesPerFrame << std::endl; + //std::cout << " sample rate = " << testDescription.mSampleRate << std::endl; + break; + } + } + + if ( !setPhysicalFormat ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") setting physical data format for device (" << device << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + } // done setting virtual/physical formats. + + // Get the stream / device latency. + UInt32 latency; + dataSize = sizeof( UInt32 ); + property.mSelector = kAudioDevicePropertyLatency; + if ( AudioObjectHasProperty( id, &property ) == true ) { + result = AudioObjectGetPropertyData( id, &property, 0, NULL, &dataSize, &latency ); + if ( result == kAudioHardwareNoError ) stream_.latency[ mode ] = latency; + else { + errorStream_ << "RtApiCore::probeDeviceOpen: system error (" << getErrorCode( result ) << ") getting device latency for device (" << device << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + } + } + + // Byte-swapping: According to AudioHardware.h, the stream data will + // always be presented in native-endian format, so we should never + // need to byte swap. + stream_.doByteSwap[mode] = false; + + // From the CoreAudio documentation, PCM data must be supplied as + // 32-bit floats. + stream_.userFormat = format; + stream_.deviceFormat[mode] = RTAUDIO_FLOAT32; + + if ( streamCount == 1 ) + stream_.nDeviceChannels[mode] = description.mChannelsPerFrame; + else // multiple streams + stream_.nDeviceChannels[mode] = channels; + stream_.nUserChannels[mode] = channels; + stream_.channelOffset[mode] = channelOffset; // offset within a CoreAudio stream + if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false; + else stream_.userInterleaved = true; + stream_.deviceInterleaved[mode] = true; + if ( monoMode == true ) stream_.deviceInterleaved[mode] = false; + + // Set flags for buffer conversion. + stream_.doConvertBuffer[mode] = false; + if ( stream_.userFormat != stream_.deviceFormat[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( stream_.nUserChannels[mode] < stream_.nDeviceChannels[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( streamCount == 1 ) { + if ( stream_.nUserChannels[mode] > 1 && + stream_.userInterleaved != stream_.deviceInterleaved[mode] ) + stream_.doConvertBuffer[mode] = true; + } + else if ( monoMode && stream_.userInterleaved ) + stream_.doConvertBuffer[mode] = true; + + // Allocate our CoreHandle structure for the stream. + CoreHandle *handle = 0; + if ( stream_.apiHandle == 0 ) { + try { + handle = new CoreHandle; + } + catch ( std::bad_alloc& ) { + errorText_ = "RtApiCore::probeDeviceOpen: error allocating CoreHandle memory."; + goto error; + } + + if ( pthread_cond_init( &handle->condition, NULL ) ) { + errorText_ = "RtApiCore::probeDeviceOpen: error initializing pthread condition variable."; + goto error; + } + stream_.apiHandle = (void *) handle; + } + else + handle = (CoreHandle *) stream_.apiHandle; + handle->iStream[mode] = firstStream; + handle->nStreams[mode] = streamCount; + handle->id[mode] = id; + + // Allocate necessary internal buffers. + unsigned long bufferBytes; + bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat ); + // stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 ); + stream_.userBuffer[mode] = (char *) malloc( bufferBytes * sizeof(char) ); + memset( stream_.userBuffer[mode], 0, bufferBytes * sizeof(char) ); + if ( stream_.userBuffer[mode] == NULL ) { + errorText_ = "RtApiCore::probeDeviceOpen: error allocating user buffer memory."; + goto error; + } + + // If possible, we will make use of the CoreAudio stream buffers as + // "device buffers". However, we can't do this if using multiple + // streams. + if ( stream_.doConvertBuffer[mode] && handle->nStreams[mode] > 1 ) { + + bool makeBuffer = true; + bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] ); + if ( mode == INPUT ) { + if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) { + unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] ); + if ( bufferBytes <= bytesOut ) makeBuffer = false; + } + } + + if ( makeBuffer ) { + bufferBytes *= *bufferSize; + if ( stream_.deviceBuffer ) free( stream_.deviceBuffer ); + stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 ); + if ( stream_.deviceBuffer == NULL ) { + errorText_ = "RtApiCore::probeDeviceOpen: error allocating device buffer memory."; + goto error; + } + } + } + + stream_.sampleRate = sampleRate; + stream_.device[mode] = device; + stream_.state = STREAM_STOPPED; + stream_.callbackInfo.object = (void *) this; + + // Setup the buffer conversion information structure. + if ( stream_.doConvertBuffer[mode] ) { + if ( streamCount > 1 ) setConvertInfo( mode, 0 ); + else setConvertInfo( mode, channelOffset ); + } + + if ( mode == INPUT && stream_.mode == OUTPUT && stream_.device[0] == device ) + // Only one callback procedure per device. + stream_.mode = DUPLEX; + else { +#if defined( MAC_OS_X_VERSION_10_5 ) && ( MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_5 ) + result = AudioDeviceCreateIOProcID( id, callbackHandler, (void *) &stream_.callbackInfo, &handle->procId[mode] ); +#else + // deprecated in favor of AudioDeviceCreateIOProcID() + result = AudioDeviceAddIOProc( id, callbackHandler, (void *) &stream_.callbackInfo ); +#endif + if ( result != noErr ) { + errorStream_ << "RtApiCore::probeDeviceOpen: system error setting callback for device (" << device << ")."; + errorText_ = errorStream_.str(); + goto error; + } + if ( stream_.mode == OUTPUT && mode == INPUT ) + stream_.mode = DUPLEX; + else + stream_.mode = mode; + } + + // Setup the device property listener for over/underload. + property.mSelector = kAudioDeviceProcessorOverload; + property.mScope = kAudioObjectPropertyScopeGlobal; + result = AudioObjectAddPropertyListener( id, &property, xrunListener, (void *) handle ); + + return SUCCESS; + + error: + if ( handle ) { + pthread_cond_destroy( &handle->condition ); + delete handle; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + stream_.state = STREAM_CLOSED; + return FAILURE; +} + +void RtApiCore :: closeStream( void ) +{ + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiCore::closeStream(): no open stream to close!"; + error( RtAudioError::WARNING ); + return; + } + + CoreHandle *handle = (CoreHandle *) stream_.apiHandle; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + if (handle) { + AudioObjectPropertyAddress property = { kAudioHardwarePropertyDevices, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster }; + + property.mSelector = kAudioDeviceProcessorOverload; + property.mScope = kAudioObjectPropertyScopeGlobal; + if (AudioObjectRemovePropertyListener( handle->id[0], &property, xrunListener, (void *) handle ) != noErr) { + errorText_ = "RtApiCore::closeStream(): error removing property listener!"; + error( RtAudioError::WARNING ); + } + } + if ( stream_.state == STREAM_RUNNING ) + AudioDeviceStop( handle->id[0], callbackHandler ); +#if defined( MAC_OS_X_VERSION_10_5 ) && ( MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_5 ) + AudioDeviceDestroyIOProcID( handle->id[0], handle->procId[0] ); +#else + // deprecated in favor of AudioDeviceDestroyIOProcID() + AudioDeviceRemoveIOProc( handle->id[0], callbackHandler ); +#endif + } + + if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && stream_.device[0] != stream_.device[1] ) ) { + if (handle) { + AudioObjectPropertyAddress property = { kAudioHardwarePropertyDevices, + kAudioObjectPropertyScopeGlobal, + kAudioObjectPropertyElementMaster }; + + property.mSelector = kAudioDeviceProcessorOverload; + property.mScope = kAudioObjectPropertyScopeGlobal; + if (AudioObjectRemovePropertyListener( handle->id[1], &property, xrunListener, (void *) handle ) != noErr) { + errorText_ = "RtApiCore::closeStream(): error removing property listener!"; + error( RtAudioError::WARNING ); + } + } + if ( stream_.state == STREAM_RUNNING ) + AudioDeviceStop( handle->id[1], callbackHandler ); +#if defined( MAC_OS_X_VERSION_10_5 ) && ( MAC_OS_X_VERSION_MIN_REQUIRED >= MAC_OS_X_VERSION_10_5 ) + AudioDeviceDestroyIOProcID( handle->id[1], handle->procId[1] ); +#else + // deprecated in favor of AudioDeviceDestroyIOProcID() + AudioDeviceRemoveIOProc( handle->id[1], callbackHandler ); +#endif + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + // Destroy pthread condition variable. + pthread_cond_destroy( &handle->condition ); + delete handle; + stream_.apiHandle = 0; + + stream_.mode = UNINITIALIZED; + stream_.state = STREAM_CLOSED; +} + +void RtApiCore :: startStream( void ) +{ + verifyStream(); + if ( stream_.state == STREAM_RUNNING ) { + errorText_ = "RtApiCore::startStream(): the stream is already running!"; + error( RtAudioError::WARNING ); + return; + } + + OSStatus result = noErr; + CoreHandle *handle = (CoreHandle *) stream_.apiHandle; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + result = AudioDeviceStart( handle->id[0], callbackHandler ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::startStream: system error (" << getErrorCode( result ) << ") starting callback procedure on device (" << stream_.device[0] << ")."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + if ( stream_.mode == INPUT || + ( stream_.mode == DUPLEX && stream_.device[0] != stream_.device[1] ) ) { + + result = AudioDeviceStart( handle->id[1], callbackHandler ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::startStream: system error starting input callback procedure on device (" << stream_.device[1] << ")."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + handle->drainCounter = 0; + handle->internalDrain = false; + stream_.state = STREAM_RUNNING; + + unlock: + if ( result == noErr ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiCore :: stopStream( void ) +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiCore::stopStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + OSStatus result = noErr; + CoreHandle *handle = (CoreHandle *) stream_.apiHandle; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + if ( handle->drainCounter == 0 ) { + handle->drainCounter = 2; + pthread_cond_wait( &handle->condition, &stream_.mutex ); // block until signaled + } + + result = AudioDeviceStop( handle->id[0], callbackHandler ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::stopStream: system error (" << getErrorCode( result ) << ") stopping callback procedure on device (" << stream_.device[0] << ")."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && stream_.device[0] != stream_.device[1] ) ) { + + result = AudioDeviceStop( handle->id[1], callbackHandler ); + if ( result != noErr ) { + errorStream_ << "RtApiCore::stopStream: system error (" << getErrorCode( result ) << ") stopping input callback procedure on device (" << stream_.device[1] << ")."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + stream_.state = STREAM_STOPPED; + + unlock: + if ( result == noErr ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiCore :: abortStream( void ) +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiCore::abortStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + CoreHandle *handle = (CoreHandle *) stream_.apiHandle; + handle->drainCounter = 2; + + stopStream(); +} + +// This function will be called by a spawned thread when the user +// callback function signals that the stream should be stopped or +// aborted. It is better to handle it this way because the +// callbackEvent() function probably should return before the AudioDeviceStop() +// function is called. +static void *coreStopStream( void *ptr ) +{ + CallbackInfo *info = (CallbackInfo *) ptr; + RtApiCore *object = (RtApiCore *) info->object; + + object->stopStream(); + pthread_exit( NULL ); +} + +bool RtApiCore :: callbackEvent( AudioDeviceID deviceId, + const AudioBufferList *inBufferList, + const AudioBufferList *outBufferList ) +{ + if ( stream_.state == STREAM_STOPPED || stream_.state == STREAM_STOPPING ) return SUCCESS; + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiCore::callbackEvent(): the stream is closed ... this shouldn't happen!"; + error( RtAudioError::WARNING ); + return FAILURE; + } + + CallbackInfo *info = (CallbackInfo *) &stream_.callbackInfo; + CoreHandle *handle = (CoreHandle *) stream_.apiHandle; + + // Check if we were draining the stream and signal is finished. + if ( handle->drainCounter > 3 ) { + ThreadHandle threadId; + + stream_.state = STREAM_STOPPING; + if ( handle->internalDrain == true ) + pthread_create( &threadId, NULL, coreStopStream, info ); + else // external call to stopStream() + pthread_cond_signal( &handle->condition ); + return SUCCESS; + } + + AudioDeviceID outputDevice = handle->id[0]; + + // Invoke user callback to get fresh output data UNLESS we are + // draining stream or duplex mode AND the input/output devices are + // different AND this function is called for the input device. + if ( handle->drainCounter == 0 && ( stream_.mode != DUPLEX || deviceId == outputDevice ) ) { + RtAudioCallback callback = (RtAudioCallback) info->callback; + double streamTime = getStreamTime(); + RtAudioStreamStatus status = 0; + if ( stream_.mode != INPUT && handle->xrun[0] == true ) { + status |= RTAUDIO_OUTPUT_UNDERFLOW; + handle->xrun[0] = false; + } + if ( stream_.mode != OUTPUT && handle->xrun[1] == true ) { + status |= RTAUDIO_INPUT_OVERFLOW; + handle->xrun[1] = false; + } + + int cbReturnValue = callback( stream_.userBuffer[0], stream_.userBuffer[1], + stream_.bufferSize, streamTime, status, info->userData ); + if ( cbReturnValue == 2 ) { + stream_.state = STREAM_STOPPING; + handle->drainCounter = 2; + abortStream(); + return SUCCESS; + } + else if ( cbReturnValue == 1 ) { + handle->drainCounter = 1; + handle->internalDrain = true; + } + } + + if ( stream_.mode == OUTPUT || ( stream_.mode == DUPLEX && deviceId == outputDevice ) ) { + + if ( handle->drainCounter > 1 ) { // write zeros to the output stream + + if ( handle->nStreams[0] == 1 ) { + memset( outBufferList->mBuffers[handle->iStream[0]].mData, + 0, + outBufferList->mBuffers[handle->iStream[0]].mDataByteSize ); + } + else { // fill multiple streams with zeros + for ( unsigned int i=0; i<handle->nStreams[0]; i++ ) { + memset( outBufferList->mBuffers[handle->iStream[0]+i].mData, + 0, + outBufferList->mBuffers[handle->iStream[0]+i].mDataByteSize ); + } + } + } + else if ( handle->nStreams[0] == 1 ) { + if ( stream_.doConvertBuffer[0] ) { // convert directly to CoreAudio stream buffer + convertBuffer( (char *) outBufferList->mBuffers[handle->iStream[0]].mData, + stream_.userBuffer[0], stream_.convertInfo[0] ); + } + else { // copy from user buffer + memcpy( outBufferList->mBuffers[handle->iStream[0]].mData, + stream_.userBuffer[0], + outBufferList->mBuffers[handle->iStream[0]].mDataByteSize ); + } + } + else { // fill multiple streams + Float32 *inBuffer = (Float32 *) stream_.userBuffer[0]; + if ( stream_.doConvertBuffer[0] ) { + convertBuffer( stream_.deviceBuffer, stream_.userBuffer[0], stream_.convertInfo[0] ); + inBuffer = (Float32 *) stream_.deviceBuffer; + } + + if ( stream_.deviceInterleaved[0] == false ) { // mono mode + UInt32 bufferBytes = outBufferList->mBuffers[handle->iStream[0]].mDataByteSize; + for ( unsigned int i=0; i<stream_.nUserChannels[0]; i++ ) { + memcpy( outBufferList->mBuffers[handle->iStream[0]+i].mData, + (void *)&inBuffer[i*stream_.bufferSize], bufferBytes ); + } + } + else { // fill multiple multi-channel streams with interleaved data + UInt32 streamChannels, channelsLeft, inJump, outJump, inOffset; + Float32 *out, *in; + + bool inInterleaved = ( stream_.userInterleaved ) ? true : false; + UInt32 inChannels = stream_.nUserChannels[0]; + if ( stream_.doConvertBuffer[0] ) { + inInterleaved = true; // device buffer will always be interleaved for nStreams > 1 and not mono mode + inChannels = stream_.nDeviceChannels[0]; + } + + if ( inInterleaved ) inOffset = 1; + else inOffset = stream_.bufferSize; + + channelsLeft = inChannels; + for ( unsigned int i=0; i<handle->nStreams[0]; i++ ) { + in = inBuffer; + out = (Float32 *) outBufferList->mBuffers[handle->iStream[0]+i].mData; + streamChannels = outBufferList->mBuffers[handle->iStream[0]+i].mNumberChannels; + + outJump = 0; + // Account for possible channel offset in first stream + if ( i == 0 && stream_.channelOffset[0] > 0 ) { + streamChannels -= stream_.channelOffset[0]; + outJump = stream_.channelOffset[0]; + out += outJump; + } + + // Account for possible unfilled channels at end of the last stream + if ( streamChannels > channelsLeft ) { + outJump = streamChannels - channelsLeft; + streamChannels = channelsLeft; + } + + // Determine input buffer offsets and skips + if ( inInterleaved ) { + inJump = inChannels; + in += inChannels - channelsLeft; + } + else { + inJump = 1; + in += (inChannels - channelsLeft) * inOffset; + } + + for ( unsigned int i=0; i<stream_.bufferSize; i++ ) { + for ( unsigned int j=0; j<streamChannels; j++ ) { + *out++ = in[j*inOffset]; + } + out += outJump; + in += inJump; + } + channelsLeft -= streamChannels; + } + } + } + } + + // Don't bother draining input + if ( handle->drainCounter ) { + handle->drainCounter++; + goto unlock; + } + + AudioDeviceID inputDevice; + inputDevice = handle->id[1]; + if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && deviceId == inputDevice ) ) { + + if ( handle->nStreams[1] == 1 ) { + if ( stream_.doConvertBuffer[1] ) { // convert directly from CoreAudio stream buffer + convertBuffer( stream_.userBuffer[1], + (char *) inBufferList->mBuffers[handle->iStream[1]].mData, + stream_.convertInfo[1] ); + } + else { // copy to user buffer + memcpy( stream_.userBuffer[1], + inBufferList->mBuffers[handle->iStream[1]].mData, + inBufferList->mBuffers[handle->iStream[1]].mDataByteSize ); + } + } + else { // read from multiple streams + Float32 *outBuffer = (Float32 *) stream_.userBuffer[1]; + if ( stream_.doConvertBuffer[1] ) outBuffer = (Float32 *) stream_.deviceBuffer; + + if ( stream_.deviceInterleaved[1] == false ) { // mono mode + UInt32 bufferBytes = inBufferList->mBuffers[handle->iStream[1]].mDataByteSize; + for ( unsigned int i=0; i<stream_.nUserChannels[1]; i++ ) { + memcpy( (void *)&outBuffer[i*stream_.bufferSize], + inBufferList->mBuffers[handle->iStream[1]+i].mData, bufferBytes ); + } + } + else { // read from multiple multi-channel streams + UInt32 streamChannels, channelsLeft, inJump, outJump, outOffset; + Float32 *out, *in; + + bool outInterleaved = ( stream_.userInterleaved ) ? true : false; + UInt32 outChannels = stream_.nUserChannels[1]; + if ( stream_.doConvertBuffer[1] ) { + outInterleaved = true; // device buffer will always be interleaved for nStreams > 1 and not mono mode + outChannels = stream_.nDeviceChannels[1]; + } + + if ( outInterleaved ) outOffset = 1; + else outOffset = stream_.bufferSize; + + channelsLeft = outChannels; + for ( unsigned int i=0; i<handle->nStreams[1]; i++ ) { + out = outBuffer; + in = (Float32 *) inBufferList->mBuffers[handle->iStream[1]+i].mData; + streamChannels = inBufferList->mBuffers[handle->iStream[1]+i].mNumberChannels; + + inJump = 0; + // Account for possible channel offset in first stream + if ( i == 0 && stream_.channelOffset[1] > 0 ) { + streamChannels -= stream_.channelOffset[1]; + inJump = stream_.channelOffset[1]; + in += inJump; + } + + // Account for possible unread channels at end of the last stream + if ( streamChannels > channelsLeft ) { + inJump = streamChannels - channelsLeft; + streamChannels = channelsLeft; + } + + // Determine output buffer offsets and skips + if ( outInterleaved ) { + outJump = outChannels; + out += outChannels - channelsLeft; + } + else { + outJump = 1; + out += (outChannels - channelsLeft) * outOffset; + } + + for ( unsigned int i=0; i<stream_.bufferSize; i++ ) { + for ( unsigned int j=0; j<streamChannels; j++ ) { + out[j*outOffset] = *in++; + } + out += outJump; + in += inJump; + } + channelsLeft -= streamChannels; + } + } + + if ( stream_.doConvertBuffer[1] ) { // convert from our internal "device" buffer + convertBuffer( stream_.userBuffer[1], + stream_.deviceBuffer, + stream_.convertInfo[1] ); + } + } + } + + unlock: + //MUTEX_UNLOCK( &stream_.mutex ); + + RtApi::tickStreamTime(); + return SUCCESS; +} + +const char* RtApiCore :: getErrorCode( OSStatus code ) +{ + switch( code ) { + + case kAudioHardwareNotRunningError: + return "kAudioHardwareNotRunningError"; + + case kAudioHardwareUnspecifiedError: + return "kAudioHardwareUnspecifiedError"; + + case kAudioHardwareUnknownPropertyError: + return "kAudioHardwareUnknownPropertyError"; + + case kAudioHardwareBadPropertySizeError: + return "kAudioHardwareBadPropertySizeError"; + + case kAudioHardwareIllegalOperationError: + return "kAudioHardwareIllegalOperationError"; + + case kAudioHardwareBadObjectError: + return "kAudioHardwareBadObjectError"; + + case kAudioHardwareBadDeviceError: + return "kAudioHardwareBadDeviceError"; + + case kAudioHardwareBadStreamError: + return "kAudioHardwareBadStreamError"; + + case kAudioHardwareUnsupportedOperationError: + return "kAudioHardwareUnsupportedOperationError"; + + case kAudioDeviceUnsupportedFormatError: + return "kAudioDeviceUnsupportedFormatError"; + + case kAudioDevicePermissionsError: + return "kAudioDevicePermissionsError"; + + default: + return "CoreAudio unknown error"; + } +} + + //******************** End of __MACOSX_CORE__ *********************// +#endif + +#if defined(__UNIX_JACK__) + +// JACK is a low-latency audio server, originally written for the +// GNU/Linux operating system and now also ported to OS-X. It can +// connect a number of different applications to an audio device, as +// well as allowing them to share audio between themselves. +// +// When using JACK with RtAudio, "devices" refer to JACK clients that +// have ports connected to the server. The JACK server is typically +// started in a terminal as follows: +// +// .jackd -d alsa -d hw:0 +// +// or through an interface program such as qjackctl. Many of the +// parameters normally set for a stream are fixed by the JACK server +// and can be specified when the JACK server is started. In +// particular, +// +// .jackd -d alsa -d hw:0 -r 44100 -p 512 -n 4 +// +// specifies a sample rate of 44100 Hz, a buffer size of 512 sample +// frames, and number of buffers = 4. Once the server is running, it +// is not possible to override these values. If the values are not +// specified in the command-line, the JACK server uses default values. +// +// The JACK server does not have to be running when an instance of +// RtApiJack is created, though the function getDeviceCount() will +// report 0 devices found until JACK has been started. When no +// devices are available (i.e., the JACK server is not running), a +// stream cannot be opened. + +#include <jack/jack.h> +#include <unistd.h> +#include <cstdio> + +// A structure to hold various information related to the Jack API +// implementation. +struct JackHandle { + jack_client_t *client; + jack_port_t **ports[2]; + std::string deviceName[2]; + bool xrun[2]; + pthread_cond_t condition; + int drainCounter; // Tracks callback counts when draining + bool internalDrain; // Indicates if stop is initiated from callback or not. + + JackHandle() + :client(0), drainCounter(0), internalDrain(false) { ports[0] = 0; ports[1] = 0; xrun[0] = false; xrun[1] = false; } +}; + +static void jackSilentError( const char * ) {}; + +RtApiJack :: RtApiJack() +{ + // Nothing to do here. +#if !defined(__RTAUDIO_DEBUG__) + // Turn off Jack's internal error reporting. + jack_set_error_function( &jackSilentError ); +#endif +} + +RtApiJack :: ~RtApiJack() +{ + if ( stream_.state != STREAM_CLOSED ) closeStream(); +} + +unsigned int RtApiJack :: getDeviceCount( void ) +{ + // See if we can become a jack client. + jack_options_t options = (jack_options_t) ( JackNoStartServer ); //JackNullOption; + jack_status_t *status = NULL; + jack_client_t *client = jack_client_open( "RtApiJackCount", options, status ); + if ( client == 0 ) return 0; + + const char **ports; + std::string port, previousPort; + unsigned int nChannels = 0, nDevices = 0; + ports = jack_get_ports( client, NULL, NULL, 0 ); + if ( ports ) { + // Parse the port names up to the first colon (:). + size_t iColon = 0; + do { + port = (char *) ports[ nChannels ]; + iColon = port.find(":"); + if ( iColon != std::string::npos ) { + port = port.substr( 0, iColon + 1 ); + if ( port != previousPort ) { + nDevices++; + previousPort = port; + } + } + } while ( ports[++nChannels] ); + free( ports ); + } + + jack_client_close( client ); + return nDevices; +} + +RtAudio::DeviceInfo RtApiJack :: getDeviceInfo( unsigned int device ) +{ + RtAudio::DeviceInfo info; + info.probed = false; + + jack_options_t options = (jack_options_t) ( JackNoStartServer ); //JackNullOption + jack_status_t *status = NULL; + jack_client_t *client = jack_client_open( "RtApiJackInfo", options, status ); + if ( client == 0 ) { + errorText_ = "RtApiJack::getDeviceInfo: Jack server not found or connection error!"; + error( RtAudioError::WARNING ); + return info; + } + + const char **ports; + std::string port, previousPort; + unsigned int nPorts = 0, nDevices = 0; + ports = jack_get_ports( client, NULL, NULL, 0 ); + if ( ports ) { + // Parse the port names up to the first colon (:). + size_t iColon = 0; + do { + port = (char *) ports[ nPorts ]; + iColon = port.find(":"); + if ( iColon != std::string::npos ) { + port = port.substr( 0, iColon ); + if ( port != previousPort ) { + if ( nDevices == device ) info.name = port; + nDevices++; + previousPort = port; + } + } + } while ( ports[++nPorts] ); + free( ports ); + } + + if ( device >= nDevices ) { + jack_client_close( client ); + errorText_ = "RtApiJack::getDeviceInfo: device ID is invalid!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + // Get the current jack server sample rate. + info.sampleRates.clear(); + + info.preferredSampleRate = jack_get_sample_rate( client ); + info.sampleRates.push_back( info.preferredSampleRate ); + + // Count the available ports containing the client name as device + // channels. Jack "input ports" equal RtAudio output channels. + unsigned int nChannels = 0; + ports = jack_get_ports( client, info.name.c_str(), NULL, JackPortIsInput ); + if ( ports ) { + while ( ports[ nChannels ] ) nChannels++; + free( ports ); + info.outputChannels = nChannels; + } + + // Jack "output ports" equal RtAudio input channels. + nChannels = 0; + ports = jack_get_ports( client, info.name.c_str(), NULL, JackPortIsOutput ); + if ( ports ) { + while ( ports[ nChannels ] ) nChannels++; + free( ports ); + info.inputChannels = nChannels; + } + + if ( info.outputChannels == 0 && info.inputChannels == 0 ) { + jack_client_close(client); + errorText_ = "RtApiJack::getDeviceInfo: error determining Jack input/output channels!"; + error( RtAudioError::WARNING ); + return info; + } + + // If device opens for both playback and capture, we determine the channels. + if ( info.outputChannels > 0 && info.inputChannels > 0 ) + info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels; + + // Jack always uses 32-bit floats. + info.nativeFormats = RTAUDIO_FLOAT32; + + // Jack doesn't provide default devices so we'll use the first available one. + if ( device == 0 && info.outputChannels > 0 ) + info.isDefaultOutput = true; + if ( device == 0 && info.inputChannels > 0 ) + info.isDefaultInput = true; + + jack_client_close(client); + info.probed = true; + return info; +} + +static int jackCallbackHandler( jack_nframes_t nframes, void *infoPointer ) +{ + CallbackInfo *info = (CallbackInfo *) infoPointer; + + RtApiJack *object = (RtApiJack *) info->object; + if ( object->callbackEvent( (unsigned long) nframes ) == false ) return 1; + + return 0; +} + +// This function will be called by a spawned thread when the Jack +// server signals that it is shutting down. It is necessary to handle +// it this way because the jackShutdown() function must return before +// the jack_deactivate() function (in closeStream()) will return. +static void *jackCloseStream( void *ptr ) +{ + CallbackInfo *info = (CallbackInfo *) ptr; + RtApiJack *object = (RtApiJack *) info->object; + + object->closeStream(); + + pthread_exit( NULL ); +} +static void jackShutdown( void *infoPointer ) +{ + CallbackInfo *info = (CallbackInfo *) infoPointer; + RtApiJack *object = (RtApiJack *) info->object; + + // Check current stream state. If stopped, then we'll assume this + // was called as a result of a call to RtApiJack::stopStream (the + // deactivation of a client handle causes this function to be called). + // If not, we'll assume the Jack server is shutting down or some + // other problem occurred and we should close the stream. + if ( object->isStreamRunning() == false ) return; + + ThreadHandle threadId; + pthread_create( &threadId, NULL, jackCloseStream, info ); + std::cerr << "\nRtApiJack: the Jack server is shutting down this client ... stream stopped and closed!!\n" << std::endl; +} + +static int jackXrun( void *infoPointer ) +{ + JackHandle *handle = (JackHandle *) infoPointer; + + if ( handle->ports[0] ) handle->xrun[0] = true; + if ( handle->ports[1] ) handle->xrun[1] = true; + + return 0; +} + +bool RtApiJack :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels, + unsigned int firstChannel, unsigned int sampleRate, + RtAudioFormat format, unsigned int *bufferSize, + RtAudio::StreamOptions *options ) +{ + JackHandle *handle = (JackHandle *) stream_.apiHandle; + + // Look for jack server and try to become a client (only do once per stream). + jack_client_t *client = 0; + if ( mode == OUTPUT || ( mode == INPUT && stream_.mode != OUTPUT ) ) { + jack_options_t jackoptions = (jack_options_t) ( JackNoStartServer ); //JackNullOption; + jack_status_t *status = NULL; + if ( options && !options->streamName.empty() ) + client = jack_client_open( options->streamName.c_str(), jackoptions, status ); + else + client = jack_client_open( "RtApiJack", jackoptions, status ); + if ( client == 0 ) { + errorText_ = "RtApiJack::probeDeviceOpen: Jack server not found or connection error!"; + error( RtAudioError::WARNING ); + return FAILURE; + } + } + else { + // The handle must have been created on an earlier pass. + client = handle->client; + } + + const char **ports; + std::string port, previousPort, deviceName; + unsigned int nPorts = 0, nDevices = 0; + ports = jack_get_ports( client, NULL, NULL, 0 ); + if ( ports ) { + // Parse the port names up to the first colon (:). + size_t iColon = 0; + do { + port = (char *) ports[ nPorts ]; + iColon = port.find(":"); + if ( iColon != std::string::npos ) { + port = port.substr( 0, iColon ); + if ( port != previousPort ) { + if ( nDevices == device ) deviceName = port; + nDevices++; + previousPort = port; + } + } + } while ( ports[++nPorts] ); + free( ports ); + } + + if ( device >= nDevices ) { + errorText_ = "RtApiJack::probeDeviceOpen: device ID is invalid!"; + return FAILURE; + } + + // Count the available ports containing the client name as device + // channels. Jack "input ports" equal RtAudio output channels. + unsigned int nChannels = 0; + unsigned long flag = JackPortIsInput; + if ( mode == INPUT ) flag = JackPortIsOutput; + ports = jack_get_ports( client, deviceName.c_str(), NULL, flag ); + if ( ports ) { + while ( ports[ nChannels ] ) nChannels++; + free( ports ); + } + + // Compare the jack ports for specified client to the requested number of channels. + if ( nChannels < (channels + firstChannel) ) { + errorStream_ << "RtApiJack::probeDeviceOpen: requested number of channels (" << channels << ") + offset (" << firstChannel << ") not found for specified device (" << device << ":" << deviceName << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Check the jack server sample rate. + unsigned int jackRate = jack_get_sample_rate( client ); + if ( sampleRate != jackRate ) { + jack_client_close( client ); + errorStream_ << "RtApiJack::probeDeviceOpen: the requested sample rate (" << sampleRate << ") is different than the JACK server rate (" << jackRate << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + stream_.sampleRate = jackRate; + + // Get the latency of the JACK port. + ports = jack_get_ports( client, deviceName.c_str(), NULL, flag ); + if ( ports[ firstChannel ] ) { + // Added by Ge Wang + jack_latency_callback_mode_t cbmode = (mode == INPUT ? JackCaptureLatency : JackPlaybackLatency); + // the range (usually the min and max are equal) + jack_latency_range_t latrange; latrange.min = latrange.max = 0; + // get the latency range + jack_port_get_latency_range( jack_port_by_name( client, ports[firstChannel] ), cbmode, &latrange ); + // be optimistic, use the min! + stream_.latency[mode] = latrange.min; + //stream_.latency[mode] = jack_port_get_latency( jack_port_by_name( client, ports[ firstChannel ] ) ); + } + free( ports ); + + // The jack server always uses 32-bit floating-point data. + stream_.deviceFormat[mode] = RTAUDIO_FLOAT32; + stream_.userFormat = format; + + if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false; + else stream_.userInterleaved = true; + + // Jack always uses non-interleaved buffers. + stream_.deviceInterleaved[mode] = false; + + // Jack always provides host byte-ordered data. + stream_.doByteSwap[mode] = false; + + // Get the buffer size. The buffer size and number of buffers + // (periods) is set when the jack server is started. + stream_.bufferSize = (int) jack_get_buffer_size( client ); + *bufferSize = stream_.bufferSize; + + stream_.nDeviceChannels[mode] = channels; + stream_.nUserChannels[mode] = channels; + + // Set flags for buffer conversion. + stream_.doConvertBuffer[mode] = false; + if ( stream_.userFormat != stream_.deviceFormat[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] && + stream_.nUserChannels[mode] > 1 ) + stream_.doConvertBuffer[mode] = true; + + // Allocate our JackHandle structure for the stream. + if ( handle == 0 ) { + try { + handle = new JackHandle; + } + catch ( std::bad_alloc& ) { + errorText_ = "RtApiJack::probeDeviceOpen: error allocating JackHandle memory."; + goto error; + } + + if ( pthread_cond_init(&handle->condition, NULL) ) { + errorText_ = "RtApiJack::probeDeviceOpen: error initializing pthread condition variable."; + goto error; + } + stream_.apiHandle = (void *) handle; + handle->client = client; + } + handle->deviceName[mode] = deviceName; + + // Allocate necessary internal buffers. + unsigned long bufferBytes; + bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat ); + stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 ); + if ( stream_.userBuffer[mode] == NULL ) { + errorText_ = "RtApiJack::probeDeviceOpen: error allocating user buffer memory."; + goto error; + } + + if ( stream_.doConvertBuffer[mode] ) { + + bool makeBuffer = true; + if ( mode == OUTPUT ) + bufferBytes = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] ); + else { // mode == INPUT + bufferBytes = stream_.nDeviceChannels[1] * formatBytes( stream_.deviceFormat[1] ); + if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) { + unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes(stream_.deviceFormat[0]); + if ( bufferBytes < bytesOut ) makeBuffer = false; + } + } + + if ( makeBuffer ) { + bufferBytes *= *bufferSize; + if ( stream_.deviceBuffer ) free( stream_.deviceBuffer ); + stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 ); + if ( stream_.deviceBuffer == NULL ) { + errorText_ = "RtApiJack::probeDeviceOpen: error allocating device buffer memory."; + goto error; + } + } + } + + // Allocate memory for the Jack ports (channels) identifiers. + handle->ports[mode] = (jack_port_t **) malloc ( sizeof (jack_port_t *) * channels ); + if ( handle->ports[mode] == NULL ) { + errorText_ = "RtApiJack::probeDeviceOpen: error allocating port memory."; + goto error; + } + + stream_.device[mode] = device; + stream_.channelOffset[mode] = firstChannel; + stream_.state = STREAM_STOPPED; + stream_.callbackInfo.object = (void *) this; + + if ( stream_.mode == OUTPUT && mode == INPUT ) + // We had already set up the stream for output. + stream_.mode = DUPLEX; + else { + stream_.mode = mode; + jack_set_process_callback( handle->client, jackCallbackHandler, (void *) &stream_.callbackInfo ); + jack_set_xrun_callback( handle->client, jackXrun, (void *) &handle ); + jack_on_shutdown( handle->client, jackShutdown, (void *) &stream_.callbackInfo ); + } + + // Register our ports. + char label[64]; + if ( mode == OUTPUT ) { + for ( unsigned int i=0; i<stream_.nUserChannels[0]; i++ ) { + snprintf( label, 64, "outport %d", i ); + handle->ports[0][i] = jack_port_register( handle->client, (const char *)label, + JACK_DEFAULT_AUDIO_TYPE, JackPortIsOutput, 0 ); + } + } + else { + for ( unsigned int i=0; i<stream_.nUserChannels[1]; i++ ) { + snprintf( label, 64, "inport %d", i ); + handle->ports[1][i] = jack_port_register( handle->client, (const char *)label, + JACK_DEFAULT_AUDIO_TYPE, JackPortIsInput, 0 ); + } + } + + // Setup the buffer conversion information structure. We don't use + // buffers to do channel offsets, so we override that parameter + // here. + if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, 0 ); + + return SUCCESS; + + error: + if ( handle ) { + pthread_cond_destroy( &handle->condition ); + jack_client_close( handle->client ); + + if ( handle->ports[0] ) free( handle->ports[0] ); + if ( handle->ports[1] ) free( handle->ports[1] ); + + delete handle; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + return FAILURE; +} + +void RtApiJack :: closeStream( void ) +{ + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiJack::closeStream(): no open stream to close!"; + error( RtAudioError::WARNING ); + return; + } + + JackHandle *handle = (JackHandle *) stream_.apiHandle; + if ( handle ) { + + if ( stream_.state == STREAM_RUNNING ) + jack_deactivate( handle->client ); + + jack_client_close( handle->client ); + } + + if ( handle ) { + if ( handle->ports[0] ) free( handle->ports[0] ); + if ( handle->ports[1] ) free( handle->ports[1] ); + pthread_cond_destroy( &handle->condition ); + delete handle; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + stream_.mode = UNINITIALIZED; + stream_.state = STREAM_CLOSED; +} + +void RtApiJack :: startStream( void ) +{ + verifyStream(); + if ( stream_.state == STREAM_RUNNING ) { + errorText_ = "RtApiJack::startStream(): the stream is already running!"; + error( RtAudioError::WARNING ); + return; + } + + JackHandle *handle = (JackHandle *) stream_.apiHandle; + int result = jack_activate( handle->client ); + if ( result ) { + errorText_ = "RtApiJack::startStream(): unable to activate JACK client!"; + goto unlock; + } + + const char **ports; + + // Get the list of available ports. + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + result = 1; + ports = jack_get_ports( handle->client, handle->deviceName[0].c_str(), NULL, JackPortIsInput); + if ( ports == NULL) { + errorText_ = "RtApiJack::startStream(): error determining available JACK input ports!"; + goto unlock; + } + + // Now make the port connections. Since RtAudio wasn't designed to + // allow the user to select particular channels of a device, we'll + // just open the first "nChannels" ports with offset. + for ( unsigned int i=0; i<stream_.nUserChannels[0]; i++ ) { + result = 1; + if ( ports[ stream_.channelOffset[0] + i ] ) + result = jack_connect( handle->client, jack_port_name( handle->ports[0][i] ), ports[ stream_.channelOffset[0] + i ] ); + if ( result ) { + free( ports ); + errorText_ = "RtApiJack::startStream(): error connecting output ports!"; + goto unlock; + } + } + free(ports); + } + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) { + result = 1; + ports = jack_get_ports( handle->client, handle->deviceName[1].c_str(), NULL, JackPortIsOutput ); + if ( ports == NULL) { + errorText_ = "RtApiJack::startStream(): error determining available JACK output ports!"; + goto unlock; + } + + // Now make the port connections. See note above. + for ( unsigned int i=0; i<stream_.nUserChannels[1]; i++ ) { + result = 1; + if ( ports[ stream_.channelOffset[1] + i ] ) + result = jack_connect( handle->client, ports[ stream_.channelOffset[1] + i ], jack_port_name( handle->ports[1][i] ) ); + if ( result ) { + free( ports ); + errorText_ = "RtApiJack::startStream(): error connecting input ports!"; + goto unlock; + } + } + free(ports); + } + + handle->drainCounter = 0; + handle->internalDrain = false; + stream_.state = STREAM_RUNNING; + + unlock: + if ( result == 0 ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiJack :: stopStream( void ) +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiJack::stopStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + JackHandle *handle = (JackHandle *) stream_.apiHandle; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + if ( handle->drainCounter == 0 ) { + handle->drainCounter = 2; + pthread_cond_wait( &handle->condition, &stream_.mutex ); // block until signaled + } + } + + jack_deactivate( handle->client ); + stream_.state = STREAM_STOPPED; +} + +void RtApiJack :: abortStream( void ) +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiJack::abortStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + JackHandle *handle = (JackHandle *) stream_.apiHandle; + handle->drainCounter = 2; + + stopStream(); +} + +// This function will be called by a spawned thread when the user +// callback function signals that the stream should be stopped or +// aborted. It is necessary to handle it this way because the +// callbackEvent() function must return before the jack_deactivate() +// function will return. +static void *jackStopStream( void *ptr ) +{ + CallbackInfo *info = (CallbackInfo *) ptr; + RtApiJack *object = (RtApiJack *) info->object; + + object->stopStream(); + pthread_exit( NULL ); +} + +bool RtApiJack :: callbackEvent( unsigned long nframes ) +{ + if ( stream_.state == STREAM_STOPPED || stream_.state == STREAM_STOPPING ) return SUCCESS; + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiCore::callbackEvent(): the stream is closed ... this shouldn't happen!"; + error( RtAudioError::WARNING ); + return FAILURE; + } + if ( stream_.bufferSize != nframes ) { + errorText_ = "RtApiCore::callbackEvent(): the JACK buffer size has changed ... cannot process!"; + error( RtAudioError::WARNING ); + return FAILURE; + } + + CallbackInfo *info = (CallbackInfo *) &stream_.callbackInfo; + JackHandle *handle = (JackHandle *) stream_.apiHandle; + + // Check if we were draining the stream and signal is finished. + if ( handle->drainCounter > 3 ) { + ThreadHandle threadId; + + stream_.state = STREAM_STOPPING; + if ( handle->internalDrain == true ) + pthread_create( &threadId, NULL, jackStopStream, info ); + else + pthread_cond_signal( &handle->condition ); + return SUCCESS; + } + + // Invoke user callback first, to get fresh output data. + if ( handle->drainCounter == 0 ) { + RtAudioCallback callback = (RtAudioCallback) info->callback; + double streamTime = getStreamTime(); + RtAudioStreamStatus status = 0; + if ( stream_.mode != INPUT && handle->xrun[0] == true ) { + status |= RTAUDIO_OUTPUT_UNDERFLOW; + handle->xrun[0] = false; + } + if ( stream_.mode != OUTPUT && handle->xrun[1] == true ) { + status |= RTAUDIO_INPUT_OVERFLOW; + handle->xrun[1] = false; + } + int cbReturnValue = callback( stream_.userBuffer[0], stream_.userBuffer[1], + stream_.bufferSize, streamTime, status, info->userData ); + if ( cbReturnValue == 2 ) { + stream_.state = STREAM_STOPPING; + handle->drainCounter = 2; + ThreadHandle id; + pthread_create( &id, NULL, jackStopStream, info ); + return SUCCESS; + } + else if ( cbReturnValue == 1 ) { + handle->drainCounter = 1; + handle->internalDrain = true; + } + } + + jack_default_audio_sample_t *jackbuffer; + unsigned long bufferBytes = nframes * sizeof( jack_default_audio_sample_t ); + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + if ( handle->drainCounter > 1 ) { // write zeros to the output stream + + for ( unsigned int i=0; i<stream_.nDeviceChannels[0]; i++ ) { + jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[0][i], (jack_nframes_t) nframes ); + memset( jackbuffer, 0, bufferBytes ); + } + + } + else if ( stream_.doConvertBuffer[0] ) { + + convertBuffer( stream_.deviceBuffer, stream_.userBuffer[0], stream_.convertInfo[0] ); + + for ( unsigned int i=0; i<stream_.nDeviceChannels[0]; i++ ) { + jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[0][i], (jack_nframes_t) nframes ); + memcpy( jackbuffer, &stream_.deviceBuffer[i*bufferBytes], bufferBytes ); + } + } + else { // no buffer conversion + for ( unsigned int i=0; i<stream_.nUserChannels[0]; i++ ) { + jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[0][i], (jack_nframes_t) nframes ); + memcpy( jackbuffer, &stream_.userBuffer[0][i*bufferBytes], bufferBytes ); + } + } + } + + // Don't bother draining input + if ( handle->drainCounter ) { + handle->drainCounter++; + goto unlock; + } + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) { + + if ( stream_.doConvertBuffer[1] ) { + for ( unsigned int i=0; i<stream_.nDeviceChannels[1]; i++ ) { + jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[1][i], (jack_nframes_t) nframes ); + memcpy( &stream_.deviceBuffer[i*bufferBytes], jackbuffer, bufferBytes ); + } + convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] ); + } + else { // no buffer conversion + for ( unsigned int i=0; i<stream_.nUserChannels[1]; i++ ) { + jackbuffer = (jack_default_audio_sample_t *) jack_port_get_buffer( handle->ports[1][i], (jack_nframes_t) nframes ); + memcpy( &stream_.userBuffer[1][i*bufferBytes], jackbuffer, bufferBytes ); + } + } + } + + unlock: + RtApi::tickStreamTime(); + return SUCCESS; +} + //******************** End of __UNIX_JACK__ *********************// +#endif + +#if defined(__WINDOWS_ASIO__) // ASIO API on Windows + +// The ASIO API is designed around a callback scheme, so this +// implementation is similar to that used for OS-X CoreAudio and Linux +// Jack. The primary constraint with ASIO is that it only allows +// access to a single driver at a time. Thus, it is not possible to +// have more than one simultaneous RtAudio stream. +// +// This implementation also requires a number of external ASIO files +// and a few global variables. The ASIO callback scheme does not +// allow for the passing of user data, so we must create a global +// pointer to our callbackInfo structure. +// +// On unix systems, we make use of a pthread condition variable. +// Since there is no equivalent in Windows, I hacked something based +// on information found in +// http://www.cs.wustl.edu/~schmidt/win32-cv-1.html. + +#include "asiosys.h" +#include "asio.h" +#include "iasiothiscallresolver.h" +#include "asiodrivers.h" +#include <cmath> + +static AsioDrivers drivers; +static ASIOCallbacks asioCallbacks; +static ASIODriverInfo driverInfo; +static CallbackInfo *asioCallbackInfo; +static bool asioXRun; + +struct AsioHandle { + int drainCounter; // Tracks callback counts when draining + bool internalDrain; // Indicates if stop is initiated from callback or not. + ASIOBufferInfo *bufferInfos; + HANDLE condition; + + AsioHandle() + :drainCounter(0), internalDrain(false), bufferInfos(0) {} +}; + +// Function declarations (definitions at end of section) +static const char* getAsioErrorString( ASIOError result ); +static void sampleRateChanged( ASIOSampleRate sRate ); +static long asioMessages( long selector, long value, void* message, double* opt ); + +RtApiAsio :: RtApiAsio() +{ + // ASIO cannot run on a multi-threaded appartment. You can call + // CoInitialize beforehand, but it must be for appartment threading + // (in which case, CoInitilialize will return S_FALSE here). + coInitialized_ = false; + HRESULT hr = CoInitialize( NULL ); + if ( FAILED(hr) ) { + errorText_ = "RtApiAsio::ASIO requires a single-threaded appartment. Call CoInitializeEx(0,COINIT_APARTMENTTHREADED)"; + error( RtAudioError::WARNING ); + } + coInitialized_ = true; + + drivers.removeCurrentDriver(); + driverInfo.asioVersion = 2; + + // See note in DirectSound implementation about GetDesktopWindow(). + driverInfo.sysRef = GetForegroundWindow(); +} + +RtApiAsio :: ~RtApiAsio() +{ + if ( stream_.state != STREAM_CLOSED ) closeStream(); + if ( coInitialized_ ) CoUninitialize(); +} + +unsigned int RtApiAsio :: getDeviceCount( void ) +{ + return (unsigned int) drivers.asioGetNumDev(); +} + +RtAudio::DeviceInfo RtApiAsio :: getDeviceInfo( unsigned int device ) +{ + RtAudio::DeviceInfo info; + info.probed = false; + + // Get device ID + unsigned int nDevices = getDeviceCount(); + if ( nDevices == 0 ) { + errorText_ = "RtApiAsio::getDeviceInfo: no devices found!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + if ( device >= nDevices ) { + errorText_ = "RtApiAsio::getDeviceInfo: device ID is invalid!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + // If a stream is already open, we cannot probe other devices. Thus, use the saved results. + if ( stream_.state != STREAM_CLOSED ) { + if ( device >= devices_.size() ) { + errorText_ = "RtApiAsio::getDeviceInfo: device ID was not present before stream was opened."; + error( RtAudioError::WARNING ); + return info; + } + return devices_[ device ]; + } + + char driverName[32]; + ASIOError result = drivers.asioGetDriverName( (int) device, driverName, 32 ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::getDeviceInfo: unable to get driver name (" << getAsioErrorString( result ) << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + info.name = driverName; + + if ( !drivers.loadDriver( driverName ) ) { + errorStream_ << "RtApiAsio::getDeviceInfo: unable to load driver (" << driverName << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + result = ASIOInit( &driverInfo ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::getDeviceInfo: error (" << getAsioErrorString( result ) << ") initializing driver (" << driverName << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Determine the device channel information. + long inputChannels, outputChannels; + result = ASIOGetChannels( &inputChannels, &outputChannels ); + if ( result != ASE_OK ) { + drivers.removeCurrentDriver(); + errorStream_ << "RtApiAsio::getDeviceInfo: error (" << getAsioErrorString( result ) << ") getting channel count (" << driverName << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + info.outputChannels = outputChannels; + info.inputChannels = inputChannels; + if ( info.outputChannels > 0 && info.inputChannels > 0 ) + info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels; + + // Determine the supported sample rates. + info.sampleRates.clear(); + for ( unsigned int i=0; i<MAX_SAMPLE_RATES; i++ ) { + result = ASIOCanSampleRate( (ASIOSampleRate) SAMPLE_RATES[i] ); + if ( result == ASE_OK ) { + info.sampleRates.push_back( SAMPLE_RATES[i] ); + + if ( !info.preferredSampleRate || ( SAMPLE_RATES[i] <= 48000 && SAMPLE_RATES[i] > info.preferredSampleRate ) ) + info.preferredSampleRate = SAMPLE_RATES[i]; + } + } + + // Determine supported data types ... just check first channel and assume rest are the same. + ASIOChannelInfo channelInfo; + channelInfo.channel = 0; + channelInfo.isInput = true; + if ( info.inputChannels <= 0 ) channelInfo.isInput = false; + result = ASIOGetChannelInfo( &channelInfo ); + if ( result != ASE_OK ) { + drivers.removeCurrentDriver(); + errorStream_ << "RtApiAsio::getDeviceInfo: error (" << getAsioErrorString( result ) << ") getting driver channel info (" << driverName << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + info.nativeFormats = 0; + if ( channelInfo.type == ASIOSTInt16MSB || channelInfo.type == ASIOSTInt16LSB ) + info.nativeFormats |= RTAUDIO_SINT16; + else if ( channelInfo.type == ASIOSTInt32MSB || channelInfo.type == ASIOSTInt32LSB ) + info.nativeFormats |= RTAUDIO_SINT32; + else if ( channelInfo.type == ASIOSTFloat32MSB || channelInfo.type == ASIOSTFloat32LSB ) + info.nativeFormats |= RTAUDIO_FLOAT32; + else if ( channelInfo.type == ASIOSTFloat64MSB || channelInfo.type == ASIOSTFloat64LSB ) + info.nativeFormats |= RTAUDIO_FLOAT64; + else if ( channelInfo.type == ASIOSTInt24MSB || channelInfo.type == ASIOSTInt24LSB ) + info.nativeFormats |= RTAUDIO_SINT24; + + if ( info.outputChannels > 0 ) + if ( getDefaultOutputDevice() == device ) info.isDefaultOutput = true; + if ( info.inputChannels > 0 ) + if ( getDefaultInputDevice() == device ) info.isDefaultInput = true; + + info.probed = true; + drivers.removeCurrentDriver(); + return info; +} + +static void bufferSwitch( long index, ASIOBool /*processNow*/ ) +{ + RtApiAsio *object = (RtApiAsio *) asioCallbackInfo->object; + object->callbackEvent( index ); +} + +void RtApiAsio :: saveDeviceInfo( void ) +{ + devices_.clear(); + + unsigned int nDevices = getDeviceCount(); + devices_.resize( nDevices ); + for ( unsigned int i=0; i<nDevices; i++ ) + devices_[i] = getDeviceInfo( i ); +} + +bool RtApiAsio :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels, + unsigned int firstChannel, unsigned int sampleRate, + RtAudioFormat format, unsigned int *bufferSize, + RtAudio::StreamOptions *options ) +{//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + + bool isDuplexInput = mode == INPUT && stream_.mode == OUTPUT; + + // For ASIO, a duplex stream MUST use the same driver. + if ( isDuplexInput && stream_.device[0] != device ) { + errorText_ = "RtApiAsio::probeDeviceOpen: an ASIO duplex stream must use the same device for input and output!"; + return FAILURE; + } + + char driverName[32]; + ASIOError result = drivers.asioGetDriverName( (int) device, driverName, 32 ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: unable to get driver name (" << getAsioErrorString( result ) << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Only load the driver once for duplex stream. + if ( !isDuplexInput ) { + // The getDeviceInfo() function will not work when a stream is open + // because ASIO does not allow multiple devices to run at the same + // time. Thus, we'll probe the system before opening a stream and + // save the results for use by getDeviceInfo(). + this->saveDeviceInfo(); + + if ( !drivers.loadDriver( driverName ) ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: unable to load driver (" << driverName << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + result = ASIOInit( &driverInfo ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: error (" << getAsioErrorString( result ) << ") initializing driver (" << driverName << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + + // keep them before any "goto error", they are used for error cleanup + goto device boundary checks + bool buffersAllocated = false; + AsioHandle *handle = (AsioHandle *) stream_.apiHandle; + unsigned int nChannels; + + + // Check the device channel count. + long inputChannels, outputChannels; + result = ASIOGetChannels( &inputChannels, &outputChannels ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: error (" << getAsioErrorString( result ) << ") getting channel count (" << driverName << ")."; + errorText_ = errorStream_.str(); + goto error; + } + + if ( ( mode == OUTPUT && (channels+firstChannel) > (unsigned int) outputChannels) || + ( mode == INPUT && (channels+firstChannel) > (unsigned int) inputChannels) ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") does not support requested channel count (" << channels << ") + offset (" << firstChannel << ")."; + errorText_ = errorStream_.str(); + goto error; + } + stream_.nDeviceChannels[mode] = channels; + stream_.nUserChannels[mode] = channels; + stream_.channelOffset[mode] = firstChannel; + + // Verify the sample rate is supported. + result = ASIOCanSampleRate( (ASIOSampleRate) sampleRate ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") does not support requested sample rate (" << sampleRate << ")."; + errorText_ = errorStream_.str(); + goto error; + } + + // Get the current sample rate + ASIOSampleRate currentRate; + result = ASIOGetSampleRate( ¤tRate ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error getting sample rate."; + errorText_ = errorStream_.str(); + goto error; + } + + // Set the sample rate only if necessary + if ( currentRate != sampleRate ) { + result = ASIOSetSampleRate( (ASIOSampleRate) sampleRate ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error setting sample rate (" << sampleRate << ")."; + errorText_ = errorStream_.str(); + goto error; + } + } + + // Determine the driver data type. + ASIOChannelInfo channelInfo; + channelInfo.channel = 0; + if ( mode == OUTPUT ) channelInfo.isInput = false; + else channelInfo.isInput = true; + result = ASIOGetChannelInfo( &channelInfo ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error (" << getAsioErrorString( result ) << ") getting data format."; + errorText_ = errorStream_.str(); + goto error; + } + + // Assuming WINDOWS host is always little-endian. + stream_.doByteSwap[mode] = false; + stream_.userFormat = format; + stream_.deviceFormat[mode] = 0; + if ( channelInfo.type == ASIOSTInt16MSB || channelInfo.type == ASIOSTInt16LSB ) { + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + if ( channelInfo.type == ASIOSTInt16MSB ) stream_.doByteSwap[mode] = true; + } + else if ( channelInfo.type == ASIOSTInt32MSB || channelInfo.type == ASIOSTInt32LSB ) { + stream_.deviceFormat[mode] = RTAUDIO_SINT32; + if ( channelInfo.type == ASIOSTInt32MSB ) stream_.doByteSwap[mode] = true; + } + else if ( channelInfo.type == ASIOSTFloat32MSB || channelInfo.type == ASIOSTFloat32LSB ) { + stream_.deviceFormat[mode] = RTAUDIO_FLOAT32; + if ( channelInfo.type == ASIOSTFloat32MSB ) stream_.doByteSwap[mode] = true; + } + else if ( channelInfo.type == ASIOSTFloat64MSB || channelInfo.type == ASIOSTFloat64LSB ) { + stream_.deviceFormat[mode] = RTAUDIO_FLOAT64; + if ( channelInfo.type == ASIOSTFloat64MSB ) stream_.doByteSwap[mode] = true; + } + else if ( channelInfo.type == ASIOSTInt24MSB || channelInfo.type == ASIOSTInt24LSB ) { + stream_.deviceFormat[mode] = RTAUDIO_SINT24; + if ( channelInfo.type == ASIOSTInt24MSB ) stream_.doByteSwap[mode] = true; + } + + if ( stream_.deviceFormat[mode] == 0 ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") data format not supported by RtAudio."; + errorText_ = errorStream_.str(); + goto error; + } + + // Set the buffer size. For a duplex stream, this will end up + // setting the buffer size based on the input constraints, which + // should be ok. + long minSize, maxSize, preferSize, granularity; + result = ASIOGetBufferSize( &minSize, &maxSize, &preferSize, &granularity ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error (" << getAsioErrorString( result ) << ") getting buffer size."; + errorText_ = errorStream_.str(); + goto error; + } + + if ( isDuplexInput ) { + // When this is the duplex input (output was opened before), then we have to use the same + // buffersize as the output, because it might use the preferred buffer size, which most + // likely wasn't passed as input to this. The buffer sizes have to be identically anyway, + // So instead of throwing an error, make them equal. The caller uses the reference + // to the "bufferSize" param as usual to set up processing buffers. + + *bufferSize = stream_.bufferSize; + + } else { + if ( *bufferSize == 0 ) *bufferSize = preferSize; + else if ( *bufferSize < (unsigned int) minSize ) *bufferSize = (unsigned int) minSize; + else if ( *bufferSize > (unsigned int) maxSize ) *bufferSize = (unsigned int) maxSize; + else if ( granularity == -1 ) { + // Make sure bufferSize is a power of two. + int log2_of_min_size = 0; + int log2_of_max_size = 0; + + for ( unsigned int i = 0; i < sizeof(long) * 8; i++ ) { + if ( minSize & ((long)1 << i) ) log2_of_min_size = i; + if ( maxSize & ((long)1 << i) ) log2_of_max_size = i; + } + + long min_delta = std::abs( (long)*bufferSize - ((long)1 << log2_of_min_size) ); + int min_delta_num = log2_of_min_size; + + for (int i = log2_of_min_size + 1; i <= log2_of_max_size; i++) { + long current_delta = std::abs( (long)*bufferSize - ((long)1 << i) ); + if (current_delta < min_delta) { + min_delta = current_delta; + min_delta_num = i; + } + } + + *bufferSize = ( (unsigned int)1 << min_delta_num ); + if ( *bufferSize < (unsigned int) minSize ) *bufferSize = (unsigned int) minSize; + else if ( *bufferSize > (unsigned int) maxSize ) *bufferSize = (unsigned int) maxSize; + } + else if ( granularity != 0 ) { + // Set to an even multiple of granularity, rounding up. + *bufferSize = (*bufferSize + granularity-1) / granularity * granularity; + } + } + + /* + // we don't use it anymore, see above! + // Just left it here for the case... + if ( isDuplexInput && stream_.bufferSize != *bufferSize ) { + errorText_ = "RtApiAsio::probeDeviceOpen: input/output buffersize discrepancy!"; + goto error; + } + */ + + stream_.bufferSize = *bufferSize; + stream_.nBuffers = 2; + + if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false; + else stream_.userInterleaved = true; + + // ASIO always uses non-interleaved buffers. + stream_.deviceInterleaved[mode] = false; + + // Allocate, if necessary, our AsioHandle structure for the stream. + if ( handle == 0 ) { + try { + handle = new AsioHandle; + } + catch ( std::bad_alloc& ) { + errorText_ = "RtApiAsio::probeDeviceOpen: error allocating AsioHandle memory."; + goto error; + } + handle->bufferInfos = 0; + + // Create a manual-reset event. + handle->condition = CreateEvent( NULL, // no security + TRUE, // manual-reset + FALSE, // non-signaled initially + NULL ); // unnamed + stream_.apiHandle = (void *) handle; + } + + // Create the ASIO internal buffers. Since RtAudio sets up input + // and output separately, we'll have to dispose of previously + // created output buffers for a duplex stream. + if ( mode == INPUT && stream_.mode == OUTPUT ) { + ASIODisposeBuffers(); + if ( handle->bufferInfos ) free( handle->bufferInfos ); + } + + // Allocate, initialize, and save the bufferInfos in our stream callbackInfo structure. + unsigned int i; + nChannels = stream_.nDeviceChannels[0] + stream_.nDeviceChannels[1]; + handle->bufferInfos = (ASIOBufferInfo *) malloc( nChannels * sizeof(ASIOBufferInfo) ); + if ( handle->bufferInfos == NULL ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: error allocating bufferInfo memory for driver (" << driverName << ")."; + errorText_ = errorStream_.str(); + goto error; + } + + ASIOBufferInfo *infos; + infos = handle->bufferInfos; + for ( i=0; i<stream_.nDeviceChannels[0]; i++, infos++ ) { + infos->isInput = ASIOFalse; + infos->channelNum = i + stream_.channelOffset[0]; + infos->buffers[0] = infos->buffers[1] = 0; + } + for ( i=0; i<stream_.nDeviceChannels[1]; i++, infos++ ) { + infos->isInput = ASIOTrue; + infos->channelNum = i + stream_.channelOffset[1]; + infos->buffers[0] = infos->buffers[1] = 0; + } + + // prepare for callbacks + stream_.sampleRate = sampleRate; + stream_.device[mode] = device; + stream_.mode = isDuplexInput ? DUPLEX : mode; + + // store this class instance before registering callbacks, that are going to use it + asioCallbackInfo = &stream_.callbackInfo; + stream_.callbackInfo.object = (void *) this; + + // Set up the ASIO callback structure and create the ASIO data buffers. + asioCallbacks.bufferSwitch = &bufferSwitch; + asioCallbacks.sampleRateDidChange = &sampleRateChanged; + asioCallbacks.asioMessage = &asioMessages; + asioCallbacks.bufferSwitchTimeInfo = NULL; + result = ASIOCreateBuffers( handle->bufferInfos, nChannels, stream_.bufferSize, &asioCallbacks ); + if ( result != ASE_OK ) { + // Standard method failed. This can happen with strict/misbehaving drivers that return valid buffer size ranges + // but only accept the preferred buffer size as parameter for ASIOCreateBuffers. eg. Creatives ASIO driver + // in that case, let's be naïve and try that instead + *bufferSize = preferSize; + stream_.bufferSize = *bufferSize; + result = ASIOCreateBuffers( handle->bufferInfos, nChannels, stream_.bufferSize, &asioCallbacks ); + } + + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error (" << getAsioErrorString( result ) << ") creating buffers."; + errorText_ = errorStream_.str(); + goto error; + } + buffersAllocated = true; + stream_.state = STREAM_STOPPED; + + // Set flags for buffer conversion. + stream_.doConvertBuffer[mode] = false; + if ( stream_.userFormat != stream_.deviceFormat[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] && + stream_.nUserChannels[mode] > 1 ) + stream_.doConvertBuffer[mode] = true; + + // Allocate necessary internal buffers + unsigned long bufferBytes; + bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat ); + stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 ); + if ( stream_.userBuffer[mode] == NULL ) { + errorText_ = "RtApiAsio::probeDeviceOpen: error allocating user buffer memory."; + goto error; + } + + if ( stream_.doConvertBuffer[mode] ) { + + bool makeBuffer = true; + bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] ); + if ( isDuplexInput && stream_.deviceBuffer ) { + unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] ); + if ( bufferBytes <= bytesOut ) makeBuffer = false; + } + + if ( makeBuffer ) { + bufferBytes *= *bufferSize; + if ( stream_.deviceBuffer ) free( stream_.deviceBuffer ); + stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 ); + if ( stream_.deviceBuffer == NULL ) { + errorText_ = "RtApiAsio::probeDeviceOpen: error allocating device buffer memory."; + goto error; + } + } + } + + // Determine device latencies + long inputLatency, outputLatency; + result = ASIOGetLatencies( &inputLatency, &outputLatency ); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::probeDeviceOpen: driver (" << driverName << ") error (" << getAsioErrorString( result ) << ") getting latency."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING); // warn but don't fail + } + else { + stream_.latency[0] = outputLatency; + stream_.latency[1] = inputLatency; + } + + // Setup the buffer conversion information structure. We don't use + // buffers to do channel offsets, so we override that parameter + // here. + if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, 0 ); + + return SUCCESS; + + error: + if ( !isDuplexInput ) { + // the cleanup for error in the duplex input, is done by RtApi::openStream + // So we clean up for single channel only + + if ( buffersAllocated ) + ASIODisposeBuffers(); + + drivers.removeCurrentDriver(); + + if ( handle ) { + CloseHandle( handle->condition ); + if ( handle->bufferInfos ) + free( handle->bufferInfos ); + + delete handle; + stream_.apiHandle = 0; + } + + + if ( stream_.userBuffer[mode] ) { + free( stream_.userBuffer[mode] ); + stream_.userBuffer[mode] = 0; + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + } + + return FAILURE; +}//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// + +void RtApiAsio :: closeStream() +{ + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiAsio::closeStream(): no open stream to close!"; + error( RtAudioError::WARNING ); + return; + } + + if ( stream_.state == STREAM_RUNNING ) { + stream_.state = STREAM_STOPPED; + ASIOStop(); + } + ASIODisposeBuffers(); + drivers.removeCurrentDriver(); + + AsioHandle *handle = (AsioHandle *) stream_.apiHandle; + if ( handle ) { + CloseHandle( handle->condition ); + if ( handle->bufferInfos ) + free( handle->bufferInfos ); + delete handle; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + stream_.mode = UNINITIALIZED; + stream_.state = STREAM_CLOSED; +} + +bool stopThreadCalled = false; + +void RtApiAsio :: startStream() +{ + verifyStream(); + if ( stream_.state == STREAM_RUNNING ) { + errorText_ = "RtApiAsio::startStream(): the stream is already running!"; + error( RtAudioError::WARNING ); + return; + } + + AsioHandle *handle = (AsioHandle *) stream_.apiHandle; + ASIOError result = ASIOStart(); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::startStream: error (" << getAsioErrorString( result ) << ") starting device."; + errorText_ = errorStream_.str(); + goto unlock; + } + + handle->drainCounter = 0; + handle->internalDrain = false; + ResetEvent( handle->condition ); + stream_.state = STREAM_RUNNING; + asioXRun = false; + + unlock: + stopThreadCalled = false; + + if ( result == ASE_OK ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiAsio :: stopStream() +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiAsio::stopStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + AsioHandle *handle = (AsioHandle *) stream_.apiHandle; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + if ( handle->drainCounter == 0 ) { + handle->drainCounter = 2; + WaitForSingleObject( handle->condition, INFINITE ); // block until signaled + } + } + + stream_.state = STREAM_STOPPED; + + ASIOError result = ASIOStop(); + if ( result != ASE_OK ) { + errorStream_ << "RtApiAsio::stopStream: error (" << getAsioErrorString( result ) << ") stopping device."; + errorText_ = errorStream_.str(); + } + + if ( result == ASE_OK ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiAsio :: abortStream() +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiAsio::abortStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + // The following lines were commented-out because some behavior was + // noted where the device buffers need to be zeroed to avoid + // continuing sound, even when the device buffers are completely + // disposed. So now, calling abort is the same as calling stop. + // AsioHandle *handle = (AsioHandle *) stream_.apiHandle; + // handle->drainCounter = 2; + stopStream(); +} + +// This function will be called by a spawned thread when the user +// callback function signals that the stream should be stopped or +// aborted. It is necessary to handle it this way because the +// callbackEvent() function must return before the ASIOStop() +// function will return. +static unsigned __stdcall asioStopStream( void *ptr ) +{ + CallbackInfo *info = (CallbackInfo *) ptr; + RtApiAsio *object = (RtApiAsio *) info->object; + + object->stopStream(); + _endthreadex( 0 ); + return 0; +} + +bool RtApiAsio :: callbackEvent( long bufferIndex ) +{ + if ( stream_.state == STREAM_STOPPED || stream_.state == STREAM_STOPPING ) return SUCCESS; + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiAsio::callbackEvent(): the stream is closed ... this shouldn't happen!"; + error( RtAudioError::WARNING ); + return FAILURE; + } + + CallbackInfo *info = (CallbackInfo *) &stream_.callbackInfo; + AsioHandle *handle = (AsioHandle *) stream_.apiHandle; + + // Check if we were draining the stream and signal if finished. + if ( handle->drainCounter > 3 ) { + + stream_.state = STREAM_STOPPING; + if ( handle->internalDrain == false ) + SetEvent( handle->condition ); + else { // spawn a thread to stop the stream + unsigned threadId; + stream_.callbackInfo.thread = _beginthreadex( NULL, 0, &asioStopStream, + &stream_.callbackInfo, 0, &threadId ); + } + return SUCCESS; + } + + // Invoke user callback to get fresh output data UNLESS we are + // draining stream. + if ( handle->drainCounter == 0 ) { + RtAudioCallback callback = (RtAudioCallback) info->callback; + double streamTime = getStreamTime(); + RtAudioStreamStatus status = 0; + if ( stream_.mode != INPUT && asioXRun == true ) { + status |= RTAUDIO_OUTPUT_UNDERFLOW; + asioXRun = false; + } + if ( stream_.mode != OUTPUT && asioXRun == true ) { + status |= RTAUDIO_INPUT_OVERFLOW; + asioXRun = false; + } + int cbReturnValue = callback( stream_.userBuffer[0], stream_.userBuffer[1], + stream_.bufferSize, streamTime, status, info->userData ); + if ( cbReturnValue == 2 ) { + stream_.state = STREAM_STOPPING; + handle->drainCounter = 2; + unsigned threadId; + stream_.callbackInfo.thread = _beginthreadex( NULL, 0, &asioStopStream, + &stream_.callbackInfo, 0, &threadId ); + return SUCCESS; + } + else if ( cbReturnValue == 1 ) { + handle->drainCounter = 1; + handle->internalDrain = true; + } + } + + unsigned int nChannels, bufferBytes, i, j; + nChannels = stream_.nDeviceChannels[0] + stream_.nDeviceChannels[1]; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + bufferBytes = stream_.bufferSize * formatBytes( stream_.deviceFormat[0] ); + + if ( handle->drainCounter > 1 ) { // write zeros to the output stream + + for ( i=0, j=0; i<nChannels; i++ ) { + if ( handle->bufferInfos[i].isInput != ASIOTrue ) + memset( handle->bufferInfos[i].buffers[bufferIndex], 0, bufferBytes ); + } + + } + else if ( stream_.doConvertBuffer[0] ) { + + convertBuffer( stream_.deviceBuffer, stream_.userBuffer[0], stream_.convertInfo[0] ); + if ( stream_.doByteSwap[0] ) + byteSwapBuffer( stream_.deviceBuffer, + stream_.bufferSize * stream_.nDeviceChannels[0], + stream_.deviceFormat[0] ); + + for ( i=0, j=0; i<nChannels; i++ ) { + if ( handle->bufferInfos[i].isInput != ASIOTrue ) + memcpy( handle->bufferInfos[i].buffers[bufferIndex], + &stream_.deviceBuffer[j++*bufferBytes], bufferBytes ); + } + + } + else { + + if ( stream_.doByteSwap[0] ) + byteSwapBuffer( stream_.userBuffer[0], + stream_.bufferSize * stream_.nUserChannels[0], + stream_.userFormat ); + + for ( i=0, j=0; i<nChannels; i++ ) { + if ( handle->bufferInfos[i].isInput != ASIOTrue ) + memcpy( handle->bufferInfos[i].buffers[bufferIndex], + &stream_.userBuffer[0][bufferBytes*j++], bufferBytes ); + } + + } + } + + // Don't bother draining input + if ( handle->drainCounter ) { + handle->drainCounter++; + goto unlock; + } + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) { + + bufferBytes = stream_.bufferSize * formatBytes(stream_.deviceFormat[1]); + + if (stream_.doConvertBuffer[1]) { + + // Always interleave ASIO input data. + for ( i=0, j=0; i<nChannels; i++ ) { + if ( handle->bufferInfos[i].isInput == ASIOTrue ) + memcpy( &stream_.deviceBuffer[j++*bufferBytes], + handle->bufferInfos[i].buffers[bufferIndex], + bufferBytes ); + } + + if ( stream_.doByteSwap[1] ) + byteSwapBuffer( stream_.deviceBuffer, + stream_.bufferSize * stream_.nDeviceChannels[1], + stream_.deviceFormat[1] ); + convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] ); + + } + else { + for ( i=0, j=0; i<nChannels; i++ ) { + if ( handle->bufferInfos[i].isInput == ASIOTrue ) { + memcpy( &stream_.userBuffer[1][bufferBytes*j++], + handle->bufferInfos[i].buffers[bufferIndex], + bufferBytes ); + } + } + + if ( stream_.doByteSwap[1] ) + byteSwapBuffer( stream_.userBuffer[1], + stream_.bufferSize * stream_.nUserChannels[1], + stream_.userFormat ); + } + } + + unlock: + // The following call was suggested by Malte Clasen. While the API + // documentation indicates it should not be required, some device + // drivers apparently do not function correctly without it. + ASIOOutputReady(); + + RtApi::tickStreamTime(); + return SUCCESS; +} + +static void sampleRateChanged( ASIOSampleRate sRate ) +{ + // The ASIO documentation says that this usually only happens during + // external sync. Audio processing is not stopped by the driver, + // actual sample rate might not have even changed, maybe only the + // sample rate status of an AES/EBU or S/PDIF digital input at the + // audio device. + + RtApi *object = (RtApi *) asioCallbackInfo->object; + try { + object->stopStream(); + } + catch ( RtAudioError &exception ) { + std::cerr << "\nRtApiAsio: sampleRateChanged() error (" << exception.getMessage() << ")!\n" << std::endl; + return; + } + + std::cerr << "\nRtApiAsio: driver reports sample rate changed to " << sRate << " ... stream stopped!!!\n" << std::endl; +} + +static long asioMessages( long selector, long value, void* /*message*/, double* /*opt*/ ) +{ + long ret = 0; + + switch( selector ) { + case kAsioSelectorSupported: + if ( value == kAsioResetRequest + || value == kAsioEngineVersion + || value == kAsioResyncRequest + || value == kAsioLatenciesChanged + // The following three were added for ASIO 2.0, you don't + // necessarily have to support them. + || value == kAsioSupportsTimeInfo + || value == kAsioSupportsTimeCode + || value == kAsioSupportsInputMonitor) + ret = 1L; + break; + case kAsioResetRequest: + // Defer the task and perform the reset of the driver during the + // next "safe" situation. You cannot reset the driver right now, + // as this code is called from the driver. Reset the driver is + // done by completely destruct is. I.e. ASIOStop(), + // ASIODisposeBuffers(), Destruction Afterwards you initialize the + // driver again. + std::cerr << "\nRtApiAsio: driver reset requested!!!" << std::endl; + ret = 1L; + break; + case kAsioResyncRequest: + // This informs the application that the driver encountered some + // non-fatal data loss. It is used for synchronization purposes + // of different media. Added mainly to work around the Win16Mutex + // problems in Windows 95/98 with the Windows Multimedia system, + // which could lose data because the Mutex was held too long by + // another thread. However a driver can issue it in other + // situations, too. + // std::cerr << "\nRtApiAsio: driver resync requested!!!" << std::endl; + asioXRun = true; + ret = 1L; + break; + case kAsioLatenciesChanged: + // This will inform the host application that the drivers were + // latencies changed. Beware, it this does not mean that the + // buffer sizes have changed! You might need to update internal + // delay data. + std::cerr << "\nRtApiAsio: driver latency may have changed!!!" << std::endl; + ret = 1L; + break; + case kAsioEngineVersion: + // Return the supported ASIO version of the host application. If + // a host application does not implement this selector, ASIO 1.0 + // is assumed by the driver. + ret = 2L; + break; + case kAsioSupportsTimeInfo: + // Informs the driver whether the + // asioCallbacks.bufferSwitchTimeInfo() callback is supported. + // For compatibility with ASIO 1.0 drivers the host application + // should always support the "old" bufferSwitch method, too. + ret = 0; + break; + case kAsioSupportsTimeCode: + // Informs the driver whether application is interested in time + // code info. If an application does not need to know about time + // code, the driver has less work to do. + ret = 0; + break; + } + return ret; +} + +static const char* getAsioErrorString( ASIOError result ) +{ + struct Messages + { + ASIOError value; + const char*message; + }; + + static const Messages m[] = + { + { ASE_NotPresent, "Hardware input or output is not present or available." }, + { ASE_HWMalfunction, "Hardware is malfunctioning." }, + { ASE_InvalidParameter, "Invalid input parameter." }, + { ASE_InvalidMode, "Invalid mode." }, + { ASE_SPNotAdvancing, "Sample position not advancing." }, + { ASE_NoClock, "Sample clock or rate cannot be determined or is not present." }, + { ASE_NoMemory, "Not enough memory to complete the request." } + }; + + for ( unsigned int i = 0; i < sizeof(m)/sizeof(m[0]); ++i ) + if ( m[i].value == result ) return m[i].message; + + return "Unknown error."; +} + +//******************** End of __WINDOWS_ASIO__ *********************// +#endif + + +#if defined(__WINDOWS_WASAPI__) // Windows WASAPI API + +// Authored by Marcus Tomlinson <themarcustomlinson@gmail.com>, April 2014 +// - Introduces support for the Windows WASAPI API +// - Aims to deliver bit streams to and from hardware at the lowest possible latency, via the absolute minimum buffer sizes required +// - Provides flexible stream configuration to an otherwise strict and inflexible WASAPI interface +// - Includes automatic internal conversion of sample rate and buffer size between hardware and the user + +#ifndef INITGUID + #define INITGUID +#endif +#include <audioclient.h> +#include <avrt.h> +#include <mmdeviceapi.h> +#include <functiondiscoverykeys_devpkey.h> + +//============================================================================= + +#define SAFE_RELEASE( objectPtr )\ +if ( objectPtr )\ +{\ + objectPtr->Release();\ + objectPtr = NULL;\ +} + +typedef HANDLE ( __stdcall *TAvSetMmThreadCharacteristicsPtr )( LPCWSTR TaskName, LPDWORD TaskIndex ); + +//----------------------------------------------------------------------------- + +// WASAPI dictates stream sample rate, format, channel count, and in some cases, buffer size. +// Therefore we must perform all necessary conversions to user buffers in order to satisfy these +// requirements. WasapiBuffer ring buffers are used between HwIn->UserIn and UserOut->HwOut to +// provide intermediate storage for read / write synchronization. +class WasapiBuffer +{ +public: + WasapiBuffer() + : buffer_( NULL ), + bufferSize_( 0 ), + inIndex_( 0 ), + outIndex_( 0 ) {} + + ~WasapiBuffer() { + free( buffer_ ); + } + + // sets the length of the internal ring buffer + void setBufferSize( unsigned int bufferSize, unsigned int formatBytes ) { + free( buffer_ ); + + buffer_ = ( char* ) calloc( bufferSize, formatBytes ); + + bufferSize_ = bufferSize; + inIndex_ = 0; + outIndex_ = 0; + } + + // attempt to push a buffer into the ring buffer at the current "in" index + bool pushBuffer( char* buffer, unsigned int bufferSize, RtAudioFormat format ) + { + if ( !buffer || // incoming buffer is NULL + bufferSize == 0 || // incoming buffer has no data + bufferSize > bufferSize_ ) // incoming buffer too large + { + return false; + } + + unsigned int relOutIndex = outIndex_; + unsigned int inIndexEnd = inIndex_ + bufferSize; + if ( relOutIndex < inIndex_ && inIndexEnd >= bufferSize_ ) { + relOutIndex += bufferSize_; + } + + // "in" index can end on the "out" index but cannot begin at it + if ( inIndex_ <= relOutIndex && inIndexEnd > relOutIndex ) { + return false; // not enough space between "in" index and "out" index + } + + // copy buffer from external to internal + int fromZeroSize = inIndex_ + bufferSize - bufferSize_; + fromZeroSize = fromZeroSize < 0 ? 0 : fromZeroSize; + int fromInSize = bufferSize - fromZeroSize; + + switch( format ) + { + case RTAUDIO_SINT8: + memcpy( &( ( char* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( char ) ); + memcpy( buffer_, &( ( char* ) buffer )[fromInSize], fromZeroSize * sizeof( char ) ); + break; + case RTAUDIO_SINT16: + memcpy( &( ( short* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( short ) ); + memcpy( buffer_, &( ( short* ) buffer )[fromInSize], fromZeroSize * sizeof( short ) ); + break; + case RTAUDIO_SINT24: + memcpy( &( ( S24* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( S24 ) ); + memcpy( buffer_, &( ( S24* ) buffer )[fromInSize], fromZeroSize * sizeof( S24 ) ); + break; + case RTAUDIO_SINT32: + memcpy( &( ( int* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( int ) ); + memcpy( buffer_, &( ( int* ) buffer )[fromInSize], fromZeroSize * sizeof( int ) ); + break; + case RTAUDIO_FLOAT32: + memcpy( &( ( float* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( float ) ); + memcpy( buffer_, &( ( float* ) buffer )[fromInSize], fromZeroSize * sizeof( float ) ); + break; + case RTAUDIO_FLOAT64: + memcpy( &( ( double* ) buffer_ )[inIndex_], buffer, fromInSize * sizeof( double ) ); + memcpy( buffer_, &( ( double* ) buffer )[fromInSize], fromZeroSize * sizeof( double ) ); + break; + } + + // update "in" index + inIndex_ += bufferSize; + inIndex_ %= bufferSize_; + + return true; + } + + // attempt to pull a buffer from the ring buffer from the current "out" index + bool pullBuffer( char* buffer, unsigned int bufferSize, RtAudioFormat format ) + { + if ( !buffer || // incoming buffer is NULL + bufferSize == 0 || // incoming buffer has no data + bufferSize > bufferSize_ ) // incoming buffer too large + { + return false; + } + + unsigned int relInIndex = inIndex_; + unsigned int outIndexEnd = outIndex_ + bufferSize; + if ( relInIndex < outIndex_ && outIndexEnd >= bufferSize_ ) { + relInIndex += bufferSize_; + } + + // "out" index can begin at and end on the "in" index + if ( outIndex_ < relInIndex && outIndexEnd > relInIndex ) { + return false; // not enough space between "out" index and "in" index + } + + // copy buffer from internal to external + int fromZeroSize = outIndex_ + bufferSize - bufferSize_; + fromZeroSize = fromZeroSize < 0 ? 0 : fromZeroSize; + int fromOutSize = bufferSize - fromZeroSize; + + switch( format ) + { + case RTAUDIO_SINT8: + memcpy( buffer, &( ( char* ) buffer_ )[outIndex_], fromOutSize * sizeof( char ) ); + memcpy( &( ( char* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( char ) ); + break; + case RTAUDIO_SINT16: + memcpy( buffer, &( ( short* ) buffer_ )[outIndex_], fromOutSize * sizeof( short ) ); + memcpy( &( ( short* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( short ) ); + break; + case RTAUDIO_SINT24: + memcpy( buffer, &( ( S24* ) buffer_ )[outIndex_], fromOutSize * sizeof( S24 ) ); + memcpy( &( ( S24* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( S24 ) ); + break; + case RTAUDIO_SINT32: + memcpy( buffer, &( ( int* ) buffer_ )[outIndex_], fromOutSize * sizeof( int ) ); + memcpy( &( ( int* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( int ) ); + break; + case RTAUDIO_FLOAT32: + memcpy( buffer, &( ( float* ) buffer_ )[outIndex_], fromOutSize * sizeof( float ) ); + memcpy( &( ( float* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( float ) ); + break; + case RTAUDIO_FLOAT64: + memcpy( buffer, &( ( double* ) buffer_ )[outIndex_], fromOutSize * sizeof( double ) ); + memcpy( &( ( double* ) buffer )[fromOutSize], buffer_, fromZeroSize * sizeof( double ) ); + break; + } + + // update "out" index + outIndex_ += bufferSize; + outIndex_ %= bufferSize_; + + return true; + } + +private: + char* buffer_; + unsigned int bufferSize_; + unsigned int inIndex_; + unsigned int outIndex_; +}; + +//----------------------------------------------------------------------------- + +// In order to satisfy WASAPI's buffer requirements, we need a means of converting sample rate +// between HW and the user. The convertBufferWasapi function is used to perform this conversion +// between HwIn->UserIn and UserOut->HwOut during the stream callback loop. +// This sample rate converter favors speed over quality, and works best with conversions between +// one rate and its multiple. +void convertBufferWasapi( char* outBuffer, + const char* inBuffer, + const unsigned int& channelCount, + const unsigned int& inSampleRate, + const unsigned int& outSampleRate, + const unsigned int& inSampleCount, + unsigned int& outSampleCount, + const RtAudioFormat& format ) +{ + // calculate the new outSampleCount and relative sampleStep + float sampleRatio = ( float ) outSampleRate / inSampleRate; + float sampleStep = 1.0f / sampleRatio; + float inSampleFraction = 0.0f; + + outSampleCount = ( unsigned int ) roundf( inSampleCount * sampleRatio ); + + // frame-by-frame, copy each relative input sample into it's corresponding output sample + for ( unsigned int outSample = 0; outSample < outSampleCount; outSample++ ) + { + unsigned int inSample = ( unsigned int ) inSampleFraction; + + switch ( format ) + { + case RTAUDIO_SINT8: + memcpy( &( ( char* ) outBuffer )[ outSample * channelCount ], &( ( char* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( char ) ); + break; + case RTAUDIO_SINT16: + memcpy( &( ( short* ) outBuffer )[ outSample * channelCount ], &( ( short* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( short ) ); + break; + case RTAUDIO_SINT24: + memcpy( &( ( S24* ) outBuffer )[ outSample * channelCount ], &( ( S24* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( S24 ) ); + break; + case RTAUDIO_SINT32: + memcpy( &( ( int* ) outBuffer )[ outSample * channelCount ], &( ( int* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( int ) ); + break; + case RTAUDIO_FLOAT32: + memcpy( &( ( float* ) outBuffer )[ outSample * channelCount ], &( ( float* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( float ) ); + break; + case RTAUDIO_FLOAT64: + memcpy( &( ( double* ) outBuffer )[ outSample * channelCount ], &( ( double* ) inBuffer )[ inSample * channelCount ], channelCount * sizeof( double ) ); + break; + } + + // jump to next in sample + inSampleFraction += sampleStep; + } +} + +//----------------------------------------------------------------------------- + +// A structure to hold various information related to the WASAPI implementation. +struct WasapiHandle +{ + IAudioClient* captureAudioClient; + IAudioClient* renderAudioClient; + IAudioCaptureClient* captureClient; + IAudioRenderClient* renderClient; + HANDLE captureEvent; + HANDLE renderEvent; + + WasapiHandle() + : captureAudioClient( NULL ), + renderAudioClient( NULL ), + captureClient( NULL ), + renderClient( NULL ), + captureEvent( NULL ), + renderEvent( NULL ) {} +}; + +//============================================================================= + +RtApiWasapi::RtApiWasapi() + : coInitialized_( false ), deviceEnumerator_( NULL ) +{ + // WASAPI can run either apartment or multi-threaded + HRESULT hr = CoInitialize( NULL ); + if ( !FAILED( hr ) ) + coInitialized_ = true; + + // Instantiate device enumerator + hr = CoCreateInstance( __uuidof( MMDeviceEnumerator ), NULL, + CLSCTX_ALL, __uuidof( IMMDeviceEnumerator ), + ( void** ) &deviceEnumerator_ ); + + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::RtApiWasapi: Unable to instantiate device enumerator"; + error( RtAudioError::DRIVER_ERROR ); + } +} + +//----------------------------------------------------------------------------- + +RtApiWasapi::~RtApiWasapi() +{ + if ( stream_.state != STREAM_CLOSED ) + closeStream(); + + SAFE_RELEASE( deviceEnumerator_ ); + + // If this object previously called CoInitialize() + if ( coInitialized_ ) + CoUninitialize(); +} + +//============================================================================= + +unsigned int RtApiWasapi::getDeviceCount( void ) +{ + unsigned int captureDeviceCount = 0; + unsigned int renderDeviceCount = 0; + + IMMDeviceCollection* captureDevices = NULL; + IMMDeviceCollection* renderDevices = NULL; + + // Count capture devices + errorText_.clear(); + HRESULT hr = deviceEnumerator_->EnumAudioEndpoints( eCapture, DEVICE_STATE_ACTIVE, &captureDevices ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceCount: Unable to retrieve capture device collection."; + goto Exit; + } + + hr = captureDevices->GetCount( &captureDeviceCount ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceCount: Unable to retrieve capture device count."; + goto Exit; + } + + // Count render devices + hr = deviceEnumerator_->EnumAudioEndpoints( eRender, DEVICE_STATE_ACTIVE, &renderDevices ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceCount: Unable to retrieve render device collection."; + goto Exit; + } + + hr = renderDevices->GetCount( &renderDeviceCount ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceCount: Unable to retrieve render device count."; + goto Exit; + } + +Exit: + // release all references + SAFE_RELEASE( captureDevices ); + SAFE_RELEASE( renderDevices ); + + if ( errorText_.empty() ) + return captureDeviceCount + renderDeviceCount; + + error( RtAudioError::DRIVER_ERROR ); + return 0; +} + +//----------------------------------------------------------------------------- + +RtAudio::DeviceInfo RtApiWasapi::getDeviceInfo( unsigned int device ) +{ + RtAudio::DeviceInfo info; + unsigned int captureDeviceCount = 0; + unsigned int renderDeviceCount = 0; + std::string defaultDeviceName; + bool isCaptureDevice = false; + + PROPVARIANT deviceNameProp; + PROPVARIANT defaultDeviceNameProp; + + IMMDeviceCollection* captureDevices = NULL; + IMMDeviceCollection* renderDevices = NULL; + IMMDevice* devicePtr = NULL; + IMMDevice* defaultDevicePtr = NULL; + IAudioClient* audioClient = NULL; + IPropertyStore* devicePropStore = NULL; + IPropertyStore* defaultDevicePropStore = NULL; + + WAVEFORMATEX* deviceFormat = NULL; + WAVEFORMATEX* closestMatchFormat = NULL; + + // probed + info.probed = false; + + // Count capture devices + errorText_.clear(); + RtAudioError::Type errorType = RtAudioError::DRIVER_ERROR; + HRESULT hr = deviceEnumerator_->EnumAudioEndpoints( eCapture, DEVICE_STATE_ACTIVE, &captureDevices ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve capture device collection."; + goto Exit; + } + + hr = captureDevices->GetCount( &captureDeviceCount ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve capture device count."; + goto Exit; + } + + // Count render devices + hr = deviceEnumerator_->EnumAudioEndpoints( eRender, DEVICE_STATE_ACTIVE, &renderDevices ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve render device collection."; + goto Exit; + } + + hr = renderDevices->GetCount( &renderDeviceCount ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve render device count."; + goto Exit; + } + + // validate device index + if ( device >= captureDeviceCount + renderDeviceCount ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Invalid device index."; + errorType = RtAudioError::INVALID_USE; + goto Exit; + } + + // determine whether index falls within capture or render devices + if ( device >= renderDeviceCount ) { + hr = captureDevices->Item( device - renderDeviceCount, &devicePtr ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve capture device handle."; + goto Exit; + } + isCaptureDevice = true; + } + else { + hr = renderDevices->Item( device, &devicePtr ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve render device handle."; + goto Exit; + } + isCaptureDevice = false; + } + + // get default device name + if ( isCaptureDevice ) { + hr = deviceEnumerator_->GetDefaultAudioEndpoint( eCapture, eConsole, &defaultDevicePtr ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve default capture device handle."; + goto Exit; + } + } + else { + hr = deviceEnumerator_->GetDefaultAudioEndpoint( eRender, eConsole, &defaultDevicePtr ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve default render device handle."; + goto Exit; + } + } + + hr = defaultDevicePtr->OpenPropertyStore( STGM_READ, &defaultDevicePropStore ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to open default device property store."; + goto Exit; + } + PropVariantInit( &defaultDeviceNameProp ); + + hr = defaultDevicePropStore->GetValue( PKEY_Device_FriendlyName, &defaultDeviceNameProp ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve default device property: PKEY_Device_FriendlyName."; + goto Exit; + } + + defaultDeviceName = convertCharPointerToStdString(defaultDeviceNameProp.pwszVal); + + // name + hr = devicePtr->OpenPropertyStore( STGM_READ, &devicePropStore ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to open device property store."; + goto Exit; + } + + PropVariantInit( &deviceNameProp ); + + hr = devicePropStore->GetValue( PKEY_Device_FriendlyName, &deviceNameProp ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve device property: PKEY_Device_FriendlyName."; + goto Exit; + } + + info.name =convertCharPointerToStdString(deviceNameProp.pwszVal); + + // is default + if ( isCaptureDevice ) { + info.isDefaultInput = info.name == defaultDeviceName; + info.isDefaultOutput = false; + } + else { + info.isDefaultInput = false; + info.isDefaultOutput = info.name == defaultDeviceName; + } + + // channel count + hr = devicePtr->Activate( __uuidof( IAudioClient ), CLSCTX_ALL, NULL, ( void** ) &audioClient ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve device audio client."; + goto Exit; + } + + hr = audioClient->GetMixFormat( &deviceFormat ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::getDeviceInfo: Unable to retrieve device mix format."; + goto Exit; + } + + if ( isCaptureDevice ) { + info.inputChannels = deviceFormat->nChannels; + info.outputChannels = 0; + info.duplexChannels = 0; + } + else { + info.inputChannels = 0; + info.outputChannels = deviceFormat->nChannels; + info.duplexChannels = 0; + } + + // sample rates + info.sampleRates.clear(); + + // allow support for all sample rates as we have a built-in sample rate converter + for ( unsigned int i = 0; i < MAX_SAMPLE_RATES; i++ ) { + info.sampleRates.push_back( SAMPLE_RATES[i] ); + } + info.preferredSampleRate = deviceFormat->nSamplesPerSec; + + // native format + info.nativeFormats = 0; + + if ( deviceFormat->wFormatTag == WAVE_FORMAT_IEEE_FLOAT || + ( deviceFormat->wFormatTag == WAVE_FORMAT_EXTENSIBLE && + ( ( WAVEFORMATEXTENSIBLE* ) deviceFormat )->SubFormat == KSDATAFORMAT_SUBTYPE_IEEE_FLOAT ) ) + { + if ( deviceFormat->wBitsPerSample == 32 ) { + info.nativeFormats |= RTAUDIO_FLOAT32; + } + else if ( deviceFormat->wBitsPerSample == 64 ) { + info.nativeFormats |= RTAUDIO_FLOAT64; + } + } + else if ( deviceFormat->wFormatTag == WAVE_FORMAT_PCM || + ( deviceFormat->wFormatTag == WAVE_FORMAT_EXTENSIBLE && + ( ( WAVEFORMATEXTENSIBLE* ) deviceFormat )->SubFormat == KSDATAFORMAT_SUBTYPE_PCM ) ) + { + if ( deviceFormat->wBitsPerSample == 8 ) { + info.nativeFormats |= RTAUDIO_SINT8; + } + else if ( deviceFormat->wBitsPerSample == 16 ) { + info.nativeFormats |= RTAUDIO_SINT16; + } + else if ( deviceFormat->wBitsPerSample == 24 ) { + info.nativeFormats |= RTAUDIO_SINT24; + } + else if ( deviceFormat->wBitsPerSample == 32 ) { + info.nativeFormats |= RTAUDIO_SINT32; + } + } + + // probed + info.probed = true; + +Exit: + // release all references + PropVariantClear( &deviceNameProp ); + PropVariantClear( &defaultDeviceNameProp ); + + SAFE_RELEASE( captureDevices ); + SAFE_RELEASE( renderDevices ); + SAFE_RELEASE( devicePtr ); + SAFE_RELEASE( defaultDevicePtr ); + SAFE_RELEASE( audioClient ); + SAFE_RELEASE( devicePropStore ); + SAFE_RELEASE( defaultDevicePropStore ); + + CoTaskMemFree( deviceFormat ); + CoTaskMemFree( closestMatchFormat ); + + if ( !errorText_.empty() ) + error( errorType ); + return info; +} + +//----------------------------------------------------------------------------- + +unsigned int RtApiWasapi::getDefaultOutputDevice( void ) +{ + for ( unsigned int i = 0; i < getDeviceCount(); i++ ) { + if ( getDeviceInfo( i ).isDefaultOutput ) { + return i; + } + } + + return 0; +} + +//----------------------------------------------------------------------------- + +unsigned int RtApiWasapi::getDefaultInputDevice( void ) +{ + for ( unsigned int i = 0; i < getDeviceCount(); i++ ) { + if ( getDeviceInfo( i ).isDefaultInput ) { + return i; + } + } + + return 0; +} + +//----------------------------------------------------------------------------- + +void RtApiWasapi::closeStream( void ) +{ + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiWasapi::closeStream: No open stream to close."; + error( RtAudioError::WARNING ); + return; + } + + if ( stream_.state != STREAM_STOPPED ) + stopStream(); + + // clean up stream memory + SAFE_RELEASE( ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient ) + SAFE_RELEASE( ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient ) + + SAFE_RELEASE( ( ( WasapiHandle* ) stream_.apiHandle )->captureClient ) + SAFE_RELEASE( ( ( WasapiHandle* ) stream_.apiHandle )->renderClient ) + + if ( ( ( WasapiHandle* ) stream_.apiHandle )->captureEvent ) + CloseHandle( ( ( WasapiHandle* ) stream_.apiHandle )->captureEvent ); + + if ( ( ( WasapiHandle* ) stream_.apiHandle )->renderEvent ) + CloseHandle( ( ( WasapiHandle* ) stream_.apiHandle )->renderEvent ); + + delete ( WasapiHandle* ) stream_.apiHandle; + stream_.apiHandle = NULL; + + for ( int i = 0; i < 2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + // update stream state + stream_.state = STREAM_CLOSED; +} + +//----------------------------------------------------------------------------- + +void RtApiWasapi::startStream( void ) +{ + verifyStream(); + + if ( stream_.state == STREAM_RUNNING ) { + errorText_ = "RtApiWasapi::startStream: The stream is already running."; + error( RtAudioError::WARNING ); + return; + } + + // update stream state + stream_.state = STREAM_RUNNING; + + // create WASAPI stream thread + stream_.callbackInfo.thread = ( ThreadHandle ) CreateThread( NULL, 0, runWasapiThread, this, CREATE_SUSPENDED, NULL ); + + if ( !stream_.callbackInfo.thread ) { + errorText_ = "RtApiWasapi::startStream: Unable to instantiate callback thread."; + error( RtAudioError::THREAD_ERROR ); + } + else { + SetThreadPriority( ( void* ) stream_.callbackInfo.thread, stream_.callbackInfo.priority ); + ResumeThread( ( void* ) stream_.callbackInfo.thread ); + } +} + +//----------------------------------------------------------------------------- + +void RtApiWasapi::stopStream( void ) +{ + verifyStream(); + + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiWasapi::stopStream: The stream is already stopped."; + error( RtAudioError::WARNING ); + return; + } + + // inform stream thread by setting stream state to STREAM_STOPPING + stream_.state = STREAM_STOPPING; + + // wait until stream thread is stopped + while( stream_.state != STREAM_STOPPED ) { + Sleep( 1 ); + } + + // Wait for the last buffer to play before stopping. + Sleep( 1000 * stream_.bufferSize / stream_.sampleRate ); + + // stop capture client if applicable + if ( ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient ) { + HRESULT hr = ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient->Stop(); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::stopStream: Unable to stop capture stream."; + error( RtAudioError::DRIVER_ERROR ); + return; + } + } + + // stop render client if applicable + if ( ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient ) { + HRESULT hr = ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient->Stop(); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::stopStream: Unable to stop render stream."; + error( RtAudioError::DRIVER_ERROR ); + return; + } + } + + // close thread handle + if ( stream_.callbackInfo.thread && !CloseHandle( ( void* ) stream_.callbackInfo.thread ) ) { + errorText_ = "RtApiWasapi::stopStream: Unable to close callback thread."; + error( RtAudioError::THREAD_ERROR ); + return; + } + + stream_.callbackInfo.thread = (ThreadHandle) NULL; +} + +//----------------------------------------------------------------------------- + +void RtApiWasapi::abortStream( void ) +{ + verifyStream(); + + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiWasapi::abortStream: The stream is already stopped."; + error( RtAudioError::WARNING ); + return; + } + + // inform stream thread by setting stream state to STREAM_STOPPING + stream_.state = STREAM_STOPPING; + + // wait until stream thread is stopped + while ( stream_.state != STREAM_STOPPED ) { + Sleep( 1 ); + } + + // stop capture client if applicable + if ( ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient ) { + HRESULT hr = ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient->Stop(); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::abortStream: Unable to stop capture stream."; + error( RtAudioError::DRIVER_ERROR ); + return; + } + } + + // stop render client if applicable + if ( ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient ) { + HRESULT hr = ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient->Stop(); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::abortStream: Unable to stop render stream."; + error( RtAudioError::DRIVER_ERROR ); + return; + } + } + + // close thread handle + if ( stream_.callbackInfo.thread && !CloseHandle( ( void* ) stream_.callbackInfo.thread ) ) { + errorText_ = "RtApiWasapi::abortStream: Unable to close callback thread."; + error( RtAudioError::THREAD_ERROR ); + return; + } + + stream_.callbackInfo.thread = (ThreadHandle) NULL; +} + +//----------------------------------------------------------------------------- + +bool RtApiWasapi::probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels, + unsigned int firstChannel, unsigned int sampleRate, + RtAudioFormat format, unsigned int* bufferSize, + RtAudio::StreamOptions* options ) +{ + bool methodResult = FAILURE; + unsigned int captureDeviceCount = 0; + unsigned int renderDeviceCount = 0; + + IMMDeviceCollection* captureDevices = NULL; + IMMDeviceCollection* renderDevices = NULL; + IMMDevice* devicePtr = NULL; + WAVEFORMATEX* deviceFormat = NULL; + unsigned int bufferBytes; + stream_.state = STREAM_STOPPED; + + // create API Handle if not already created + if ( !stream_.apiHandle ) + stream_.apiHandle = ( void* ) new WasapiHandle(); + + // Count capture devices + errorText_.clear(); + RtAudioError::Type errorType = RtAudioError::DRIVER_ERROR; + HRESULT hr = deviceEnumerator_->EnumAudioEndpoints( eCapture, DEVICE_STATE_ACTIVE, &captureDevices ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve capture device collection."; + goto Exit; + } + + hr = captureDevices->GetCount( &captureDeviceCount ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve capture device count."; + goto Exit; + } + + // Count render devices + hr = deviceEnumerator_->EnumAudioEndpoints( eRender, DEVICE_STATE_ACTIVE, &renderDevices ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve render device collection."; + goto Exit; + } + + hr = renderDevices->GetCount( &renderDeviceCount ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve render device count."; + goto Exit; + } + + // validate device index + if ( device >= captureDeviceCount + renderDeviceCount ) { + errorType = RtAudioError::INVALID_USE; + errorText_ = "RtApiWasapi::probeDeviceOpen: Invalid device index."; + goto Exit; + } + + // determine whether index falls within capture or render devices + if ( device >= renderDeviceCount ) { + if ( mode != INPUT ) { + errorType = RtAudioError::INVALID_USE; + errorText_ = "RtApiWasapi::probeDeviceOpen: Capture device selected as output device."; + goto Exit; + } + + // retrieve captureAudioClient from devicePtr + IAudioClient*& captureAudioClient = ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient; + + hr = captureDevices->Item( device - renderDeviceCount, &devicePtr ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve capture device handle."; + goto Exit; + } + + hr = devicePtr->Activate( __uuidof( IAudioClient ), CLSCTX_ALL, + NULL, ( void** ) &captureAudioClient ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve device audio client."; + goto Exit; + } + + hr = captureAudioClient->GetMixFormat( &deviceFormat ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve device mix format."; + goto Exit; + } + + stream_.nDeviceChannels[mode] = deviceFormat->nChannels; + captureAudioClient->GetStreamLatency( ( long long* ) &stream_.latency[mode] ); + } + else { + if ( mode != OUTPUT ) { + errorType = RtAudioError::INVALID_USE; + errorText_ = "RtApiWasapi::probeDeviceOpen: Render device selected as input device."; + goto Exit; + } + + // retrieve renderAudioClient from devicePtr + IAudioClient*& renderAudioClient = ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient; + + hr = renderDevices->Item( device, &devicePtr ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve render device handle."; + goto Exit; + } + + hr = devicePtr->Activate( __uuidof( IAudioClient ), CLSCTX_ALL, + NULL, ( void** ) &renderAudioClient ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve device audio client."; + goto Exit; + } + + hr = renderAudioClient->GetMixFormat( &deviceFormat ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::probeDeviceOpen: Unable to retrieve device mix format."; + goto Exit; + } + + stream_.nDeviceChannels[mode] = deviceFormat->nChannels; + renderAudioClient->GetStreamLatency( ( long long* ) &stream_.latency[mode] ); + } + + // fill stream data + if ( ( stream_.mode == OUTPUT && mode == INPUT ) || + ( stream_.mode == INPUT && mode == OUTPUT ) ) { + stream_.mode = DUPLEX; + } + else { + stream_.mode = mode; + } + + stream_.device[mode] = device; + stream_.doByteSwap[mode] = false; + stream_.sampleRate = sampleRate; + stream_.bufferSize = *bufferSize; + stream_.nBuffers = 1; + stream_.nUserChannels[mode] = channels; + stream_.channelOffset[mode] = firstChannel; + stream_.userFormat = format; + stream_.deviceFormat[mode] = getDeviceInfo( device ).nativeFormats; + + if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) + stream_.userInterleaved = false; + else + stream_.userInterleaved = true; + stream_.deviceInterleaved[mode] = true; + + // Set flags for buffer conversion. + stream_.doConvertBuffer[mode] = false; + if ( stream_.userFormat != stream_.deviceFormat[mode] || + stream_.nUserChannels != stream_.nDeviceChannels ) + stream_.doConvertBuffer[mode] = true; + else if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] && + stream_.nUserChannels[mode] > 1 ) + stream_.doConvertBuffer[mode] = true; + + if ( stream_.doConvertBuffer[mode] ) + setConvertInfo( mode, 0 ); + + // Allocate necessary internal buffers + bufferBytes = stream_.nUserChannels[mode] * stream_.bufferSize * formatBytes( stream_.userFormat ); + + stream_.userBuffer[mode] = ( char* ) calloc( bufferBytes, 1 ); + if ( !stream_.userBuffer[mode] ) { + errorType = RtAudioError::MEMORY_ERROR; + errorText_ = "RtApiWasapi::probeDeviceOpen: Error allocating user buffer memory."; + goto Exit; + } + + if ( options && options->flags & RTAUDIO_SCHEDULE_REALTIME ) + stream_.callbackInfo.priority = 15; + else + stream_.callbackInfo.priority = 0; + + ///! TODO: RTAUDIO_MINIMIZE_LATENCY // Provide stream buffers directly to callback + ///! TODO: RTAUDIO_HOG_DEVICE // Exclusive mode + + methodResult = SUCCESS; + +Exit: + //clean up + SAFE_RELEASE( captureDevices ); + SAFE_RELEASE( renderDevices ); + SAFE_RELEASE( devicePtr ); + CoTaskMemFree( deviceFormat ); + + // if method failed, close the stream + if ( methodResult == FAILURE ) + closeStream(); + + if ( !errorText_.empty() ) + error( errorType ); + return methodResult; +} + +//============================================================================= + +DWORD WINAPI RtApiWasapi::runWasapiThread( void* wasapiPtr ) +{ + if ( wasapiPtr ) + ( ( RtApiWasapi* ) wasapiPtr )->wasapiThread(); + + return 0; +} + +DWORD WINAPI RtApiWasapi::stopWasapiThread( void* wasapiPtr ) +{ + if ( wasapiPtr ) + ( ( RtApiWasapi* ) wasapiPtr )->stopStream(); + + return 0; +} + +DWORD WINAPI RtApiWasapi::abortWasapiThread( void* wasapiPtr ) +{ + if ( wasapiPtr ) + ( ( RtApiWasapi* ) wasapiPtr )->abortStream(); + + return 0; +} + +//----------------------------------------------------------------------------- + +void RtApiWasapi::wasapiThread() +{ + // as this is a new thread, we must CoInitialize it + CoInitialize( NULL ); + + HRESULT hr; + + IAudioClient* captureAudioClient = ( ( WasapiHandle* ) stream_.apiHandle )->captureAudioClient; + IAudioClient* renderAudioClient = ( ( WasapiHandle* ) stream_.apiHandle )->renderAudioClient; + IAudioCaptureClient* captureClient = ( ( WasapiHandle* ) stream_.apiHandle )->captureClient; + IAudioRenderClient* renderClient = ( ( WasapiHandle* ) stream_.apiHandle )->renderClient; + HANDLE captureEvent = ( ( WasapiHandle* ) stream_.apiHandle )->captureEvent; + HANDLE renderEvent = ( ( WasapiHandle* ) stream_.apiHandle )->renderEvent; + + WAVEFORMATEX* captureFormat = NULL; + WAVEFORMATEX* renderFormat = NULL; + float captureSrRatio = 0.0f; + float renderSrRatio = 0.0f; + WasapiBuffer captureBuffer; + WasapiBuffer renderBuffer; + + // declare local stream variables + RtAudioCallback callback = ( RtAudioCallback ) stream_.callbackInfo.callback; + BYTE* streamBuffer = NULL; + unsigned long captureFlags = 0; + unsigned int bufferFrameCount = 0; + unsigned int numFramesPadding = 0; + unsigned int convBufferSize = 0; + bool callbackPushed = false; + bool callbackPulled = false; + bool callbackStopped = false; + int callbackResult = 0; + + // convBuffer is used to store converted buffers between WASAPI and the user + char* convBuffer = NULL; + unsigned int convBuffSize = 0; + unsigned int deviceBuffSize = 0; + + errorText_.clear(); + RtAudioError::Type errorType = RtAudioError::DRIVER_ERROR; + + // Attempt to assign "Pro Audio" characteristic to thread + HMODULE AvrtDll = LoadLibrary( (LPCTSTR) "AVRT.dll" ); + if ( AvrtDll ) { + DWORD taskIndex = 0; + TAvSetMmThreadCharacteristicsPtr AvSetMmThreadCharacteristicsPtr = ( TAvSetMmThreadCharacteristicsPtr ) GetProcAddress( AvrtDll, "AvSetMmThreadCharacteristicsW" ); + AvSetMmThreadCharacteristicsPtr( L"Pro Audio", &taskIndex ); + FreeLibrary( AvrtDll ); + } + + // start capture stream if applicable + if ( captureAudioClient ) { + hr = captureAudioClient->GetMixFormat( &captureFormat ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve device mix format."; + goto Exit; + } + + captureSrRatio = ( ( float ) captureFormat->nSamplesPerSec / stream_.sampleRate ); + + // initialize capture stream according to desire buffer size + float desiredBufferSize = stream_.bufferSize * captureSrRatio; + REFERENCE_TIME desiredBufferPeriod = ( REFERENCE_TIME ) ( ( float ) desiredBufferSize * 10000000 / captureFormat->nSamplesPerSec ); + + if ( !captureClient ) { + hr = captureAudioClient->Initialize( AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_EVENTCALLBACK, + desiredBufferPeriod, + desiredBufferPeriod, + captureFormat, + NULL ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to initialize capture audio client."; + goto Exit; + } + + hr = captureAudioClient->GetService( __uuidof( IAudioCaptureClient ), + ( void** ) &captureClient ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve capture client handle."; + goto Exit; + } + + // configure captureEvent to trigger on every available capture buffer + captureEvent = CreateEvent( NULL, FALSE, FALSE, NULL ); + if ( !captureEvent ) { + errorType = RtAudioError::SYSTEM_ERROR; + errorText_ = "RtApiWasapi::wasapiThread: Unable to create capture event."; + goto Exit; + } + + hr = captureAudioClient->SetEventHandle( captureEvent ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to set capture event handle."; + goto Exit; + } + + ( ( WasapiHandle* ) stream_.apiHandle )->captureClient = captureClient; + ( ( WasapiHandle* ) stream_.apiHandle )->captureEvent = captureEvent; + } + + unsigned int inBufferSize = 0; + hr = captureAudioClient->GetBufferSize( &inBufferSize ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to get capture buffer size."; + goto Exit; + } + + // scale outBufferSize according to stream->user sample rate ratio + unsigned int outBufferSize = ( unsigned int ) ( stream_.bufferSize * captureSrRatio ) * stream_.nDeviceChannels[INPUT]; + inBufferSize *= stream_.nDeviceChannels[INPUT]; + + // set captureBuffer size + captureBuffer.setBufferSize( inBufferSize + outBufferSize, formatBytes( stream_.deviceFormat[INPUT] ) ); + + // reset the capture stream + hr = captureAudioClient->Reset(); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to reset capture stream."; + goto Exit; + } + + // start the capture stream + hr = captureAudioClient->Start(); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to start capture stream."; + goto Exit; + } + } + + // start render stream if applicable + if ( renderAudioClient ) { + hr = renderAudioClient->GetMixFormat( &renderFormat ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve device mix format."; + goto Exit; + } + + renderSrRatio = ( ( float ) renderFormat->nSamplesPerSec / stream_.sampleRate ); + + // initialize render stream according to desire buffer size + float desiredBufferSize = stream_.bufferSize * renderSrRatio; + REFERENCE_TIME desiredBufferPeriod = ( REFERENCE_TIME ) ( ( float ) desiredBufferSize * 10000000 / renderFormat->nSamplesPerSec ); + + if ( !renderClient ) { + hr = renderAudioClient->Initialize( AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_EVENTCALLBACK, + desiredBufferPeriod, + desiredBufferPeriod, + renderFormat, + NULL ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to initialize render audio client."; + goto Exit; + } + + hr = renderAudioClient->GetService( __uuidof( IAudioRenderClient ), + ( void** ) &renderClient ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve render client handle."; + goto Exit; + } + + // configure renderEvent to trigger on every available render buffer + renderEvent = CreateEvent( NULL, FALSE, FALSE, NULL ); + if ( !renderEvent ) { + errorType = RtAudioError::SYSTEM_ERROR; + errorText_ = "RtApiWasapi::wasapiThread: Unable to create render event."; + goto Exit; + } + + hr = renderAudioClient->SetEventHandle( renderEvent ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to set render event handle."; + goto Exit; + } + + ( ( WasapiHandle* ) stream_.apiHandle )->renderClient = renderClient; + ( ( WasapiHandle* ) stream_.apiHandle )->renderEvent = renderEvent; + } + + unsigned int outBufferSize = 0; + hr = renderAudioClient->GetBufferSize( &outBufferSize ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to get render buffer size."; + goto Exit; + } + + // scale inBufferSize according to user->stream sample rate ratio + unsigned int inBufferSize = ( unsigned int ) ( stream_.bufferSize * renderSrRatio ) * stream_.nDeviceChannels[OUTPUT]; + outBufferSize *= stream_.nDeviceChannels[OUTPUT]; + + // set renderBuffer size + renderBuffer.setBufferSize( inBufferSize + outBufferSize, formatBytes( stream_.deviceFormat[OUTPUT] ) ); + + // reset the render stream + hr = renderAudioClient->Reset(); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to reset render stream."; + goto Exit; + } + + // start the render stream + hr = renderAudioClient->Start(); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to start render stream."; + goto Exit; + } + } + + if ( stream_.mode == INPUT ) { + convBuffSize = ( size_t ) ( stream_.bufferSize * captureSrRatio ) * stream_.nDeviceChannels[INPUT] * formatBytes( stream_.deviceFormat[INPUT] ); + deviceBuffSize = stream_.bufferSize * stream_.nDeviceChannels[INPUT] * formatBytes( stream_.deviceFormat[INPUT] ); + } + else if ( stream_.mode == OUTPUT ) { + convBuffSize = ( size_t ) ( stream_.bufferSize * renderSrRatio ) * stream_.nDeviceChannels[OUTPUT] * formatBytes( stream_.deviceFormat[OUTPUT] ); + deviceBuffSize = stream_.bufferSize * stream_.nDeviceChannels[OUTPUT] * formatBytes( stream_.deviceFormat[OUTPUT] ); + } + else if ( stream_.mode == DUPLEX ) { + convBuffSize = std::max( ( size_t ) ( stream_.bufferSize * captureSrRatio ) * stream_.nDeviceChannels[INPUT] * formatBytes( stream_.deviceFormat[INPUT] ), + ( size_t ) ( stream_.bufferSize * renderSrRatio ) * stream_.nDeviceChannels[OUTPUT] * formatBytes( stream_.deviceFormat[OUTPUT] ) ); + deviceBuffSize = std::max( stream_.bufferSize * stream_.nDeviceChannels[INPUT] * formatBytes( stream_.deviceFormat[INPUT] ), + stream_.bufferSize * stream_.nDeviceChannels[OUTPUT] * formatBytes( stream_.deviceFormat[OUTPUT] ) ); + } + + convBuffer = ( char* ) malloc( convBuffSize ); + stream_.deviceBuffer = ( char* ) malloc( deviceBuffSize ); + if ( !convBuffer || !stream_.deviceBuffer ) { + errorType = RtAudioError::MEMORY_ERROR; + errorText_ = "RtApiWasapi::wasapiThread: Error allocating device buffer memory."; + goto Exit; + } + + // stream process loop + while ( stream_.state != STREAM_STOPPING ) { + if ( !callbackPulled ) { + // Callback Input + // ============== + // 1. Pull callback buffer from inputBuffer + // 2. If 1. was successful: Convert callback buffer to user sample rate and channel count + // Convert callback buffer to user format + + if ( captureAudioClient ) { + // Pull callback buffer from inputBuffer + callbackPulled = captureBuffer.pullBuffer( convBuffer, + ( unsigned int ) ( stream_.bufferSize * captureSrRatio ) * stream_.nDeviceChannels[INPUT], + stream_.deviceFormat[INPUT] ); + + if ( callbackPulled ) { + // Convert callback buffer to user sample rate + convertBufferWasapi( stream_.deviceBuffer, + convBuffer, + stream_.nDeviceChannels[INPUT], + captureFormat->nSamplesPerSec, + stream_.sampleRate, + ( unsigned int ) ( stream_.bufferSize * captureSrRatio ), + convBufferSize, + stream_.deviceFormat[INPUT] ); + + if ( stream_.doConvertBuffer[INPUT] ) { + // Convert callback buffer to user format + convertBuffer( stream_.userBuffer[INPUT], + stream_.deviceBuffer, + stream_.convertInfo[INPUT] ); + } + else { + // no further conversion, simple copy deviceBuffer to userBuffer + memcpy( stream_.userBuffer[INPUT], + stream_.deviceBuffer, + stream_.bufferSize * stream_.nUserChannels[INPUT] * formatBytes( stream_.userFormat ) ); + } + } + } + else { + // if there is no capture stream, set callbackPulled flag + callbackPulled = true; + } + + // Execute Callback + // ================ + // 1. Execute user callback method + // 2. Handle return value from callback + + // if callback has not requested the stream to stop + if ( callbackPulled && !callbackStopped ) { + // Execute user callback method + callbackResult = callback( stream_.userBuffer[OUTPUT], + stream_.userBuffer[INPUT], + stream_.bufferSize, + getStreamTime(), + captureFlags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY ? RTAUDIO_INPUT_OVERFLOW : 0, + stream_.callbackInfo.userData ); + + // Handle return value from callback + if ( callbackResult == 1 ) { + // instantiate a thread to stop this thread + HANDLE threadHandle = CreateThread( NULL, 0, stopWasapiThread, this, 0, NULL ); + if ( !threadHandle ) { + errorType = RtAudioError::THREAD_ERROR; + errorText_ = "RtApiWasapi::wasapiThread: Unable to instantiate stream stop thread."; + goto Exit; + } + else if ( !CloseHandle( threadHandle ) ) { + errorType = RtAudioError::THREAD_ERROR; + errorText_ = "RtApiWasapi::wasapiThread: Unable to close stream stop thread handle."; + goto Exit; + } + + callbackStopped = true; + } + else if ( callbackResult == 2 ) { + // instantiate a thread to stop this thread + HANDLE threadHandle = CreateThread( NULL, 0, abortWasapiThread, this, 0, NULL ); + if ( !threadHandle ) { + errorType = RtAudioError::THREAD_ERROR; + errorText_ = "RtApiWasapi::wasapiThread: Unable to instantiate stream abort thread."; + goto Exit; + } + else if ( !CloseHandle( threadHandle ) ) { + errorType = RtAudioError::THREAD_ERROR; + errorText_ = "RtApiWasapi::wasapiThread: Unable to close stream abort thread handle."; + goto Exit; + } + + callbackStopped = true; + } + } + } + + // Callback Output + // =============== + // 1. Convert callback buffer to stream format + // 2. Convert callback buffer to stream sample rate and channel count + // 3. Push callback buffer into outputBuffer + + if ( renderAudioClient && callbackPulled ) { + if ( stream_.doConvertBuffer[OUTPUT] ) { + // Convert callback buffer to stream format + convertBuffer( stream_.deviceBuffer, + stream_.userBuffer[OUTPUT], + stream_.convertInfo[OUTPUT] ); + + } + + // Convert callback buffer to stream sample rate + convertBufferWasapi( convBuffer, + stream_.deviceBuffer, + stream_.nDeviceChannels[OUTPUT], + stream_.sampleRate, + renderFormat->nSamplesPerSec, + stream_.bufferSize, + convBufferSize, + stream_.deviceFormat[OUTPUT] ); + + // Push callback buffer into outputBuffer + callbackPushed = renderBuffer.pushBuffer( convBuffer, + convBufferSize * stream_.nDeviceChannels[OUTPUT], + stream_.deviceFormat[OUTPUT] ); + } + else { + // if there is no render stream, set callbackPushed flag + callbackPushed = true; + } + + // Stream Capture + // ============== + // 1. Get capture buffer from stream + // 2. Push capture buffer into inputBuffer + // 3. If 2. was successful: Release capture buffer + + if ( captureAudioClient ) { + // if the callback input buffer was not pulled from captureBuffer, wait for next capture event + if ( !callbackPulled ) { + WaitForSingleObject( captureEvent, INFINITE ); + } + + // Get capture buffer from stream + hr = captureClient->GetBuffer( &streamBuffer, + &bufferFrameCount, + &captureFlags, NULL, NULL ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve capture buffer."; + goto Exit; + } + + if ( bufferFrameCount != 0 ) { + // Push capture buffer into inputBuffer + if ( captureBuffer.pushBuffer( ( char* ) streamBuffer, + bufferFrameCount * stream_.nDeviceChannels[INPUT], + stream_.deviceFormat[INPUT] ) ) + { + // Release capture buffer + hr = captureClient->ReleaseBuffer( bufferFrameCount ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to release capture buffer."; + goto Exit; + } + } + else + { + // Inform WASAPI that capture was unsuccessful + hr = captureClient->ReleaseBuffer( 0 ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to release capture buffer."; + goto Exit; + } + } + } + else + { + // Inform WASAPI that capture was unsuccessful + hr = captureClient->ReleaseBuffer( 0 ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to release capture buffer."; + goto Exit; + } + } + } + + // Stream Render + // ============= + // 1. Get render buffer from stream + // 2. Pull next buffer from outputBuffer + // 3. If 2. was successful: Fill render buffer with next buffer + // Release render buffer + + if ( renderAudioClient ) { + // if the callback output buffer was not pushed to renderBuffer, wait for next render event + if ( callbackPulled && !callbackPushed ) { + WaitForSingleObject( renderEvent, INFINITE ); + } + + // Get render buffer from stream + hr = renderAudioClient->GetBufferSize( &bufferFrameCount ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve render buffer size."; + goto Exit; + } + + hr = renderAudioClient->GetCurrentPadding( &numFramesPadding ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve render buffer padding."; + goto Exit; + } + + bufferFrameCount -= numFramesPadding; + + if ( bufferFrameCount != 0 ) { + hr = renderClient->GetBuffer( bufferFrameCount, &streamBuffer ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to retrieve render buffer."; + goto Exit; + } + + // Pull next buffer from outputBuffer + // Fill render buffer with next buffer + if ( renderBuffer.pullBuffer( ( char* ) streamBuffer, + bufferFrameCount * stream_.nDeviceChannels[OUTPUT], + stream_.deviceFormat[OUTPUT] ) ) + { + // Release render buffer + hr = renderClient->ReleaseBuffer( bufferFrameCount, 0 ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to release render buffer."; + goto Exit; + } + } + else + { + // Inform WASAPI that render was unsuccessful + hr = renderClient->ReleaseBuffer( 0, 0 ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to release render buffer."; + goto Exit; + } + } + } + else + { + // Inform WASAPI that render was unsuccessful + hr = renderClient->ReleaseBuffer( 0, 0 ); + if ( FAILED( hr ) ) { + errorText_ = "RtApiWasapi::wasapiThread: Unable to release render buffer."; + goto Exit; + } + } + } + + // if the callback buffer was pushed renderBuffer reset callbackPulled flag + if ( callbackPushed ) { + callbackPulled = false; + } + + // tick stream time + RtApi::tickStreamTime(); + } + +Exit: + // clean up + CoTaskMemFree( captureFormat ); + CoTaskMemFree( renderFormat ); + + free ( convBuffer ); + + CoUninitialize(); + + // update stream state + stream_.state = STREAM_STOPPED; + + if ( errorText_.empty() ) + return; + else + error( errorType ); +} + +//******************** End of __WINDOWS_WASAPI__ *********************// +#endif + + +#if defined(__WINDOWS_DS__) // Windows DirectSound API + +// Modified by Robin Davies, October 2005 +// - Improvements to DirectX pointer chasing. +// - Bug fix for non-power-of-two Asio granularity used by Edirol PCR-A30. +// - Auto-call CoInitialize for DSOUND and ASIO platforms. +// Various revisions for RtAudio 4.0 by Gary Scavone, April 2007 +// Changed device query structure for RtAudio 4.0.7, January 2010 + +#include <dsound.h> +#include <assert.h> +#include <algorithm> + +#if defined(__MINGW32__) + // missing from latest mingw winapi +#define WAVE_FORMAT_96M08 0x00010000 /* 96 kHz, Mono, 8-bit */ +#define WAVE_FORMAT_96S08 0x00020000 /* 96 kHz, Stereo, 8-bit */ +#define WAVE_FORMAT_96M16 0x00040000 /* 96 kHz, Mono, 16-bit */ +#define WAVE_FORMAT_96S16 0x00080000 /* 96 kHz, Stereo, 16-bit */ +#endif + +#define MINIMUM_DEVICE_BUFFER_SIZE 32768 + +#ifdef _MSC_VER // if Microsoft Visual C++ +#pragma comment( lib, "winmm.lib" ) // then, auto-link winmm.lib. Otherwise, it has to be added manually. +#endif + +static inline DWORD dsPointerBetween( DWORD pointer, DWORD laterPointer, DWORD earlierPointer, DWORD bufferSize ) +{ + if ( pointer > bufferSize ) pointer -= bufferSize; + if ( laterPointer < earlierPointer ) laterPointer += bufferSize; + if ( pointer < earlierPointer ) pointer += bufferSize; + return pointer >= earlierPointer && pointer < laterPointer; +} + +// A structure to hold various information related to the DirectSound +// API implementation. +struct DsHandle { + unsigned int drainCounter; // Tracks callback counts when draining + bool internalDrain; // Indicates if stop is initiated from callback or not. + void *id[2]; + void *buffer[2]; + bool xrun[2]; + UINT bufferPointer[2]; + DWORD dsBufferSize[2]; + DWORD dsPointerLeadTime[2]; // the number of bytes ahead of the safe pointer to lead by. + HANDLE condition; + + DsHandle() + :drainCounter(0), internalDrain(false) { id[0] = 0; id[1] = 0; buffer[0] = 0; buffer[1] = 0; xrun[0] = false; xrun[1] = false; bufferPointer[0] = 0; bufferPointer[1] = 0; } +}; + +// Declarations for utility functions, callbacks, and structures +// specific to the DirectSound implementation. +static BOOL CALLBACK deviceQueryCallback( LPGUID lpguid, + LPCTSTR description, + LPCTSTR module, + LPVOID lpContext ); + +static const char* getErrorString( int code ); + +static unsigned __stdcall callbackHandler( void *ptr ); + +struct DsDevice { + LPGUID id[2]; + bool validId[2]; + bool found; + std::string name; + + DsDevice() + : found(false) { validId[0] = false; validId[1] = false; } +}; + +struct DsProbeData { + bool isInput; + std::vector<struct DsDevice>* dsDevices; +}; + +RtApiDs :: RtApiDs() +{ + // Dsound will run both-threaded. If CoInitialize fails, then just + // accept whatever the mainline chose for a threading model. + coInitialized_ = false; + HRESULT hr = CoInitialize( NULL ); + if ( !FAILED( hr ) ) coInitialized_ = true; +} + +RtApiDs :: ~RtApiDs() +{ + if ( coInitialized_ ) CoUninitialize(); // balanced call. + if ( stream_.state != STREAM_CLOSED ) closeStream(); +} + +// The DirectSound default output is always the first device. +unsigned int RtApiDs :: getDefaultOutputDevice( void ) +{ + return 0; +} + +// The DirectSound default input is always the first input device, +// which is the first capture device enumerated. +unsigned int RtApiDs :: getDefaultInputDevice( void ) +{ + return 0; +} + +unsigned int RtApiDs :: getDeviceCount( void ) +{ + // Set query flag for previously found devices to false, so that we + // can check for any devices that have disappeared. + for ( unsigned int i=0; i<dsDevices.size(); i++ ) + dsDevices[i].found = false; + + // Query DirectSound devices. + struct DsProbeData probeInfo; + probeInfo.isInput = false; + probeInfo.dsDevices = &dsDevices; + HRESULT result = DirectSoundEnumerate( (LPDSENUMCALLBACK) deviceQueryCallback, &probeInfo ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::getDeviceCount: error (" << getErrorString( result ) << ") enumerating output devices!"; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + } + + // Query DirectSoundCapture devices. + probeInfo.isInput = true; + result = DirectSoundCaptureEnumerate( (LPDSENUMCALLBACK) deviceQueryCallback, &probeInfo ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::getDeviceCount: error (" << getErrorString( result ) << ") enumerating input devices!"; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + } + + // Clean out any devices that may have disappeared (code update submitted by Eli Zehngut). + for ( unsigned int i=0; i<dsDevices.size(); ) { + if ( dsDevices[i].found == false ) dsDevices.erase( dsDevices.begin() + i ); + else i++; + } + + return static_cast<unsigned int>(dsDevices.size()); +} + +RtAudio::DeviceInfo RtApiDs :: getDeviceInfo( unsigned int device ) +{ + RtAudio::DeviceInfo info; + info.probed = false; + + if ( dsDevices.size() == 0 ) { + // Force a query of all devices + getDeviceCount(); + if ( dsDevices.size() == 0 ) { + errorText_ = "RtApiDs::getDeviceInfo: no devices found!"; + error( RtAudioError::INVALID_USE ); + return info; + } + } + + if ( device >= dsDevices.size() ) { + errorText_ = "RtApiDs::getDeviceInfo: device ID is invalid!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + HRESULT result; + if ( dsDevices[ device ].validId[0] == false ) goto probeInput; + + LPDIRECTSOUND output; + DSCAPS outCaps; + result = DirectSoundCreate( dsDevices[ device ].id[0], &output, NULL ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::getDeviceInfo: error (" << getErrorString( result ) << ") opening output device (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + goto probeInput; + } + + outCaps.dwSize = sizeof( outCaps ); + result = output->GetCaps( &outCaps ); + if ( FAILED( result ) ) { + output->Release(); + errorStream_ << "RtApiDs::getDeviceInfo: error (" << getErrorString( result ) << ") getting capabilities!"; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + goto probeInput; + } + + // Get output channel information. + info.outputChannels = ( outCaps.dwFlags & DSCAPS_PRIMARYSTEREO ) ? 2 : 1; + + // Get sample rate information. + info.sampleRates.clear(); + for ( unsigned int k=0; k<MAX_SAMPLE_RATES; k++ ) { + if ( SAMPLE_RATES[k] >= (unsigned int) outCaps.dwMinSecondarySampleRate && + SAMPLE_RATES[k] <= (unsigned int) outCaps.dwMaxSecondarySampleRate ) { + info.sampleRates.push_back( SAMPLE_RATES[k] ); + + if ( !info.preferredSampleRate || ( SAMPLE_RATES[k] <= 48000 && SAMPLE_RATES[k] > info.preferredSampleRate ) ) + info.preferredSampleRate = SAMPLE_RATES[k]; + } + } + + // Get format information. + if ( outCaps.dwFlags & DSCAPS_PRIMARY16BIT ) info.nativeFormats |= RTAUDIO_SINT16; + if ( outCaps.dwFlags & DSCAPS_PRIMARY8BIT ) info.nativeFormats |= RTAUDIO_SINT8; + + output->Release(); + + if ( getDefaultOutputDevice() == device ) + info.isDefaultOutput = true; + + if ( dsDevices[ device ].validId[1] == false ) { + info.name = dsDevices[ device ].name; + info.probed = true; + return info; + } + + probeInput: + + LPDIRECTSOUNDCAPTURE input; + result = DirectSoundCaptureCreate( dsDevices[ device ].id[1], &input, NULL ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::getDeviceInfo: error (" << getErrorString( result ) << ") opening input device (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + DSCCAPS inCaps; + inCaps.dwSize = sizeof( inCaps ); + result = input->GetCaps( &inCaps ); + if ( FAILED( result ) ) { + input->Release(); + errorStream_ << "RtApiDs::getDeviceInfo: error (" << getErrorString( result ) << ") getting object capabilities (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Get input channel information. + info.inputChannels = inCaps.dwChannels; + + // Get sample rate and format information. + std::vector<unsigned int> rates; + if ( inCaps.dwChannels >= 2 ) { + if ( inCaps.dwFormats & WAVE_FORMAT_1S16 ) info.nativeFormats |= RTAUDIO_SINT16; + if ( inCaps.dwFormats & WAVE_FORMAT_2S16 ) info.nativeFormats |= RTAUDIO_SINT16; + if ( inCaps.dwFormats & WAVE_FORMAT_4S16 ) info.nativeFormats |= RTAUDIO_SINT16; + if ( inCaps.dwFormats & WAVE_FORMAT_96S16 ) info.nativeFormats |= RTAUDIO_SINT16; + if ( inCaps.dwFormats & WAVE_FORMAT_1S08 ) info.nativeFormats |= RTAUDIO_SINT8; + if ( inCaps.dwFormats & WAVE_FORMAT_2S08 ) info.nativeFormats |= RTAUDIO_SINT8; + if ( inCaps.dwFormats & WAVE_FORMAT_4S08 ) info.nativeFormats |= RTAUDIO_SINT8; + if ( inCaps.dwFormats & WAVE_FORMAT_96S08 ) info.nativeFormats |= RTAUDIO_SINT8; + + if ( info.nativeFormats & RTAUDIO_SINT16 ) { + if ( inCaps.dwFormats & WAVE_FORMAT_1S16 ) rates.push_back( 11025 ); + if ( inCaps.dwFormats & WAVE_FORMAT_2S16 ) rates.push_back( 22050 ); + if ( inCaps.dwFormats & WAVE_FORMAT_4S16 ) rates.push_back( 44100 ); + if ( inCaps.dwFormats & WAVE_FORMAT_96S16 ) rates.push_back( 96000 ); + } + else if ( info.nativeFormats & RTAUDIO_SINT8 ) { + if ( inCaps.dwFormats & WAVE_FORMAT_1S08 ) rates.push_back( 11025 ); + if ( inCaps.dwFormats & WAVE_FORMAT_2S08 ) rates.push_back( 22050 ); + if ( inCaps.dwFormats & WAVE_FORMAT_4S08 ) rates.push_back( 44100 ); + if ( inCaps.dwFormats & WAVE_FORMAT_96S08 ) rates.push_back( 96000 ); + } + } + else if ( inCaps.dwChannels == 1 ) { + if ( inCaps.dwFormats & WAVE_FORMAT_1M16 ) info.nativeFormats |= RTAUDIO_SINT16; + if ( inCaps.dwFormats & WAVE_FORMAT_2M16 ) info.nativeFormats |= RTAUDIO_SINT16; + if ( inCaps.dwFormats & WAVE_FORMAT_4M16 ) info.nativeFormats |= RTAUDIO_SINT16; + if ( inCaps.dwFormats & WAVE_FORMAT_96M16 ) info.nativeFormats |= RTAUDIO_SINT16; + if ( inCaps.dwFormats & WAVE_FORMAT_1M08 ) info.nativeFormats |= RTAUDIO_SINT8; + if ( inCaps.dwFormats & WAVE_FORMAT_2M08 ) info.nativeFormats |= RTAUDIO_SINT8; + if ( inCaps.dwFormats & WAVE_FORMAT_4M08 ) info.nativeFormats |= RTAUDIO_SINT8; + if ( inCaps.dwFormats & WAVE_FORMAT_96M08 ) info.nativeFormats |= RTAUDIO_SINT8; + + if ( info.nativeFormats & RTAUDIO_SINT16 ) { + if ( inCaps.dwFormats & WAVE_FORMAT_1M16 ) rates.push_back( 11025 ); + if ( inCaps.dwFormats & WAVE_FORMAT_2M16 ) rates.push_back( 22050 ); + if ( inCaps.dwFormats & WAVE_FORMAT_4M16 ) rates.push_back( 44100 ); + if ( inCaps.dwFormats & WAVE_FORMAT_96M16 ) rates.push_back( 96000 ); + } + else if ( info.nativeFormats & RTAUDIO_SINT8 ) { + if ( inCaps.dwFormats & WAVE_FORMAT_1M08 ) rates.push_back( 11025 ); + if ( inCaps.dwFormats & WAVE_FORMAT_2M08 ) rates.push_back( 22050 ); + if ( inCaps.dwFormats & WAVE_FORMAT_4M08 ) rates.push_back( 44100 ); + if ( inCaps.dwFormats & WAVE_FORMAT_96M08 ) rates.push_back( 96000 ); + } + } + else info.inputChannels = 0; // technically, this would be an error + + input->Release(); + + if ( info.inputChannels == 0 ) return info; + + // Copy the supported rates to the info structure but avoid duplication. + bool found; + for ( unsigned int i=0; i<rates.size(); i++ ) { + found = false; + for ( unsigned int j=0; j<info.sampleRates.size(); j++ ) { + if ( rates[i] == info.sampleRates[j] ) { + found = true; + break; + } + } + if ( found == false ) info.sampleRates.push_back( rates[i] ); + } + std::sort( info.sampleRates.begin(), info.sampleRates.end() ); + + // If device opens for both playback and capture, we determine the channels. + if ( info.outputChannels > 0 && info.inputChannels > 0 ) + info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels; + + if ( device == 0 ) info.isDefaultInput = true; + + // Copy name and return. + info.name = dsDevices[ device ].name; + info.probed = true; + return info; +} + +bool RtApiDs :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels, + unsigned int firstChannel, unsigned int sampleRate, + RtAudioFormat format, unsigned int *bufferSize, + RtAudio::StreamOptions *options ) +{ + if ( channels + firstChannel > 2 ) { + errorText_ = "RtApiDs::probeDeviceOpen: DirectSound does not support more than 2 channels per device."; + return FAILURE; + } + + size_t nDevices = dsDevices.size(); + if ( nDevices == 0 ) { + // This should not happen because a check is made before this function is called. + errorText_ = "RtApiDs::probeDeviceOpen: no devices found!"; + return FAILURE; + } + + if ( device >= nDevices ) { + // This should not happen because a check is made before this function is called. + errorText_ = "RtApiDs::probeDeviceOpen: device ID is invalid!"; + return FAILURE; + } + + if ( mode == OUTPUT ) { + if ( dsDevices[ device ].validId[0] == false ) { + errorStream_ << "RtApiDs::probeDeviceOpen: device (" << device << ") does not support output!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + else { // mode == INPUT + if ( dsDevices[ device ].validId[1] == false ) { + errorStream_ << "RtApiDs::probeDeviceOpen: device (" << device << ") does not support input!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + + // According to a note in PortAudio, using GetDesktopWindow() + // instead of GetForegroundWindow() is supposed to avoid problems + // that occur when the application's window is not the foreground + // window. Also, if the application window closes before the + // DirectSound buffer, DirectSound can crash. In the past, I had + // problems when using GetDesktopWindow() but it seems fine now + // (January 2010). I'll leave it commented here. + // HWND hWnd = GetForegroundWindow(); + HWND hWnd = GetDesktopWindow(); + + // Check the numberOfBuffers parameter and limit the lowest value to + // two. This is a judgement call and a value of two is probably too + // low for capture, but it should work for playback. + int nBuffers = 0; + if ( options ) nBuffers = options->numberOfBuffers; + if ( options && options->flags & RTAUDIO_MINIMIZE_LATENCY ) nBuffers = 2; + if ( nBuffers < 2 ) nBuffers = 3; + + // Check the lower range of the user-specified buffer size and set + // (arbitrarily) to a lower bound of 32. + if ( *bufferSize < 32 ) *bufferSize = 32; + + // Create the wave format structure. The data format setting will + // be determined later. + WAVEFORMATEX waveFormat; + ZeroMemory( &waveFormat, sizeof(WAVEFORMATEX) ); + waveFormat.wFormatTag = WAVE_FORMAT_PCM; + waveFormat.nChannels = channels + firstChannel; + waveFormat.nSamplesPerSec = (unsigned long) sampleRate; + + // Determine the device buffer size. By default, we'll use the value + // defined above (32K), but we will grow it to make allowances for + // very large software buffer sizes. + DWORD dsBufferSize = MINIMUM_DEVICE_BUFFER_SIZE; + DWORD dsPointerLeadTime = 0; + + void *ohandle = 0, *bhandle = 0; + HRESULT result; + if ( mode == OUTPUT ) { + + LPDIRECTSOUND output; + result = DirectSoundCreate( dsDevices[ device ].id[0], &output, NULL ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") opening output device (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + DSCAPS outCaps; + outCaps.dwSize = sizeof( outCaps ); + result = output->GetCaps( &outCaps ); + if ( FAILED( result ) ) { + output->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") getting capabilities (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Check channel information. + if ( channels + firstChannel == 2 && !( outCaps.dwFlags & DSCAPS_PRIMARYSTEREO ) ) { + errorStream_ << "RtApiDs::getDeviceInfo: the output device (" << dsDevices[ device ].name << ") does not support stereo playback."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Check format information. Use 16-bit format unless not + // supported or user requests 8-bit. + if ( outCaps.dwFlags & DSCAPS_PRIMARY16BIT && + !( format == RTAUDIO_SINT8 && outCaps.dwFlags & DSCAPS_PRIMARY8BIT ) ) { + waveFormat.wBitsPerSample = 16; + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + } + else { + waveFormat.wBitsPerSample = 8; + stream_.deviceFormat[mode] = RTAUDIO_SINT8; + } + stream_.userFormat = format; + + // Update wave format structure and buffer information. + waveFormat.nBlockAlign = waveFormat.nChannels * waveFormat.wBitsPerSample / 8; + waveFormat.nAvgBytesPerSec = waveFormat.nSamplesPerSec * waveFormat.nBlockAlign; + dsPointerLeadTime = nBuffers * (*bufferSize) * (waveFormat.wBitsPerSample / 8) * channels; + + // If the user wants an even bigger buffer, increase the device buffer size accordingly. + while ( dsPointerLeadTime * 2U > dsBufferSize ) + dsBufferSize *= 2; + + // Set cooperative level to DSSCL_EXCLUSIVE ... sound stops when window focus changes. + // result = output->SetCooperativeLevel( hWnd, DSSCL_EXCLUSIVE ); + // Set cooperative level to DSSCL_PRIORITY ... sound remains when window focus changes. + result = output->SetCooperativeLevel( hWnd, DSSCL_PRIORITY ); + if ( FAILED( result ) ) { + output->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") setting cooperative level (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Even though we will write to the secondary buffer, we need to + // access the primary buffer to set the correct output format + // (since the default is 8-bit, 22 kHz!). Setup the DS primary + // buffer description. + DSBUFFERDESC bufferDescription; + ZeroMemory( &bufferDescription, sizeof( DSBUFFERDESC ) ); + bufferDescription.dwSize = sizeof( DSBUFFERDESC ); + bufferDescription.dwFlags = DSBCAPS_PRIMARYBUFFER; + + // Obtain the primary buffer + LPDIRECTSOUNDBUFFER buffer; + result = output->CreateSoundBuffer( &bufferDescription, &buffer, NULL ); + if ( FAILED( result ) ) { + output->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") accessing primary buffer (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Set the primary DS buffer sound format. + result = buffer->SetFormat( &waveFormat ); + if ( FAILED( result ) ) { + output->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") setting primary buffer format (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Setup the secondary DS buffer description. + ZeroMemory( &bufferDescription, sizeof( DSBUFFERDESC ) ); + bufferDescription.dwSize = sizeof( DSBUFFERDESC ); + bufferDescription.dwFlags = ( DSBCAPS_STICKYFOCUS | + DSBCAPS_GLOBALFOCUS | + DSBCAPS_GETCURRENTPOSITION2 | + DSBCAPS_LOCHARDWARE ); // Force hardware mixing + bufferDescription.dwBufferBytes = dsBufferSize; + bufferDescription.lpwfxFormat = &waveFormat; + + // Try to create the secondary DS buffer. If that doesn't work, + // try to use software mixing. Otherwise, there's a problem. + result = output->CreateSoundBuffer( &bufferDescription, &buffer, NULL ); + if ( FAILED( result ) ) { + bufferDescription.dwFlags = ( DSBCAPS_STICKYFOCUS | + DSBCAPS_GLOBALFOCUS | + DSBCAPS_GETCURRENTPOSITION2 | + DSBCAPS_LOCSOFTWARE ); // Force software mixing + result = output->CreateSoundBuffer( &bufferDescription, &buffer, NULL ); + if ( FAILED( result ) ) { + output->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") creating secondary buffer (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + + // Get the buffer size ... might be different from what we specified. + DSBCAPS dsbcaps; + dsbcaps.dwSize = sizeof( DSBCAPS ); + result = buffer->GetCaps( &dsbcaps ); + if ( FAILED( result ) ) { + output->Release(); + buffer->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") getting buffer settings (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + dsBufferSize = dsbcaps.dwBufferBytes; + + // Lock the DS buffer + LPVOID audioPtr; + DWORD dataLen; + result = buffer->Lock( 0, dsBufferSize, &audioPtr, &dataLen, NULL, NULL, 0 ); + if ( FAILED( result ) ) { + output->Release(); + buffer->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") locking buffer (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Zero the DS buffer + ZeroMemory( audioPtr, dataLen ); + + // Unlock the DS buffer + result = buffer->Unlock( audioPtr, dataLen, NULL, 0 ); + if ( FAILED( result ) ) { + output->Release(); + buffer->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") unlocking buffer (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + ohandle = (void *) output; + bhandle = (void *) buffer; + } + + if ( mode == INPUT ) { + + LPDIRECTSOUNDCAPTURE input; + result = DirectSoundCaptureCreate( dsDevices[ device ].id[1], &input, NULL ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") opening input device (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + DSCCAPS inCaps; + inCaps.dwSize = sizeof( inCaps ); + result = input->GetCaps( &inCaps ); + if ( FAILED( result ) ) { + input->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") getting input capabilities (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Check channel information. + if ( inCaps.dwChannels < channels + firstChannel ) { + errorText_ = "RtApiDs::getDeviceInfo: the input device does not support requested input channels."; + return FAILURE; + } + + // Check format information. Use 16-bit format unless user + // requests 8-bit. + DWORD deviceFormats; + if ( channels + firstChannel == 2 ) { + deviceFormats = WAVE_FORMAT_1S08 | WAVE_FORMAT_2S08 | WAVE_FORMAT_4S08 | WAVE_FORMAT_96S08; + if ( format == RTAUDIO_SINT8 && inCaps.dwFormats & deviceFormats ) { + waveFormat.wBitsPerSample = 8; + stream_.deviceFormat[mode] = RTAUDIO_SINT8; + } + else { // assume 16-bit is supported + waveFormat.wBitsPerSample = 16; + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + } + } + else { // channel == 1 + deviceFormats = WAVE_FORMAT_1M08 | WAVE_FORMAT_2M08 | WAVE_FORMAT_4M08 | WAVE_FORMAT_96M08; + if ( format == RTAUDIO_SINT8 && inCaps.dwFormats & deviceFormats ) { + waveFormat.wBitsPerSample = 8; + stream_.deviceFormat[mode] = RTAUDIO_SINT8; + } + else { // assume 16-bit is supported + waveFormat.wBitsPerSample = 16; + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + } + } + stream_.userFormat = format; + + // Update wave format structure and buffer information. + waveFormat.nBlockAlign = waveFormat.nChannels * waveFormat.wBitsPerSample / 8; + waveFormat.nAvgBytesPerSec = waveFormat.nSamplesPerSec * waveFormat.nBlockAlign; + dsPointerLeadTime = nBuffers * (*bufferSize) * (waveFormat.wBitsPerSample / 8) * channels; + + // If the user wants an even bigger buffer, increase the device buffer size accordingly. + while ( dsPointerLeadTime * 2U > dsBufferSize ) + dsBufferSize *= 2; + + // Setup the secondary DS buffer description. + DSCBUFFERDESC bufferDescription; + ZeroMemory( &bufferDescription, sizeof( DSCBUFFERDESC ) ); + bufferDescription.dwSize = sizeof( DSCBUFFERDESC ); + bufferDescription.dwFlags = 0; + bufferDescription.dwReserved = 0; + bufferDescription.dwBufferBytes = dsBufferSize; + bufferDescription.lpwfxFormat = &waveFormat; + + // Create the capture buffer. + LPDIRECTSOUNDCAPTUREBUFFER buffer; + result = input->CreateCaptureBuffer( &bufferDescription, &buffer, NULL ); + if ( FAILED( result ) ) { + input->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") creating input buffer (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Get the buffer size ... might be different from what we specified. + DSCBCAPS dscbcaps; + dscbcaps.dwSize = sizeof( DSCBCAPS ); + result = buffer->GetCaps( &dscbcaps ); + if ( FAILED( result ) ) { + input->Release(); + buffer->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") getting buffer settings (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + dsBufferSize = dscbcaps.dwBufferBytes; + + // NOTE: We could have a problem here if this is a duplex stream + // and the play and capture hardware buffer sizes are different + // (I'm actually not sure if that is a problem or not). + // Currently, we are not verifying that. + + // Lock the capture buffer + LPVOID audioPtr; + DWORD dataLen; + result = buffer->Lock( 0, dsBufferSize, &audioPtr, &dataLen, NULL, NULL, 0 ); + if ( FAILED( result ) ) { + input->Release(); + buffer->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") locking input buffer (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Zero the buffer + ZeroMemory( audioPtr, dataLen ); + + // Unlock the buffer + result = buffer->Unlock( audioPtr, dataLen, NULL, 0 ); + if ( FAILED( result ) ) { + input->Release(); + buffer->Release(); + errorStream_ << "RtApiDs::probeDeviceOpen: error (" << getErrorString( result ) << ") unlocking input buffer (" << dsDevices[ device ].name << ")!"; + errorText_ = errorStream_.str(); + return FAILURE; + } + + ohandle = (void *) input; + bhandle = (void *) buffer; + } + + // Set various stream parameters + DsHandle *handle = 0; + stream_.nDeviceChannels[mode] = channels + firstChannel; + stream_.nUserChannels[mode] = channels; + stream_.bufferSize = *bufferSize; + stream_.channelOffset[mode] = firstChannel; + stream_.deviceInterleaved[mode] = true; + if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false; + else stream_.userInterleaved = true; + + // Set flag for buffer conversion + stream_.doConvertBuffer[mode] = false; + if (stream_.nUserChannels[mode] != stream_.nDeviceChannels[mode]) + stream_.doConvertBuffer[mode] = true; + if (stream_.userFormat != stream_.deviceFormat[mode]) + stream_.doConvertBuffer[mode] = true; + if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] && + stream_.nUserChannels[mode] > 1 ) + stream_.doConvertBuffer[mode] = true; + + // Allocate necessary internal buffers + long bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat ); + stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 ); + if ( stream_.userBuffer[mode] == NULL ) { + errorText_ = "RtApiDs::probeDeviceOpen: error allocating user buffer memory."; + goto error; + } + + if ( stream_.doConvertBuffer[mode] ) { + + bool makeBuffer = true; + bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] ); + if ( mode == INPUT ) { + if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) { + unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] ); + if ( bufferBytes <= (long) bytesOut ) makeBuffer = false; + } + } + + if ( makeBuffer ) { + bufferBytes *= *bufferSize; + if ( stream_.deviceBuffer ) free( stream_.deviceBuffer ); + stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 ); + if ( stream_.deviceBuffer == NULL ) { + errorText_ = "RtApiDs::probeDeviceOpen: error allocating device buffer memory."; + goto error; + } + } + } + + // Allocate our DsHandle structures for the stream. + if ( stream_.apiHandle == 0 ) { + try { + handle = new DsHandle; + } + catch ( std::bad_alloc& ) { + errorText_ = "RtApiDs::probeDeviceOpen: error allocating AsioHandle memory."; + goto error; + } + + // Create a manual-reset event. + handle->condition = CreateEvent( NULL, // no security + TRUE, // manual-reset + FALSE, // non-signaled initially + NULL ); // unnamed + stream_.apiHandle = (void *) handle; + } + else + handle = (DsHandle *) stream_.apiHandle; + handle->id[mode] = ohandle; + handle->buffer[mode] = bhandle; + handle->dsBufferSize[mode] = dsBufferSize; + handle->dsPointerLeadTime[mode] = dsPointerLeadTime; + + stream_.device[mode] = device; + stream_.state = STREAM_STOPPED; + if ( stream_.mode == OUTPUT && mode == INPUT ) + // We had already set up an output stream. + stream_.mode = DUPLEX; + else + stream_.mode = mode; + stream_.nBuffers = nBuffers; + stream_.sampleRate = sampleRate; + + // Setup the buffer conversion information structure. + if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, firstChannel ); + + // Setup the callback thread. + if ( stream_.callbackInfo.isRunning == false ) { + unsigned threadId; + stream_.callbackInfo.isRunning = true; + stream_.callbackInfo.object = (void *) this; + stream_.callbackInfo.thread = _beginthreadex( NULL, 0, &callbackHandler, + &stream_.callbackInfo, 0, &threadId ); + if ( stream_.callbackInfo.thread == 0 ) { + errorText_ = "RtApiDs::probeDeviceOpen: error creating callback thread!"; + goto error; + } + + // Boost DS thread priority + SetThreadPriority( (HANDLE) stream_.callbackInfo.thread, THREAD_PRIORITY_HIGHEST ); + } + return SUCCESS; + + error: + if ( handle ) { + if ( handle->buffer[0] ) { // the object pointer can be NULL and valid + LPDIRECTSOUND object = (LPDIRECTSOUND) handle->id[0]; + LPDIRECTSOUNDBUFFER buffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0]; + if ( buffer ) buffer->Release(); + object->Release(); + } + if ( handle->buffer[1] ) { + LPDIRECTSOUNDCAPTURE object = (LPDIRECTSOUNDCAPTURE) handle->id[1]; + LPDIRECTSOUNDCAPTUREBUFFER buffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1]; + if ( buffer ) buffer->Release(); + object->Release(); + } + CloseHandle( handle->condition ); + delete handle; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + stream_.state = STREAM_CLOSED; + return FAILURE; +} + +void RtApiDs :: closeStream() +{ + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiDs::closeStream(): no open stream to close!"; + error( RtAudioError::WARNING ); + return; + } + + // Stop the callback thread. + stream_.callbackInfo.isRunning = false; + WaitForSingleObject( (HANDLE) stream_.callbackInfo.thread, INFINITE ); + CloseHandle( (HANDLE) stream_.callbackInfo.thread ); + + DsHandle *handle = (DsHandle *) stream_.apiHandle; + if ( handle ) { + if ( handle->buffer[0] ) { // the object pointer can be NULL and valid + LPDIRECTSOUND object = (LPDIRECTSOUND) handle->id[0]; + LPDIRECTSOUNDBUFFER buffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0]; + if ( buffer ) { + buffer->Stop(); + buffer->Release(); + } + object->Release(); + } + if ( handle->buffer[1] ) { + LPDIRECTSOUNDCAPTURE object = (LPDIRECTSOUNDCAPTURE) handle->id[1]; + LPDIRECTSOUNDCAPTUREBUFFER buffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1]; + if ( buffer ) { + buffer->Stop(); + buffer->Release(); + } + object->Release(); + } + CloseHandle( handle->condition ); + delete handle; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + stream_.mode = UNINITIALIZED; + stream_.state = STREAM_CLOSED; +} + +void RtApiDs :: startStream() +{ + verifyStream(); + if ( stream_.state == STREAM_RUNNING ) { + errorText_ = "RtApiDs::startStream(): the stream is already running!"; + error( RtAudioError::WARNING ); + return; + } + + DsHandle *handle = (DsHandle *) stream_.apiHandle; + + // Increase scheduler frequency on lesser windows (a side-effect of + // increasing timer accuracy). On greater windows (Win2K or later), + // this is already in effect. + timeBeginPeriod( 1 ); + + buffersRolling = false; + duplexPrerollBytes = 0; + + if ( stream_.mode == DUPLEX ) { + // 0.5 seconds of silence in DUPLEX mode while the devices spin up and synchronize. + duplexPrerollBytes = (int) ( 0.5 * stream_.sampleRate * formatBytes( stream_.deviceFormat[1] ) * stream_.nDeviceChannels[1] ); + } + + HRESULT result = 0; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + LPDIRECTSOUNDBUFFER buffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0]; + result = buffer->Play( 0, 0, DSBPLAY_LOOPING ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::startStream: error (" << getErrorString( result ) << ") starting output buffer!"; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) { + + LPDIRECTSOUNDCAPTUREBUFFER buffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1]; + result = buffer->Start( DSCBSTART_LOOPING ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::startStream: error (" << getErrorString( result ) << ") starting input buffer!"; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + handle->drainCounter = 0; + handle->internalDrain = false; + ResetEvent( handle->condition ); + stream_.state = STREAM_RUNNING; + + unlock: + if ( FAILED( result ) ) error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiDs :: stopStream() +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiDs::stopStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + HRESULT result = 0; + LPVOID audioPtr; + DWORD dataLen; + DsHandle *handle = (DsHandle *) stream_.apiHandle; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + if ( handle->drainCounter == 0 ) { + handle->drainCounter = 2; + WaitForSingleObject( handle->condition, INFINITE ); // block until signaled + } + + stream_.state = STREAM_STOPPED; + + MUTEX_LOCK( &stream_.mutex ); + + // Stop the buffer and clear memory + LPDIRECTSOUNDBUFFER buffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0]; + result = buffer->Stop(); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") stopping output buffer!"; + errorText_ = errorStream_.str(); + goto unlock; + } + + // Lock the buffer and clear it so that if we start to play again, + // we won't have old data playing. + result = buffer->Lock( 0, handle->dsBufferSize[0], &audioPtr, &dataLen, NULL, NULL, 0 ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") locking output buffer!"; + errorText_ = errorStream_.str(); + goto unlock; + } + + // Zero the DS buffer + ZeroMemory( audioPtr, dataLen ); + + // Unlock the DS buffer + result = buffer->Unlock( audioPtr, dataLen, NULL, 0 ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") unlocking output buffer!"; + errorText_ = errorStream_.str(); + goto unlock; + } + + // If we start playing again, we must begin at beginning of buffer. + handle->bufferPointer[0] = 0; + } + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) { + LPDIRECTSOUNDCAPTUREBUFFER buffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1]; + audioPtr = NULL; + dataLen = 0; + + stream_.state = STREAM_STOPPED; + + if ( stream_.mode != DUPLEX ) + MUTEX_LOCK( &stream_.mutex ); + + result = buffer->Stop(); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") stopping input buffer!"; + errorText_ = errorStream_.str(); + goto unlock; + } + + // Lock the buffer and clear it so that if we start to play again, + // we won't have old data playing. + result = buffer->Lock( 0, handle->dsBufferSize[1], &audioPtr, &dataLen, NULL, NULL, 0 ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") locking input buffer!"; + errorText_ = errorStream_.str(); + goto unlock; + } + + // Zero the DS buffer + ZeroMemory( audioPtr, dataLen ); + + // Unlock the DS buffer + result = buffer->Unlock( audioPtr, dataLen, NULL, 0 ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::stopStream: error (" << getErrorString( result ) << ") unlocking input buffer!"; + errorText_ = errorStream_.str(); + goto unlock; + } + + // If we start recording again, we must begin at beginning of buffer. + handle->bufferPointer[1] = 0; + } + + unlock: + timeEndPeriod( 1 ); // revert to normal scheduler frequency on lesser windows. + MUTEX_UNLOCK( &stream_.mutex ); + + if ( FAILED( result ) ) error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiDs :: abortStream() +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiDs::abortStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + DsHandle *handle = (DsHandle *) stream_.apiHandle; + handle->drainCounter = 2; + + stopStream(); +} + +void RtApiDs :: callbackEvent() +{ + if ( stream_.state == STREAM_STOPPED || stream_.state == STREAM_STOPPING ) { + Sleep( 50 ); // sleep 50 milliseconds + return; + } + + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiDs::callbackEvent(): the stream is closed ... this shouldn't happen!"; + error( RtAudioError::WARNING ); + return; + } + + CallbackInfo *info = (CallbackInfo *) &stream_.callbackInfo; + DsHandle *handle = (DsHandle *) stream_.apiHandle; + + // Check if we were draining the stream and signal is finished. + if ( handle->drainCounter > stream_.nBuffers + 2 ) { + + stream_.state = STREAM_STOPPING; + if ( handle->internalDrain == false ) + SetEvent( handle->condition ); + else + stopStream(); + return; + } + + // Invoke user callback to get fresh output data UNLESS we are + // draining stream. + if ( handle->drainCounter == 0 ) { + RtAudioCallback callback = (RtAudioCallback) info->callback; + double streamTime = getStreamTime(); + RtAudioStreamStatus status = 0; + if ( stream_.mode != INPUT && handle->xrun[0] == true ) { + status |= RTAUDIO_OUTPUT_UNDERFLOW; + handle->xrun[0] = false; + } + if ( stream_.mode != OUTPUT && handle->xrun[1] == true ) { + status |= RTAUDIO_INPUT_OVERFLOW; + handle->xrun[1] = false; + } + int cbReturnValue = callback( stream_.userBuffer[0], stream_.userBuffer[1], + stream_.bufferSize, streamTime, status, info->userData ); + if ( cbReturnValue == 2 ) { + stream_.state = STREAM_STOPPING; + handle->drainCounter = 2; + abortStream(); + return; + } + else if ( cbReturnValue == 1 ) { + handle->drainCounter = 1; + handle->internalDrain = true; + } + } + + HRESULT result; + DWORD currentWritePointer, safeWritePointer; + DWORD currentReadPointer, safeReadPointer; + UINT nextWritePointer; + + LPVOID buffer1 = NULL; + LPVOID buffer2 = NULL; + DWORD bufferSize1 = 0; + DWORD bufferSize2 = 0; + + char *buffer; + long bufferBytes; + + MUTEX_LOCK( &stream_.mutex ); + if ( stream_.state == STREAM_STOPPED ) { + MUTEX_UNLOCK( &stream_.mutex ); + return; + } + + if ( buffersRolling == false ) { + if ( stream_.mode == DUPLEX ) { + //assert( handle->dsBufferSize[0] == handle->dsBufferSize[1] ); + + // It takes a while for the devices to get rolling. As a result, + // there's no guarantee that the capture and write device pointers + // will move in lockstep. Wait here for both devices to start + // rolling, and then set our buffer pointers accordingly. + // e.g. Crystal Drivers: the capture buffer starts up 5700 to 9600 + // bytes later than the write buffer. + + // Stub: a serious risk of having a pre-emptive scheduling round + // take place between the two GetCurrentPosition calls... but I'm + // really not sure how to solve the problem. Temporarily boost to + // Realtime priority, maybe; but I'm not sure what priority the + // DirectSound service threads run at. We *should* be roughly + // within a ms or so of correct. + + LPDIRECTSOUNDBUFFER dsWriteBuffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0]; + LPDIRECTSOUNDCAPTUREBUFFER dsCaptureBuffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1]; + + DWORD startSafeWritePointer, startSafeReadPointer; + + result = dsWriteBuffer->GetCurrentPosition( NULL, &startSafeWritePointer ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current write position!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + result = dsCaptureBuffer->GetCurrentPosition( NULL, &startSafeReadPointer ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current read position!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + while ( true ) { + result = dsWriteBuffer->GetCurrentPosition( NULL, &safeWritePointer ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current write position!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + result = dsCaptureBuffer->GetCurrentPosition( NULL, &safeReadPointer ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current read position!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + if ( safeWritePointer != startSafeWritePointer && safeReadPointer != startSafeReadPointer ) break; + Sleep( 1 ); + } + + //assert( handle->dsBufferSize[0] == handle->dsBufferSize[1] ); + + handle->bufferPointer[0] = safeWritePointer + handle->dsPointerLeadTime[0]; + if ( handle->bufferPointer[0] >= handle->dsBufferSize[0] ) handle->bufferPointer[0] -= handle->dsBufferSize[0]; + handle->bufferPointer[1] = safeReadPointer; + } + else if ( stream_.mode == OUTPUT ) { + + // Set the proper nextWritePosition after initial startup. + LPDIRECTSOUNDBUFFER dsWriteBuffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0]; + result = dsWriteBuffer->GetCurrentPosition( ¤tWritePointer, &safeWritePointer ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current write position!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + handle->bufferPointer[0] = safeWritePointer + handle->dsPointerLeadTime[0]; + if ( handle->bufferPointer[0] >= handle->dsBufferSize[0] ) handle->bufferPointer[0] -= handle->dsBufferSize[0]; + } + + buffersRolling = true; + } + + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + LPDIRECTSOUNDBUFFER dsBuffer = (LPDIRECTSOUNDBUFFER) handle->buffer[0]; + + if ( handle->drainCounter > 1 ) { // write zeros to the output stream + bufferBytes = stream_.bufferSize * stream_.nUserChannels[0]; + bufferBytes *= formatBytes( stream_.userFormat ); + memset( stream_.userBuffer[0], 0, bufferBytes ); + } + + // Setup parameters and do buffer conversion if necessary. + if ( stream_.doConvertBuffer[0] ) { + buffer = stream_.deviceBuffer; + convertBuffer( buffer, stream_.userBuffer[0], stream_.convertInfo[0] ); + bufferBytes = stream_.bufferSize * stream_.nDeviceChannels[0]; + bufferBytes *= formatBytes( stream_.deviceFormat[0] ); + } + else { + buffer = stream_.userBuffer[0]; + bufferBytes = stream_.bufferSize * stream_.nUserChannels[0]; + bufferBytes *= formatBytes( stream_.userFormat ); + } + + // No byte swapping necessary in DirectSound implementation. + + // Ahhh ... windoze. 16-bit data is signed but 8-bit data is + // unsigned. So, we need to convert our signed 8-bit data here to + // unsigned. + if ( stream_.deviceFormat[0] == RTAUDIO_SINT8 ) + for ( int i=0; i<bufferBytes; i++ ) buffer[i] = (unsigned char) ( buffer[i] + 128 ); + + DWORD dsBufferSize = handle->dsBufferSize[0]; + nextWritePointer = handle->bufferPointer[0]; + + DWORD endWrite, leadPointer; + while ( true ) { + // Find out where the read and "safe write" pointers are. + result = dsBuffer->GetCurrentPosition( ¤tWritePointer, &safeWritePointer ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current write position!"; + errorText_ = errorStream_.str(); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + + // We will copy our output buffer into the region between + // safeWritePointer and leadPointer. If leadPointer is not + // beyond the next endWrite position, wait until it is. + leadPointer = safeWritePointer + handle->dsPointerLeadTime[0]; + //std::cout << "safeWritePointer = " << safeWritePointer << ", leadPointer = " << leadPointer << ", nextWritePointer = " << nextWritePointer << std::endl; + if ( leadPointer > dsBufferSize ) leadPointer -= dsBufferSize; + if ( leadPointer < nextWritePointer ) leadPointer += dsBufferSize; // unwrap offset + endWrite = nextWritePointer + bufferBytes; + + // Check whether the entire write region is behind the play pointer. + if ( leadPointer >= endWrite ) break; + + // If we are here, then we must wait until the leadPointer advances + // beyond the end of our next write region. We use the + // Sleep() function to suspend operation until that happens. + double millis = ( endWrite - leadPointer ) * 1000.0; + millis /= ( formatBytes( stream_.deviceFormat[0]) * stream_.nDeviceChannels[0] * stream_.sampleRate); + if ( millis < 1.0 ) millis = 1.0; + Sleep( (DWORD) millis ); + } + + if ( dsPointerBetween( nextWritePointer, safeWritePointer, currentWritePointer, dsBufferSize ) + || dsPointerBetween( endWrite, safeWritePointer, currentWritePointer, dsBufferSize ) ) { + // We've strayed into the forbidden zone ... resync the read pointer. + handle->xrun[0] = true; + nextWritePointer = safeWritePointer + handle->dsPointerLeadTime[0] - bufferBytes; + if ( nextWritePointer >= dsBufferSize ) nextWritePointer -= dsBufferSize; + handle->bufferPointer[0] = nextWritePointer; + endWrite = nextWritePointer + bufferBytes; + } + + // Lock free space in the buffer + result = dsBuffer->Lock( nextWritePointer, bufferBytes, &buffer1, + &bufferSize1, &buffer2, &bufferSize2, 0 ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") locking buffer during playback!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + + // Copy our buffer into the DS buffer + CopyMemory( buffer1, buffer, bufferSize1 ); + if ( buffer2 != NULL ) CopyMemory( buffer2, buffer+bufferSize1, bufferSize2 ); + + // Update our buffer offset and unlock sound buffer + dsBuffer->Unlock( buffer1, bufferSize1, buffer2, bufferSize2 ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") unlocking buffer during playback!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + nextWritePointer = ( nextWritePointer + bufferSize1 + bufferSize2 ) % dsBufferSize; + handle->bufferPointer[0] = nextWritePointer; + } + + // Don't bother draining input + if ( handle->drainCounter ) { + handle->drainCounter++; + goto unlock; + } + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) { + + // Setup parameters. + if ( stream_.doConvertBuffer[1] ) { + buffer = stream_.deviceBuffer; + bufferBytes = stream_.bufferSize * stream_.nDeviceChannels[1]; + bufferBytes *= formatBytes( stream_.deviceFormat[1] ); + } + else { + buffer = stream_.userBuffer[1]; + bufferBytes = stream_.bufferSize * stream_.nUserChannels[1]; + bufferBytes *= formatBytes( stream_.userFormat ); + } + + LPDIRECTSOUNDCAPTUREBUFFER dsBuffer = (LPDIRECTSOUNDCAPTUREBUFFER) handle->buffer[1]; + long nextReadPointer = handle->bufferPointer[1]; + DWORD dsBufferSize = handle->dsBufferSize[1]; + + // Find out where the write and "safe read" pointers are. + result = dsBuffer->GetCurrentPosition( ¤tReadPointer, &safeReadPointer ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current read position!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + + if ( safeReadPointer < (DWORD)nextReadPointer ) safeReadPointer += dsBufferSize; // unwrap offset + DWORD endRead = nextReadPointer + bufferBytes; + + // Handling depends on whether we are INPUT or DUPLEX. + // If we're in INPUT mode then waiting is a good thing. If we're in DUPLEX mode, + // then a wait here will drag the write pointers into the forbidden zone. + // + // In DUPLEX mode, rather than wait, we will back off the read pointer until + // it's in a safe position. This causes dropouts, but it seems to be the only + // practical way to sync up the read and write pointers reliably, given the + // the very complex relationship between phase and increment of the read and write + // pointers. + // + // In order to minimize audible dropouts in DUPLEX mode, we will + // provide a pre-roll period of 0.5 seconds in which we return + // zeros from the read buffer while the pointers sync up. + + if ( stream_.mode == DUPLEX ) { + if ( safeReadPointer < endRead ) { + if ( duplexPrerollBytes <= 0 ) { + // Pre-roll time over. Be more agressive. + int adjustment = endRead-safeReadPointer; + + handle->xrun[1] = true; + // Two cases: + // - large adjustments: we've probably run out of CPU cycles, so just resync exactly, + // and perform fine adjustments later. + // - small adjustments: back off by twice as much. + if ( adjustment >= 2*bufferBytes ) + nextReadPointer = safeReadPointer-2*bufferBytes; + else + nextReadPointer = safeReadPointer-bufferBytes-adjustment; + + if ( nextReadPointer < 0 ) nextReadPointer += dsBufferSize; + + } + else { + // In pre=roll time. Just do it. + nextReadPointer = safeReadPointer - bufferBytes; + while ( nextReadPointer < 0 ) nextReadPointer += dsBufferSize; + } + endRead = nextReadPointer + bufferBytes; + } + } + else { // mode == INPUT + while ( safeReadPointer < endRead && stream_.callbackInfo.isRunning ) { + // See comments for playback. + double millis = (endRead - safeReadPointer) * 1000.0; + millis /= ( formatBytes(stream_.deviceFormat[1]) * stream_.nDeviceChannels[1] * stream_.sampleRate); + if ( millis < 1.0 ) millis = 1.0; + Sleep( (DWORD) millis ); + + // Wake up and find out where we are now. + result = dsBuffer->GetCurrentPosition( ¤tReadPointer, &safeReadPointer ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") getting current read position!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + + if ( safeReadPointer < (DWORD)nextReadPointer ) safeReadPointer += dsBufferSize; // unwrap offset + } + } + + // Lock free space in the buffer + result = dsBuffer->Lock( nextReadPointer, bufferBytes, &buffer1, + &bufferSize1, &buffer2, &bufferSize2, 0 ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") locking capture buffer!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + + if ( duplexPrerollBytes <= 0 ) { + // Copy our buffer into the DS buffer + CopyMemory( buffer, buffer1, bufferSize1 ); + if ( buffer2 != NULL ) CopyMemory( buffer+bufferSize1, buffer2, bufferSize2 ); + } + else { + memset( buffer, 0, bufferSize1 ); + if ( buffer2 != NULL ) memset( buffer + bufferSize1, 0, bufferSize2 ); + duplexPrerollBytes -= bufferSize1 + bufferSize2; + } + + // Update our buffer offset and unlock sound buffer + nextReadPointer = ( nextReadPointer + bufferSize1 + bufferSize2 ) % dsBufferSize; + dsBuffer->Unlock( buffer1, bufferSize1, buffer2, bufferSize2 ); + if ( FAILED( result ) ) { + errorStream_ << "RtApiDs::callbackEvent: error (" << getErrorString( result ) << ") unlocking capture buffer!"; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + handle->bufferPointer[1] = nextReadPointer; + + // No byte swapping necessary in DirectSound implementation. + + // If necessary, convert 8-bit data from unsigned to signed. + if ( stream_.deviceFormat[1] == RTAUDIO_SINT8 ) + for ( int j=0; j<bufferBytes; j++ ) buffer[j] = (signed char) ( buffer[j] - 128 ); + + // Do buffer conversion if necessary. + if ( stream_.doConvertBuffer[1] ) + convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] ); + } + + unlock: + MUTEX_UNLOCK( &stream_.mutex ); + RtApi::tickStreamTime(); +} + +// Definitions for utility functions and callbacks +// specific to the DirectSound implementation. + +static unsigned __stdcall callbackHandler( void *ptr ) +{ + CallbackInfo *info = (CallbackInfo *) ptr; + RtApiDs *object = (RtApiDs *) info->object; + bool* isRunning = &info->isRunning; + + while ( *isRunning == true ) { + object->callbackEvent(); + } + + _endthreadex( 0 ); + return 0; +} + +static BOOL CALLBACK deviceQueryCallback( LPGUID lpguid, + LPCTSTR description, + LPCTSTR /*module*/, + LPVOID lpContext ) +{ + struct DsProbeData& probeInfo = *(struct DsProbeData*) lpContext; + std::vector<struct DsDevice>& dsDevices = *probeInfo.dsDevices; + + HRESULT hr; + bool validDevice = false; + if ( probeInfo.isInput == true ) { + DSCCAPS caps; + LPDIRECTSOUNDCAPTURE object; + + hr = DirectSoundCaptureCreate( lpguid, &object, NULL ); + if ( hr != DS_OK ) return TRUE; + + caps.dwSize = sizeof(caps); + hr = object->GetCaps( &caps ); + if ( hr == DS_OK ) { + if ( caps.dwChannels > 0 && caps.dwFormats > 0 ) + validDevice = true; + } + object->Release(); + } + else { + DSCAPS caps; + LPDIRECTSOUND object; + hr = DirectSoundCreate( lpguid, &object, NULL ); + if ( hr != DS_OK ) return TRUE; + + caps.dwSize = sizeof(caps); + hr = object->GetCaps( &caps ); + if ( hr == DS_OK ) { + if ( caps.dwFlags & DSCAPS_PRIMARYMONO || caps.dwFlags & DSCAPS_PRIMARYSTEREO ) + validDevice = true; + } + object->Release(); + } + + // If good device, then save its name and guid. + std::string name = convertCharPointerToStdString( description ); + //if ( name == "Primary Sound Driver" || name == "Primary Sound Capture Driver" ) + if ( lpguid == NULL ) + name = "Default Device"; + if ( validDevice ) { + for ( unsigned int i=0; i<dsDevices.size(); i++ ) { + if ( dsDevices[i].name == name ) { + dsDevices[i].found = true; + if ( probeInfo.isInput ) { + dsDevices[i].id[1] = lpguid; + dsDevices[i].validId[1] = true; + } + else { + dsDevices[i].id[0] = lpguid; + dsDevices[i].validId[0] = true; + } + return TRUE; + } + } + + DsDevice device; + device.name = name; + device.found = true; + if ( probeInfo.isInput ) { + device.id[1] = lpguid; + device.validId[1] = true; + } + else { + device.id[0] = lpguid; + device.validId[0] = true; + } + dsDevices.push_back( device ); + } + + return TRUE; +} + +static const char* getErrorString( int code ) +{ + switch ( code ) { + + case DSERR_ALLOCATED: + return "Already allocated"; + + case DSERR_CONTROLUNAVAIL: + return "Control unavailable"; + + case DSERR_INVALIDPARAM: + return "Invalid parameter"; + + case DSERR_INVALIDCALL: + return "Invalid call"; + + case DSERR_GENERIC: + return "Generic error"; + + case DSERR_PRIOLEVELNEEDED: + return "Priority level needed"; + + case DSERR_OUTOFMEMORY: + return "Out of memory"; + + case DSERR_BADFORMAT: + return "The sample rate or the channel format is not supported"; + + case DSERR_UNSUPPORTED: + return "Not supported"; + + case DSERR_NODRIVER: + return "No driver"; + + case DSERR_ALREADYINITIALIZED: + return "Already initialized"; + + case DSERR_NOAGGREGATION: + return "No aggregation"; + + case DSERR_BUFFERLOST: + return "Buffer lost"; + + case DSERR_OTHERAPPHASPRIO: + return "Another application already has priority"; + + case DSERR_UNINITIALIZED: + return "Uninitialized"; + + default: + return "DirectSound unknown error"; + } +} +//******************** End of __WINDOWS_DS__ *********************// +#endif + + +#if defined(__LINUX_ALSA__) + +#include <alsa/asoundlib.h> +#include <unistd.h> + + // A structure to hold various information related to the ALSA API + // implementation. +struct AlsaHandle { + snd_pcm_t *handles[2]; + bool synchronized; + bool xrun[2]; + pthread_cond_t runnable_cv; + bool runnable; + + AlsaHandle() + :synchronized(false), runnable(false) { xrun[0] = false; xrun[1] = false; } +}; + +static void *alsaCallbackHandler( void * ptr ); + +RtApiAlsa :: RtApiAlsa() +{ + // Nothing to do here. +} + +RtApiAlsa :: ~RtApiAlsa() +{ + if ( stream_.state != STREAM_CLOSED ) closeStream(); +} + +unsigned int RtApiAlsa :: getDeviceCount( void ) +{ + unsigned nDevices = 0; + int result, subdevice, card; + char name[64]; + snd_ctl_t *handle; + + // Count cards and devices + card = -1; + snd_card_next( &card ); + while ( card >= 0 ) { + sprintf( name, "hw:%d", card ); + result = snd_ctl_open( &handle, name, 0 ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::getDeviceCount: control open, card = " << card << ", " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + goto nextcard; + } + subdevice = -1; + while( 1 ) { + result = snd_ctl_pcm_next_device( handle, &subdevice ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::getDeviceCount: control next device, card = " << card << ", " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + break; + } + if ( subdevice < 0 ) + break; + nDevices++; + } + nextcard: + snd_ctl_close( handle ); + snd_card_next( &card ); + } + + result = snd_ctl_open( &handle, "default", 0 ); + if (result == 0) { + nDevices++; + snd_ctl_close( handle ); + } + + return nDevices; +} + +RtAudio::DeviceInfo RtApiAlsa :: getDeviceInfo( unsigned int device ) +{ + RtAudio::DeviceInfo info; + info.probed = false; + + unsigned nDevices = 0; + int result, subdevice, card; + char name[64]; + snd_ctl_t *chandle; + + // Count cards and devices + card = -1; + subdevice = -1; + snd_card_next( &card ); + while ( card >= 0 ) { + sprintf( name, "hw:%d", card ); + result = snd_ctl_open( &chandle, name, SND_CTL_NONBLOCK ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::getDeviceInfo: control open, card = " << card << ", " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + goto nextcard; + } + subdevice = -1; + while( 1 ) { + result = snd_ctl_pcm_next_device( chandle, &subdevice ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::getDeviceInfo: control next device, card = " << card << ", " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + break; + } + if ( subdevice < 0 ) break; + if ( nDevices == device ) { + sprintf( name, "hw:%d,%d", card, subdevice ); + goto foundDevice; + } + nDevices++; + } + nextcard: + snd_ctl_close( chandle ); + snd_card_next( &card ); + } + + result = snd_ctl_open( &chandle, "default", SND_CTL_NONBLOCK ); + if ( result == 0 ) { + if ( nDevices == device ) { + strcpy( name, "default" ); + goto foundDevice; + } + nDevices++; + } + + if ( nDevices == 0 ) { + errorText_ = "RtApiAlsa::getDeviceInfo: no devices found!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + if ( device >= nDevices ) { + errorText_ = "RtApiAlsa::getDeviceInfo: device ID is invalid!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + foundDevice: + + // If a stream is already open, we cannot probe the stream devices. + // Thus, use the saved results. + if ( stream_.state != STREAM_CLOSED && + ( stream_.device[0] == device || stream_.device[1] == device ) ) { + snd_ctl_close( chandle ); + if ( device >= devices_.size() ) { + errorText_ = "RtApiAlsa::getDeviceInfo: device ID was not present before stream was opened."; + error( RtAudioError::WARNING ); + return info; + } + return devices_[ device ]; + } + + int openMode = SND_PCM_ASYNC; + snd_pcm_stream_t stream; + snd_pcm_info_t *pcminfo; + snd_pcm_info_alloca( &pcminfo ); + snd_pcm_t *phandle; + snd_pcm_hw_params_t *params; + snd_pcm_hw_params_alloca( ¶ms ); + + // First try for playback unless default device (which has subdev -1) + stream = SND_PCM_STREAM_PLAYBACK; + snd_pcm_info_set_stream( pcminfo, stream ); + if ( subdevice != -1 ) { + snd_pcm_info_set_device( pcminfo, subdevice ); + snd_pcm_info_set_subdevice( pcminfo, 0 ); + + result = snd_ctl_pcm_info( chandle, pcminfo ); + if ( result < 0 ) { + // Device probably doesn't support playback. + goto captureProbe; + } + } + + result = snd_pcm_open( &phandle, name, stream, openMode | SND_PCM_NONBLOCK ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_open error for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + goto captureProbe; + } + + // The device is open ... fill the parameter structure. + result = snd_pcm_hw_params_any( phandle, params ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_hw_params error for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + goto captureProbe; + } + + // Get output channel information. + unsigned int value; + result = snd_pcm_hw_params_get_channels_max( params, &value ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::getDeviceInfo: error getting device (" << name << ") output channels, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + goto captureProbe; + } + info.outputChannels = value; + snd_pcm_close( phandle ); + + captureProbe: + stream = SND_PCM_STREAM_CAPTURE; + snd_pcm_info_set_stream( pcminfo, stream ); + + // Now try for capture unless default device (with subdev = -1) + if ( subdevice != -1 ) { + result = snd_ctl_pcm_info( chandle, pcminfo ); + snd_ctl_close( chandle ); + if ( result < 0 ) { + // Device probably doesn't support capture. + if ( info.outputChannels == 0 ) return info; + goto probeParameters; + } + } + else + snd_ctl_close( chandle ); + + result = snd_pcm_open( &phandle, name, stream, openMode | SND_PCM_NONBLOCK); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_open error for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + if ( info.outputChannels == 0 ) return info; + goto probeParameters; + } + + // The device is open ... fill the parameter structure. + result = snd_pcm_hw_params_any( phandle, params ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_hw_params error for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + if ( info.outputChannels == 0 ) return info; + goto probeParameters; + } + + result = snd_pcm_hw_params_get_channels_max( params, &value ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::getDeviceInfo: error getting device (" << name << ") input channels, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + if ( info.outputChannels == 0 ) return info; + goto probeParameters; + } + info.inputChannels = value; + snd_pcm_close( phandle ); + + // If device opens for both playback and capture, we determine the channels. + if ( info.outputChannels > 0 && info.inputChannels > 0 ) + info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels; + + // ALSA doesn't provide default devices so we'll use the first available one. + if ( device == 0 && info.outputChannels > 0 ) + info.isDefaultOutput = true; + if ( device == 0 && info.inputChannels > 0 ) + info.isDefaultInput = true; + + probeParameters: + // At this point, we just need to figure out the supported data + // formats and sample rates. We'll proceed by opening the device in + // the direction with the maximum number of channels, or playback if + // they are equal. This might limit our sample rate options, but so + // be it. + + if ( info.outputChannels >= info.inputChannels ) + stream = SND_PCM_STREAM_PLAYBACK; + else + stream = SND_PCM_STREAM_CAPTURE; + snd_pcm_info_set_stream( pcminfo, stream ); + + result = snd_pcm_open( &phandle, name, stream, openMode | SND_PCM_NONBLOCK); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_open error for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // The device is open ... fill the parameter structure. + result = snd_pcm_hw_params_any( phandle, params ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::getDeviceInfo: snd_pcm_hw_params error for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Test our discrete set of sample rate values. + info.sampleRates.clear(); + for ( unsigned int i=0; i<MAX_SAMPLE_RATES; i++ ) { + if ( snd_pcm_hw_params_test_rate( phandle, params, SAMPLE_RATES[i], 0 ) == 0 ) { + info.sampleRates.push_back( SAMPLE_RATES[i] ); + + if ( !info.preferredSampleRate || ( SAMPLE_RATES[i] <= 48000 && SAMPLE_RATES[i] > info.preferredSampleRate ) ) + info.preferredSampleRate = SAMPLE_RATES[i]; + } + } + if ( info.sampleRates.size() == 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::getDeviceInfo: no supported sample rates found for device (" << name << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Probe the supported data formats ... we don't care about endian-ness just yet + snd_pcm_format_t format; + info.nativeFormats = 0; + format = SND_PCM_FORMAT_S8; + if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 ) + info.nativeFormats |= RTAUDIO_SINT8; + format = SND_PCM_FORMAT_S16; + if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 ) + info.nativeFormats |= RTAUDIO_SINT16; + format = SND_PCM_FORMAT_S24; + if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 ) + info.nativeFormats |= RTAUDIO_SINT24; + format = SND_PCM_FORMAT_S32; + if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 ) + info.nativeFormats |= RTAUDIO_SINT32; + format = SND_PCM_FORMAT_FLOAT; + if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 ) + info.nativeFormats |= RTAUDIO_FLOAT32; + format = SND_PCM_FORMAT_FLOAT64; + if ( snd_pcm_hw_params_test_format( phandle, params, format ) == 0 ) + info.nativeFormats |= RTAUDIO_FLOAT64; + + // Check that we have at least one supported format + if ( info.nativeFormats == 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::getDeviceInfo: pcm device (" << name << ") data format not supported by RtAudio."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Get the device name + char *cardname; + result = snd_card_get_name( card, &cardname ); + if ( result >= 0 ) { + sprintf( name, "hw:%s,%d", cardname, subdevice ); + free( cardname ); + } + info.name = name; + + // That's all ... close the device and return + snd_pcm_close( phandle ); + info.probed = true; + return info; +} + +void RtApiAlsa :: saveDeviceInfo( void ) +{ + devices_.clear(); + + unsigned int nDevices = getDeviceCount(); + devices_.resize( nDevices ); + for ( unsigned int i=0; i<nDevices; i++ ) + devices_[i] = getDeviceInfo( i ); +} + +bool RtApiAlsa :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels, + unsigned int firstChannel, unsigned int sampleRate, + RtAudioFormat format, unsigned int *bufferSize, + RtAudio::StreamOptions *options ) + +{ +#if defined(__RTAUDIO_DEBUG__) + snd_output_t *out; + snd_output_stdio_attach(&out, stderr, 0); +#endif + + // I'm not using the "plug" interface ... too much inconsistent behavior. + + unsigned nDevices = 0; + int result, subdevice, card; + char name[64]; + snd_ctl_t *chandle; + + if ( options && options->flags & RTAUDIO_ALSA_USE_DEFAULT ) + snprintf(name, sizeof(name), "%s", "default"); + else { + // Count cards and devices + card = -1; + snd_card_next( &card ); + while ( card >= 0 ) { + sprintf( name, "hw:%d", card ); + result = snd_ctl_open( &chandle, name, SND_CTL_NONBLOCK ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::probeDeviceOpen: control open, card = " << card << ", " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + subdevice = -1; + while( 1 ) { + result = snd_ctl_pcm_next_device( chandle, &subdevice ); + if ( result < 0 ) break; + if ( subdevice < 0 ) break; + if ( nDevices == device ) { + sprintf( name, "hw:%d,%d", card, subdevice ); + snd_ctl_close( chandle ); + goto foundDevice; + } + nDevices++; + } + snd_ctl_close( chandle ); + snd_card_next( &card ); + } + + result = snd_ctl_open( &chandle, "default", SND_CTL_NONBLOCK ); + if ( result == 0 ) { + if ( nDevices == device ) { + strcpy( name, "default" ); + goto foundDevice; + } + nDevices++; + } + + if ( nDevices == 0 ) { + // This should not happen because a check is made before this function is called. + errorText_ = "RtApiAlsa::probeDeviceOpen: no devices found!"; + return FAILURE; + } + + if ( device >= nDevices ) { + // This should not happen because a check is made before this function is called. + errorText_ = "RtApiAlsa::probeDeviceOpen: device ID is invalid!"; + return FAILURE; + } + } + + foundDevice: + + // The getDeviceInfo() function will not work for a device that is + // already open. Thus, we'll probe the system before opening a + // stream and save the results for use by getDeviceInfo(). + if ( mode == OUTPUT || ( mode == INPUT && stream_.mode != OUTPUT ) ) // only do once + this->saveDeviceInfo(); + + snd_pcm_stream_t stream; + if ( mode == OUTPUT ) + stream = SND_PCM_STREAM_PLAYBACK; + else + stream = SND_PCM_STREAM_CAPTURE; + + snd_pcm_t *phandle; + int openMode = SND_PCM_ASYNC; + result = snd_pcm_open( &phandle, name, stream, openMode ); + if ( result < 0 ) { + if ( mode == OUTPUT ) + errorStream_ << "RtApiAlsa::probeDeviceOpen: pcm device (" << name << ") won't open for output."; + else + errorStream_ << "RtApiAlsa::probeDeviceOpen: pcm device (" << name << ") won't open for input."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Fill the parameter structure. + snd_pcm_hw_params_t *hw_params; + snd_pcm_hw_params_alloca( &hw_params ); + result = snd_pcm_hw_params_any( phandle, hw_params ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error getting pcm device (" << name << ") parameters, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + +#if defined(__RTAUDIO_DEBUG__) + fprintf( stderr, "\nRtApiAlsa: dump hardware params just after device open:\n\n" ); + snd_pcm_hw_params_dump( hw_params, out ); +#endif + + // Set access ... check user preference. + if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) { + stream_.userInterleaved = false; + result = snd_pcm_hw_params_set_access( phandle, hw_params, SND_PCM_ACCESS_RW_NONINTERLEAVED ); + if ( result < 0 ) { + result = snd_pcm_hw_params_set_access( phandle, hw_params, SND_PCM_ACCESS_RW_INTERLEAVED ); + stream_.deviceInterleaved[mode] = true; + } + else + stream_.deviceInterleaved[mode] = false; + } + else { + stream_.userInterleaved = true; + result = snd_pcm_hw_params_set_access( phandle, hw_params, SND_PCM_ACCESS_RW_INTERLEAVED ); + if ( result < 0 ) { + result = snd_pcm_hw_params_set_access( phandle, hw_params, SND_PCM_ACCESS_RW_NONINTERLEAVED ); + stream_.deviceInterleaved[mode] = false; + } + else + stream_.deviceInterleaved[mode] = true; + } + + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting pcm device (" << name << ") access, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Determine how to set the device format. + stream_.userFormat = format; + snd_pcm_format_t deviceFormat = SND_PCM_FORMAT_UNKNOWN; + + if ( format == RTAUDIO_SINT8 ) + deviceFormat = SND_PCM_FORMAT_S8; + else if ( format == RTAUDIO_SINT16 ) + deviceFormat = SND_PCM_FORMAT_S16; + else if ( format == RTAUDIO_SINT24 ) + deviceFormat = SND_PCM_FORMAT_S24; + else if ( format == RTAUDIO_SINT32 ) + deviceFormat = SND_PCM_FORMAT_S32; + else if ( format == RTAUDIO_FLOAT32 ) + deviceFormat = SND_PCM_FORMAT_FLOAT; + else if ( format == RTAUDIO_FLOAT64 ) + deviceFormat = SND_PCM_FORMAT_FLOAT64; + + if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat) == 0) { + stream_.deviceFormat[mode] = format; + goto setFormat; + } + + // The user requested format is not natively supported by the device. + deviceFormat = SND_PCM_FORMAT_FLOAT64; + if ( snd_pcm_hw_params_test_format( phandle, hw_params, deviceFormat ) == 0 ) { + stream_.deviceFormat[mode] = RTAUDIO_FLOAT64; + goto setFormat; + } + + deviceFormat = SND_PCM_FORMAT_FLOAT; + if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) { + stream_.deviceFormat[mode] = RTAUDIO_FLOAT32; + goto setFormat; + } + + deviceFormat = SND_PCM_FORMAT_S32; + if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) { + stream_.deviceFormat[mode] = RTAUDIO_SINT32; + goto setFormat; + } + + deviceFormat = SND_PCM_FORMAT_S24; + if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) { + stream_.deviceFormat[mode] = RTAUDIO_SINT24; + goto setFormat; + } + + deviceFormat = SND_PCM_FORMAT_S16; + if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) { + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + goto setFormat; + } + + deviceFormat = SND_PCM_FORMAT_S8; + if ( snd_pcm_hw_params_test_format(phandle, hw_params, deviceFormat ) == 0 ) { + stream_.deviceFormat[mode] = RTAUDIO_SINT8; + goto setFormat; + } + + // If we get here, no supported format was found. + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: pcm device " << device << " data format not supported by RtAudio."; + errorText_ = errorStream_.str(); + return FAILURE; + + setFormat: + result = snd_pcm_hw_params_set_format( phandle, hw_params, deviceFormat ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting pcm device (" << name << ") data format, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Determine whether byte-swaping is necessary. + stream_.doByteSwap[mode] = false; + if ( deviceFormat != SND_PCM_FORMAT_S8 ) { + result = snd_pcm_format_cpu_endian( deviceFormat ); + if ( result == 0 ) + stream_.doByteSwap[mode] = true; + else if (result < 0) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error getting pcm device (" << name << ") endian-ness, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + + // Set the sample rate. + result = snd_pcm_hw_params_set_rate_near( phandle, hw_params, (unsigned int*) &sampleRate, 0 ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting sample rate on device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Determine the number of channels for this device. We support a possible + // minimum device channel number > than the value requested by the user. + stream_.nUserChannels[mode] = channels; + unsigned int value; + result = snd_pcm_hw_params_get_channels_max( hw_params, &value ); + unsigned int deviceChannels = value; + if ( result < 0 || deviceChannels < channels + firstChannel ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: requested channel parameters not supported by device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + result = snd_pcm_hw_params_get_channels_min( hw_params, &value ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error getting minimum channels for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + deviceChannels = value; + if ( deviceChannels < channels + firstChannel ) deviceChannels = channels + firstChannel; + stream_.nDeviceChannels[mode] = deviceChannels; + + // Set the device channels. + result = snd_pcm_hw_params_set_channels( phandle, hw_params, deviceChannels ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting channels for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Set the buffer (or period) size. + int dir = 0; + snd_pcm_uframes_t periodSize = *bufferSize; + result = snd_pcm_hw_params_set_period_size_near( phandle, hw_params, &periodSize, &dir ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting period size for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + *bufferSize = periodSize; + + // Set the buffer number, which in ALSA is referred to as the "period". + unsigned int periods = 0; + if ( options && options->flags & RTAUDIO_MINIMIZE_LATENCY ) periods = 2; + if ( options && options->numberOfBuffers > 0 ) periods = options->numberOfBuffers; + if ( periods < 2 ) periods = 4; // a fairly safe default value + result = snd_pcm_hw_params_set_periods_near( phandle, hw_params, &periods, &dir ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error setting periods for device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // If attempting to setup a duplex stream, the bufferSize parameter + // MUST be the same in both directions! + if ( stream_.mode == OUTPUT && mode == INPUT && *bufferSize != stream_.bufferSize ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: system error setting buffer size for duplex stream on device (" << name << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + stream_.bufferSize = *bufferSize; + + // Install the hardware configuration + result = snd_pcm_hw_params( phandle, hw_params ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error installing hardware configuration on device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + +#if defined(__RTAUDIO_DEBUG__) + fprintf(stderr, "\nRtApiAlsa: dump hardware params after installation:\n\n"); + snd_pcm_hw_params_dump( hw_params, out ); +#endif + + // Set the software configuration to fill buffers with zeros and prevent device stopping on xruns. + snd_pcm_sw_params_t *sw_params = NULL; + snd_pcm_sw_params_alloca( &sw_params ); + snd_pcm_sw_params_current( phandle, sw_params ); + snd_pcm_sw_params_set_start_threshold( phandle, sw_params, *bufferSize ); + snd_pcm_sw_params_set_stop_threshold( phandle, sw_params, ULONG_MAX ); + snd_pcm_sw_params_set_silence_threshold( phandle, sw_params, 0 ); + + // The following two settings were suggested by Theo Veenker + //snd_pcm_sw_params_set_avail_min( phandle, sw_params, *bufferSize ); + //snd_pcm_sw_params_set_xfer_align( phandle, sw_params, 1 ); + + // here are two options for a fix + //snd_pcm_sw_params_set_silence_size( phandle, sw_params, ULONG_MAX ); + snd_pcm_uframes_t val; + snd_pcm_sw_params_get_boundary( sw_params, &val ); + snd_pcm_sw_params_set_silence_size( phandle, sw_params, val ); + + result = snd_pcm_sw_params( phandle, sw_params ); + if ( result < 0 ) { + snd_pcm_close( phandle ); + errorStream_ << "RtApiAlsa::probeDeviceOpen: error installing software configuration on device (" << name << "), " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + return FAILURE; + } + +#if defined(__RTAUDIO_DEBUG__) + fprintf(stderr, "\nRtApiAlsa: dump software params after installation:\n\n"); + snd_pcm_sw_params_dump( sw_params, out ); +#endif + + // Set flags for buffer conversion + stream_.doConvertBuffer[mode] = false; + if ( stream_.userFormat != stream_.deviceFormat[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( stream_.nUserChannels[mode] < stream_.nDeviceChannels[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] && + stream_.nUserChannels[mode] > 1 ) + stream_.doConvertBuffer[mode] = true; + + // Allocate the ApiHandle if necessary and then save. + AlsaHandle *apiInfo = 0; + if ( stream_.apiHandle == 0 ) { + try { + apiInfo = (AlsaHandle *) new AlsaHandle; + } + catch ( std::bad_alloc& ) { + errorText_ = "RtApiAlsa::probeDeviceOpen: error allocating AlsaHandle memory."; + goto error; + } + + if ( pthread_cond_init( &apiInfo->runnable_cv, NULL ) ) { + errorText_ = "RtApiAlsa::probeDeviceOpen: error initializing pthread condition variable."; + goto error; + } + + stream_.apiHandle = (void *) apiInfo; + apiInfo->handles[0] = 0; + apiInfo->handles[1] = 0; + } + else { + apiInfo = (AlsaHandle *) stream_.apiHandle; + } + apiInfo->handles[mode] = phandle; + phandle = 0; + + // Allocate necessary internal buffers. + unsigned long bufferBytes; + bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat ); + stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 ); + if ( stream_.userBuffer[mode] == NULL ) { + errorText_ = "RtApiAlsa::probeDeviceOpen: error allocating user buffer memory."; + goto error; + } + + if ( stream_.doConvertBuffer[mode] ) { + + bool makeBuffer = true; + bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] ); + if ( mode == INPUT ) { + if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) { + unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] ); + if ( bufferBytes <= bytesOut ) makeBuffer = false; + } + } + + if ( makeBuffer ) { + bufferBytes *= *bufferSize; + if ( stream_.deviceBuffer ) free( stream_.deviceBuffer ); + stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 ); + if ( stream_.deviceBuffer == NULL ) { + errorText_ = "RtApiAlsa::probeDeviceOpen: error allocating device buffer memory."; + goto error; + } + } + } + + stream_.sampleRate = sampleRate; + stream_.nBuffers = periods; + stream_.device[mode] = device; + stream_.state = STREAM_STOPPED; + + // Setup the buffer conversion information structure. + if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, firstChannel ); + + // Setup thread if necessary. + if ( stream_.mode == OUTPUT && mode == INPUT ) { + // We had already set up an output stream. + stream_.mode = DUPLEX; + // Link the streams if possible. + apiInfo->synchronized = false; + if ( snd_pcm_link( apiInfo->handles[0], apiInfo->handles[1] ) == 0 ) + apiInfo->synchronized = true; + else { + errorText_ = "RtApiAlsa::probeDeviceOpen: unable to synchronize input and output devices."; + error( RtAudioError::WARNING ); + } + } + else { + stream_.mode = mode; + + // Setup callback thread. + stream_.callbackInfo.object = (void *) this; + + // Set the thread attributes for joinable and realtime scheduling + // priority (optional). The higher priority will only take affect + // if the program is run as root or suid. Note, under Linux + // processes with CAP_SYS_NICE privilege, a user can change + // scheduling policy and priority (thus need not be root). See + // POSIX "capabilities". + pthread_attr_t attr; + pthread_attr_init( &attr ); + pthread_attr_setdetachstate( &attr, PTHREAD_CREATE_JOINABLE ); + +#ifdef SCHED_RR // Undefined with some OSes (eg: NetBSD 1.6.x with GNU Pthread) + if ( options && options->flags & RTAUDIO_SCHEDULE_REALTIME ) { + // We previously attempted to increase the audio callback priority + // to SCHED_RR here via the attributes. However, while no errors + // were reported in doing so, it did not work. So, now this is + // done in the alsaCallbackHandler function. + stream_.callbackInfo.doRealtime = true; + int priority = options->priority; + int min = sched_get_priority_min( SCHED_RR ); + int max = sched_get_priority_max( SCHED_RR ); + if ( priority < min ) priority = min; + else if ( priority > max ) priority = max; + stream_.callbackInfo.priority = priority; + } +#endif + + stream_.callbackInfo.isRunning = true; + result = pthread_create( &stream_.callbackInfo.thread, &attr, alsaCallbackHandler, &stream_.callbackInfo ); + pthread_attr_destroy( &attr ); + if ( result ) { + stream_.callbackInfo.isRunning = false; + errorText_ = "RtApiAlsa::error creating callback thread!"; + goto error; + } + } + + return SUCCESS; + + error: + if ( apiInfo ) { + pthread_cond_destroy( &apiInfo->runnable_cv ); + if ( apiInfo->handles[0] ) snd_pcm_close( apiInfo->handles[0] ); + if ( apiInfo->handles[1] ) snd_pcm_close( apiInfo->handles[1] ); + delete apiInfo; + stream_.apiHandle = 0; + } + + if ( phandle) snd_pcm_close( phandle ); + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + stream_.state = STREAM_CLOSED; + return FAILURE; +} + +void RtApiAlsa :: closeStream() +{ + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiAlsa::closeStream(): no open stream to close!"; + error( RtAudioError::WARNING ); + return; + } + + AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle; + stream_.callbackInfo.isRunning = false; + MUTEX_LOCK( &stream_.mutex ); + if ( stream_.state == STREAM_STOPPED ) { + apiInfo->runnable = true; + pthread_cond_signal( &apiInfo->runnable_cv ); + } + MUTEX_UNLOCK( &stream_.mutex ); + pthread_join( stream_.callbackInfo.thread, NULL ); + + if ( stream_.state == STREAM_RUNNING ) { + stream_.state = STREAM_STOPPED; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) + snd_pcm_drop( apiInfo->handles[0] ); + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) + snd_pcm_drop( apiInfo->handles[1] ); + } + + if ( apiInfo ) { + pthread_cond_destroy( &apiInfo->runnable_cv ); + if ( apiInfo->handles[0] ) snd_pcm_close( apiInfo->handles[0] ); + if ( apiInfo->handles[1] ) snd_pcm_close( apiInfo->handles[1] ); + delete apiInfo; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + stream_.mode = UNINITIALIZED; + stream_.state = STREAM_CLOSED; +} + +void RtApiAlsa :: startStream() +{ + // This method calls snd_pcm_prepare if the device isn't already in that state. + + verifyStream(); + if ( stream_.state == STREAM_RUNNING ) { + errorText_ = "RtApiAlsa::startStream(): the stream is already running!"; + error( RtAudioError::WARNING ); + return; + } + + MUTEX_LOCK( &stream_.mutex ); + + int result = 0; + snd_pcm_state_t state; + AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle; + snd_pcm_t **handle = (snd_pcm_t **) apiInfo->handles; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + state = snd_pcm_state( handle[0] ); + if ( state != SND_PCM_STATE_PREPARED ) { + result = snd_pcm_prepare( handle[0] ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::startStream: error preparing output pcm device, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + } + + if ( ( stream_.mode == INPUT || stream_.mode == DUPLEX ) && !apiInfo->synchronized ) { + result = snd_pcm_drop(handle[1]); // fix to remove stale data received since device has been open + state = snd_pcm_state( handle[1] ); + if ( state != SND_PCM_STATE_PREPARED ) { + result = snd_pcm_prepare( handle[1] ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::startStream: error preparing input pcm device, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + } + + stream_.state = STREAM_RUNNING; + + unlock: + apiInfo->runnable = true; + pthread_cond_signal( &apiInfo->runnable_cv ); + MUTEX_UNLOCK( &stream_.mutex ); + + if ( result >= 0 ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiAlsa :: stopStream() +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiAlsa::stopStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + stream_.state = STREAM_STOPPED; + MUTEX_LOCK( &stream_.mutex ); + + int result = 0; + AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle; + snd_pcm_t **handle = (snd_pcm_t **) apiInfo->handles; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + if ( apiInfo->synchronized ) + result = snd_pcm_drop( handle[0] ); + else + result = snd_pcm_drain( handle[0] ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::stopStream: error draining output pcm device, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + if ( ( stream_.mode == INPUT || stream_.mode == DUPLEX ) && !apiInfo->synchronized ) { + result = snd_pcm_drop( handle[1] ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::stopStream: error stopping input pcm device, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + unlock: + apiInfo->runnable = false; // fixes high CPU usage when stopped + MUTEX_UNLOCK( &stream_.mutex ); + + if ( result >= 0 ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiAlsa :: abortStream() +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiAlsa::abortStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + stream_.state = STREAM_STOPPED; + MUTEX_LOCK( &stream_.mutex ); + + int result = 0; + AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle; + snd_pcm_t **handle = (snd_pcm_t **) apiInfo->handles; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + result = snd_pcm_drop( handle[0] ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::abortStream: error aborting output pcm device, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + if ( ( stream_.mode == INPUT || stream_.mode == DUPLEX ) && !apiInfo->synchronized ) { + result = snd_pcm_drop( handle[1] ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::abortStream: error aborting input pcm device, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + unlock: + apiInfo->runnable = false; // fixes high CPU usage when stopped + MUTEX_UNLOCK( &stream_.mutex ); + + if ( result >= 0 ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiAlsa :: callbackEvent() +{ + AlsaHandle *apiInfo = (AlsaHandle *) stream_.apiHandle; + if ( stream_.state == STREAM_STOPPED ) { + MUTEX_LOCK( &stream_.mutex ); + while ( !apiInfo->runnable ) + pthread_cond_wait( &apiInfo->runnable_cv, &stream_.mutex ); + + if ( stream_.state != STREAM_RUNNING ) { + MUTEX_UNLOCK( &stream_.mutex ); + return; + } + MUTEX_UNLOCK( &stream_.mutex ); + } + + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiAlsa::callbackEvent(): the stream is closed ... this shouldn't happen!"; + error( RtAudioError::WARNING ); + return; + } + + int doStopStream = 0; + RtAudioCallback callback = (RtAudioCallback) stream_.callbackInfo.callback; + double streamTime = getStreamTime(); + RtAudioStreamStatus status = 0; + if ( stream_.mode != INPUT && apiInfo->xrun[0] == true ) { + status |= RTAUDIO_OUTPUT_UNDERFLOW; + apiInfo->xrun[0] = false; + } + if ( stream_.mode != OUTPUT && apiInfo->xrun[1] == true ) { + status |= RTAUDIO_INPUT_OVERFLOW; + apiInfo->xrun[1] = false; + } + doStopStream = callback( stream_.userBuffer[0], stream_.userBuffer[1], + stream_.bufferSize, streamTime, status, stream_.callbackInfo.userData ); + + if ( doStopStream == 2 ) { + abortStream(); + return; + } + + MUTEX_LOCK( &stream_.mutex ); + + // The state might change while waiting on a mutex. + if ( stream_.state == STREAM_STOPPED ) goto unlock; + + int result; + char *buffer; + int channels; + snd_pcm_t **handle; + snd_pcm_sframes_t frames; + RtAudioFormat format; + handle = (snd_pcm_t **) apiInfo->handles; + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) { + + // Setup parameters. + if ( stream_.doConvertBuffer[1] ) { + buffer = stream_.deviceBuffer; + channels = stream_.nDeviceChannels[1]; + format = stream_.deviceFormat[1]; + } + else { + buffer = stream_.userBuffer[1]; + channels = stream_.nUserChannels[1]; + format = stream_.userFormat; + } + + // Read samples from device in interleaved/non-interleaved format. + if ( stream_.deviceInterleaved[1] ) + result = snd_pcm_readi( handle[1], buffer, stream_.bufferSize ); + else { + void *bufs[channels]; + size_t offset = stream_.bufferSize * formatBytes( format ); + for ( int i=0; i<channels; i++ ) + bufs[i] = (void *) (buffer + (i * offset)); + result = snd_pcm_readn( handle[1], bufs, stream_.bufferSize ); + } + + if ( result < (int) stream_.bufferSize ) { + // Either an error or overrun occured. + if ( result == -EPIPE ) { + snd_pcm_state_t state = snd_pcm_state( handle[1] ); + if ( state == SND_PCM_STATE_XRUN ) { + apiInfo->xrun[1] = true; + result = snd_pcm_prepare( handle[1] ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::callbackEvent: error preparing device after overrun, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + } + } + else { + errorStream_ << "RtApiAlsa::callbackEvent: error, current state is " << snd_pcm_state_name( state ) << ", " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + } + } + else { + errorStream_ << "RtApiAlsa::callbackEvent: audio read error, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + } + error( RtAudioError::WARNING ); + goto tryOutput; + } + + // Do byte swapping if necessary. + if ( stream_.doByteSwap[1] ) + byteSwapBuffer( buffer, stream_.bufferSize * channels, format ); + + // Do buffer conversion if necessary. + if ( stream_.doConvertBuffer[1] ) + convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] ); + + // Check stream latency + result = snd_pcm_delay( handle[1], &frames ); + if ( result == 0 && frames > 0 ) stream_.latency[1] = frames; + } + + tryOutput: + + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + // Setup parameters and do buffer conversion if necessary. + if ( stream_.doConvertBuffer[0] ) { + buffer = stream_.deviceBuffer; + convertBuffer( buffer, stream_.userBuffer[0], stream_.convertInfo[0] ); + channels = stream_.nDeviceChannels[0]; + format = stream_.deviceFormat[0]; + } + else { + buffer = stream_.userBuffer[0]; + channels = stream_.nUserChannels[0]; + format = stream_.userFormat; + } + + // Do byte swapping if necessary. + if ( stream_.doByteSwap[0] ) + byteSwapBuffer(buffer, stream_.bufferSize * channels, format); + + // Write samples to device in interleaved/non-interleaved format. + if ( stream_.deviceInterleaved[0] ) + result = snd_pcm_writei( handle[0], buffer, stream_.bufferSize ); + else { + void *bufs[channels]; + size_t offset = stream_.bufferSize * formatBytes( format ); + for ( int i=0; i<channels; i++ ) + bufs[i] = (void *) (buffer + (i * offset)); + result = snd_pcm_writen( handle[0], bufs, stream_.bufferSize ); + } + + if ( result < (int) stream_.bufferSize ) { + // Either an error or underrun occured. + if ( result == -EPIPE ) { + snd_pcm_state_t state = snd_pcm_state( handle[0] ); + if ( state == SND_PCM_STATE_XRUN ) { + apiInfo->xrun[0] = true; + result = snd_pcm_prepare( handle[0] ); + if ( result < 0 ) { + errorStream_ << "RtApiAlsa::callbackEvent: error preparing device after underrun, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + } + else + errorText_ = "RtApiAlsa::callbackEvent: audio write error, underrun."; + } + else { + errorStream_ << "RtApiAlsa::callbackEvent: error, current state is " << snd_pcm_state_name( state ) << ", " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + } + } + else { + errorStream_ << "RtApiAlsa::callbackEvent: audio write error, " << snd_strerror( result ) << "."; + errorText_ = errorStream_.str(); + } + error( RtAudioError::WARNING ); + goto unlock; + } + + // Check stream latency + result = snd_pcm_delay( handle[0], &frames ); + if ( result == 0 && frames > 0 ) stream_.latency[0] = frames; + } + + unlock: + MUTEX_UNLOCK( &stream_.mutex ); + + RtApi::tickStreamTime(); + if ( doStopStream == 1 ) this->stopStream(); +} + +static void *alsaCallbackHandler( void *ptr ) +{ + CallbackInfo *info = (CallbackInfo *) ptr; + RtApiAlsa *object = (RtApiAlsa *) info->object; + bool *isRunning = &info->isRunning; + +#ifdef SCHED_RR // Undefined with some OSes (eg: NetBSD 1.6.x with GNU Pthread) + if ( &info->doRealtime ) { + pthread_t tID = pthread_self(); // ID of this thread + sched_param prio = { info->priority }; // scheduling priority of thread + pthread_setschedparam( tID, SCHED_RR, &prio ); + } +#endif + + while ( *isRunning == true ) { + pthread_testcancel(); + object->callbackEvent(); + } + + pthread_exit( NULL ); +} + +//******************** End of __LINUX_ALSA__ *********************// +#endif + +#if defined(__LINUX_PULSE__) + +// Code written by Peter Meerwald, pmeerw@pmeerw.net +// and Tristan Matthews. + +#include <pulse/error.h> +#include <pulse/simple.h> +#include <cstdio> + +static const unsigned int SUPPORTED_SAMPLERATES[] = { 8000, 16000, 22050, 32000, + 44100, 48000, 96000, 0}; + +struct rtaudio_pa_format_mapping_t { + RtAudioFormat rtaudio_format; + pa_sample_format_t pa_format; +}; + +static const rtaudio_pa_format_mapping_t supported_sampleformats[] = { + {RTAUDIO_SINT16, PA_SAMPLE_S16LE}, + {RTAUDIO_SINT32, PA_SAMPLE_S32LE}, + {RTAUDIO_FLOAT32, PA_SAMPLE_FLOAT32LE}, + {0, PA_SAMPLE_INVALID}}; + +struct PulseAudioHandle { + pa_simple *s_play; + pa_simple *s_rec; + pthread_t thread; + pthread_cond_t runnable_cv; + bool runnable; + PulseAudioHandle() : s_play(0), s_rec(0), runnable(false) { } +}; + +RtApiPulse::~RtApiPulse() +{ + if ( stream_.state != STREAM_CLOSED ) + closeStream(); +} + +unsigned int RtApiPulse::getDeviceCount( void ) +{ + return 1; +} + +RtAudio::DeviceInfo RtApiPulse::getDeviceInfo( unsigned int /*device*/ ) +{ + RtAudio::DeviceInfo info; + info.probed = true; + info.name = "PulseAudio"; + info.outputChannels = 2; + info.inputChannels = 2; + info.duplexChannels = 2; + info.isDefaultOutput = true; + info.isDefaultInput = true; + + for ( const unsigned int *sr = SUPPORTED_SAMPLERATES; *sr; ++sr ) + info.sampleRates.push_back( *sr ); + + info.preferredSampleRate = 48000; + info.nativeFormats = RTAUDIO_SINT16 | RTAUDIO_SINT32 | RTAUDIO_FLOAT32; + + return info; +} + +static void *pulseaudio_callback( void * user ) +{ + CallbackInfo *cbi = static_cast<CallbackInfo *>( user ); + RtApiPulse *context = static_cast<RtApiPulse *>( cbi->object ); + volatile bool *isRunning = &cbi->isRunning; + + while ( *isRunning ) { + pthread_testcancel(); + context->callbackEvent(); + } + + pthread_exit( NULL ); +} + +void RtApiPulse::closeStream( void ) +{ + PulseAudioHandle *pah = static_cast<PulseAudioHandle *>( stream_.apiHandle ); + + stream_.callbackInfo.isRunning = false; + if ( pah ) { + MUTEX_LOCK( &stream_.mutex ); + if ( stream_.state == STREAM_STOPPED ) { + pah->runnable = true; + pthread_cond_signal( &pah->runnable_cv ); + } + MUTEX_UNLOCK( &stream_.mutex ); + + pthread_join( pah->thread, 0 ); + if ( pah->s_play ) { + pa_simple_flush( pah->s_play, NULL ); + pa_simple_free( pah->s_play ); + } + if ( pah->s_rec ) + pa_simple_free( pah->s_rec ); + + pthread_cond_destroy( &pah->runnable_cv ); + delete pah; + stream_.apiHandle = 0; + } + + if ( stream_.userBuffer[0] ) { + free( stream_.userBuffer[0] ); + stream_.userBuffer[0] = 0; + } + if ( stream_.userBuffer[1] ) { + free( stream_.userBuffer[1] ); + stream_.userBuffer[1] = 0; + } + + stream_.state = STREAM_CLOSED; + stream_.mode = UNINITIALIZED; +} + +void RtApiPulse::callbackEvent( void ) +{ + PulseAudioHandle *pah = static_cast<PulseAudioHandle *>( stream_.apiHandle ); + + if ( stream_.state == STREAM_STOPPED ) { + MUTEX_LOCK( &stream_.mutex ); + while ( !pah->runnable ) + pthread_cond_wait( &pah->runnable_cv, &stream_.mutex ); + + if ( stream_.state != STREAM_RUNNING ) { + MUTEX_UNLOCK( &stream_.mutex ); + return; + } + MUTEX_UNLOCK( &stream_.mutex ); + } + + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiPulse::callbackEvent(): the stream is closed ... " + "this shouldn't happen!"; + error( RtAudioError::WARNING ); + return; + } + + RtAudioCallback callback = (RtAudioCallback) stream_.callbackInfo.callback; + double streamTime = getStreamTime(); + RtAudioStreamStatus status = 0; + int doStopStream = callback( stream_.userBuffer[OUTPUT], stream_.userBuffer[INPUT], + stream_.bufferSize, streamTime, status, + stream_.callbackInfo.userData ); + + if ( doStopStream == 2 ) { + abortStream(); + return; + } + + MUTEX_LOCK( &stream_.mutex ); + void *pulse_in = stream_.doConvertBuffer[INPUT] ? stream_.deviceBuffer : stream_.userBuffer[INPUT]; + void *pulse_out = stream_.doConvertBuffer[OUTPUT] ? stream_.deviceBuffer : stream_.userBuffer[OUTPUT]; + + if ( stream_.state != STREAM_RUNNING ) + goto unlock; + + int pa_error; + size_t bytes; + if (stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + if ( stream_.doConvertBuffer[OUTPUT] ) { + convertBuffer( stream_.deviceBuffer, + stream_.userBuffer[OUTPUT], + stream_.convertInfo[OUTPUT] ); + bytes = stream_.nDeviceChannels[OUTPUT] * stream_.bufferSize * + formatBytes( stream_.deviceFormat[OUTPUT] ); + } else + bytes = stream_.nUserChannels[OUTPUT] * stream_.bufferSize * + formatBytes( stream_.userFormat ); + + if ( pa_simple_write( pah->s_play, pulse_out, bytes, &pa_error ) < 0 ) { + errorStream_ << "RtApiPulse::callbackEvent: audio write error, " << + pa_strerror( pa_error ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + } + } + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX) { + if ( stream_.doConvertBuffer[INPUT] ) + bytes = stream_.nDeviceChannels[INPUT] * stream_.bufferSize * + formatBytes( stream_.deviceFormat[INPUT] ); + else + bytes = stream_.nUserChannels[INPUT] * stream_.bufferSize * + formatBytes( stream_.userFormat ); + + if ( pa_simple_read( pah->s_rec, pulse_in, bytes, &pa_error ) < 0 ) { + errorStream_ << "RtApiPulse::callbackEvent: audio read error, " << + pa_strerror( pa_error ) << "."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + } + if ( stream_.doConvertBuffer[INPUT] ) { + convertBuffer( stream_.userBuffer[INPUT], + stream_.deviceBuffer, + stream_.convertInfo[INPUT] ); + } + } + + unlock: + MUTEX_UNLOCK( &stream_.mutex ); + RtApi::tickStreamTime(); + + if ( doStopStream == 1 ) + stopStream(); +} + +void RtApiPulse::startStream( void ) +{ + PulseAudioHandle *pah = static_cast<PulseAudioHandle *>( stream_.apiHandle ); + + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiPulse::startStream(): the stream is not open!"; + error( RtAudioError::INVALID_USE ); + return; + } + if ( stream_.state == STREAM_RUNNING ) { + errorText_ = "RtApiPulse::startStream(): the stream is already running!"; + error( RtAudioError::WARNING ); + return; + } + + MUTEX_LOCK( &stream_.mutex ); + + stream_.state = STREAM_RUNNING; + + pah->runnable = true; + pthread_cond_signal( &pah->runnable_cv ); + MUTEX_UNLOCK( &stream_.mutex ); +} + +void RtApiPulse::stopStream( void ) +{ + PulseAudioHandle *pah = static_cast<PulseAudioHandle *>( stream_.apiHandle ); + + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiPulse::stopStream(): the stream is not open!"; + error( RtAudioError::INVALID_USE ); + return; + } + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiPulse::stopStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + stream_.state = STREAM_STOPPED; + MUTEX_LOCK( &stream_.mutex ); + + if ( pah && pah->s_play ) { + int pa_error; + if ( pa_simple_drain( pah->s_play, &pa_error ) < 0 ) { + errorStream_ << "RtApiPulse::stopStream: error draining output device, " << + pa_strerror( pa_error ) << "."; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + } + + stream_.state = STREAM_STOPPED; + MUTEX_UNLOCK( &stream_.mutex ); +} + +void RtApiPulse::abortStream( void ) +{ + PulseAudioHandle *pah = static_cast<PulseAudioHandle*>( stream_.apiHandle ); + + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiPulse::abortStream(): the stream is not open!"; + error( RtAudioError::INVALID_USE ); + return; + } + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiPulse::abortStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + stream_.state = STREAM_STOPPED; + MUTEX_LOCK( &stream_.mutex ); + + if ( pah && pah->s_play ) { + int pa_error; + if ( pa_simple_flush( pah->s_play, &pa_error ) < 0 ) { + errorStream_ << "RtApiPulse::abortStream: error flushing output device, " << + pa_strerror( pa_error ) << "."; + errorText_ = errorStream_.str(); + MUTEX_UNLOCK( &stream_.mutex ); + error( RtAudioError::SYSTEM_ERROR ); + return; + } + } + + stream_.state = STREAM_STOPPED; + MUTEX_UNLOCK( &stream_.mutex ); +} + +bool RtApiPulse::probeDeviceOpen( unsigned int device, StreamMode mode, + unsigned int channels, unsigned int firstChannel, + unsigned int sampleRate, RtAudioFormat format, + unsigned int *bufferSize, RtAudio::StreamOptions *options ) +{ + PulseAudioHandle *pah = 0; + unsigned long bufferBytes = 0; + pa_sample_spec ss; + + if ( device != 0 ) return false; + if ( mode != INPUT && mode != OUTPUT ) return false; + if ( channels != 1 && channels != 2 ) { + errorText_ = "RtApiPulse::probeDeviceOpen: unsupported number of channels."; + return false; + } + ss.channels = channels; + + if ( firstChannel != 0 ) return false; + + bool sr_found = false; + for ( const unsigned int *sr = SUPPORTED_SAMPLERATES; *sr; ++sr ) { + if ( sampleRate == *sr ) { + sr_found = true; + stream_.sampleRate = sampleRate; + ss.rate = sampleRate; + break; + } + } + if ( !sr_found ) { + errorText_ = "RtApiPulse::probeDeviceOpen: unsupported sample rate."; + return false; + } + + bool sf_found = 0; + for ( const rtaudio_pa_format_mapping_t *sf = supported_sampleformats; + sf->rtaudio_format && sf->pa_format != PA_SAMPLE_INVALID; ++sf ) { + if ( format == sf->rtaudio_format ) { + sf_found = true; + stream_.userFormat = sf->rtaudio_format; + stream_.deviceFormat[mode] = stream_.userFormat; + ss.format = sf->pa_format; + break; + } + } + if ( !sf_found ) { // Use internal data format conversion. + stream_.userFormat = format; + stream_.deviceFormat[mode] = RTAUDIO_FLOAT32; + ss.format = PA_SAMPLE_FLOAT32LE; + } + + // Set other stream parameters. + if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) stream_.userInterleaved = false; + else stream_.userInterleaved = true; + stream_.deviceInterleaved[mode] = true; + stream_.nBuffers = 1; + stream_.doByteSwap[mode] = false; + stream_.nUserChannels[mode] = channels; + stream_.nDeviceChannels[mode] = channels + firstChannel; + stream_.channelOffset[mode] = 0; + std::string streamName = "RtAudio"; + + // Set flags for buffer conversion. + stream_.doConvertBuffer[mode] = false; + if ( stream_.userFormat != stream_.deviceFormat[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( stream_.nUserChannels[mode] < stream_.nDeviceChannels[mode] ) + stream_.doConvertBuffer[mode] = true; + + // Allocate necessary internal buffers. + bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat ); + stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 ); + if ( stream_.userBuffer[mode] == NULL ) { + errorText_ = "RtApiPulse::probeDeviceOpen: error allocating user buffer memory."; + goto error; + } + stream_.bufferSize = *bufferSize; + + if ( stream_.doConvertBuffer[mode] ) { + + bool makeBuffer = true; + bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] ); + if ( mode == INPUT ) { + if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) { + unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] ); + if ( bufferBytes <= bytesOut ) makeBuffer = false; + } + } + + if ( makeBuffer ) { + bufferBytes *= *bufferSize; + if ( stream_.deviceBuffer ) free( stream_.deviceBuffer ); + stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 ); + if ( stream_.deviceBuffer == NULL ) { + errorText_ = "RtApiPulse::probeDeviceOpen: error allocating device buffer memory."; + goto error; + } + } + } + + stream_.device[mode] = device; + + // Setup the buffer conversion information structure. + if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, firstChannel ); + + if ( !stream_.apiHandle ) { + PulseAudioHandle *pah = new PulseAudioHandle; + if ( !pah ) { + errorText_ = "RtApiPulse::probeDeviceOpen: error allocating memory for handle."; + goto error; + } + + stream_.apiHandle = pah; + if ( pthread_cond_init( &pah->runnable_cv, NULL ) != 0 ) { + errorText_ = "RtApiPulse::probeDeviceOpen: error creating condition variable."; + goto error; + } + } + pah = static_cast<PulseAudioHandle *>( stream_.apiHandle ); + + int error; + if ( options && !options->streamName.empty() ) streamName = options->streamName; + switch ( mode ) { + case INPUT: + pa_buffer_attr buffer_attr; + buffer_attr.fragsize = bufferBytes; + buffer_attr.maxlength = -1; + + pah->s_rec = pa_simple_new( NULL, streamName.c_str(), PA_STREAM_RECORD, NULL, "Record", &ss, NULL, &buffer_attr, &error ); + if ( !pah->s_rec ) { + errorText_ = "RtApiPulse::probeDeviceOpen: error connecting input to PulseAudio server."; + goto error; + } + break; + case OUTPUT: + pah->s_play = pa_simple_new( NULL, "RtAudio", PA_STREAM_PLAYBACK, NULL, "Playback", &ss, NULL, NULL, &error ); + if ( !pah->s_play ) { + errorText_ = "RtApiPulse::probeDeviceOpen: error connecting output to PulseAudio server."; + goto error; + } + break; + default: + goto error; + } + + if ( stream_.mode == UNINITIALIZED ) + stream_.mode = mode; + else if ( stream_.mode == mode ) + goto error; + else + stream_.mode = DUPLEX; + + if ( !stream_.callbackInfo.isRunning ) { + stream_.callbackInfo.object = this; + stream_.callbackInfo.isRunning = true; + if ( pthread_create( &pah->thread, NULL, pulseaudio_callback, (void *)&stream_.callbackInfo) != 0 ) { + errorText_ = "RtApiPulse::probeDeviceOpen: error creating thread."; + goto error; + } + } + + stream_.state = STREAM_STOPPED; + return true; + + error: + if ( pah && stream_.callbackInfo.isRunning ) { + pthread_cond_destroy( &pah->runnable_cv ); + delete pah; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + return FAILURE; +} + +//******************** End of __LINUX_PULSE__ *********************// +#endif + +#if defined(__LINUX_OSS__) + +#include <unistd.h> +#include <sys/ioctl.h> +#include <unistd.h> +#include <fcntl.h> +#include <sys/soundcard.h> +#include <errno.h> +#include <math.h> + +static void *ossCallbackHandler(void * ptr); + +// A structure to hold various information related to the OSS API +// implementation. +struct OssHandle { + int id[2]; // device ids + bool xrun[2]; + bool triggered; + pthread_cond_t runnable; + + OssHandle() + :triggered(false) { id[0] = 0; id[1] = 0; xrun[0] = false; xrun[1] = false; } +}; + +RtApiOss :: RtApiOss() +{ + // Nothing to do here. +} + +RtApiOss :: ~RtApiOss() +{ + if ( stream_.state != STREAM_CLOSED ) closeStream(); +} + +unsigned int RtApiOss :: getDeviceCount( void ) +{ + int mixerfd = open( "/dev/mixer", O_RDWR, 0 ); + if ( mixerfd == -1 ) { + errorText_ = "RtApiOss::getDeviceCount: error opening '/dev/mixer'."; + error( RtAudioError::WARNING ); + return 0; + } + + oss_sysinfo sysinfo; + if ( ioctl( mixerfd, SNDCTL_SYSINFO, &sysinfo ) == -1 ) { + close( mixerfd ); + errorText_ = "RtApiOss::getDeviceCount: error getting sysinfo, OSS version >= 4.0 is required."; + error( RtAudioError::WARNING ); + return 0; + } + + close( mixerfd ); + return sysinfo.numaudios; +} + +RtAudio::DeviceInfo RtApiOss :: getDeviceInfo( unsigned int device ) +{ + RtAudio::DeviceInfo info; + info.probed = false; + + int mixerfd = open( "/dev/mixer", O_RDWR, 0 ); + if ( mixerfd == -1 ) { + errorText_ = "RtApiOss::getDeviceInfo: error opening '/dev/mixer'."; + error( RtAudioError::WARNING ); + return info; + } + + oss_sysinfo sysinfo; + int result = ioctl( mixerfd, SNDCTL_SYSINFO, &sysinfo ); + if ( result == -1 ) { + close( mixerfd ); + errorText_ = "RtApiOss::getDeviceInfo: error getting sysinfo, OSS version >= 4.0 is required."; + error( RtAudioError::WARNING ); + return info; + } + + unsigned nDevices = sysinfo.numaudios; + if ( nDevices == 0 ) { + close( mixerfd ); + errorText_ = "RtApiOss::getDeviceInfo: no devices found!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + if ( device >= nDevices ) { + close( mixerfd ); + errorText_ = "RtApiOss::getDeviceInfo: device ID is invalid!"; + error( RtAudioError::INVALID_USE ); + return info; + } + + oss_audioinfo ainfo; + ainfo.dev = device; + result = ioctl( mixerfd, SNDCTL_AUDIOINFO, &ainfo ); + close( mixerfd ); + if ( result == -1 ) { + errorStream_ << "RtApiOss::getDeviceInfo: error getting device (" << ainfo.name << ") info."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Probe channels + if ( ainfo.caps & PCM_CAP_OUTPUT ) info.outputChannels = ainfo.max_channels; + if ( ainfo.caps & PCM_CAP_INPUT ) info.inputChannels = ainfo.max_channels; + if ( ainfo.caps & PCM_CAP_DUPLEX ) { + if ( info.outputChannels > 0 && info.inputChannels > 0 && ainfo.caps & PCM_CAP_DUPLEX ) + info.duplexChannels = (info.outputChannels > info.inputChannels) ? info.inputChannels : info.outputChannels; + } + + // Probe data formats ... do for input + unsigned long mask = ainfo.iformats; + if ( mask & AFMT_S16_LE || mask & AFMT_S16_BE ) + info.nativeFormats |= RTAUDIO_SINT16; + if ( mask & AFMT_S8 ) + info.nativeFormats |= RTAUDIO_SINT8; + if ( mask & AFMT_S32_LE || mask & AFMT_S32_BE ) + info.nativeFormats |= RTAUDIO_SINT32; + if ( mask & AFMT_FLOAT ) + info.nativeFormats |= RTAUDIO_FLOAT32; + if ( mask & AFMT_S24_LE || mask & AFMT_S24_BE ) + info.nativeFormats |= RTAUDIO_SINT24; + + // Check that we have at least one supported format + if ( info.nativeFormats == 0 ) { + errorStream_ << "RtApiOss::getDeviceInfo: device (" << ainfo.name << ") data format not supported by RtAudio."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + return info; + } + + // Probe the supported sample rates. + info.sampleRates.clear(); + if ( ainfo.nrates ) { + for ( unsigned int i=0; i<ainfo.nrates; i++ ) { + for ( unsigned int k=0; k<MAX_SAMPLE_RATES; k++ ) { + if ( ainfo.rates[i] == SAMPLE_RATES[k] ) { + info.sampleRates.push_back( SAMPLE_RATES[k] ); + + if ( !info.preferredSampleRate || ( SAMPLE_RATES[k] <= 48000 && SAMPLE_RATES[k] > info.preferredSampleRate ) ) + info.preferredSampleRate = SAMPLE_RATES[k]; + + break; + } + } + } + } + else { + // Check min and max rate values; + for ( unsigned int k=0; k<MAX_SAMPLE_RATES; k++ ) { + if ( ainfo.min_rate <= (int) SAMPLE_RATES[k] && ainfo.max_rate >= (int) SAMPLE_RATES[k] ) { + info.sampleRates.push_back( SAMPLE_RATES[k] ); + + if ( !info.preferredSampleRate || ( SAMPLE_RATES[k] <= 48000 && SAMPLE_RATES[k] > info.preferredSampleRate ) ) + info.preferredSampleRate = SAMPLE_RATES[k]; + } + } + } + + if ( info.sampleRates.size() == 0 ) { + errorStream_ << "RtApiOss::getDeviceInfo: no supported sample rates found for device (" << ainfo.name << ")."; + errorText_ = errorStream_.str(); + error( RtAudioError::WARNING ); + } + else { + info.probed = true; + info.name = ainfo.name; + } + + return info; +} + + +bool RtApiOss :: probeDeviceOpen( unsigned int device, StreamMode mode, unsigned int channels, + unsigned int firstChannel, unsigned int sampleRate, + RtAudioFormat format, unsigned int *bufferSize, + RtAudio::StreamOptions *options ) +{ + int mixerfd = open( "/dev/mixer", O_RDWR, 0 ); + if ( mixerfd == -1 ) { + errorText_ = "RtApiOss::probeDeviceOpen: error opening '/dev/mixer'."; + return FAILURE; + } + + oss_sysinfo sysinfo; + int result = ioctl( mixerfd, SNDCTL_SYSINFO, &sysinfo ); + if ( result == -1 ) { + close( mixerfd ); + errorText_ = "RtApiOss::probeDeviceOpen: error getting sysinfo, OSS version >= 4.0 is required."; + return FAILURE; + } + + unsigned nDevices = sysinfo.numaudios; + if ( nDevices == 0 ) { + // This should not happen because a check is made before this function is called. + close( mixerfd ); + errorText_ = "RtApiOss::probeDeviceOpen: no devices found!"; + return FAILURE; + } + + if ( device >= nDevices ) { + // This should not happen because a check is made before this function is called. + close( mixerfd ); + errorText_ = "RtApiOss::probeDeviceOpen: device ID is invalid!"; + return FAILURE; + } + + oss_audioinfo ainfo; + ainfo.dev = device; + result = ioctl( mixerfd, SNDCTL_AUDIOINFO, &ainfo ); + close( mixerfd ); + if ( result == -1 ) { + errorStream_ << "RtApiOss::getDeviceInfo: error getting device (" << ainfo.name << ") info."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Check if device supports input or output + if ( ( mode == OUTPUT && !( ainfo.caps & PCM_CAP_OUTPUT ) ) || + ( mode == INPUT && !( ainfo.caps & PCM_CAP_INPUT ) ) ) { + if ( mode == OUTPUT ) + errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") does not support output."; + else + errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") does not support input."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + int flags = 0; + OssHandle *handle = (OssHandle *) stream_.apiHandle; + if ( mode == OUTPUT ) + flags |= O_WRONLY; + else { // mode == INPUT + if (stream_.mode == OUTPUT && stream_.device[0] == device) { + // We just set the same device for playback ... close and reopen for duplex (OSS only). + close( handle->id[0] ); + handle->id[0] = 0; + if ( !( ainfo.caps & PCM_CAP_DUPLEX ) ) { + errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") does not support duplex mode."; + errorText_ = errorStream_.str(); + return FAILURE; + } + // Check that the number previously set channels is the same. + if ( stream_.nUserChannels[0] != channels ) { + errorStream_ << "RtApiOss::probeDeviceOpen: input/output channels must be equal for OSS duplex device (" << ainfo.name << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + flags |= O_RDWR; + } + else + flags |= O_RDONLY; + } + + // Set exclusive access if specified. + if ( options && options->flags & RTAUDIO_HOG_DEVICE ) flags |= O_EXCL; + + // Try to open the device. + int fd; + fd = open( ainfo.devnode, flags, 0 ); + if ( fd == -1 ) { + if ( errno == EBUSY ) + errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") is busy."; + else + errorStream_ << "RtApiOss::probeDeviceOpen: error opening device (" << ainfo.name << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // For duplex operation, specifically set this mode (this doesn't seem to work). + /* + if ( flags | O_RDWR ) { + result = ioctl( fd, SNDCTL_DSP_SETDUPLEX, NULL ); + if ( result == -1) { + errorStream_ << "RtApiOss::probeDeviceOpen: error setting duplex mode for device (" << ainfo.name << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + } + */ + + // Check the device channel support. + stream_.nUserChannels[mode] = channels; + if ( ainfo.max_channels < (int)(channels + firstChannel) ) { + close( fd ); + errorStream_ << "RtApiOss::probeDeviceOpen: the device (" << ainfo.name << ") does not support requested channel parameters."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Set the number of channels. + int deviceChannels = channels + firstChannel; + result = ioctl( fd, SNDCTL_DSP_CHANNELS, &deviceChannels ); + if ( result == -1 || deviceChannels < (int)(channels + firstChannel) ) { + close( fd ); + errorStream_ << "RtApiOss::probeDeviceOpen: error setting channel parameters on device (" << ainfo.name << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + stream_.nDeviceChannels[mode] = deviceChannels; + + // Get the data format mask + int mask; + result = ioctl( fd, SNDCTL_DSP_GETFMTS, &mask ); + if ( result == -1 ) { + close( fd ); + errorStream_ << "RtApiOss::probeDeviceOpen: error getting device (" << ainfo.name << ") data formats."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Determine how to set the device format. + stream_.userFormat = format; + int deviceFormat = -1; + stream_.doByteSwap[mode] = false; + if ( format == RTAUDIO_SINT8 ) { + if ( mask & AFMT_S8 ) { + deviceFormat = AFMT_S8; + stream_.deviceFormat[mode] = RTAUDIO_SINT8; + } + } + else if ( format == RTAUDIO_SINT16 ) { + if ( mask & AFMT_S16_NE ) { + deviceFormat = AFMT_S16_NE; + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + } + else if ( mask & AFMT_S16_OE ) { + deviceFormat = AFMT_S16_OE; + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + stream_.doByteSwap[mode] = true; + } + } + else if ( format == RTAUDIO_SINT24 ) { + if ( mask & AFMT_S24_NE ) { + deviceFormat = AFMT_S24_NE; + stream_.deviceFormat[mode] = RTAUDIO_SINT24; + } + else if ( mask & AFMT_S24_OE ) { + deviceFormat = AFMT_S24_OE; + stream_.deviceFormat[mode] = RTAUDIO_SINT24; + stream_.doByteSwap[mode] = true; + } + } + else if ( format == RTAUDIO_SINT32 ) { + if ( mask & AFMT_S32_NE ) { + deviceFormat = AFMT_S32_NE; + stream_.deviceFormat[mode] = RTAUDIO_SINT32; + } + else if ( mask & AFMT_S32_OE ) { + deviceFormat = AFMT_S32_OE; + stream_.deviceFormat[mode] = RTAUDIO_SINT32; + stream_.doByteSwap[mode] = true; + } + } + + if ( deviceFormat == -1 ) { + // The user requested format is not natively supported by the device. + if ( mask & AFMT_S16_NE ) { + deviceFormat = AFMT_S16_NE; + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + } + else if ( mask & AFMT_S32_NE ) { + deviceFormat = AFMT_S32_NE; + stream_.deviceFormat[mode] = RTAUDIO_SINT32; + } + else if ( mask & AFMT_S24_NE ) { + deviceFormat = AFMT_S24_NE; + stream_.deviceFormat[mode] = RTAUDIO_SINT24; + } + else if ( mask & AFMT_S16_OE ) { + deviceFormat = AFMT_S16_OE; + stream_.deviceFormat[mode] = RTAUDIO_SINT16; + stream_.doByteSwap[mode] = true; + } + else if ( mask & AFMT_S32_OE ) { + deviceFormat = AFMT_S32_OE; + stream_.deviceFormat[mode] = RTAUDIO_SINT32; + stream_.doByteSwap[mode] = true; + } + else if ( mask & AFMT_S24_OE ) { + deviceFormat = AFMT_S24_OE; + stream_.deviceFormat[mode] = RTAUDIO_SINT24; + stream_.doByteSwap[mode] = true; + } + else if ( mask & AFMT_S8) { + deviceFormat = AFMT_S8; + stream_.deviceFormat[mode] = RTAUDIO_SINT8; + } + } + + if ( stream_.deviceFormat[mode] == 0 ) { + // This really shouldn't happen ... + close( fd ); + errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") data format not supported by RtAudio."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Set the data format. + int temp = deviceFormat; + result = ioctl( fd, SNDCTL_DSP_SETFMT, &deviceFormat ); + if ( result == -1 || deviceFormat != temp ) { + close( fd ); + errorStream_ << "RtApiOss::probeDeviceOpen: error setting data format on device (" << ainfo.name << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Attempt to set the buffer size. According to OSS, the minimum + // number of buffers is two. The supposed minimum buffer size is 16 + // bytes, so that will be our lower bound. The argument to this + // call is in the form 0xMMMMSSSS (hex), where the buffer size (in + // bytes) is given as 2^SSSS and the number of buffers as 2^MMMM. + // We'll check the actual value used near the end of the setup + // procedure. + int ossBufferBytes = *bufferSize * formatBytes( stream_.deviceFormat[mode] ) * deviceChannels; + if ( ossBufferBytes < 16 ) ossBufferBytes = 16; + int buffers = 0; + if ( options ) buffers = options->numberOfBuffers; + if ( options && options->flags & RTAUDIO_MINIMIZE_LATENCY ) buffers = 2; + if ( buffers < 2 ) buffers = 3; + temp = ((int) buffers << 16) + (int)( log10( (double)ossBufferBytes ) / log10( 2.0 ) ); + result = ioctl( fd, SNDCTL_DSP_SETFRAGMENT, &temp ); + if ( result == -1 ) { + close( fd ); + errorStream_ << "RtApiOss::probeDeviceOpen: error setting buffer size on device (" << ainfo.name << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + stream_.nBuffers = buffers; + + // Save buffer size (in sample frames). + *bufferSize = ossBufferBytes / ( formatBytes(stream_.deviceFormat[mode]) * deviceChannels ); + stream_.bufferSize = *bufferSize; + + // Set the sample rate. + int srate = sampleRate; + result = ioctl( fd, SNDCTL_DSP_SPEED, &srate ); + if ( result == -1 ) { + close( fd ); + errorStream_ << "RtApiOss::probeDeviceOpen: error setting sample rate (" << sampleRate << ") on device (" << ainfo.name << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + + // Verify the sample rate setup worked. + if ( abs( srate - sampleRate ) > 100 ) { + close( fd ); + errorStream_ << "RtApiOss::probeDeviceOpen: device (" << ainfo.name << ") does not support sample rate (" << sampleRate << ")."; + errorText_ = errorStream_.str(); + return FAILURE; + } + stream_.sampleRate = sampleRate; + + if ( mode == INPUT && stream_.mode == OUTPUT && stream_.device[0] == device) { + // We're doing duplex setup here. + stream_.deviceFormat[0] = stream_.deviceFormat[1]; + stream_.nDeviceChannels[0] = deviceChannels; + } + + // Set interleaving parameters. + stream_.userInterleaved = true; + stream_.deviceInterleaved[mode] = true; + if ( options && options->flags & RTAUDIO_NONINTERLEAVED ) + stream_.userInterleaved = false; + + // Set flags for buffer conversion + stream_.doConvertBuffer[mode] = false; + if ( stream_.userFormat != stream_.deviceFormat[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( stream_.nUserChannels[mode] < stream_.nDeviceChannels[mode] ) + stream_.doConvertBuffer[mode] = true; + if ( stream_.userInterleaved != stream_.deviceInterleaved[mode] && + stream_.nUserChannels[mode] > 1 ) + stream_.doConvertBuffer[mode] = true; + + // Allocate the stream handles if necessary and then save. + if ( stream_.apiHandle == 0 ) { + try { + handle = new OssHandle; + } + catch ( std::bad_alloc& ) { + errorText_ = "RtApiOss::probeDeviceOpen: error allocating OssHandle memory."; + goto error; + } + + if ( pthread_cond_init( &handle->runnable, NULL ) ) { + errorText_ = "RtApiOss::probeDeviceOpen: error initializing pthread condition variable."; + goto error; + } + + stream_.apiHandle = (void *) handle; + } + else { + handle = (OssHandle *) stream_.apiHandle; + } + handle->id[mode] = fd; + + // Allocate necessary internal buffers. + unsigned long bufferBytes; + bufferBytes = stream_.nUserChannels[mode] * *bufferSize * formatBytes( stream_.userFormat ); + stream_.userBuffer[mode] = (char *) calloc( bufferBytes, 1 ); + if ( stream_.userBuffer[mode] == NULL ) { + errorText_ = "RtApiOss::probeDeviceOpen: error allocating user buffer memory."; + goto error; + } + + if ( stream_.doConvertBuffer[mode] ) { + + bool makeBuffer = true; + bufferBytes = stream_.nDeviceChannels[mode] * formatBytes( stream_.deviceFormat[mode] ); + if ( mode == INPUT ) { + if ( stream_.mode == OUTPUT && stream_.deviceBuffer ) { + unsigned long bytesOut = stream_.nDeviceChannels[0] * formatBytes( stream_.deviceFormat[0] ); + if ( bufferBytes <= bytesOut ) makeBuffer = false; + } + } + + if ( makeBuffer ) { + bufferBytes *= *bufferSize; + if ( stream_.deviceBuffer ) free( stream_.deviceBuffer ); + stream_.deviceBuffer = (char *) calloc( bufferBytes, 1 ); + if ( stream_.deviceBuffer == NULL ) { + errorText_ = "RtApiOss::probeDeviceOpen: error allocating device buffer memory."; + goto error; + } + } + } + + stream_.device[mode] = device; + stream_.state = STREAM_STOPPED; + + // Setup the buffer conversion information structure. + if ( stream_.doConvertBuffer[mode] ) setConvertInfo( mode, firstChannel ); + + // Setup thread if necessary. + if ( stream_.mode == OUTPUT && mode == INPUT ) { + // We had already set up an output stream. + stream_.mode = DUPLEX; + if ( stream_.device[0] == device ) handle->id[0] = fd; + } + else { + stream_.mode = mode; + + // Setup callback thread. + stream_.callbackInfo.object = (void *) this; + + // Set the thread attributes for joinable and realtime scheduling + // priority. The higher priority will only take affect if the + // program is run as root or suid. + pthread_attr_t attr; + pthread_attr_init( &attr ); + pthread_attr_setdetachstate( &attr, PTHREAD_CREATE_JOINABLE ); +#ifdef SCHED_RR // Undefined with some OSes (eg: NetBSD 1.6.x with GNU Pthread) + if ( options && options->flags & RTAUDIO_SCHEDULE_REALTIME ) { + struct sched_param param; + int priority = options->priority; + int min = sched_get_priority_min( SCHED_RR ); + int max = sched_get_priority_max( SCHED_RR ); + if ( priority < min ) priority = min; + else if ( priority > max ) priority = max; + param.sched_priority = priority; + pthread_attr_setschedparam( &attr, ¶m ); + pthread_attr_setschedpolicy( &attr, SCHED_RR ); + } + else + pthread_attr_setschedpolicy( &attr, SCHED_OTHER ); +#else + pthread_attr_setschedpolicy( &attr, SCHED_OTHER ); +#endif + + stream_.callbackInfo.isRunning = true; + result = pthread_create( &stream_.callbackInfo.thread, &attr, ossCallbackHandler, &stream_.callbackInfo ); + pthread_attr_destroy( &attr ); + if ( result ) { + stream_.callbackInfo.isRunning = false; + errorText_ = "RtApiOss::error creating callback thread!"; + goto error; + } + } + + return SUCCESS; + + error: + if ( handle ) { + pthread_cond_destroy( &handle->runnable ); + if ( handle->id[0] ) close( handle->id[0] ); + if ( handle->id[1] ) close( handle->id[1] ); + delete handle; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + return FAILURE; +} + +void RtApiOss :: closeStream() +{ + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiOss::closeStream(): no open stream to close!"; + error( RtAudioError::WARNING ); + return; + } + + OssHandle *handle = (OssHandle *) stream_.apiHandle; + stream_.callbackInfo.isRunning = false; + MUTEX_LOCK( &stream_.mutex ); + if ( stream_.state == STREAM_STOPPED ) + pthread_cond_signal( &handle->runnable ); + MUTEX_UNLOCK( &stream_.mutex ); + pthread_join( stream_.callbackInfo.thread, NULL ); + + if ( stream_.state == STREAM_RUNNING ) { + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) + ioctl( handle->id[0], SNDCTL_DSP_HALT, 0 ); + else + ioctl( handle->id[1], SNDCTL_DSP_HALT, 0 ); + stream_.state = STREAM_STOPPED; + } + + if ( handle ) { + pthread_cond_destroy( &handle->runnable ); + if ( handle->id[0] ) close( handle->id[0] ); + if ( handle->id[1] ) close( handle->id[1] ); + delete handle; + stream_.apiHandle = 0; + } + + for ( int i=0; i<2; i++ ) { + if ( stream_.userBuffer[i] ) { + free( stream_.userBuffer[i] ); + stream_.userBuffer[i] = 0; + } + } + + if ( stream_.deviceBuffer ) { + free( stream_.deviceBuffer ); + stream_.deviceBuffer = 0; + } + + stream_.mode = UNINITIALIZED; + stream_.state = STREAM_CLOSED; +} + +void RtApiOss :: startStream() +{ + verifyStream(); + if ( stream_.state == STREAM_RUNNING ) { + errorText_ = "RtApiOss::startStream(): the stream is already running!"; + error( RtAudioError::WARNING ); + return; + } + + MUTEX_LOCK( &stream_.mutex ); + + stream_.state = STREAM_RUNNING; + + // No need to do anything else here ... OSS automatically starts + // when fed samples. + + MUTEX_UNLOCK( &stream_.mutex ); + + OssHandle *handle = (OssHandle *) stream_.apiHandle; + pthread_cond_signal( &handle->runnable ); +} + +void RtApiOss :: stopStream() +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiOss::stopStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + MUTEX_LOCK( &stream_.mutex ); + + // The state might change while waiting on a mutex. + if ( stream_.state == STREAM_STOPPED ) { + MUTEX_UNLOCK( &stream_.mutex ); + return; + } + + int result = 0; + OssHandle *handle = (OssHandle *) stream_.apiHandle; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + // Flush the output with zeros a few times. + char *buffer; + int samples; + RtAudioFormat format; + + if ( stream_.doConvertBuffer[0] ) { + buffer = stream_.deviceBuffer; + samples = stream_.bufferSize * stream_.nDeviceChannels[0]; + format = stream_.deviceFormat[0]; + } + else { + buffer = stream_.userBuffer[0]; + samples = stream_.bufferSize * stream_.nUserChannels[0]; + format = stream_.userFormat; + } + + memset( buffer, 0, samples * formatBytes(format) ); + for ( unsigned int i=0; i<stream_.nBuffers+1; i++ ) { + result = write( handle->id[0], buffer, samples * formatBytes(format) ); + if ( result == -1 ) { + errorText_ = "RtApiOss::stopStream: audio write error."; + error( RtAudioError::WARNING ); + } + } + + result = ioctl( handle->id[0], SNDCTL_DSP_HALT, 0 ); + if ( result == -1 ) { + errorStream_ << "RtApiOss::stopStream: system error stopping callback procedure on device (" << stream_.device[0] << ")."; + errorText_ = errorStream_.str(); + goto unlock; + } + handle->triggered = false; + } + + if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && handle->id[0] != handle->id[1] ) ) { + result = ioctl( handle->id[1], SNDCTL_DSP_HALT, 0 ); + if ( result == -1 ) { + errorStream_ << "RtApiOss::stopStream: system error stopping input callback procedure on device (" << stream_.device[0] << ")."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + unlock: + stream_.state = STREAM_STOPPED; + MUTEX_UNLOCK( &stream_.mutex ); + + if ( result != -1 ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiOss :: abortStream() +{ + verifyStream(); + if ( stream_.state == STREAM_STOPPED ) { + errorText_ = "RtApiOss::abortStream(): the stream is already stopped!"; + error( RtAudioError::WARNING ); + return; + } + + MUTEX_LOCK( &stream_.mutex ); + + // The state might change while waiting on a mutex. + if ( stream_.state == STREAM_STOPPED ) { + MUTEX_UNLOCK( &stream_.mutex ); + return; + } + + int result = 0; + OssHandle *handle = (OssHandle *) stream_.apiHandle; + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + result = ioctl( handle->id[0], SNDCTL_DSP_HALT, 0 ); + if ( result == -1 ) { + errorStream_ << "RtApiOss::abortStream: system error stopping callback procedure on device (" << stream_.device[0] << ")."; + errorText_ = errorStream_.str(); + goto unlock; + } + handle->triggered = false; + } + + if ( stream_.mode == INPUT || ( stream_.mode == DUPLEX && handle->id[0] != handle->id[1] ) ) { + result = ioctl( handle->id[1], SNDCTL_DSP_HALT, 0 ); + if ( result == -1 ) { + errorStream_ << "RtApiOss::abortStream: system error stopping input callback procedure on device (" << stream_.device[0] << ")."; + errorText_ = errorStream_.str(); + goto unlock; + } + } + + unlock: + stream_.state = STREAM_STOPPED; + MUTEX_UNLOCK( &stream_.mutex ); + + if ( result != -1 ) return; + error( RtAudioError::SYSTEM_ERROR ); +} + +void RtApiOss :: callbackEvent() +{ + OssHandle *handle = (OssHandle *) stream_.apiHandle; + if ( stream_.state == STREAM_STOPPED ) { + MUTEX_LOCK( &stream_.mutex ); + pthread_cond_wait( &handle->runnable, &stream_.mutex ); + if ( stream_.state != STREAM_RUNNING ) { + MUTEX_UNLOCK( &stream_.mutex ); + return; + } + MUTEX_UNLOCK( &stream_.mutex ); + } + + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApiOss::callbackEvent(): the stream is closed ... this shouldn't happen!"; + error( RtAudioError::WARNING ); + return; + } + + // Invoke user callback to get fresh output data. + int doStopStream = 0; + RtAudioCallback callback = (RtAudioCallback) stream_.callbackInfo.callback; + double streamTime = getStreamTime(); + RtAudioStreamStatus status = 0; + if ( stream_.mode != INPUT && handle->xrun[0] == true ) { + status |= RTAUDIO_OUTPUT_UNDERFLOW; + handle->xrun[0] = false; + } + if ( stream_.mode != OUTPUT && handle->xrun[1] == true ) { + status |= RTAUDIO_INPUT_OVERFLOW; + handle->xrun[1] = false; + } + doStopStream = callback( stream_.userBuffer[0], stream_.userBuffer[1], + stream_.bufferSize, streamTime, status, stream_.callbackInfo.userData ); + if ( doStopStream == 2 ) { + this->abortStream(); + return; + } + + MUTEX_LOCK( &stream_.mutex ); + + // The state might change while waiting on a mutex. + if ( stream_.state == STREAM_STOPPED ) goto unlock; + + int result; + char *buffer; + int samples; + RtAudioFormat format; + + if ( stream_.mode == OUTPUT || stream_.mode == DUPLEX ) { + + // Setup parameters and do buffer conversion if necessary. + if ( stream_.doConvertBuffer[0] ) { + buffer = stream_.deviceBuffer; + convertBuffer( buffer, stream_.userBuffer[0], stream_.convertInfo[0] ); + samples = stream_.bufferSize * stream_.nDeviceChannels[0]; + format = stream_.deviceFormat[0]; + } + else { + buffer = stream_.userBuffer[0]; + samples = stream_.bufferSize * stream_.nUserChannels[0]; + format = stream_.userFormat; + } + + // Do byte swapping if necessary. + if ( stream_.doByteSwap[0] ) + byteSwapBuffer( buffer, samples, format ); + + if ( stream_.mode == DUPLEX && handle->triggered == false ) { + int trig = 0; + ioctl( handle->id[0], SNDCTL_DSP_SETTRIGGER, &trig ); + result = write( handle->id[0], buffer, samples * formatBytes(format) ); + trig = PCM_ENABLE_INPUT|PCM_ENABLE_OUTPUT; + ioctl( handle->id[0], SNDCTL_DSP_SETTRIGGER, &trig ); + handle->triggered = true; + } + else + // Write samples to device. + result = write( handle->id[0], buffer, samples * formatBytes(format) ); + + if ( result == -1 ) { + // We'll assume this is an underrun, though there isn't a + // specific means for determining that. + handle->xrun[0] = true; + errorText_ = "RtApiOss::callbackEvent: audio write error."; + error( RtAudioError::WARNING ); + // Continue on to input section. + } + } + + if ( stream_.mode == INPUT || stream_.mode == DUPLEX ) { + + // Setup parameters. + if ( stream_.doConvertBuffer[1] ) { + buffer = stream_.deviceBuffer; + samples = stream_.bufferSize * stream_.nDeviceChannels[1]; + format = stream_.deviceFormat[1]; + } + else { + buffer = stream_.userBuffer[1]; + samples = stream_.bufferSize * stream_.nUserChannels[1]; + format = stream_.userFormat; + } + + // Read samples from device. + result = read( handle->id[1], buffer, samples * formatBytes(format) ); + + if ( result == -1 ) { + // We'll assume this is an overrun, though there isn't a + // specific means for determining that. + handle->xrun[1] = true; + errorText_ = "RtApiOss::callbackEvent: audio read error."; + error( RtAudioError::WARNING ); + goto unlock; + } + + // Do byte swapping if necessary. + if ( stream_.doByteSwap[1] ) + byteSwapBuffer( buffer, samples, format ); + + // Do buffer conversion if necessary. + if ( stream_.doConvertBuffer[1] ) + convertBuffer( stream_.userBuffer[1], stream_.deviceBuffer, stream_.convertInfo[1] ); + } + + unlock: + MUTEX_UNLOCK( &stream_.mutex ); + + RtApi::tickStreamTime(); + if ( doStopStream == 1 ) this->stopStream(); +} + +static void *ossCallbackHandler( void *ptr ) +{ + CallbackInfo *info = (CallbackInfo *) ptr; + RtApiOss *object = (RtApiOss *) info->object; + bool *isRunning = &info->isRunning; + + while ( *isRunning == true ) { + pthread_testcancel(); + object->callbackEvent(); + } + + pthread_exit( NULL ); +} + +//******************** End of __LINUX_OSS__ *********************// +#endif + + +// *************************************************** // +// +// Protected common (OS-independent) RtAudio methods. +// +// *************************************************** // + +// This method can be modified to control the behavior of error +// message printing. +void RtApi :: error( RtAudioError::Type type ) +{ + errorStream_.str(""); // clear the ostringstream + + RtAudioErrorCallback errorCallback = (RtAudioErrorCallback) stream_.callbackInfo.errorCallback; + if ( errorCallback ) { + // abortStream() can generate new error messages. Ignore them. Just keep original one. + + if ( firstErrorOccurred_ ) + return; + + firstErrorOccurred_ = true; + const std::string errorMessage = errorText_; + + if ( type != RtAudioError::WARNING && stream_.state != STREAM_STOPPED) { + stream_.callbackInfo.isRunning = false; // exit from the thread + abortStream(); + } + + errorCallback( type, errorMessage ); + firstErrorOccurred_ = false; + return; + } + + if ( type == RtAudioError::WARNING && showWarnings_ == true ) + std::cerr << '\n' << errorText_ << "\n\n"; + else if ( type != RtAudioError::WARNING ) + throw( RtAudioError( errorText_, type ) ); +} + +void RtApi :: verifyStream() +{ + if ( stream_.state == STREAM_CLOSED ) { + errorText_ = "RtApi:: a stream is not open!"; + error( RtAudioError::INVALID_USE ); + } +} + +void RtApi :: clearStreamInfo() +{ + stream_.mode = UNINITIALIZED; + stream_.state = STREAM_CLOSED; + stream_.sampleRate = 0; + stream_.bufferSize = 0; + stream_.nBuffers = 0; + stream_.userFormat = 0; + stream_.userInterleaved = true; + stream_.streamTime = 0.0; + stream_.apiHandle = 0; + stream_.deviceBuffer = 0; + stream_.callbackInfo.callback = 0; + stream_.callbackInfo.userData = 0; + stream_.callbackInfo.isRunning = false; + stream_.callbackInfo.errorCallback = 0; + for ( int i=0; i<2; i++ ) { + stream_.device[i] = 11111; + stream_.doConvertBuffer[i] = false; + stream_.deviceInterleaved[i] = true; + stream_.doByteSwap[i] = false; + stream_.nUserChannels[i] = 0; + stream_.nDeviceChannels[i] = 0; + stream_.channelOffset[i] = 0; + stream_.deviceFormat[i] = 0; + stream_.latency[i] = 0; + stream_.userBuffer[i] = 0; + stream_.convertInfo[i].channels = 0; + stream_.convertInfo[i].inJump = 0; + stream_.convertInfo[i].outJump = 0; + stream_.convertInfo[i].inFormat = 0; + stream_.convertInfo[i].outFormat = 0; + stream_.convertInfo[i].inOffset.clear(); + stream_.convertInfo[i].outOffset.clear(); + } +} + +unsigned int RtApi :: formatBytes( RtAudioFormat format ) +{ + if ( format == RTAUDIO_SINT16 ) + return 2; + else if ( format == RTAUDIO_SINT32 || format == RTAUDIO_FLOAT32 ) + return 4; + else if ( format == RTAUDIO_FLOAT64 ) + return 8; + else if ( format == RTAUDIO_SINT24 ) + return 3; + else if ( format == RTAUDIO_SINT8 ) + return 1; + + errorText_ = "RtApi::formatBytes: undefined format."; + error( RtAudioError::WARNING ); + + return 0; +} + +void RtApi :: setConvertInfo( StreamMode mode, unsigned int firstChannel ) +{ + if ( mode == INPUT ) { // convert device to user buffer + stream_.convertInfo[mode].inJump = stream_.nDeviceChannels[1]; + stream_.convertInfo[mode].outJump = stream_.nUserChannels[1]; + stream_.convertInfo[mode].inFormat = stream_.deviceFormat[1]; + stream_.convertInfo[mode].outFormat = stream_.userFormat; + } + else { // convert user to device buffer + stream_.convertInfo[mode].inJump = stream_.nUserChannels[0]; + stream_.convertInfo[mode].outJump = stream_.nDeviceChannels[0]; + stream_.convertInfo[mode].inFormat = stream_.userFormat; + stream_.convertInfo[mode].outFormat = stream_.deviceFormat[0]; + } + + if ( stream_.convertInfo[mode].inJump < stream_.convertInfo[mode].outJump ) + stream_.convertInfo[mode].channels = stream_.convertInfo[mode].inJump; + else + stream_.convertInfo[mode].channels = stream_.convertInfo[mode].outJump; + + // Set up the interleave/deinterleave offsets. + if ( stream_.deviceInterleaved[mode] != stream_.userInterleaved ) { + if ( ( mode == OUTPUT && stream_.deviceInterleaved[mode] ) || + ( mode == INPUT && stream_.userInterleaved ) ) { + for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) { + stream_.convertInfo[mode].inOffset.push_back( k * stream_.bufferSize ); + stream_.convertInfo[mode].outOffset.push_back( k ); + stream_.convertInfo[mode].inJump = 1; + } + } + else { + for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) { + stream_.convertInfo[mode].inOffset.push_back( k ); + stream_.convertInfo[mode].outOffset.push_back( k * stream_.bufferSize ); + stream_.convertInfo[mode].outJump = 1; + } + } + } + else { // no (de)interleaving + if ( stream_.userInterleaved ) { + for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) { + stream_.convertInfo[mode].inOffset.push_back( k ); + stream_.convertInfo[mode].outOffset.push_back( k ); + } + } + else { + for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) { + stream_.convertInfo[mode].inOffset.push_back( k * stream_.bufferSize ); + stream_.convertInfo[mode].outOffset.push_back( k * stream_.bufferSize ); + stream_.convertInfo[mode].inJump = 1; + stream_.convertInfo[mode].outJump = 1; + } + } + } + + // Add channel offset. + if ( firstChannel > 0 ) { + if ( stream_.deviceInterleaved[mode] ) { + if ( mode == OUTPUT ) { + for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) + stream_.convertInfo[mode].outOffset[k] += firstChannel; + } + else { + for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) + stream_.convertInfo[mode].inOffset[k] += firstChannel; + } + } + else { + if ( mode == OUTPUT ) { + for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) + stream_.convertInfo[mode].outOffset[k] += ( firstChannel * stream_.bufferSize ); + } + else { + for ( int k=0; k<stream_.convertInfo[mode].channels; k++ ) + stream_.convertInfo[mode].inOffset[k] += ( firstChannel * stream_.bufferSize ); + } + } + } +} + +void RtApi :: convertBuffer( char *outBuffer, char *inBuffer, ConvertInfo &info ) +{ + // This function does format conversion, input/output channel compensation, and + // data interleaving/deinterleaving. 24-bit integers are assumed to occupy + // the lower three bytes of a 32-bit integer. + + // Clear our device buffer when in/out duplex device channels are different + if ( outBuffer == stream_.deviceBuffer && stream_.mode == DUPLEX && + ( stream_.nDeviceChannels[0] < stream_.nDeviceChannels[1] ) ) + memset( outBuffer, 0, stream_.bufferSize * info.outJump * formatBytes( info.outFormat ) ); + + int j; + if (info.outFormat == RTAUDIO_FLOAT64) { + Float64 scale; + Float64 *out = (Float64 *)outBuffer; + + if (info.inFormat == RTAUDIO_SINT8) { + signed char *in = (signed char *)inBuffer; + scale = 1.0 / 127.5; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float64) in[info.inOffset[j]]; + out[info.outOffset[j]] += 0.5; + out[info.outOffset[j]] *= scale; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT16) { + Int16 *in = (Int16 *)inBuffer; + scale = 1.0 / 32767.5; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float64) in[info.inOffset[j]]; + out[info.outOffset[j]] += 0.5; + out[info.outOffset[j]] *= scale; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT24) { + Int24 *in = (Int24 *)inBuffer; + scale = 1.0 / 8388607.5; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float64) (in[info.inOffset[j]].asInt()); + out[info.outOffset[j]] += 0.5; + out[info.outOffset[j]] *= scale; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT32) { + Int32 *in = (Int32 *)inBuffer; + scale = 1.0 / 2147483647.5; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float64) in[info.inOffset[j]]; + out[info.outOffset[j]] += 0.5; + out[info.outOffset[j]] *= scale; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT32) { + Float32 *in = (Float32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float64) in[info.inOffset[j]]; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT64) { + // Channel compensation and/or (de)interleaving only. + Float64 *in = (Float64 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = in[info.inOffset[j]]; + } + in += info.inJump; + out += info.outJump; + } + } + } + else if (info.outFormat == RTAUDIO_FLOAT32) { + Float32 scale; + Float32 *out = (Float32 *)outBuffer; + + if (info.inFormat == RTAUDIO_SINT8) { + signed char *in = (signed char *)inBuffer; + scale = (Float32) ( 1.0 / 127.5 ); + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float32) in[info.inOffset[j]]; + out[info.outOffset[j]] += 0.5; + out[info.outOffset[j]] *= scale; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT16) { + Int16 *in = (Int16 *)inBuffer; + scale = (Float32) ( 1.0 / 32767.5 ); + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float32) in[info.inOffset[j]]; + out[info.outOffset[j]] += 0.5; + out[info.outOffset[j]] *= scale; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT24) { + Int24 *in = (Int24 *)inBuffer; + scale = (Float32) ( 1.0 / 8388607.5 ); + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float32) (in[info.inOffset[j]].asInt()); + out[info.outOffset[j]] += 0.5; + out[info.outOffset[j]] *= scale; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT32) { + Int32 *in = (Int32 *)inBuffer; + scale = (Float32) ( 1.0 / 2147483647.5 ); + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float32) in[info.inOffset[j]]; + out[info.outOffset[j]] += 0.5; + out[info.outOffset[j]] *= scale; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT32) { + // Channel compensation and/or (de)interleaving only. + Float32 *in = (Float32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = in[info.inOffset[j]]; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT64) { + Float64 *in = (Float64 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Float32) in[info.inOffset[j]]; + } + in += info.inJump; + out += info.outJump; + } + } + } + else if (info.outFormat == RTAUDIO_SINT32) { + Int32 *out = (Int32 *)outBuffer; + if (info.inFormat == RTAUDIO_SINT8) { + signed char *in = (signed char *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) in[info.inOffset[j]]; + out[info.outOffset[j]] <<= 24; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT16) { + Int16 *in = (Int16 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) in[info.inOffset[j]]; + out[info.outOffset[j]] <<= 16; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT24) { + Int24 *in = (Int24 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) in[info.inOffset[j]].asInt(); + out[info.outOffset[j]] <<= 8; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT32) { + // Channel compensation and/or (de)interleaving only. + Int32 *in = (Int32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = in[info.inOffset[j]]; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT32) { + Float32 *in = (Float32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] * 2147483647.5 - 0.5); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT64) { + Float64 *in = (Float64 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] * 2147483647.5 - 0.5); + } + in += info.inJump; + out += info.outJump; + } + } + } + else if (info.outFormat == RTAUDIO_SINT24) { + Int24 *out = (Int24 *)outBuffer; + if (info.inFormat == RTAUDIO_SINT8) { + signed char *in = (signed char *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] << 16); + //out[info.outOffset[j]] <<= 16; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT16) { + Int16 *in = (Int16 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] << 8); + //out[info.outOffset[j]] <<= 8; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT24) { + // Channel compensation and/or (de)interleaving only. + Int24 *in = (Int24 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = in[info.inOffset[j]]; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT32) { + Int32 *in = (Int32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] >> 8); + //out[info.outOffset[j]] >>= 8; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT32) { + Float32 *in = (Float32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] * 8388607.5 - 0.5); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT64) { + Float64 *in = (Float64 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int32) (in[info.inOffset[j]] * 8388607.5 - 0.5); + } + in += info.inJump; + out += info.outJump; + } + } + } + else if (info.outFormat == RTAUDIO_SINT16) { + Int16 *out = (Int16 *)outBuffer; + if (info.inFormat == RTAUDIO_SINT8) { + signed char *in = (signed char *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int16) in[info.inOffset[j]]; + out[info.outOffset[j]] <<= 8; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT16) { + // Channel compensation and/or (de)interleaving only. + Int16 *in = (Int16 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = in[info.inOffset[j]]; + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT24) { + Int24 *in = (Int24 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int16) (in[info.inOffset[j]].asInt() >> 8); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT32) { + Int32 *in = (Int32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int16) ((in[info.inOffset[j]] >> 16) & 0x0000ffff); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT32) { + Float32 *in = (Float32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int16) (in[info.inOffset[j]] * 32767.5 - 0.5); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT64) { + Float64 *in = (Float64 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (Int16) (in[info.inOffset[j]] * 32767.5 - 0.5); + } + in += info.inJump; + out += info.outJump; + } + } + } + else if (info.outFormat == RTAUDIO_SINT8) { + signed char *out = (signed char *)outBuffer; + if (info.inFormat == RTAUDIO_SINT8) { + // Channel compensation and/or (de)interleaving only. + signed char *in = (signed char *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = in[info.inOffset[j]]; + } + in += info.inJump; + out += info.outJump; + } + } + if (info.inFormat == RTAUDIO_SINT16) { + Int16 *in = (Int16 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (signed char) ((in[info.inOffset[j]] >> 8) & 0x00ff); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT24) { + Int24 *in = (Int24 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (signed char) (in[info.inOffset[j]].asInt() >> 16); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_SINT32) { + Int32 *in = (Int32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (signed char) ((in[info.inOffset[j]] >> 24) & 0x000000ff); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT32) { + Float32 *in = (Float32 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (signed char) (in[info.inOffset[j]] * 127.5 - 0.5); + } + in += info.inJump; + out += info.outJump; + } + } + else if (info.inFormat == RTAUDIO_FLOAT64) { + Float64 *in = (Float64 *)inBuffer; + for (unsigned int i=0; i<stream_.bufferSize; i++) { + for (j=0; j<info.channels; j++) { + out[info.outOffset[j]] = (signed char) (in[info.inOffset[j]] * 127.5 - 0.5); + } + in += info.inJump; + out += info.outJump; + } + } + } +} + +//static inline uint16_t bswap_16(uint16_t x) { return (x>>8) | (x<<8); } +//static inline uint32_t bswap_32(uint32_t x) { return (bswap_16(x&0xffff)<<16) | (bswap_16(x>>16)); } +//static inline uint64_t bswap_64(uint64_t x) { return (((unsigned long long)bswap_32(x&0xffffffffull))<<32) | (bswap_32(x>>32)); } + +void RtApi :: byteSwapBuffer( char *buffer, unsigned int samples, RtAudioFormat format ) +{ + register char val; + register char *ptr; + + ptr = buffer; + if ( format == RTAUDIO_SINT16 ) { + for ( unsigned int i=0; i<samples; i++ ) { + // Swap 1st and 2nd bytes. + val = *(ptr); + *(ptr) = *(ptr+1); + *(ptr+1) = val; + + // Increment 2 bytes. + ptr += 2; + } + } + else if ( format == RTAUDIO_SINT32 || + format == RTAUDIO_FLOAT32 ) { + for ( unsigned int i=0; i<samples; i++ ) { + // Swap 1st and 4th bytes. + val = *(ptr); + *(ptr) = *(ptr+3); + *(ptr+3) = val; + + // Swap 2nd and 3rd bytes. + ptr += 1; + val = *(ptr); + *(ptr) = *(ptr+1); + *(ptr+1) = val; + + // Increment 3 more bytes. + ptr += 3; + } + } + else if ( format == RTAUDIO_SINT24 ) { + for ( unsigned int i=0; i<samples; i++ ) { + // Swap 1st and 3rd bytes. + val = *(ptr); + *(ptr) = *(ptr+2); + *(ptr+2) = val; + + // Increment 2 more bytes. + ptr += 2; + } + } + else if ( format == RTAUDIO_FLOAT64 ) { + for ( unsigned int i=0; i<samples; i++ ) { + // Swap 1st and 8th bytes + val = *(ptr); + *(ptr) = *(ptr+7); + *(ptr+7) = val; + + // Swap 2nd and 7th bytes + ptr += 1; + val = *(ptr); + *(ptr) = *(ptr+5); + *(ptr+5) = val; + + // Swap 3rd and 6th bytes + ptr += 1; + val = *(ptr); + *(ptr) = *(ptr+3); + *(ptr+3) = val; + + // Swap 4th and 5th bytes + ptr += 1; + val = *(ptr); + *(ptr) = *(ptr+1); + *(ptr+1) = val; + + // Increment 5 more bytes. + ptr += 5; + } + } +} + + // Indentation settings for Vim and Emacs + // + // Local Variables: + // c-basic-offset: 2 + // indent-tabs-mode: nil + // End: + // + // vim: et sts=2 sw=2 + +#endif diff --git a/drivers/speex/config.h b/drivers/speex/config.h index d31382702c..8c48e3b99d 100644 --- a/drivers/speex/config.h +++ b/drivers/speex/config.h @@ -1,52 +1,52 @@ -/*
- Copyright (C) 2003 Commonwealth Scientific and Industrial Research
- Organisation (CSIRO) Australia
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- - Neither the name of CSIRO Australia nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A
- PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ORGANISATION OR
- CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef CONFIG_H
-#define CONFIG_H
-
-/* An inline macro is required for use of the inline keyword as not all C compilers support */
-/* inline. It is officially C99 and C++ only */
-
-
-/* Use only fixed point arithmetic */
-
-//#ifdef _MSC_VER
-//#define inline _inline
-//#endif
-
-#define FIXED_POINT 1
-
-#ifdef _MSC_VER
-#define inline __inline
-#endif
-
-#endif /* ! CONFIG_H */
+/* + Copyright (C) 2003 Commonwealth Scientific and Industrial Research + Organisation (CSIRO) Australia + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of CSIRO Australia nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A + PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE ORGANISATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef CONFIG_H +#define CONFIG_H + +/* An inline macro is required for use of the inline keyword as not all C compilers support */ +/* inline. It is officially C99 and C++ only */ + + +/* Use only fixed point arithmetic */ + +//#ifdef _MSC_VER +//#define inline _inline +//#endif + +#define FIXED_POINT 1 + +#ifdef _MSC_VER +#define inline __inline +#endif + +#endif /* ! CONFIG_H */ diff --git a/drivers/speex/lsp.h b/drivers/speex/lsp.h index 648652fb9e..b55bd42f2c 100644 --- a/drivers/speex/lsp.h +++ b/drivers/speex/lsp.h @@ -1,64 +1,64 @@ -/*---------------------------------------------------------------------------*\
-Original Copyright
- FILE........: AK2LSPD.H
- TYPE........: Turbo C header file
- COMPANY.....: Voicetronix
- AUTHOR......: James Whitehall
- DATE CREATED: 21/11/95
-
-Modified by Jean-Marc Valin
-
- This file contains functions for converting Linear Prediction
- Coefficients (LPC) to Line Spectral Pair (LSP) and back. Note that the
- LSP coefficients are not in radians format but in the x domain of the
- unit circle.
-
-\*---------------------------------------------------------------------------*/
-/**
- @file lsp.h
- @brief Line Spectral Pair (LSP) functions.
-*/
-/* Speex License:
-
- Redistribution and use in source and binary forms, with or without
- modification, are permitted provided that the following conditions
- are met:
-
- - Redistributions of source code must retain the above copyright
- notice, this list of conditions and the following disclaimer.
-
- - Redistributions in binary form must reproduce the above copyright
- notice, this list of conditions and the following disclaimer in the
- documentation and/or other materials provided with the distribution.
-
- - Neither the name of the Xiph.org Foundation nor the names of its
- contributors may be used to endorse or promote products derived from
- this software without specific prior written permission.
-
- THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR
- CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
- EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
-*/
-
-#ifndef __AK2LSPD__
-#define __AK2LSPD__
-
-#include "arch.h"
-
-int lpc_to_lsp (spx_coef_t *a, int lpcrdr, spx_lsp_t *freq, int nb, spx_word16_t delta, char *stack);
-void lsp_to_lpc(spx_lsp_t *freq, spx_coef_t *ak, int lpcrdr, char *stack);
-
-/*Added by JMV*/
-void lsp_enforce_margin(spx_lsp_t *lsp, int len, spx_word16_t margin);
-
-void lsp_interpolate(spx_lsp_t *old_lsp, spx_lsp_t *new_lsp, spx_lsp_t *interp_lsp, int len, int subframe, int nb_subframes);
-
-#endif /* __AK2LSPD__ */
+/*---------------------------------------------------------------------------*\ +Original Copyright + FILE........: AK2LSPD.H + TYPE........: Turbo C header file + COMPANY.....: Voicetronix + AUTHOR......: James Whitehall + DATE CREATED: 21/11/95 + +Modified by Jean-Marc Valin + + This file contains functions for converting Linear Prediction + Coefficients (LPC) to Line Spectral Pair (LSP) and back. Note that the + LSP coefficients are not in radians format but in the x domain of the + unit circle. + +\*---------------------------------------------------------------------------*/ +/** + @file lsp.h + @brief Line Spectral Pair (LSP) functions. +*/ +/* Speex License: + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions + are met: + + - Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + - Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + + - Neither the name of the Xiph.org Foundation nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, + EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, + PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR + PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF + LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING + NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +*/ + +#ifndef __AK2LSPD__ +#define __AK2LSPD__ + +#include "arch.h" + +int lpc_to_lsp (spx_coef_t *a, int lpcrdr, spx_lsp_t *freq, int nb, spx_word16_t delta, char *stack); +void lsp_to_lpc(spx_lsp_t *freq, spx_coef_t *ak, int lpcrdr, char *stack); + +/*Added by JMV*/ +void lsp_enforce_margin(spx_lsp_t *lsp, int len, spx_word16_t margin); + +void lsp_interpolate(spx_lsp_t *old_lsp, spx_lsp_t *new_lsp, spx_lsp_t *interp_lsp, int len, int subframe, int nb_subframes); + +#endif /* __AK2LSPD__ */ diff --git a/drivers/speex/speex_bind.cpp b/drivers/speex/speex_bind.cpp index 6e9eb638a2..d15bb3da8c 100644 --- a/drivers/speex/speex_bind.cpp +++ b/drivers/speex/speex_bind.cpp @@ -1,64 +1,64 @@ -
-#include "memory.h"
-#include "speex_bind.h"
-#include
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-void *speex_alloc (int size) {
-
- uint8_t * mem = (uint8_t*)memalloc(size);
- for(int i=0;i<size;i++)
- mem[i]=0;
- return mem;
-}
-
-void *speex_alloc_scratch (int size) {
-
- return memalloc(size);
-}
-
-void *speex_realloc (void *ptr, int size) {
-
- return memrealloc(ptr,size);
-}
-
-void speex_free (void *ptr) {
-
- memfree(ptr);
-}
-
-void speex_free_scratch (void *ptr) {
-
- memfree(ptr);
-}
-
-void _speex_fatal(const char *str, const char *file, int line) {
-
- _err_print_error("SPEEX ERROR",p_file,p_line,str);
-}
-
-void speex_warning(const char *str) {
-
- _err_print_error("SPEEX WARNING","",0,str);
-}
-
-void speex_warning_int(const char *str, int val) {
-
- _err_print_error("SPEEX WARNING INT","Value",val,str);
-}
-
-void speex_notify(const char *str) {
-
- print_line(str);
-}
-
-void _speex_putc(int ch, void *file) {
-
- // will not putc, no.
-}
-
-#ifdef __cplusplus
-}
-#endif
+ +#include "memory.h" +#include "speex_bind.h" +#include +#ifdef __cplusplus +extern "C" { +#endif + +void *speex_alloc (int size) { + + uint8_t * mem = (uint8_t*)memalloc(size); + for(int i=0;i<size;i++) + mem[i]=0; + return mem; +} + +void *speex_alloc_scratch (int size) { + + return memalloc(size); +} + +void *speex_realloc (void *ptr, int size) { + + return memrealloc(ptr,size); +} + +void speex_free (void *ptr) { + + memfree(ptr); +} + +void speex_free_scratch (void *ptr) { + + memfree(ptr); +} + +void _speex_fatal(const char *str, const char *file, int line) { + + _err_print_error("SPEEX ERROR",p_file,p_line,str); +} + +void speex_warning(const char *str) { + + _err_print_error("SPEEX WARNING","",0,str); +} + +void speex_warning_int(const char *str, int val) { + + _err_print_error("SPEEX WARNING INT","Value",val,str); +} + +void speex_notify(const char *str) { + + print_line(str); +} + +void _speex_putc(int ch, void *file) { + + // will not putc, no. +} + +#ifdef __cplusplus +} +#endif diff --git a/drivers/speex/speex_bind.h b/drivers/speex/speex_bind.h index e842960d3c..c928430a33 100644 --- a/drivers/speex/speex_bind.h +++ b/drivers/speex/speex_bind.h @@ -1,48 +1,48 @@ -#ifndef SPEEX_BIND_H
-#define SPEEX_BIND_H
-
-#ifdef __cplusplus
-extern "C" {
-#endif
-
-/*
-#define OVERRIDE_SPEEX_ALLOC
-#define OVERRIDE_SPEEX_ALLOC_SCRATCH
-#define OVERRIDE_SPEEX_REALLOC
-#define OVERRIDE_SPEEX_FREE
-#define OVERRIDE_SPEEX_FREE_SCRATCH
-#define OVERRIDE_SPEEX_FATAL
-#define OVERRIDE_SPEEX_WARNING
-#define OVERRIDE_SPEEX_WARNING_INT
-#define OVERRIDE_SPEEX_NOTIFY
-#define OVERRIDE_SPEEX_PUTC
-
-void *speex_alloc (int size);
-void *speex_alloc_scratch (int size);
-void *speex_realloc (void *ptr, int size);
-void speex_free (void *ptr);
-void speex_free_scratch (void *ptr);
-void _speex_fatal(const char *str, const char *file, int line);
-void speex_warning(const char *str);
-void speex_warning_int(const char *str, int val);
-void speex_notify(const char *str);
-void _speex_putc(int ch, void *file);
-
-
-*/
-#define RELEASE
-#define SPEEX_PI 3.14159265358979323846
-
-#ifdef _MSC_VER
-#define SPEEX_INLINE __inline
-#else
-#define SPEEX_INLINE inline
-#endif
-
-
-
-#ifdef __cplusplus
-}
-#endif
-
-#endif // SPEEX_BIND_H
+#ifndef SPEEX_BIND_H +#define SPEEX_BIND_H + +#ifdef __cplusplus +extern "C" { +#endif + +/* +#define OVERRIDE_SPEEX_ALLOC +#define OVERRIDE_SPEEX_ALLOC_SCRATCH +#define OVERRIDE_SPEEX_REALLOC +#define OVERRIDE_SPEEX_FREE +#define OVERRIDE_SPEEX_FREE_SCRATCH +#define OVERRIDE_SPEEX_FATAL +#define OVERRIDE_SPEEX_WARNING +#define OVERRIDE_SPEEX_WARNING_INT +#define OVERRIDE_SPEEX_NOTIFY +#define OVERRIDE_SPEEX_PUTC + +void *speex_alloc (int size); +void *speex_alloc_scratch (int size); +void *speex_realloc (void *ptr, int size); +void speex_free (void *ptr); +void speex_free_scratch (void *ptr); +void _speex_fatal(const char *str, const char *file, int line); +void speex_warning(const char *str); +void speex_warning_int(const char *str, int val); +void speex_notify(const char *str); +void _speex_putc(int ch, void *file); + + +*/ +#define RELEASE +#define SPEEX_PI 3.14159265358979323846 + +#ifdef _MSC_VER +#define SPEEX_INLINE __inline +#else +#define SPEEX_INLINE inline +#endif + + + +#ifdef __cplusplus +} +#endif + +#endif // SPEEX_BIND_H |