From bf2f2a715fa495af31761aaf9bb86766a1e598dc Mon Sep 17 00:00:00 2001 From: FernandoS27 Date: Thu, 20 Sep 2018 23:06:53 -0400 Subject: [PATCH 1/6] Improved fast swizzle and removed restrictions to it --- src/video_core/textures/decoders.cpp | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 20ba6d4f6..225c999dc 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -46,6 +46,7 @@ void CopySwizzledData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_ } } +//This table represents the internal swizzle of a gob. template struct alignas(64) SwizzleTable { constexpr SwizzleTable() { @@ -69,21 +70,25 @@ void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u8* swizzled_da u8* unswizzled_data, bool unswizzle, u32 block_height) { std::array data_ptrs; const std::size_t stride{width * bytes_per_pixel}; - const std::size_t image_width_in_gobs{(stride + 63) / 64}; + const std::size_t gobs_in_x = 64; + const std::size_t gobs_in_y = 8; + const std::size_t gobs_size = gobs_in_x*gobs_in_y; + const std::size_t image_width_in_gobs{(stride + gobs_in_x - 1) / gobs_in_x}; const std::size_t copy_size{16}; for (std::size_t y = 0; y < height; ++y) { const std::size_t initial_gob = - (y / (8 * block_height)) * 512 * block_height * image_width_in_gobs + - (y % (8 * block_height) / 8) * 512; + (y / (gobs_in_y * block_height)) * gobs_size * block_height * image_width_in_gobs + + (y % (gobs_in_y * block_height) / gobs_in_y) * gobs_size; const std::size_t pixel_base{y * width * bytes_per_pixel}; - const auto& table = swizzle_table[y % 8]; + const auto& table = swizzle_table[y % gobs_in_y]; for (std::size_t xb = 0; xb < stride; xb += copy_size) { - const std::size_t gob_address{initial_gob + (xb / 64) * 512 * block_height}; + const std::size_t truncated_copy = std::min(copy_size, stride - xb); + const std::size_t gob_address{initial_gob + (xb / gobs_in_x) * gobs_size * block_height}; const std::size_t swizzle_offset{gob_address + table[(xb / 16) % 4]}; const std::size_t pixel_index{xb + pixel_base}; data_ptrs[unswizzle] = swizzled_data + swizzle_offset; data_ptrs[!unswizzle] = unswizzled_data + pixel_index; - std::memcpy(data_ptrs[0], data_ptrs[1], copy_size); + std::memcpy(data_ptrs[0], data_ptrs[1], truncated_copy); } } } @@ -134,7 +139,7 @@ u32 BytesPerPixel(TextureFormat format) { std::vector UnswizzleTexture(VAddr address, u32 tile_size, u32 bytes_per_pixel, u32 width, u32 height, u32 block_height) { std::vector unswizzled_data(width * height * bytes_per_pixel); - if (bytes_per_pixel % 3 != 0 && (width * bytes_per_pixel) % 16 == 0) { + if (bytes_per_pixel % 3 != 0) { FastSwizzleData(width / tile_size, height / tile_size, bytes_per_pixel, Memory::GetPointer(address), unswizzled_data.data(), true, block_height); } else { From 68aaa83836af2b582204c264670086ea94a2ee58 Mon Sep 17 00:00:00 2001 From: FernandoS27 Date: Thu, 20 Sep 2018 23:48:24 -0400 Subject: [PATCH 2/6] Improved Legacy Swizzler to be better documented and work better --- src/video_core/textures/decoders.cpp | 36 ++++++++++++++++------------ 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 225c999dc..89d218bc7 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -14,17 +14,12 @@ namespace Tegra::Texture { /** * Calculates the offset of an (x, y) position within a swizzled texture. - * Taken from the Tegra X1 TRM. + * Taken from the Tegra X1 Technical Reference Manual. pages 1187-1188 */ -static u32 GetSwizzleOffset(u32 x, u32 y, u32 image_width, u32 bytes_per_pixel, u32 block_height) { +static u32 GetSwizzleOffset(u32 x, u32 y, u32 bytes_per_pixel, u32 gob_address) { // Round up to the next gob - const u32 image_width_in_gobs{(image_width * bytes_per_pixel + 63) / 64}; - - u32 GOB_address = 0 + (y / (8 * block_height)) * 512 * block_height * image_width_in_gobs + - (x * bytes_per_pixel / 64) * 512 * block_height + - (y % (8 * block_height) / 8) * 512; x *= bytes_per_pixel; - u32 address = GOB_address + ((x % 64) / 32) * 256 + ((y % 8) / 2) * 64 + ((x % 32) / 16) * 32 + + u32 address = gob_address + ((x % 64) / 32) * 256 + ((y % 8) / 2) * 64 + ((x % 32) / 16) * 32 + (y % 2) * 16 + (x % 16); return address; @@ -32,21 +27,31 @@ static u32 GetSwizzleOffset(u32 x, u32 y, u32 image_width, u32 bytes_per_pixel, void CopySwizzledData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { - u8* data_ptrs[2]; + std::array data_ptrs; + const u32 stride = width * bytes_per_pixel; + const u32 gobs_in_x = 64; + const u32 gobs_in_y = 8; + const u32 gobs_size = gobs_in_x * gobs_in_y; + const u32 image_width_in_gobs{(stride + gobs_in_x - 1) / gobs_in_x}; for (unsigned y = 0; y < height; ++y) { + const u32 gob_y_address = + (y / (gobs_in_y * block_height)) * gobs_size * block_height * image_width_in_gobs + + (y % (gobs_in_y * block_height) / gobs_in_y) * gobs_size; for (unsigned x = 0; x < width; ++x) { - u32 swizzle_offset = GetSwizzleOffset(x, y, width, bytes_per_pixel, block_height); - u32 pixel_index = (x + y * width) * out_bytes_per_pixel; + const u32 gob_address = + gob_y_address + (x * bytes_per_pixel / gobs_in_x) * gobs_size * block_height; + const u32 swizzle_offset = GetSwizzleOffset(x, y, bytes_per_pixel, gob_address); + const u32 pixel_index = (x + y * width) * out_bytes_per_pixel; data_ptrs[unswizzle] = swizzled_data + swizzle_offset; - data_ptrs[!unswizzle] = &unswizzled_data[pixel_index]; + data_ptrs[!unswizzle] = unswizzled_data + pixel_index; std::memcpy(data_ptrs[0], data_ptrs[1], bytes_per_pixel); } } } -//This table represents the internal swizzle of a gob. +// This table represents the internal swizzle of a gob. template struct alignas(64) SwizzleTable { constexpr SwizzleTable() { @@ -72,7 +77,7 @@ void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u8* swizzled_da const std::size_t stride{width * bytes_per_pixel}; const std::size_t gobs_in_x = 64; const std::size_t gobs_in_y = 8; - const std::size_t gobs_size = gobs_in_x*gobs_in_y; + const std::size_t gobs_size = gobs_in_x * gobs_in_y; const std::size_t image_width_in_gobs{(stride + gobs_in_x - 1) / gobs_in_x}; const std::size_t copy_size{16}; for (std::size_t y = 0; y < height; ++y) { @@ -83,7 +88,8 @@ void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u8* swizzled_da const auto& table = swizzle_table[y % gobs_in_y]; for (std::size_t xb = 0; xb < stride; xb += copy_size) { const std::size_t truncated_copy = std::min(copy_size, stride - xb); - const std::size_t gob_address{initial_gob + (xb / gobs_in_x) * gobs_size * block_height}; + const std::size_t gob_address{initial_gob + + (xb / gobs_in_x) * gobs_size * block_height}; const std::size_t swizzle_offset{gob_address + table[(xb / 16) % 4]}; const std::size_t pixel_index{xb + pixel_base}; data_ptrs[unswizzle] = swizzled_data + swizzle_offset; From f020319a45dcac183ec50d12593658670717fd59 Mon Sep 17 00:00:00 2001 From: FernandoS27 Date: Fri, 21 Sep 2018 11:10:44 -0400 Subject: [PATCH 3/6] Remove same output bpp restriction on FastSwizzle --- src/video_core/textures/decoders.cpp | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 89d218bc7..c61195969 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -71,7 +71,7 @@ struct alignas(64) SwizzleTable { constexpr auto swizzle_table = SwizzleTable<8, 4>(); -void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u8* swizzled_data, +void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { std::array data_ptrs; const std::size_t stride{width * bytes_per_pixel}; @@ -84,14 +84,15 @@ void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u8* swizzled_da const std::size_t initial_gob = (y / (gobs_in_y * block_height)) * gobs_size * block_height * image_width_in_gobs + (y % (gobs_in_y * block_height) / gobs_in_y) * gobs_size; - const std::size_t pixel_base{y * width * bytes_per_pixel}; + const std::size_t pixel_base{y * width * out_bytes_per_pixel}; const auto& table = swizzle_table[y % gobs_in_y]; for (std::size_t xb = 0; xb < stride; xb += copy_size) { const std::size_t truncated_copy = std::min(copy_size, stride - xb); const std::size_t gob_address{initial_gob + (xb / gobs_in_x) * gobs_size * block_height}; const std::size_t swizzle_offset{gob_address + table[(xb / 16) % 4]}; - const std::size_t pixel_index{xb + pixel_base}; + const std::size_t out_x = xb*out_bytes_per_pixel / bytes_per_pixel; + const std::size_t pixel_index{out_x + pixel_base}; data_ptrs[unswizzle] = swizzled_data + swizzle_offset; data_ptrs[!unswizzle] = unswizzled_data + pixel_index; std::memcpy(data_ptrs[0], data_ptrs[1], truncated_copy); @@ -146,7 +147,7 @@ std::vector UnswizzleTexture(VAddr address, u32 tile_size, u32 bytes_per_pix u32 height, u32 block_height) { std::vector unswizzled_data(width * height * bytes_per_pixel); if (bytes_per_pixel % 3 != 0) { - FastSwizzleData(width / tile_size, height / tile_size, bytes_per_pixel, + FastSwizzleData(width / tile_size, height / tile_size, bytes_per_pixel, bytes_per_pixel, Memory::GetPointer(address), unswizzled_data.data(), true, block_height); } else { CopySwizzledData(width / tile_size, height / tile_size, bytes_per_pixel, bytes_per_pixel, From 41c6c4593aa35ce1f9f9a59415e5e159fad75d82 Mon Sep 17 00:00:00 2001 From: FernandoS27 Date: Fri, 21 Sep 2018 11:29:54 -0400 Subject: [PATCH 4/6] Standarized Legacy Swizzle to look alike FastSwizzle and use a Swizzling Table instead --- src/video_core/textures/decoders.cpp | 84 +++++++++++++--------------- 1 file changed, 40 insertions(+), 44 deletions(-) diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index c61195969..b7d53ced1 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -13,53 +13,20 @@ namespace Tegra::Texture { /** + * This table represents the internal swizzle of a gob, + * in format 16 bytes x 2 sector packing. * Calculates the offset of an (x, y) position within a swizzled texture. * Taken from the Tegra X1 Technical Reference Manual. pages 1187-1188 */ -static u32 GetSwizzleOffset(u32 x, u32 y, u32 bytes_per_pixel, u32 gob_address) { - // Round up to the next gob - x *= bytes_per_pixel; - u32 address = gob_address + ((x % 64) / 32) * 256 + ((y % 8) / 2) * 64 + ((x % 32) / 16) * 32 + - (y % 2) * 16 + (x % 16); - - return address; -} - -void CopySwizzledData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, - u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { - std::array data_ptrs; - const u32 stride = width * bytes_per_pixel; - const u32 gobs_in_x = 64; - const u32 gobs_in_y = 8; - const u32 gobs_size = gobs_in_x * gobs_in_y; - const u32 image_width_in_gobs{(stride + gobs_in_x - 1) / gobs_in_x}; - for (unsigned y = 0; y < height; ++y) { - const u32 gob_y_address = - (y / (gobs_in_y * block_height)) * gobs_size * block_height * image_width_in_gobs + - (y % (gobs_in_y * block_height) / gobs_in_y) * gobs_size; - for (unsigned x = 0; x < width; ++x) { - const u32 gob_address = - gob_y_address + (x * bytes_per_pixel / gobs_in_x) * gobs_size * block_height; - const u32 swizzle_offset = GetSwizzleOffset(x, y, bytes_per_pixel, gob_address); - const u32 pixel_index = (x + y * width) * out_bytes_per_pixel; - - data_ptrs[unswizzle] = swizzled_data + swizzle_offset; - data_ptrs[!unswizzle] = unswizzled_data + pixel_index; - - std::memcpy(data_ptrs[0], data_ptrs[1], bytes_per_pixel); - } - } -} - -// This table represents the internal swizzle of a gob. -template +template struct alignas(64) SwizzleTable { + static_assert(M * Align == 64, "Swizzle Table does not align to GOB"); constexpr SwizzleTable() { for (u32 y = 0; y < N; ++y) { for (u32 x = 0; x < M; ++x) { - const u32 x2 = x * 16; + const u32 x2 = x * Align; values[y][x] = static_cast(((x2 % 64) / 32) * 256 + ((y % 8) / 2) * 64 + - ((x2 % 32) / 16) * 32 + (y % 2) * 16); + ((x2 % 32) / 16) * 32 + (y % 2) * 16 + (x2 % 16)); } } } @@ -69,10 +36,39 @@ struct alignas(64) SwizzleTable { std::array, N> values{}; }; -constexpr auto swizzle_table = SwizzleTable<8, 4>(); +constexpr auto legacy_swizzle_table = SwizzleTable<8, 64, 1>(); +constexpr auto fast_swizzle_table = SwizzleTable<8, 4, 16>(); -void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, u8* swizzled_data, - u8* unswizzled_data, bool unswizzle, u32 block_height) { +void CopySwizzledData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, + u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { + std::array data_ptrs; + const std::size_t stride = width * bytes_per_pixel; + const std::size_t gobs_in_x = 64; + const std::size_t gobs_in_y = 8; + const std::size_t gobs_size = gobs_in_x * gobs_in_y; + const std::size_t image_width_in_gobs{(stride + gobs_in_x - 1) / gobs_in_x}; + for (std::size_t y = 0; y < height; ++y) { + const std::size_t gob_y_address = + (y / (gobs_in_y * block_height)) * gobs_size * block_height * image_width_in_gobs + + (y % (gobs_in_y * block_height) / gobs_in_y) * gobs_size; + const auto& table = legacy_swizzle_table[y % gobs_in_y]; + for (std::size_t x = 0; x < width; ++x) { + const std::size_t gob_address = + gob_y_address + (x * bytes_per_pixel / gobs_in_x) * gobs_size * block_height; + const std::size_t x2 = x * bytes_per_pixel; + const std::size_t swizzle_offset = gob_address + table[x2 % gobs_in_x]; + const std::size_t pixel_index = (x + y * width) * out_bytes_per_pixel; + + data_ptrs[unswizzle] = swizzled_data + swizzle_offset; + data_ptrs[!unswizzle] = unswizzled_data + pixel_index; + + std::memcpy(data_ptrs[0], data_ptrs[1], bytes_per_pixel); + } + } +} + +void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, + u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { std::array data_ptrs; const std::size_t stride{width * bytes_per_pixel}; const std::size_t gobs_in_x = 64; @@ -85,13 +81,13 @@ void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_p (y / (gobs_in_y * block_height)) * gobs_size * block_height * image_width_in_gobs + (y % (gobs_in_y * block_height) / gobs_in_y) * gobs_size; const std::size_t pixel_base{y * width * out_bytes_per_pixel}; - const auto& table = swizzle_table[y % gobs_in_y]; + const auto& table = fast_swizzle_table[y % gobs_in_y]; for (std::size_t xb = 0; xb < stride; xb += copy_size) { const std::size_t truncated_copy = std::min(copy_size, stride - xb); const std::size_t gob_address{initial_gob + (xb / gobs_in_x) * gobs_size * block_height}; const std::size_t swizzle_offset{gob_address + table[(xb / 16) % 4]}; - const std::size_t out_x = xb*out_bytes_per_pixel / bytes_per_pixel; + const std::size_t out_x = xb * out_bytes_per_pixel / bytes_per_pixel; const std::size_t pixel_index{out_x + pixel_base}; data_ptrs[unswizzle] = swizzled_data + swizzle_offset; data_ptrs[!unswizzle] = unswizzled_data + pixel_index; From d2dd1289bda29fba164bdd45854324e45a80e4c7 Mon Sep 17 00:00:00 2001 From: FernandoS27 Date: Fri, 21 Sep 2018 11:42:34 -0400 Subject: [PATCH 5/6] Join both Swizzle methods within one interface function --- src/video_core/textures/decoders.cpp | 30 ++++++++++++++++++---------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index b7d53ced1..5a2a0b84d 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -39,8 +39,9 @@ struct alignas(64) SwizzleTable { constexpr auto legacy_swizzle_table = SwizzleTable<8, 64, 1>(); constexpr auto fast_swizzle_table = SwizzleTable<8, 4, 16>(); -void CopySwizzledData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, - u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { +static void LegacySwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, + u8* swizzled_data, u8* unswizzled_data, bool unswizzle, + u32 block_height) { std::array data_ptrs; const std::size_t stride = width * bytes_per_pixel; const std::size_t gobs_in_x = 64; @@ -67,8 +68,9 @@ void CopySwizzledData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_ } } -void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, - u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { +static void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, + u8* swizzled_data, u8* unswizzled_data, bool unswizzle, + u32 block_height) { std::array data_ptrs; const std::size_t stride{width * bytes_per_pixel}; const std::size_t gobs_in_x = 64; @@ -96,6 +98,17 @@ void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_p } } +void CopySwizzledData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, + u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { + if (bytes_per_pixel % 3 != 0) { + FastSwizzleData(width, height, bytes_per_pixel, out_bytes_per_pixel, swizzled_data, + unswizzled_data, unswizzle, block_height); + } else { + LegacySwizzleData(width, height, bytes_per_pixel, out_bytes_per_pixel, swizzled_data, + unswizzled_data, unswizzle, block_height); + } +} + u32 BytesPerPixel(TextureFormat format) { switch (format) { case TextureFormat::DXT1: @@ -142,13 +155,8 @@ u32 BytesPerPixel(TextureFormat format) { std::vector UnswizzleTexture(VAddr address, u32 tile_size, u32 bytes_per_pixel, u32 width, u32 height, u32 block_height) { std::vector unswizzled_data(width * height * bytes_per_pixel); - if (bytes_per_pixel % 3 != 0) { - FastSwizzleData(width / tile_size, height / tile_size, bytes_per_pixel, bytes_per_pixel, - Memory::GetPointer(address), unswizzled_data.data(), true, block_height); - } else { - CopySwizzledData(width / tile_size, height / tile_size, bytes_per_pixel, bytes_per_pixel, - Memory::GetPointer(address), unswizzled_data.data(), true, block_height); - } + CopySwizzledData(width / tile_size, height / tile_size, bytes_per_pixel, bytes_per_pixel, + Memory::GetPointer(address), unswizzled_data.data(), true, block_height); return unswizzled_data; } From 57b44200a2bf37cc4775ea88b5ff8ed9f67bc145 Mon Sep 17 00:00:00 2001 From: FernandoS27 Date: Fri, 21 Sep 2018 12:09:59 -0400 Subject: [PATCH 6/6] Reverse stride align restriction on FastSwizzle due to lost performance --- src/video_core/textures/decoders.cpp | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/src/video_core/textures/decoders.cpp b/src/video_core/textures/decoders.cpp index 5a2a0b84d..3d5476e5d 100644 --- a/src/video_core/textures/decoders.cpp +++ b/src/video_core/textures/decoders.cpp @@ -85,7 +85,6 @@ static void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_ const std::size_t pixel_base{y * width * out_bytes_per_pixel}; const auto& table = fast_swizzle_table[y % gobs_in_y]; for (std::size_t xb = 0; xb < stride; xb += copy_size) { - const std::size_t truncated_copy = std::min(copy_size, stride - xb); const std::size_t gob_address{initial_gob + (xb / gobs_in_x) * gobs_size * block_height}; const std::size_t swizzle_offset{gob_address + table[(xb / 16) % 4]}; @@ -93,14 +92,14 @@ static void FastSwizzleData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_ const std::size_t pixel_index{out_x + pixel_base}; data_ptrs[unswizzle] = swizzled_data + swizzle_offset; data_ptrs[!unswizzle] = unswizzled_data + pixel_index; - std::memcpy(data_ptrs[0], data_ptrs[1], truncated_copy); + std::memcpy(data_ptrs[0], data_ptrs[1], copy_size); } } } void CopySwizzledData(u32 width, u32 height, u32 bytes_per_pixel, u32 out_bytes_per_pixel, u8* swizzled_data, u8* unswizzled_data, bool unswizzle, u32 block_height) { - if (bytes_per_pixel % 3 != 0) { + if (bytes_per_pixel % 3 != 0 && (width * bytes_per_pixel) % 16 == 0) { FastSwizzleData(width, height, bytes_per_pixel, out_bytes_per_pixel, swizzled_data, unswizzled_data, unswizzle, block_height); } else {