astc_averages_and_directions.cpp 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627
  1. /*----------------------------------------------------------------------------*/
  2. /**
  3. * This confidential and proprietary software may be used only as
  4. * authorised by a licensing agreement from ARM Limited
  5. * (C) COPYRIGHT 2011-2012 ARM Limited
  6. * ALL RIGHTS RESERVED
  7. *
  8. * The entire notice above must be reproduced on all authorised
  9. * copies and copies may only be made to the extent permitted
  10. * by a licensing agreement from ARM Limited.
  11. *
  12. * @brief Implements functions for finding dominant direction of a set of
  13. * colors, using ARM patent pending method.
  14. */
  15. /*----------------------------------------------------------------------------*/
  16. #include "astc_codec_internals.h"
  17. #include <math.h>
  18. #include "mathlib.h"
  19. #ifdef DEBUG_CAPTURE_NAN
  20. #ifndef _GNU_SOURCE
  21. #define _GNU_SOURCE
  22. #endif
  23. #include <fenv.h>
  24. #endif
  25. /* routines to compute average colors and dominant directions for blocks with 3 and 4 components. */
  26. /*
  27. for a full block, functions to compute averages and dominant directions. The averages and directions are computed separately for each partition.
  28. We have separate versions for blocks with and without alpha, since the processing for blocks with alpha is significantly more expensive.
  29. The direction vectors it produces are NOT normalized.
  30. */
  31. void compute_averages_and_directions_rgba(const partition_info * pt,
  32. const imageblock * blk,
  33. const error_weight_block * ewb,
  34. const float4 * color_scalefactors,
  35. float4 * averages, float4 * directions_rgba, float3 * directions_gba, float3 * directions_rba, float3 * directions_rga, float3 * directions_rgb)
  36. {
  37. int i;
  38. int partition_count = pt->partition_count;
  39. int partition;
  40. for (partition = 0; partition < partition_count; partition++)
  41. {
  42. const uint8_t *weights = pt->texels_of_partition[partition];
  43. int texelcount = pt->texels_per_partition[partition];
  44. float4 base_sum = float4(0, 0, 0, 0);
  45. float partition_weight = 0.0f;
  46. for (i = 0; i < texelcount; i++)
  47. {
  48. int iwt = weights[i];
  49. float weight = ewb->texel_weight[iwt];
  50. float4 texel_datum = float4(blk->work_data[4 * iwt],
  51. blk->work_data[4 * iwt + 1],
  52. blk->work_data[4 * iwt + 2],
  53. blk->work_data[4 * iwt + 3]) * weight;
  54. partition_weight += weight;
  55. base_sum = base_sum + texel_datum;
  56. }
  57. float4 average = base_sum * 1.0f / MAX(partition_weight, 1e-7f);
  58. averages[partition] = average * color_scalefactors[partition];
  59. float4 sum_xp = float4(0, 0, 0, 0);
  60. float4 sum_yp = float4(0, 0, 0, 0);
  61. float4 sum_zp = float4(0, 0, 0, 0);
  62. float4 sum_wp = float4(0, 0, 0, 0);
  63. for (i = 0; i < texelcount; i++)
  64. {
  65. int iwt = weights[i];
  66. float weight = ewb->texel_weight[iwt];
  67. float4 texel_datum = float4(blk->work_data[4 * iwt],
  68. blk->work_data[4 * iwt + 1],
  69. blk->work_data[4 * iwt + 2],
  70. blk->work_data[4 * iwt + 3]);
  71. texel_datum = (texel_datum - average) * weight;
  72. if (texel_datum.x > 0.0f)
  73. sum_xp = sum_xp + texel_datum;
  74. if (texel_datum.y > 0.0f)
  75. sum_yp = sum_yp + texel_datum;
  76. if (texel_datum.z > 0.0f)
  77. sum_zp = sum_zp + texel_datum;
  78. if (texel_datum.w > 0.0f)
  79. sum_wp = sum_wp + texel_datum;
  80. }
  81. float prod_xp = dot(sum_xp, sum_xp);
  82. float prod_yp = dot(sum_yp, sum_yp);
  83. float prod_zp = dot(sum_zp, sum_zp);
  84. float prod_wp = dot(sum_wp, sum_wp);
  85. float4 best_vector = sum_xp;
  86. float best_sum = prod_xp;
  87. if (prod_yp > best_sum)
  88. {
  89. best_vector = sum_yp;
  90. best_sum = prod_yp;
  91. }
  92. if (prod_zp > best_sum)
  93. {
  94. best_vector = sum_zp;
  95. best_sum = prod_zp;
  96. }
  97. if (prod_wp > best_sum)
  98. {
  99. best_vector = sum_wp;
  100. best_sum = prod_wp;
  101. }
  102. directions_rgba[partition] = best_vector;
  103. directions_rgb[partition] = best_vector.xyz;
  104. directions_rga[partition] = best_vector.xyw;
  105. directions_rba[partition] = best_vector.xzw;
  106. directions_gba[partition] = best_vector.yzw;
  107. }
  108. }
  109. void compute_averages_and_directions_rgb(const partition_info * pt,
  110. const imageblock * blk,
  111. const error_weight_block * ewb,
  112. const float4 * color_scalefactors, float3 * averages, float3 * directions_rgb, float2 * directions_rg, float2 * directions_rb, float2 * directions_gb)
  113. {
  114. int i;
  115. int partition_count = pt->partition_count;
  116. int partition;
  117. const float *texel_weights = ewb->texel_weight_rgb;
  118. for (partition = 0; partition < partition_count; partition++)
  119. {
  120. const uint8_t *weights = pt->texels_of_partition[partition];
  121. int texelcount = pt->texels_per_partition[partition];
  122. float3 base_sum = float3(0, 0, 0);
  123. float partition_weight = 0.0f;
  124. for (i = 0; i < texelcount; i++)
  125. {
  126. int iwt = weights[i];
  127. float weight = texel_weights[iwt];
  128. float3 texel_datum = float3(blk->work_data[4 * iwt],
  129. blk->work_data[4 * iwt + 1],
  130. blk->work_data[4 * iwt + 2]) * weight;
  131. partition_weight += weight;
  132. base_sum = base_sum + texel_datum;
  133. }
  134. float4 csf = color_scalefactors[partition];
  135. float3 average = base_sum * 1.0f / MAX(partition_weight, 1e-7f);
  136. averages[partition] = average * csf.xyz;
  137. float3 sum_xp = float3(0, 0, 0);
  138. float3 sum_yp = float3(0, 0, 0);
  139. float3 sum_zp = float3(0, 0, 0);
  140. for (i = 0; i < texelcount; i++)
  141. {
  142. int iwt = weights[i];
  143. float weight = texel_weights[iwt];
  144. float3 texel_datum = float3(blk->work_data[4 * iwt],
  145. blk->work_data[4 * iwt + 1],
  146. blk->work_data[4 * iwt + 2]);
  147. texel_datum = (texel_datum - average) * weight;
  148. if (texel_datum.x > 0.0f)
  149. sum_xp = sum_xp + texel_datum;
  150. if (texel_datum.y > 0.0f)
  151. sum_yp = sum_yp + texel_datum;
  152. if (texel_datum.z > 0.0f)
  153. sum_zp = sum_zp + texel_datum;
  154. }
  155. float prod_xp = dot(sum_xp, sum_xp);
  156. float prod_yp = dot(sum_yp, sum_yp);
  157. float prod_zp = dot(sum_zp, sum_zp);
  158. float3 best_vector = sum_xp;
  159. float best_sum = prod_xp;
  160. if (prod_yp > best_sum)
  161. {
  162. best_vector = sum_yp;
  163. best_sum = prod_yp;
  164. }
  165. if (prod_zp > best_sum)
  166. {
  167. best_vector = sum_zp;
  168. best_sum = prod_zp;
  169. }
  170. directions_rgb[partition] = best_vector;
  171. directions_gb[partition] = best_vector.yz;
  172. directions_rb[partition] = best_vector.xz;
  173. directions_rg[partition] = best_vector.xy;
  174. }
  175. }
  176. void compute_averages_and_directions_3_components(const partition_info * pt,
  177. const imageblock * blk,
  178. const error_weight_block * ewb,
  179. const float3 * color_scalefactors, int component1, int component2, int component3, float3 * averages, float3 * directions)
  180. {
  181. int i;
  182. int partition_count = pt->partition_count;
  183. int partition;
  184. const float *texel_weights;
  185. if (component1 == 1 && component2 == 2 && component3 == 3)
  186. texel_weights = ewb->texel_weight_gba;
  187. else if (component1 == 0 && component2 == 2 && component3 == 3)
  188. texel_weights = ewb->texel_weight_rba;
  189. else if (component1 == 0 && component2 == 1 && component3 == 3)
  190. texel_weights = ewb->texel_weight_rga;
  191. else if (component1 == 0 && component2 == 1 && component3 == 2)
  192. texel_weights = ewb->texel_weight_rgb;
  193. else
  194. {
  195. texel_weights = ewb->texel_weight_gba;
  196. ASTC_CODEC_INTERNAL_ERROR;
  197. }
  198. for (partition = 0; partition < partition_count; partition++)
  199. {
  200. const uint8_t *weights = pt->texels_of_partition[partition];
  201. int texelcount = pt->texels_per_partition[partition];
  202. float3 base_sum = float3(0, 0, 0);
  203. float partition_weight = 0.0f;
  204. for (i = 0; i < texelcount; i++)
  205. {
  206. int iwt = weights[i];
  207. float weight = texel_weights[iwt];
  208. float3 texel_datum = float3(blk->work_data[4 * iwt + component1],
  209. blk->work_data[4 * iwt + component2],
  210. blk->work_data[4 * iwt + component3]) * weight;
  211. partition_weight += weight;
  212. base_sum = base_sum + texel_datum;
  213. }
  214. float3 csf = color_scalefactors[partition];
  215. float3 average = base_sum * 1.0f / MAX(partition_weight, 1e-7f);
  216. averages[partition] = average * csf.xyz;
  217. float3 sum_xp = float3(0, 0, 0);
  218. float3 sum_yp = float3(0, 0, 0);
  219. float3 sum_zp = float3(0, 0, 0);
  220. for (i = 0; i < texelcount; i++)
  221. {
  222. int iwt = weights[i];
  223. float weight = texel_weights[iwt];
  224. float3 texel_datum = float3(blk->work_data[4 * iwt + component1],
  225. blk->work_data[4 * iwt + component2],
  226. blk->work_data[4 * iwt + component3]);
  227. texel_datum = (texel_datum - average) * weight;
  228. if (texel_datum.x > 0.0f)
  229. sum_xp = sum_xp + texel_datum;
  230. if (texel_datum.y > 0.0f)
  231. sum_yp = sum_yp + texel_datum;
  232. if (texel_datum.z > 0.0f)
  233. sum_zp = sum_zp + texel_datum;
  234. }
  235. float prod_xp = dot(sum_xp, sum_xp);
  236. float prod_yp = dot(sum_yp, sum_yp);
  237. float prod_zp = dot(sum_zp, sum_zp);
  238. float3 best_vector = sum_xp;
  239. float best_sum = prod_xp;
  240. if (prod_yp > best_sum)
  241. {
  242. best_vector = sum_yp;
  243. best_sum = prod_yp;
  244. }
  245. if (prod_zp > best_sum)
  246. {
  247. best_vector = sum_zp;
  248. best_sum = prod_zp;
  249. }
  250. if (dot(best_vector, best_vector) < 1e-18)
  251. best_vector = float3(1, 1, 1);
  252. directions[partition] = best_vector;
  253. }
  254. }
  255. void compute_averages_and_directions_2_components(const partition_info * pt,
  256. const imageblock * blk,
  257. const error_weight_block * ewb, const float2 * color_scalefactors, int component1, int component2, float2 * averages, float2 * directions)
  258. {
  259. int i;
  260. int partition_count = pt->partition_count;
  261. int partition;
  262. const float *texel_weights;
  263. if (component1 == 0 && component2 == 1)
  264. texel_weights = ewb->texel_weight_rg;
  265. else if (component1 == 0 && component2 == 2)
  266. texel_weights = ewb->texel_weight_rb;
  267. else if (component1 == 1 && component2 == 2)
  268. texel_weights = ewb->texel_weight_gb;
  269. else
  270. {
  271. texel_weights = ewb->texel_weight_rg;
  272. // unsupported set of color components.
  273. ASTC_CODEC_INTERNAL_ERROR;
  274. exit(1);
  275. }
  276. for (partition = 0; partition < partition_count; partition++)
  277. {
  278. const uint8_t *weights = pt->texels_of_partition[partition];
  279. int texelcount = pt->texels_per_partition[partition];
  280. float2 base_sum = float2(0, 0);
  281. float partition_weight = 0.0f;
  282. for (i = 0; i < texelcount; i++)
  283. {
  284. int iwt = weights[i];
  285. float weight = texel_weights[iwt];
  286. float2 texel_datum = float2(blk->work_data[4 * iwt + component1],
  287. blk->work_data[4 * iwt + component2]) * weight;
  288. partition_weight += weight;
  289. base_sum = base_sum + texel_datum;
  290. }
  291. float2 csf = color_scalefactors[partition];
  292. float2 average = base_sum * 1.0f / MAX(partition_weight, 1e-7f);
  293. averages[partition] = average * csf.xy;
  294. float2 sum_xp = float2(0, 0);
  295. float2 sum_yp = float2(0, 0);
  296. for (i = 0; i < texelcount; i++)
  297. {
  298. int iwt = weights[i];
  299. float weight = texel_weights[iwt];
  300. float2 texel_datum = float2(blk->work_data[4 * iwt + component1],
  301. blk->work_data[4 * iwt + component2]);
  302. texel_datum = (texel_datum - average) * weight;
  303. if (texel_datum.x > 0.0f)
  304. sum_xp = sum_xp + texel_datum;
  305. if (texel_datum.y > 0.0f)
  306. sum_yp = sum_yp + texel_datum;
  307. }
  308. float prod_xp = dot(sum_xp, sum_xp);
  309. float prod_yp = dot(sum_yp, sum_yp);
  310. float2 best_vector = sum_xp;
  311. float best_sum = prod_xp;
  312. if (prod_yp > best_sum)
  313. {
  314. best_vector = sum_yp;
  315. best_sum = prod_yp;
  316. }
  317. directions[partition] = best_vector;
  318. }
  319. }
  320. #define XPASTE(x,y) x##y
  321. #define PASTE(x,y) XPASTE(x,y)
  322. #define TWO_COMPONENT_ERROR_FUNC( funcname, c0_iwt, c1_iwt, c01_name, c01_rname ) \
  323. float funcname( \
  324. const partition_info *pt, \
  325. const imageblock *blk, \
  326. const error_weight_block *ewb, \
  327. const processed_line2 *plines, \
  328. float *length_of_lines \
  329. ) \
  330. { \
  331. int i; \
  332. float errorsum = 0.0f; \
  333. int partition; \
  334. for(partition=0; partition<pt->partition_count; partition++) \
  335. { \
  336. const uint8_t *weights = pt->texels_of_partition[ partition ]; \
  337. int texelcount = pt->texels_per_partition[ partition ]; \
  338. float lowparam = 1e10f; \
  339. float highparam = -1e10f; \
  340. processed_line2 l = plines[partition]; \
  341. if( ewb->contains_zeroweight_texels ) \
  342. { \
  343. for(i=0;i<texelcount;i++) \
  344. { \
  345. int iwt = weights[i]; \
  346. float texel_weight = ewb-> PASTE(texel_weight_ , c01_rname) [i]; \
  347. if( texel_weight > 1e-20f ) \
  348. { \
  349. float2 point = float2(blk->work_data[4*iwt + c0_iwt], blk->work_data[4*iwt + c1_iwt] ); \
  350. float param = dot( point, l.bs ); \
  351. float2 rp1 = l.amod + param*l.bis; \
  352. float2 dist = rp1 - point; \
  353. float4 ews = ewb->error_weights[iwt]; \
  354. errorsum += dot( ews. c01_name, dist*dist ); \
  355. if( param < lowparam ) lowparam = param; \
  356. if( param > highparam ) highparam = param; \
  357. } \
  358. } \
  359. } \
  360. else \
  361. { \
  362. for(i=0;i<texelcount;i++) \
  363. { \
  364. int iwt = weights[i]; \
  365. float2 point = float2(blk->work_data[4*iwt + c0_iwt], blk->work_data[4*iwt + c1_iwt] ); \
  366. float param = dot( point, l.bs ); \
  367. float2 rp1 = l.amod + param*l.bis; \
  368. float2 dist = rp1 - point; \
  369. float4 ews = ewb->error_weights[iwt]; \
  370. errorsum += dot( ews. c01_name, dist*dist ); \
  371. if( param < lowparam ) lowparam = param; \
  372. if( param > highparam ) highparam = param; \
  373. } \
  374. } \
  375. float linelen = highparam - lowparam; \
  376. if( !(linelen > 1e-7f) ) \
  377. linelen = 1e-7f; \
  378. length_of_lines[partition] = linelen; \
  379. } \
  380. return errorsum; \
  381. }
  382. TWO_COMPONENT_ERROR_FUNC(compute_error_squared_rg, 0, 1, xy, rg)
  383. TWO_COMPONENT_ERROR_FUNC(compute_error_squared_rb, 0, 2, xz, rb)
  384. TWO_COMPONENT_ERROR_FUNC(compute_error_squared_gb, 1, 2, yz, gb)
  385. TWO_COMPONENT_ERROR_FUNC(compute_error_squared_ra, 0, 3, zw, ra)
  386. // function to compute the error across a tile when using a particular set of
  387. // lines for a particular partitioning. Also compute the length of each
  388. // color-space line in each partitioning.
  389. #define THREE_COMPONENT_ERROR_FUNC( funcname, c0_iwt, c1_iwt, c2_iwt, c012_name, c012_rname ) \
  390. float funcname( \
  391. const partition_info *pt, \
  392. const imageblock *blk, \
  393. const error_weight_block *ewb, \
  394. const processed_line3 *plines, \
  395. float *length_of_lines \
  396. ) \
  397. { \
  398. int i; \
  399. float errorsum = 0.0f; \
  400. int partition; \
  401. for(partition=0; partition<pt->partition_count; partition++) \
  402. { \
  403. const uint8_t *weights = pt->texels_of_partition[ partition ]; \
  404. int texelcount = pt->texels_per_partition[ partition ]; \
  405. float lowparam = 1e10f; \
  406. float highparam = -1e10f; \
  407. processed_line3 l = plines[partition]; \
  408. if( ewb->contains_zeroweight_texels ) \
  409. { \
  410. for(i=0;i<texelcount;i++) \
  411. { \
  412. int iwt = weights[i]; \
  413. float texel_weight = ewb-> PASTE(texel_weight_ , c012_rname) [i]; \
  414. if( texel_weight > 1e-20f ) \
  415. { \
  416. float3 point = float3(blk->work_data[4*iwt + c0_iwt], blk->work_data[4*iwt + c1_iwt], blk->work_data[4*iwt + c2_iwt] ); \
  417. float param = dot( point, l.bs ); \
  418. float3 rp1 = l.amod + param*l.bis; \
  419. float3 dist = rp1 - point; \
  420. float4 ews = ewb->error_weights[iwt]; \
  421. errorsum += dot( ews. c012_name, dist*dist ); \
  422. if( param < lowparam ) lowparam = param; \
  423. if( param > highparam ) highparam = param; \
  424. } \
  425. } \
  426. } \
  427. else \
  428. { \
  429. for(i=0;i<texelcount;i++) \
  430. { \
  431. int iwt = weights[i]; \
  432. float3 point = float3(blk->work_data[4*iwt + c0_iwt], blk->work_data[4*iwt + c1_iwt], blk->work_data[4*iwt + c2_iwt] ); \
  433. float param = dot( point, l.bs ); \
  434. float3 rp1 = l.amod + param*l.bis; \
  435. float3 dist = rp1 - point; \
  436. float4 ews = ewb->error_weights[iwt]; \
  437. errorsum += dot( ews. c012_name, dist*dist ); \
  438. if( param < lowparam ) lowparam = param; \
  439. if( param > highparam ) highparam = param; \
  440. } \
  441. } \
  442. float linelen = highparam - lowparam; \
  443. if( !(linelen > 1e-7f) ) \
  444. linelen = 1e-7f; \
  445. length_of_lines[partition] = linelen; \
  446. } \
  447. return errorsum; \
  448. }
  449. THREE_COMPONENT_ERROR_FUNC(compute_error_squared_gba, 1, 2, 3, yzw, gba)
  450. THREE_COMPONENT_ERROR_FUNC(compute_error_squared_rba, 0, 2, 3, xzw, rba)
  451. THREE_COMPONENT_ERROR_FUNC(compute_error_squared_rga, 0, 1, 3, xyw, rga)
  452. THREE_COMPONENT_ERROR_FUNC(compute_error_squared_rgb, 0, 1, 2, xyz, rgb)
  453. float compute_error_squared_rgba(const partition_info * pt, // the partition that we use when computing the squared-error.
  454. const imageblock * blk, const error_weight_block * ewb, const processed_line4 * plines, float *length_of_lines)
  455. {
  456. int i;
  457. float errorsum = 0.0f;
  458. int partition;
  459. for (partition = 0; partition < pt->partition_count; partition++)
  460. {
  461. const uint8_t *weights = pt->texels_of_partition[partition];
  462. int texelcount = pt->texels_per_partition[partition];
  463. float lowparam = 1e10;
  464. float highparam = -1e10;
  465. processed_line4 l = plines[partition];
  466. if (ewb->contains_zeroweight_texels)
  467. {
  468. for (i = 0; i < texelcount; i++)
  469. {
  470. int iwt = weights[i];
  471. if (ewb->texel_weight[iwt] > 1e-20)
  472. {
  473. float4 point = float4(blk->work_data[4 * iwt], blk->work_data[4 * iwt + 1], blk->work_data[4 * iwt + 2], blk->work_data[4 * iwt + 3]);
  474. float param = dot(point, l.bs);
  475. float4 rp1 = l.amod + param * l.bis;
  476. float4 dist = rp1 - point;
  477. float4 ews = ewb->error_weights[iwt];
  478. errorsum += dot(ews, dist * dist);
  479. if (param < lowparam)
  480. lowparam = param;
  481. if (param > highparam)
  482. highparam = param;
  483. }
  484. }
  485. }
  486. else
  487. {
  488. for (i = 0; i < texelcount; i++)
  489. {
  490. int iwt = weights[i];
  491. float4 point = float4(blk->work_data[4 * iwt], blk->work_data[4 * iwt + 1], blk->work_data[4 * iwt + 2], blk->work_data[4 * iwt + 3]);
  492. float param = dot(point, l.bs);
  493. float4 rp1 = l.amod + param * l.bis;
  494. float4 dist = rp1 - point;
  495. float4 ews = ewb->error_weights[iwt];
  496. errorsum += dot(ews, dist * dist);
  497. if (param < lowparam)
  498. lowparam = param;
  499. if (param > highparam)
  500. highparam = param;
  501. }
  502. }
  503. float linelen = highparam - lowparam;
  504. if (!(linelen > 1e-7f))
  505. linelen = 1e-7f;
  506. length_of_lines[partition] = linelen;
  507. }
  508. return errorsum;
  509. }
  510. // function to compute the error across a tile when using a particular line for
  511. // a particular partition.
  512. float compute_error_squared_rgb_single_partition(int partition_to_test, int xdim, int ydim, int zdim, const partition_info * pt, // the partition that we use when computing the squared-error.
  513. const imageblock * blk, const error_weight_block * ewb, const processed_line3 * lin // the line for the partition.
  514. )
  515. {
  516. int i;
  517. int texels_per_block = xdim * ydim * zdim;
  518. float errorsum = 0.0f;
  519. for (i = 0; i < texels_per_block; i++)
  520. {
  521. int partition = pt->partition_of_texel[i];
  522. float texel_weight = ewb->texel_weight_rgb[i];
  523. if (partition != partition_to_test || texel_weight < 1e-20)
  524. continue;
  525. float3 point = float3(blk->work_data[4 * i], blk->work_data[4 * i + 1], blk->work_data[4 * i + 2]);
  526. float param = dot(point, lin->bs);
  527. float3 rp1 = lin->amod + param * lin->bis;
  528. float3 dist = rp1 - point;
  529. float4 ews = ewb->error_weights[i];
  530. errorsum += dot(ews.xyz, dist * dist);
  531. }
  532. return errorsum;
  533. }