Fix integer overflow for bigger textures
Mathias Gumz akira at fluxbox dot org
1 files changed,
6 insertions(+),
2 deletions(-)
jump to
M
src/FbTk/TextureRender.cc
→
src/FbTk/TextureRender.cc
@@ -111,6 +111,10 @@ return (x * other_y) - (other_x * y);
} }; +template <typename T> int sign(T val) { + return (T(0) < val) - (val < T(0)); +} + std::vector<char>& getGradientBuffer(size_t size) { static std::vector<char> buffer; if (buffer.size() < size)@@ -469,10 +473,10 @@ for (x = 0; x < static_cast<int>(width); ++x, ++i) {
// check, if the point (x, y) is left or right of the vectors // 'a' and 'b'. if the point is on the same side for both 'a' and - // 'b' (a.cross() is equal to b.cross()) then use the y_gradient, + // 'b' (a.cross() is equal to b.cross()) then use the x_gradient, // otherwise use y_gradient - if ((a.cross(x, y) * b.cross(x, b.y + y)) < 0) { + if (sign(a.cross(x, y)) * sign(b.cross(x, b.y + y)) < 0) { rgba[i] = x_gradient[x]; } else { rgba[i] = y_gradient[y];