added all graphics 'state' tests

.github/workflows/main.yml

@@ -49,6 +49,16 @@ jobs:
       with:
         name: love-x86_64-AppImage-debug
         path: love-${{ github.sha }}.AppImage-debug.tar.gz
+    - name: Make Runnable
+      run: chmod a+x love-linux-x86_64.AppImage
+    - name: Run All Tests
+      run: xvfb-run love-linux-x86_64.AppImage testing
+    - name: Love Test Report
+      uses: ellraiser/love-test-report@main
+      with:
+        name: Love Testsuite Linux
+        title: linux-test-report
+        path: testing/output/lovetest_runAllTests.md
   windows-os:
     runs-on: windows-latest
     strategy:
@@ -208,25 +218,22 @@ jobs:
     - name: Build Test Exe
       if: steps.vars.outputs.arch != 'ARM64'
       run: cmake --build build --config Release --target install
-    - name: Run All Tests (OpenGL)
+    - name: Install Mesa
       if: steps.vars.outputs.arch != 'ARM64'
-      run: install\lovec.exe testing/main.lua
-    - name: Love Test Report (OpenGL)
-      if: steps.vars.outputs.arch != 'ARM64'
-      uses: ellraiser/love-test-report@main
-      with:
-        name: Love Testsuite Windows (OpenGL)
-        title: windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-test-report-opengl
-        path: testing/output/lovetest_runAllTests.md
-    - name: Run All Tests (Vulkan)
+      run: |
+        curl.exe -L --output mesa.7z --url https://github.com/pal1000/mesa-dist-win/releases/download/23.2.1/mesa3d-23.2.1-release-msvc.7z
+        7z x mesa.7z
+        mklink opengl32.dll "x64\opengl32.dll"
+        mklink libglapi.dll "x64\libglapi.dll"
+    - name: Run All Tests
       if: steps.vars.outputs.arch != 'ARM64'
-      run: install\lovec.exe testing/main.lua --renderers vulkan
-    - name: Love Test Report (Vulkan)
+      run: install\lovec.exe testing/main.lua
+    - name: Love Test Report
       if: steps.vars.outputs.arch != 'ARM64'
       uses: ellraiser/love-test-report@main
       with:
-        name: Love Testsuite Windows (Vulkan)
-        title: windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-test-report-vulkan
+        name: Love Testsuite Windows
+        title: windows-${{ steps.vars.outputs.arch }}${{ steps.vars.outputs.compatname }}-test-report
         path: testing/output/lovetest_runAllTests.md
   macOS:
     runs-on: macos-latest
@@ -256,21 +263,13 @@ jobs:
       with:
         name: love-macos
         path: love-macos.zip
-    - name: Run All Tests (OpenGL)
+    - name: Run All Tests
       run: love-macos/love.app/Contents/MacOS/love testing
-    - name: Love Test Report (OpenGL)
-      uses: ellraiser/love-test-report@main
-      with:
-        name: Love Testsuite MacOS (OpenGL)
-        title: macos-test-report-opengl
-        path: testing/output/lovetest_runAllTests.md
-    - name: Run All Tests (metal)
-      run: love-macos/love.app/Contents/MacOS/love testing --renderers metal
-    - name: Love Test Report (metal)
+    - name: Love Test Report
       uses: ellraiser/love-test-report@main
       with:
-        name: Love Testsuite MacOS (Metal)
-        title: macos-test-report-metal
+        name: Love Testsuite MacOS
+        title: macos-test-report
         path: testing/output/lovetest_runAllTests.md
   iOS-Simulator:
     runs-on: macos-latest

testing/classes/TestMethod.lua

@@ -25,11 +25,19 @@ TestMethod = {
       result = {},
       colors = {
         red = {1, 0, 0, 1},
+        redpale = {1, 0.5, 0.5, 1},
+        red07 = {0.7, 0, 0, 1},
         green = {0, 1, 0, 1},
+        greenhalf = {0, 0.5, 0, 1},
+        greenfade = {0, 1, 0, 0.5},
         blue = {0, 0, 1, 1},
+        bluefade = {0, 0, 1, 0.5},
+        yellow = {1, 1, 0, 1},
         black = {0, 0, 0, 1},
         white = {1, 1, 1, 1}
-      }
+      },
+      delay = 0,
+      delayed = false
     }
     setmetatable(test, self)
     self.__index = self
@@ -69,6 +77,11 @@ TestMethod = {
         local coord = pixels[p]
         local tr, tg, tb, ta = imgdata:getPixel(coord[1], coord[2])
         local compare_id = tostring(coord[1]) .. ',' .. tostring(coord[2])
+        -- prevent us getting stuff like 0.501960785 for 0.5 red 
+        tr = math.floor((tr*10)+0.5)/10
+        tg = math.floor((tg*10)+0.5)/10
+        tb = math.floor((tb*10)+0.5)/10
+        ta = math.floor((ta*10)+0.5)/10
         -- @TODO add some sort pixel tolerance to the coords
         self:assertEquals(col[1], tr, 'check pixel r for ' .. i .. ' at ' .. compare_id .. '(' .. label .. ')')
         self:assertEquals(col[2], tg, 'check pixel g for ' .. i .. ' at ' .. compare_id .. '(' .. label .. ')')
@@ -201,12 +214,19 @@ TestMethod = {
   -- @desc - quick assert for value not nil 
   -- @param {any} value - value to check not nil
   -- @return {nil}
-  assertNotNil = function (self, value)
+  assertNotNil = function (self, value, err)
     self:assertNotEquals(nil, value, 'check not nil')
+    if err ~= nil then
+      table.insert(self.asserts, {
+        key = 'assert #' .. tostring(self.count),
+        passed = false,
+        message = err,
+        test = 'assert not nil catch'
+      })
+    end
   end,
 
 
-
   -- @method - TestMethod:skipTest()
   -- @desc - used to mark this test as skipped for a specific reason
   -- @param {string} reason - reason why method is being skipped
@@ -217,6 +237,17 @@ TestMethod = {
   end,
 
 
+  -- currently unused
+  setDelay = function(self, frames)
+    self.delay = frames
+    self.delayed = true
+    love.test.delayed = self
+  end,
+  isDelayed = function(self)
+    return self.delayed
+  end,
+
+
   -- @method - TestMethod:evaluateTest()
   -- @desc - evaluates the results of all assertions for a final restult
   -- @return {nil}

testing/classes/TestSuite.lua

@@ -10,6 +10,7 @@ TestSuite = {
       -- testsuite internals
       modules = {},
       module = nil,
+      test = nil,
       testcanvas = nil,
       current = 1,
       output = '',
@@ -19,6 +20,7 @@ TestSuite = {
       html = '',
       mdrows = '',
       mdfailures = '',
+      delayed = nil,
       fakequit = false,
       windowmode = true,
 
@@ -70,7 +72,8 @@ TestSuite = {
             if self.module.called[self.module.index] == nil then
               self.module.called[self.module.index] = true
               local method = self.module.running[self.module.index]
-              local test = TestMethod:new(method, self.module)
+              self.test = TestMethod:new(method, self.module)
+              TextRun:set('love.' .. self.module.module .. '.' .. method)
 
               -- check method exists in love first
               if self.module.module ~= 'objects' and (love[self.module.module] == nil or love[self.module.module][method] == nil) then
@@ -80,26 +83,52 @@ TestSuite = {
                   tested .. matching,
                   ' ==> FAIL (0/0) - call failed - method does not exist'
                 )
-              -- otherwise run the test method then eval the asserts
+              -- otherwise run the test method
               else
-                local ok, chunk, err = pcall(self[self.module.module][method], test)
+                local ok, chunk, err = pcall(self[self.module.module][method], self.test)
                 if ok == false then
                   print("FATAL", chunk, err)
-                  test.fatal = tostring(chunk) .. tostring(err)
+                  self.test.fatal = tostring(chunk) .. tostring(err)
                 end
-                local ok, chunk, err = pcall(test.evaluateTest, test)
+              end
+
+            -- once we've run check delay + eval
+            else
+
+              -- @TODO use coroutines?
+              -- if we have a test method that needs a delay
+              -- we wait for the delay to run out first
+              if self.delayed ~= nil then
+                self.delayed.delay = self.delayed.delay - 1
+                -- re-run the test method again when delay ends
+                -- its up to the test to handle the :isDelayed() property
+                if self.delayed.delay <= 0 then
+                  local ok, chunk, err = pcall(self[self.module.module][self.delayed.method], self.test)
+                  if ok == false then
+                    print("FATAL", chunk, err)
+                    self.test.fatal = tostring(chunk) .. tostring(err)
+                  end
+                  self.delayed = nil
+                end
+              else
+
+                -- now we're all done evaluate the test 
+                local ok, chunk, err = pcall(self.test.evaluateTest, self.test)
                 if ok == false then
                   print("FATAL", chunk, err)
-                  test.fatal = tostring(chunk) .. tostring(err)
+                  self.test.fatal = tostring(chunk) .. tostring(err)
                 end
+                -- save having to :release() anything we made in the last test
+                -- 7251ms > 7543ms
+                collectgarbage("collect")
+                -- move onto the next test
+                self.module.index = self.module.index + 1
+
               end
-              -- save having to :release() anything we made in the last test
-              -- 7251ms > 7543ms
-              collectgarbage("collect")
-              -- move onto the next test
-              self.module.index = self.module.index + 1
+
             end
 
+          -- once all tests have run
           else
 
             -- print module results and add to output

testing/conf.lua

@@ -1,8 +1,8 @@
 function love.conf(t)
   t.console = true
   t.window.name = 'love.test'
-  t.window.width = 256
-  t.window.height = 256
+  t.window.width = 360
+  t.window.height = 240
   t.window.resizable = true
   t.renderers = {"opengl"}
   t.modules.audio = true

testing/main.lua

@@ -33,7 +33,7 @@ love.load = function(args)
 
   -- setup basic img to display
   if love.window ~= nil then
-    love.window.setMode(256, 256, {
+    love.window.setMode(360, 240, {
       fullscreen = false,
       resizable = true,
       centered = true
@@ -47,6 +47,9 @@ love.load = function(args)
         img = nil
       }
       Logo.img = love.graphics.newQuad(0, 0, 64, 64, Logo.texture)
+      Font = love.graphics.newFont('resources/font.ttf', 8, 'normal')
+      TextCommand = love.graphics.newTextBatch(Font, 'Loading...')
+      TextRun = love.graphics.newTextBatch(Font, '')
     end
   end
 
@@ -63,6 +66,7 @@ love.load = function(args)
   local testcmd = '--runAllTests'
   local module = ''
   local method = ''
+  local cmderr = 'Invalid flag used'
   local modules = {
     'audio', 'data', 'event', 'filesystem', 'font', 'graphics',
     'image', 'math', 'objects', 'physics', 'sound', 'system',
@@ -97,9 +101,14 @@ love.load = function(args)
   if testcmd == '--runSpecificMethod' then
     local testmodule = TestModule:new(module, method)
     table.insert(love.test.modules, testmodule)
-    love.test.module = testmodule
-    love.test.module:log('grey', '--runSpecificMethod "' .. module .. '" "' .. method .. '"')
-    love.test.output = 'lovetest_runSpecificMethod_' .. module .. '_' .. method
+    if module ~= '' and method ~= '' then
+      love.test.module = testmodule
+      love.test.module:log('grey', '--runSpecificMethod "' .. module .. '" "' .. method .. '"')
+      love.test.output = 'lovetest_runSpecificMethod_' .. module .. '_' .. method
+    else
+      if method == '' then cmderr = 'No valid method specified' end
+      if module == '' then cmderr = 'No valid module specified' end
+    end
   end
 
   -- runSpecificModules runs all methods for all the modules given
@@ -110,10 +119,13 @@ love.load = function(args)
       table.insert(love.test.modules, testmodule)
       table.insert(modulelist, modules[m])
     end
-    
-    love.test.module = love.test.modules[1]
-    love.test.module:log('grey', '--runSpecificModules "' .. table.concat(modulelist, '" "') .. '"')
-    love.test.output = 'lovetest_runSpecificModules_' .. table.concat(modulelist, '_')
+    if #modulelist > 0 then
+      love.test.module = love.test.modules[1]
+      love.test.module:log('grey', '--runSpecificModules "' .. table.concat(modulelist, '" "') .. '"')
+      love.test.output = 'lovetest_runSpecificModules_' .. table.concat(modulelist, '_')
+    else
+      cmderr = 'No modules specified'
+    end
   end
 
   -- otherwise default runs all methods for all modules
@@ -129,12 +141,14 @@ love.load = function(args)
 
   -- invalid command
   if love.test.module == nil then
-    print("Wrong flags used")
+    print(cmderr)
+    love.event.quit(0)
+  else 
+    -- start first module
+    TextCommand:set(testcmd)
+    love.test.module:runTests()
   end
 
-  -- start first module
-  love.test.module:runTests()
-
 end
 
 -- love.update
@@ -147,7 +161,11 @@ end
 -- love.draw
 -- draw a little logo to the screen
 love.draw = function()
-  love.graphics.draw(Logo.texture, Logo.img, 64, 64, 0, 2, 2)
+  local lw = (love.graphics.getPixelWidth() - 128) / 2
+  local lh = (love.graphics.getPixelHeight() - 128) / 2
+  love.graphics.draw(Logo.texture, Logo.img, lw, lh, 0, 2, 2)
+  love.graphics.draw(TextCommand, 4, 12, 0, 2, 2)
+  love.graphics.draw(TextRun, 4, 32, 0, 2, 2)
 end
 
 

testing/readme.md

@@ -12,6 +12,7 @@ Currently written for löve 12
 - [x] No platform-specific dependencies / scripts
 - [x] Ability to run a subset of tests
 - [x] Ability to easily run an individual test.
+- [x] Automatic testing that happens after every commit
 
 ---
 
@@ -81,27 +82,25 @@ For sanity-checking, if it's currently not covered or we're not sure how to test
 ## Coverage
 This is the status of all module tests currently.  
 "objects" is a special module to cover any object specific tests, i.e. testing a File object functions as expected
-```lua
--- [x] audio        26 PASSED |  0 FAILED |  0 SKIPPED
--- [x] data          7 PASSED |  0 FAILED |  3 SKIPPED      [SEE BELOW]
--- [x] event         4 PASSED |  0 FAILED |  2 SKIPPED      [SEE BELOW]
--- [x] filesystem   27 PASSED |  0 FAILED |  2 SKIPPED
--- [x] font          4 PASSED |  0 FAILED |  1 SKIPPED      [SEE BELOW]
--- [ ] graphics     65 PASSED |  0 FAILED | 31 SKIPPED      [SEE BELOW]
--- [x] image         3 PASSED |  0 FAILED |  0 SKIPPED
--- [x] math         17 PASSED |  0 FAILED |  0 SKIPPED
--- [x] physics      22 PASSED |  0 FAILED |  0 SKIPPED
--- [x] sound         2 PASSED |  0 FAILED |  0 SKIPPED
--- [x] system        6 PASSED |  0 FAILED |  2 SKIPPED
--- [x] thread        3 PASSED |  0 FAILED |  0 SKIPPED
--- [x] timer         6 PASSED |  0 FAILED |  0 SKIPPED
--- [x] video         1 PASSED |  0 FAILED |  0 SKIPPED
--- [x] window       32 PASSED |  2 FAILED |  2 SKIPPED      [SEE BELOW]
-
--- [ ] objects      STILL TO BE DONE
---------------------------------------------------------------------------------
--- [x] totals      226 PASSED |  4 FAILED | 43 SKIPPED
-```
+| Module                | Passed | Failed | Skipped | Time   |
+| --------------------- | ------ | ------ | ------- | ------ |
+| 🟢 love.audio | 26 | 0 | 0 | 2.602s |
+| 🟢 love.data | 7 | 0 | 3 | 1.003s |
+| 🟢 love.event | 4 | 0 | 2 | 0.599s |
+| 🟢 love.filesystem | 27 | 0 | 2 | 2.900s |
+| 🟢 love.font | 4 | 0 | 1 | 0.500s |
+| 🟢 love.graphics | 81 | 0 | 15 | 10.678s |
+| 🟢 love.image | 3 | 0 | 0 | 0.300s |
+| 🟢 love.math | 17 | 0 | 0 | 1.678s |
+| 🟢 love.objects | 1 | 0 | 0 | 0.121s |
+| 🟢 love.physics | 22 | 0 | 0 | 2.197s |
+| 🟢 love.sound | 2 | 0 | 0 | 0.200s |
+| 🟢 love.system | 6 | 0 | 2 | 0.802s |
+| 🟢 love.thread | 3 | 0 | 0 | 0.300s |
+| 🟢 love.timer | 6 | 0 | 0 | 2.358s |
+| 🟢 love.video | 1 | 0 | 0 | 0.100s |
+| 🟢 love.window | 34 | 0 | 2 | 8.050s |
+**271** tests were completed in **34.387s** with **244** passed, **0** failed, and **27** skipped
 
 The following modules are not covered as we can't really emulate input nicely:  
 `joystick`, `keyboard`, `mouse`, and `touch`
@@ -113,23 +112,17 @@ Modules with some small bits needed or needing sense checking:
 - **love.data** - packing methods need writing cos i dont really get what they are
 - **love.event** - love.event.wait or love.event.pump need writing if possible I dunno how to check
 - **love.font** - newBMFontRasterizer() wiki entry is wrong so not sure whats expected
-- **love.graphics** - still need to do tests for the drawing and state methods
+- **love.graphics** - still need to do tests for the main drawing methods
 - **love.image** - ideally isCompressed should have an example of all compressed files love can take
 - **love.math** - linearToGamma + gammaToLinear using direct formulas don't get same value back
 - **love.objects** - not started properly yet
-
----
-
-## Failures
-- **love.window.isMaximized()** - returns false after calling love.window.maximize?
-- **love.window.maximize()** - same as above
+- **love.graphics.setStencilTest** - deprecated, replaced by setStencilMode()
 
 ---
 
 ## Stretch Goals
 - [ ] Tests can compare visual results to a reference image
 - [ ] Ability to see all visual results at a glance
-- [ ] Automatic testing that happens after every commit
 - [ ] Ability to test loading different combinations of modules
 - [ ] Performance tests
 

testing/tests/graphics.lua

@@ -112,11 +112,10 @@ love.test.graphics.rectangle = function(test)
   love.graphics.setCanvas()
   local imgdata1 = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
   -- test, check red bg and blue central square
-  local comparepixels = {
+  test:assertPixels(imgdata1, {
     red = {{0,0},{15,0},{15,15},{0,15}},
     blue = {{6,6},{9,6},{9,9},{6,9}}
-  }
-  test:assertPixels(imgdata1, comparepixels, 'fill')
+  }, 'fill')
   -- clear canvas to do some line testing
   love.graphics.setCanvas(canvas)
     love.graphics.clear(0, 0, 0, 1)
@@ -126,16 +125,19 @@ love.test.graphics.rectangle = function(test)
     love.graphics.rectangle('line', 1, 1, 2, 15) -- 3x16 left aligned blue outline
     love.graphics.setColor(0, 1, 0, 1)
     love.graphics.rectangle('line', 11, 1, 5, 15) -- 6x16 right aligned green outline
+    love.graphics.setColor(1, 1, 1, 1)
   love.graphics.setCanvas()
   local imgdata2 = love.graphics.readbackTexture(canvas, {1, 1, 0, 0, 16, 16})
   -- -- check corners and inner corners
-  comparepixels = {
+  test:assertPixels(imgdata2, {
     red = {{3,0},{9,0},{3,15,9,15}},
     blue = {{0,0},{2,0},{0,15},{2,15}},
     green = {{10,0},{15,0},{10,15},{15,15}},
-    black = {{1,1},{1,14},{3,1},{9,1},{3,14},{9,14},{11,1},{14,1},{11,14},{14,14}}
-  }
-  test:assertPixels(imgdata2, comparepixels, 'line')
+    black = {
+      {1,1},{1,14},{3,1},{9,1},{3,14},
+      {9,14},{11,1},{14,1},{11,14},{14,14}
+    }
+  }, 'line')
 end
 
 
@@ -147,10 +149,15 @@ end
 
 
 -- love.graphics.captureScreenshot
--- @NOTE could test this but not with current setup as we need to wait for the 
--- draw frame to finish before we could assert the file was created
 love.test.graphics.captureScreenshot = function(test)
-  test:skipTest('cant test this worked (easily)')
+  if test:isDelayed() == false then
+    love.graphics.captureScreenshot('example-screenshot.png')
+    test:setDelay(10)
+  -- need to wait until end of the frame for the screenshot
+  else
+    test:assertNotNil(love.filesystem.openFile('example-screenshot.png', 'r'))
+    love.filesystem.remove('example-screenshot.png')
+  end
 end
 
 
@@ -611,34 +618,117 @@ love.test.graphics.setBackgroundColor = function(test)
   test:assertEquals(0, g, 'check set bg g')
   test:assertEquals(0, b, 'check set bg b')
   test:assertEquals(1, a, 'check set bg a')
+  love.graphics.setBackgroundColor(0, 0, 0, 1)
 end
 
 
 -- love.graphics.setBlendMode
 love.test.graphics.setBlendMode = function(test)
-  -- set mode, write to canvas, check output
-  test:skipTest('test method needs writing')
+  -- create fully white canvas, then draw diff. pixels through blendmodes
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0.5, 0.5, 0.5, 1)
+    love.graphics.setBlendMode('add', 'alphamultiply')
+    love.graphics.setColor(1, 0, 0, 1)
+    love.graphics.points({1,1})
+    love.graphics.setBlendMode('subtract', 'alphamultiply')
+    love.graphics.setColor(1, 1, 1, 0.5)
+    love.graphics.points({16,1})
+    love.graphics.setBlendMode('multiply', 'premultiplied')
+    love.graphics.setColor(0, 1, 0, 1)
+    love.graphics.points({16,16})
+    love.graphics.setBlendMode('replace', 'premultiplied')
+    love.graphics.setColor(0, 0, 1, 0.5)
+    love.graphics.points({1,16})
+    love.graphics.setColor(1, 1, 1, 1)
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  -- check the 4 corners
+  test:assertPixels(imgdata, {
+    redpale = {{0,0}},
+    black = {{15,0}},
+    greenhalf = {{15,15}},
+    bluefade = {{0,15}}
+  }, 'blend mode')
+  love.graphics.setBlendMode('alpha', 'alphamultiply') -- reset 
 end
 
 
 -- love.graphics.setCanvas
 love.test.graphics.setCanvas = function(test)
   -- make 2 canvas, set to each, draw one to the other, check output
-  test:skipTest('test method needs writing')
+  local canvas1 = love.graphics.newCanvas(16, 16)
+  local canvas2 = love.graphics.newCanvas(16, 16)
+  love.graphics.setCanvas(canvas1)
+    test:assertEquals(canvas1, love.graphics.getCanvas(), 'check canvas 1 set')
+    love.graphics.clear(1, 0, 0, 1)
+  love.graphics.setCanvas(canvas2)
+    test:assertEquals(canvas2, love.graphics.getCanvas(), 'check canvas 2 set')
+    love.graphics.clear(0, 0, 0, 1)
+    love.graphics.draw(canvas1, 0, 0)
+  love.graphics.setCanvas()
+  test:assertEquals(nil, love.graphics.getCanvas(), 'check no canvas set')
+  local imgdata = love.graphics.readbackTexture(canvas2, {16, 0, 0, 0, 16, 16})
+  -- check 2nd canvas is red
+  test:assertPixels(imgdata, {
+    red = {{0,0},{15,0},{15,15},{0,15}}
+  }, 'set canvas')
 end
 
 
 -- love.graphics.setColor
 love.test.graphics.setColor = function(test)
   -- set colors, draw rect, check color 
-  test:skipTest('test method needs writing')
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    love.graphics.setColor(1, 0, 0, 1)
+    local r, g, b, a = love.graphics.getColor()
+    test:assertEquals(1, r, 'check r set')
+    test:assertEquals(0, g, 'check g set')
+    test:assertEquals(0, b, 'check b set')
+    test:assertEquals(1, a, 'check a set')
+    love.graphics.points({{1,1},{6,1},{11,1},{16,1}})
+    love.graphics.setColor(1, 1, 0, 1)
+    love.graphics.points({{1,2},{6,2},{11,2},{16,2}})
+    love.graphics.setColor(0, 1, 0, 0.5)
+    love.graphics.points({{1,3},{6,3},{11,3},{16,3}})
+    love.graphics.setColor(0, 0, 1, 1)
+    love.graphics.points({{1,4},{6,4},{11,4},{16,4}})
+    love.graphics.setColor(1, 1, 1, 1)
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, {
+    red = {{0,0},{5,0},{10,0},{15,0}},
+    yellow = {{0,1},{5,1},{10,1},{15,1}},
+    greenhalf = {{0,2},{5,2},{10,2},{15,2}},
+    blue = {{0,3},{5,3},{10,3},{15,3}}
+  }, 'set color')
 end
 
 
 -- love.graphics.setColorMask
 love.test.graphics.setColorMask = function(test)
   -- set mask, draw stuff, check output pixels
-  test:skipTest('test method needs writing')
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    -- mask off blue
+    love.graphics.setColorMask(true, true, false, true)
+    local r, g, b, a = love.graphics.getColorMask()
+    test:assertEquals(r, true, 'check r mask')
+    test:assertEquals(g, true, 'check g mask')
+    test:assertEquals(b, false, 'check b mask')
+    test:assertEquals(a, true, 'check a mask')
+    -- draw "black" which should then turn to yellow
+    love.graphics.setColor(1, 1, 1, 1)
+    love.graphics.rectangle('fill', 0, 0, 16, 16)
+    love.graphics.setColorMask(true, true, true, true)
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, {
+    yellow = {{0,0},{0,15},{15,15},{15,0}}
+  }, 'set color mask')
 end
 
 
@@ -656,67 +746,243 @@ end
 
 -- love.graphics.setDepthMode
 love.test.graphics.setDepthMode = function(test)
-  test:skipTest('test method needs writing')
+  -- check documented modes are valid
+  local comparemode, write = love.graphics.getDepthMode()
+  local modes = {
+    'equal', 'notequal', 'less', 'lequal', 'gequal',
+    'greater', 'never', 'always'
+  }
+  for m=1,#modes do
+    love.graphics.setDepthMode(modes[m], true)
+    test:assertEquals(modes[m], love.graphics.getDepthMode(), 'check depth mode ' .. modes[m] .. ' set')
+  end
+  love.graphics.setDepthMode(comparemode, write)
+  -- @TODO better graphics drawing specific test
 end
 
 
 -- love.graphics.setFont
 love.test.graphics.setFont = function(test)
-  test:skipTest('test method needs writing')
+  -- set font doesnt return anything so draw with the test font
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setFont(Font)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    love.graphics.setColor(1, 0, 0, 1)
+    love.graphics.print('love', 0, 3)
+    love.graphics.setColor(1, 1, 1, 1)
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, {
+    red = {
+      {0,0},{0,6},{2,6},{6,2},
+      {4,4},{8,4},{6,6},{10,2},
+      {14,2},{12,6}
+    }
+  }, 'set font for print')
 end
 
 
 -- love.graphics.setFrontFaceWinding
 love.test.graphics.setFrontFaceWinding = function(test)
-  test:skipTest('test method needs writing')
+  -- check documented modes are valid
+  local original = love.graphics.getFrontFaceWinding()
+  love.graphics.setFrontFaceWinding('cw')
+  test:assertEquals('cw', love.graphics.getFrontFaceWinding(), 'check ffw cw set')
+  love.graphics.setFrontFaceWinding('ccw')
+  test:assertEquals('ccw', love.graphics.getFrontFaceWinding(), 'check ffw ccw set')
+  love.graphics.setFrontFaceWinding(original)
+  -- @TODO better graphics drawing specific test
 end
 
 
 -- love.graphics.setLineJoin
 love.test.graphics.setLineJoin = function(test)
-  test:skipTest('test method needs writing')
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setFont(Font)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    local line = {0,1,8,1,8,8}
+    love.graphics.setLineStyle('rough')
+    love.graphics.setLineWidth(2)
+    love.graphics.setColor(1, 0, 0)
+    love.graphics.setLineJoin('bevel')
+    love.graphics.line(line)
+    love.graphics.translate(0, 4)
+    love.graphics.setColor(1, 1, 0)
+    love.graphics.setLineJoin('none')
+    love.graphics.line(line)
+    love.graphics.translate(0, 4)
+    love.graphics.setColor(0, 0, 1)
+    love.graphics.setLineJoin('miter')
+    love.graphics.line(line)
+    love.graphics.setColor(1, 1, 1)
+    love.graphics.setLineWidth(1)
+    love.graphics.origin()
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, {
+    black = {{8,0}},
+    red = {{8,4}},
+    yellow = {{8,7}},
+    blue = {{8,8}}
+  }, 'set line join')
 end
 
 
 -- love.graphics.setLineStyle
 love.test.graphics.setLineStyle = function(test)
-  test:skipTest('test method needs writing')
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setFont(Font)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    love.graphics.setColor(1, 0, 0)
+    local line = {0,1,16,1}
+    love.graphics.setLineStyle('rough')
+    love.graphics.line(line)
+    love.graphics.translate(0, 4)
+    love.graphics.setLineStyle('smooth')
+    love.graphics.line(line)
+    love.graphics.setLineStyle('rough')
+    love.graphics.setColor(1, 1, 1)
+    love.graphics.origin()
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, {
+    red = {{0,0},{7,0},{15,0}},
+    red07 = {{0,4},{7,4},{15,4}}
+  }, 'set line style')
 end
 
 
 -- love.graphics.setLineWidth
 love.test.graphics.setLineWidth = function(test)
-  test:skipTest('test method needs writing')
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setFont(Font)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    local line = {0,1,8,1,8,8}
+    love.graphics.setColor(1, 0, 0)
+    love.graphics.setLineWidth(2)
+    love.graphics.line(line)
+    love.graphics.translate(0, 4)
+    love.graphics.setColor(1, 1, 0)
+    love.graphics.setLineWidth(3)
+    love.graphics.line(line)
+    love.graphics.translate(0, 4)
+    love.graphics.setColor(0, 0, 1)
+    love.graphics.setLineWidth(4)
+    love.graphics.line(line)
+    love.graphics.setColor(1, 1, 1)
+    love.graphics.setLineWidth(1)
+    love.graphics.origin()
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, {
+    black = {{0,2},{6,2},{0,6},{5,6},{0,11},{5,11}},
+    red = {{0,0},{0,1},{7,2},{8,2}},
+    yellow = {{0,3},{0,5},{6,6},{8,6}},
+    blue = {{0,7},{0,10},{6,15},{9,15}}
+  }, 'set line width')
 end
 
 
 -- love.graphics.setMeshCullMode
 love.test.graphics.setMeshCullMode = function(test)
-  test:skipTest('test method needs writing')
+  -- check documented modes are valid
+  local original = love.graphics.getMeshCullMode()
+  local modes = {'back', 'front', 'none'}
+  for m=1,#modes do
+    love.graphics.setMeshCullMode(modes[m])
+    test:assertEquals(modes[m], love.graphics.getMeshCullMode(), 'check mesh cull mode ' .. modes[m] .. ' was set')
+  end
+  love.graphics.setMeshCullMode(original)
+  -- @TODO better graphics drawing specific test
 end
 
 
 -- love.graphics.setScissor
 love.test.graphics.setScissor = function(test)
-  test:skipTest('test method needs writing')
+  -- make a scissor for the left half
+  -- then we should be able to fill the canvas with red and only left is filled
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    love.graphics.origin()
+    love.graphics.setScissor(0, 0, 8, 16)
+    love.graphics.clear(1, 0, 0, 1)
+    love.graphics.setColor(1, 1, 1, 1)
+    love.graphics.setScissor()
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, { 
+    red = {{0,0},{7,0},{0,15},{7,15}},
+    black ={{8,0},{8,15},{15,0},{15,15}}
+  }, 'set scissor')
 end
 
 
 -- love.graphics.setShader
 love.test.graphics.setShader = function(test)
-  test:skipTest('test method needs writing')
+  -- make a shader that will only ever draw yellow
+  local pixelcode = 'vec4 effect(vec4 color, Image tex, vec2 texture_coords, vec2 screen_coords) { vec4 texturecolor = Texel(tex, texture_coords); return vec4(1.0,1.0,0.0,1.0);}'
+  local vertexcode = 'vec4 position(mat4 transform_projection, vec4 vertex_position) { return transform_projection * vertex_position; }'
+  local shader = love.graphics.newShader(pixelcode, vertexcode)
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    love.graphics.setShader(shader)
+      -- draw red rectangle
+      love.graphics.setColor(1, 0, 0, 1)
+      love.graphics.rectangle('fill', 0, 0, 16, 16)
+    love.graphics.setShader()
+    love.graphics.setColor(1, 1, 1, 1)
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, { 
+    yellow = {{0,0},{15,0},{0,15},{15,15}},
+  }, 'check shader set to yellow')
 end
 
 
 -- love.graphics.setStencilTest
-love.test.graphics.setStencilMode = function(test)
-  test:skipTest('test method needs writing')
+love.test.graphics.setStencilTest = function(test)
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setCanvas({canvas, stencil=true})
+    love.graphics.clear(0, 0, 0, 1)
+    love.graphics.stencil(function()
+      love.graphics.circle('fill', 8, 8, 6)
+    end, 'replace', 1)
+    love.graphics.setStencilTest('greater', 0)
+    love.graphics.setColor(1, 0, 0, 1)
+    love.graphics.rectangle('fill', 0, 0, 16, 16)
+    love.graphics.setColor(1, 1, 1, 1)
+    love.graphics.setStencilTest()
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, { 
+    red = {{6,2},{9,2},{2,6},{2,9},{13,6},{9,6},{6,13},{9,13}}
+  }, 'check stencil test')
 end
 
 
 -- love.graphics.setWireframe
 love.test.graphics.setWireframe = function(test)
-  test:skipTest('test method needs writing')
+  -- check wireframe outlines
+  love.graphics.setWireframe(true)
+  local canvas = love.graphics.newCanvas(16, 16)
+  love.graphics.setCanvas(canvas)
+    love.graphics.clear(0, 0, 0, 1)
+    love.graphics.setColor(1, 1, 0, 1)
+    love.graphics.rectangle('fill', 2, 2, 13, 13)
+    love.graphics.setColor(1, 1, 1, 1)
+    love.graphics.setWireframe(false)
+  love.graphics.setCanvas()
+  local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
+  test:assertPixels(imgdata, { 
+    yellow = {{1,14},{14,1},{14,14},{2,2},{13,13}},
+    black = {{2,13},{13,2}}
+  }, 'set wireframe')
 end
 
 
@@ -860,7 +1126,6 @@ love.test.graphics.rotate = function(test)
   love.graphics.setCanvas()
   local imgdata = love.graphics.readbackTexture(canvas, {16, 0, 0, 0, 16, 16})
   test:assertPixels(imgdata, { red = {{0,0},{3,0},{3,3},{0,3}} }, 'rotate 90')
-  imgdata:encode('png', 'rotate.png')
 end
 
 

testing/tests/system.lua

@@ -30,10 +30,10 @@ love.test.system.getPowerInfo = function(test)
   test:assertMatch(states, state, 'check value matches')
   -- if percent/seconds check within expected range
   if percent ~= nil then
-    test:assertRange(percent, 0, 100, 'check value within range')
+    test:assertRange(percent, 0, 100, 'check battery percent within range')
   end
   if seconds ~= nil then
-    test:assertRange(seconds, 0, 100, 'check value within range')
+    test:assertNotNil(seconds)
   end
 end
 

testing/tests/window.lua

@@ -5,15 +5,13 @@
 love.test.window.close = function(test)
   -- closing window should cause graphics to not be active
   love.window.close()
-  local active = false
-  if love.graphics ~= nil then 
-    active = love.graphics.isActive() 
-  end
-  test:assertEquals(false, active, 'check window active')
-  love.window.setMode(256, 256) -- reset 
+  local active = love.graphics.isActive()
+  test:assertEquals(false, active, 'check window not active')
+  love.window.updateMode(360, 240) -- reset
+  active = love.graphics.isActive() 
+  test:assertEquals(true, active, 'check window active again')
 end
 
-
 -- love.window.fromPixels
 love.test.window.fromPixels = function(test)
   -- check dpi/pixel ratio as expected
@@ -93,8 +91,8 @@ end
 -- @NOTE could prob add more checks on the flags here based on conf.lua
 love.test.window.getMode = function(test)
   local w, h, flags = love.window.getMode()
-  test:assertEquals(256, w, 'check w')
-  test:assertEquals(256, h, 'check h')
+  test:assertEquals(360, w, 'check w')
+  test:assertEquals(240, h, 'check h')
   test:assertEquals(false, flags["fullscreen"], 'check fullscreen')
 end
 
@@ -169,13 +167,14 @@ end
 
 -- love.window.isMaximized
 love.test.window.isMaximized = function(test)
-  -- check minimized to start
-  love.window.minimize()
-  test:assertEquals(false, love.window.isMaximized(), 'check window maximized')
-  -- try to mazimize
-  love.window.maximize()
-  test:assertEquals(true, love.window.isMaximized(), 'check window not maximized')
-  love.window.restore()
+  if test:isDelayed() == false then
+    love.window.maximize()
+    test:setDelay(10)
+  else
+    -- on MACOS maximize wont get recognised immedietely so wait a few frames
+    test:assertEquals(true, love.window.isMaximized(), 'check window now maximized')
+    love.window.restore()
+  end
 end
 
 
@@ -197,7 +196,7 @@ love.test.window.isOpen = function(test)
   -- try closing
   love.window.close()
   test:assertEquals(false, love.window.isOpen(), 'check window closed')
-  love.window.setMode(256, 256) -- reset 
+  love.window.updateMode(360, 240) -- reset 
 end
 
 
@@ -208,16 +207,21 @@ love.test.window.isVisible = function(test)
   -- check closing makes window not visible
   love.window.close()
   test:assertEquals(false, love.window.isVisible(), 'check window not visible')
-  love.window.setMode(256, 256) -- reset 
+  love.window.updateMode(360, 240) -- reset 
 end
 
 
 -- love.window.maximize
 love.test.window.maximize = function(test)
-  -- check maximizing is set
-  love.window.maximize()
-  test:assertEquals(true, love.window.isMaximized(), 'check window maximized')
-  love.window.restore()
+  if test:isDelayed() == false then
+    -- check maximizing is set
+    love.window.maximize()
+    test:setDelay(10)
+  else
+    -- on macos we need to wait a few frames
+    test:assertEquals(true, love.window.isMaximized(), 'check window maximized')
+    love.window.restore()
+  end
 end
 
 
@@ -293,7 +297,7 @@ love.test.window.setMode = function(test)
   test:assertEquals(512, height, 'check window h match')
   test:assertEquals(false, flags["fullscreen"], 'check window not fullscreen')
   test:assertEquals(false, flags["resizable"], 'check window not resizeable')
-  love.window.setMode(256, 256, {
+  love.window.setMode(360, 240, {
     fullscreen = false,
     resizable = true
   })
@@ -353,14 +357,14 @@ love.test.window.updateMode = function(test)
     resizable = false
   })
   -- update mode with some props but not others
-  love.window.updateMode(256, 256, nil)
+  love.window.updateMode(360, 240, nil)
   -- check only changed values changed
   local width, height, flags = love.window.getMode()
-  test:assertEquals(256, width, 'check window w match')
-  test:assertEquals(256, height, 'check window h match')
+  test:assertEquals(360, width, 'check window w match')
+  test:assertEquals(240, height, 'check window h match')
   test:assertEquals(false, flags["fullscreen"], 'check window not fullscreen')
   test:assertEquals(false, flags["resizable"], 'check window not resizeable')
-  love.window.setMode(256, 256, { -- reset
+  love.window.setMode(360, 240, { -- reset
     fullscreen = false,
     resizable = true
   })

testing/todo.md

@@ -1,18 +1,34 @@
 `/Applications/love_12.app/Contents/MacOS/love ./testing`
 
-## CI
-- [ ] ignore test suite for windows AMD
-- [ ] add test run to linux (opengl+vulkan) + ios builds (opengl+metal)
-
 ## TESTSUITE
-- [ ] finish graphics state methods
+- [ ] setStencilMode to replace setStencilTest
 - [ ] start graphics drawing methods
 - [ ] start object methods
 
+## GRAPHICS
+Methods that need a better actual graphics check if possible:
+- [ ] setDepthMode
+- [ ] setFrontFaceWinding
+- [ ] setMeshCullMode
+
 ## FUTURE
-- [ ] pass in err string returns to the test output
-  maybe even assertNotNil could use the second value automatically
-  test:assertNotNil(love.filesystem.openFile('file2', 'r')) wouldn't have to change
-- [ ] some joystick/input stuff could be at least nil checked maybe?
 - [ ] need a platform: format table somewhere for compressed formats (i.e. DXT not supported)
-  could add platform as global to command and then use in tests?
+      could add platform as global to command and then use in tests?
+- [ ] use coroutines for the delay action? i.e. wrap each test call in coroutine 
+      and then every test can use coroutine.yield() if needed
+- [ ] could nil check some joystick and keyboard methods?
+
+## GITHUB ACTION CI
+- [ ] linux needs to run xvfb-run with the appimage
+- [ ] windows can try installing mesa for opengl replacement
+- [ ] ios test run?
+
+Can't run --renderers metal on github action images:
+Run love-macos/love.app/Contents/MacOS/love testing --renderers metal
+Cannot create Metal renderer: Metal is not supported on this system.
+Cannot create graphics: no supported renderer on this system.
+Error: Cannot create graphics: no supported renderer on this system.
+
+Can't run test suite on windows as it stands:
+Unable to create renderer
+This program requires a graphics card and video drivers which support OpenGL 2.1 or OpenGL ES 2.