Merge pull request #2041 from ellraiser/main

test suite updates

testing/classes/TestMethod.lua

@@ -374,23 +374,39 @@ TestMethod = {
   end,
 
 
+  -- @method - TestMethod:isOS()
+  -- @desc - checks for a specific OS (or list of OSs)
+  -- @param {string/s} - each arg passed will be checked as a valid OS, as long
+  --                     as one passed the function will return true
+  -- @return {boolean} - returns true if one of the OSs given matches actual OS
+  isOS = function(self, ...)
+    for os=1,select("#", ...) do
+      if select(os, ...) == love.test.current_os then return true end
+    end
+    return false
+  end,
+
   -- @method - TestMethod:evaluateTest()
   -- @desc - evaluates the results of all assertions for a final restult
   -- @return {nil}
   evaluateTest = function(self)
     local failure = ''
     local failures = 0
+
+    -- check all asserts for failures, additional failures are also printed
+    local assert_failures = {}
     for a=1,#self.asserts do
-      -- @TODO show all failed assertion methods?
-      -- currently just shows the first assert that failed
       if not self.asserts[a].passed and not self.skipped then
         if failure == '' then failure = self.asserts[a] end
+        table.insert(assert_failures, self.asserts[a])
         failures = failures + 1
       end
     end
     if self.fatal ~= '' then failure = self.fatal end
     local passed = tostring(#self.asserts - failures)
     local total = '(' .. passed .. '/' .. tostring(#self.asserts) .. ')'
+
+    -- skipped tests have a special log
     if self.skipped then
       self.testmodule.skipped = self.testmodule.skipped + 1
       love.test.totals[3] = love.test.totals[3] + 1
@@ -398,9 +414,12 @@ TestMethod = {
         total = '', 
         result = "SKIP", 
         passed = false, 
-        message = '(0/0) - method skipped [' .. self.skipreason .. ']'
+        message = '(0/0) - method skipped [' .. self.skipreason .. ']',
+        failures = {}
       }
     else
+
+      -- if no failure but has asserts, then passed
       if failure == '' and #self.asserts > 0 then
         self.passed = true
         self.testmodule.passed = self.testmodule.passed + 1
@@ -409,38 +428,48 @@ TestMethod = {
           total = total, 
           result = 'PASS', 
           passed = true, 
-          message = nil
+          message = nil,
+          failures = {}
         }
+
+      -- otherwise it failed
       else
         self.passed = false
         self.testmodule.failed = self.testmodule.failed + 1
         love.test.totals[2] = love.test.totals[2] + 1
+
+        -- no asserts means invalid test
         if #self.asserts == 0 then
           local msg = 'no asserts defined'
           if self.fatal ~= '' then msg = self.fatal end
-          self.result = { 
-            total = total, 
-            result = 'FAIL', 
-            passed = false, 
-            key = 'test', 
-            message = msg 
+          self.result = {
+            total = total,
+            result = 'FAIL',
+            passed = false,
+            key = 'test',
+            message = msg,
+            failures = {}
           }
+
+        -- otherwise we had failures, log the first and supply the list of
+        -- additional failures if any for printResult()
         else
           local key = failure['key']
           if failure['test'] ~= nil then
             key = key .. ' [' .. failure['test'] .. ']'
           end
           local msg = failure['message']
-          if self.fatal ~= '' then 
+          if self.fatal ~= '' then
             key = 'code'
             msg = self.fatal
           end
-          self.result = { 
-            total = total, 
-            result = 'FAIL', 
-            passed = false, 
+          self.result = {
+            total = total,
+            result = 'FAIL',
+            passed = false,
             key = key,
-            message = msg
+            message = msg,
+            failures = assert_failures
           }
         end
       end
@@ -535,6 +564,21 @@ TestMethod = {
       ' ==> ' .. self.result.result .. ' - ' .. endtime .. 's ' ..
       self.result.total .. msg
     )
+
+    -- if we failed on multiple asserts, list them here - makes it easier for 
+    -- debugging new methods added that are failing multiple asserts
+    if #self.result.failures > 1 then
+      for f=2,#self.result.failures do
+        local addf = self.result.failures[f]
+        self.testmodule:log(
+          self.testmodule.colors[self.result.result],
+          '  ' .. tested .. matching,
+          ' ==> ' ..
+          addf['key'] .. ' [' .. addf['test'] .. '] failed - ' .. addf['message']
+        )
+      end
+    end
+
   end
 
 

testing/classes/TestSuite.lua

@@ -23,6 +23,7 @@ TestSuite = {
       delayed = nil,
       fakequit = false,
       windowmode = true,
+      current_os = love._os,
 
       -- love modules to test
       audio = {},
@@ -76,7 +77,10 @@ TestSuite = {
               TextRun = 'love.' .. self.module.module .. '.' .. method
 
               self.test.co = coroutine.create(function()
-                local ok, chunk, err = pcall(love.test[love.test.module.module][method], love.test.test)
+                local ok, chunk, err = pcall(
+                  love.test[love.test.module.module][method],
+                  love.test.test
+                )
                 if ok == false then
                   love.test.test['passed'] = false
                   love.test.test['fatal'] = tostring(chunk) .. tostring(err)
@@ -140,7 +144,14 @@ TestSuite = {
   printResult = function(self)
     local finaltime = UtilTimeFormat(self.time)
 
-    local name, version, vendor, device = love.graphics.getRendererInfo()
+    -- in case we dont have love.graphics loaded, for future module specific disabling
+    local name = 'NONE'
+    local version = 'NONE'
+    local vendor = 'NONE'
+    local device = 'NONE'
+    if love.graphics then
+      name, version, vendor, device = love.graphics.getRendererInfo()
+    end
     
     local md = '<!-- PASSED ' .. tostring(self.totals[1]) ..
       ' || FAILED ' .. tostring(self.totals[2]) ..

testing/main.lua

@@ -41,6 +41,8 @@ love.load = function(args)
       resizable = true,
       centered = true
     })
+
+    -- set up some graphics to draw if enabled
     if love.graphics ~= nil then
       love.graphics.setDefaultFilter("nearest", "nearest")
       love.graphics.setLineStyle('rough')
@@ -54,6 +56,7 @@ love.load = function(args)
       TextCommand = 'Loading...'
       TextRun = ''
     end
+
   end
 
   -- mount for output later

testing/readme.md

@@ -25,15 +25,15 @@ This is the status of all module tests.
 See the **Todo** section for outstanding tasks if you want to contribute!
 | Module            | Done | Skip | Modules          | Done | Skip |
 | ----------------- | ---- | ---- | ---------------- | ---- | ---- |
-| 🟢 audio          |   28 |   0  | 🟢 mouse          |   18 |   0  |
+| 🟢 audio          |   31 |   0  | 🟢 mouse          |   18 |   0  |
 | 🟢 data           |   12 |   0  | 🟢 physics        |   26 |   0  |
 | 🟢 event          |    4 |   2  | 🟢 sensor         |    1 |   0  |
-| 🟢 filesystem     |   29 |   2  | 🟢 sound          |    4 |   0  |
-| 🟢 font           |    7 |   0  | 🟢 system         |    6 |   2  |
-| 🟢 graphics       |  104 |   1  | 🟢 thread         |    5 |   0  |
+| 🟢 filesystem     |   33 |   2  | 🟢 sound          |    4 |   0  |
+| 🟢 font           |    7 |   0  | 🟢 system         |    7 |   2  |
+| 🟢 graphics       |  105 |   1  | 🟢 thread         |    5 |   0  |
 | 🟢 image          |    5 |   0  | 🟢 timer          |    6 |   0  |
 | 🟢 joystick       |    6 |   0  | 🟢 touch          |    3 |   0  |
-| 🟢 keyboard       |    9 |   0  | 🟢 video          |    2 |   0  |
+| 🟢 keyboard       |   10 |   0  | 🟢 video          |    2 |   0  |
 | 🟢 love           |    6 |   0  | 🟢 window         |   34 |   2  |
 | 🟢 math           |   20 |   0  | 
 
@@ -102,7 +102,7 @@ love.test.filesystem.read = function(test)
 end
 ```
 
-Each test is run inside it's own coroutine - you can use `test:waitFrames(frames)` to pause the test for a small period if you need to check things that won't happen for a few seconds.
+Each test is run inside it's own coroutine - you can use `test:waitFrames(frames)` or `test:waitSeconds(seconds)` to pause the test for a small period if you need to check things that won't happen for a few frames/seconds.
 
 After each test method is ran, the assertions are totalled up, printed, and we move onto the next method! Once all methods in the suite are run a total pass/fail/skip is given for that module and we move onto the next module (if any)
 
@@ -133,7 +133,7 @@ The automated tests through Github work for the most part however there are a fe
 
 These exceptions are either skipped, or handled by using a 1px or 1/255rgba tolerance - when run locally on real hardware, these tests pass fine at the default 0 tolerance.  
 You can specify the test suite is being run on a runner by adding the `--isRunner` flag in your workflow file, i.e.:  
-`& 'c:\Program Files\LOVE\love.exe' PATH_TO_TESTING_FOLDER/main.lua --console --runAllTests --isRunner`
+`& 'c:\Program Files\LOVE\love.exe' PATH_TO_TESTING_FOLDER/main.lua --console --all --isRunner`
 | Test                       |    OS     |      Exception      | Reason |
 | -------------------------- | --------- | ------------------- | ------ |
 | love.graphics.setWireframe |   MacOS   |    1px tolerance    | Wireframes are offset by 1,1 when drawn |

testing/tests/audio.lua

@@ -469,7 +469,7 @@ love.test.audio.setPlaybackDevice = function(test)
   -- rn on macos all 3 return false
   -- whereas linux/windows return true for blank/current, which is expected
   -- as openalsoft treats blank as current
-  if love.system.getOS() == 'OS X' then
+  if test:isOS('OS X') then
     test:assertFalse(success1, 'check blank device fails')
     test:assertFalse(success2, 'check invalid device fails')
     test:assertFalse(success3, 'check existing device fails')

testing/tests/filesystem.lua

@@ -393,7 +393,7 @@ love.test.filesystem.mountCommonPath = function(test)
   local mount3 = love.filesystem.mountCommonPath('userhome', 'userhome', 'readwrite')
   local mount4 = love.filesystem.mountCommonPath('userappdata', 'userappdata', 'readwrite')
   -- userdesktop isnt valid on linux
-  if love.system.getOS() ~= 'Linux' then
+  if not test:isOS('Linux') then
     local mount5 = love.filesystem.mountCommonPath('userdesktop', 'userdesktop', 'readwrite')
     test:assertTrue(mount5, 'check mount userdesktop')
   end

testing/tests/graphics.lua

@@ -10,56 +10,53 @@
 
 -- GraphicsBuffer (love.graphics.newBuffer)
 love.test.graphics.Buffer = function(test)
+
+  -- setup vertex data and create some buffers
   local vertexformat = {
     {name="VertexPosition", format="floatvec2"},
     {name="VertexTexCoord", format="floatvec2"},
     {name="VertexColor", format="unorm8vec4"},
   }
-
   local vertexdata = {
     {0,  0,  0, 0, 1, 0, 1, 1},
     {10, 0,  1, 0, 0, 1, 1, 1},
     {10, 10, 1, 1, 0, 0, 1, 1},
     {0,  10, 0, 1, 1, 0, 0, 1},
   }
-
   local flatvertexdata = {}
   for i, vert in ipairs(vertexdata) do
     for j, v in ipairs(vert) do
       table.insert(flatvertexdata, v)
     end
   end
-
   local vertexbuffer1 = love.graphics.newBuffer(vertexformat, 4, {vertex=true, debugname='testvertexbuffer'})
   local vertexbuffer2 = love.graphics.newBuffer(vertexformat, vertexdata, {vertex=true})
-
   test:assertObject(vertexbuffer1)
   test:assertObject(vertexbuffer2)
 
+  -- check buffer properties
   test:assertEquals(4, vertexbuffer1:getElementCount(), 'check vertex count 1')
   test:assertEquals(4, vertexbuffer2:getElementCount(), 'check vertex count 2')
-
   -- vertex buffers have their elements tightly packed.
   test:assertEquals(20, vertexbuffer1:getElementStride(), 'check vertex array stride')
-
   test:assertEquals(20 * 4, vertexbuffer1:getSize(), 'check vertex buffer size')
-
   vertexbuffer1:setArrayData(vertexdata)
   vertexbuffer1:setArrayData(flatvertexdata)
-
   vertexbuffer1:clear(8, 8) -- partial clear (the first texcoord)
 
+  -- check buffer types
   test:assertTrue(vertexbuffer1:isBufferType('vertex'), 'check is vertex buffer')
   test:assertFalse(vertexbuffer1:isBufferType('index'), 'check is not index buffer')
   test:assertFalse(vertexbuffer1:isBufferType('texel'), 'check is not texel buffer')
   test:assertFalse(vertexbuffer1:isBufferType('shaderstorage'), 'check is not shader storage buffer')
 
+  -- check debug name
   test:assertEquals('testvertexbuffer', vertexbuffer1:getDebugName(), 'check buffer debug name')
 
+  -- check buffer format and format properties
   local format = vertexbuffer1:getFormat()
   test:assertEquals('table', type(format), 'check buffer format is table')
   test:assertEquals(#vertexformat, #format, 'check buffer format length')
-
   for i, v in ipairs(vertexformat) do
     test:assertEquals(v.name, format[i].name, string.format('check buffer format %d name', i))
     test:assertEquals(v.format, format[i].format, string.format('check buffer format %d format', i))
@@ -67,8 +64,10 @@ love.test.graphics.Buffer = function(test)
     test:assertNotNil(format[i].offset)
   end
 
+  -- check index buffer
   local indexbuffer = love.graphics.newBuffer('uint16', 128, {index=true})
   test:assertTrue(indexbuffer:isBufferType('index'), 'check is index buffer')
+
 end
 
 
@@ -80,22 +79,22 @@ love.test.graphics.ShaderStorageBuffer = function(test)
     return
   end
 
+  -- setup buffer
   local format = {
     { name="1", format="float" },
     { name="2", format="floatmat4x4" },
     { name="3", format="floatvec2" }
   }
-
   local buffer = love.graphics.newBuffer(format, 1, {shaderstorage = true})
-
   test:assertEquals(96, buffer:getElementStride(), 'check shader storage buffer element stride')
 
+  -- set new data
   local data = {}
   for i = 1, 19 do
     data[i] = 0
   end
-
   buffer:setArrayData(data)
+
 end
 
 
@@ -1170,7 +1169,7 @@ love.test.graphics.arc = function(test)
     love.graphics.setColor(1, 1, 1, 1)
   love.graphics.setCanvas()
   local imgdata3 = love.graphics.readbackTexture(canvas)
-  if GITHUB_RUNNER and love.system.getOS() == 'OS X' then
+  if GITHUB_RUNNER and test:isOS('OS X') then
     -- on macosx runners, the arcs are not drawn as accurately at low res
     -- there's a couple pixels different in the curve of the arc but as we
     -- are at such a low resolution I think that can be expected
@@ -1468,7 +1467,7 @@ love.test.graphics.captureScreenshot = function(test)
   TextCommand = prevtextcommand
   test:assertNotNil(cbdata)
 
-  if love.system.getOS() == "iOS" or love.system.getOS() == "Android" then
+  if test:isOS('iOS', 'Android') then
     -- Mobile operating systems don't let us control the window resolution,
     -- so we can't compare the reference image properly.
     test:assertTrue(true, 'skip test')
@@ -1586,7 +1585,7 @@ love.test.graphics.newSpriteBatch = function(test)
 end
 
 
--- love.graphics.newText
+-- love.graphics.newTextBatch
 -- @NOTE this is just basic nil checking, objs have their own test method
 love.test.graphics.newTextBatch = function(test)
   local font = love.graphics.newFont('resources/font.ttf')
@@ -1594,6 +1593,14 @@ love.test.graphics.newTextBatch = function(test)
 end
 
 
+-- love.graphics.newTexture
+-- @NOTE this is just basic nil checking, objs have their own test method
+love.test.graphics.newTexture = function(test)
+  local imgdata = love.image.newImageData('resources/love.png')
+  test:assertObject(love.graphics.newTexture(imgdata))
+end
+
+
 -- love.graphics.newVideo
 -- @NOTE this is just basic nil checking, objs have their own test method
 love.test.graphics.newVideo = function(test)
@@ -2167,7 +2174,7 @@ love.test.graphics.setLineStyle = function(test)
   love.graphics.setCanvas()
   local imgdata = love.graphics.readbackTexture(canvas)
   -- linux runner needs a 1/255 tolerance for the blend between a rough line + bg 
-  if GITHUB_RUNNER and love.system.getOS() == 'Linux' then
+  if GITHUB_RUNNER and test:isOS('Linux') then
     test.rgba_tolerance = 1
   end
   test:compareImg(imgdata)
@@ -2300,7 +2307,7 @@ love.test.graphics.setWireframe = function(test)
     love.graphics.setWireframe(false)
     local imgdata = love.graphics.readbackTexture(canvas)
     -- on macOS runners wireframes are drawn 1px off from the target
-    if GITHUB_RUNNER and love.system.getOS() == 'OS X' then
+    if GITHUB_RUNNER and test:isOS('OS X') then
       test.pixel_tolerance = 1
     end
     test:compareImg(imgdata)
@@ -2667,8 +2674,8 @@ love.test.graphics.getSupported = function(test)
     'clampzero', 'lighten', 'glsl3', 'instancing', 'fullnpot', 
     'pixelshaderhighp', 'shaderderivatives', 'indirectdraw', 'mipmaprange',
     'copyrendertargettobuffer', 'copytexturetobuffer', 'copybuffer',
-    'indexbuffer32bit', 'multirendertargetformats', 'clampone', 'blendminmax',
-    'glsl4'
+    'copybuffertotexture', 'indexbuffer32bit', 'multirendertargetformats', 
+    'clampone', 'blendminmax', 'glsl4'
   }
   local features = love.graphics.getSupported()
   for g=1,#gfs do

testing/todo.md

@@ -3,7 +3,6 @@ These are all the outstanding methods that require test coverage, along with a f
 
 ## General 
 - ability to test loading different combinations of modules if needed?
-- performance tests? need to discuss what + how, might be better as a seperate thing
 - check expected behaviour of mount + unmount with common path
   try uncommenting love.filesystem.unmountCommonPath and you'll see the issues
 - revisit love.audio.setPlaybackDevice when we update openal soft for MacOS
@@ -19,7 +18,6 @@ These are all the outstanding methods that require test coverage, along with a f
 - love.graphics.newComputeShader()
 - love.graphics.dispatchThreadgroups()
 - love.graphics.dispatchIndirect()
-- love.graphics.newTexture()
 - love.graphics.drawFromShader()
 - love.graphics.drawFromShaderIndirect()
 - love.graphics.drawIndirect()