Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: support SARIF output formatter #120

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/busted.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ jobs:
luarocks install busted
luarocks install lanes # required for parallel execution
luarocks install luautf8 # required for decoder unit test
luarocks install dkjson # required for decoder unit test
luarocks install luasocket # required for profiler unit test
luarocks install luacov-coveralls

Expand Down
2 changes: 1 addition & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -115,7 +115,7 @@ Use the Luacheck issue tracker on GitHub to submit bugs, suggestions and questio

## Building and testing

After the Luacheck repo is cloned and changes are made, run `luarocks make` (using `sudo` if necessary) from its root directory to install dev version of Luacheck. To run Luacheck using sources in current directory without installing it, run `lua -e 'package.path="./src/?.lua;./src/?/init.lua;"..package.path' bin/luacheck.lua ...`. To test Luacheck, ensure that you have [busted](http://olivinelabs.com/busted/), [luautf8](https://github.com/starwing/luautf8), and [luasocket](https://github.com/lunarmodules/luasocket) installed and run `busted`.
After the Luacheck repo is cloned and changes are made, run `luarocks make` (using `sudo` if necessary) from its root directory to install dev version of Luacheck. To run Luacheck using sources in current directory without installing it, run `lua -e 'package.path="./src/?.lua;./src/?/init.lua;"..package.path' bin/luacheck.lua ...`. To test Luacheck, ensure that you have [busted](http://olivinelabs.com/busted/), [luautf8](https://github.com/starwing/luautf8), [dkjson](https://luarocks.org/modules/dhkolf/dkjson) and [luasocket](https://github.com/lunarmodules/luasocket) installed and run `busted`.

## Docker

Expand Down
1 change: 1 addition & 0 deletions docsrc/cli.rst
Original file line number Diff line number Diff line change
Expand Up @@ -142,6 +142,7 @@ Option Meaning

* ``TAP`` - Test Anything Protocol formatter;
* ``JUnit`` - JUnit XML formatter;
* ``Sarif`` - Static Analysis Results Interchange Format formatter;
* ``visual_studio`` - MSBuild/Visual Studio aware formatter;
* ``plain`` - simple warning-per-line formatter;
* ``default`` - standard formatter.
Expand Down
3 changes: 2 additions & 1 deletion luacheck-dev-1.rockspec
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,8 @@ description = {
dependencies = {
"lua >= 5.1",
"argparse >= 0.6.0",
"luafilesystem >= 1.6.3"
"luafilesystem >= 1.6.3",
"dkjson >= 2.1.0"
}

test_dependencies = {
Expand Down
10 changes: 10 additions & 0 deletions spec/cli_spec.lua
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
local json = require "dkjson"
local utils = require "luacheck.utils"
local multithreading = require "luacheck.multithreading"
local helper = require "spec.helper"
Expand Down Expand Up @@ -979,6 +980,15 @@ not ok 8 spec/samples/python_code.lua:1:6: (E011) expected '=' near '__future__'
]], get_output "bad_file spec/samples/good_code.lua spec/samples/bad_code.lua spec/samples/python_code.lua --std=lua52 --formatter JUnit --no-config")
end)

it("has built-in Sarif formatter", function()
-- luacheck: ignore 631
local expect = [[
{"runs":[{"tool":{"driver":{"version":"1.0.0","name":"luacheck","informationUri":"https://github.com/lunarmodules/luacheck"}},"results":[{"locations":[{"physicalLocation":{"artifactLocation":{"uri":"bad_file"}}}],"ruleId":"I/O error","level":"error","message":{"text":"luacheck : fatal error F1: couldn't check bad_file: couldn't read: No such file or directory"}},{"locations":[{"physicalLocation":{"region":{"startLine":3,"startColumn":16},"artifactLocation":{"uri":"spec/samples/bad_code.lua"}}}],"ruleId":"W211","level":"warning","message":{"text":"spec/samples/bad_code.lua:3:16: unused function 'helper'"}},{"locations":[{"physicalLocation":{"region":{"startLine":3,"startColumn":23},"artifactLocation":{"uri":"spec/samples/bad_code.lua"}}}],"ruleId":"W212","level":"warning","message":{"text":"spec/samples/bad_code.lua:3:23: unused variable length argument"}},{"locations":[{"physicalLocation":{"region":{"startLine":7,"startColumn":10},"artifactLocation":{"uri":"spec/samples/bad_code.lua"}}}],"ruleId":"W111","level":"warning","message":{"text":"spec/samples/bad_code.lua:7:10: setting non-standard global variable 'embrace'"}},{"locations":[{"physicalLocation":{"region":{"startLine":8,"startColumn":10},"artifactLocation":{"uri":"spec/samples/bad_code.lua"}}}],"ruleId":"W412","level":"warning","message":{"text":"spec/samples/bad_code.lua:8:10: variable 'opt' was previously defined as an argument on line 7"}},{"locations":[{"physicalLocation":{"region":{"startLine":9,"startColumn":11},"artifactLocation":{"uri":"spec/samples/bad_code.lua"}}}],"ruleId":"W113","level":"warning","message":{"text":"spec/samples/bad_code.lua:9:11: accessing undefined variable 'hepler'"}},{"locations":[{"physicalLocation":{"region":{"startLine":1,"startColumn":6},"artifactLocation":{"uri":"spec/samples/python_code.lua"}}}],"ruleId":"E011","level":"error","message":{"text":"spec/samples/python_code.lua:1:6: expected '=' near '__future__'"}}]}],"version":"2.1.0","$schema":"https://json.schemastore.org/sarif-2.1.0.json"}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One nuance about Lua that makes it tricky to test reliably is that table key order is not deterministic. Obviously this isn't a Lua table it is a serialized JSON object, but what is to prevent a similar problem here? Both this expectation and the actual value are deserialized into Lua and then compared. How is that going to work out with the Lua table not having deterministic key ordering?

Copy link
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I tried to compare the json strings directly. And although the content is the same, the key order of the json is different:
image

That's why I'm deserializing JSON to Lua Table for comparison.

The internal implementation of assert.same ignores the order of the lua table keys, ref: https://github.com/lunarmodules/luassert/blob/8b5fd81d942532877091b68f1f3bd0f4e78fba83/src/util.lua#L12

]]
local actual = get_output "bad_file spec/samples/good_code.lua spec/samples/bad_code.lua spec/samples/python_code.lua --std=lua52 --formatter Sarif --no-config"
assert.same(json.decode(expect), json.decode(actual))
yeshan333 marked this conversation as resolved.
Show resolved Hide resolved
end)

it("has built-in Visual Studio aware formatter", function()
assert.equal([[
luacheck : fatal error F1: couldn't check bad_file: couldn't read: No such file or directory
Expand Down
74 changes: 74 additions & 0 deletions src/luacheck/format.lua
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
local json = require "dkjson"
local stages = require "luacheck.stages"
local utils = require "luacheck.utils"

Expand Down Expand Up @@ -317,6 +318,79 @@ function format.builtin_formatters.plain(report, file_names, opts)
return table.concat(buf, "\n")
end

-- https://docs.github.com/en/code-security/code-scanning/integrating-with-code-scanning/sarif-support-for-code-scanning
function format.builtin_formatters.Sarif(report, file_names, opts)
opts.color = false
local get_level = function(event)
return event.code:sub(1, 1) == "0" and "error" or "warning"
end

-- SARIF support for GitHub code scanning
local sarif_output = {
["$schema"] = "https://json.schemastore.org/sarif-2.1.0.json",
version = "2.1.0",
runs = {
{
tool = {
driver = {
informationUri = "https://github.com/lunarmodules/luacheck",
name = "luacheck",
version = "1.0.0"
}
},
results = {}
}
}
}

for file_i, file_report in ipairs(report) do
if file_report.fatal then
table.insert(sarif_output.runs[1].results, {
level = "error",
ruleId = fatal_type(file_report),
message = {
text = ("luacheck : fatal error %s: couldn't check %s: %s"):format(
fatal_error_codes[file_report.fatal], file_names[file_i], file_report.msg)
},
locations = {
{
physicalLocation = {
artifactLocation = {
uri = file_names[file_i]
}
}
}
}
})
else
for _, event in ipairs(file_report) do
table.insert(sarif_output.runs[1].results, {
level = get_level(event),
ruleId = event_code(event),
message = {
text = format_event(file_names[file_i], event, opts)
},
locations = {
{
physicalLocation = {
artifactLocation = {
uri = file_names[file_i]
},
region = {
startLine = event.line,
startColumn = event.column
}
}
}
}
})
end
end
end

return json.encode(sarif_output)
end

--- Formats a report.
-- Recognized options:
-- `options.formatter`: name of used formatter. Default: "default".
Expand Down
1 change: 1 addition & 0 deletions src/luacheck/main.lua
Original file line number Diff line number Diff line change
Expand Up @@ -244,6 +244,7 @@ Links:
parser:option("--formatter" , "Use custom formatter. <formatter> must be a module name or one of:\n" ..
" TAP - Test Anything Protocol formatter;\n" ..
" JUnit - JUnit XML formatter;\n" ..
" Sarif - Static Analysis Results Interchange Format formatter;\n" ..
" visual_studio - MSBuild/Visual Studio aware formatter;\n" ..
" plain - simple warning-per-line formatter;\n" ..
" default - standard formatter."),
Expand Down