Skip to content

Commit d174050

Browse files
sumnekosumneko
authored
Merge pull request #2638 from emmericp/multi-threaded-check
Add multi-process support to --check.
2 parents 9123d4d + 38d8332 commit d174050

File tree

7 files changed

+256
-111
lines changed

7 files changed

+256
-111
lines changed

‎locale/en-us/script.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ CLI_CHECK_PROGRESS =
650650
'Found {} problems in {} files'
651651
CLI_CHECK_RESULTS =
652652
'Diagnosis complete, {} problems found, see {}'
653+
CLI_CHECK_MULTIPLE_WORKERS =
654+
'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
653655
CLI_DOC_INITING =
654656
'Loading documents ...'
655657
CLI_DOC_DONE =

‎locale/pt-br/script.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ CLI_CHECK_PROGRESS = -- TODO: need translate!
650650
'Found {} problems in {} files'
651651
CLI_CHECK_RESULTS =
652652
'Diagnóstico completo, {} problemas encontrados, veja {}'
653+
CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
654+
'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
653655
CLI_DOC_INITING = -- TODO: need translate!
654656
'Loading documents ...'
655657
CLI_DOC_DONE = -- TODO: need translate!

‎locale/zh-cn/script.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ CLI_CHECK_PROGRESS = -- TODO: need translate!
650650
'Found {} problems in {} files'
651651
CLI_CHECK_RESULTS =
652652
'诊断完成，共有 {} 个问题，请查看 {}'
653+
CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
654+
'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
653655
CLI_DOC_INITING =
654656
'加载文档 ...'
655657
CLI_DOC_DONE =

‎locale/zh-tw/script.lua

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -650,6 +650,8 @@ CLI_CHECK_PROGRESS = -- TODO: need translate!
650650
'Found {} problems in {} files'
651651
CLI_CHECK_RESULTS =
652652
'診斷完成，共有 {} 個問題，請查看 {}'
653+
CLI_CHECK_MULTIPLE_WORKERS = -- TODO: need translate!
654+
'Starting {} worker tasks, progress output will be disabled. This may take a few minutes.'
653655
CLI_DOC_INITING = -- TODO: need translate!
654656
'Loading documents ...'
655657
CLI_DOC_DONE = -- TODO: need translate!

‎script/cli/check.lua

Lines changed: 77 additions & 111 deletions
Original file line numberDiff line numberDiff line change
@@ -1,130 +1,96 @@
1-
local lclient = require 'lclient'()
2-
local furi = require 'file-uri'
3-
local ws = require 'workspace'
4-
local files = require 'files'
5-
local diag = require 'provider.diagnostic'
6-
local util = require 'utility'
7-
local jsonb = require 'json-beautify'
8-
local lang = require 'language'
9-
local define = require 'proto.define'
10-
local config = require 'config.config'
11-
local fs = require 'bee.filesystem'
12-
local provider = require 'provider'
1+
local lang = require 'language'
2+
local platform = require 'bee.platform'
3+
local subprocess = require 'bee.subprocess'
4+
local json = require 'json'
5+
local jsonb = require 'json-beautify'
6+
local util = require 'utility'
137

14-
require 'plugin'
15-
require 'vm'
168

17-
lang(LOCALE)
9+
local numThreads = tonumber(NUM_THREADS or 1)
1810

19-
if type(CHECK) ~= 'string' then
20-
print(lang.script('CLI_CHECK_ERROR_TYPE', type(CHECK)))
21-
return
11+
local exe = arg[-1]
12+
-- TODO: is this necessary? got it from the shell.lua helper in bee.lua tests
13+
if platform.os == 'windows' and not exe:match('%.[eE][xX][eE]$') then
14+
exe = exe..'.exe'
2215
end
2316

24-
local rootPath = fs.absolute(fs.path(CHECK)):string()
25-
local rootUri = furi.encode(rootPath)
26-
if not rootUri then
27-
print(lang.script('CLI_CHECK_ERROR_URI', rootPath))
28-
return
17+
local function logFileForThread(threadId)
18+
return LOGPATH .. '/check-partial-' .. threadId .. '.json'
2919
end
30-
rootUri = rootUri:gsub("/$", "")
3120

32-
if CHECKLEVEL then
33-
if not define.DiagnosticSeverity[CHECKLEVEL] then
34-
print(lang.script('CLI_CHECK_ERROR_LEVEL', 'Error, Warning, Information, Hint'))
35-
return
21+
local function buildArgs(threadId)
22+
local args = {exe}
23+
local skipNext = false
24+
for i = 1, #arg do
25+
local arg = arg[i]
26+
-- --check needs to be transformed into --check_worker
27+
if arg:lower():match('^%-%-check$') or arg:lower():match('^%-%-check=') then
28+
args[#args + 1] = arg:gsub('%-%-%w*', '--check_worker')
29+
-- --check_out_path needs to be removed if we have more than one thread
30+
elseif arg:lower():match('%-%-check_out_path') and numThreads > 1 then
31+
if not arg:match('%-%-%w*=') then
32+
skipNext = true
33+
end
34+
else
35+
if skipNext then
36+
skipNext = false
37+
else
38+
args[#args + 1] = arg
39+
end
40+
end
41+
end
42+
args[#args + 1] = '--thread_id'
43+
args[#args + 1] = tostring(threadId)
44+
if numThreads > 1 then
45+
args[#args + 1] = '--quiet'
46+
args[#args + 1] = '--check_out_path'
47+
args[#args + 1] = logFileForThread(threadId)
3648
end
49+
return args
3750
end
38-
local checkLevel = define.DiagnosticSeverity[CHECKLEVEL] or define.DiagnosticSeverity.Warning
39-
40-
util.enableCloseFunction()
4151

42-
local lastClock = os.clock()
43-
local results = {}
44-
45-
local function errorhandler(err)
46-
print(err)
47-
print(debug.traceback())
52+
if numThreads > 1 then
53+
print(lang.script('CLI_CHECK_MULTIPLE_WORKERS', numThreads))
4854
end
4955

50-
---@async
51-
xpcall(lclient.start, errorhandler, lclient, function (client)
52-
client:registerFakers()
53-
54-
client:initialize {
55-
rootUri = rootUri,
56-
}
57-
58-
client:register('textDocument/publishDiagnostics', function (params)
59-
results[params.uri] = params.diagnostics
60-
end)
61-
62-
io.write(lang.script('CLI_CHECK_INITING'))
63-
64-
provider.updateConfig(rootUri)
65-
66-
ws.awaitReady(rootUri)
67-
68-
local disables = util.arrayToHash(config.get(rootUri, 'Lua.diagnostics.disable'))
69-
for name, serverity in pairs(define.DiagnosticDefaultSeverity) do
70-
serverity = config.get(rootUri, 'Lua.diagnostics.severity')[name] or 'Warning'
71-
if serverity:sub(-1) == '!' then
72-
serverity = serverity:sub(1, -2)
73-
end
74-
if define.DiagnosticSeverity[serverity] > checkLevel then
75-
disables[name] = true
76-
end
56+
local procs = {}
57+
for i = 1, numThreads do
58+
local process, err = subprocess.spawn({buildArgs(i)})
59+
if err then
60+
print(err)
7761
end
78-
config.set(rootUri, 'Lua.diagnostics.disable', util.getTableKeys(disables, true))
79-
80-
local uris = files.getChildFiles(rootUri)
81-
local max = #uris
82-
for i, uri in ipairs(uris) do
83-
files.open(uri)
84-
diag.doDiagnostic(uri, true)
85-
-- Print regularly but always print the last entry to ensure that logs written to files don't look incomplete.
86-
if os.clock() - lastClock > 0.2 or i == #uris then
87-
lastClock = os.clock()
88-
client:update()
89-
local output = '\x0D'
90-
.. ('>'):rep(math.ceil(i / max * 20))
91-
.. ('='):rep(20 - math.ceil(i / max * 20))
92-
.. ' '
93-
.. ('0'):rep(#tostring(max) - #tostring(i))
94-
.. tostring(i) .. '/' .. tostring(max)
95-
io.write(output)
96-
local filesWithErrors = 0
97-
local errors = 0
98-
for _, diags in pairs(results) do
99-
filesWithErrors = filesWithErrors + 1
100-
errors = errors + #diags
101-
end
102-
if errors > 0 then
103-
local errorDetails = ' [' .. lang.script('CLI_CHECK_PROGRESS', errors, filesWithErrors) .. ']'
104-
io.write(errorDetails)
105-
end
106-
io.flush()
107-
end
62+
if process then
63+
procs[#procs + 1] = process
10864
end
109-
io.write('\x0D')
110-
end)
65+
end
11166

112-
local count = 0
113-
for uri, result in pairs(results) do
114-
count = count + #result
115-
if #result == 0 then
116-
results[uri] = nil
117-
end
67+
for _, process in ipairs(procs) do
68+
process:wait()
11869
end
11970

120-
if count == 0 then
121-
print(lang.script('CLI_CHECK_SUCCESS'))
122-
else
123-
local outpath = CHECK_OUT_PATH
124-
if outpath == nil then
125-
outpath = LOGPATH .. '/check.json'
126-
end
127-
util.saveFile(outpath, jsonb.beautify(results))
71+
local outpath = CHECK_OUT_PATH
72+
if outpath == nil then
73+
outpath = LOGPATH .. '/check.json'
74+
end
12875

129-
print(lang.script('CLI_CHECK_RESULTS', count, outpath))
76+
if numThreads > 1 then
77+
local mergedResults = {}
78+
local count = 0
79+
for i = 1, numThreads do
80+
local result = json.decode(util.loadFile(logFileForThread(i)) or '[]')
81+
for k, v in pairs(result) do
82+
local entries = mergedResults[k] or {}
83+
mergedResults[k] = entries
84+
for _, entry in ipairs(v) do
85+
entries[#entries + 1] = entry
86+
count = count + 1
87+
end
88+
end
89+
end
90+
util.saveFile(outpath, jsonb.beautify(mergedResults))
91+
if count == 0 then
92+
print(lang.script('CLI_CHECK_SUCCESS'))
93+
else
94+
print(lang.script('CLI_CHECK_RESULTS', count, outpath))
95+
end
13096
end

0 commit comments

Comments
 (0)