diff --git a/ts/util/lint/linter.ts b/ts/util/lint/linter.ts index 803e0ad56a..3c93dc2b1e 100644 --- a/ts/util/lint/linter.ts +++ b/ts/util/lint/linter.ts @@ -1,26 +1,27 @@ // tslint:disable no-console - -import { readFileSync } from 'fs'; +import * as fs from 'fs'; import { join, relative } from 'path'; import normalizePath from 'normalize-path'; - -import { sync as fgSync } from 'fast-glob'; -import { forEach, some, values } from 'lodash'; +import pMap from 'p-map'; +import FastGlob from 'fast-glob'; import { ExceptionType, REASONS, RuleType } from './types'; import { ENCODING, loadJSON, sortExceptions } from './util'; const ALL_REASONS = REASONS.join('|'); -const now = new Date(); -function getExceptionKey(exception: any) { +function getExceptionKey( + exception: Pick +): string { return `${exception.rule}-${exception.path}-${exception.lineNumber}`; } -function createLookup(list: Array) { +function createLookup( + list: ReadonlyArray +): { [key: string]: ExceptionType } { const lookup = Object.create(null); - forEach(list, exception => { + list.forEach((exception: ExceptionType) => { const key = getExceptionKey(exception); if (lookup[key]) { @@ -39,16 +40,7 @@ const basePath = join(__dirname, '../../..'); const searchPattern = normalizePath(join(basePath, '**/*.{js,ts,tsx}')); -const rules: Array = loadJSON(rulesPath); -const exceptions: Array = loadJSON(exceptionsPath); -const exceptionsLookup = createLookup(exceptions); -let scannedCount = 0; - -const allSourceFiles = fgSync(searchPattern, { onlyFiles: true }); - -const results: Array = []; - -const excludedFiles = [ +const excludedFilesRegexps = [ // Non-distributed files '\\.d\\.ts$', @@ -295,10 +287,10 @@ const excludedFiles = [ '^node_modules/webpack-hot-middleware/.+', '^node_modules/webpack-merge/.+', '^node_modules/webpack/.+', -]; +].map(str => new RegExp(str)); function setupRules(allRules: Array) { - forEach(allRules, (rule, index) => { + allRules.forEach((rule: RuleType, index: number) => { if (!rule.name) { throw new Error(`Rule at index ${index} is missing a name`); } @@ -311,93 +303,108 @@ function setupRules(allRules: Array) { }); } -setupRules(rules); +async function main(): Promise { + const now = new Date(); -forEach(allSourceFiles, file => { - const relativePath = relative(basePath, file).replace(/\\/g, '/'); - if ( - some(excludedFiles, excluded => { - const regex = new RegExp(excluded); + const rules: Array = loadJSON(rulesPath); + setupRules(rules); - return regex.test(relativePath); - }) - ) { - return; + const exceptions: Array = loadJSON(exceptionsPath); + const exceptionsLookup = createLookup(exceptions); + + const results: Array = []; + let scannedCount = 0; + + await pMap( + await FastGlob(searchPattern, { onlyFiles: true }), + async (file: string) => { + const relativePath = relative(basePath, file).replace(/\\/g, '/'); + + const isFileExcluded = excludedFilesRegexps.some(excludedRegexp => + excludedRegexp.test(relativePath) + ); + if (isFileExcluded) { + return; + } + + scannedCount += 1; + + const lines = (await fs.promises.readFile(file, ENCODING)).split(/\r?\n/); + + rules.forEach((rule: RuleType) => { + const excludedModules = rule.excludedModules || []; + if (excludedModules.some(module => relativePath.startsWith(module))) { + return; + } + + lines.forEach((line: string, lineIndex: number) => { + if (!rule.regex.test(line)) { + return; + } + // recreate this rule since it has g flag, and carries local state + if (rule.expression) { + rule.regex = new RegExp(rule.expression, 'g'); + } + + const lineNumber = lineIndex + 1; + + const exceptionKey = getExceptionKey({ + rule: rule.name, + path: relativePath, + lineNumber, + }); + + const exception = exceptionsLookup[exceptionKey]; + if (exception && (!exception.line || exception.line === line)) { + // tslint:disable-next-line no-dynamic-delete + delete exceptionsLookup[exceptionKey]; + + return; + } + + results.push({ + rule: rule.name, + path: relativePath, + line: line.length < 300 ? line : undefined, + lineNumber, + reasonCategory: ALL_REASONS, + updated: now.toJSON(), + reasonDetail: '', + }); + }); + }); + }, + // Without this, we may run into "too many open files" errors. + { concurrency: 100 } + ); + + const unusedExceptions = Object.values(exceptionsLookup); + + console.log( + `${scannedCount} files scanned.`, + `${results.length} questionable lines,`, + `${unusedExceptions.length} unused exceptions,`, + `${exceptions.length} total exceptions.` + ); + + if (results.length === 0 && unusedExceptions.length === 0) { + process.exit(); } - scannedCount += 1; - - const fileContents = readFileSync(file, ENCODING); - const lines = fileContents.split('\n'); - - forEach(rules, (rule: RuleType) => { - const excludedModules = rule.excludedModules || []; - if (some(excludedModules, module => relativePath.startsWith(module))) { - return; - } - - forEach(lines, (rawLine, lineIndex) => { - const line = rawLine.replace(/\r/g, ''); - - if (!rule.regex.test(line)) { - return; - } - // recreate this rule since it has g flag, and carries local state - if (rule.expression) { - rule.regex = new RegExp(rule.expression, 'g'); - } - - const path = relativePath; - const lineNumber = lineIndex + 1; - - const exceptionKey = getExceptionKey({ - rule: rule.name, - path: relativePath, - lineNumber, - }); - - const exception = exceptionsLookup[exceptionKey]; - if (exception && (!exception.line || exception.line === line)) { - // tslint:disable-next-line no-dynamic-delete - delete exceptionsLookup[exceptionKey]; - - return; - } - - results.push({ - rule: rule.name, - path, - line: line.length < 300 ? line : undefined, - lineNumber, - reasonCategory: ALL_REASONS, - updated: now.toJSON(), - reasonDetail: '', - }); - }); - }); -}); - -const unusedExceptions = values(exceptionsLookup); - -console.log( - `${scannedCount} files scanned.`, - `${results.length} questionable lines,`, - `${unusedExceptions.length} unused exceptions,`, - `${exceptions.length} total exceptions.` -); - -if (results.length === 0 && unusedExceptions.length === 0) { - process.exit(); -} - -console.log(); -console.log('Questionable lines:'); -console.log(JSON.stringify(sortExceptions(results), null, ' ')); - -if (unusedExceptions.length) { console.log(); - console.log('Unused exceptions!'); - console.log(JSON.stringify(sortExceptions(unusedExceptions), null, ' ')); + console.log('Questionable lines:'); + console.log(JSON.stringify(sortExceptions(results), null, ' ')); + + if (unusedExceptions.length) { + console.log(); + console.log('Unused exceptions!'); + console.log(JSON.stringify(sortExceptions(unusedExceptions), null, ' ')); + } + + process.exit(1); } -process.exit(1); +main().catch(err => { + console.error(err); + process.exit(1); +});