diff --git a/docs/new-rule.md b/docs/new-rule.md
index 191d04d21e..7b688433df 100644
--- a/docs/new-rule.md
+++ b/docs/new-rule.md
@@ -24,6 +24,6 @@ Use the [`astexplorer` site](https://astexplorer.net) with the `espree` parser a
 	*(The description should be the same as the heading of the documentation file).*
 - Run `$ npm test` to ensure the tests pass.
 - Run `$ npm run integration` to run the rules against real projects to ensure your rule does not fail on real-world code.
-- Run `$ npm run lint` to run the rules against this repository to ensure code in the repository are following your rule.
 - Open a pull request with a title in exactly the format `` Add `rule-name` rule ``, for example, `` Add `no-unused-properties` rule ``.
 - The pull request description should include the issue it fixes, for example, `Fixes #123`.
+- Run `$ npm run lint` to run the rules against codebase to ensure code in the repository are following your rule, _you can ignore this step until your PR is reviewed_.
diff --git a/test/lint/lint.js b/test/lint/lint.js
index 6ac13aa258..04e1ee34f1 100755
--- a/test/lint/lint.js
+++ b/test/lint/lint.js
@@ -3,16 +3,15 @@
 const {CLIEngine} = require('eslint');
 const unicorn = require('../..');
 
-const {configs: {recommended}} = unicorn;
-const {rules} = recommended;
+const {recommended} = unicorn.configs;
 const files = [process.argv[2] || '.'];
 const fix = process.argv.includes('--fix');
+const ruleIds = Object.keys(unicorn.rules);
+const unicornRules = new Map(Object.entries(unicorn.rules));
 
 const cli = new CLIEngine({
-	...recommended,
+	baseConfig: recommended,
 	rules: {
-		...rules,
-
 		// TODO: remove this override, when #391 is fixed
 		'unicorn/consistent-function-scoping': 'off'
 	},
@@ -22,14 +21,39 @@ const cli = new CLIEngine({
 
 cli.addPlugin('eslint-plugin-unicorn', unicorn);
 
+// Make sure rules are loaded from codebase
+const loadedRules = cli.getRules();
+if (!ruleIds.every(ruleId => unicornRules.get(ruleId) === loadedRules.get(`unicorn/${ruleId}`))) {
+	console.error('`eslint-plugin-unicorn` rules are not loaded from codebase.');
+	process.exit(1);
+}
+
 const report = cli.executeOnFiles(files);
 
-if (fix) {
+const {errorCount, warningCount, fixableErrorCount, fixableWarningCount} = report;
+
+const hasFixable = fixableErrorCount || fixableWarningCount;
+
+if (fix && hasFixable) {
 	CLIEngine.outputFixes(report);
 }
 
-const formatter = cli.getFormatter();
+if (errorCount || warningCount) {
+	const formatter = cli.getFormatter();
+	console.log(formatter(report.results));
+
+	console.log();
+	console.log(`You need to fix the failed test${errorCount + warningCount > 1 ? 's' : ''} above and run \`npm run lint <file>\` to check again.`);
 
-console.log(formatter(report.results));
+	if (hasFixable) {
+		console.log();
+		console.log('You may also want run `npm run lint <file> --fix` to fix fixable problems.');
+	}
+
+	console.log();
+	console.log('* If you\'re making a new rule, you can fix this later. *');
+} else {
+	console.log('All tests have passed.');
+}
 
-process.exit(report.errorCount);
+process.exit(errorCount);