diff --git a/src/action.js b/src/action.js
index 6c2c4085..553cac4d 100644
--- a/src/action.js
+++ b/src/action.js
@@ -15,8 +15,6 @@ import { signAddon as signAddonV4 } from "sign-addon";
 import * as github from "@actions/github";
 import * as core from "@actions/core";
 
-import CheckRun from "./checkrun.js";
-
 const KNOWN_LICENSES = new Set([
   "all-rights-reserved", "MPL-2.0", "GPL-2.0-or-later", "GPL-3.0-or-later", "LGPL-2.1-or-later",
   "LGPL-3.0-or-later", "MIT", "BSD-2-Clause"
@@ -52,27 +50,20 @@ export default class WebExtAction {
   }
 
   async cmd_lint() {
-    function linterToAnnotation(message) {
-      let level = message._type == "error" ? "failure" : message._type;
+    function linterToAnnotationProperty(data) {
       return {
-        path: message.file || "none",
-        start_line: message.line || 1,
-        end_line: message.line || 1,
-        start_column: message.column,
-        end_column: message.column,
-        annotation_level: level,
-        message: message.description,
-        title: message.message
+        title: data.message,
+        file: data.file,
+        startLine: data.line || 1,
+        startColumn: data.column
       };
     }
+
     function linterToString(message) {
       let prefix = message._type[0].toUpperCase() + message._type.substr(1);
       return `${prefix}: ${message.file}${":" + (message.line || "")} - ${message.message}`;
     }
 
-    let check = new CheckRun("web-ext lint", github.context, this.options.token);
-    await check.create();
-
     let results = await webExt.cmd.lint({
       sourceDir: this.options.sourceDir,
       artifactsDir: this.options.artifactsDir,
@@ -83,20 +74,24 @@ export default class WebExtAction {
       shouldExitProgram: false
     });
 
-    let nonfatal = results.notices.concat(results.warnings).map(linterToAnnotation);
-    let fatal = results.errors.map(linterToAnnotation);
-    let summary = results.summary;
-    let summaryLine = `${summary.errors} Errors, ${summary.warnings} Warnings, ${summary.notices} Notices`;
+    let annotations = results.errors.concat(results.warnings).concat(results.notices);
 
-    await check.complete(summaryLine, nonfatal, fatal);
+    if (annotations.length > 10) {
+      core[annotations[9]._type]("Only the first 9 linting messages are shown, please fix them first");
+      console.log(annotations.map(linterToString).join("\n") + "\n");
+      annotations.splice(9);
+    }
 
-    if (!check.ready) {
-      console.log(results.notices.concat(results.warnings).concat(results.errors).map(linterToString).join("\n") + "\n");
+    for (let message of annotations) {
+      core[message._type](message.description, linterToAnnotationProperty(message));
     }
-    console.log(summaryLine);
 
-    if (fatal.length) {
+    let summary = results.summary;
+    let summaryLine = `${summary.errors} Errors, ${summary.warnings} Warnings, ${summary.notices} Notices`;
+    if (results.errors.length) {
       throw new Error(summaryLine);
+    } else {
+      console.log(summaryLine);
     }
   }
 
diff --git a/src/checkrun.js b/src/checkrun.js
deleted file mode 100644
index b4260fba..00000000
--- a/src/checkrun.js
+++ /dev/null
@@ -1,75 +0,0 @@
-/* This Source Code Form is subject to the terms of the Mozilla Public
- * License, v. 2.0. If a copy of the MPL was not distributed with this
- * file, You can obtain one at http://mozilla.org/MPL/2.0/.
- * Portions Copyright (C) Philipp Kewisch, 2019 */
-
-import * as github from "@actions/github";
-
-export default class CheckRun {
-  constructor(name, context, token) {
-    this.id = null;
-    this.name = name;
-    this.context = context;
-    if (token) {
-      this.octokit = github.getOctokit(token);
-    }
-
-    this.ready = !!token;
-  }
-
-  async create() {
-    if (!this.ready) {
-      return;
-    }
-
-    let data = {
-      ...this.context.repo,
-      head_sha: this.context.sha,
-      name: "Test: " + this.name,
-      status: "in_progress",
-      started_at: new Date().toISOString(),
-    };
-
-    try {
-      let res = await this.octokit.rest.checks.create(data);
-      this.id = res.data.id;
-    } catch (e) {
-      // No permissions, likely running in a pull request
-      this.ready = false;
-    }
-  }
-
-  async complete(errorCount, warningCount, annotations) {
-    if (!this.ready) {
-      return;
-    }
-
-    let conclusion = "success";
-    if (errorCount > 0) {
-      conclusion = "failure";
-    } else if (warningCount > 0) {
-      conclusion = "neutral";
-    }
-
-    let data = {
-      ...this.context.repo,
-      head_sha: this.context.sha,
-      check_run_id: this.id,
-      status: "completed",
-      completed_at: new Date().toISOString(),
-      conclusion: conclusion,
-      output: {
-        title: this.name,
-        summary: `${errorCount} errors, ${warningCount} warnings`,
-        annotations: annotations,
-      },
-    };
-
-    try {
-      await this.octokit.rest.checks.update(data);
-    } catch (e) {
-      console.log(JSON.stringify(data, null, 2));
-      throw e;
-    }
-  }
-}