Add Markdown Grammar (w/ Highlighting) (#42)

* Add Markdown Grammar * Add test cases
CodeEditApp · Jun 3, 2023 · d0a99dd · d0a99dd
1 parent 9e7b7c4
commit d0a99dd
Show file tree

Hide file tree

Showing 15 changed files with 238 additions and 0 deletions.
diff --git a/CodeLanguages-Container/CodeLanguages-Container.xcodeproj/project.pbxproj b/CodeLanguages-Container/CodeLanguages-Container.xcodeproj/project.pbxproj
@@ -39,6 +39,7 @@
 		6CEC70FE29C3A85000B61C7A /* TreeSitterRegex in Frameworks */ = {isa = PBXBuildFile; productRef = 6CEC70FD29C3A85000B61C7A /* TreeSitterRegex */; };
 		7DB18E9729FDC51C00F8EC00 /* TreeSitterScala in Frameworks */ = {isa = PBXBuildFile; productRef = 7DB18E9629FDC51C00F8EC00 /* TreeSitterScala */; };
 		9D6DA3B8298F1A4600E69066 /* TreeSitterOCaml in Frameworks */ = {isa = PBXBuildFile; productRef = 9D6DA3B7298F1A4600E69066 /* TreeSitterOCaml */; };
+		9DFDC3662A02D9BE0023B3BC /* TreeSitterMarkdown in Frameworks */ = {isa = PBXBuildFile; productRef = 9DFDC3652A02D9BE0023B3BC /* TreeSitterMarkdown */; };
 /* End PBXBuildFile section */
 
 /* Begin PBXFileReference section */
@@ -52,6 +53,7 @@
 			isa = PBXFrameworksBuildPhase;
 			buildActionMask = 2147483647;
 			files = (
+				9DFDC3662A02D9BE0023B3BC /* TreeSitterMarkdown in Frameworks */,
 				28B3F051290C36B1000CD04D /* TreeSitterPHP in Frameworks */,
 				28B3F042290C365C000CD04D /* TreeSitterHaskell in Frameworks */,
 				285BF67329AAA45B00641989 /* TreeSitterLua in Frameworks */,
@@ -190,6 +192,7 @@
 				6CA62EA829F9D36700785B11 /* TreeSitterTS */,
 				6CA62EAA29F9D36700785B11 /* TreeSitterTSX */,
 				7DB18E9629FDC51C00F8EC00 /* TreeSitterScala */,
+				9DFDC3652A02D9BE0023B3BC /* TreeSitterMarkdown */,
 			);
 			productName = "CodeLanguages-Container";
 			productReference = 28B3F00C290C207D000CD04D /* CodeLanguages_Container.framework */;
@@ -250,6 +253,7 @@
 				28AAB6AC29CA57D40087654B /* XCRemoteSwiftPackageReference "tree-sitter-dart" */,
 				6CA62EA729F9D36700785B11 /* XCRemoteSwiftPackageReference "tree-sitter-typescript" */,
 				7DB18E9529FDC51C00F8EC00 /* XCRemoteSwiftPackageReference "tree-sitter-scala" */,
+				9DFDC3642A02D9BE0023B3BC /* XCRemoteSwiftPackageReference "tree-sitter-markdown" */,
 			);
 			productRefGroup = 28B3F00D290C207D000CD04D /* Products */;
 			projectDirPath = "";
@@ -739,6 +743,14 @@
 				kind = branch;
 			};
 		};
+		9DFDC3642A02D9BE0023B3BC /* XCRemoteSwiftPackageReference "tree-sitter-markdown" */ = {
+			isa = XCRemoteSwiftPackageReference;
+			repositoryURL = "https://github.com/MDeiml/tree-sitter-markdown";
+			requirement = {
+				branch = split_parser;
+				kind = branch;
+			};
+		};
 /* End XCRemoteSwiftPackageReference section */
 
 /* Begin XCSwiftPackageProductDependency section */
@@ -897,6 +909,11 @@
 			package = 9D6DA3B6298F1A4500E69066 /* XCRemoteSwiftPackageReference "tree-sitter-ocaml" */;
 			productName = TreeSitterOCaml;
 		};
+		9DFDC3652A02D9BE0023B3BC /* TreeSitterMarkdown */ = {
+			isa = XCSwiftPackageProductDependency;
+			package = 9DFDC3642A02D9BE0023B3BC /* XCRemoteSwiftPackageReference "tree-sitter-markdown" */;
+			productName = TreeSitterMarkdown;
+		};
 /* End XCSwiftPackageProductDependency section */
 	};
 	rootObject = 28B3F003290C207D000CD04D /* Project object */;

diff --git a/...deLanguages-Container.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved b/...deLanguages-Container.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved
@@ -162,6 +162,15 @@
         "revision" : "47d56065016862b85ad327d6976b1e664e1c034b"
       }
     },
+    {
+      "identity" : "tree-sitter-markdown",
+      "kind" : "remoteSourceControl",
+      "location" : "https://github.com/MDeiml/tree-sitter-markdown",
+      "state" : {
+        "branch" : "split_parser",
+        "revision" : "fa6bfd51727e4bef99f7eec5f43947f73d64ea7d"
+      }
+    },
     {
       "identity" : "tree-sitter-objc",
       "kind" : "remoteSourceControl",

diff --git a/CodeLanguages-Container/CodeLanguages-Container/CodeLanguages_Container.h b/CodeLanguages-Container/CodeLanguages-Container/CodeLanguages_Container.h
@@ -39,6 +39,8 @@ extern TSLanguage *tree_sitter_javascript();
 extern TSLanguage *tree_sitter_json();
 extern TSLanguage *tree_sitter_kotlin();
 extern TSLanguage *tree_sitter_lua();
+extern TSLanguage *tree_sitter_markdown();
+extern TSLanguage *tree_sitter_markdown_inline();
 extern TSLanguage *tree_sitter_objc();
 extern TSLanguage *tree_sitter_ocaml();
 extern TSLanguage *tree_sitter_ocaml_interface();

diff --git a/CodeLanguagesContainer.xcframework.zip b/CodeLanguagesContainer.xcframework.zip
diff --git a/Sources/CodeEditLanguages/CodeLanguage+Definitions.swift b/Sources/CodeEditLanguages/CodeLanguage+Definitions.swift
@@ -29,6 +29,8 @@ public extension CodeLanguage {
         .jsx,
         .kotlin,
         .lua,
+        .markdown,
+        .markdownInline,
         .objc,
         .ocaml,
         .ocamlInterface,
@@ -179,6 +181,22 @@ public extension CodeLanguage {
         highlights: ["injections"]
     )
 
+    /// A language structure for `Markdown`
+    static let markdown: CodeLanguage = .init(
+        id: .markdown,
+        tsName: "markdown",
+        extensions: ["md", "mkd", "mkdn", "mdwn", "mdown", "markdown"],
+        highlights: ["injections"]
+    )
+
+    /// A language structure for `Markdown Inline`
+    static let markdownInline: CodeLanguage = .init(
+        id: .markdownInline,
+        tsName: "markdown-inline",
+        extensions: [],
+        highlights: ["injections"]
+    )
+
     /// A language structure for `Objective C`
     static let objc: CodeLanguage = .init(
         id: .objc,

diff --git a/Sources/CodeEditLanguages/CodeLanguage.swift b/Sources/CodeEditLanguages/CodeLanguage.swift
@@ -102,6 +102,10 @@ public struct CodeLanguage {
             return tree_sitter_kotlin()
         case .lua:
             return tree_sitter_lua()
+        case .markdown:
+            return tree_sitter_markdown()
+        case .markdownInline:
+            return tree_sitter_markdown_inline()
         case .objc:
             return tree_sitter_objc()
         case .ocaml:

diff --git a/Sources/CodeEditLanguages/Documentation.docc/CodeLanguage.md b/Sources/CodeEditLanguages/Documentation.docc/CodeLanguage.md
@@ -37,6 +37,7 @@ let language = CodeLanguage.detectLanguageFrom(url: fileURL)
 - JSX
 - Kotlin
 - Lua
+- Markdown
 - Objective C
 - OCaml / OCaml Interface
 - PHP
@@ -88,6 +89,8 @@ let language = CodeLanguage.detectLanguageFrom(url: fileURL)
 - ``jsx``
 - ``kotlin``
 - ``lua``
+- ``markdown``
+- ``markdownInline``
 - ``objc``
 - ``php``
 - ``python``

diff --git a/Sources/CodeEditLanguages/Documentation.docc/TreeSitterModel.md b/Sources/CodeEditLanguages/Documentation.docc/TreeSitterModel.md
@@ -50,6 +50,8 @@ let query = TreeSitterModel.shared.swiftQuery
 - ``jsxQuery``
 - ``kotlinQuery``
 - ``luaQuery``
+- ``markdownQuery``
+- ``markdownInlineQuery``
 - ``objcQuery``
 - ``ocamlQuery``
 - ``ocamlInterfaceQuery``

diff --git a/Sources/CodeEditLanguages/Resources/tree-sitter-markdown-inline/highlights.scm b/Sources/CodeEditLanguages/Resources/tree-sitter-markdown-inline/highlights.scm
@@ -0,0 +1,37 @@
+;; From nvim-treesitter/nvim-treesitter
+[
+  (code_span)
+  (link_title)
+] @text.literal
+
+[
+  (emphasis_delimiter)
+  (code_span_delimiter)
+] @punctuation.delimiter
+
+(emphasis) @text.emphasis
+
+(strong_emphasis) @text.strong
+
+[
+  (link_destination)
+  (uri_autolink)
+] @text.uri
+
+[
+  (link_label)
+  (link_text)
+  (image_description)
+] @text.reference
+
+[
+  (backslash_escape)
+  (hard_line_break)
+] @string.escape
+
+; ")" not part of query because of
+; https://github.com/nvim-treesitter/nvim-treesitter/issues/2206
+; TODO: Find better fix for this
+(image ["!" "[" "]" "("] @punctuation.delimiter)
+(inline_link ["[" "]" "("] @punctuation.delimiter)
+(shortcut_link ["[" "]"] @punctuation.delimiter)
diff --git a/Sources/CodeEditLanguages/Resources/tree-sitter-markdown-inline/injections.scm b/Sources/CodeEditLanguages/Resources/tree-sitter-markdown-inline/injections.scm
@@ -0,0 +1,2 @@
+((html_tag) @injection.content (#set! injection.language "html"))
+((latex_block) @injection.content (#set! injection.language "latex"))
diff --git a/Sources/CodeEditLanguages/Resources/tree-sitter-markdown/highlights.scm b/Sources/CodeEditLanguages/Resources/tree-sitter-markdown/highlights.scm
@@ -0,0 +1,52 @@
+;From nvim-treesitter/nvim-treesitter
+(atx_heading (inline) @text.title)
+(setext_heading (paragraph) @text.title)
+
+[
+  (atx_h1_marker)
+  (atx_h2_marker)
+  (atx_h3_marker)
+  (atx_h4_marker)
+  (atx_h5_marker)
+  (atx_h6_marker)
+  (setext_h1_underline)
+  (setext_h2_underline)
+] @punctuation.special
+
+[
+  (link_title)
+  (indented_code_block)
+  (fenced_code_block)
+] @text.literal
+
+[
+  (fenced_code_block_delimiter)
+] @punctuation.delimiter
+
+(code_fence_content) @none
+
+[
+  (link_destination)
+] @text.uri
+
+[
+  (link_label)
+] @text.reference
+
+[
+  (list_marker_plus)
+  (list_marker_minus)
+  (list_marker_star)
+  (list_marker_dot)
+  (list_marker_parenthesis)
+  (thematic_break)
+] @punctuation.special
+
+[
+  (block_continuation)
+  (block_quote_marker)
+] @punctuation.special
+
+[
+  (backslash_escape)
+] @string.escape
diff --git a/Sources/CodeEditLanguages/Resources/tree-sitter-markdown/injections.scm b/Sources/CodeEditLanguages/Resources/tree-sitter-markdown/injections.scm
@@ -0,0 +1,12 @@
+(fenced_code_block
+  (info_string
+    (language) @injection.language)
+  (code_fence_content) @injection.content)
+
+((html_block) @injection.content (#set! injection.language "html"))
+
+(document . (section . (thematic_break) (_) @injection.content (thematic_break)) (#set! injection.language "yaml"))
+
+([(minus_metadata) (plus_metadata)] @injection.content (#set! injection.language "yml"))
+
+((inline) @injection.content (#set! injection.language "markdown_inline"))
diff --git a/Sources/CodeEditLanguages/TreeSitterLanguage.swift b/Sources/CodeEditLanguages/TreeSitterLanguage.swift
@@ -27,6 +27,8 @@ public enum TreeSitterLanguage: String {
     case jsx
     case kotlin
     case lua
+    case markdown
+    case markdownInline
     case objc
     case ocaml
     case ocamlInterface

diff --git a/Sources/CodeEditLanguages/TreeSitterModel.swift b/Sources/CodeEditLanguages/TreeSitterModel.swift
@@ -56,6 +56,10 @@ public class TreeSitterModel {
             return kotlinQuery
         case .lua:
             return luaQuery
+        case .markdown:
+            return markdownQuery
+        case .markdownInline:
+            return markdownInlineQuery
         case .objc:
             return objcQuery
         case .ocaml:
@@ -181,6 +185,16 @@ public class TreeSitterModel {
         return queryFor(.lua)
     }()
 
+    /// Query for `Markdown` files.
+    public private(set) lazy var markdownQuery: Query? = {
+        return queryFor(.markdown)
+    }()
+
+    /// Query for `Markdown Inline` files.
+    public private(set) lazy var markdownInlineQuery: Query? = {
+        return queryFor(.markdownInline)
+    }()
+
     /// Query for `Objective C` files.
     public private(set) lazy var objcQuery: Query? = {
         return queryFor(.objc)

diff --git a/Tests/CodeEditLanguagesTests/CodeEditLanguagesTests.swift b/Tests/CodeEditLanguagesTests/CodeEditLanguagesTests.swift
@@ -398,6 +398,70 @@ final class CodeEditLanguagesTests: XCTestCase {
         var language = CodeLanguage.lua
         language.resourceURL = bundleURL
 
+        let data = try Data(contentsOf: language.queryURL!)
+        let query = try? Query(language: language.language!, data: data)
+        XCTAssertNotNil(query)
+        XCTAssertNotEqual(query?.patternCount, 0)
+    }
+// MARK: - Markdown
+
+    func test_CodeLanguageMarkdown() throws {
+        let url = URL(fileURLWithPath: "~/path/to/file.md")
+        let language = CodeLanguage.detectLanguageFrom(url: url)
+
+        XCTAssertEqual(language.id, .markdown)
+    }
+
+    func test_CodeLanguageMarkdown2() throws {
+        let url = URL(fileURLWithPath: "~/path/to/file.mkd")
+        let language = CodeLanguage.detectLanguageFrom(url: url)
+
+        XCTAssertEqual(language.id, .markdown)
+    }
+
+    func test_CodeLanguageMarkdown3() throws {
+        let url = URL(fileURLWithPath: "~/path/to/file.mkdn")
+        let language = CodeLanguage.detectLanguageFrom(url: url)
+
+        XCTAssertEqual(language.id, .markdown)
+    }
+
+    func test_CodeLanguageMarkdown4() throws {
+        let url = URL(fileURLWithPath: "~/path/to/file.mdwn")
+        let language = CodeLanguage.detectLanguageFrom(url: url)
+
+        XCTAssertEqual(language.id, .markdown)
+    }
+
+    func test_CodeLanguageMarkdown5() throws {
+        let url = URL(fileURLWithPath: "~/path/to/file.mdown")
+        let language = CodeLanguage.detectLanguageFrom(url: url)
+
+        XCTAssertEqual(language.id, .markdown)
+    }
+
+    func test_CodeLanguageMarkdown6() throws {
+        let url = URL(fileURLWithPath: "~/path/to/file.markdown")
+        let language = CodeLanguage.detectLanguageFrom(url: url)
+
+        XCTAssertEqual(language.id, .markdown)
+    }
+
+    func test_FetchQueryMarkdown() throws {
+        var language = CodeLanguage.markdown
+        language.resourceURL = bundleURL
+
+        let data = try Data(contentsOf: language.queryURL!)
+        let query = try? Query(language: language.language!, data: data)
+        XCTAssertNotNil(query)
+        XCTAssertNotEqual(query?.patternCount, 0)
+    }
+
+// MARK: - Markdown Inline
+    func test_FetchQueryMarkdownInline() throws {
+        var language = CodeLanguage.markdownInline
+        language.resourceURL = bundleURL
+
         let data = try Data(contentsOf: language.queryURL!)
         let query = try? Query(language: language.language!, data: data)
         XCTAssertNotNil(query)
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1,2 @@
		((html_tag) @injection.content (#set! injection.language "html"))
		((latex_block) @injection.content (#set! injection.language "latex"))