Add article and tutorial about Documentation Provider (#413)

* Initial version of a simple DocumentationProvider
* Correct usage of DocumentationMarkup, fix quick navigation
* Include Yann's suggestions. Not finished yet.
* Fix issues Yann pointed out

code_samples/simple_language_plugin/src/main/java/org/intellij/sdk/language/SimpleDocumentationProvider.java

@@ -0,0 +1,96 @@
+package org.intellij.sdk.language;
+
+import com.intellij.lang.documentation.AbstractDocumentationProvider;
+import com.intellij.lang.documentation.DocumentationMarkup;
+import com.intellij.psi.PsiElement;
+import com.intellij.psi.presentation.java.SymbolPresentationUtil;
+import org.intellij.sdk.language.psi.SimpleProperty;
+import org.jetbrains.annotations.NotNull;
+import org.jetbrains.annotations.Nullable;
+
+import java.util.List;
+
+public class SimpleDocumentationProvider extends AbstractDocumentationProvider {
+
+  /**
+   * For the Simple Language, we don't have online documentation. However, if your language provides
+   * references pages online, URLs for the element can be returned here.
+   */
+  @Override
+  public @Nullable List<String> getUrlFor(PsiElement element, PsiElement originalElement) {
+    return null;
+  }
+
+  /**
+   * Extracts the key, value, file and documentation comment of a Simple key/value entry and returns
+   * a formatted representation of the information.
+   */
+  @Override
+  public @Nullable String generateDoc(PsiElement element, @Nullable PsiElement originalElement) {
+    if (element instanceof SimpleProperty) {
+      final String key = ((SimpleProperty) element).getKey();
+      final String value = ((SimpleProperty) element).getValue();
+      final String file = SymbolPresentationUtil.getFilePathPresentation(element.getContainingFile());
+      final String docComment = SimpleUtil.findDocumentationComment((SimpleProperty) element);
+
+      return renderFullDoc(key, value, file, docComment);
+    }
+    return null;
+  }
+
+  /**
+   * Provides the information in which file the Simple language key/value is defined.
+   */
+  @Override
+  public @Nullable String getQuickNavigateInfo(PsiElement element, PsiElement originalElement) {
+    if (element instanceof SimpleProperty) {
+      final String key = ((SimpleProperty) element).getKey();
+      final String file = SymbolPresentationUtil.getFilePathPresentation(element.getContainingFile());
+      return "\"" + key + "\" in " + file;
+    }
+    return null;
+  }
+
+  /**
+   * Provides documentation when a Simple Language element is hovered with the mouse.
+   */
+  @Override
+  public @Nullable String generateHoverDoc(@NotNull PsiElement element, @Nullable PsiElement originalElement) {
+    return generateDoc(element, originalElement);
+  }
+
+  /**
+   * Creates a key/value row for the rendered documentation.
+   */
+  private void addKeyValueSection(String key, String value, StringBuilder sb) {
+    sb.append(DocumentationMarkup.SECTION_HEADER_START);
+    sb.append(key);
+    sb.append(DocumentationMarkup.SECTION_SEPARATOR);
+    sb.append("<p>");
+    sb.append(value);
+    sb.append(DocumentationMarkup.SECTION_END);
+  }
+
+  /**
+   * Creates the formatted documentation using {@link DocumentationMarkup}. See the Java doc of
+   * {@link com.intellij.lang.documentation.DocumentationProvider#generateDoc(PsiElement, PsiElement)} for more
+   * information about building the layout.
+   */
+  private String renderFullDoc(String key, String value, String file, String docComment) {
+    StringBuilder sb = new StringBuilder();
+    sb.append(DocumentationMarkup.DEFINITION_START);
+    sb.append("Simple Property");
+    sb.append(DocumentationMarkup.DEFINITION_END);
+    sb.append(DocumentationMarkup.CONTENT_START);
+    sb.append(value);
+    sb.append(DocumentationMarkup.CONTENT_END);
+    sb.append(DocumentationMarkup.SECTIONS_START);
+    addKeyValueSection("Key:", key, sb);
+    addKeyValueSection("Value:", value, sb);
+    addKeyValueSection("File:", file, sb);
+    addKeyValueSection("Comment:", docComment, sb);
+    sb.append(DocumentationMarkup.SECTIONS_END);
+    return sb.toString();
+  }
+
+}

code_samples/simple_language_plugin/src/main/java/org/intellij/sdk/language/SimpleUtil.java

@@ -2,19 +2,22 @@
 
 package org.intellij.sdk.language;
 
+import com.google.common.collect.Lists;
 import com.intellij.openapi.project.Project;
+import com.intellij.openapi.util.text.StringUtil;
 import com.intellij.openapi.vfs.VirtualFile;
+import com.intellij.psi.PsiComment;
+import com.intellij.psi.PsiElement;
 import com.intellij.psi.PsiManager;
+import com.intellij.psi.PsiWhiteSpace;
 import com.intellij.psi.search.FileTypeIndex;
 import com.intellij.psi.search.GlobalSearchScope;
 import com.intellij.psi.util.PsiTreeUtil;
 import org.intellij.sdk.language.psi.SimpleFile;
 import org.intellij.sdk.language.psi.SimpleProperty;
+import org.jetbrains.annotations.NotNull;
 
-import java.util.ArrayList;
+import java.util.*;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.List;
 
 public class SimpleUtil {
 
@@ -61,4 +64,20 @@ public class SimpleUtil {
     return result;
   }
 
+  /**
+   * Attempts to collect any comment elements above the Simple key/value pair.
+   */
+  public static @NotNull String findDocumentationComment(SimpleProperty property) {
+    List<String> result = new LinkedList<>();
+    PsiElement element = property.getPrevSibling();
+    while (element instanceof PsiComment || element instanceof PsiWhiteSpace) {
+      if (element instanceof PsiComment) {
+        String commentText = element.getText().replaceFirst("[!# ]+", "");
+        result.add(commentText);
+      }
+      element = element.getPrevSibling();
+    }
+    return StringUtil.join(Lists.reverse(result),"\n ");
+  }
+
 }

code_samples/simple_language_plugin/src/main/resources/META-INF/plugin.xml

@@ -66,6 +66,7 @@
     <codeStyleSettingsProvider implementation="org.intellij.sdk.language.SimpleCodeStyleSettingsProvider"/>
     <langCodeStyleSettingsProvider implementation="org.intellij.sdk.language.SimpleLanguageCodeStyleSettingsProvider"/>
     <lang.commenter language="Simple" implementationClass="org.intellij.sdk.language.SimpleCommenter"/>
+    <lang.documentationProvider language="Simple" implementationClass="org.intellij.sdk.language.SimpleDocumentationProvider"/>
   </extensions>
 
 </idea-plugin>

ijs.tree

@@ -260,6 +260,7 @@
             <toc-element id="code_style_settings.md"/>
             <toc-element id="commenter.md"/>
             <toc-element id="quick_fix.md"/>
+            <toc-element id="documentation_provider.md"/>
         </toc-element>
         <toc-element id="writing_tests_for_plugins.md">
             <toc-element id="tests_prerequisites.md"/>

topics/intro/content_updates.md

@@ -8,6 +8,11 @@ See [GitHub Changelog](https://github.com/JetBrains/intellij-sdk-docs/commits/ma
 
 ## 2021
 
+### June-21
+
+Documentation Provider
+: Add [Documentation](documentation.md) section with an [accompanying tutorial](documentation_provider.md) that show how to implement a `DocumentationProvider` for custom languages.
+
 ### May-21
             
 IDE specific Extension Point Lists

topics/reference_guide/custom_language_support/documentation.md

@@ -2,14 +2,56 @@
 
 <!-- Copyright 2000-2021 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. -->
 
-To provide different kinds of documentation support, the plugin needs to provide an implementation of the [`DocumentationProvider`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/DocumentationProvider.java) interface and register it in the `com.intellij.lang.documentationProvider` extension point.
+Custom languages can use the `com.intellij.lang.documentationProvider` extension point (EP) to show documentation for functions,
-A standard base class for such implementations is available in [`AbstractDocumentationProvider`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/AbstractDocumentationProvider.java).
+methods, classes, or other constructs right inside the IDE.
+Accessing the documentation is done by calling
+<menupath>[View | Quick Documentation](https://www.jetbrains.com/help/idea/viewing-reference-information.html#inline-quick-documentation)</menupath>
+or hovering over a symbol, which will open a popup to show type information, parameters, usage descriptions, or examples.
+The source of the documentation contents can vary.
+Often it is extracted from comments (e.g. JavaDoc comments) in the source code,
+but it’s also possible to access external resources like web pages.
 
-The `getQuickNavigateInfo()` method returns the text to be displayed when the user holds the mouse over an element with <shortcut>Ctrl/Cmd</shortcut> pressed.
+In addition to showing the documentation, the `getQuickNavigateInfo()` method returns the text to be displayed
+when the user hovers over an element with <shortcut>Ctrl</shortcut>/<shortcut>Cmd</shortcut> pressed.
 
-When generating complete documentation via `generateDoc()`, use [`DocumentationMarkup`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/DocumentationMarkup.java) to layout contents (see JavaDoc for details).
+Custom actions can also be added to documentation inlays and documentation popups via
+`com.intellij.codeInsight.documentation.DocumentationActionProvider` registered in the
+`com.intellij.documentationActionProvider` extension point.
 
-Additional custom actions can be added to documentation inlays and documentation popup via `com.intellij.codeInsight.documentation.DocumentationActionProvider` registered in `com.intellij.documentationActionProvider` extension point. (2020.3)
 
-**Example**:
+# Implementation
-[`DocumentationProvider`](upsource:///plugins/properties/src/com/intellij/lang/properties/PropertiesDocumentationProvider.java) for [Properties language plugin](upsource:///plugins/properties/)
+
+Custom language developers usually extend from
+[`AbstractDocumentationProvider`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/AbstractDocumentationProvider.java)
+instead of implementing the
+[`DocumentationProvider`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/DocumentationProvider.java) interface.
+This implementation needs to be registered as `com.intellij.lang.documentationProvider` in the <path>plugin.xml</path>.
+
+The main work is done in `generateDoc()`, which has two PSI element arguments:
+the target element for which the documentation is requested and the original element under the cursor.
+If  IntelliJ Platform's choice of target element isn't suitable for your language, you can override `getCustomDocumentationElement()`
+and provide the correct element.
+
+How the documentation for the target element is created is up to the custom language developer.
+A common choice is to extract and format documentation comments.
+To format the documentation contents, you should use
+[`DocumentationMarkup`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/DocumentationMarkup.java)
+to achieve a consistent output.
+
+Once these steps are completed, the following additional features can be implemented:
+
+* Implement `getQuickNavigateInfo()` to provide the text that should be displayed when an element is hovered over with <shortcut>Ctrl</shortcut>/<shortcut>Cmd</shortcut> pressed.
+* Implement `generateHoverDoc()` to show different contents on mouse hover.
+* Implement `getDocumentationElementForLookupItem()` to return a suitable PSI element for the given lookup element when 
+  <menupath>View | Quick Documentation</menupath> is called on an element of the autocompletion popup.
+* Implement `getUrlFor()` and [`ExternalDocumentationProvider`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/ExternalDocumentationProvider.java) to fetch documentation for elements from online resources.
+
+
+# Examples
+
+The [custom language tutorial](documentation_provider.md) contains a step-by-step guide for the `DocumentationProvider` of the Simple language.
+In addition, several implementations of other languages exist in the IntelliJ Platform code, for instance:
+
+* The [Properties Language plugin](upsource:///plugins/properties/) has a small and easy-to-understand [`DocumentationProvider`](upsource:///plugins/properties/src/com/intellij/lang/properties/PropertiesDocumentationProvider.java) similar to the one shown in the custom language tutorial.
+* The [`CssDocumentationProvider`](upsource:///CSS/src/com/intellij/psi/css/impl/util/CssDocumentationProvider.java) is an example of an `ExternalDocumentationProvider`, which accesses online resources to provide documentation.
+* Usage examples for DocumentationMarkup can be found in [`ThemeJsonDocumentationProvider`](upsource:///plugins/devkit/devkit-core/src/themes/ThemeJsonDocumentationProvider.java).

topics/tutorials/custom_language_support/documentation_provider.md

@@ -0,0 +1,232 @@
+[//]: # (title: 19. Documentation)
+
+<!-- Copyright 2000-2021 JetBrains s.r.o. and other contributors. Use of this source code is governed by the Apache 2.0 license that can be found in the LICENSE file. -->
+
+A [`DocumentationProvider`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/DocumentationProvider.java)
+helps users by showing documentation for symbols like method calls inside the editor.
+For the custom language tutorial, we’re implementing a version of this EP for the Simple Language that shows the key/value,
+the file where it is defined, and any related documentation comment.
+
+**Reference:** [Documentation](documentation.md)
+
+
+## Implement DocumentationProvider and Register the EP
+
+In the first step, we create an empty class that extends 
+[`AbstractDocumentationProvider`](upsource:///platform/analysis-api/src/com/intellij/lang/documentation/AbstractDocumentationProvider.java)
+and registers it in the <path>plugin.xml</path>.
+
+
+```java
+public class SimpleDocumentationProvider extends AbstractDocumentationProvider { }
+```
+
+Make sure the class is registered in the <path>plugin.xml</path> between the `extensions` tags, as shown below:
+
+```xml
+<extensions defaultExtensionNs="com.intellij">
+  <!-- Other extensions… -->
+  <lang.documentationProvider language="Simple"
+       implementationClass="org.intellij.sdk.language.SimpleDocumentationProvider"/>
+</extensions>
+```
+
+
+## Ensure That the Correct PSI Element Is Used
+
+For the Simple Language, we consider two use-cases:
+
+1. A Simple key is [used inside a Java string literal](reference_contributor.md), 
+   and we would like to show documentation for the key/value right from the reference inside the Java file.
+2. The cursor is already over a key/value definition inside a Simple file, in which case we would also like to show its documentation.
+
+To ensure that the IntelliJ Platform chooses the correct element of type `SimpleProperty` when <menupath>View | Quick Documentation</menupath> is called,
+we create a dummy implementation of `generateDoc()`:
+
+```java
+@Override
+public @Nullable String generateDoc(PsiElement element, @Nullable PsiElement originalElement) {
+ return super.generateDoc(element, originalElement);
+}
+```
+
+Now, we set a breakpoint in our dummy implementation, debug the plugin, and call <menupath>View | Quick Documentation</menupath>
+for the Simple property both in the Java file and the Simple file.
+We do this by placing the cursor over the key and [pressing the shortcut](https://www.jetbrains.com/help/idea/viewing-reference-information.html#view-quick-docs)
+for showing the documentation.
+
+In both cases, we find that the element provided is `SimplePropertyImpl`, which is exactly what we hoped for.
+However, there are two drawbacks: inside a Java string, your cursor needs to be directly over `key` in the string `"simple:key"` to make <emphasis>Quick Documentation</emphasis> work.
+Since the Simple Language only allows for one property per string,
+it would be nice if <emphasis>Quick Documentation</emphasis> worked no matter where your cursor was positioned in the string as long as the string contained a Simple property.
+Inside a Simple file, the situation is similar, and calling <menupath>View | Quick Documentation</menupath> only works when the cursor is positioned on the key.
+
+Please refer to the Addendum below, which explains how to improve on this situation by additionally overriding `getCustomDocumentationElement()` method.
+
+
+## Extract Documentation Comments from Key/Value Definitions
+
+While `SimpleProperty` elements will provide us with their key and value, we have no direct access to a possible comment that is preceding the key/value definition.
+Since we would like to show this comment in the documentation as well, we need a small helper function that extracts the text from the comment.
+This function will reside in the `SimpleUtil` class and will find for instance the comment preceding `apikey` in the following short example:
+
+```text
+#An application programming interface key (API key) is a unique identifier used
+#to authenticate a user, developer, or calling program to an API.
+apikey=ph342m91337h4xX0r5k!11Zz!
+```
+
+The following implementation will check if there is any comment preceding a `SimpleProperty`, and if there is,
+it will collect all comment lines until it reaches either the previous key/value definition or the beginning of the file.
+One caveat is that since we’re collecting the comment lines backwards, we need to reverse the list before joining them into a single string.
+A simple regex is used to remove the leading hash characters and whitespaces from each line.
+
+```java
+public static @NotNull String findDocumentationComment(SimpleProperty property) {
+ List<String> result = new LinkedList<>();
+ PsiElement element = property.getPrevSibling();
+ while (element instanceof PsiComment || element instanceof PsiWhiteSpace) {
+   if (element instanceof PsiComment) {
+     String commentText = element.getText().replaceFirst("[# ]+", "");
+     result.add(commentText);
+   }
+   element = element.getPrevSibling();
+ }
+ return StringUtil.join(Lists.reverse(result),"\n ");
+}
+```
+
+
+## Render the Documentation
+
+With easy ways to access the key, the value, the file, and a possible documentation comment,
+we now have everything in place to provide a useful implementation of `generateDoc()`.
+
+```java
+@Override
+public @Nullable String generateDoc(PsiElement element, @Nullable PsiElement originalElement) {
+  if (element instanceof SimpleProperty) {
+    final String key = ((SimpleProperty) element).getKey();
+    final String value = ((SimpleProperty) element).getValue();
+    final String file = SymbolPresentationUtil.getFilePathPresentation(element.getContainingFile());
+    final String docComment = SimpleUtil.findDocumentationComment((SimpleProperty) element);
+
+    return renderFullDoc(key, value, file, docComment);
+  }
+  return null;
+}
+```
+
+The creation of the rendered documentation is done in a separate method for clarity.
+It uses `DocumentationMarkup` to align and format the contents.
+
+```java
+private String renderFullDoc(String key, String value, String file, String docComment) {
+  StringBuilder sb = new StringBuilder();
+  sb.append(DocumentationMarkup.DEFINITION_START);
+  sb.append("Simple Property");
+  sb.append(DocumentationMarkup.DEFINITION_END);
+  sb.append(DocumentationMarkup.CONTENT_START);
+  sb.append(value);
+  sb.append(DocumentationMarkup.CONTENT_END);
+  sb.append(DocumentationMarkup.SECTIONS_START);
+  addKeyValueSection("Key:", key, sb);
+  addKeyValueSection("Value:", value, sb);
+  addKeyValueSection("File:", file, sb);
+  addKeyValueSection("Comment:", docComment, sb);
+  sb.append(DocumentationMarkup.SECTIONS_END);
+  return sb.toString();
+}
+```
+
+The `addKeyValueSection()` method used is just a small helper function to reduce repetition.
+
+```java
+private void addKeyValueSection(String key, String value, StringBuilder sb) {
+  sb.append(DocumentationMarkup.SECTION_HEADER_START);
+  sb.append(key);
+  sb.append(DocumentationMarkup.SECTION_SEPARATOR);
+  sb.append("<p>");
+  sb.append(value);
+  sb.append(DocumentationMarkup.SECTION_END);
+}
+```
+
+After implementing all the steps above, the IDE will show the rendered documentation for a Simple key when called with <menupath>View | Quick Documentation</menupath>.
+
+
+## Implement Additional Functionality
+
+We can provide implementations for additional functionality that comes with a `DocumentationProvider`.
+For instance, when simply hovering the mouse over the code, it also shows documentation after a short delay.
+It’s not necessary that this popup show the exact same information as when calling _Quick Documentation_, but for the purpose of this tutorial, we’ll do just that.
+
+```java
+public @Nullable String generateHoverDoc(@NotNull PsiElement element, @Nullable PsiElement originalElement) {
+   return generateDoc(element, originalElement);
+}
+```
+
+When the mouse hovers over code with <shortcut>Ctrl</shortcut>/<shortcut>Cmd</shortcut> pressed, the IDE shows navigation information of the symbol under the cursor,
+such as its namespace or package.
+The implementation below will show the Simple key and the file where it is defined.
+
+```java
+@Override
+public @Nullable String getQuickNavigateInfo(PsiElement element, PsiElement originalElement) {
+  if (element instanceof SimpleProperty) {
+    final String key = ((SimpleProperty) element).getKey();
+    final String file = SymbolPresentationUtil.getFilePathPresentation(element.getContainingFile());
+    return "\"" + key + "\" in " + file;
+  }
+  return null;
+}
+```
+
+
+Finally, <menupath>View | Quick Documentation</menupath> can also be called from a selected entry within the autocompletion popup.
+In that case, language developers need to ensure that the correct PSI element for generating the documentation is provided.
+In the case of Simple Language, the lookup element is already a `SimpleProperty` and no additional work needs to be done.
+In other circumstances, you can override `getDocumentationElementForLookupItem() `and return the correct PSI element.
+
+
+## Addendum: Choosing a Better Target Element
+
+To be able to call <menupath>View | Quick Documentation</menupath> for Simple properties in all places of a Java string literal, two steps are required:
+
+1. The extension point needs to be changed from `lang.documentationProvider` to `documentationProvider` because only then 
+   the Simple DocumentationProvider is called for PSI elements with a different language.
+2. The `getCustomDocumentationElement()` method needs to be implemented to find the correct target PSI element for creating the documentation.
+
+Therefore, the current version of the code could be extended to check whether <menupath>View | Quick Documentation</menupath> was called from inside a Java string or a Simple file.
+It then uses PSI and `PsiReference` functionalities to determine the correct target element.
+This allows getting documentation for a Simple property no matter where it was called inside a Java string literal or a Simple property definition.
+
+```java
+@Override
+public @Nullable PsiElement getCustomDocumentationElement(@NotNull Editor editor, @NotNull PsiFile file, @Nullable PsiElement contextElement, int targetOffset) {
+ if (contextElement != null) {
+   // In this part the SimpleProperty element is extracted from inside a Java string
+   if (contextElement instanceof PsiJavaToken && ((PsiJavaToken) contextElement).getTokenType().equals(JavaTokenType.STRING_LITERAL)) {
+     final PsiElement parent = contextElement.getParent();
+     final PsiReference[] references = parent.getReferences();
+     for (PsiReference ref : references) {
+       if (ref instanceof SimpleReference) {
+         final PsiElement property = ref.resolve();
+         if (property instanceof SimpleProperty) {
+           return property;
+         }
+       }
+     }
+   }
+   // In this part the SimpleProperty element is extracted when inside a .simple file
+   else if (contextElement.getLanguage() == SimpleLanguage.INSTANCE) {
+     final PsiElement property = PsiTreeUtil.getParentOfType(contextElement, SimpleProperty.class);
+     if (property != null) {
+       return property;
+     }
+   }
+ }
+ return super.getCustomDocumentationElement(editor, file, contextElement, targetOffset);
+}
+```