android_studio_releases.md: parse an external XML file instead of web scrapping AS releases page

.github/scripts/android_studio_releases.main.kts

@@ -2,31 +2,23 @@
 
 /**
  * This script is used to update the Android Studio releases page.
- * At first, it fetches the list of Android Studio updates from the official `updates.xml` file.
+ * At first, it fetches the list of Android Studio updates from an XML file generated on TeamCity.
  * Parsed list is used to generate the Markdown table.
- * The actual IntelliJ IDEA release version is obtained with the help of the JetBrains Data Services API.
  */
-@file:DependsOn("org.jsoup:jsoup:1.14.3")
 @file:DependsOn("net.swiftzer.semver:semver:1.1.2")
 @file:DependsOn("org.simpleframework:simple-xml:2.7.1")
-@file:DependsOn("org.json:json:20211205")
 
 import net.swiftzer.semver.SemVer
-import org.jsoup.Jsoup
 import org.simpleframework.xml.Attribute
+import org.simpleframework.xml.Element
 import org.simpleframework.xml.ElementList
 import org.simpleframework.xml.Root
 import org.simpleframework.xml.core.Persister
-import java.io.BufferedInputStream
 import java.io.File
-import java.io.FileOutputStream
 import java.net.URL
-import java.util.zip.ZipFile
 
+val RELEASES_LIST = "https://jb.gg/android-studio-releases-list.xml"
 val RELEASES_FILE_PATH_MD = "topics/_generated/android_studio_releases.md"
-val RELEASES_FILE_PATH_XML = "topics/_generated/android_studio_releases.xml"
-val INTELLIJ_RELEASES = "https://www.jetbrains.com/intellij-repository/releases/"
-val ANDROID_STUDIO_HOST = "https://developer.android.com"
 val CHANNEL_BADGES_LIST = """
   [release]: https://img.shields.io/badge/-release-blue?style=flat-square
   [patch]: https://img.shields.io/badge/-patch-orange?style=flat-square
@@ -36,81 +28,30 @@ val CHANNEL_BADGES_LIST = """
   [preview]: https://img.shields.io/badge/-preview-darktgrey?style=flat-square
 """
 
-val platformBuildToVersionMapping = INTELLIJ_RELEASES.fetch { content ->
+val content = URL(RELEASES_LIST).readText()
-  Jsoup.parse(content, "").select("h2:contains(com.jetbrains.intellij.idea) + table tbody tr").mapNotNull { tr ->
+        .run { Persister().read(Content::class.java, this) }
-    val (version, build) = tr.select("td:nth-child(odd)").map { SemVer.parse(it.text()) }
+        ?: throw RuntimeException("Failed to parse releases list")
-    (build to version).takeIf { version.major > 2000 }
-  }.toMap().toSortedMap()
-}
 
-val frameUrl = "$ANDROID_STUDIO_HOST/studio/archive".fetch { content ->
+val xml = """
-  Jsoup.parse(content, "").select("devsite-iframe iframe[src]").firstOrNull()?.attr("src")
-}.let { "$ANDROID_STUDIO_HOST/$it" }
 
-frameUrl.fetch { content ->
+<chunk id="releases_table">
-  val contentFile = file(RELEASES_FILE_PATH_XML)
+${
-  val current = contentFile.takeIf { it.length() > 0 }?.let {
+  content.items.groupBy { it.version.toLooseVersion().major }.entries.joinToString("\n\n") {
-    Persister().read(Content::class.java, it)
+    """
-  } ?: Content()
+      ## ${it.key}.*
-  val nameToBuildMapping = current.items.associate { it.name to it.build }
+      ${it.value.renderTable()}
-
+    """
-  Jsoup.parse(content, "").select("section.expandable").run {
-    mapIndexed { index, item ->
-      val title = item.select("p").firstOrNull()?.text() ?: throw IllegalStateException("No title found")
-      val (name, channelRaw, date) = """^([\w ]+ \(?[\d.]+\)? ?(?:(\w+) \d+)?) (\w+ \d+, \d+)$""".toRegex().find(title)?.groupValues?.drop(1)
-              ?: emptyList()
-
-      println("# $name")
-      println("  ${index + 1}/$size")
-
-      val href = item.select(".downloads a[href$=.zip]").firstOrNull()?.attr("href")
-      val version = href?.split('/')?.let { it[it.indexOf("ide-zips") + 1] }
-              ?: throw IllegalStateException("No version found for $name")
-      val build = nameToBuildMapping[name].takeUnless(String?::isNullOrBlank) ?: run { href.resolveBuild() }
-      val platformBuild = build.split('-').last().toLooseVersion()
-
-      val platformVersion = platformBuildToVersionMapping[platformBuild] ?: run {
-        platformBuildToVersionMapping.entries.findLast { it.key < platformBuild }?.value
-      }
-      val channel = channelRaw.takeIf { it.isNotBlank() } ?: "Release"
-
-      println("  version='${version}'")
-      println("  build='${build}'")
-      println("  platformBuild='${platformBuild}'")
-      println("  platformVersion='${platformVersion}'")
-
-      Item(name, build, version, channel, platformBuild.toString(), platformVersion.toString(), date)
-    }
-  }.let {
-    val version = with(current) {
-      when (items.hashCode() != it.hashCode()) {
-        true -> version + 1
-        false -> version
-      }
-    }
-    Content(version, it)
-  }.also {
-    Persister().write(it, contentFile)
-  }.also { (_, items) ->
-    ("""
-    <chunk id="releases_table">
-    """ + items.groupBy { it.version.toLooseVersion().major }.entries.joinToString("\n\n") {
-      """
-        ## ${it.key}.*
-
-        ${it.value.renderTable()}
-      """
-    } + """
-      $CHANNEL_BADGES_LIST
-    </chunk>
-
-    <chunk id="releases_table_short">
-      ${items.distinctBy(Item::version).take(5).renderTable()}
-      $CHANNEL_BADGES_LIST
-    </chunk>
-  """).split("\n").joinToString("\n", transform = String::trim).let(file(RELEASES_FILE_PATH_MD)::writeText)
   }
 }
+$CHANNEL_BADGES_LIST
+</chunk>
+
+<chunk id="releases_table_short">
+${content.items.distinctBy(Item::version).take(5).renderTable()}
+$CHANNEL_BADGES_LIST
+</chunk>
+
+""".split("\n").joinToString("\n", transform = String::trim).let(file(RELEASES_FILE_PATH_MD)::writeText)
 
 fun List<Item>.renderTable() = """
   | Release Name | Channel | Release Date | Version | IntelliJ IDEA Version |
@@ -125,41 +66,6 @@ fun List<Item>.renderTable() = """
   "| $name | $channel | $date | $version | $platform |"
 }
 
-fun <T> String.fetch(block: (String) -> T) = URL(this).openStream().use { inputStream ->
-  block(inputStream.readBytes().toString(Charsets.UTF_8))
-}
-
-fun <T> String.download(block: (File) -> T) = URL(this).openStream().use { inputStream ->
-  BufferedInputStream(inputStream).use { bis ->
-    File.createTempFile("android-studio", ".zip").also(File::deleteOnExit).let { tempFile ->
-      FileOutputStream(tempFile).use { outputStream ->
-        println("  Downloading $this to $tempFile")
-        ByteArray(1024).let { data ->
-          var count: Int
-          while (bis.read(data, 0, data.size).also { count = it } != -1) {
-            outputStream.write(data, 0, count)
-          }
-        }
-      }
-      block(tempFile)
-    }
-  }
-}
-
-fun String.resolveBuild() = download { file ->
-  ZipFile(file).use { zip ->
-    zip.getEntry("android-studio/build.txt").let { entry ->
-      zip.getInputStream(entry).use { inputStream ->
-        inputStream.readBytes().toString(Charsets.UTF_8)
-      }.also {
-        println("  Resolved build number: $it")
-      }
-    }
-  }.also {
-    file.delete()
-  }
-}
-
 fun String.toLooseVersion() = split('.').map { it.take(4).toInt() }.let {
   val (major, minor, patch) = it + 0
   SemVer(major, minor, patch)
@@ -167,18 +73,48 @@ fun String.toLooseVersion() = split('.').map { it.take(4).toInt() }.let {
 
 fun file(path: String) = File(System.getenv("GITHUB_WORKSPACE") ?: "../../").resolve(path).also(File::createNewFile)
 
-@Root(strict = false)
+@Root(strict = false, name = "content")
 data class Content(
-        @field:Attribute var version: Int = 1,
+        @field:Attribute
-        @field:ElementList(inline = true, required = false) var items: List<Item> = mutableListOf(),
+        var version: Int = 1,
+
+        @field:ElementList(inline = true, entry = "item")
+        var items: List<Item> = mutableListOf(),
 )
 
 data class Item(
+        @field:Element
         var name: String = "",
+
+        @field:Element
         var build: String = "",
+
+        @field:Element
         var version: String = "",
+
+        @field:Element
         var channel: String = "",
+
+        @field:Element
         var platformBuild: String? = null,
+
+        @field:Element
         var platformVersion: String? = null,
+
+        @field:Element
         var date: String = "",
+
+        @field:ElementList(inline = true, entry = "download")
+        var downloads: List<Download> = mutableListOf(),
+)
+
+data class Download(
+        @field:Element
+        var link: String = "",
+
+        @field:Element
+        var size: Int = 0,
+
+        @field:Element
+        var checksum: String = "",
 )

topics/_generated/android_studio_releases.md

@@ -1,9 +1,9 @@
 
+
 <chunk id="releases_table">
 
 ## 2022.*
 
-
 | Release Name | Channel | Release Date | Version | IntelliJ IDEA Version |
 |--------------|:-------:|--------------|---------|-----------------------|
 | Electric Eel (2022.1.1) Canary 9 | ![canary][canary] | August 3, 2022 | **2022.1.1.9** <br/> AI-221.5921.22.2211.8881706 | **2022.1.3** <br/> 221.5921.22 |
@@ -20,7 +20,6 @@
 
 ## 2021.*
 
-
 | Release Name | Channel | Release Date | Version | IntelliJ IDEA Version |
 |--------------|:-------:|--------------|---------|-----------------------|
 | Dolphin (2021.3.1) Beta 5 | ![beta][beta] | July 7, 2022 | **2021.3.1.14** <br/> AI-213.7172.25.2113.8774922 | **2021.3.3** <br/> 213.7172.25 |
@@ -80,7 +79,6 @@
 
 ## 2020.*
 
-
 | Release Name | Channel | Release Date | Version | IntelliJ IDEA Version |
 |--------------|:-------:|--------------|---------|-----------------------|
 | Arctic Fox (2020.3.1) Patch 4 | ![patch][patch] | December 8, 2021 | **2020.3.1.26** <br/> AI-203.7717.56.2031.7935034 | **2020.3.3** <br/> 203.7717.56 |
@@ -114,7 +112,6 @@
 
 ## 4.*
 
-
 | Release Name | Channel | Release Date | Version | IntelliJ IDEA Version |
 |--------------|:-------:|--------------|---------|-----------------------|
 | 4.2.2 | ![release][release] | June 30, 2021 | **4.2.2.0** <br/> AI-202.7660.26.42.7486908 | **2020.2.3** <br/> 202.7660.26 |
@@ -188,7 +185,6 @@
 
 ## 3.*
 
-
 | Release Name | Channel | Release Date | Version | IntelliJ IDEA Version |
 |--------------|:-------:|--------------|---------|-----------------------|
 | 3.6.3 | ![release][release] | April 17, 2020 | **3.6.3.0** <br/> AI-192.7142.36.36.6392135 | **2019.2.4** <br/> 192.7142.36 |
@@ -357,7 +353,6 @@
 
 ## 2.*
 
-
 | Release Name | Channel | Release Date | Version | IntelliJ IDEA Version |
 |--------------|:-------:|--------------|---------|-----------------------|
 | 2.4 Preview 7 | ![preview][preview] | April 28, 2017 | **2.4.0.6** <br/> AI-171.3934896 | **2017.1.0** <br/> 171.3934.0 |
@@ -395,3 +390,4 @@
 [preview]: https://img.shields.io/badge/-preview-darktgrey?style=flat-square
 
 </chunk>
+