GT-2897: Gradle installation script for external dependencies

DevGuide.md

@@ -71,6 +71,30 @@ Ghidra's build uses artifacts named as available in Maven Central and Bintray JC
 Unfortunately, in some cases, the artifact or the particular version we desire is not available.
 So, in addition to mavenCentral and jcenter, you must configure a flatDir-style repository for manually-downloaded dependencies.
 
+#### Dependency Setup Script
+The flat repository mentioned above can be done automatically by running a simple Gradle script. 
+Navigate to the Ghidra clone you just created and from the top-level ghidra folder, run the following:
+```
+gradle --init-script gradle/init.gradle tasks
+```
+The Gradle task to be executed, in this case _tasks_, is unimportant. The point is to have Gradle execute
+the __init.gradle__ script. If it ran correctly you will have a new folder, __flatRepo/__, in your home directory populated with the following jar files:
+ * AXMLPrinter2
+ * csframework
+ * dex-ir-2.0
+ * dex-reader-2.0
+ * dex-reader-api-2.0
+ * dex-tools-2.0
+ * dex-translator-2.0
+ * dex-writer-2.0
+ * hfsx
+ * hfsx_dmglib
+ * iharder-base64 
+
+There will also be a new archive, yajsw-stable-12.12.zip, placed in __ghidra.bin/Ghidra/Features/GhidraServer/__.  
+
+If you see these, congrats! Skip to [importing the Gradle project](#import-gradle-project). If not, continue with manual configuration below...
+
 Create `~/.gradle/init.d/repos.gradle` with the following contents:
 
 ```groovy
@@ -216,6 +240,8 @@ To build the full Ghidra distribution, you must also build the GhidraServer.
 
 ## Get Dependencies for GhidraServer
 
+_Note_: If you already ran the [dependency setup script](#dependency-setup-script), you can skip this section and continue on to [building the package](#building-the-package).
+
 Building the GhidraServer requires "Yet another Java service wrapper" (yajsw) version 12.12.
 Download `yajsw-stable-12.12.zip` from their project on www.sourceforge.net, and place it in a directory named:
 `ghidra.bin/Ghidra/Features/GhidraServer/`. Note that `ghidra.bin` must be a sibling of `ghidra`:

gradle/certification.manifest

@@ -5,6 +5,7 @@ distributableGhidraExtension.gradle||GHIDRA||||END|
 distributableGhidraModule.gradle||GHIDRA||||END|
 externalGhidraExtension.gradle||GHIDRA||||END|
 helpProject.gradle||GHIDRA||||END|
+init.gradle||GHIDRA||||END|
 jacocoProject.gradle||GHIDRA||||END|
 javaProject.gradle||GHIDRA||||END|
 javaTestProject.gradle||GHIDRA||||END|

gradle/init.gradle

@@ -0,0 +1,357 @@
+/*******************************************************************************
+ * init.gradle                                                                 *
+ *                                                                             *      
+ * Sets up the gradle configuration for external users and downloads           *
+ * any required dependencies that aren't available in the                      *
+ * other online repositories (eg: maven). This should be run                   *
+ * immediately after cloning the Ghidra repository before any other gradle     *
+ * tasks are run.                                                              *
+ *                                                                             *
+ * Specifically, this task:                                                    *
+ *                                                                             *
+ *   1. Downloads various jars required by the ghidra build and                *
+ *      puts them in <USER_HOME>/flatRepo. The jars to be                      *
+ *      downloaded:                                                            *
+ *          - dex-tools-2.0.zip                                                *
+ *          - AXMLPrinter2.jar                                                 *
+ *          - hfsexplorer-0_21-bin.zip                                         *
+ *          - yajsw-stable-12.12.zip (placed in GhidraServer location)         *
+ *                                                                             *
+ *   2. Creates a gradle configuration file (repos.config) in                  *
+ *      <USER_HOME>/.gradle/init.d/. This contains repository                  *
+ *      information used by gradle to find dependencies (it points             *
+ *      gradle to the flatRepo location created above).                        *
+ *                                                                             *
+ * usage: from the command line in the main ghidra repository                  *
+ *        directory, run the following:                                        *
+ *                                                                             *
+ *        gradle --init-script gradle/init.gradle <any_task>                   *
+ *                                                                             *
+ * Note: Running this script multiple times will cause the config              *
+ *       file to be recreated and all dependencies re-downloaded.              *
+ *                                                                             *
+ * Note: All files are downloaded to a the standard java temporary folder      *
+ *       location, in a sub-folder called 'ghidra. This is cleaned up and      *
+ *       removed when the script completes.                                    *
+ *       TODO: make sure this folder is cleaned up in EVERY case, especially   *
+ *       if the script fails at some point                                     *
+ *                                                                             *
+ *******************************************************************************/
+
+import java.util.zip.*;
+import java.nio.file.*;
+import java.security.MessageDigest;
+import org.apache.commons.io.*;
+import org.apache.commons.io.filefilter.*;
+
+ext.HOME_DIR = System.getProperty('user.home')
+ext.FLAT_REPO_DIR = new File(HOME_DIR + "/flatRepo")
+ext.TMP_FILE = File.createTempFile("downloads", ".tmp", null)
+File TMP_DIR = TMP_FILE.getParentFile()
+ext.DOWNLOADS_DIR = new File(TMP_DIR, "ghidra")
+
+// The URLs for each of the archives to be downloaded
+ext.DEX_ZIP = 'https://github.com/pxb1988/dex2jar/releases/download/2.0/dex-tools-2.0.zip'
+ext.AXML_ZIP = 'https://storage.googleapis.com/google-code-archive-downloads/v2/code.google.com/android4me/AXMLPrinter2.jar'
+ext.HFS_ZIP = 'https://sourceforge.net/projects/catacombae/files/HFSExplorer/0.21/hfsexplorer-0_21-bin.zip'
+ext.YAJSW_ZIP = 'https://sourceforge.net/projects/yajsw/files/yajsw/yajsw-stable-12.12/yajsw-stable-12.12.zip'
+
+// Store the MD5s for each of the downloads so we can verify that we retrieved them
+// all successfully
+ext.DEX_MD5 = '032456b9db9e6059376611553aecf31f'
+ext.AXML_MD5 = '55d70be9862c2b456cc91a933c197934'
+ext.HFS_MD5 = 'cc1713d634d2cd1fd7f21e18ae4d5d5c'
+ext.YAJSW_MD5 = 'e490ea92554f0238d74d4ef6161cb2c7'
+
+// Number of times to try and establish a connection when downloading files before
+// failing
+ext.NUM_RETRIES = 1
+
+initscript {
+    repositories {
+        mavenCentral()
+    }
+    dependencies {
+        classpath 'commons-io:commons-io:2.5'
+    }
+}
+
+try {
+	createDirs()
+	createConfigFile()
+	populateFlatRepo()
+}
+finally {
+	cleanup()  
+}
+
+/**
+ * Creates the directories where the dependencies will be downloaded and stored
+ */
+def createDirs() {   
+   if (!DOWNLOADS_DIR.exists()) {
+      DOWNLOADS_DIR.mkdirs()
+   }
+   if (!FLAT_REPO_DIR.exists()) {
+      FLAT_REPO_DIR.mkdirs()
+   }
+}
+
+/**
+ * Creates the repos.gradle configuration file that tells Gradle 
+ * where to look for dependencies. This ensures that Gradle will 
+ * find the jars we store in the local flat repo.
+ */
+def createConfigFile() {
+
+   ext.repoConfigDir = new File(HOME_DIR + "/.gradle/init.d")
+   ext.repoConfigFile = new File(repoConfigDir, "repos.gradle")
+
+   if (!repoConfigDir.exists()) {
+      repoConfigDir.mkdirs()
+   }
+
+   repoConfigFile.write("ext.HOME = System.getProperty('user.home')")
+   repoConfigFile.append("\nallprojects {")
+   repoConfigFile.append("\nrepositories {")
+   repoConfigFile.append("\nmavenCentral()")
+   repoConfigFile.append("\njcenter()")
+   repoConfigFile.append('\nflatDir name: "flat", dirs:["$HOME/flatRepo"]')
+   repoConfigFile.append("\n}")
+   repoConfigFile.append("\n}")
+}
+
+/**
+ * Downloads a file from a URL. If there is a problem connecting to the given
+ * URL the attempt will be retried NUM_RETRIES times before failing.
+ *
+ * Progress is shown on the command line in the form of the number of bytes
+ * downloaded; the total size of the file to download is not known during the
+ * download so no percentage of the total is given.
+ *
+ * @param url the file to download
+ * @param filename the local file to create for the download
+ */
+def download(url, filename) {
+
+   BufferedInputStream istream = establishConnection(url, NUM_RETRIES);
+   assert istream != null : "***CONNECTION FAILURE***\nmax attempts exceeded; exiting\n"
+   
+   FileOutputStream ostream = new FileOutputStream(filename);
+   def dataBuffer = new byte[1024];
+   int bytesRead;
+   int totalRead;
+   while ((bytesRead = istream.read(dataBuffer, 0, 1024)) != -1) { 
+	   
+     ostream.write(dataBuffer, 0, bytesRead);
+     totalRead += bytesRead
+	      
+     // print progress on the same line in the console...
+     print("\r")
+     print("Downloading: " + filename + " " + totalRead)
+     System.out.flush()
+   }
+   println("")
+	
+   istream.close();
+   ostream.close();
+}
+
+/**
+ * Attemps to establish a connection to the given URL. This will attempt to retry the
+ * connection in the event of a failure.
+ *
+ * @param url the site to connect to
+ * @param retries the number of times to attempt to reconnect if there is a failure
+ */
+def establishConnection(url, retries) {
+   println("Download file: " + url)
+   for (int i=0; i<retries; i++) {
+      try {
+         println("connect attempt " + (i+1) + " of " + retries)
+         return new BufferedInputStream(new URL(url).openStream()); 
+      }
+      catch (Exception e) {
+         println("connection error! " + e)
+      }
+   }
+}
+
+/**
+ * Unzips a file to a directory
+ *
+ * @param sourceDir the directory where the zip file resides
+ * @param targetDir the directory where the unzipped files should be placed
+ * @param zipFileName the name of the file to unpack
+ */
+def unzip(sourceDir, targetDir, zipFileName) {
+   def zip = new ZipFile(new File(sourceDir, zipFileName))
+   
+   zip.entries().findAll { !it.directory }.each { e ->
+      (e.name as File).with { f -> 
+         if (f.parentFile != null) {
+	         File destPath = new File(targetDir.path, f.parentFile.path)
+	         destPath.mkdirs()
+	         File targetFile = new File(destPath.path, f.name)
+	         targetFile.withOutputStream { w -> 
+	            w << zip.getInputStream(e)
+	         }
+	      }
+      }
+   }
+}
+
+/**
+ * Downloads and stores the necessary dependencies in the local
+ * flat repository.
+ */
+def populateFlatRepo() {
+
+   // 1. Download all the dependencies.
+   download (DEX_ZIP, DOWNLOADS_DIR.path + '/dex-tools-2.0.zip')
+   download (AXML_ZIP, FLAT_REPO_DIR.path + '/AXMLPrinter2.jar')
+   download (HFS_ZIP, DOWNLOADS_DIR.path + '/hfsexplorer-0_21-bin.zip')
+   download (YAJSW_ZIP, DOWNLOADS_DIR.path + '/yajsw-stable-12.12.zip')
+   
+   validateChecksum(DOWNLOADS_DIR.path + '/dex-tools-2.0.zip', DEX_MD5);
+   validateChecksum(FLAT_REPO_DIR.path + '/AXMLPrinter2.jar', AXML_MD5);
+   validateChecksum(DOWNLOADS_DIR.path + '/hfsexplorer-0_21-bin.zip', HFS_MD5);
+   validateChecksum(DOWNLOADS_DIR.path + '/yajsw-stable-12.12.zip', YAJSW_MD5);
+
+   // 2. Unzip the dependencies; for some of these we don't need everything in 
+   //    the download so unzip them and only copy what we need. 
+   // 
+   unzip(DOWNLOADS_DIR, DOWNLOADS_DIR, "dex-tools-2.0.zip")
+   unzipHfsx()
+   unzipYajsw()
+
+   // 3. Copy the necessary jars to the flatRepo directory. Yajsw is the
+   //    exception; it needs to go in a GhidraServer folder.
+   copyDexTools()
+   copyHfsx()
+   copyYajsw()
+}
+
+/**
+ * Generates the md5 for the given file and compares it against the
+ * expected result. If there is no match an assert exception will be
+ * generated.
+ *
+ * @param filename the fully-qualified file path+name
+ * @param expectedMd5 the expected md5 for the file
+ */
+def validateChecksum(filename, expectedMd5) {
+    MessageDigest md = MessageDigest.getInstance("MD5");
+    md.update(Files.readAllBytes(Paths.get(filename)));
+    byte[] digest = md.digest();
+    StringBuilder sb = new StringBuilder();
+    for (byte b : digest) {
+       sb.append(String.format("%02x", b));
+    }
+    assert(sb.toString().equals(expectedMd5));
+}
+
+/**
+ * Unzips the hfsx zip file
+ */
+def unzipHfsx() {
+   def hfsxdir = getOrCreateTempHfsxDir()
+   unzip (DOWNLOADS_DIR, hfsxdir, "hfsexplorer-0_21-bin.zip")
+}
+
+/**
+ * Unzips the yajsw zip file
+ */
+def unzipYajsw() {
+   def yajswdir = getOrCreateTempYajswDir()
+   unzip (DOWNLOADS_DIR, yajswdir, "yajsw-stable-12.12.zip")
+}
+
+/**
+ * Copies the dex-tools jars to the flat repository
+ *
+ * Note: This will only copy files beginning with "dex-"
+ */
+def copyDexTools() {
+   FileUtils.copyDirectory(new File(DOWNLOADS_DIR, 'dex2jar-2.0/lib/'), FLAT_REPO_DIR, new WildcardFileFilter("dex-*"));
+}
+
+/**
+ * Copies the hfsx jars to the flat repository
+ */
+def copyHfsx() {
+   FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/csframework.jar"), new File(FLAT_REPO_DIR, "csframework.jar"));
+   FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/hfsx_dmglib.jar"), new File(FLAT_REPO_DIR, "hfsx_dmglib.jar"));
+   FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/hfsx.jar"), new File(FLAT_REPO_DIR, "hfsx.jar"));
+   FileUtils.copyFile(new File(DOWNLOADS_DIR, "hfsx/lib/iharder-base64.jar"), new File(FLAT_REPO_DIR, "iharder-base64.jar"));
+}
+
+/**
+ * Copies the yajswdir zip to its location in the GhidraServer project.
+ *
+ * Note: There is a Gradle task to do all of this (yajswDevUnpack) but we cannot execute
+ *       that from here so just copy everything manually
+ */
+def copyYajsw() {
+   def userdir = System.getProperty("user.dir")
+   def yajswdir = new File(getOrCreateTempYajswDir(),"yajsw-stable-12.12")
+   def yajswdirTarget = new File(userdir, "/../ghidra/Ghidra/Features/GhidraServer/build/data/yajsw-stable-12.12")
+   
+   FileUtils.copyFile(new File(yajswdir, "yajsw.policy.txt"), new File(yajswdirTarget, "yajsw.policy.txt"));
+   FileUtils.copyFile(new File(yajswdir, "LICENSE.txt"), new File(yajswdirTarget, "LICENSE.txt"));
+   FileUtils.copyDirectory(yajswdir, yajswdirTarget, new WildcardFileFilter("*.jar"));
+   FileUtils.copyDirectory(new File(yajswdir, "templates"), new File(yajswdirTarget, "templates"));
+   FileUtils.copyDirectory(new File(yajswdir, "lib/extended"), new File(yajswdirTarget, "lib/extended"));
+   FileUtils.copyDirectory(new File(yajswdir, "doc"), new File(yajswdirTarget, "doc"));
+   FileUtils.copyDirectory(new File(yajswdir, "lib/core"), new File(yajswdirTarget, "lib/core"));
+}
+
+/**
+ * Creates a temporary folder to house the hfsx zip contents
+ *
+ * @return the newly-created hfsx directory object
+ */
+def getOrCreateTempHfsxDir() {
+   def hfsxdir = new File (DOWNLOADS_DIR, "hfsx")
+   if (!hfsxdir.exists()) {
+       hfsxdir.mkdir()
+   }
+
+   return hfsxdir;
+}
+
+/**
+ * Creates a temporary folder to house the yajsw zip contents
+ */
+def getOrCreateTempYajswDir() {
+   def yajswdir = new File (DOWNLOADS_DIR, "yajsw")
+   if (!yajswdir.exists()) {
+       yajswdir.mkdir()
+   }
+
+   return yajswdir;
+}
+
+/**
+ * Performs any cleanup operations that need to be performed after the flat repo has
+ * been populated.
+ */
+def cleanup() {
+   remove(DOWNLOADS_DIR)
+   remove(TMP_FILE)
+}
+
+/**
+ * Deletes the given file (or directory)
+ *
+ * @param fileOrDirectory the item (either a file or directory) to be deleted
+ */ 
+def remove(fileOrDirectory) {
+   if (fileOrDirectory.isDirectory()) {
+      for (File child : fileOrDirectory.listFiles()) {
+         remove(child);
+      }
+   }
+
+   fileOrDirectory.delete();
+}