Move Dartium reference build scripts to dartium_tools/perf

BUG=
R=rmacnak@google.com, vsm@google.com

Review URL: https://codereview.chromium.org//169183004

git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@32907 260f80e4-7a28-3924-810f-c04153c831b5

tools/bots/fetch_reference_build.py

@@ -0,0 +1,49 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Fetches an archived chromium build into
+  src/chrome/tools/test/reference_build unless
+  src/chrome/tools/test/reference_build/REQUESTED_REVISION is the same as
+  src/chrome/tools/test/reference_build/CURRENT_REVISION.
+  Must be run from the root of a Dartium or multivm checkout.
+
+Usage:
+  $ ./src/dart/tools/bots/fetch_reference_build_revision.py
+"""
+
+import os
+import subprocess
+import sys
+
+def main(argv):
+  dirname = os.path.join('src', 'chrome', 'tools',
+                        'test', 'reference_build')
+  request = os.path.join(dirname, 'REQUESTED_REVISION')
+  found = os.path.join(dirname, 'CURRENT_REVISION')
+  if not os.path.exists(request):
+    return
+  with file(request, 'r') as f:
+    request_revision = f.read()
+
+  if os.path.exists(found):
+    with file(found, 'r') as f:
+      found_revision = f.read()
+    if found_revision == request_revision:
+      return
+
+  get_script = os.path.join('src', 'dart', 'tools',
+                            'bots', 'get_chromium_build.py')
+  get_script = os.path.abspath(get_script)
+  exit_code = subprocess.call(['python', get_script,
+                               '-r', request_revision,
+                               '-t', dirname])
+  if exit_code == 0:
+    with file(found, 'w') as f:
+      f.write(request_revision)
+  return exit_code
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

tools/bots/get_chromium_build.py

@@ -0,0 +1,171 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Gets a Chromium archived build, and unpacks it
+   into a target directory.
+
+  Use -r option to specify the revison number
+  Use -t option to specify the directory to unzip the build into.
+
+Usage:
+  $ get_chromium_build.py -r <revision> -t <target>
+"""
+
+import logging
+import optparse
+import os
+import platform
+import shutil
+import subprocess
+import sys
+import time
+import urllib
+import urllib2
+import zipfile
+
+# Example chromium build location:
+# gs://chromium-browser-snapshots/Linux_x64/228977/chrome-linux.zip
+CHROMIUM_URL_FMT = ('http://commondatastorage.googleapis.com/'
+                    'chromium-browser-snapshots/%s/%s/%s')
+
+class BuildUpdater(object):
+  _PLATFORM_PATHS_MAP = {
+      'Linux': { 'zipfiles': ['chrome-linux.zip'],
+                 'folder': 'chrome_linux',
+                 'archive_path': 'Linux_x64'},
+      'Darwin': {'zipfiles': ['chrome-mac.zip'],
+                 'folder': 'chrome_mac',
+                 'archive_path': 'Mac'},
+      'Windows': {'zipfiles': ['chrome-win32.zip',
+                               'chrome-win32-syms.zip'],
+                 'folder': 'chrome_win',
+                 'archive_path': 'Win'}}
+
+  def __init__(self, options):
+    platform_data = BuildUpdater._PLATFORM_PATHS_MAP[platform.system()]
+    self._zipfiles = platform_data['zipfiles']
+    self._folder = platform_data['folder']
+    self._archive_path = platform_data['archive_path']
+    self._revision = int(options.revision)
+    self._target_dir = options.target_dir
+    self._download_dir = os.path.join(self._target_dir, 'downloads')
+
+  def _GetBuildUrl(self, revision, filename):
+    return CHROMIUM_URL_FMT % (self._archive_path, revision, filename)
+
+  def _FindBuildRevision(self, revision, filename):
+    MAX_REVISIONS_PER_BUILD = 100
+    for revision_guess in xrange(revision, revision + MAX_REVISIONS_PER_BUILD):
+      if self._DoesBuildExist(revision_guess, filename):
+        return revision_guess
+      else:
+        time.sleep(.1)
+    return None
+
+  def _DoesBuildExist(self, revision_guess, filename):
+    url = self._GetBuildUrl(revision_guess, filename)
+
+    r = urllib2.Request(url)
+    r.get_method = lambda: 'HEAD'
+    try:
+      urllib2.urlopen(r)
+      return True
+    except urllib2.HTTPError, err:
+      if err.code == 404:
+        return False
+
+  def _DownloadBuild(self):
+    if not os.path.exists(self._download_dir):
+      os.makedirs(self._download_dir)
+    for zipfile in self._zipfiles:
+      build_revision = self._FindBuildRevision(self._revision, zipfile)
+      if not build_revision:
+        logging.critical('Failed to find %s build for r%s\n',
+                         self._archive_path,
+                         self._revision)
+        sys.exit(1)
+      url = self._GetBuildUrl(build_revision, zipfile)
+      logging.info('Downloading %s', url)
+      r = urllib2.urlopen(url)
+      with file(os.path.join(self._download_dir, zipfile), 'wb') as f:
+        f.write(r.read())
+
+  def _UnzipFile(self, dl_file, dest_dir):
+    if not zipfile.is_zipfile(dl_file):
+      return False
+    logging.info('Unzipping %s', dl_file)
+    with zipfile.ZipFile(dl_file, 'r') as z:
+      for content in z.namelist():
+        dest = os.path.join(dest_dir, content[content.find('/')+1:])
+        # Create dest parent dir if it does not exist.
+        if not os.path.isdir(os.path.dirname(dest)):
+          logging.info('Making %s', dest)
+          os.makedirs(os.path.dirname(dest))
+        # If dest is just a dir listing, do nothing.
+        if not os.path.basename(dest):
+          continue
+        with z.open(content) as unzipped_content:
+          logging.info('Extracting %s to %s (%s)', content, dest, dl_file)
+          with file(dest, 'wb') as dest_file:
+            dest_file.write(unzipped_content.read())
+          permissions = z.getinfo(content).external_attr >> 16
+          if permissions:
+            os.chmod(dest, permissions)
+    return True
+
+  def _ClearDir(self, dir):
+    """Clears all files in |dir| except for hidden files and folders."""
+    for root, dirs, files in os.walk(dir):
+      # Skip hidden files and folders (like .svn and .git).
+      files = [f for f in files if f[0] != '.']
+      dirs[:] = [d for d in dirs if d[0] != '.']
+
+      for f in files:
+        os.remove(os.path.join(root, f))
+
+  def _ExtractBuild(self):
+    dest_dir = os.path.join(self._target_dir, self._folder)
+    self._ClearDir(dest_dir)
+    for root, _, dl_files in os.walk(os.path.join(self._download_dir)):
+      for dl_file in dl_files:
+        dl_file = os.path.join(root, dl_file)
+        if not self._UnzipFile(dl_file, dest_dir):
+          logging.info('Copying %s to %s', dl_file, dest_dir)
+          shutil.copy(dl_file, dest_dir)
+    shutil.rmtree(self._download_dir)
+
+  def DownloadAndUpdateBuild(self):
+    self._DownloadBuild()
+    self._ExtractBuild()
+
+
+def ParseOptions(argv):
+  parser = optparse.OptionParser()
+  usage = 'usage: %prog <options>'
+  parser.set_usage(usage)
+  parser.add_option('-r', dest='revision',
+                    help='Revision to download.')
+  parser.add_option('-t', dest='target_dir',
+                    help='Target directory for unzipped Chromium.')
+
+  (options, _) = parser.parse_args(argv)
+  if not options.revision:
+    logging.critical('Must specify -r.\n')
+    sys.exit(1)
+  if not options.target_dir:
+    logging.critical('Must specify -t.\n')
+    sys.exit(1)
+  return options
+
+def main(argv):
+  logging.getLogger().setLevel(logging.DEBUG)
+  options = ParseOptions(argv)
+  b = BuildUpdater(options)
+  b.DownloadAndUpdateBuild()
+  logging.info('Successfully got archived Chromium build.')
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))

tools/bots/set_reference_build_revision.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python
+#
+# Copyright (c) 2014 The Chromium Authors. All rights reserved.
+# Use of this source code is governed by a BSD-style license that can be
+# found in the LICENSE file.
+
+"""Writes a revision number into src/chrome/tools/test/reference_build/REVISON
+   Must be run from the root of a Dartium or multivm checkout.
+
+Usage:
+  $ ./src/dart/tools/bots/set_reference_build_revision.py <revision>
+"""
+
+import os
+import sys
+
+def main(argv):
+  revision = argv[1]
+  output = os.path.join('src', 'chrome', 'tools',
+                        'test', 'reference_build',
+                        'REQUESTED_REVISION')
+  dirname = os.path.dirname(output)
+  if dirname and not os.path.exists(dirname):
+    os.makedirs(dirname)
+  with file(output, 'w') as f:
+    f.write(revision)
+
+if __name__ == '__main__':
+  sys.exit(main(sys.argv))