mirror of
https://github.com/dart-lang/sdk
synced 2024-09-20 05:41:37 +00:00
c7d5711688
Cleans up Python Code in Build Tools TEST: build everything and run tests BUG: https://code.google.com/p/dart/issues/detail?id=19592 R=ricow@google.com Review URL: https://codereview.chromium.org//350483003 git-svn-id: https://dart.googlecode.com/svn/branches/bleeding_edge/dart@38573 260f80e4-7a28-3924-810f-c04153c831b5
435 lines
17 KiB
Python
Executable file
435 lines
17 KiB
Python
Executable file
#!/usr/bin/python
|
|
|
|
# Copyright (c) 2012, the Dart project authors. Please see the AUTHORS file
|
|
# for details. All rights reserved. Use of this source code is governed by a
|
|
# BSD-style license that can be found in the LICENSE file.
|
|
|
|
# Run to install the necessary components to run webdriver on the buildbots or
|
|
# on your local machine.
|
|
# Note: The setup steps can be done fairly easily by hand. This script is
|
|
# intended to simply and reduce the time for setup since there are a fair number
|
|
# of steps.
|
|
|
|
# TODO(efortuna): Rewrite this script in Dart when the Process module has a
|
|
# better high level API.
|
|
import HTMLParser
|
|
import optparse
|
|
import os
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import string
|
|
import subprocess
|
|
import sys
|
|
import urllib
|
|
import urllib2
|
|
import zipfile
|
|
|
|
def run_cmd(cmd, stdin=None):
|
|
"""Run the command on the command line in the shell. We print the output of
|
|
the command.
|
|
"""
|
|
print cmd
|
|
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
|
|
stdin=subprocess.PIPE, shell=True)
|
|
output, stderr = p.communicate(input=stdin)
|
|
if output:
|
|
print output
|
|
if stderr:
|
|
print stderr
|
|
|
|
def parse_args():
|
|
parser = optparse.OptionParser()
|
|
parser.add_option('--firefox', '-f', dest='firefox',
|
|
help="Don't install Firefox", action='store_true', default=False)
|
|
parser.add_option('--opera', '-o', dest='opera', default=False,
|
|
help="Don't install Opera", action='store_true')
|
|
parser.add_option('--chromedriver', '-c', dest='chromedriver',
|
|
help="Don't install chromedriver.", action='store_true', default=False)
|
|
parser.add_option('--iedriver', '-i', dest='iedriver',
|
|
help="Don't install iedriver (only used on Windows).",
|
|
action='store_true', default=False)
|
|
parser.add_option('--seleniumrc', '-s', dest='seleniumrc',
|
|
help="Don't install the Selenium RC server (used for Safari and Opera "
|
|
"tests).", action='store_true', default=False)
|
|
parser.add_option('--python', '-p', dest='python',
|
|
help="Don't install Selenium python bindings.", action='store_true',
|
|
default=False)
|
|
parser.add_option('--buildbot', '-b', dest='buildbot', action='store_true',
|
|
help='Perform a buildbot selenium setup (buildbots have a different' +
|
|
'location for their python executable).', default=False)
|
|
args, _ = parser.parse_args()
|
|
return args
|
|
|
|
def find_depot_tools_location(is_buildbot):
|
|
"""Depot_tools is our default install location for chromedriver, so we find
|
|
its location on the filesystem.
|
|
Arguments:
|
|
is_buildbot - True if we are running buildbot machine setup (we can't detect
|
|
this automatically because this script is not run at build time).
|
|
"""
|
|
if is_buildbot:
|
|
depot_tools = os.sep + os.path.join('b', 'depot_tools')
|
|
if 'win32' in sys.platform or 'cygwin' in sys.platform:
|
|
depot_tools = os.path.join('e:', depot_tools)
|
|
return depot_tools
|
|
else:
|
|
path = os.environ['PATH'].split(os.pathsep)
|
|
for loc in path:
|
|
if 'depot_tools' in loc:
|
|
return loc
|
|
raise Exception("Could not find depot_tools in your path.")
|
|
|
|
class GoogleBasedInstaller(object):
|
|
"""Install a project from a Google source, pulling latest version."""
|
|
|
|
def __init__(self, project_name, destination, download_path_func):
|
|
"""Create an object that will install the project.
|
|
Arguments:
|
|
project_name - Google code name of the project, such as "selenium" or
|
|
"chromedriver."
|
|
destination - Where to download the desired file on our filesystem.
|
|
download_path_func - A function that takes a dictionary (currently with keys
|
|
"os" and "version", but more can be added) that calculates the string
|
|
representing the path of the download we want.
|
|
"""
|
|
self.project_name = project_name
|
|
self.destination = destination
|
|
self.download_path_func = download_path_func
|
|
|
|
@property
|
|
def get_os_str(self):
|
|
"""The strings to indicate what OS a download is for."""
|
|
os_str = 'win'
|
|
if 'darwin' in sys.platform:
|
|
os_str = 'mac'
|
|
elif 'linux' in sys.platform:
|
|
os_str = 'linux32'
|
|
if '64bit' in platform.architecture()[0]:
|
|
os_str = 'linux64'
|
|
if self.project_name == 'chromedriver' and (
|
|
os_str == 'mac' or os_str == 'win'):
|
|
os_str += '32'
|
|
return os_str
|
|
|
|
def run(self):
|
|
"""Download and install the project."""
|
|
print 'Installing %s' % self.project_name
|
|
os_str = self.get_os_str
|
|
version = self.find_latest_version()
|
|
download_path = self.download_path_func({'os': os_str, 'version': version})
|
|
download_name = os.path.basename(download_path)
|
|
urllib.urlretrieve(os.path.join(self.source_path(), download_path),
|
|
os.path.join(self.destination, download_name))
|
|
if download_name.endswith('.zip'):
|
|
if platform.system() != 'Windows':
|
|
# The Python zip utility does not preserve executable permissions, but
|
|
# this does not seem to be a problem for Windows, which does not have a
|
|
# built in zip utility. :-/
|
|
run_cmd('unzip -u %s -d %s' % (os.path.join(self.destination,
|
|
download_name), self.destination), stdin='y')
|
|
else:
|
|
z = zipfile.ZipFile(os.path.join(self.destination, download_name))
|
|
z.extractall(self.destination)
|
|
z.close()
|
|
os.remove(os.path.join(self.destination, download_name))
|
|
chrome_path = os.path.join(os.path.dirname(os.path.abspath(__file__)),
|
|
'orig-chromedriver')
|
|
if self.project_name == 'chromedriver' and os.path.exists(chrome_path):
|
|
# We have one additional location to make sure chromedriver is updated.
|
|
# TODO(efortuna): Remove this. See move_chrome_driver_if_needed in
|
|
# perf_testing/run_perf_tests.py
|
|
driver = 'chromedriver'
|
|
if platform.system() == 'Windows':
|
|
driver += '.exe'
|
|
shutil.copy(os.path.join(self.destination, driver),
|
|
os.path.join(chrome_path, driver))
|
|
|
|
class ChromeDriverInstaller(GoogleBasedInstaller):
|
|
"""Install chromedriver from Google Storage."""
|
|
|
|
def __init__(self, destination):
|
|
"""Create an object to install ChromeDriver
|
|
destination - Where to download the desired file on our filesystem.
|
|
"""
|
|
super(ChromeDriverInstaller, self).__init__('chromedriver', destination,
|
|
lambda x: '%(version)s/chromedriver_%(os)s.zip' % x)
|
|
|
|
def find_latest_version(self):
|
|
"""Find the latest version number of ChromeDriver."""
|
|
source_page = urllib2.urlopen(self.source_path())
|
|
source_text = source_page.read()
|
|
regex = re.compile('(?:<Key>)(\d+\.\d+)')
|
|
latest = max(regex.findall(source_text))
|
|
return latest
|
|
|
|
def source_path(self):
|
|
return 'http://chromedriver.storage.googleapis.com'
|
|
|
|
class GoogleCodeInstaller(GoogleBasedInstaller):
|
|
"""Install a project from Google Code."""
|
|
|
|
def google_code_downloads_page(self):
|
|
return 'http://code.google.com/p/%s/downloads/list' % self.project_name
|
|
|
|
def find_latest_version(self):
|
|
"""Find the latest version number of some code available for download on a
|
|
Google code page. This was unfortunately done in an ad hoc manner because
|
|
Google Code does not seem to have an API for their list of current
|
|
downloads(!).
|
|
"""
|
|
google_code_site = self.google_code_downloads_page()
|
|
f = urllib2.urlopen(google_code_site)
|
|
latest = ''
|
|
|
|
download_regex_str = self.download_path_func({'os': self.get_os_str,
|
|
'version': '.+'})
|
|
|
|
for line in f.readlines():
|
|
if re.search(download_regex_str, line):
|
|
suffix_index = line.find(
|
|
download_regex_str[download_regex_str.rfind('.'):])
|
|
name_end = download_regex_str.rfind('.+')
|
|
name = self.download_path_func({'os': self.get_os_str, 'version': ''})
|
|
name = name[:name.rfind('.')]
|
|
version_str = line[line.find(name) + len(name) : suffix_index]
|
|
orig_version_str = version_str
|
|
if version_str.count('.') == 0:
|
|
version_str = version_str.replace('_', '.')
|
|
version_str = re.compile(r'[^\d.]+').sub('', version_str)
|
|
if latest == '':
|
|
latest = '0.' * version_str.count('.')
|
|
latest += '0'
|
|
orig_latest_str = latest
|
|
else:
|
|
orig_latest_str = latest
|
|
latest = latest.replace('_', '.')
|
|
latest = re.compile(r'[^\d.]+').sub('', latest)
|
|
nums = version_str.split('.')
|
|
latest_nums = latest.split('.')
|
|
for (num, latest_num) in zip(nums, latest_nums):
|
|
if int(num) > int(latest_num):
|
|
latest = orig_version_str
|
|
break
|
|
else:
|
|
latest = orig_latest_str
|
|
if latest == '':
|
|
raise Exception("Couldn't find the desired download on " + \
|
|
' %s.' % google_code_site)
|
|
return latest
|
|
|
|
def source_path(self):
|
|
return 'http://%s.googlecode.com/files/' % self.project_name
|
|
|
|
|
|
class FirefoxInstaller(object):
|
|
"""Installs the latest version of Firefox on the machine."""
|
|
|
|
def ff_download_site(self, os_name):
|
|
return 'http://releases.mozilla.org/pub/mozilla.org/firefox/releases/' + \
|
|
'latest/%s/en-US/' % os_name
|
|
|
|
@property
|
|
def get_os_str(self):
|
|
"""Returns the string that Mozilla uses to denote which operating system a
|
|
Firefox binary is for."""
|
|
os_str = ('win32', '.exe')
|
|
if 'darwin' in sys.platform:
|
|
os_str = ('mac', '.dmg')
|
|
elif 'linux' in sys.platform:
|
|
os_str = ('linux-i686', '.tar.bz2')
|
|
if '64bit' in platform.architecture()[0]:
|
|
os_str = ('linux-x86_64', '.tar.bz2')
|
|
return os_str
|
|
|
|
def get_download_url(self):
|
|
"""Parse the html on the page to determine what is the latest download
|
|
appropriate for our system."""
|
|
f = urllib2.urlopen(self.ff_download_site(self.get_os_str[0]))
|
|
download_name = ''
|
|
for line in f.readlines():
|
|
suffix = self.get_os_str[1]
|
|
if (suffix + '"') in line:
|
|
link_str = '<a href="'
|
|
download_name = line[line.find(link_str) + len(link_str) : \
|
|
line.find(suffix) + len(suffix)]
|
|
break
|
|
return '%s%s' % (self.ff_download_site(self.get_os_str[0]), download_name)
|
|
|
|
def run(self):
|
|
print 'Installing Firefox'
|
|
if 'darwin' in sys.platform:
|
|
urllib.urlretrieve(self.get_download_url(), 'firefox.dmg')
|
|
run_cmd('hdiutil mount firefox.dmg')
|
|
run_cmd('sudo cp -R /Volumes/firefox/Firefox.app /Applications')
|
|
run_cmd('hdiutil unmount /Volumes/firefox/')
|
|
elif 'win' in sys.platform:
|
|
urllib.urlretrieve(self.get_download_url(), 'firefox_install.exe')
|
|
run_cmd('firefox_install.exe -ms')
|
|
else:
|
|
run_cmd('wget -O - %s | tar -C ~ -jxv' % self.get_download_url())
|
|
|
|
|
|
class SeleniumBindingsInstaller(object):
|
|
"""Install the Selenium Webdriver bindings for Python."""
|
|
|
|
SETUPTOOLS_SITE = 'http://python-distribute.org/distribute_setup.py'
|
|
PIP_SITE = 'https://raw.github.com/pypa/pip/master/contrib/get-pip.py'
|
|
def __init__(self, is_buildbot):
|
|
self.is_buildbot = is_buildbot
|
|
|
|
def run(self):
|
|
print 'Installing Selenium Python Bindings'
|
|
admin_keyword = ''
|
|
python_cmd = 'python'
|
|
pip_cmd = 'pip'
|
|
if 'win32' not in sys.platform and 'cygwin' not in sys.platform:
|
|
admin_keyword = 'sudo'
|
|
pip_cmd = '/usr/local/bin/pip'
|
|
else:
|
|
# The python installation is "special" on Windows buildbots.
|
|
if self.is_buildbot:
|
|
python_loc = os.path.join(
|
|
find_depot_tools_location(self.is_buildbot), 'python_bin')
|
|
python_cmd = os.path.join(python_loc, 'python')
|
|
pip_cmd = os.path.join(python_loc, 'Scripts', pip_cmd)
|
|
else:
|
|
path = os.environ['PATH'].split(os.pathsep)
|
|
for loc in path:
|
|
if 'python' in loc or 'Python' in loc:
|
|
pip_cmd = os.path.join(loc, 'Scripts', pip_cmd)
|
|
break
|
|
page = urllib2.urlopen(self.SETUPTOOLS_SITE)
|
|
run_cmd('%s %s' % (admin_keyword, python_cmd), page.read())
|
|
page = urllib2.urlopen(self.PIP_SITE)
|
|
run_cmd('%s %s' % (admin_keyword, python_cmd), page.read())
|
|
run_cmd('%s %s install -U selenium' % (admin_keyword, pip_cmd))
|
|
|
|
class OperaHtmlParser(HTMLParser.HTMLParser):
|
|
"""A helper class to parse Opera pages listing available downloads to find the
|
|
correct download we want."""
|
|
|
|
def initialize(self, rejection_func, accept_func):
|
|
"""Initialize some state for our parser.
|
|
Arguments:
|
|
rejection_func: A function that accepts the value of the URL and determines
|
|
if it is of the type we are looking for.
|
|
accept_func: A function that takes the URL and the "current best" URL and
|
|
determines if it is better than our current download url."""
|
|
self.latest = 0
|
|
self.rejection_func = rejection_func
|
|
self.accept_func = accept_func
|
|
|
|
def handle_starttag(self, tag, attrs):
|
|
"""Find the latest version."""
|
|
if (tag == 'a' and attrs[0][0] == 'href' and
|
|
self.rejection_func(attrs[0][1])):
|
|
self.latest = self.accept_func(attrs[0][1], self.latest)
|
|
|
|
class OperaInstaller(object):
|
|
"""Install from the Opera FTP website."""
|
|
|
|
def find_latest_version(self, download_page, rejection_func, accept_func):
|
|
"""Get the latest non-beta version.
|
|
Arguments:
|
|
download_page: The initial page that lists all the download options.
|
|
rejection_func: A function that accepts the value of the URL and determines
|
|
if it is of the type we are looking for.
|
|
accept_func: A function that takes the URL and the "current best" URL and
|
|
determines if it is better than our current download url."""
|
|
f = urllib2.urlopen(download_page)
|
|
parser = OperaHtmlParser()
|
|
parser.initialize(rejection_func, accept_func)
|
|
parser.feed(f.read())
|
|
return str(parser.latest)
|
|
|
|
def run(self):
|
|
"""Download and install Opera."""
|
|
print 'Installing Opera'
|
|
os_str = self.get_os_str
|
|
download_name = 'http://ftp.opera.com/pub/opera/%s/' % os_str
|
|
|
|
def higher_revision(new_version_str, current):
|
|
version_string = new_version_str[:-1]
|
|
if int(version_string) > current:
|
|
return int(version_string)
|
|
return current
|
|
|
|
version = self.find_latest_version(
|
|
download_name,
|
|
lambda x: x[0] in string.digits and 'b' not in x and 'rc' not in x,
|
|
higher_revision)
|
|
download_name += version
|
|
if ('linux' in sys.platform and
|
|
platform.linux_distribution()[0] == 'Ubuntu'):
|
|
# Last time I tried, the .deb file you download directly from opera was
|
|
# not installing correctly on Ubuntu. This installs Opera more nicely.
|
|
os.system("sudo sh -c 'wget -O - http://deb.opera.com/archive.key | "
|
|
"apt-key add -'")
|
|
os.system("""sudo sh -c 'echo "deb http://deb.opera.com/opera/ """
|
|
"""stable non-free" > /etc/apt/sources.list.d/opera.list'""")
|
|
run_cmd('sudo apt-get update')
|
|
run_cmd('sudo apt-get install opera', stdin='y')
|
|
else:
|
|
if 'darwin' in sys.platform:
|
|
dotted_version = '%s.%s' % (version[:2], version[2:])
|
|
download_name += '/Opera_%s_Setup_Intel.dmg' % dotted_version
|
|
urllib.urlretrieve(download_name, 'opera.dmg')
|
|
run_cmd('hdiutil mount opera.dmg', stdin='qY\n')
|
|
run_cmd('sudo cp -R /Volumes/Opera/Opera.app /Applications')
|
|
run_cmd('hdiutil unmount /Volumes/Opera/')
|
|
elif 'win' in sys.platform:
|
|
download_name += '/en/Opera_%s_en_Setup.exe' % version
|
|
urllib.urlretrieve(download_name, 'opera_install.exe')
|
|
run_cmd('opera_install.exe -ms')
|
|
else:
|
|
# For all other flavors of linux, download the tar.
|
|
download_name += '/'
|
|
extension = '.tar.bz2'
|
|
if '64bit' in platform.architecture()[0]:
|
|
platform_str = '.x86_64'
|
|
else:
|
|
platform_str = '.i386'
|
|
def get_acceptable_file(new_version_str, current):
|
|
return new_version_str
|
|
latest = self.find_latest_version(
|
|
download_name,
|
|
lambda x: x.startswith('opera') and x.endswith(extension)
|
|
and platform_str in x,
|
|
get_acceptable_file)
|
|
download_name += latest
|
|
run_cmd('wget -O - %s | tar -C ~ -jxv' % download_name)
|
|
print ('PLEASE MANUALLY RUN "~/%s/install" TO COMPLETE OPERA '
|
|
'INSTALLATION' %
|
|
download_name[download_name.rfind('/') + 1:-len(extension)])
|
|
|
|
@property
|
|
def get_os_str(self):
|
|
"""The strings to indicate what OS a download is."""
|
|
os_str = 'win'
|
|
if 'darwin' in sys.platform:
|
|
os_str = 'mac'
|
|
elif 'linux' in sys.platform:
|
|
os_str = 'linux'
|
|
return os_str
|
|
|
|
def main():
|
|
args = parse_args()
|
|
if not args.python:
|
|
SeleniumBindingsInstaller(args.buildbot).run()
|
|
if not args.chromedriver:
|
|
ChromeDriverInstaller(find_depot_tools_location(args.buildbot)).run()
|
|
if not args.seleniumrc:
|
|
GoogleCodeInstaller('selenium', os.path.dirname(os.path.abspath(__file__)),
|
|
lambda x: 'selenium-server-standalone-%(version)s.jar' % x).run()
|
|
if not args.iedriver and platform.system() == 'Windows':
|
|
GoogleCodeInstaller('selenium', find_depot_tools_location(args.buildbot),
|
|
lambda x: 'IEDriverServer_Win32_%(version)s.zip' % x).run()
|
|
if not args.firefox:
|
|
FirefoxInstaller().run()
|
|
if not args.opera:
|
|
OperaInstaller().run()
|
|
|
|
if __name__ == '__main__':
|
|
main()
|