diff --git a/tools/build.py b/tools/build.py new file mode 100755 index 00000000000..d97746fc1ef --- /dev/null +++ b/tools/build.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. +# + +import optparse +import os +import shutil +import subprocess +import sys +import utils + +HOST_OS = utils.GuessOS() +HOST_CPUS = utils.GuessCpus() +armcompilerlocation = '/opt/codesourcery/arm-2009q1' + +def BuildOptions(): + result = optparse.OptionParser() + result.add_option("-m", "--mode", + help='Build variants (comma-separated).', + metavar='[all,debug,release]', + default='debug') + result.add_option("-v", "--verbose", + help='Verbose output.', + default=False, action="store_true") + result.add_option("--arch", + help='Target architectures (comma-separated).', + metavar='[all,ia32,x64,simarm,arm,dartc]', + default=utils.GuessArchitecture()) + result.add_option("-j", + help='The number of parallel jobs to run.', + metavar=HOST_CPUS, + default=str(HOST_CPUS)) + result.add_option("--devenv", + help='Path containing devenv.com on Windows', + default='C:\\Program Files (x86)\\Microsoft Visual Studio 9.0\\Common7\\IDE') + return result + + +def ProcessOptions(options): + if options.arch == 'all': + options.arch = 'ia32,x64,simarm' + if options.mode == 'all': + options.mode = 'debug,release' + options.mode = options.mode.split(',') + options.arch = options.arch.split(',') + for mode in options.mode: + if not mode in ['debug', 'release']: + print "Unknown mode %s" % mode + return False + for arch in options.arch: + if not arch in ['ia32', 'x64', 'simarm', 'arm', 'dartc']: + print "Unknown arch %s" % arch + return False + return True + + +def setTools(arch): + if arch == 'arm': + toolsOverride = { + "CC" : armcompilerlocation + "/bin/arm-none-linux-gnueabi-gcc", + "CXX" : armcompilerlocation + "/bin/arm-none-linux-gnueabi-g++", + "AR" : armcompilerlocation + "/bin/arm-none-linux-gnueabi-ar", + "LINK": armcompilerlocation + "/bin/arm-none-linux-gnueabi-g++", + "NM" : armcompilerlocation + "/bin/arm-none-linux-gnueabi-nm", + } + return toolsOverride + + +def Execute(args): + print "#" + ' '.join(args) + process = subprocess.Popen(args) + process.wait() + if process.returncode != 0: + raise Error(args[0] + " failed") + + +def CurrentDirectoryBaseName(): + """Returns the name of the current directory""" + return os.path.relpath(os.curdir, start=os.pardir) + +def Main(): + utils.ConfigureJava() + # Parse the options. + parser = BuildOptions() + (options, args) = parser.parse_args() + if not ProcessOptions(options): + parser.print_help() + return 1 + # Determine which targets to build. By default we build the "all" target. + if len(args) == 0: + if HOST_OS == 'macos': + target = 'All' + else: + target = 'all' + else: + target = args[0] + # Build the targets for each requested configuration. + for mode in options.mode: + for arch in options.arch: + build_config = utils.GetBuildConf(mode, arch) + if HOST_OS == 'macos': + project_file = 'dart.xcodeproj' + if os.path.exists('dart-%s.gyp' % CurrentDirectoryBaseName()): + project_file = 'dart-%s.xcodeproj' % CurrentDirectoryBaseName() + args = ['xcodebuild', + '-project', + project_file, + '-target', + target, + '-parallelizeTargets', + '-configuration', + build_config, + 'SYMROOT=%s' % os.path.abspath('xcodebuild') + ] + elif HOST_OS == 'win32': + project_file = 'dart.sln' + if os.path.exists('dart-%s.gyp' % CurrentDirectoryBaseName()): + project_file = 'dart-%s.sln' % CurrentDirectoryBaseName() + args = [options.devenv + os.sep + 'devenv.com', + '/build', + build_config, + project_file + ] + else: + make = 'make' + if HOST_OS == 'freebsd': + make = 'gmake' + # work around lack of flock + os.environ['LINK'] = '$(CXX)' + args = [make, + '-j', + options.j, + 'BUILDTYPE=' + build_config, + ] + if options.verbose: + args += ['V=1'] + + args += [target] + + toolsOverride = setTools(arch) + if toolsOverride: + for k, v in toolsOverride.iteritems(): + args.append( k + "=" + v) + print k + " = " + v + + print ' '.join(args) + process = subprocess.Popen(args) + process.wait() + if process.returncode != 0: + print "BUILD FAILED" + return 1 + + return 0 + + +if __name__ == '__main__': + sys.exit(Main()) diff --git a/tools/compile_java/compile_java.py b/tools/compile_java/compile_java.py new file mode 100644 index 00000000000..bc35bf1b3c8 --- /dev/null +++ b/tools/compile_java/compile_java.py @@ -0,0 +1,204 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +# This python script compiles a set of java files and puts them all into a +# single .jar file. + +import os +import shutil +import sys +import tempfile +from optparse import OptionParser + +# Filters out all arguments until the next '--' argument +# occurs. +def ListArgCallback(option, opt_str, value, parser): + if value is None: + value = [] + + for arg in parser.rargs: + if arg[:2].startswith('--'): + break + value.append(arg) + + del parser.rargs[:len(value)] + setattr(parser.values, option.dest, value) + + +# Compiles all the java source files in srcDir. +def javaCompile(javac, srcDirectories, srcList, classpath, + classOutputDir, buildConfig, javacArgs, + sources): + # Check & prepare directories. + for srcDir in srcDirectories: + if not os.path.exists(srcDir): + sys.stderr.write('source directory not found: ' + srcDir + '\n') + return False + + if os.path.exists(classOutputDir): + shutil.rmtree(classOutputDir) + os.makedirs(classOutputDir) + + # Find all java files and write them in a temp file. + (tempFileDescriptor, javaFilesTempFileName) = tempfile.mkstemp() + javaFilesTempFile = os.fdopen(tempFileDescriptor, "w") + try: + if srcDirectories: + def findJavaFiles(arg, dirName, names): + for fileName in names: + (base, ext) = os.path.splitext(fileName) + if ext == '.java': + javaFilesTempFile.write(os.path.join(dirName, fileName) + '\n') + for srcDir in srcDirectories: + os.path.walk(srcDir, findJavaFiles, None) + + if srcList: + f = open(srcList, 'r') + for line in f: + javaFilesTempFile.write(os.path.abspath(line)) + f.close() + + javaFilesTempFile.flush() + javaFilesTempFile.close() + + # Prepare javac command. + command = [javac] + + # Use a large enough heap to be able to compile all of the classes in one + # big compilation step. + command.append('-J-Xmx256m') + + if buildConfig == 'Debug': + command.append('-g') + + if srcDirectories: + command.append('-sourcepath') + command.append(os.pathsep.join(srcDirectories)) + + if classpath: + command.append('-classpath') + command.append(os.pathsep.join(classpath)) + + command.append('-d') + command.append(classOutputDir) + + if srcDirectories or srcList: + command.append('@' + javaFilesTempFileName) + + command = command + javacArgs + + abs_sources = [os.path.abspath(source) for source in sources] + + command = [' '.join(command)] + abs_sources + + # Compile. + sys.stdout.write(' \\\n '.join(command) + '\n') + if os.system(' '.join(command)): + sys.stderr.write('java compilation failed\n') + return False + return True + finally: + os.remove(javaFilesTempFileName) + +def copyProperties(propertyFiles, classOutputDir): + for file in propertyFiles: + if not os.path.isfile(file): + sys.stderr.write('property file not found: ' + file + '\n') + return False + + if not os.path.exists(classOutputDir): + sys.stderr.write('classes dir not found: ' + classOutputDir + '\n') + return False + + if propertyFiles == []: + return True + + command = ['cp'] + propertyFiles + [classOutputDir] + commandStr = ' '.join(command) + sys.stdout.write(commandStr + '\n') + if os.system(commandStr): + sys.stderr.write('property file copy failed\n') + return False + return True + +def createJar(classOutputDir, jarFileName): + if not os.path.exists(classOutputDir): + sys.stderr.write('classes dir not found: ' + classOutputDir + '\n') + return False + + command = ['jar', 'cf', jarFileName, '-C', classOutputDir, '.'] + commandStr = ' '.join(command) + sys.stdout.write(commandStr + '\n') + if os.system(commandStr): + sys.stderr.write('jar creation failed\n') + return False + return True + + +def main(args): + try: + # Parse input. + parser = OptionParser() + parser.add_option("--javac", default="javac", + action="store", type="string", + help="location of javac command") + parser.add_option("--sourceDir", dest="sourceDirs", default=[], + action="callback", callback=ListArgCallback, + help="specify a list of directories to look for " + + ".java files to compile") + parser.add_option("--sources", dest="sources", default=[], + action="callback", callback=ListArgCallback, + help="specify a list of source files to compile") + parser.add_option("--sourceList", dest="sourceList", + action="store", type="string", + help="specify the file that contains the list of Java source files") + parser.add_option("--classpath", dest="classpath", default=[], + action="append", type="string", + help="specify referenced jar files") + parser.add_option("--classesDir", + action="store", type="string", + help="location of intermediate .class files") + parser.add_option("--jarFileName", + action="store", type="string", + help="name of the output jar file") + parser.add_option("--buildConfig", + action="store", type="string", default='Release', + help="Debug or Release") + parser.add_option("--javacArgs", dest="javacArgs", default=[], + action="callback", callback=ListArgCallback, + help="remaining args are passed directly to javac") + parser.add_option("--propertyFiles", dest="propertyFiles", default=[], + action="callback", callback=ListArgCallback, + help="specify a list of property files to copy") + + (options, args) = parser.parse_args() + if not options.classesDir: + sys.stderr.write('--classesDir not specified\n') + return -1 + if not options.jarFileName: + sys.stderr.write('--jarFileName not specified\n') + return -1 + if len(options.sourceDirs) > 0 and options.sourceList: + sys.stderr.write("--sourceDir and --sourceList cannot be both specified"); + return -1 + + # Compile and put into .jar file. + if not javaCompile(options.javac, options.sourceDirs, + options.sourceList, options.classpath, + options.classesDir, options.buildConfig, + options.javacArgs, options.sources): + return -1 + if not copyProperties(options.propertyFiles, options.classesDir): + return -1 + if not createJar(options.classesDir, options.jarFileName): + return -1 + + return 0 + except Exception, inst: + sys.stderr.write('compile_java.py exception\n') + sys.stderr.write(str(inst)) + return -1 + +if __name__ == '__main__': + sys.exit(main(sys.argv)) diff --git a/tools/dartc b/tools/dartc new file mode 100644 index 00000000000..19cc6bdbd70 --- /dev/null +++ b/tools/dartc @@ -0,0 +1,20 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import os +import subprocess +import sys + +import utils + +# Figure out the full path to the real dashc from the cwd. +scriptPath = os.path.dirname(__file__) +buildRoot = utils.GetBuildRoot(utils.GuessOS()) +dashcPath = os.path.join(scriptPath, "../compiler", buildRoot, + "Debug/ant-out/dist/bin/dashc") + +args = [dashcPath] + sys.argv[1:] +sys.exit(subprocess.call(args)) diff --git a/tools/generate_projects.py b/tools/generate_projects.py new file mode 100755 index 00000000000..a2f2cae0483 --- /dev/null +++ b/tools/generate_projects.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import os +import sys +import platform + +def NormJoin(path1, path2): + return os.path.normpath(os.path.join(path1, path2)) + + +def GetProjectGypFile(project_name): + project_file = os.path.join(project_name, 'dart-%s.gyp' % project_name) + if not os.path.exists(project_file): + sys.stderr.write('%s does not exist\n' % project_file) + project_file = os.path.join(project_name, 'dart.gyp') + return project_file + +dart_src = NormJoin(os.path.dirname(sys.argv[0]), os.pardir) +project_src = sys.argv[1] +gyp_pylib = os.path.join(dart_src, 'third_party', 'gyp', 'pylib') + +if __name__ == '__main__': + # If this script is invoked with the -fmake option, we assume that + # it is being run automatically via a project Makefile. That can be + # problematic (because GYP is really bad at setting up the paths + # correctly), so we try to run "gclient runhooks" instead. + if '-fmake' in sys.argv: + try: + sys.exit(os.execvp("gclient", ["gclient", "runhooks"])) + except OSError: + # Sometimes gclient is not on the PATH. Let the user know that + # he must run "gclient runhooks" manually. + sys.stderr.write('Error: GYP files are out of date.\n') + sys.stderr.write('\n\n*** Please run "gclient runhooks" ***\n\n\n') + sys.exit(1) + + +# Add gyp to the imports and if needed get it from the third_party location +# inside the standalone dart gclient checkout. +try: + import gyp +except ImportError, e: + sys.path.append(os.path.abspath(gyp_pylib)) + import gyp + + +if __name__ == '__main__': + # Make our own location absolute as it is stored in Makefiles. + sys.argv[0] = os.path.abspath(sys.argv[0]) + + # Add any extra arguments. Needed by compiler/dart.gyp to build v8. + args = sys.argv[2:] + + args += ['--depth', project_src] + args += ['-I', './tools/gyp/common.gypi'] + if platform.system() == 'Linux': + # We need to fiddle with toplevel-dir to work around a GYP bug + # that breaks building v8 from compiler/dart.gyp. + args += ['--toplevel-dir', os.curdir] + args += ['--generator-output', project_src] + else: + # On at least the Mac, the toplevel-dir should be where the + # sources are. Otherwise, Xcode won't show sources correctly. + args += ['--toplevel-dir', project_src] + + # Change into the dart directory as we want the project to be rooted here. + # Also, GYP is very sensitive to exacly from where it is being run. + os.chdir(dart_src) + + args += [GetProjectGypFile(project_src)] + + # Generate the projects. + sys.exit(gyp.main(args)) diff --git a/tools/get_dartium.py b/tools/get_dartium.py new file mode 100755 index 00000000000..cea805a6d9f --- /dev/null +++ b/tools/get_dartium.py @@ -0,0 +1,145 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +# Gets or updates our local build of Dartium. This is used for testing Dart +# apps, without the need to build Dartium locally + + +import os +import sys +import platform +import errno +import tempfile +import shutil +import subprocess + +def NormJoin(path1, path2): + return os.path.normpath(os.path.join(path1, path2)) + +# Change into the dart directory as we want the project to be rooted here. +dart_src = NormJoin(os.path.dirname(sys.argv[0]), os.pardir) +os.chdir(dart_src) + +GSUTIL_DIR = 'third_party/gsutil/20110627' +GSUTIL = GSUTIL_DIR + '/gsutil' +DARTIUM_DIR = 'client/tests/dartium' +VERSION = DARTIUM_DIR + '/BUILD_VERSION' + +sys.path.append(GSUTIL_DIR + '/boto') +import boto + + +def execute_command(*cmd): + """Execute a command in a subprocess.""" + pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + output, error = pipe.communicate() + if pipe.returncode != 0: + raise Exception('Execution of "%s" failed: %s' % (cmd, str(output))) + return output + + +def execute_command_visible(*cmd): + """Execute a command in a subprocess, but show stdout/stderr.""" + result = subprocess.call(cmd, stdout=sys.stdout, stderr=sys.stderr, + stdin=sys.stdin) + if result != 0: + raise Exception('Execution of "%s" failed' % cmd) + + +def gsutil(*cmd): + return execute_command(GSUTIL, *cmd) + + +def gsutil_visible(*cmd): + execute_command_visible(GSUTIL, *cmd) + + +def has_boto_config(): + """Returns true if boto config exists.""" + + config_paths = boto.pyami.config.BotoConfigLocations + if 'AWS_CREDENTIAL_FILE' in os.environ: + config_paths.append(os.environ['AWS_CREDENTIAL_FILE']) + for config_path in config_paths: + if os.path.exists(config_path): + return True + + return False + + +def in_runhooks(): + '''True if this script was called by "gclient runhooks" or "gclient sync"''' + return 'runhooks' in sys.argv + + +def ensure_config(): + # If ~/.boto doesn't exist, tell the user to run "gsutil config" + if not has_boto_config(): + print >>sys.stderr, ''' +******************************************************************************* +* WARNING: Can't download Dartium binaries! These are required to test client. +* You need to do a one-time configuration step to access Google Storage. +* Please run this command and follow the instructions: +* %s config +* +* NOTE: When prompted you can leave "project-id" blank. Just hit enter. +******************************************************************************* +''' % GSUTIL + sys.exit(1) + + +def main(): + system = platform.system() + if system == 'Darwin': + osname = 'mac' + elif system == 'Linux': + osname = 'lucid64' + else: + print >>sys.stderr, ('WARNING: platform "%s" does not support' + 'DumpRenderTree for tests') % system + return 1 + + ensure_config() + + # Query the last known good build + latest = gsutil('ls', 'gs://dashium-archive/latest/dashium-%s-full-*.zip' % + osname).split()[-1] + + # Check if we need to update the file + if os.path.exists(VERSION): + v = open(VERSION, 'r').read() + if v == latest: + if not in_runhooks(): + print 'Dartium is up to date.\nVersion: ' + latest + return 0 # up to date + + if os.path.exists(DARTIUM_DIR): + print 'Removing old dartium tree %s' % DARTIUM_DIR + shutil.rmtree(DARTIUM_DIR) + + # download the zip file to a temporary path, and unzip to the target location + temp_dir = tempfile.mkdtemp() + try: + temp_zip = temp_dir + '/dashium.zip' + # It's nice to show download progress + gsutil_visible('cp', latest, temp_zip) + + execute_command('unzip', temp_zip, '-d', temp_dir) + unzipped_dir = temp_dir + '/' + os.path.basename(latest)[:-4] # remove .zip + shutil.move(unzipped_dir, DARTIUM_DIR) + finally: + shutil.rmtree(temp_dir) + + # create the version stamp + v = open(VERSION, 'w') + v.write(latest) + v.close() + + print 'Successfully downloaded to %s' % DARTIUM_DIR + return 0 + +if __name__ == '__main__': + sys.exit(main()) diff --git a/tools/gyp/all.gypi b/tools/gyp/all.gypi new file mode 100644 index 00000000000..63e9e75379a --- /dev/null +++ b/tools/gyp/all.gypi @@ -0,0 +1,28 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +# A set of variables needed to build some of the Chrome based subparts of the +# Dart project (e.g. V8). This is in no way a complete list of variables being +# defined by Chrome, but just the minimally needed subset. +{ + 'variables': { + 'library': 'static_library', + 'component': 'static_library', + 'host_arch': 'ia32', + 'target_arch': 'ia32', + 'v8_location': '<(DEPTH)/third_party/v8', + }, + 'conditions': [ + [ 'OS=="linux"', { + 'target_defaults': { + 'ldflags': [ '-pthread', ], + }, + }], + ], + 'includes': [ + 'xcode.gypi', + 'configurations.gypi', + 'source_filter.gypi', + ], +} diff --git a/tools/gyp/common.gypi b/tools/gyp/common.gypi new file mode 100644 index 00000000000..3577f922002 --- /dev/null +++ b/tools/gyp/common.gypi @@ -0,0 +1,28 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +# A set of variables needed to build some of the Chrome based subparts of the +# Dash project (e.g. V8). This is in no way a complete list of variables being +# defined by Chrome, but just the minimally needed subset. +{ + 'variables': { + 'library': 'static_library', + 'component': 'static_library', + 'host_arch': 'ia32', + 'target_arch': 'ia32', + 'v8_location': '<(DEPTH)/../third_party/v8', + }, + 'conditions': [ + [ 'OS=="linux"', { + 'target_defaults': { + 'ldflags': [ '-pthread', ], + }, + }], + ], + 'includes': [ + 'xcode.gypi', + 'configurations.gypi', + 'source_filter.gypi', + ], +} diff --git a/tools/gyp/configurations.gypi b/tools/gyp/configurations.gypi new file mode 100644 index 00000000000..051ac7f2bda --- /dev/null +++ b/tools/gyp/configurations.gypi @@ -0,0 +1,126 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +{ + 'variables': { + 'common_gcc_warning_flags': [ + '-Wall', + '-Wextra', # Also known as -W. + '-Wno-unused-parameter', + # TODO(v8-team): Fix V8 build. + #'-Wold-style-cast', + ], + + # Default value. This may be overridden in a containing project gyp. + 'target_arch%': 'ia32', + }, + 'conditions': [ + [ 'OS=="linux"', { 'includes': [ 'configurations_make.gypi', ], } ], + [ 'OS=="mac"', { 'includes': [ 'configurations_xcode.gypi', ], } ], + [ 'OS=="win"', { 'includes': [ 'configurations_msvs.gypi', ], } ], + ], + 'target_defaults': { + 'default_configuration': 'Debug_ia32', + 'configurations': { + 'Dart_Base': { + 'abstract': 1, + }, + + 'Dart_ia32_Base': { + 'abstract': 1, + }, + + 'Dart_x64_Base': { + 'abstract': 1, + }, + + 'Dart_simarm_Base': { + 'abstract': 1, + 'defines': [ + 'TARGET_ARCH_ARM', + ] + }, + + 'Dart_arm_Base': { + 'abstract': 1, + 'defines': [ + 'TARGET_ARCH_ARM', + ], + }, + + 'Dart_Debug': { + 'abstract': 1, + }, + + 'Dart_Release': { + 'abstract': 1, + 'defines': [ + 'NDEBUG', + ], + }, + + 'Debug_ia32': { + 'inherit_from': ['Dart_Base', 'Dart_ia32_Base', 'Dart_Debug'], + }, + + 'Release_ia32': { + 'inherit_from': ['Dart_Base', 'Dart_ia32_Base', 'Dart_Release'], + }, + + 'Debug_x64': { + 'inherit_from': ['Dart_Base', 'Dart_x64_Base', 'Dart_Debug'], + }, + + 'Release_x64': { + 'inherit_from': ['Dart_Base', 'Dart_x64_Base', 'Dart_Release'], + }, + + 'Debug_simarm': { + # Should not inherit from Dart_Debug because Dart_simarm_Base defines + # the optimization level to be -O3, as the simulator runs too slow + # otherwise. + 'inherit_from': ['Dart_Base', 'Dart_simarm_Base'], + 'defines': [ + 'DEBUG', + ], + }, + + 'Release_simarm': { + # Should not inherit from Dart_Release (see Debug_simarm). + 'inherit_from': ['Dart_Base', 'Dart_simarm_Base'], + 'defines': [ + 'NDEBUG', + ], + }, + + 'Debug_arm': { + 'inherit_from': ['Dart_Base', 'Dart_arm_Base', 'Dart_Debug'], + }, + + 'Release_arm': { + 'inherit_from': ['Dart_Base', 'Dart_arm_Base', 'Dart_Release'], + }, + + 'Debug_dartc': { + # If we build any native code (e.g. V8), then we should just use the + # release version. + 'inherit_from': ['Release_ia32'], + }, + + 'Release_dartc': { + 'inherit_from': ['Release_ia32'], + }, + + # These targets assume that target_arch is passed in explicitly + # by the containing project (e.g., chromium). + 'Debug': { + 'inherit_from': ['Debug_<(target_arch)'] + }, + + 'Release': { + 'inherit_from': ['Release_<(target_arch)'] + }, + }, + }, +} diff --git a/tools/gyp/configurations_make.gypi b/tools/gyp/configurations_make.gypi new file mode 100644 index 00000000000..2a956a76166 --- /dev/null +++ b/tools/gyp/configurations_make.gypi @@ -0,0 +1,68 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +{ + 'variables': { + 'arm_cross_libc%': '/opt/codesourcery/arm-2009q1/arm-none-linux-gnueabi/libc', + }, + 'target_defaults': { + 'configurations': { + 'Dart_Base': { + 'cflags': [ + '-Werror', + '<@(common_gcc_warning_flags)', + '-Wnon-virtual-dtor', + # TODO(v8-team): Fix V8 build. + #'-Woverloaded-virtual', + '-g', + '-ansi', + '-fno-rtti', + '-fno-exceptions', + '-fPIC', + '-fvisibility=hidden', + '-fvisibility-inlines-hidden', + ], + 'ldflags': [ + '-rdynamic', + '-Wl,-rpath,<(PRODUCT_DIR)/lib.target', + ], + }, + + 'Dart_ia32_Base': { + 'cflags': [ '-m32', ], + 'ldflags': [ '-m32', ], + }, + + 'Dart_x64_Base': { + 'cflags': [ '-m64', ], + 'ldflags': [ '-m64', ], + }, + + 'Dart_simarm_Base': { + 'cflags': [ '-O3', '-m32', ], + 'ldflags': [ '-m32', ], + }, + + 'Dart_arm_Base': { + 'cflags': [ + '-march=armv7-a', + '-mfpu=vfp', + '-mfloat-abi=softfp', + '-fno-strict-overflow', + ], + 'ldflags': [ + '-Wl,-rpath=<(arm_cross_libc)/usr/lib', + ], + }, + + 'Dart_Debug': { + 'cflags': [ '-O0', ], + }, + + 'Dart_Release': { + 'cflags': [ '-O3', ], + }, + }, + }, +} diff --git a/tools/gyp/configurations_msvs.gypi b/tools/gyp/configurations_msvs.gypi new file mode 100644 index 00000000000..49a2d64b1d7 --- /dev/null +++ b/tools/gyp/configurations_msvs.gypi @@ -0,0 +1,46 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +{ + 'target_defaults': { + 'configurations': { + 'Dart_Debug': { + 'msvs_settings': { + 'VCCLCompilerTool': { + 'Optimization': '0', + 'DebugInformationFormat': '3', + 'ExceptionHandling': '0', + 'RuntimeTypeInfo': 'false', + }, + 'VCLinkerTool': { + 'LinkIncremental': '2', + 'GenerateDebugInformation': 'true', + 'StackReserveSize': '2097152', + }, + }, + }, + + 'Dart_Release': { + 'msvs_settings': { + 'VCCLCompilerTool': { + 'Optimization': '2', + 'InlineFunctionExpansion': '2', + 'EnableIntrinsicFunctions': 'true', + 'FavorSizeOrSpeed': '0', + 'ExceptionHandling': '0', + 'RuntimeTypeInfo': 'false', + 'OmitFramePointers': 'true', + 'StringPooling': 'true', + }, + 'VCLinkerTool': { + 'LinkIncremental': '1', + 'OptimizeReferences': '2', + 'EnableCOMDATFolding': '2', + 'StackReserveSize': '2097152', + }, + }, + }, + }, + }, +} diff --git a/tools/gyp/configurations_xcode.gypi b/tools/gyp/configurations_xcode.gypi new file mode 100644 index 00000000000..11061916511 --- /dev/null +++ b/tools/gyp/configurations_xcode.gypi @@ -0,0 +1,79 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +{ + 'target_defaults': { + 'configurations': { + 'Dart_Base': { + 'xcode_settings': { + # To switch to the LLVM based backend change the two lines below. + #'GCC_VERSION': 'com.apple.compilers.llvmgcc42', + 'GCC_VERSION': '4.2', + 'GCC_C_LANGUAGE_STANDARD': 'ansi', + 'GCC_ENABLE_CPP_EXCEPTIONS': 'NO', # -fno-exceptions + 'GCC_ENABLE_CPP_RTTI': 'NO', # -fno-rtti + 'GCC_DEBUGGING_SYMBOLS': 'default', # -g + 'GCC_GENERATE_DEBUGGING_SYMBOLS': 'YES', # Do not strip symbols + 'GCC_SYMBOLS_PRIVATE_EXTERN': 'YES', # -fvisibility=hidden + 'GCC_INLINES_ARE_PRIVATE_EXTERN': 'YES', # -fvisibility-inlines-hidden + 'GCC_WARN_NON_VIRTUAL_DESTRUCTOR': 'YES', # -Wnon-virtual-dtor + # TODO(v8-team): Fix V8 build. + #'GCC_WARN_HIDDEN_VIRTUAL_FUNCTIONS': 'YES', # -Woverloaded-virtual + 'GCC_TREAT_WARNINGS_AS_ERRORS': 'YES', # -Werror + 'WARNING_CFLAGS': [ + '<@(common_gcc_warning_flags)', + '-Wtrigraphs', # Disable Xcode default. + ], + + # Generate PIC code as Chrome is switching to this. + 'GCC_DYNAMIC_NO_PIC': 'NO', + + # When searching for header files, do not search + # subdirectories. Without this option, vm/assert.h conflicts + # with the system header assert.h. Apple also recommend + # setting this to NO. + 'ALWAYS_SEARCH_USER_PATHS': 'NO', + + # Attempt to remove compiler options that Xcode adds by default. + 'GCC_CW_ASM_SYNTAX': 'NO', # Remove -fasm-blocks. + + 'GCC_ENABLE_PASCAL_STRINGS': 'NO', + 'GCC_ENABLE_TRIGRAPHS': 'NO', + 'PREBINDING': 'NO', + }, + }, + + 'Dart_ia32_Base': { + 'xcode_settings': { + 'ARCHS': 'i386', + }, + }, + + 'Dart_x64_Base': { + 'xcode_settings': { + 'ARCHS': 'x86_64', + }, + }, + + 'Dart_simarm_Base': { + 'xcode_settings': { + 'ARCHS': 'i386', + 'GCC_OPTIMIZATION_LEVEL': '3', + }, + }, + + 'Dart_Debug': { + 'xcode_settings': { + 'GCC_OPTIMIZATION_LEVEL': '0', + }, + }, + + 'Dart_Release': { + 'xcode_settings': { + 'GCC_OPTIMIZATION_LEVEL': '3', + }, + }, + }, + }, +} diff --git a/tools/gyp/source_filter.gypi b/tools/gyp/source_filter.gypi new file mode 100644 index 00000000000..19fba77ee10 --- /dev/null +++ b/tools/gyp/source_filter.gypi @@ -0,0 +1,13 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +{ + 'target_defaults': { + 'conditions': [ + ['OS!="linux"', {'sources/': [['exclude', '_linux\\.(cc|h)$']]}], + ['OS!="mac"', {'sources/': [['exclude', '_macos\\.(cc|h)$']]}], + ['OS!="win"', {'sources/': [['exclude', '_win\\.(cc|h)$']]}], + ], + }, +} diff --git a/tools/gyp/xcode.gypi b/tools/gyp/xcode.gypi new file mode 100644 index 00000000000..26afb4adf2e --- /dev/null +++ b/tools/gyp/xcode.gypi @@ -0,0 +1,56 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +{ + 'variables': { + # Mac OS X SDK and deployment target support. + # The SDK identifies the version of the system headers that will be used, + # and corresponds to the MAC_OS_X_VERSION_MAX_ALLOWED compile-time macro. + # "Maximum allowed" refers to the operating system version whose APIs are + # available in the headers. + # The deployment target identifies the minimum system version that the + # built products are expected to function on. It corresponds to the + # MAC_OS_X_VERSION_MIN_REQUIRED compile-time macro. + # To ensure these macros are available, #include . + # Additional documentation on these macros is available at + # http://developer.apple.com/mac/library/technotes/tn2002/tn2064.html#SECTION3 + # Chrome normally builds with the Mac OS X 10.5 SDK and sets the + # deployment target to 10.5. Other projects, such as O3D, may override + # these defaults. + 'mac_sdk%': '10.5', + 'mac_deployment_target%': '10.5', + }, + 'xcode_settings': { + # DON'T ADD ANYTHING NEW TO THIS BLOCK UNLESS YOU REALLY REALLY NEED IT! + # This block adds *project-wide* configuration settings to each project + # file. It's almost always wrong to put things here. Specify your + # custom xcode_settings in target_defaults to add them to targets instead. + + # In an Xcode Project Info window, the "Base SDK for All Configurations" + # setting sets the SDK on a project-wide basis. In order to get the + # configured SDK to show properly in the Xcode UI, SDKROOT must be set + # here at the project level. + 'SDKROOT': 'macosx<(mac_sdk)', # -isysroot + + # The Xcode generator will look for an xcode_settings section at the root + # of each dict and use it to apply settings on a file-wide basis. Most + # settings should not be here, they should be in target-specific + # xcode_settings sections, or better yet, should use non-Xcode-specific + # settings in target dicts. SYMROOT is a special case, because many other + # Xcode variables depend on it, including variables such as + # PROJECT_DERIVED_FILE_DIR. When a source group corresponding to something + # like PROJECT_DERIVED_FILE_DIR is added to a project, in order for the + # files to appear (when present) in the UI as actual files and not red + # red "missing file" proxies, the correct path to PROJECT_DERIVED_FILE_DIR, + # and therefore SYMROOT, needs to be set at the project level. + 'SYMROOT': '<(DEPTH)/xcodebuild', + + # It is here to make it possible to run binaries with shared libraries. + # However, it may create problems if you move the shared library. Also, + # this may not be appropriate for "release release" builds. + # Perhaps we should set it to @rpath instead. See + # http://developer.apple.com/library/mac/#documentation/DeveloperTools/Conceptual/DynamicLibraries/100-Articles/RunpathDependentLibraries.html#//apple_ref/doc/uid/TP40008306-SW1 + 'INSTALL_PATH': '$(TARGET_BUILD_DIR)', + }, +} diff --git a/tools/make_bundle.py b/tools/make_bundle.py new file mode 100755 index 00000000000..c34baaa881f --- /dev/null +++ b/tools/make_bundle.py @@ -0,0 +1,229 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. +# + +"""Tool for automating creation of a Dart bundle.""" + +import optparse +import os +from os import path +import shutil +import subprocess +import sys + +import utils + + +class BundleMaker(object): + """Main class for building a Dart bundle.""" + + def __init__(self, top_dir=None, dest=None, verbose=False, skip_build=False): + self._top_dir = top_dir + self._dest = dest + self._verbose = verbose + self._skip_build = skip_build + self._os = utils.GuessOS() + self._release_build_root = utils.GetBuildRoot(self._os, mode='release', + arch='ia32') + self._debug_build_root = utils.GetBuildRoot(self._os, mode='debug', + arch='ia32') + self._dartc_build_root = utils.GetBuildRoot(self._os, mode='release', + arch='dartc') + + @staticmethod + def BuildOptions(): + """Make an option parser with the options supported by this tool. + + Returns: + A newly created OptionParser. + """ + op = optparse.OptionParser('usage: %prog [options]') + op.add_option('-d', '--dest') + op.add_option('-v', '--verbose', default=False, action='store_true') + op.add_option('--skip-build', default=False, action='store_true') + return op + + @staticmethod + def CheckOptions(op, top_dir, cmd_line_args): + """Check the command line arguments. + + Args: + op: An OptionParser (see BuildOptions). + top_dir: The top-level source directory. + cmd_line_args: The command line arguments. + + Returns: + A dict with the analyzed options and other values. The dict + includes these keys: + dest: The destition directory for storing the bundle. + verbose: Whether the tool should be verbose. + top_dir: Same as top_dir argument. + skip_build: Whether the tool should skip the build steps. + """ + (options, args) = op.parse_args(args=cmd_line_args) + if args: + # Terminate program. + op.error('extra arguments on command line') + dest = options.dest + if not dest: + dest = path.normpath(path.join(top_dir, 'new_bundle')) + print 'Bundle is saved to %r' % dest + if not path.exists(dest): + os.makedirs(dest) + elif not path.isdir(dest): + # Terminate program. + op.error('%s: is not a directory' % dest) + return { + 'dest': dest, + 'verbose': options.verbose, + 'top_dir': top_dir, + 'skip_build': options.skip_build, + } + + def _PrintConfiguration(self): + for member in [m for m in dir(self) if not m.startswith('_')]: + value = getattr(self, member) + if not callable(value): + print '%s = %r' % (member, value) + + def _GetTool(self, name): + return self._GetLocation('tools', name) + + def _GetLocation(self, *arguments): + location = path.join(self._top_dir, *arguments) + if not path.exists(location): + raise utils.Error('%s: does not exist' % location) + return location + + def _InvokeTool(self, project, name, *arguments): + location = self._GetLocation(project) + tool = path.relpath(self._GetTool(name), location) + command_array = [tool] + for argument in arguments: + command_array.append(str(argument)) + stdout = subprocess.PIPE + if self._verbose: + print 'Invoking', ' '.join(command_array) + print 'in', location + stdout = None # In verbose mode we want to see the output from the tool. + proc = subprocess.Popen(command_array, + cwd=location, + stdout=stdout, + stderr=subprocess.STDOUT) + stdout = proc.communicate()[0] + exit_code = proc.wait() + if exit_code != 0: + sys.stderr.write(stdout) + raise utils.Error('%s returned %s' % (name, exit_code)) + elif self._verbose: + print name, 'returned', exit_code + + def _GetReleaseOutput(self, project, name): + return self._GetLocation(project, self._release_build_root, name) + + def _GetDebugOutput(self, project, name): + return self._GetLocation(project, self._debug_build_root, name) + + def _GetDartcOutput(self, project, name): + return self._GetLocation(project, self._dartc_build_root, name) + + def _GetNativeDest(self, mode, name): + return path.join('native', self._os, utils.GetBuildConf(mode, 'ia32'), name) + + def _EnsureExists(self, artifact): + if not path.exists(artifact): + raise utils.Error('%s: does not exist' % artifact) + + def _BuildArtifacts(self): + if not self._skip_build: + self._InvokeTool('runtime', 'build.py', '--arch=ia32', + '--mode=release,debug') + self._InvokeTool('compiler', 'build.py', '--arch=dartc', '--mode=release') + self._InvokeTool('language', 'build.py', '--arch=ia32', '--mode=release') + + release_vm = self._GetReleaseOutput('runtime', 'dart_bin') + self._EnsureExists(release_vm) + release_vm_dest = self._GetNativeDest('release', 'dart_bin') + + debug_vm = self._GetDebugOutput('runtime', 'dart_bin') + self._EnsureExists(debug_vm) + debug_vm_dest = self._GetNativeDest('debug', 'dart_bin') + + dartc_bundle = self._GetDartcOutput('compiler', 'compiler') + self._EnsureExists(dartc_bundle) + return ( + (self._GetLocation('bundle', 'bin', 'dart'), 'dart', False), + (release_vm, release_vm_dest, True), + (debug_vm, debug_vm_dest, True), + (dartc_bundle, 'compiler', False), + (self._GetLocation('bundle', 'samples'), 'samples', False), + (self._GetLocation('bundle', 'README'), 'README.txt', False), + (self._GetReleaseOutput('language', 'guide'), 'guide', False), + ) + + def _CopyCorelib(self): + def ReadSources(sources, *paths): + p = path.join(*paths) + return [self._GetLocation(p, s) for s in sources if s.endswith('.dart')] + gypi = self._GetLocation('corelib', 'src', 'corelib_sources.gypi') + sources = [] + with open(gypi, 'r') as f: + text = f.read() + sources.extend(ReadSources(eval(text)['sources'], 'corelib', 'src')) + gypi = self._GetLocation('runtime', 'lib', 'lib_sources.gypi') + with open(gypi, 'r') as f: + text = f.read() + sources.extend(ReadSources(eval(text)['sources'], 'runtime', 'lib')) + dest = path.join(self._dest, 'lib', 'core') + if not path.exists(dest): + os.makedirs(dest) + for source in sources: + if self._verbose: + print 'Copying', source, 'to', dest + shutil.copy2(source, dest) + + def MakeBundle(self): + """Build and install all the components of a bundle. + + Returns: + 0 if the bundle was created successfully. + """ + if self._verbose: + self._PrintConfiguration() + for artifact, reldest, strip in self._BuildArtifacts(): + dest = path.join(self._dest, reldest) + if not path.exists(path.dirname(dest)): + os.makedirs(path.dirname(dest)) + if self._verbose: + print 'Copying', artifact, 'to', dest + if path.isdir(artifact): + assert not strip + if path.exists(dest): + shutil.rmtree(dest) + shutil.copytree(artifact, dest) + else: + if strip: + os.system('strip -o %s %s' % (dest, artifact)) + else: + shutil.copy2(artifact, dest) + self._CopyCorelib() + os.system('chmod -R a+rX %s' % self._dest) + return 0 + + +def main(): + top_dir = path.normpath(path.join(path.dirname(sys.argv[0]), os.pardir)) + cmd_line_args = sys.argv[1:] + try: + op = BundleMaker.BuildOptions() + options = BundleMaker.CheckOptions(op, top_dir, cmd_line_args) + except utils.Error: + return 1 + sys.exit(BundleMaker(**options).MakeBundle()) + + +if __name__ == '__main__': + main() diff --git a/tools/make_bundle_unittest.py b/tools/make_bundle_unittest.py new file mode 100755 index 00000000000..6d6ea217a00 --- /dev/null +++ b/tools/make_bundle_unittest.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. +# + +"""Unit tests for make_bundle.py.""" + +import os +from os import path +import shutil +import subprocess +import sys +import tempfile +import unittest + +import make_bundle + + +class BundleMakerTest(unittest.TestCase): + """Unit test class for BundleMaker.""" + + def setUp(self): + self._tempdir = tempfile.mkdtemp() + self._top_dir = path.normpath(path.join(path.dirname(sys.argv[0]), + os.pardir)) + self._dest = path.join(self._tempdir, 'new_bundle') + + def tearDown(self): + shutil.rmtree(self._tempdir) + + def testBuildOptions(self): + op = make_bundle.BundleMaker.BuildOptions() + op.parse_args(args=[]) + + def testCheckOptions(self): + op = make_bundle.BundleMaker.BuildOptions() + options = make_bundle.BundleMaker.CheckOptions(op, self._top_dir, + ['--dest', self._dest]) + self.failUnless(path.exists(self._dest)) + os.rmdir(self._dest) + self.assertEquals(self._dest, options['dest']) + self.assertEquals(self._top_dir, options['top_dir']) + self.failIf(options['verbose']) + self.failIf(options['skip_build']) + options = make_bundle.BundleMaker.CheckOptions(op, self._top_dir, + ['--dest', self._dest, '-v']) + self.failUnless(path.exists(self._dest)) + self.assertEquals(self._dest, options['dest']) + self.assertEquals(self._top_dir, options['top_dir']) + self.failUnless(options['verbose']) + self.failIf(options['skip_build']) + + def _RunCommand(self, *args): + proc = subprocess.Popen(args, + cwd=self._dest, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + stdout = proc.communicate()[0] + self.assertEqual(0, proc.wait(), msg='%s\n%s' % (' '.join(args), stdout)) + + def testMakeBundle(self): + os.mkdir(self._dest) + maker = make_bundle.BundleMaker(dest=self._dest, top_dir=self._top_dir) + self.assertEquals(0, maker.MakeBundle()) + commands = [ + './dart samples/hello.dart', + './dart samples/deltablue.dart', + './dart samples/mandelbrot.dart', + './dart samples/towers.dart', + ] + for command in commands: + args = command.split(' ') + self._RunCommand(*args) + args.append('--arch=dartc') + self._RunCommand(*args) + + +if __name__ == '__main__': + unittest.main() diff --git a/tools/presubmit.sh b/tools/presubmit.sh new file mode 100755 index 00000000000..b9700317516 --- /dev/null +++ b/tools/presubmit.sh @@ -0,0 +1,163 @@ +#!/bin/bash +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. +# + +# A quick check over a subset the tests in the runtime, compiler +# and client directories. + +# Currently builds and checks: +# runtime - release mode +# compiler - debug mode (non-optimized) +# client - chromium debug mode + +DO_OPTIMIZE=0 +DO_DARTIUM=0 +TESTS_FAILED=0 + +function usage { + echo "usage: $0 [ --help ] [ --optimize ] [ --dartium ] " + echo + echo "Runs a quick set of tests on runtime, client, and compiler dirs" + echo + echo " --optimize: Also run dartc/release tests optimize" + echo " --dartium : Also run dartium/debug tests" + echo +} + +# Compile the vm/runtime +# $1 directory to build in +# $2 arch +# $3 mode +function doBuild { + cd $1 + ../tools/build.py --arch $2 --mode $3 + if [ $? != 0 ] ; then + echo "Build of $1 failed" + exit 1 + fi + cd .. +} + +# Execute a set of tests +# $1 directory to test in +# $2 arch +# $3 mode +# Returns the output from the subcommand +function doTest { + cd $1 + ../tools/test.py --arch $2 --mode $3 + RESULT=$? + cd .. + if [ ${RESULT} != 0 ] ; then + TESTS_FAILED=1 + fi + return ${RESULT} +} + +# Main + +while [ ! -z "$1" ] ; do + case $1 in + "-h"|"-?"|"-help"|"--help") + usage + exit 1 + ;; + "--optimize") + DO_OPTIMIZE=1 + ;; + "--dartium") + DO_DARTIUM=1 + ;; + *) + echo "Unrecognized argument: $1" + usage + exit 1 + ;; + esac + shift +done + +if [ ! -d compiler -o ! -d runtime -o ! -d tests ] ; then + echo "This doesn't look like the dart source tree." + echo "Change your directory to the dart trunk source" + exit 1 +fi + +echo +echo "--- Building runtime ---" +doBuild runtime ia32 release + +echo +echo "--- Building compiler ---" +doBuild compiler dartc debug + +if [ ${DO_OPTIMIZE} == 1 ] ; then + # echo "Syncing compiler debug build to release" + # rsync -a out/Debug_dartc out/Release_dartc + doBuild compiler dartc release +fi + +# TODO(zundel): Potential shortcut: don't rebuild all of dartc again. +# Tried using rsync, but it doesn't work - client rebuilds anyway + +# Build in client dir +echo +echo "--- Building client ---" +doBuild client dartc debug + + +echo +echo "=== Runtime tests === " +doTest runtime ia32 release +RUNTIME_RESULT=$? + + +echo +echo "=== Compiler tests ===" +echo " Debug mode" +doTest compiler dartc debug +COMPILER_RESULT=$? + +if [ ${DO_OPTIMIZE} == 1 ] ; then + echo " Release mode (--optimize)" + doTest compiler dartc release + RESULT=$? + if [ ${RESULT} != 0 ] ; then + COMPILER_RESULT=$RESULT + fi +fi + +echo +echo "=== Client tests ===" +echo " Chromium" +doTest client chromium debug +CLIENT_RESULT=$? + +if [ ${DO_DARTIUM} == 1 ] ; then + echo " Dartium" + doTest client dartium release + RESULT=$? + if [ ${RESULT} != 0 ] ; then + CLIENT_RESULT=${RESULT} + fi +fi + +# Print summary of results +if [ ${RUNTIME_RESULT} != 0 ] ; then + echo "*** Runtime tests failed" +fi + +if [ ${COMPILER_RESULT} != 0 ] ; then + echo "*** Compiler tests failed" +fi + +if [ ${CLIENT_RESULT} != 0 ] ; then + echo "*** Client tests failed" +fi + +if [ ${TESTS_FAILED} == 0 ] ; then + echo "All presubmit tests passed!" +fi diff --git a/tools/run.py b/tools/run.py new file mode 100755 index 00000000000..46010d301ae --- /dev/null +++ b/tools/run.py @@ -0,0 +1,506 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. +# + +""" +Runs a Dart unit test in different configurations: dartium, chromium, ia32, x64, +arm, simarm, and dartc. Example: + +run.py --arch=dartium --mode=release --test=Test.dart +""" + +import optparse +import os +from os.path import join, abspath, dirname, basename, relpath +import platform +import re +import shutil +import subprocess +import sys +import tempfile +import utils + +OS_GUESS = utils.GuessOS() + +HTML_CONTENTS = ''' + + + Test %(title)s + + + +

Running %(title)s

+ + + + + +''' + +DART_TEST_AS_LIBRARY = ''' +#library('test'); +#source('%(test)s'); +''' + +#TODO(sigmund): remove when prefixes are working +DART_MAIN_TRAMPOLINE = ''' +#library('main_trampoline'); + +#import('%(library)s'); + +mainTrampoline() { + main(); +} +''' + +DART_CONTENTS = ''' +#library('test'); + +#import('%(dom_library)s'); +#import('%(test_framework)s'); + +#import('main_trampoline.dart'); + +pass() { + document.body.innerHTML = 'PASS'; + window.postMessage('unittest-suite-done', '*'); +} + +fail(e, trace) { + document.body.innerHTML = 'FAIL: $e, $trace'; + window.postMessage('unittest-suite-done', '*'); +} + +// All tests are registered as async tests on the UnitTestSuite. +// If the test uses the [:TestRunner:] we will a callback to wait for the +// done callback. +// Otherwise we will call [:testSuite.done():] immediately after the test +// finished. +main() { + bool needsToWait = false; + bool mainIsFinished = false; + TestRunner.waitForDoneCallback = () { needsToWait = true; }; + TestRunner.doneCallback = () { + if (mainIsFinished) { + pass(); + } else { + needsToWait = false; + } + }; + try { + mainTrampoline(); + if (!needsToWait) pass(); + mainIsFinished = true; + } catch(var e, var trace) { + fail(e, trace); + } +} +''' + + +# Patterns for matching test options in .dart files. +VM_OPTIONS_PATTERN = re.compile(r"// VMOptions=(.*)") +DART_OPTIONS_PATTERN = re.compile(r"// DartOptions=(.*)") + +# Pattern for checking if the test is a web test. +DOM_IMPORT_PATTERN = re.compile(r"^#import.*(dart:dom|html.dart)'\);", + re.MULTILINE) + +# Pattern for matching the output of a browser test. +BROWSER_OUTPUT_PASS_PATTERN = re.compile(r"^Content-Type: text/plain\nPASS$", + re.MULTILINE) + +# Pattern for checking if the test is a library in itself. +LIBRARY_DEFINITION_PATTERN = re.compile(r"^#library(.*);", + re.MULTILINE) + +class Error(Exception): + pass + + +def IsWebTest(test, source): + return DOM_IMPORT_PATTERN.search(source) + +def IsLibraryDefinition(test, source): + return LIBRARY_DEFINITION_PATTERN.search(source) + + +class Architecture(object): + def __init__(self, root_path, arch, mode, test): + self.root_path = root_path; + self.arch = arch; + self.mode = mode; + self.test = test; + self.build_root = utils.GetBuildRoot(OS_GUESS, self.mode, self.arch) + source = file(test).read() + self.vm_options = utils.ParseTestOptions(VM_OPTIONS_PATTERN, + source, + root_path) + if not self.vm_options: self.vm_options = [] + + self.dart_options = utils.ParseTestOptions(DART_OPTIONS_PATTERN, + source, + root_path) + self.is_web_test = IsWebTest(test, source) + self.temp_dir = None + + def HasFatalTypeErrors(self): + return False + + def GetTestFrameworkPath(self): + return join(self.root_path, 'tests', 'isolate', 'src', + 'TestFramework.dart') + +class BrowserArchitecture(Architecture): + def __init__(self, root_path, arch, mode, test): + super(BrowserArchitecture, self).__init__(root_path, arch, mode, test) + self.temp_dir = tempfile.mkdtemp(); + if not self.is_web_test: self.GenerateWebTestScript() + + def GetTestScriptFile(self): + """Returns the name of the .dart file to compile.""" + if self.is_web_test: return abspath(self.test) + return join(self.temp_dir, 'test.dart') + + def GetHtmlContents(self): + script_type = self.GetScriptType() + + controller_path = join(self.root_path, + 'client', 'testing', 'unittest', 'test_controller.js') + + return HTML_CONTENTS % { 'title' : self.test, + 'controller_script': controller_path, + 'script_type' : script_type, + 'source_script' : self.GetScriptPath() } + + def GetHtmlPath(self): + # Resources for web tests are relative to the 'html' file. We + # output the 'html' file in the 'out' directory instead of the temporary + # directory because we can easily go the the resources in 'client' through + # 'out'. + if self.is_web_test: + html_path = join(self.root_path, 'client', self.build_root) + if not os.path.exists(html_path): os.makedirs(html_path) + return html_path + + return self.temp_dir + + def GetTestContents(self): + unittest_path = join(self.root_path, + 'client', 'testing', 'unittest', 'unittest.dart') + if self.arch == 'chromium': + dom_path = join(self.root_path, + 'client', 'testing', 'unittest', 'dom_for_unittest.dart') + else: + dom_path = join('dart:dom') + + test_framework_path = self.GetTestFrameworkPath() + test_name = basename(self.test) + test_path = abspath(self.test) + + inputs = { 'unittest': unittest_path, + 'test': test_path, + 'dom_library': dom_path, + 'test_framework': test_framework_path } + return DART_CONTENTS % inputs + + def GenerateWebTestScript(self): + if IsLibraryDefinition(self.test, file(self.test).read()): + library_file = abspath(self.test) + else: + library_file = 'test_as_library.dart' + test_as_library = DART_TEST_AS_LIBRARY % { 'test': abspath(self.test) } + test_as_library_file = join(self.temp_dir, library_file) + f = open(test_as_library_file, 'w') + f.write(test_as_library) + f.close() + + trampoline_contents = DART_MAIN_TRAMPOLINE % { 'library': library_file } + trampoline_file = join(self.temp_dir, 'main_trampoline.dart') + f = open(trampoline_file, 'w') + f.write(trampoline_contents) + f.close() + + app_output_file = self.GetTestScriptFile() + f = open(app_output_file, 'w') + f.write(self.GetTestContents()) + f.close() + + def GetRunCommand(self, fatal_static_type_errors = False): + # For some reason, DRT needs to be called via an absolute path + drt_location = join(self.root_path, + 'client', 'tests', 'dartium', 'DumpRenderTree') + + # On Mac DumpRenderTree is a .app folder + if platform.system() == 'Darwin': + drt_location += '.app/Contents/MacOS/DumpRenderTree' + + drt_flags = [ '--no-timeout' ] + dart_flags = '--dart-flags=--enable_asserts --enable_type_checks ' + dart_flags += ' '.join(self.vm_options) + + if self.arch == 'chromium' and self.mode == 'release': + dart_flags += ' --optimize ' + drt_flags.append(dart_flags) + + html_output_file = join(self.GetHtmlPath(), self.GetHtmlName()) + f = open(html_output_file, 'w') + f.write(self.GetHtmlContents()) + f.close() + + drt_flags.append(html_output_file) + + return [drt_location] + drt_flags + + def HasFailed(self, output): + return not BROWSER_OUTPUT_PASS_PATTERN.search(output) + + def RunTest(self, verbose): + retcode = self.Compile() + if retcode != 0: return 1 + + command = self.GetRunCommand() + + status, output, err = ExecutePipedCommand(command, verbose) + if not self.HasFailed(output): + self.Cleanup() + return 0 + + # TODO(sigmund): print better error message, including how to run test + # locally, and translate error traces using source map info. + print "(FAIL) test page:\033[31m " + command[2] + " \033[0m" + if verbose: + print 'Additional info: ' + print output + print err + return 1 + + def Cleanup(self): + shutil.rmtree(self.temp_dir) + self.temp_dir = None + + +class ChromiumArchitecture(BrowserArchitecture): + def __init__(self, root_path, arch, mode, test): + super(ChromiumArchitecture, self).__init__(root_path, arch, mode, test) + + def GetScriptType(self): + return 'text/javascript' + + def GetScriptPath(self): + """ Returns the name of the output .js file to create """ + path = self.GetTestScriptFile() + return abspath(os.path.join(self.temp_dir, + os.path.basename(path) + '.js')) + + + def GetHtmlName(self): + return relpath(self.test, self.root_path).replace(os.sep, '_') + '.html' + + def GetCompileCommand(self, fatal_static_type_errors=False): + """ Returns cmdline as an array to invoke the compiler on this test""" + # We need an absolute path because the compilation will run + # in a temporary directory. + + dartc = abspath(join(utils.GetBuildRoot(OS_GUESS, self.mode, 'dartc'), + 'compiler', + 'bin', + 'dartc')) + if utils.IsWindows(): dartc += '.exe' + cmd = [dartc, '--work', self.temp_dir] + cmd += self.vm_options + cmd += ['--out', self.GetScriptPath()] + if fatal_static_type_errors: + cmd.append('-fatal-type-errors') + cmd.append(self.GetTestScriptFile()) + return cmd + + def Compile(self): + return ExecuteCommand(self.GetCompileCommand()) + + +class DartiumArchitecture(BrowserArchitecture): + def __init__(self, root_path, arch, mode, test): + super(DartiumArchitecture, self).__init__(root_path, arch, mode, test) + + def GetScriptType(self): + return 'application/dart' + + def GetScriptPath(self): + return 'file:///' + self.GetTestScriptFile() + + def GetHtmlName(self): + path = relpath(self.test, self.root_path).replace(os.sep, '_') + return path + '.dartium.html' + + def GetCompileCommand(self, fatal_static_type_errors=False): + return None + + def Compile(self): + return 0 + + +class StandaloneArchitecture(Architecture): + def __init__(self, root_path, arch, mode, test): + super(StandaloneArchitecture, self).__init__(root_path, arch, mode, test) + + def GetCompileCommand(self, fatal_static_type_errors=False): + return None + + def GetRunCommand(self, fatal_static_type_errors=False): + dart = self.GetExecutable() + test_name = basename(self.test) + test_path = abspath(self.test) + command = [dart] + self.vm_options + (classname, extension) = os.path.splitext(test_name) + if self.dart_options: + command += self.dart_options + elif (extension == '.dart'): + if fatal_static_type_errors: + command += self.GetFatalTypeErrorsFlags() + if '_' in classname: + (classname, sep, tag) = classname.rpartition('_') + command += [test_path] + else: + command += ['--', test_path] + + return command + + def GetFatalTypeErrorsFlags(self): + return [] + + def RunTest(self, verbose): + command = self.GetRunCommand() + return ExecuteCommand(command, verbose) + + def Cleanup(self): + return + + +# Long term, we should do the running machinery that is currently in +# DartRunner.java +class DartcArchitecture(StandaloneArchitecture): + def __init__(self, root_path, arch, mode, test): + super(DartcArchitecture, self).__init__(root_path, arch, mode, test) + + def GetExecutable(self): + return abspath(join(self.build_root, 'compiler', 'bin', 'dartc_test')) + + def GetFatalTypeErrorsFlags(self): + return ['--fatal-type-errors'] + + def HasFatalTypeErrors(self): + return True + + def GetRunCommand(self, fatal_static_type_errors=False): + cmd = super(DartcArchitecture, self).GetRunCommand( + fatal_static_type_errors) + return cmd + + +class RuntimeArchitecture(StandaloneArchitecture): + def __init__(self, root_path, arch, mode, test): + super(RuntimeArchitecture, self).__init__(root_path, arch, mode, test) + + def GetExecutable(self): + return abspath(join(self.build_root, 'dart_bin')) + + +def ExecutePipedCommand(cmd, verbose): + """Execute a command in a subprocess. + """ + if verbose: print 'Executing: ' + ' '.join(cmd) + pipe = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) + (output, err) = pipe.communicate() + if pipe.returncode != 0 and verbose: + print 'Execution failed: ' + output + '\n' + err + print output + print err + return pipe.returncode, output, err + + +def ExecuteCommand(cmd, verbose = False): + """Execute a command in a subprocess. + """ + if verbose: print 'Executing: ' + ' '.join(cmd) + return subprocess.call(cmd) + + +def AreOptionsValid(options): + if not options.arch in ['ia32', 'x64', 'arm', 'simarm', 'dartc', 'dartium', + 'chromium']: + print 'Unknown arch %s' % options.arch + return None + + return options.test + + +def Flags(): + result = optparse.OptionParser() + result.add_option("-v", "--verbose", + help="Print messages", + default=False, + action="store_true") + result.add_option("-t", "--test", + help="App or Dart file containing the test", + type="string", + action="store", + default=None) + result.add_option("--arch", + help="The architecture to run tests for", + metavar="[ia32,x64,arm,simarm,dartc,chromium,dartium]", + default=utils.GuessArchitecture()) + result.add_option("-m", "--mode", + help="The test modes in which to run", + metavar='[debug,release]', + default='debug') + result.set_usage("run.py --arch ARCH --mode MODE -t TEST") + return result + + +def GetArchitecture(arch, mode, test): + root_path = abspath(join(dirname(sys.argv[0]), '..')) + if arch == 'chromium': + return ChromiumArchitecture(root_path, arch, mode, test) + + elif arch == 'dartium': + return DartiumArchitecture(root_path, arch, mode, test) + + elif arch in ['ia32', 'x64', 'simarm', 'arm']: + return RuntimeArchitecture(root_path, arch, mode, test) + + elif arch == 'dartc': + return DartcArchitecture(root_path, arch, mode, test) + + +def Main(): + parser = Flags() + (options, args) = parser.parse_args() + if not AreOptionsValid(options): + parser.print_help() + return 1 + + return GetArchitecture(options.arch, options.mode, + options.test).RunTest(options.verbose) + + +if __name__ == '__main__': + sys.exit(Main()) diff --git a/tools/test.py b/tools/test.py new file mode 100755 index 00000000000..f3f6170afab --- /dev/null +++ b/tools/test.py @@ -0,0 +1,1723 @@ +#!/usr/bin/env python +# +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. +# + +import imp +import optparse +import os +from os.path import join, dirname, abspath, basename, isdir, exists, realpath +import platform +import re +import run +import select +import signal +import subprocess +import sys +import tempfile +import time +import threading +import traceback +from Queue import Queue, Empty + +import utils + +TIMEOUT_SECS = 60 +VERBOSE = False +ARCH_GUESS = utils.GuessArchitecture() +OS_GUESS = utils.GuessOS() +HOST_CPUS = utils.GuessCpus() +USE_DEFAULT_CPUS = -1 +BUILT_IN_TESTS = ['dartc', 'vm', 'dart', 'corelib', 'language', 'co19', + 'samples', 'isolate', 'stub-generator', 'client'] + +# Patterns for matching test options in .dart files. +VM_OPTIONS_PATTERN = re.compile(r"// VMOptions=(.*)") +DART_OPTIONS_PATTERN = re.compile(r"// DartOptions=(.*)") +ISOLATE_STUB_PATTERN = re.compile(r"// IsolateStubs=(.*)") + +# --------------------------------------------- +# --- P r o g r e s s I n d i c a t o r s --- +# --------------------------------------------- + + +class ProgressIndicator(object): + + def __init__(self, cases, context): + self.abort = False + self.terminate = False + self.cases = cases + self.queue = Queue(len(cases)) + self.batch_queues = {}; + self.context = context + + # Extract batchable cases. + found_cmds = {} + batch_cases = [] + for case in cases: + cmd = case.case.GetCommand()[0] + if not utils.IsWindows(): + # Diagnostic check for executable (if an absolute pathname) + if not cmd in found_cmds: + if os.path.isabs(cmd) and not os.path.isfile(cmd): + msg = "Can't find command %s\n" % cmd \ + + "(Did you build first? " \ + + "Are you running in the correct directory?)" + raise Exception(msg) + else: + found_cmds[cmd] = 1 + + if case.case.IsBatchable(): + if not self.batch_queues.has_key(cmd): + self.batch_queues[cmd] = Queue(len(cases)) + self.batch_queues[cmd].put(case) + else: + self.queue.put_nowait(case) + + self.succeeded = 0 + self.remaining = len(cases) + self.total = len(cases) + self.failed = [ ] + self.crashed = 0 + self.lock = threading.Lock() + + def PrintFailureHeader(self, test): + if test.IsNegative(): + negative_marker = '[negative] ' + else: + negative_marker = '' + print "=== %(label)s %(negative)s===" % { + 'label': test.GetLabel(), + 'negative': negative_marker + } + print "Path: %s" % "/".join(test.path) + + def Run(self, tasks): + self.Starting() + + # Scale the number of tasks to the nubmer of CPUs on the machine + if tasks == USE_DEFAULT_CPUS: + tasks = HOST_CPUS + + # TODO(zundel): Refactor BatchSingle method and TestRunner to + # share code and simplify this method. + + # Start the non-batchable items first - there are some long running + # jobs we don't want to wait on at the end. + threads = [] + # Spawn N-1 threads and then use this thread as the last one. + # That way -j1 avoids threading altogether which is a nice fallback + # in case of threading problems. + for i in xrange(tasks - 1): + thread = threading.Thread(target=self.RunSingle, args=[]) + threads.append(thread) + thread.start() + + + # Next, crank up the batchable tasks. Note that this will start + # 'tasks' more threads, but the assumption is that if batching is + # enabled that almost all tests are batchable. + for (cmd, queue) in self.batch_queues.items(): + if not queue.empty(): + batch_len = queue.qsize(); + try: + batch_tester = BatchTester(queue, tasks, self, + [cmd, '-batch']) + except Exception, e: + print "Aborting batch test for " + cmd + ". Problem on startup." + batch_tester.Shutdown() + raise + + try: + batch_tester.WaitForCompletion() + except: + print "Aborting batch cmd " + cmd + "while waiting for completion." + batch_tester.Shutdown() + raise + + try: + self.RunSingle() + if self.abort: + raise Exception("Aborted") + # Wait for the remaining non-batched threads. + for thread in threads: + # Use a timeout so that signals (ctrl-c) will be processed. + thread.join(timeout=10000000) + if self.abort: + raise Exception("Aborted") + except Exception, e: + # If there's an exception we schedule an interruption for any + # remaining threads. + self.terminate = True + # ...and then reraise the exception to bail out + raise + + self.Done() + return not self.failed + + def RunSingle(self): + while not self.terminate: + try: + test = self.queue.get_nowait() + except Empty: + return + case = test.case + with self.lock: + self.AboutToRun(case) + try: + start = time.time() + output = case.Run() + case.duration = (time.time() - start) + except KeyboardInterrupt: + self.abort = True + self.terminate = True + raise + except IOError, e: + self.abort = True + self.terminate = True + raise + if self.terminate: + return + with self.lock: + if output.UnexpectedOutput(): + self.failed.append(output) + if output.HasCrashed(): + self.crashed += 1 + else: + self.succeeded += 1 + self.remaining -= 1 + self.HasRun(output) + + +class BatchTester(object): + """Implements communication with a set of subprocesses using threads.""" + + def __init__(self, work_queue, tasks, progress, batch_cmd): + self.work_queue = work_queue + self.terminate = False + self.progress = progress + self.threads = [] + self.runners = {} + self.last_activity = {} + self.context = progress.context + self.shutdown_lock = threading.Lock() + + # Scale the number of tasks to the nubmer of CPUs on the machine + # 1:1 is too much of an overload on many machines in batch mode, + # so scale the ratio of threads to CPUs back. + if tasks == USE_DEFAULT_CPUS: + tasks = .75 * HOST_CPUS + + # Start threads + for i in xrange(tasks): + thread = threading.Thread(target=self.RunThread, args=[batch_cmd, i]) + self.threads.append(thread) + thread.daemon = True + thread.start() + + def RunThread(self, batch_cmd, thread_number): + """A thread started to feed a single TestRunner.""" + try: + while not self.terminate and not self.work_queue.empty(): + runner = subprocess.Popen(batch_cmd, + stdin=subprocess.PIPE, + stderr=subprocess.STDOUT, + stdout=subprocess.PIPE) + self.runners[thread_number] = runner + self.FeedTestRunner(runner, thread_number) + if self.last_activity.has_key(thread_number): + del self.last_activity[thread_number] + + # cleanup + self.EndRunner(runner) + + except: + self.Shutdown() + raise + finally: + if self.last_activity.has_key(thread_number): + del self.last_activity[thread_number] + self.EndRunner(runner) + + def EndRunner(self, runner): + """ Cleans up a single runner, killing the child if necessary""" + with self.shutdown_lock: + if runner: + returncode = runner.poll() + if returncode == None: + runner.kill() + for (found_runner, thread_number) in self.runners.items(): + if runner == found_runner: + del self.runners[thread_number] + break + try: + runner.communicate(); + except ValueError: + pass + + + + def CheckForTimeouts(self): + now = time.time() + for (thread_number, start_time) in self.last_activity.items(): + if now - start_time > self.context.timeout: + self.runners[thread_number].kill() + + def WaitForCompletion(self): + """ Wait for threads to finish, and monitor test runners for timeouts.""" + for t in self.threads: + while True: + self.CheckForTimeouts() + t.join(timeout=5) + if not t.isAlive(): + break + + def FeedTestRunner(self, runner, thread_number): + """Feed commands to the fork'ed TestRunner through a Popen object.""" + + last_case = {} + last_buf = '' + + while not self.terminate: + # Is the runner still alive? + returninfo = runner.poll() + if returninfo is not None: + buf = last_buf + '\n' + runner.stdout.read() + if last_case: + self.RecordPassFail(last_case, buf, CRASH) + else: + with self.progress.lock: + print >>sys. stderr, ("%s: runner unexpectedly exited: %d" + % (threading.currentThread().name, returninfo)) + print 'Crash Output: ' + print + print buf + return + + try: + case = self.work_queue.get_nowait() + with self.progress.lock: + self.progress.AboutToRun(case.case) + + except Empty: + return + test_case = case.case + cmd = " ".join(test_case.GetCommand()[1:]) + + try: + print >>runner.stdin, cmd + except IOError: + with self.progress.lock: + traceback.print_exc() + + # Child exited before starting the next command. + buf = last_buf + '\n' + runner.stdout.read() + self.RecordPassFail(last_case, buf, CRASH) + + # We never got a chance to run this command - queue it back up. + self.work_queue.put(case) + return + + buf = "" + self.last_activity[thread_number] = time.time() + while not self.terminate: + line = runner.stdout.readline() + if self.terminate: + break; + case.case.duration = time.time() - self.last_activity[thread_number]; + if not line: + # EOF. Child has exited. + if case.case.duration > self.context.timeout: + with self.progress.lock: + print "Child timed out after %d seconds" % self.context.timeout + self.RecordPassFail(case, buf, TIMEOUT) + elif buf: + self.RecordPassFail(case, buf, CRASH) + return + + # Look for TestRunner batch status escape sequence. e.g. + # >>> TEST PASS + if line.startswith('>>> '): + result = line.split() + if result[1] == 'TEST': + outcome = result[2].lower() + + # Read the rest of the output buffer (possible crash output) + if outcome == CRASH: + buf += runner.stdout.read() + + self.RecordPassFail(case, buf, outcome) + + # Always handle crashes by restarting the runner. + if outcome == CRASH: + return + break + elif result[1] == 'BATCH': + pass + else: + print 'Unknown cmd from batch runner: %s' % line + else: + buf += line + + # If the process crashes before the next command is executed, + # save info to report diagnostics. + last_buf = buf + last_case = case + + def RecordPassFail(self, case, stdout_buf, outcome): + """An unexpected failure occurred.""" + if outcome == PASS or outcome == OKAY: + exit_code = 0 + elif outcome == CRASH: + exit_code = -1 + elif outcome == FAIL or outcome == TIMEOUT: + exit_code = 1 + else: + assert false, "Unexpected outcome: %s" % outcome + + cmd_output = CommandOutput(0, exit_code, + outcome == TIMEOUT, stdout_buf, "") + test_output = TestOutput(case.case, + case.case.GetCommand(), + cmd_output) + with self.progress.lock: + if test_output.UnexpectedOutput(): + self.progress.failed.append(test_output) + else: + self.progress.succeeded += 1 + if outcome == CRASH: + self.progress.crashed += 1 + self.progress.remaining -= 1 + self.progress.HasRun(test_output) + + def Shutdown(self): + """Kill all active runners""" + print "Shutting down remaining runners" + self.terminate = True; + for runner in self.runners.values(): + runner.kill() + # Give threads a chance to exit gracefully + time.sleep(2) + for runner in self.runners.values(): + self.EndRunner(runner) + + +def EscapeCommand(command): + parts = [] + for part in command: + if ' ' in part: + # Escape spaces. We may need to escape more characters for this + # to work properly. + parts.append('"%s"' % part) + else: + parts.append(part) + return " ".join(parts) + + +class SimpleProgressIndicator(ProgressIndicator): + + def Starting(self): + print 'Running %i tests' % len(self.cases) + + def Done(self): + print + for failed in self.failed: + self.PrintFailureHeader(failed.test) + if failed.output.stderr: + print "--- stderr ---" + print failed.output.stderr.strip() + if failed.output.stdout: + print "--- stdout ---" + print failed.output.stdout.strip() + print "Command: %s" % EscapeCommand(failed.command) + if failed.HasCrashed(): + print "--- CRASHED ---" + if failed.HasTimedOut(): + print "--- TIMEOUT ---" + if len(self.failed) == 0: + print "===" + print "=== All tests succeeded" + print "===" + else: + print + print "===" + if len(self.failed) == 1: + print "=== 1 test failed" + else: + print "=== %i tests failed" % len(self.failed) + if self.crashed > 0: + if self.crashed == 1: + print "=== 1 test CRASHED" + else: + print "=== %i tests CRASHED" % self.crashed + print "===" + + +class VerboseProgressIndicator(SimpleProgressIndicator): + + def AboutToRun(self, case): + print 'Starting %s...' % case.GetLabel() + sys.stdout.flush() + + def HasRun(self, output): + if output.UnexpectedOutput(): + if output.HasCrashed(): + outcome = 'CRASH' + else: + outcome = 'FAIL' + else: + outcome = 'PASS' + print 'Done running %s: %s' % (output.test.GetLabel(), outcome) + + +class OneLineProgressIndicator(SimpleProgressIndicator): + + def AboutToRun(self, case): + pass + + def HasRun(self, output): + if output.UnexpectedOutput(): + if output.HasCrashed(): + outcome = 'CRASH' + else: + outcome = 'FAIL' + else: + outcome = 'pass' + print 'Done %s: %s' % (output.test.GetLabel(), outcome) + + +class OneLineProgressIndicatorForBuildBot(OneLineProgressIndicator): + + def HasRun(self, output): + super(OneLineProgressIndicatorForBuildBot, self).HasRun(output) + percent = (((self.total - self.remaining) * 100) // self.total) + print '@@@STEP_CLEAR@@@' + print '@@@STEP_TEXT@ %3d%% +%d -%d @@@' % ( + percent, self.succeeded, len(self.failed)) + + +class CompactProgressIndicator(ProgressIndicator): + + def __init__(self, cases, context, templates): + super(CompactProgressIndicator, self).__init__(cases, context) + self.templates = templates + self.last_status_length = 0 + self.start_time = time.time() + + def Starting(self): + pass + + def Done(self): + self.PrintProgress('Done') + + def AboutToRun(self, case): + self.PrintProgress(case.GetLabel()) + + def HasRun(self, output): + if output.UnexpectedOutput(): + self.ClearLine(self.last_status_length) + self.PrintFailureHeader(output.test) + stdout = output.output.stdout.strip() + if len(stdout): + print self.templates['stdout'] % stdout + stderr = output.output.stderr.strip() + if len(stderr): + print self.templates['stderr'] % stderr + print "Command: %s" % EscapeCommand(output.command) + if output.HasCrashed(): + print "--- CRASHED ---" + if output.HasTimedOut(): + print "--- TIMEOUT ---" + + def Truncate(self, str, length): + if length and (len(str) > (length - 3)): + return str[:(length-3)] + "..." + else: + return str + + def PrintProgress(self, name): + self.ClearLine(self.last_status_length) + elapsed = time.time() - self.start_time + status = self.templates['status_line'] % { + 'passed': self.succeeded, + 'percent': (((self.total - self.remaining) * 100) // self.total), + 'failed': len(self.failed), + 'test': name, + 'mins': int(elapsed) / 60, + 'secs': int(elapsed) % 60 + } + status = self.Truncate(status, 78) + self.last_status_length = len(status) + print status, + sys.stdout.flush() + + +class MonochromeProgressIndicator(CompactProgressIndicator): + + def __init__(self, cases, context): + templates = { + 'status_line': "[%(mins)02i:%(secs)02i|%%%(percent) 4d|+%(passed) 4d|-%(failed) 4d]: %(test)s", + 'stdout': '%s', + 'stderr': '%s', + 'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"), + 'max_length': 78 + } + super(MonochromeProgressIndicator, self).__init__(cases, context, templates) + + def ClearLine(self, last_line_length): + print ("\r" + (" " * last_line_length) + "\r"), + +class ColorProgressIndicator(CompactProgressIndicator): + + def __init__(self, cases, context): + templates = { + 'status_line': ("[%(mins)02i:%(secs)02i|%%%(percent) 4d|" + "\033[32m+%(passed) 4d" + "\033[0m|\033[31m-%(failed) 4d\033[0m]: %(test)s"), + 'stdout': '%s', + 'stderr': '%s', + 'clear': lambda last_line_length: ("\r" + (" " * last_line_length) + "\r"), + 'max_length': 78 + } + super(ColorProgressIndicator, self).__init__(cases, context, templates) + + def ClearLine(self, last_line_length): + print ("\r" + (" " * last_line_length) + "\r"), + + +PROGRESS_INDICATORS = { + 'verbose': VerboseProgressIndicator, + 'mono': MonochromeProgressIndicator, + 'color': ColorProgressIndicator, + 'line': OneLineProgressIndicator, + 'buildbot': OneLineProgressIndicatorForBuildBot +} + + +# ------------------------- +# --- F r a m e w o r k --- +# ------------------------- + + +class CommandOutput(object): + + def __init__(self, pid, exit_code, timed_out, stdout, stderr): + self.pid = pid + self.exit_code = exit_code + self.timed_out = timed_out + self.stdout = stdout + self.stderr = stderr + self.failed = None + + +class TestCase(object): + + def __init__(self, context, path): + self.path = path + self.context = context + self.duration = None + self.arch = [] + + def IsBatchable(self): + if self.context.use_batch: + if self.arch and 'dartc' in self.arch: + return True + return False + + def IsNegative(self): + return False + + def CompareTime(self, other): + return cmp(other.duration, self.duration) + + def DidFail(self, output): + if output.failed is None: + output.failed = self.IsFailureOutput(output) + return output.failed + + def IsFailureOutput(self, output): + return output.exit_code != 0 + + def RunCommand(self, command, cwd=None): + full_command = self.context.processor(command) + try: + output = Execute(full_command, self.context, self.context.timeout, cwd) + except OSError as e: + raise utils.ToolError("%s: %s" % (full_command[0], e.strerror)) + test_output = TestOutput(self, full_command, output) + self.Cleanup() + return test_output + + def BeforeRun(self): + pass + + def AfterRun(self): + pass + + def Run(self): + self.BeforeRun() + cmd = self.GetCommand() + try: + result = self.RunCommand(cmd) + finally: + self.AfterRun() + return result + + def Cleanup(self): + return + + +class TestOutput(object): + + def __init__(self, test, command, output): + self.test = test + self.command = command + self.output = output + + def UnexpectedOutput(self): + if self.HasCrashed(): + outcome = CRASH + elif self.HasTimedOut(): + outcome = TIMEOUT + elif self.HasFailed(): + outcome = FAIL + else: + outcome = PASS + return not outcome in self.test.outcomes + + def HasCrashed(self): + if utils.IsWindows(): + if self.output.exit_code == 3: + # The VM uses std::abort to terminate on asserts. + # std::abort terminates with exit code 3 on Windows. + return True + return 0x80000000 & self.output.exit_code and not (0x3FFFFF00 & self.output.exit_code) + else: + # Timed out tests will have exit_code -signal.SIGTERM. + if self.output.timed_out: + return False + if self.output.exit_code == 253: + # The Java dartc runners exit 253 in case of unhandled exceptions. + return True + return self.output.exit_code < 0 + + def HasTimedOut(self): + return self.output.timed_out; + + def HasFailed(self): + execution_failed = self.test.DidFail(self.output) + if self.test.IsNegative(): + return not execution_failed + else: + return execution_failed + + +def KillProcessWithID(pid): + if utils.IsWindows(): + os.popen('taskkill /T /F /PID %d' % pid) + else: + os.kill(pid, signal.SIGTERM) + + +MAX_SLEEP_TIME = 0.1 +INITIAL_SLEEP_TIME = 0.0001 +SLEEP_TIME_FACTOR = 1.25 + +SEM_INVALID_VALUE = -1 +SEM_NOGPFAULTERRORBOX = 0x0002 # Microsoft Platform SDK WinBase.h + +def Win32SetErrorMode(mode): + prev_error_mode = SEM_INVALID_VALUE + try: + import ctypes + prev_error_mode = ctypes.windll.kernel32.SetErrorMode(mode); + except ImportError: + pass + return prev_error_mode + +def RunProcess(context, timeout, args, **rest): + if context.verbose: print "#", " ".join(args) + popen_args = args + prev_error_mode = SEM_INVALID_VALUE; + if utils.IsWindows(): + popen_args = '"' + subprocess.list2cmdline(args) + '"' + if context.suppress_dialogs: + # Try to change the error mode to avoid dialogs on fatal errors. Don't + # touch any existing error mode flags by merging the existing error mode. + # See http://blogs.msdn.com/oldnewthing/archive/2004/07/27/198410.aspx. + error_mode = SEM_NOGPFAULTERRORBOX; + prev_error_mode = Win32SetErrorMode(error_mode); + Win32SetErrorMode(error_mode | prev_error_mode); + process = subprocess.Popen( + shell = utils.IsWindows(), + args = popen_args, + **rest + ) + if utils.IsWindows() and context.suppress_dialogs and prev_error_mode != SEM_INVALID_VALUE: + Win32SetErrorMode(prev_error_mode) + # Compute the end time - if the process crosses this limit we + # consider it timed out. + if timeout is None: end_time = None + else: end_time = time.time() + timeout + timed_out = False + # Repeatedly check the exit code from the process in a + # loop and keep track of whether or not it times out. + exit_code = None + sleep_time = INITIAL_SLEEP_TIME + while exit_code is None: + if (not end_time is None) and (time.time() >= end_time): + # Kill the process and wait for it to exit. + KillProcessWithID(process.pid) + # Drain the output pipe from the process to avoid deadlock + process.communicate() + exit_code = process.wait() + timed_out = True + else: + exit_code = process.poll() + time.sleep(sleep_time) + sleep_time = sleep_time * SLEEP_TIME_FACTOR + if sleep_time > MAX_SLEEP_TIME: + sleep_time = MAX_SLEEP_TIME + return (process, exit_code, timed_out) + + +def PrintError(str): + sys.stderr.write(str) + sys.stderr.write('\n') + + +def CheckedUnlink(name): + try: + os.unlink(name) + except OSError, e: + PrintError("os.unlink() " + str(e)) + + +def Execute(args, context, timeout=None, cwd=None): + (fd_out, outname) = tempfile.mkstemp() + (fd_err, errname) = tempfile.mkstemp() + (process, exit_code, timed_out) = RunProcess( + context, + timeout, + args = args, + stdout = fd_out, + stderr = fd_err, + cwd = cwd + ) + os.close(fd_out) + os.close(fd_err) + output = file(outname).read() + errors = file(errname).read() + CheckedUnlink(outname) + CheckedUnlink(errname) + result = CommandOutput(process.pid, exit_code, timed_out, output, errors) + return result + + +class TestConfiguration(object): + + def __init__(self, context, root): + self.context = context + self.root = root + + def Contains(self, path, file): + if len(path) > len(file): + return False + for i in xrange(len(path)): + if not path[i].match(file[i]): + return False + return True + + def GetTestStatus(self, sections, defs): + pass + + +class TestSuite(object): + + def __init__(self, name): + self.name = name + + def GetName(self): + return self.name + + +class TestRepository(TestSuite): + + def __init__(self, path): + normalized_path = abspath(path) + super(TestRepository, self).__init__(basename(normalized_path)) + self.path = normalized_path + self.is_loaded = False + self.config = None + + def GetConfiguration(self, context): + if self.is_loaded: + return self.config + self.is_loaded = True + file = None + try: + (file, pathname, description) = imp.find_module('testcfg', [ self.path ]) + module = imp.load_module('testcfg', file, pathname, description) + self.config = module.GetConfiguration(context, self.path) + finally: + if file: + file.close() + return self.config + + def ListTests(self, current_path, path, context, mode, arch): + return self.GetConfiguration(context).ListTests(current_path, + path, + mode, + arch) + + def GetTestStatus(self, context, sections, defs): + self.GetConfiguration(context).GetTestStatus(sections, defs) + + +class LiteralTestSuite(TestSuite): + + def __init__(self, tests): + super(LiteralTestSuite, self).__init__('root') + self.tests = tests + + def ListTests(self, current_path, path, context, mode, arch): + name = path[0] + result = [ ] + for test in self.tests: + test_name = test.GetName() + if name.match(test_name): + full_path = current_path + [test_name] + result += test.ListTests(full_path, path, context, mode, arch) + return result + + def GetTestStatus(self, context, sections, defs): + for test in self.tests: + test.GetTestStatus(context, sections, defs) + +class Context(object): + + def __init__(self, workspace, verbose, os, timeout, + processor, suppress_dialogs, executable, flags, + keep_temporary_files, use_batch): + self.workspace = workspace + self.verbose = verbose + self.os = os + self.timeout = timeout + self.processor = processor + self.suppress_dialogs = suppress_dialogs + self.executable = executable + self.flags = flags + self.keep_temporary_files = keep_temporary_files + self.use_batch = use_batch == "true" + + def GetBuildRoot(self, mode, arch): + result = utils.GetBuildRoot(self.os, mode, arch) + return result + + def GetBuildConf(self, mode, arch): + result = utils.GetBuildConf(mode, arch) + return result + + def GetExecutable(self, mode, arch, name): + if self.executable is not None: + return self.executable + path = abspath(join(self.GetBuildRoot(mode, arch), name)) + if utils.IsWindows() and not path.endswith('.exe'): + return path + '.exe' + else: + return path + + def GetDart(self, mode, arch): + if arch == 'dartc': + command = [ abspath(join(self.GetBuildRoot(mode, arch), + 'compiler', 'bin', 'dartc_test')) ] + else: + command = [ self.GetExecutable(mode, arch, 'dart_bin') ] + + return command + + def GetDartC(self, mode, arch): + dartc = abspath(os.path.join(self.GetBuildRoot(mode, arch), + 'compiler', 'bin', 'dartc')) + if utils.IsWindows(): dartc += '.exe' + command = [ dartc ] + + # Add the flags from the context to the command line. + command += self.flags + return command + + def GetRunTests(self, mode, arch): + return [ self.GetExecutable(mode, arch, 'run_vm_tests') ] + +def RunTestCases(cases_to_run, progress, tasks, context): + progress = PROGRESS_INDICATORS[progress](cases_to_run, context) + return progress.Run(tasks) + + +# ------------------------------------------- +# --- T e s t C o n f i g u r a t i o n --- +# ------------------------------------------- + + +SKIP = 'skip' +FAIL = 'fail' +PASS = 'pass' +OKAY = 'okay' +TIMEOUT = 'timeout' +CRASH = 'crash' +SLOW = 'slow' + + +class Expression(object): + pass + + +class Constant(Expression): + + def __init__(self, value): + self.value = value + + def Evaluate(self, env, defs): + return self.value + + +class Variable(Expression): + + def __init__(self, name): + self.name = name + + def GetOutcomes(self, env, defs): + if self.name in env: return ListSet([env[self.name]]) + else: return Nothing() + + +class Outcome(Expression): + + def __init__(self, name): + self.name = name + + def GetOutcomes(self, env, defs): + if self.name in defs: + return defs[self.name].GetOutcomes(env, defs) + else: + return ListSet([self.name]) + + +class Set(object): + pass + + +class ListSet(Set): + + def __init__(self, elms): + self.elms = elms + + def __str__(self): + return "ListSet%s" % str(self.elms) + + def Intersect(self, that): + if not isinstance(that, ListSet): + return that.Intersect(self) + return ListSet([ x for x in self.elms if x in that.elms ]) + + def Union(self, that): + if not isinstance(that, ListSet): + return that.Union(self) + return ListSet(self.elms + [ x for x in that.elms if x not in self.elms ]) + + def IsEmpty(self): + return len(self.elms) == 0 + + +class Everything(Set): + + def Intersect(self, that): + return that + + def Union(self, that): + return self + + def IsEmpty(self): + return False + + +class Nothing(Set): + + def Intersect(self, that): + return self + + def Union(self, that): + return that + + def IsEmpty(self): + return True + + +class Operation(Expression): + + def __init__(self, left, op, right): + self.left = left + self.op = op + self.right = right + + def Evaluate(self, env, defs): + if self.op == '||' or self.op == ',': + return self.left.Evaluate(env, defs) or self.right.Evaluate(env, defs) + elif self.op == 'if': + return False + elif self.op == '==': + inter = self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs)) + return not inter.IsEmpty() + else: + assert self.op == '&&' + return self.left.Evaluate(env, defs) and self.right.Evaluate(env, defs) + + def GetOutcomes(self, env, defs): + if self.op == '||' or self.op == ',': + return self.left.GetOutcomes(env, defs).Union(self.right.GetOutcomes(env, defs)) + elif self.op == 'if': + if self.right.Evaluate(env, defs): return self.left.GetOutcomes(env, defs) + else: return Nothing() + else: + assert self.op == '&&' + return self.left.GetOutcomes(env, defs).Intersect(self.right.GetOutcomes(env, defs)) + + +def IsAlpha(str): + for char in str: + if not (char.isalpha() or char.isdigit() or char == '_'): + return False + return True + + +class Tokenizer(object): + """A simple string tokenizer that chops expressions into variables, + parens and operators""" + + def __init__(self, expr): + self.index = 0 + self.expr = expr + self.length = len(expr) + self.tokens = None + + def Current(self, length = 1): + if not self.HasMore(length): return "" + return self.expr[self.index:self.index+length] + + def HasMore(self, length = 1): + return self.index < self.length + (length - 1) + + def Advance(self, count = 1): + self.index = self.index + count + + def AddToken(self, token): + self.tokens.append(token) + + def SkipSpaces(self): + while self.HasMore() and self.Current().isspace(): + self.Advance() + + def Tokenize(self): + self.tokens = [ ] + while self.HasMore(): + self.SkipSpaces() + if not self.HasMore(): + return None + if self.Current() == '(': + self.AddToken('(') + self.Advance() + elif self.Current() == ')': + self.AddToken(')') + self.Advance() + elif self.Current() == '$': + self.AddToken('$') + self.Advance() + elif self.Current() == ',': + self.AddToken(',') + self.Advance() + elif IsAlpha(self.Current()): + buf = "" + while self.HasMore() and IsAlpha(self.Current()): + buf += self.Current() + self.Advance() + self.AddToken(buf) + elif self.Current(2) == '&&': + self.AddToken('&&') + self.Advance(2) + elif self.Current(2) == '||': + self.AddToken('||') + self.Advance(2) + elif self.Current(2) == '==': + self.AddToken('==') + self.Advance(2) + else: + return None + return self.tokens + + +class Scanner(object): + """A simple scanner that can serve out tokens from a given list""" + + def __init__(self, tokens): + self.tokens = tokens + self.length = len(tokens) + self.index = 0 + + def HasMore(self): + return self.index < self.length + + def Current(self): + return self.tokens[self.index] + + def Advance(self): + self.index = self.index + 1 + + +def ParseAtomicExpression(scan): + if scan.Current() == "true": + scan.Advance() + return Constant(True) + elif scan.Current() == "false": + scan.Advance() + return Constant(False) + elif IsAlpha(scan.Current()): + name = scan.Current() + scan.Advance() + return Outcome(name.lower()) + elif scan.Current() == '$': + scan.Advance() + if not IsAlpha(scan.Current()): + return None + name = scan.Current() + scan.Advance() + return Variable(name.lower()) + elif scan.Current() == '(': + scan.Advance() + result = ParseLogicalExpression(scan) + if (not result) or (scan.Current() != ')'): + return None + scan.Advance() + return result + else: + return None + + +BINARIES = ['=='] +def ParseOperatorExpression(scan): + left = ParseAtomicExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() in BINARIES): + op = scan.Current() + scan.Advance() + right = ParseOperatorExpression(scan) + if not right: + return None + left = Operation(left, op, right) + return left + + +def ParseConditionalExpression(scan): + left = ParseOperatorExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() == 'if'): + scan.Advance() + right = ParseOperatorExpression(scan) + if not right: + return None + left= Operation(left, 'if', right) + return left + + +LOGICALS = ["&&", "||", ","] +def ParseLogicalExpression(scan): + left = ParseConditionalExpression(scan) + if not left: return None + while scan.HasMore() and (scan.Current() in LOGICALS): + op = scan.Current() + scan.Advance() + right = ParseConditionalExpression(scan) + if not right: + return None + left = Operation(left, op, right) + return left + + +def ParseCondition(expr): + """Parses a logical expression into an Expression object""" + tokens = Tokenizer(expr).Tokenize() + if not tokens: + print "Malformed expression: '%s'" % expr + return None + scan = Scanner(tokens) + ast = ParseLogicalExpression(scan) + if not ast: + print "Malformed expression: '%s'" % expr + return None + if scan.HasMore(): + print "Malformed expression: '%s'" % expr + return None + return ast + + +class ClassifiedTest(object): + + def __init__(self, case, outcomes): + self.case = case + self.outcomes = outcomes + + +class Configuration(object): + """The parsed contents of a configuration file""" + + def __init__(self, sections, defs): + self.sections = sections + self.defs = defs + + def ClassifyTests(self, cases, env): + sections = [s for s in self.sections if s.condition.Evaluate(env, self.defs)] + all_rules = reduce(list.__add__, [s.rules for s in sections], []) + unused_rules = set(all_rules) + result = [ ] + all_outcomes = set([]) + for case in cases: + matches = [ r for r in all_rules if r.Contains(case.path) ] + outcomes = set([]) + for rule in matches: + outcomes = outcomes.union(rule.GetOutcomes(env, self.defs)) + unused_rules.discard(rule) + if not outcomes: + outcomes = [PASS] + case.outcomes = outcomes + all_outcomes = all_outcomes.union(outcomes) + result.append(ClassifiedTest(case, outcomes)) + return (result, list(unused_rules), all_outcomes) + + +class Section(object): + """A section of the configuration file. Sections are enabled or + disabled prior to running the tests, based on their conditions""" + + def __init__(self, condition): + self.condition = condition + self.rules = [ ] + + def AddRule(self, rule): + self.rules.append(rule) + + +class Rule(object): + """A single rule that specifies the expected outcome for a single + test.""" + + def __init__(self, raw_path, path, value): + self.raw_path = raw_path + self.path = path + self.value = value + + def GetOutcomes(self, env, defs): + set = self.value.GetOutcomes(env, defs) + assert isinstance(set, ListSet) + return set.elms + + def Contains(self, path): + if len(self.path) > len(path): + return False + for i in xrange(len(self.path)): + if not self.path[i].match(path[i]): + return False + return True + + +HEADER_PATTERN = re.compile(r'\[([^]]+)\]') +RULE_PATTERN = re.compile(r'\s*([^: ]*)\s*:(.*)') +DEF_PATTERN = re.compile(r'^def\s*(\w+)\s*=(.*)$') +PREFIX_PATTERN = re.compile(r'^\s*prefix\s+([\w\_\.\-\/]+)$') + + +def ReadConfigurationInto(path, sections, defs): + current_section = Section(Constant(True)) + sections.append(current_section) + prefix = [] + for line in utils.ReadLinesFrom(path): + header_match = HEADER_PATTERN.match(line) + if header_match: + condition_str = header_match.group(1).strip() + condition = ParseCondition(condition_str) + new_section = Section(condition) + sections.append(new_section) + current_section = new_section + continue + rule_match = RULE_PATTERN.match(line) + if rule_match: + path = prefix + SplitPath(rule_match.group(1).strip()) + value_str = rule_match.group(2).strip() + value = ParseCondition(value_str) + if not value: + return False + current_section.AddRule(Rule(rule_match.group(1), path, value)) + continue + def_match = DEF_PATTERN.match(line) + if def_match: + name = def_match.group(1).lower() + value = ParseCondition(def_match.group(2).strip()) + if not value: + return False + defs[name] = value + continue + prefix_match = PREFIX_PATTERN.match(line) + if prefix_match: + prefix = SplitPath(prefix_match.group(1).strip()) + continue + print "Malformed line: '%s'." % line + return False + return True + + +# --------------- +# --- M a i n --- +# --------------- + + +def BuildOptions(): + result = optparse.OptionParser() + result.add_option("-m", "--mode", + help="The test modes in which to run (comma-separated)", + metavar='[all,debug,release]', + default='debug') + result.add_option("-v", "--verbose", + help="Verbose output", + default=False, + action="store_true") + result.add_option("-p", "--progress", + help="The style of progress indicator (verbose, line, color, mono)", + choices=PROGRESS_INDICATORS.keys(), + default=None) + result.add_option("--report", + help="Print a summary of the tests to be run", + default=False, + action="store_true") + result.add_option("--list", + help="List all the tests, but don't run them", + default=False, + action="store_true") + result.add_option("-s", "--suite", + help="A test suite", + default=[], + action="append") + result.add_option("-t", "--timeout", + help="Timeout in seconds", + default=None, + type="int") + result.add_option("--checked", + help="Run tests in checked mode", + default=False, + action="store_true") + result.add_option("--flag", + help="Pass this additional flag to the VM", + default=[], + action="append") + result.add_option("--arch", + help="The architecture to run tests for", + metavar="[all,ia32,x64,simarm,arm,dartc]", + default=ARCH_GUESS) + result.add_option("--os", + help="The OS to run tests on", + default=OS_GUESS) + result.add_option("--valgrind", + help="Run tests through valgrind", + default=False, + action="store_true") + result.add_option("-j", "--tasks", + help="The number of parallel tasks to run", + metavar=HOST_CPUS, + default=USE_DEFAULT_CPUS, + type="int") + result.add_option("--time", + help="Print timing information after running", + default=False, + action="store_true") + result.add_option("--executable", + help="The executable with which to run the tests", + default=None) + result.add_option("--keep_temporary_files", + help="Do not delete temporary files after running the tests", + default=False, + action="store_true") + result.add_option("--batch", + help="Run multiple tests for dartc architecture in a single vm", + choices=["true","false"], + default="true", + type="choice"); + result.add_option("--optimize", + help="Invoke dart compiler with --optimize flag", + default=False, + action="store_true") + + return result + + +def ProcessOptions(options): + global VERBOSE + VERBOSE = options.verbose + if options.arch == 'all': + options.arch = 'ia32,x64,simarm' + if options.mode == 'all': + options.mode = 'debug,release' + # By default we run with a higher timeout setting in when running on + # a simulated architecture and in debug mode. + if not options.timeout: + options.timeout = TIMEOUT_SECS + if 'dartc' in options.arch: options.timeout *= 4 + elif 'chromium' in options.arch: options.timeout *= 4 + elif 'dartium' in options.arch: options.timeout *= 4 + elif 'debug' in options.mode: options.timeout *= 2 + # TODO(zundel): is arch 'sim' out of date? + if 'sim' in options.arch: options.timeout *= 4 + options.mode = options.mode.split(',') + options.arch = options.arch.split(',') + for mode in options.mode: + if not mode in ['debug', 'release']: + print "Unknown mode %s" % mode + return False + for arch in options.arch: + if not arch in ['ia32', 'x64', 'simarm', 'arm', 'dartc', 'dartium', + 'chromium']: + print "Unknown arch %s" % arch + return False + options.flags = [] + if (arch == 'dartc' or arch == 'chromium') and mode == 'release': + options.flags.append('--optimize') + options.flags.append('--ignore-unrecognized-flags') + if options.checked: + options.flags.append('--enable_asserts') + options.flags.append('--enable_type_checks') + if options.optimize: + options.flags.append('--optimize') + for flag in options.flag: + options.flags.append(flag) + if options.verbose: + print "Flags on the command line:" + for x in options.flags: + print x + # If the user hasn't specified the progress indicator, we pick + # a good one depending on the setting of the verbose option. + if not options.progress: + if options.verbose: options.progress = 'verbose' + else: options.progress = 'mono' + # Options for future use. Such as Windows runner support. + options.suppress_dialogs = True + options.special_command = None + return True + + +REPORT_TEMPLATE = """\ +Total: %(total)i tests + * %(skipped)4d tests will be skipped + * %(nocrash)4d tests are expected to be flaky but not crash + * %(pass)4d tests are expected to pass + * %(fail_ok)4d tests are expected to fail that we won't fix + * %(fail)4d tests are expected to fail that we should fix + * %(crash)4d tests are expected to crash that we should fix + * %(batched)4d tests are running in batch mode\ +""" + +def PrintReport(cases): + """Print a breakdown of which tests are marked pass/skip/fail """ + def IsFlaky(o): + return (PASS in o) and (FAIL in o) and (not CRASH in o) and (not OKAY in o) + def IsFailOk(o): + return (len(o) == 2) and (FAIL in o) and (OKAY in o) + unskipped = [c for c in cases if not SKIP in c.outcomes] + print REPORT_TEMPLATE % { + 'total': len(cases), + 'skipped': len(cases) - len(unskipped), + 'nocrash': len([t for t in unskipped if IsFlaky(t.outcomes)]), + 'pass': len([t for t in unskipped if list(t.outcomes) == [PASS]]), + 'fail_ok': len([t for t in unskipped if IsFailOk(t.outcomes)]), + 'fail': len([t for t in unskipped if list(t.outcomes) == [FAIL]]), + 'crash': len([t for t in unskipped if list(t.outcomes) == [CRASH]]), + 'batched' : len([t for t in unskipped if t.case.IsBatchable()]) + } + + +def PrintTests(cases): + has_errors = False + for case in cases: + try: + case.case.GetCommand() + except: + sys.stderr.write(case.case.filename + '\n') + has_errors = True + if has_errors: + raise Exception('Errors in above files') + for case in [c for c in cases if not SKIP in c.outcomes]: + print "%s\t%s\t%s\t%s" %('/'.join(case.case.path), + ','.join(case.outcomes), + case.case.IsNegative(), + '\t'.join(case.case.GetCommand()[1:])) + + +class Pattern(object): + + def __init__(self, pattern): + self.pattern = pattern + self.compiled = None + + def match(self, str): + if not self.compiled: + pattern = "^" + self.pattern.replace('*', '.*') + "$" + self.compiled = re.compile(pattern) + return self.compiled.match(str) + + def __str__(self): + return self.pattern + + +def SplitPath(s): + stripped = [ c.strip() for c in s.split('/') ] + return [ Pattern(s) for s in stripped if len(s) > 0 ] + + +def GetSpecialCommandProcessor(value): + if (not value) or (value.find('@') == -1): + def ExpandCommand(args): + return args + return ExpandCommand + else: + pos = value.find('@') + import urllib + prefix = urllib.unquote(value[:pos]).split() + suffix = urllib.unquote(value[pos+1:]).split() + def ExpandCommand(args): + return prefix + args + suffix + return ExpandCommand + + +def GetSuites(test_root): + def IsSuite(path): + return isdir(path) and exists(join(path, 'testcfg.py')) + return [ f for f in os.listdir(test_root) if IsSuite(join(test_root, f)) ] + + +def FormatTime(d): + millis = round(d * 1000) % 1000 + return time.strftime("%M:%S.", time.gmtime(d)) + ("%03i" % millis) + + +def Main(): + utils.ConfigureJava() + parser = BuildOptions() + (options, args) = parser.parse_args() + if not ProcessOptions(options): + parser.print_help() + return 1 + + client = abspath(join(dirname(sys.argv[0]), '..')) + repositories = [] + for component in os.listdir(client) + ['.']: + test_path = join(client, component, 'tests') + if exists(test_path) and isdir(test_path): + suites = GetSuites(test_path) + repositories += [TestRepository(join(test_path, name)) for name in suites] + repositories += [TestRepository(a) for a in options.suite] + + root = LiteralTestSuite(repositories) + if len(args) == 0: + paths = [SplitPath(t) for t in BUILT_IN_TESTS] + else: + paths = [ ] + for arg in args: + path = SplitPath(arg) + paths.append(path) + + # Check for --valgrind option. If enabled, we overwrite the special + # command flag with a command that uses the tools/valgrind.py script. + if options.valgrind: + run_valgrind = join(client, 'runtime', 'tools', 'valgrind.py') + options.special_command = "python -u " + run_valgrind + " @" + + context = Context(client, + VERBOSE, + options.os, + options.timeout, + GetSpecialCommandProcessor(options.special_command), + options.suppress_dialogs, + options.executable, + options.flags, + options.keep_temporary_files, + options.batch) + + # Get status for tests + sections = [ ] + defs = { } + root.GetTestStatus(context, sections, defs) + config = Configuration(sections, defs) + + # List the tests + all_cases = [ ] + all_unused = [ ] + unclassified_tests = [ ] + globally_unused_rules = None + for path in paths: + for mode in options.mode: + for arch in options.arch: + env = { + 'mode': mode, + 'system': utils.GuessOS(), + 'arch': arch, + } + test_list = root.ListTests([], path, context, mode, arch) + unclassified_tests += test_list + (cases, unused_rules, all_outcomes) = config.ClassifyTests(test_list, env) + if globally_unused_rules is None: + globally_unused_rules = set(unused_rules) + else: + globally_unused_rules = globally_unused_rules.intersection(unused_rules) + all_cases += cases + all_unused.append(unused_rules) + + if options.report: + PrintReport(all_cases) + + if options.list: + PrintTests(all_cases) + return 0; + + result = None + def DoSkip(case): + return SKIP in case.outcomes or SLOW in case.outcomes + cases_to_run = [ c for c in all_cases if not DoSkip(c) ] + if len(cases_to_run) == 0: + print "No tests to run." + return 0 + else: + try: + start = time.time() + if RunTestCases(cases_to_run, options.progress, options.tasks, + context): + result = 0 + else: + result = 1 + duration = time.time() - start + except KeyboardInterrupt: + print "Exiting on KeyboardInterrupt" + return 1 + + if options.time: + print + print "--- Total time: %s ---" % FormatTime(duration) + timed_tests = [ t.case for t in cases_to_run if not t.case.duration is None ] + timed_tests.sort(lambda a, b: a.CompareTime(b)) + index = 1 + for entry in timed_tests[:20]: + t = FormatTime(entry.duration) + print "%4i (%s) %s" % (index, t, entry.GetLabel()) + index += 1 + + return result + + +if __name__ == '__main__': + sys.exit(Main()) diff --git a/tools/testing.py b/tools/testing.py new file mode 100644 index 00000000000..a2601a41f2b --- /dev/null +++ b/tools/testing.py @@ -0,0 +1,365 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +import atexit +import fileinput +import os +import test +import platform +import re +import run +import sys +import tempfile + +import test +import utils + +from os.path import join, exists, basename + +import utils # This is tools.utils + +class Error(Exception): + pass + +class TestConfigurationError(Error): + pass + +class StandardTestCase(test.TestCase): + def __init__(self, context, path, filename, mode, arch): + super(StandardTestCase, self).__init__(context, path) + self.filename = filename + self.mode = mode + self.arch = arch + self.run_arch = run.GetArchitecture(self.arch, self.mode, self.filename) + for flag in context.flags: + self.run_arch.vm_options.append(flag) + + def IsNegative(self): + return self.GetName().endswith("NegativeTest") + + def GetLabel(self): + return "%s%s %s" % (self.mode, self.arch, '/'.join(self.path)) + + def GetCommand(self): + return self.run_arch.GetRunCommand(); + + def GetName(self): + return self.path[-1] + + def GetPath(self): + return os.path.dirname(self.filename) + + def GetSource(self): + return file(self.filename).read() + + +class MultiTestCase(StandardTestCase): + + def __init__(self, context, path, filename, kind, mode, arch): + super(MultiTestCase, self).__init__(context, path, filename, mode, arch) + self.kind = kind + + def GetCommand(self): + return self.run_arch.GetRunCommand( + fatal_static_type_errors=(self.kind == 'static type error')); + + def IsNegative(self): + if self.kind == 'compile-time error': + return True + if self.kind == 'runtime error': + return False + if self.kind == 'static type error': + return self.run_arch.HasFatalTypeErrors() + return False + + +class StandardTestConfiguration(test.TestConfiguration): + LEGAL_KINDS = set(['compile-time error', + 'runtime error', + 'static type error', + 'dynamic type error']) + + def __init__(self, context, root): + super(StandardTestConfiguration, self).__init__(context, root) + + def _cleanup(self, tests): + if self.context.keep_temporary_files: return + + dirs = [] + for t in tests: + if t.run_arch != None: + temp_dir = t.run_arch.temp_dir + if temp_dir != None: dirs.append(temp_dir) + + if len(dirs) == 0: return + if not utils.Daemonize(): return + + os.execlp('rm', *(['rm', '-rf'] + dirs)) + + + def CreateTestCases(self, test_path, path, filename, mode, arch): + tags = {} + if filename.endswith(".dart"): + tags = self.SplitMultiTest(test_path, filename) + if arch in ['dartium', 'chromium']: + if tags: + return [] + else: + return [BrowserTestCase( + self.context, test_path, filename, False, mode, arch)] + else: + tests = [] + if tags: + for tag in sorted(tags): + kind, test_source = tags[tag] + if not self.Contains(path, test_path + [tag]): + continue + tests.append(MultiTestCase(self.context, + test_path + [tag], + test_source, + kind, + mode, + arch)) + else: + tests.append(StandardTestCase(self.context, + test_path, + filename, + mode, + arch)) + return tests + + def ListTests(self, current_path, path, mode, arch): + tests = [] + for root, dirs, files in os.walk(join(self.root, 'src')): + for f in [x for x in files if self.IsTest(x)]: + if f.endswith(".dart"): + test_path = current_path + [ f[:-5] ] # Remove .dart suffix. + elif f.endswith(".app"): + test_path = current_path + [ f[:-4] ] # Remove .app suffix. + if not self.Contains(path, test_path): + continue + tests.extend(self.CreateTestCases(test_path, path, join(root, f), + mode, arch)) + atexit.register(lambda: self._cleanup(tests)) + return tests + + def IsTest(self, name): + return name.endswith('Test.dart') or name.endswith("Test.app") + + def GetTestStatus(self, sections, defs): + status = join(self.root, basename(self.root) + '.status') + if exists(status): + test.ReadConfigurationInto(status, sections, defs) + + def SplitMultiTest(self, test_path, filename): + (name, extension) = os.path.splitext(os.path.basename(filename)) + with open(filename, 'r') as s: + source = s.read() + lines = source.splitlines() + tags = {} + for line in lines: + (code, sep, info) = line.partition(' /// ') + if sep: + (tag, sep, kind) = info.partition(': ') + if tags.has_key(tag): + raise utils.Error('duplicated tag %s' % tag) + if kind not in StandardTestConfiguration.LEGAL_KINDS: + raise utils.Error('unrecognized kind %s' % kind) + tags[tag] = kind + if not tags: + return {} + tests = {} + generated_test_dir = os.path.join(self.context.workspace, 'generated_tests') + generated_test_dir = os.path.join(generated_test_dir, *test_path[:-1]) + if not os.path.exists(generated_test_dir): + os.makedirs(generated_test_dir) + for tag in tags: + test_lines = [] + for line in lines: + if ' /// ' in line: + if ' /// %s:' % tag in line: + test_lines.append(line) + else: + test_lines.append('// %s' % line) + else: + test_lines.append(line) + test_filename = os.path.join(generated_test_dir, + '%s_%s%s' % (name, tag, extension)) + with open(test_filename, 'w') as test_file: + for line in test_lines: + print >> test_file, line + tests[tag] = (tags[tag], test_filename) + test_filename = os.path.join(generated_test_dir, + '%s%s' % (name, extension)) + with open(test_filename, 'w') as test_file: + for line in lines: + if ' /// ' not in line: + print >> test_file, line + else: + print >> test_file, '//', line + tests['none'] = ('', test_filename) + return tests + +class BrowserTestCase(StandardTestCase): + def __init__(self, context, path, filename, + fatal_static_type_errors, mode, arch): + super(BrowserTestCase, self).__init__(context, path, filename, mode, arch) + self.fatal_static_type_errors = fatal_static_type_errors + + def Run(self): + command = self.run_arch.GetCompileCommand(self.fatal_static_type_errors) + if command != None: + # We change the directory where dartc will be launched because + # it is not predictable on the location of the compiled file. In + # case the test is a web test, we make sure the app file is not + # in a subdirectory. + cwd = None + if self.run_arch.is_web_test: cwd = self.run_arch.temp_dir + command = command[:1] + self.context.flags + command[1:] + test_output = self.RunCommand(command, cwd=cwd) + + # If errors were found, fail fast and show compile errors: + if test_output.output.exit_code != 0: + if not self.context.keep_temporary_files: + self.run_arch.Cleanup() + return test_output + + command = self.run_arch.GetRunCommand(); + test_output = self.RunCommand(command) + # The return value of DumpRenderedTree does not indicate test failing, but + # the output does. + if self.run_arch.HasFailed(test_output.output.stdout): + test_output.output.exit_code = 1 + + # TODO(ngeoffray): We run out of space on the build bots for these tests if + # the temp directories are not removed right after running the test. + if not self.context.keep_temporary_files: + self.run_arch.Cleanup() + + return test_output + + +class BrowserTestConfiguration(StandardTestConfiguration): + def __init__(self, context, root, fatal_static_type_errors=False): + super(BrowserTestConfiguration, self).__init__(context, root) + self.fatal_static_type_errors = fatal_static_type_errors + + def ListTests(self, current_path, path, mode, arch): + tests = [] + for root, dirs, files in os.walk(self.root): + for f in [x for x in files if self.IsTest(x)]: + relative = os.path.relpath(root, self.root).split(os.path.sep) + test_path = current_path + relative + [os.path.splitext(f)[0]] + if not self.Contains(path, test_path): + continue + tests.append(BrowserTestCase(self.context, + test_path, + join(root, f), + self.fatal_static_type_errors, + mode, + arch)) + atexit.register(lambda: self._cleanup(tests)) + return tests + + def IsTest(self, name): + return name.endswith('_tests.dart') + + +class CompilationTestCase(test.TestCase): + """ Run the dartc compiler on a given top level dart file """ + def __init__(self, path, context, filename, mode, arch): + super(CompilationTestCase, self).__init__(context, path) + self.filename = filename + self.mode = mode + self.arch = arch + self.run_arch = run.GetArchitecture(self.arch, self.mode, + self.filename) + self.temp_dir = tempfile.mkdtemp(prefix='dartc-output-') + + def IsNegative(self): + return False + + def GetLabel(self): + return "%s/%s %s" % (self.mode, self.arch, '/'.join(self.path)) + + def GetCommand(self): + cmd = self.context.GetDartC(self.mode, self.arch); + cmd += self.context.flags + cmd += ['-check-only', + '-fatal-type-errors', + '-Werror', + '-out', self.temp_dir, + self.filename] + + return cmd + + def GetName(self): + return self.path[-1] + + +class CompilationTestConfiguration(test.TestConfiguration): + """ Configuration that searches specific directories for apps to compile + + Expects a status file named dartc.status + """ + def __init__(self, context, root): + super(CompilationTestConfiguration, self).__init__(context, root) + + def ListTests(self, current_path, path, mode, arch): + tests = [] + client_path = os.path.normpath(os.path.join(self.root, '..', '..')) + + for src_dir in self.SourceDirs(): + for root, dirs, files in os.walk(os.path.join(client_path, src_dir)): + ignore_dirs = [d for d in dirs if d.startswith('.')] + for d in ignore_dirs: + dirs.remove(d) + for f in files: + filename = [os.path.basename(client_path)] + filename.extend(root[len(client_path) + 1:].split(os.path.sep)) + filename.append(f) # Remove .lib or .app suffix. + test_path = current_path + filename + test_dart_file = os.path.join(root, f) + if ((not self.Contains(path, test_path)) + or (not self.IsTest(test_dart_file))): + continue + tests.append(CompilationTestCase(test_path, + self.context, + test_dart_file, + mode, + arch)) + atexit.register(lambda: self._cleanup(tests)) + return tests + + def SourceDirs(self): + """ Returns a list of directories to scan for files to compile """ + raise TestConfigurationError( + "Subclasses must implement SourceDirs()") + + def IsTest(self, name): + if (not name.endswith('.dart')): + return False + if (os.path.exists(name)): + # TODO(dgrove): can we end reading the input early? + for line in fileinput.input(name): + if (re.match('#', line)): + fileinput.close() + return True + fileinput.close() + return False + return False + + def GetTestStatus(self, sections, defs): + status = os.path.join(self.root, 'dartc.status') + if os.path.exists(status): + test.ReadConfigurationInto(status, sections, defs) + + def _cleanup(self, tests): + if not utils.Daemonize(): return + os.execlp('rm', *(['rm', '-rf'] + [t.temp_dir for t in tests])) + raise + + +def GetConfiguration(context, root): + return CompilationTestConfiguration(context, root) diff --git a/tools/utils.py b/tools/utils.py new file mode 100644 index 00000000000..adeb5410037 --- /dev/null +++ b/tools/utils.py @@ -0,0 +1,222 @@ +# Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +# for details. All rights reserved. Use of this source code is governed by a +# BSD-style license that can be found in the LICENSE file. + +# This file contains a set of utilities functions used by other Python-based +# scripts. + +import commands +import os +import platform +import re +import subprocess +import sys + + +# Try to guess the host operating system. +def GuessOS(): + id = platform.system() + if id == "Linux": + return "linux" + elif id == "Darwin": + return "macos" + elif id == "Windows" or id == "Microsoft": + # On Windows Vista platform.system() can return "Microsoft" with some + # versions of Python, see http://bugs.python.org/issue1082 for details. + return "win32" + elif id == 'FreeBSD': + return 'freebsd' + elif id == 'OpenBSD': + return 'openbsd' + elif id == 'SunOS': + return 'solaris' + else: + return None + + +# Try to guess the host architecture. +def GuessArchitecture(): + id = platform.machine() + if id.startswith('arm'): + return 'arm' + elif (not id) or (not re.match('(x|i[3-6])86', id) is None): + return 'ia32' + elif id == 'i86pc': + return 'ia32' + else: + return None + + +# Try to guess the number of cpus on this machine. +def GuessCpus(): + if os.path.exists("/proc/cpuinfo"): + return int(commands.getoutput("grep -E '^processor' /proc/cpuinfo | wc -l")) + if os.path.exists("/usr/bin/hostinfo"): + return int(commands.getoutput('/usr/bin/hostinfo | grep "processors are logically available." | awk "{ print \$1 }"')) + win_cpu_count = os.getenv("NUMBER_OF_PROCESSORS") + if win_cpu_count: + return int(win_cpu_count) + return int(os.getenv("DART_NUMBER_OF_CORES", 2)) + + +# Returns true if we're running under Windows. +def IsWindows(): + return GuessOS() == 'win32' + + +# Reads a text file into an array of strings - one for each +# line. Strips comments in the process. +def ReadLinesFrom(name): + result = [] + for line in open(name): + if '#' in line: + line = line[:line.find('#')] + line = line.strip() + if len(line) == 0: + continue + result.append(line) + return result + +# Filters out all arguments until the next '--' argument +# occurs. +def ListArgCallback(option, opt_str, value, parser): + if value is None: + value = [] + + for arg in parser.rargs: + if arg[:2].startswith('--'): + break + value.append(arg) + + del parser.rargs[:len(value)] + setattr(parser.values, option.dest, value) + + +# Filters out all argument until the first non '-' or the +# '--' argument occurs. +def ListDartArgCallback(option, opt_str, value, parser): + if value is None: + value = [] + + for arg in parser.rargs: + if arg[:2].startswith('--') or arg[0] != '-': + break + value.append(arg) + + del parser.rargs[:len(value)] + setattr(parser.values, option.dest, value) + + +# Mapping table between build mode and build configuration. +BUILD_MODES = { + 'debug': 'Debug', + 'release': 'Release', +} + + +# Mapping table between OS and build output location. +BUILD_ROOT = { + 'win32': os.path.join(''), + 'linux': os.path.join('out'), + 'freebsd': os.path.join('out'), + 'macos': os.path.join('xcodebuild'), +} + +def GetBuildMode(mode): + global BUILD_MODES + return BUILD_MODES[mode] + + +def GetBuildConf(mode, arch): + return GetBuildMode(mode) + "_" + arch + +# Temporary variables that we should remove once the buildbots build 'ia32' +# instead of 'dartc'. +RUN_FROM_TOP_DIR = os.path.basename(os.path.abspath(os.curdir)) == 'dart' +ARCH_GUESS = GuessArchitecture() + +def GetBuildRoot(target_os, mode=None, arch=None): + if arch == 'dartc' and RUN_FROM_TOP_DIR: arch = ARCH_GUESS + global BUILD_ROOT + if mode: + return os.path.join(BUILD_ROOT[target_os], GetBuildConf(mode, arch)) + else: + return BUILD_ROOT[target_os] + + +def ParseTestOptions(pattern, source, workspace): + match = pattern.search(source) + # Options should be a single line in the test case no splits. + if match: + def rewrite(path, workspace): + # Paths in test files are always specified using '/' + # as the path separator. Replace with the actual + # path separator before use. + if ('/' in path): + split_path = path.split('/') + path = os.sep.join(split_path) + path = os.path.join(workspace, path) + if not os.path.exists(path): + raise Exception(path) + return path + return [rewrite(o, workspace) for o in match.group(1).split(' ')] + else: + return None + + +def ConfigureJava(): + java_home = '/usr/libexec/java_home' + if os.path.exists(java_home): + proc = subprocess.Popen([java_home, '-v', '1.6+'], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE) + (stdout, stderr) = proc.communicate() + if proc.wait() != 0: + raise ToolError('non-zero exit code from ' + java_home) + new = stdout.strip() + current = os.getenv('JAVA_HOME', default=new) + if current != new: + sys.stderr.write('Please set JAVA_HOME to %s\n' % new) + os.putenv('JAVA_HOME', new) + + +def Daemonize(): + """ + Create a detached background process (daemon). Returns True for + the daemon, False for the parent process. + See: http://www.faqs.org/faqs/unix-faq/programmer/faq/ + "1.7 How do I get my program to act like a daemon?" + """ + if os.fork() > 0: + return False + os.setsid() + if os.fork() > 0: + os._exit(0) + raise + return True + + +def Main(argv): + print "GuessOS() -> ", GuessOS() + print "GuessArchitecture() -> ", GuessArchitecture() + print "GuessCpus() -> ", GuessCpus() + print "IsWindows() -> ", IsWindows() + + +class Error(Exception): + pass + + +class ToolError(Exception): + """Deprecated exception, use Error instead.""" + + def __init__(self, value): + self.value = value + + def __str__(self): + return repr(self.value) + + +if __name__ == "__main__": + import sys + Main(sys.argv) diff --git a/tools/utils/elisp/dart-mode.el b/tools/utils/elisp/dart-mode.el new file mode 100644 index 00000000000..9d6abbd792f --- /dev/null +++ b/tools/utils/elisp/dart-mode.el @@ -0,0 +1,165 @@ +;;; dart-mode.el --- a Dart mode for emacs based upon CC mode. + +;; This program is free software; you can redistribute it and/or modify +;; it under the terms of the GNU General Public License as published by +;; the Free Software Foundation; either version 2 of the License, or +;; (at your option) any later version. +;; +;; This program is distributed in the hope that it will be useful, +;; but WITHOUT ANY WARRANTY; without even the implied warranty of +;; MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +;; GNU General Public License for more details. +;; +;; You should have received a copy of the GNU General Public License +;; along with this program; see the file COPYING. If not, write to +;; the Free Software Foundation, Inc., 59 Temple Place - Suite 330, +;; Boston, MA 02111-1307, USA. + +;;; Commentary: + +;; This is a basic Dart mode based on Martin Stjernholm's GPL'd +;; derived-mode-ex.el template. It uses cc-mode and falls back on +;; Java for basic rules. +;; +;; Note: The interface used in this file requires CC Mode 5.30 or +;; later. + +;;; Code: + +(require 'cc-mode) + +;; These are only required at compile time to get the sources for the +;; language constants. (The cc-fonts require and the font-lock +;; related constants could additionally be put inside an +;; (eval-after-load "font-lock" ...) but then some trickery is +;; necessary to get them compiled.) +(eval-when-compile + (require 'cc-langs) + (require 'cc-fonts)) + +(eval-and-compile + ;; Make our mode known to the language constant system. Use Java + ;; mode as the fallback for the constants we don't change here. + ;; This needs to be done also at compile time since the language + ;; constants are evaluated then. + (c-add-language 'dart-mode 'java-mode)) + +;; Dart has no boolean but a string and a vector type. +(c-lang-defconst c-primitive-type-kwds + dart (append '("bool" "var") + (delete "boolean" + ;; Use append to not be destructive on the + ;; return value below. + (append + ;; Due to the fallback to Java, we need not give + ;; a language to `c-lang-const'. + (c-lang-const c-primitive-type-kwds) + nil)))) + +;; Recognize member init lists after colons in Dart. +(c-lang-defconst c-nonlabel-token-key + dart (concat "\\s\(\\|" (c-lang-const c-nonlabel-token-key))) + +;; No cpp in this language, but there's still a "#include" directive to +;; fontify. (The definitions for the extra keywords above are enough +;; to incorporate them into the fontification regexps for types and +;; keywords, so no additional font-lock patterns are required.) +(c-lang-defconst c-cpp-matchers + dart (cons + ;; Use the eval form for `font-lock-keywords' to be able to use + ;; the `c-preprocessor-face-name' variable that maps to a + ;; suitable face depending on the (X)Emacs version. + '(eval . (list "^\\s *\\(#include\\)\\>\\(.*\\)" + (list 1 c-preprocessor-face-name) + '(2 font-lock-string-face))) + ;; There are some other things in `c-cpp-matchers' besides the + ;; preprocessor support, so include it. + (c-lang-const c-cpp-matchers))) + +(defcustom dart-font-lock-extra-types nil + "*List of extra types (aside from the type keywords) to recognize in Dart mode. +Each list item should be a regexp matching a single identifier.") + +(defconst dart-font-lock-keywords-1 (c-lang-const c-matchers-1 dart) + "Minimal highlighting for Dart mode.") + +(defconst dart-font-lock-keywords-2 (c-lang-const c-matchers-2 dart) + "Fast normal highlighting for Dart mode.") + +(defconst dart-font-lock-keywords-3 (c-lang-const c-matchers-3 dart) + "Accurate normal highlighting for Dart mode.") + +(defvar dart-font-lock-keywords dart-font-lock-keywords-3 + "Default expressions to highlight in Dart mode.") + +(defvar dart-mode-syntax-table nil + "Syntax table used in dart-mode buffers.") +(or dart-mode-syntax-table + (setq dart-mode-syntax-table + (funcall (c-lang-const c-make-mode-syntax-table dart)))) + +(defvar dart-mode-abbrev-table nil + "Abbreviation table used in dart-mode buffers.") +(c-define-abbrev-table 'dart-mode-abbrev-table + ;; Keywords that if they occur first on a line might alter the + ;; syntactic context, and which therefore should trig reindentation + ;; when they are completed. + '(("else" "else" c-electric-continued-statement 0) + ("while" "while" c-electric-continued-statement 0) + ("catch" "catch" c-electric-continued-statement 0) + ("finally" "finally" c-electric-continued-statement 0))) + +(defvar dart-mode-map (let ((map (c-make-inherited-keymap))) + ;; Add bindings which are only useful for Dart + map) + "Keymap used in dart-mode buffers.") + +(easy-menu-define dart-menu dart-mode-map "Dart Mode Commands" + ;; Can use `dart' as the language for `c-mode-menu' + ;; since its definition covers any language. In + ;; this case the language is used to adapt to the + ;; nonexistence of a cpp pass and thus removing some + ;; irrelevant menu alternatives. + (cons "Dart" (c-lang-const c-mode-menu dart))) + +;;;###autoload +(add-to-list 'auto-mode-alist '("\\.dart\\'" . dart-mode)) + +;;;###autoload +(defun dart-mode () + "Major mode for editing Dart code. +This is a simple example of a separate mode derived from CC Mode to +support a language with syntax similar to C/C++/ObjC/Java/IDL/Pike. + +The hook `c-mode-common-hook' is run with no args at mode +initialization, then `dart-mode-hook'. + +Key bindings: +\\{dart-mode-map}" + (interactive) + (kill-all-local-variables) + (c-initialize-cc-mode t) + (set-syntax-table dart-mode-syntax-table) + (setq major-mode 'dart-mode + mode-name "Dart" + local-abbrev-table dart-mode-abbrev-table + abbrev-mode t) + (use-local-map c-mode-map) + ;; `c-init-language-vars' is a macro that is expanded at compile + ;; time to a large `setq' with all the language variables and their + ;; customized values for our language. + (c-init-language-vars dart-mode) + ;; `c-common-init' initializes most of the components of a CC Mode + ;; buffer, including setup of the mode menu, font-lock, etc. + ;; There's also a lower level routine `c-basic-common-init' that + ;; only makes the necessary initialization to get the syntactic + ;; analysis and similar things working. + (c-common-init 'dart-mode) + (easy-menu-add dart-menu) + (run-hooks 'c-mode-common-hook) + (run-hooks 'dart-mode-hook) + (c-update-modeline)) + +(provide 'dart-mode) + +;;; dart-mode.el ends here diff --git a/tools/utils/vim/syntax/dart.vim b/tools/utils/vim/syntax/dart.vim new file mode 100644 index 00000000000..c8cc719d0bb --- /dev/null +++ b/tools/utils/vim/syntax/dart.vim @@ -0,0 +1,94 @@ +" Vim syntax file " Language: Dart +" Copyright (c) 2011, the Dart project authors. Please see the AUTHORS file +" for details. All rights reserved. Use of this source code is governed by a +" BSD-style license that can be found in the LICENSE file. + +" Quit when a syntax file was already loaded +if !exists("main_syntax") + if version < 600 + syntax clear + elseif exists("b:current_syntax") + finish + endif + " we define it here so that included files can test for it + let main_syntax='dart' + syn region dartFold start="{" end="}" transparent fold +endif + +" keyword definitions +syn keyword dartConditional if else switch +syn keyword dartRepeat while for +syn keyword dartBoolean true false +syn keyword dartConstant null +syn keyword dartTypedef this super class typedef +syn keyword dartOperator new is in factory +syn match dartOperator "+\|-\|*\|[~]\=/\|%\|||\|&&\|!\|==[=]\=" +syn keyword dartType void var const bool int double num +syn keyword dartStatement return +syn keyword dartStorageClass static final +syn keyword dartExceptions throw try catch finally +syn keyword dartAssert assert +syn keyword dartClassDecl extends implements interface +" TODO(antonm): check if labels on break and continue are supported. +syn keyword dartBranch break continue nextgroup=dartUserLabelRef skipwhite +syn keyword dartKeyword function get set +syn match dartUserLabelRef "\k\+" contained +syn match dartVarArg "\.\.\." + +" TODO(antonm): consider conditional highlighting of corelib classes. + +syn region dartLabelRegion transparent matchgroup=dartLabel start="\" matchgroup=NONE end=":" +syn keyword dartLabel default + +" Comments +syn keyword dartTodo contained TODO FIXME XXX +syn region dartComment start="/\*" end="\*/" contains=dartTodo,dartDocLink,@Spell +syn match dartLineComment "//.*" contains=dartTodo,@Spell +syn region dartDocLink contained start=+\[+ end=+\]+ + +" Strings +syn region dartString start=+"+ end=+"+ contains=@Spell,dartInterpolation,dartSpecialChar +syn region dartString start=+'+ end=+'+ contains=@Spell,dartInterpolation,dartSpecialChar +syn match dartInterpolation contained "\$\(\w+\|{\w\+}\)" +syn match dartSpecialChar contained "\\\([4-9]\d\|[0-3]\d\d\|[\"\\'ntbrf]\|u\x\{4\}\)" + +" Numbers +syn match dartNumber "\<\d\+\(\.\d\+\)\=\>" + +" The default highlighting. +command! -nargs=+ HiLink hi def link +HiLink dartVarArg Function +HiLink dartBranch Conditional +HiLink dartUserLabelRef dartUserLabel +HiLink dartLabel Label +HiLink dartUserLabel Label +HiLink dartConditional Conditional +HiLink dartRepeat Repeat +HiLink dartExceptions Exception +HiLink dartAssert Statement +HiLink dartStorageClass StorageClass +HiLink dartClassDecl dartStorageClass +HiLink dartBoolean Boolean +HiLink dartString String +HiLink dartNumber Number +HiLink dartStatement Statement +HiLink dartOperator Operator +HiLink dartComment Comment +HiLink dartLineComment Comment +HiLink dartConstant Constant +HiLink dartTypedef Typedef +HiLink dartTodo Todo +HiLink dartKeyword Keyword +HiLink dartType Type +HiLink dartInterpolation PreProc +HiLink dartDocLink SpecialComment +HiLink dartSpecialChar SpecialChar +delcommand HiLink + +let b:current_syntax = "dart" + +if main_syntax == 'dart' + unlet main_syntax +endif + +let b:spell_options="contained"