3 # Copyright (c) 2009 Google Inc. All rights reserved.
4 # Use of this source code is governed by a BSD-style license that can be
5 # found in the LICENSE file.
8 TestGyp.py: a testing framework for GYP integration tests.
18 from TestCommon import __all__
25 class TestGypBase(TestCommon.TestCommon):
27 Class for controlling end-to-end tests of gyp generators.
29 Instantiating this class will create a temporary directory and
30 arrange for its destruction (via the TestCmd superclass) and
31 copy all of the non-gyptest files in the directory hierarchy of the
34 The default behavior is to test the 'gyp' or 'gyp.bat' file in the
35 current directory. An alternative may be specified explicitly on
36 instantiation, or by setting the TESTGYP_GYP environment variable.
38 This class should be subclassed for each supported gyp generator
39 (format). Various abstract methods below define calling signatures
40 used by the test scripts to invoke builds on the generated build
41 configuration and to run executables generated by those builds.
47 _exe = TestCommon.exe_suffix
48 _obj = TestCommon.obj_suffix
49 shobj_ = TestCommon.shobj_prefix
50 _shobj = TestCommon.shobj_suffix
51 lib_ = TestCommon.lib_prefix
52 _lib = TestCommon.lib_suffix
53 dll_ = TestCommon.dll_prefix
54 _dll = TestCommon.dll_suffix
56 # Constants to represent different targets.
58 DEFAULT = '__default__'
60 # Constants for different target types.
61 EXECUTABLE = '__executable__'
62 STATIC_LIB = '__static_lib__'
63 SHARED_LIB = '__shared_lib__'
65 def __init__(self, gyp=None, *args, **kw):
66 self.origin_cwd = os.path.abspath(os.path.dirname(sys.argv[0]))
69 gyp = os.environ.get('TESTGYP_GYP')
71 if sys.platform == 'win32':
75 self.gyp = os.path.abspath(gyp)
77 self.initialize_build_tool()
79 if not kw.has_key('match'):
80 kw['match'] = TestCommon.match_exact
82 if not kw.has_key('workdir'):
83 # Default behavior: the null string causes TestCmd to create
84 # a temporary directory for us.
87 formats = kw.get('formats', [])
88 if kw.has_key('formats'):
91 super(TestGypBase, self).__init__(*args, **kw)
93 excluded_formats = set([f for f in formats if f[0] == '!'])
94 included_formats = set(formats) - excluded_formats
95 if ('!'+self.format in excluded_formats or
96 included_formats and self.format not in included_formats):
97 msg = 'Invalid test for %r format; skipping test.\n'
98 self.skip_test(msg % self.format)
100 self.copy_test_configuration(self.origin_cwd, self.workdir)
101 self.set_configuration(None)
103 def built_file_must_exist(self, name, type=None, **kw):
105 Fails the test if the specified built file name does not exist.
107 return self.must_exist(self.built_file_path(name, type, **kw))
109 def built_file_must_not_exist(self, name, type=None, **kw):
111 Fails the test if the specified built file name exists.
113 return self.must_not_exist(self.built_file_path(name, type, **kw))
115 def built_file_must_match(self, name, contents, **kw):
117 Fails the test if the contents of the specified built file name
118 do not match the specified contents.
120 return self.must_match(self.built_file_path(name, **kw), contents)
122 def built_file_must_not_match(self, name, contents, **kw):
124 Fails the test if the contents of the specified built file name
125 match the specified contents.
127 return self.must_not_match(self.built_file_path(name, **kw), contents)
129 def copy_test_configuration(self, source_dir, dest_dir):
131 Copies the test configuration from the specified source_dir
132 (the directory in which the test script lives) to the
133 specified dest_dir (a temporary working directory).
135 This ignores all files and directories that begin with
136 the string 'gyptest', and all '.svn' subdirectories.
138 for root, dirs, files in os.walk(source_dir):
141 dirs = [ d for d in dirs if not d.startswith('gyptest') ]
142 files = [ f for f in files if not f.startswith('gyptest') ]
144 source = os.path.join(root, dirname)
145 destination = source.replace(source_dir, dest_dir)
146 os.mkdir(destination)
147 if sys.platform != 'win32':
148 shutil.copystat(source, destination)
149 for filename in files:
150 source = os.path.join(root, filename)
151 destination = source.replace(source_dir, dest_dir)
152 shutil.copy2(source, destination)
154 def initialize_build_tool(self):
156 Initializes the .build_tool attribute.
158 Searches the .build_tool_list for an executable name on the user's
159 $PATH. The first tool on the list is used as-is if nothing is found
160 on the current $PATH.
162 for build_tool in self.build_tool_list:
165 if os.path.isabs(build_tool):
166 self.build_tool = build_tool
168 build_tool = self.where_is(build_tool)
170 self.build_tool = build_tool
173 if self.build_tool_list:
174 self.build_tool = self.build_tool_list[0]
176 def relocate(self, source, destination):
178 Renames (relocates) the specified source (usually a directory)
179 to the specified destination, creating the destination directory
182 Note: Don't use this as a generic "rename" operation. In the
183 future, "relocating" parts of a GYP tree may affect the state of
184 the test to modify the behavior of later method calls.
186 destination_dir = os.path.dirname(destination)
187 if not os.path.exists(destination_dir):
188 self.subdir(destination_dir)
189 os.rename(source, destination)
191 def report_not_up_to_date(self):
193 Reports that a build is not up-to-date.
195 This provides common reporting for formats that have complicated
196 conditions for checking whether a build is up-to-date. Formats
197 that expect exact output from the command (make, scons) can
198 just set stdout= when they call the run_build() method.
200 print "Build is not up-to-date:"
201 print self.banner('STDOUT ')
203 stderr = self.stderr()
205 print self.banner('STDERR ')
208 def run_gyp(self, gyp_file, *args, **kw):
210 Runs gyp against the specified gyp_file with the specified args.
212 # TODO: --depth=. works around Chromium-specific tree climbing.
213 args = ('--depth=.', '--format='+self.format, gyp_file) + args
214 return self.run(program=self.gyp, arguments=args, **kw)
216 def run(self, *args, **kw):
218 Executes a program by calling the superclass .run() method.
220 This exists to provide a common place to filter out keyword
221 arguments implemented in this layer, without having to update
222 the tool-specific subclasses or clutter the tests themselves
223 with platform-specific code.
225 if kw.has_key('SYMROOT'):
227 super(TestGypBase, self).run(*args, **kw)
229 def set_configuration(self, configuration):
231 Sets the configuration, to be used for invoking the build
232 tool and testing potential built output.
234 self.configuration = configuration
236 def configuration_dirname(self):
237 if self.configuration:
238 return self.configuration.split('|')[0]
242 def configuration_buildname(self):
243 if self.configuration:
244 return self.configuration
249 # Abstract methods to be defined by format-specific subclasses.
252 def build(self, gyp_file, target=None, **kw):
254 Runs a build of the specified target against the configuration
255 generated from the specified gyp_file.
257 A 'target' argument of None or the special value TestGyp.DEFAULT
258 specifies the default argument for the underlying build tool.
259 A 'target' argument of TestGyp.ALL specifies the 'all' target
260 (if any) of the underlying build tool.
262 raise NotImplementedError
264 def built_file_path(self, name, type=None, **kw):
266 Returns a path to the specified file name, of the specified type.
268 raise NotImplementedError
270 def built_file_basename(self, name, type=None, **kw):
272 Returns the base name of the specified file name, of the specified type.
274 A bare=True keyword argument specifies that prefixes and suffixes shouldn't
277 if not kw.get('bare'):
278 if type == self.EXECUTABLE:
279 name = name + self._exe
280 elif type == self.STATIC_LIB:
281 name = self.lib_ + name + self._lib
282 elif type == self.SHARED_LIB:
283 name = self.dll_ + name + self._dll
286 def run_built_executable(self, name, *args, **kw):
288 Runs an executable program built from a gyp-generated configuration.
290 The specified name should be independent of any particular generator.
291 Subclasses should find the output executable in the appropriate
292 output build directory, tack on any necessary executable suffix, etc.
294 raise NotImplementedError
296 def up_to_date(self, gyp_file, target=None, **kw):
298 Verifies that a build of the specified target is up to date.
300 The subclass should implement this by calling build()
301 (or a reasonable equivalent), checking whatever conditions
302 will tell it the build was an "up to date" null build, and
305 raise NotImplementedError
308 class TestGypGypd(TestGypBase):
310 Subclass for testing the GYP 'gypd' generator (spit out the
311 internal data structure as pretty-printed Python).
316 class TestGypMake(TestGypBase):
318 Subclass for testing the GYP Make generator.
321 build_tool_list = ['make']
323 def build(self, gyp_file, target=None, **kw):
325 Runs a Make build using the Makefiles generated from the specified
328 arguments = kw.get('arguments', [])[:]
329 if self.configuration:
330 arguments.append('BUILDTYPE=' + self.configuration)
331 if target not in (None, self.DEFAULT):
332 arguments.append(target)
333 # Sub-directory builds provide per-gyp Makefiles (i.e.
334 # Makefile.gyp_filename), so use that if there is no Makefile.
335 chdir = kw.get('chdir', '')
336 if not os.path.exists(os.path.join(chdir, 'Makefile')):
337 print "NO Makefile in " + os.path.join(chdir, 'Makefile')
338 arguments.insert(0, '-f')
339 arguments.insert(1, os.path.splitext(gyp_file)[0] + '.Makefile')
340 kw['arguments'] = arguments
341 return self.run(program=self.build_tool, **kw)
342 def up_to_date(self, gyp_file, target=None, **kw):
344 Verifies that a build of the specified Make target is up to date.
346 if target in (None, self.DEFAULT):
347 message_target = 'all'
349 message_target = target
350 kw['stdout'] = "make: Nothing to be done for `%s'.\n" % message_target
351 return self.build(gyp_file, target, **kw)
352 def run_built_executable(self, name, *args, **kw):
354 Runs an executable built by Make.
356 configuration = self.configuration_dirname()
357 libdir = os.path.join('out', configuration, 'lib')
358 # TODO(piman): when everything is cross-compile safe, remove lib.target
359 os.environ['LD_LIBRARY_PATH'] = libdir + '.host:' + libdir + '.target'
360 # Enclosing the name in a list avoids prepending the original dir.
361 program = [self.built_file_path(name, type=self.EXECUTABLE, **kw)]
362 return self.run(program=program, *args, **kw)
363 def built_file_path(self, name, type=None, **kw):
365 Returns a path to the specified file name, of the specified type,
368 Built files are in the subdirectory 'out/{configuration}'.
369 The default is 'out/Default'.
371 A chdir= keyword argument specifies the source directory
372 relative to which the output subdirectory can be found.
374 "type" values of STATIC_LIB or SHARED_LIB append the necessary
375 prefixes and suffixes to a platform-independent library base name.
377 A libdir= keyword argument specifies a library subdirectory other
378 than the default 'obj.target'.
381 chdir = kw.get('chdir')
384 configuration = self.configuration_dirname()
385 result.extend(['out', configuration])
386 if type == self.STATIC_LIB:
387 result.append(kw.get('libdir', 'obj.target'))
388 elif type == self.SHARED_LIB:
389 result.append(kw.get('libdir', 'lib.target'))
390 result.append(self.built_file_basename(name, type, **kw))
391 return self.workpath(*result)
394 class TestGypMSVS(TestGypBase):
396 Subclass for testing the GYP Visual Studio generator.
400 u = r'=== Build: (\d+) succeeded, 0 failed, (\d+) up-to-date, 0 skipped ==='
401 up_to_date_re = re.compile(u, re.M)
403 # Initial None element will indicate to our .initialize_build_tool()
404 # method below that 'devenv' was not found on %PATH%.
406 # Note: we must use devenv.com to be able to capture build output.
407 # Directly executing devenv.exe only sends output to BuildLog.htm.
408 build_tool_list = [None, 'devenv.com']
410 def initialize_build_tool(self):
411 """ Initializes the Visual Studio .build_tool and .uses_msbuild parameters.
413 We use the value specified by GYP_MSVS_VERSION. If not specified, we
414 search %PATH% and %PATHEXT% for a devenv.{exe,bat,...} executable.
415 Failing that, we search for likely deployment paths.
417 super(TestGypMSVS, self).initialize_build_tool()
418 possible_roots = ['C:\\Program Files (x86)', 'C:\\Program Files']
420 '2010': r'Microsoft Visual Studio 10.0\Common7\IDE\devenv.com',
421 '2008': r'Microsoft Visual Studio 9.0\Common7\IDE\devenv.com',
422 '2005': r'Microsoft Visual Studio 8\Common7\IDE\devenv.com'}
423 msvs_version = os.environ.get('GYP_MSVS_VERSION', 'auto')
424 if msvs_version in possible_paths:
425 # Check that the path to the specified GYP_MSVS_VERSION exists.
426 path = possible_paths[msvs_version]
427 for r in possible_roots:
428 bt = os.path.join(r, path)
429 if os.path.exists(bt):
431 self.uses_msbuild = msvs_version >= '2010'
434 print ('Warning: Environment variable GYP_MSVS_VERSION specifies "%s" '
435 'but corresponding "%s" was not found.' % (msvs_version, path))
437 # We found 'devenv' on the path, use that and try to guess the version.
438 for version, path in possible_paths.iteritems():
439 if self.build_tool.find(path) >= 0:
440 self.uses_msbuild = version >= '2010'
443 # If not, assume not MSBuild.
444 self.uses_msbuild = False
446 # Neither GYP_MSVS_VERSION nor the path help us out. Iterate through
447 # the choices looking for a match.
448 for version, path in possible_paths.iteritems():
449 for r in possible_roots:
450 bt = os.path.join(r, path)
451 if os.path.exists(bt):
453 self.uses_msbuild = msvs_version >= '2010'
455 print 'Error: could not find devenv'
457 def build(self, gyp_file, target=None, rebuild=False, **kw):
459 Runs a Visual Studio build using the configuration generated
460 from the specified gyp_file.
462 configuration = self.configuration_buildname()
467 arguments = kw.get('arguments', [])[:]
468 arguments.extend([gyp_file.replace('.gyp', '.sln'),
469 build, configuration])
470 # Note: the Visual Studio generator doesn't add an explicit 'all'
471 # target, so we just treat it the same as the default.
472 if target not in (None, self.ALL, self.DEFAULT):
473 arguments.extend(['/Project', target])
474 if self.configuration:
475 arguments.extend(['/ProjectConfig', self.configuration])
476 kw['arguments'] = arguments
477 return self.run(program=self.build_tool, **kw)
478 def up_to_date(self, gyp_file, target=None, **kw):
480 Verifies that a build of the specified Visual Studio target is up to date.
482 result = self.build(gyp_file, target, **kw)
484 stdout = self.stdout()
485 m = self.up_to_date_re.search(stdout)
488 succeeded = m.group(1)
489 up_to_date = m.group(2)
490 up_to_date = succeeded == '0' and up_to_date == '1'
491 # Figuring out if the build is up to date changed with VS2010.
492 # For builds that should be up to date, I sometimes get
493 # "1 succeeded and 0 up to date". As an ad-hoc measure, we check
494 # this and also verify that th number of output lines is small.
495 # I don't know if this is caused by VS itself or is due to
496 # interaction with virus checkers.
497 if self.uses_msbuild and (succeeded == '1' and
498 up_to_date == '0' and
499 stdout.count('\n') <= 6):
502 self.report_not_up_to_date()
505 def run_built_executable(self, name, *args, **kw):
507 Runs an executable built by Visual Studio.
509 configuration = self.configuration_dirname()
510 # Enclosing the name in a list avoids prepending the original dir.
511 program = [self.built_file_path(name, type=self.EXECUTABLE, **kw)]
512 return self.run(program=program, *args, **kw)
513 def built_file_path(self, name, type=None, **kw):
515 Returns a path to the specified file name, of the specified type,
516 as built by Visual Studio.
518 Built files are in a subdirectory that matches the configuration
519 name. The default is 'Default'.
521 A chdir= keyword argument specifies the source directory
522 relative to which the output subdirectory can be found.
524 "type" values of STATIC_LIB or SHARED_LIB append the necessary
525 prefixes and suffixes to a platform-independent library base name.
528 chdir = kw.get('chdir')
531 result.append(self.configuration_dirname())
532 if type == self.STATIC_LIB:
534 result.append(self.built_file_basename(name, type, **kw))
535 return self.workpath(*result)
538 class TestGypSCons(TestGypBase):
540 Subclass for testing the GYP SCons generator.
543 build_tool_list = ['scons', 'scons.py']
545 def build(self, gyp_file, target=None, **kw):
547 Runs a scons build using the SCons configuration generated from the
550 arguments = kw.get('arguments', [])[:]
551 dirname = os.path.dirname(gyp_file)
553 arguments.extend(['-C', dirname])
554 if self.configuration:
555 arguments.append('--mode=' + self.configuration)
556 if target not in (None, self.DEFAULT):
557 arguments.append(target)
558 kw['arguments'] = arguments
559 return self.run(program=self.build_tool, **kw)
560 def up_to_date(self, gyp_file, target=None, **kw):
562 Verifies that a build of the specified SCons target is up to date.
564 if target in (None, self.DEFAULT):
565 up_to_date_targets = 'all'
567 up_to_date_targets = target
568 up_to_date_lines = []
569 for arg in up_to_date_targets.split():
570 up_to_date_lines.append("scons: `%s' is up to date.\n" % arg)
571 kw['stdout'] = ''.join(up_to_date_lines)
572 arguments = kw.get('arguments', [])[:]
573 arguments.append('-Q')
574 kw['arguments'] = arguments
575 return self.build(gyp_file, target, **kw)
576 def run_built_executable(self, name, *args, **kw):
578 Runs an executable built by scons.
580 configuration = self.configuration_dirname()
581 os.environ['LD_LIBRARY_PATH'] = os.path.join(configuration, 'lib')
582 # Enclosing the name in a list avoids prepending the original dir.
583 program = [self.built_file_path(name, type=self.EXECUTABLE, **kw)]
584 return self.run(program=program, *args, **kw)
585 def built_file_path(self, name, type=None, **kw):
587 Returns a path to the specified file name, of the specified type,
590 Built files are in a subdirectory that matches the configuration
591 name. The default is 'Default'.
593 A chdir= keyword argument specifies the source directory
594 relative to which the output subdirectory can be found.
596 "type" values of STATIC_LIB or SHARED_LIB append the necessary
597 prefixes and suffixes to a platform-independent library base name.
600 chdir = kw.get('chdir')
603 result.append(self.configuration_dirname())
604 if type in (self.STATIC_LIB, self.SHARED_LIB):
606 result.append(self.built_file_basename(name, type, **kw))
607 return self.workpath(*result)
610 class TestGypXcode(TestGypBase):
612 Subclass for testing the GYP Xcode generator.
615 build_tool_list = ['xcodebuild']
617 phase_script_execution = ("\n"
618 "PhaseScriptExecution /\\S+/Script-[0-9A-F]+\\.sh\n"
620 " /bin/sh -c /\\S+/Script-[0-9A-F]+\\.sh\n"
621 "(make: Nothing to be done for `all'\\.\n)?")
623 strip_up_to_date_expressions = [
624 # Various actions or rules can run even when the overall build target
625 # is up to date. Strip those phases' GYP-generated output.
626 re.compile(phase_script_execution, re.S),
628 # The message from distcc_pump can trail the "BUILD SUCCEEDED"
629 # message, so strip that, too.
630 re.compile('__________Shutting down distcc-pump include server\n', re.S),
633 up_to_date_endings = (
634 'Checking Dependencies...\n** BUILD SUCCEEDED **\n', # Xcode 3.0/3.1
635 'Check dependencies\n** BUILD SUCCEEDED **\n\n', # Xcode 3.2
638 def build(self, gyp_file, target=None, **kw):
640 Runs an xcodebuild using the .xcodeproj generated from the specified
643 # Be sure we're working with a copy of 'arguments' since we modify it.
644 # The caller may not be expecting it to be modified.
645 arguments = kw.get('arguments', [])[:]
646 arguments.extend(['-project', gyp_file.replace('.gyp', '.xcodeproj')])
647 if target == self.ALL:
648 arguments.append('-alltargets',)
649 elif target not in (None, self.DEFAULT):
650 arguments.extend(['-target', target])
651 if self.configuration:
652 arguments.extend(['-configuration', self.configuration])
653 symroot = kw.get('SYMROOT', '$SRCROOT/build')
655 arguments.append('SYMROOT='+symroot)
656 kw['arguments'] = arguments
657 return self.run(program=self.build_tool, **kw)
658 def up_to_date(self, gyp_file, target=None, **kw):
660 Verifies that a build of the specified Xcode target is up to date.
662 result = self.build(gyp_file, target, **kw)
664 output = self.stdout()
665 for expression in self.strip_up_to_date_expressions:
666 output = expression.sub('', output)
667 if not output.endswith(self.up_to_date_endings):
668 self.report_not_up_to_date()
671 def run_built_executable(self, name, *args, **kw):
673 Runs an executable built by xcodebuild.
675 configuration = self.configuration_dirname()
676 os.environ['DYLD_LIBRARY_PATH'] = os.path.join('build', configuration)
677 # Enclosing the name in a list avoids prepending the original dir.
678 program = [self.built_file_path(name, type=self.EXECUTABLE, **kw)]
679 return self.run(program=program, *args, **kw)
680 def built_file_path(self, name, type=None, **kw):
682 Returns a path to the specified file name, of the specified type,
685 Built files are in the subdirectory 'build/{configuration}'.
686 The default is 'build/Default'.
688 A chdir= keyword argument specifies the source directory
689 relative to which the output subdirectory can be found.
691 "type" values of STATIC_LIB or SHARED_LIB append the necessary
692 prefixes and suffixes to a platform-independent library base name.
695 chdir = kw.get('chdir')
698 configuration = self.configuration_dirname()
699 result.extend(['build', configuration])
700 result.append(self.built_file_basename(name, type, **kw))
701 return self.workpath(*result)
704 format_class_list = [
712 def TestGyp(*args, **kw):
714 Returns an appropriate TestGyp* instance for a specified GYP format.
716 format = kw.get('format')
720 format = os.environ.get('TESTGYP_FORMAT')
721 for format_class in format_class_list:
722 if format == format_class.format:
723 return format_class(*args, **kw)
724 raise Exception, "unknown format %r" % format