3 # Copyright (C) 2007, 2008, 2011 Apple Inc. All rights reserved.
5 # Redistribution and use in source and binary forms, with or without
6 # modification, are permitted provided that the following conditions
9 # 1. Redistributions of source code must retain the above copyright
10 # notice, this list of conditions and the following disclaimer.
11 # 2. Redistributions in binary form must reproduce the above copyright
12 # notice, this list of conditions and the following disclaimer in the
13 # documentation and/or other materials provided with the distribution.
14 # 3. Neither the name of Apple Computer, Inc. ("Apple") nor the names of
15 # its contributors may be used to endorse or promote products derived
16 # from this software without specific prior written permission.
18 # THIS SOFTWARE IS PROVIDED BY APPLE AND ITS CONTRIBUTORS "AS IS" AND ANY
19 # EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
20 # WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
21 # DISCLAIMED. IN NO EVENT SHALL APPLE OR ITS CONTRIBUTORS BE LIABLE FOR ANY
22 # DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
23 # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
24 # LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND
25 # ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
26 # (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
27 # THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29 # This script attempts to find the point at which a regression (or progression)
30 # of behavior occurred by searching WebKit nightly builds.
32 # To override the location where the nightly builds are downloaded or the path
33 # to the Safari web browser, create a ~/.bisect-buildsrc file with one or more of
34 # the following lines (use "~/" to specify a path from your home directory):
36 # $branch = "branch-name";
37 # $nightlyDownloadDirectory = "~/path/to/nightly/downloads";
38 # $safariPath = "/path/to/Safari.app";
45 use File::Temp qw(tempfile);
47 use Time::HiRes qw(usleep);
49 sub createTempFile($);
50 sub downloadNightly($$$);
51 sub findMacOSXVersion();
52 sub findNearestNightlyIndex(\@$$);
53 sub findSafariVersion($);
55 sub makeNightlyList($$$$);
56 sub max($$) { return $_[0] > $_[1] ? $_[0] : $_[1]; }
57 sub mountAndRunNightly($$$$);
58 sub parseRevisions($$;$);
60 sub printTracLink($$);
65 my %validBranches = map { $_ => 1 } qw(feature-branch trunk);
66 my $branch = $Settings::branch;
67 my $nightlyDownloadDirectory = $Settings::nightlyDownloadDirectory;
68 my $safariPath = $Settings::safariPath;
79 # Fix up -r switches in @ARGV
80 @ARGV = map { /^(-r)(.+)$/ ? ($1, $2) : $_ } @ARGV;
82 my $result = GetOptions(
83 "b|branch=s" => \$branch,
84 "d|download-directory=s" => \$nightlyDownloadDirectory,
85 "h|help" => \$showHelp,
86 "l|local!" => \$localOnly,
87 "p|progression!" => \$isProgression,
88 "r|revisions=s" => \&parseRevisions,
89 "safari-path=s" => \$safariPath,
90 "s|sanity-check!" => \$sanityCheck,
92 $testURL = shift @ARGV;
94 $branch = "feature-branch" if $branch eq "feature";
95 if (!exists $validBranches{$branch}) {
96 print STDERR "ERROR: Invalid branch '$branch'\n";
100 if (!$result || $showHelp || scalar(@ARGV) > 0) {
101 print STDERR "Search WebKit nightly builds for changes in behavior.\n";
102 print STDERR "Usage: " . basename($0) . " [options] [url]\n";
104 [-b|--branch name] name of the nightly build branch (default: trunk)
105 [-d|--download-directory dir] nightly build download directory (default: ~/Library/Caches/WebKit-Nightlies)
106 [-h|--help] show this help message
107 [-l|--local] only use local (already downloaded) nightlies
108 [-p|--progression] searching for a progression, not a regression
109 [-r|--revision M[:N]] specify starting (and optional ending) revisions to search
110 [--safari-path path] path to Safari application bundle (default: /Applications/Safari.app)
111 [-s|--sanity-check] verify both starting and ending revisions before bisecting
116 my $nightlyWebSite = "http://nightly.webkit.org";
117 my $nightlyBuildsURLBase = $nightlyWebSite . File::Spec->catdir("/builds", $branch, "mac");
118 my $nightlyFilesURLBase = $nightlyWebSite . File::Spec->catdir("/files", $branch, "mac");
120 $nightlyDownloadDirectory = glob($nightlyDownloadDirectory) if $nightlyDownloadDirectory =~ /^~/;
121 $safariPath = glob($safariPath) if $safariPath =~ /^~/;
122 $safariPath = File::Spec->catdir($safariPath, "Contents/MacOS/Safari") if $safariPath =~ m#\.app/*#;
124 $nightlyDownloadDirectory = File::Spec->catdir($nightlyDownloadDirectory, $branch);
125 if (! -d $nightlyDownloadDirectory) {
126 mkpath($nightlyDownloadDirectory, 0, 0755) || die "Could not create $nightlyDownloadDirectory: $!";
129 @nightlies = makeNightlyList($localOnly, $nightlyDownloadDirectory, findMacOSXVersion(), findSafariVersion($safariPath));
131 my $startIndex = $revisions[0] ? findNearestNightlyIndex(@nightlies, $revisions[0], 'ceil') : 0;
132 my $endIndex = $revisions[1] ? findNearestNightlyIndex(@nightlies, $revisions[1], 'floor') : $#nightlies;
134 my $tempFile = createTempFile($testURL);
140 printf "\nChecking starting revision r%s...\n",
141 $nightlies[$startIndex]->{rev};
142 downloadNightly($nightlies[$startIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
143 mountAndRunNightly($nightlies[$startIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
144 $didReproduceBug = promptForTest($nightlies[$startIndex]->{rev});
145 $startIndex-- if $didReproduceBug < 0;
146 } while ($didReproduceBug < 0);
147 die "ERROR: Bug reproduced in starting revision! Do you need to test an earlier revision or for a progression?"
148 if $didReproduceBug && !$isProgression;
149 die "ERROR: Bug not reproduced in starting revision! Do you need to test an earlier revision or for a regression?"
150 if !$didReproduceBug && $isProgression;
153 printf "\nChecking ending revision r%s...\n",
154 $nightlies[$endIndex]->{rev};
155 downloadNightly($nightlies[$endIndex]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
156 mountAndRunNightly($nightlies[$endIndex]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
157 $didReproduceBug = promptForTest($nightlies[$endIndex]->{rev});
158 $endIndex++ if $didReproduceBug < 0;
159 } while ($didReproduceBug < 0);
160 die "ERROR: Bug NOT reproduced in ending revision! Do you need to test a later revision or for a progression?"
161 if !$didReproduceBug && !$isProgression;
162 die "ERROR: Bug reproduced in ending revision! Do you need to test a later revision or for a regression?"
163 if $didReproduceBug && $isProgression;
166 printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression);
168 my %brokenRevisions = ();
169 while (abs($endIndex - $startIndex) > 1) {
170 my $index = $startIndex + int(($endIndex - $startIndex) / 2);
174 if (exists $nightlies[$index]) {
175 my $buildsLeft = max(max(0, $endIndex - $index - 1), max(0, $index - $startIndex - 1));
176 my $plural = $buildsLeft == 1 ? "" : "s";
177 printf "\nChecking revision r%s (%d build%s left to test after this)...\n", $nightlies[$index]->{rev}, $buildsLeft, $plural;
178 downloadNightly($nightlies[$index]->{file}, $nightlyFilesURLBase, $nightlyDownloadDirectory);
179 mountAndRunNightly($nightlies[$index]->{file}, $nightlyDownloadDirectory, $safariPath, $tempFile);
180 $didReproduceBug = promptForTest($nightlies[$index]->{rev});
182 if ($didReproduceBug < 0) {
183 $brokenRevisions{$nightlies[$index]->{rev}} = $nightlies[$index]->{file};
184 delete $nightlies[$index];
186 $index = $startIndex + int(($endIndex - $startIndex) / 2);
188 } while ($didReproduceBug < 0);
190 if ($didReproduceBug && !$isProgression || !$didReproduceBug && $isProgression) {
193 $startIndex = $index;
196 print "\nBroken revisions skipped: r" . join(", r", keys %brokenRevisions) . "\n"
197 if scalar keys %brokenRevisions > 0;
198 printStatus($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev}, $isProgression);
201 printTracLink($nightlies[$startIndex]->{rev}, $nightlies[$endIndex]->{rev});
203 unlink $tempFile if $tempFile;
207 sub createTempFile($)
211 return undef if !$url;
213 my ($fh, $tempFile) = tempfile(
214 basename($0) . "-XXXXXXXX",
215 DIR => File::Spec->tmpdir(),
219 print $fh "<meta http-equiv=\"refresh\" content=\"0; $url\">\n";
225 sub downloadNightly($$$)
227 my ($filename, $urlBase, $directory) = @_;
228 my $path = File::Spec->catfile($directory, $filename);
230 print "Downloading $filename to $directory...\n";
231 `curl -# -o '$path' '$urlBase/$filename'`;
235 sub findMacOSXVersion()
238 open(SW_VERS, "-|", "/usr/bin/sw_vers") || die;
240 $version = $1 if /^ProductVersion:\s+([^\s]+)/;
246 sub findNearestNightlyIndex(\@$$)
248 my ($nightlies, $revision, $round) = @_;
251 my $highIndex = $#{$nightlies};
253 return $highIndex if uc($revision) eq 'HEAD' || $revision >= $nightlies->[$highIndex]->{rev};
254 return $lowIndex if $revision <= $nightlies->[$lowIndex]->{rev};
256 while (abs($highIndex - $lowIndex) > 1) {
257 my $index = $lowIndex + int(($highIndex - $lowIndex) / 2);
258 if ($revision < $nightlies->[$index]->{rev}) {
260 } elsif ($revision > $nightlies->[$index]->{rev}) {
267 return ($round eq "floor") ? $lowIndex : $highIndex;
270 sub findSafariVersion($)
273 my $versionPlist = File::Spec->catdir(dirname(dirname($path)), "version.plist");
275 open(PLIST, "< $versionPlist") || die;
277 if (m#^\s*<key>CFBundleShortVersionString</key>#) {
279 $version =~ s#^\s*<string>([0-9.]+)[^<]*</string>\s*[\r\n]*#$1#;
290 our $branch = "trunk";
291 our $nightlyDownloadDirectory = File::Spec->catdir($ENV{HOME}, "Library/Caches/WebKit-Nightlies");
292 our $safariPath = "/Applications/Safari.app";
294 my $rcfile = File::Spec->catdir($ENV{HOME}, ".bisect-buildsrc");
295 return if !-f $rcfile;
297 my $result = do $rcfile;
298 die "Could not parse $rcfile: $@" if $@;
301 sub makeNightlyList($$$$)
303 my ($useLocalFiles, $localDirectory, $macOSXVersion, $safariVersion) = @_;
306 if ($useLocalFiles) {
307 opendir(DIR, $localDirectory) || die "$!";
308 foreach my $file (readdir(DIR)) {
309 if ($file =~ /^WebKit-SVN-r([0-9]+)\.dmg$/) {
310 push(@files, +{ rev => $1, file => $file });
315 open(NIGHTLIES, "curl -s $nightlyBuildsURLBase/all |") || die;
317 while (my $line = <NIGHTLIES>) {
319 my ($revision, $timestamp, $url) = split(/,/, $line);
320 my $nightly = basename($url);
321 push(@files, +{ rev => $revision, file => $nightly });
326 if (eval "v$macOSXVersion" ge v10.5) {
327 if ($safariVersion eq "4 Public Beta") {
328 @files = grep { $_->{rev} >= 39682 } @files;
329 } elsif (eval "v$safariVersion" ge v3.2) {
330 @files = grep { $_->{rev} >= 37348 } @files;
331 } elsif (eval "v$safariVersion" ge v3.1) {
332 @files = grep { $_->{rev} >= 29711 } @files;
333 } elsif (eval "v$safariVersion" ge v3.0) {
334 @files = grep { $_->{rev} >= 25124 } @files;
335 } elsif (eval "v$safariVersion" ge v2.0) {
336 @files = grep { $_->{rev} >= 19594 } @files;
338 die "Requires Safari 2.0 or newer";
340 } elsif (eval "v$macOSXVersion" ge v10.4) {
341 if ($safariVersion eq "4 Public Beta") {
342 @files = grep { $_->{rev} >= 39682 } @files;
343 } elsif (eval "v$safariVersion" ge v3.2) {
344 @files = grep { $_->{rev} >= 37348 } @files;
345 } elsif (eval "v$safariVersion" ge v3.1) {
346 @files = grep { $_->{rev} >= 29711 } @files;
347 } elsif (eval "v$safariVersion" ge v3.0) {
348 @files = grep { $_->{rev} >= 19992 } @files;
349 } elsif (eval "v$safariVersion" ge v2.0) {
350 @files = grep { $_->{rev} >= 11976 } @files;
352 die "Requires Safari 2.0 or newer";
355 die "Requires Mac OS X 10.4 (Tiger) or 10.5 (Leopard)";
358 my $nightlycmp = sub { return $a->{rev} <=> $b->{rev}; };
360 return sort $nightlycmp @files;
363 sub mountAndRunNightly($$$$)
365 my ($filename, $directory, $safari, $tempFile) = @_;
366 my $mountPath = "/Volumes/WebKit";
367 my $webkitApp = File::Spec->catfile($mountPath, "WebKit.app");
368 my $diskImage = File::Spec->catfile($directory, $filename);
369 my $devNull = File::Spec->devnull();
372 while (-e $mountPath) {
374 usleep 100 if $i > 1;
375 `hdiutil detach '$mountPath' 2> $devNull`;
376 die "Could not unmount $diskImage at $mountPath" if $i > 100;
378 die "Can't mount $diskImage: $mountPath already exists!" if -e $mountPath;
380 print "Mounting disk image and running WebKit...\n";
381 `hdiutil attach '$diskImage'`;
383 while (! -e $webkitApp) {
386 die "Could not mount $diskImage at $mountPath" if $i > 100;
390 if (-d "/Volumes/WebKit/WebKit.app/Contents/Frameworks") {
391 my $osXVersion = join('.', (split(/\./, findMacOSXVersion()))[0..1]);
392 $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Frameworks/$osXVersion";
394 $frameworkPath = "/Volumes/WebKit/WebKit.app/Contents/Resources";
398 `DYLD_FRAMEWORK_PATH=$frameworkPath WEBKIT_UNSET_DYLD_FRAMEWORK_PATH=YES $safari $tempFile`;
400 `hdiutil detach '$mountPath' 2> $devNull`;
403 sub parseRevisions($$;$)
405 my ($optionName, $value, $ignored) = @_;
407 if ($value =~ /^r?([0-9]+|HEAD):?$/i) {
408 push(@revisions, $1);
409 die "Too many revision arguments specified" if scalar @revisions > 2;
410 } elsif ($value =~ /^r?([0-9]+):?r?([0-9]+|HEAD)$/i) {
414 die "Unknown revision '$value': expected 'M' or 'M:N'";
420 my ($startRevision, $endRevision, $isProgression) = @_;
421 printf "\n%s: r%s %s: r%s\n",
422 $isProgression ? "Fails" : "Works", $startRevision,
423 $isProgression ? "Works" : "Fails", $endRevision;
426 sub printTracLink($$)
428 my ($startRevision, $endRevision) = @_;
429 printf("http://trac.webkit.org/log/trunk/?rev=%s&stop_rev=%s\n", $endRevision, $startRevision + 1);
435 print "Did the bug reproduce in r$revision (yes/no/broken)? ";
436 my $answer = <STDIN>;
437 return 1 if $answer =~ /^(1|y.*)$/i;
438 return -1 if $answer =~ /^(-1|b.*)$/i; # Broken