Tools/ChangeLog

112011-01-27 Dirk Pranke <dpranke@chromium.org>

22

        Reviewed by NOBODY (OOPS!).

        Clean up of the filesystem-related modules used in webkitpy.

        I've added relpath() to the filesystem interface, modified

        ospath.relpath() so that it could work with the filesystem

        interface, and modified the fileset* routines to use the

        filesystem interface consistently.

        This patch also adds a close() routine to the fileset routines

        to indicate that the caller is done with the fileset. This

        allows zipfileset to clean up after itself when it creates

        tempfiles to store downloads.

        * Scripts/webkitpy/common/system/directoryfileset.py:

        * Scripts/webkitpy/common/system/fileset.py:

        * Scripts/webkitpy/common/system/filesystem.py:

        * Scripts/webkitpy/common/system/filesystem_mock.py:

        * Scripts/webkitpy/common/system/ospath.py:

        * Scripts/webkitpy/common/system/zipfileset.py:

        * Scripts/webkitpy/common/system/zipfileset_unittest.py:

2011-01-27  Dirk Pranke  <dpranke@chromium.org>

        Reviewed by Mihai Parparita.

        new-run-webkit-tests: turn off pixel tests correctly by default

Tools/Scripts/webkitpy/common/system/directoryfileset.py

# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF

# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from __future__ import with_statement

import os

import shutil

2924from webkitpy.common.system.fileset import FileSetFileHandle

3025from webkitpy.common.system.filesystem import FileSystem

31 import webkitpy.common.system.ospath as ospath

class DirectoryFileSet(object):

@@class DirectoryFileSet(object):

    def __init__(self, path, filesystem=None):

        self._path = path

        self._filesystem = filesystem or FileSystem()

39 if not self._path.endswith(os~~.pa~~th.sep):

40 self._path += os~~.pa~~th.sep

33 if not self._path.endswith(self._filesystem.sep):

34 self._path += self._filesystem.sep

    def _full_path(self, filename):

        assert self._is_under(self._path, filename)

@@class DirectoryFileSet(object):

        return self._filesystem.files_under(self._path)

    def _is_under(self, dir, filename):

55 return bool(o~~spa~~th.relpath(self._filesystem.join(dir, filename), dir))

49 return bool(self._filesystem.relpath(self._filesystem.join(dir, filename), dir))

    def open(self, filename):

        return FileSetFileHandle(self, filename, self._filesystem)

@@class DirectoryFileSet(object):

        dest = self._filesystem.join(path, filename)

        # As filename may have slashes in it, we must ensure that the same

        # directory hierarchy exists at the output path.

72 self._filesystem.maybe_make_directory(os.p~~ath.split~~(dest)~~[0]~~)

66 self._filesystem.maybe_make_directory(self._filesystem.dirname(dest))

        self._filesystem.copyfile(src, dest)

    def delete(self, filename):

Tools/Scripts/webkitpy/common/system/fileset.py

# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

from __future__ import with_statement

25 import os

from webkitpy.common.system.filesystem import FileSystem

@@class FileSetFileHandle(object):

    def __str__(self):

        return "%s:%s" % (self._fileset, self._filename)

    def close(self):

        pass

    def contents(self):

        if self._contents is None:

            self._contents = self._fileset.read(self._filename)

@@class FileSetFileHandle(object):

        return self._filename

    def splitext(self):

64 return os~~.pa~~th.splitext(self.name())

66 return self._filesystem.splitext(self.name())

Tools/Scripts/webkitpy/common/system/filesystem.py

@@import shutil

import tempfile

import time

42from webkitpy.common.system import ospath

43

class FileSystem(object):

    """FileSystem interface for webkitpy.

@@class FileSystem(object):

        with codecs.open(path, 'r', 'utf8') as f:

            return f.read()

    def relpath(self, path, start='.'):

        """Returns the relative path from the starting point."""

        return ospath.relpath(path, start)

    class _WindowsError(exceptions.OSError):

        """Fake exception for Linux and Mac."""

        pass

Tools/Scripts/webkitpy/common/system/filesystem_mock.py

import errno

import os

31 import path

3231import re

3332

from webkitpy.common.system import path

from webkitpy.common.system import ospath

class MockFileSystem(object):

    def __init__(self, files=None):

@@class MockFileSystem(object):

        raise IOError(errno.ENOENT, path, os.strerror(errno.ENOENT))

    def _split(self, path):

54 idx = path.rfind(~~'/'~~)

56 idx = path.rfind(self.sep)

        return (path[0:idx], path[idx + 1:])

    def abspath(self, path):

60 if path.endswith(self.sep):

61 return path[:-1]

        return path

    def basename(self, path):

@@class MockFileSystem(object):

        if self.basename(path) in dirs_to_skip:

            return []

94 if not path.endswith(~~'/'~~):

95 path += ~~'/'~~

98 if not path.endswith(self.sep):

99 path += self.sep

96100

97 dir_substrings = [~~'/'~~ + d + ~~'/'~~ for d in dirs_to_skip]

101 dir_substrings = [self.sep + d + self.sep for d in dirs_to_skip]

        for filename in self.files:

            if not filename.startswith(path):

                continue

@@class MockFileSystem(object):

            return [f for f in self.files if f == path]

    def isabs(self, path):

121 return path.startswith(~~'/'~~)

125 return path.startswith(self.sep)

    def isfile(self, path):

        return path in self.files and self.files[path] is not None

@@class MockFileSystem(object):

    def isdir(self, path):

        if path in self.files:

            return False

129 if not path.endswith(~~'/'~~):

130 path += ~~'/'~~

133 if not path.endswith(self.sep):

134 path += self.sep

        # We need to use a copy of the keys here in order to avoid switching

        # to a different thread and potentially modifying the dict in

@@class MockFileSystem(object):

        return any(f.startswith(path) for f in files)

    def join(self, *comps):

139 return re.sub(re.escape(os.path.sep), '/', os.path.join(*comps))

        # FIXME: might want tests for this and/or a better comment about how

        # it works.

        return re.sub(re.escape(os.path.sep), self.sep, os.path.join(*comps))

    def listdir(self, path):

        if not self.isdir(path):

            raise OSError("%s is not a directory" % path)

145 if not path.endswith(~~'/'~~):

146 path += ~~'/'~~

151 if not path.endswith(self.sep):

152 path += self.sep

        dirs = []

        files = []

        for f in self.files:

            if self.exists(f) and f.startswith(path):

                remaining = f[len(path):]

153 if ~~'/'~~ in remaining:

154 dir = remaining[:remaining.index(~~'/'~~)]

159 if self.sep in remaining:

160 dir = remaining[:remaining.index(self.sep)]

                    if not dir in dirs:

                        dirs.append(dir)

                else:

@@class MockFileSystem(object):

    def _mktemp(self, suffix='', prefix='tmp', dir=None, **kwargs):

        if dir is None:

168 dir = '/__im_tmp'

174 dir = self.sep + '__im_tmp'

        curno = self.current_tmpno

        self.current_tmpno += 1

        return self.join(dir, "%s_%u_%s" % (prefix, curno, suffix))

@@class MockFileSystem(object):

            self._raise_not_found(path)

        return self.files[path]

    def relpath(self, path, start='.'):

        return ospath.relpath(path, start, self.abspath, self.sep)

    def remove(self, path):

        if self.files[path] is None:

            self._raise_not_found(path)

@@class MockFileSystem(object):

        self.written_files[path] = None

    def rmtree(self, path):

234 if not path.endswith(~~'/'~~):

235 path += ~~'/'~~

243 if not path.endswith(self.sep):

244 path += self.sep

        for f in self.files:

            if f.startswith(path):

Tools/Scripts/webkitpy/common/system/ospath.py

@@import os

#

# It should behave essentially the same as os.path.relpath(), except for

# returning None on paths not contained in abs_start_path.

35 def relpath(path, start_path, os_path_abspath=None):

35def relpath(path, start_path, os_path_abspath=None, sep=None):

    """Return a path relative to the given start path, or None.

    Returns None if the path is not contained in the directory start_path.

@@def relpath(path, start_path, os_path_abspath=None):

      os_path_abspath: A replacement function for unit testing.  This

                       function should strip trailing slashes just like

                       os.path.abspath().  Defaults to os.path.abspath.

47 sep: Path separator. Defaults to os.path.sep

    """

    if os_path_abspath is None:

        os_path_abspath = os.path.abspath

52 sep = sep or os.sep

    # Since os_path_abspath() calls os.path.normpath()--

    #

@@def relpath(path, start_path, os_path_abspath=None):

    if not rel_path:

        # Then the paths are the same.

        pass

70 elif rel_path[0] == ~~os.~~sep:

72 elif rel_path[0] == sep:

        # It is probably sufficient to remove just the first character

        # since os.path.normpath() collapses separators, but we use

        # lstrip() just to be sure.

74 rel_path = rel_path.lstrip(~~os.~~sep)

76 rel_path = rel_path.lstrip(sep)

    else:

        # We are in the case typified by the following example:

        #

Tools/Scripts/webkitpy/common/system/zipfileset.py

@@class ZipFileSet(object):

    """The set of files in a zip file that resides at a URL (local or remote)"""

    def __init__(self, zip_url, filesystem=None, zip_factory=None):

        self._zip_url = zip_url

36 self._temp_file = None

        self._zip_file = None

        self._filesystem = filesystem or FileSystem()

        self._zip_factory = zip_factory or self._retrieve_zip_file

    def _retrieve_zip_file(self, zip_url):

        temp_file = NetworkTransaction().run(lambda: urllib.urlretrieve(zip_url)[0])

42 return zipfile.ZipFile(temp_file)

43 return (temp_file, zipfile.ZipFile(temp_file))

    def _load(self):

        if self._zip_file is None:

46 self._zip_file = self._zip_factory(self._zip_url)

47 self._temp_file, self._zip_file = self._zip_factory(self._zip_url)

    def open(self, filename):

        self._load()

        return FileSetFileHandle(self, filename, self._filesystem)

    def close(self):

        if self._temp_file:

            self._filesystem.remove(self._temp_file)

            self._temp_file = None

    def namelist(self):

        self._load()

        return self._zip_file.namelist()

Tools/Scripts/webkitpy/common/system/zipfileset_unittest.py

@@class ZipFileSetTest(unittest.TestCase):

        result = FakeZip(self._filesystem)

        result.add_file('some-file', 'contents')

        result.add_file('a/b/some-other-file', 'other contents')

67 return result

67 return (None, result)

    def test_open(self):

        file = self._zip.open('a/b/some-other-file')

        self.assertEquals('a/b/some-other-file', file.name())

        self.assertEquals('other contents', file.contents())

    def test_close(self):

        zipfileset = ZipFileSet('blah', self._filesystem, self.make_fake_zip)

        zipfileset.close()

    def test_read(self):

        self.assertEquals('contents', self._zip.read('some-file'))