#!/usr/bin/python # Copyright 2014 Google Inc. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. """Sync files from/to an Android device.""" from __future__ import unicode_literals import sys reload(sys) sys.setdefaultencoding("utf-8") import argparse import glob import locale import os import re import stat import subprocess import sys import time import fnmatch def _sprintf(s, *args): # To be able to use string formatting, we first have to covert to # unicode strings; however, we must do so in a way that preserves all # bytes, and convert back at the end. An encoding that maps all byte # values to different Unicode codepoints is cp437. return (s.decode('cp437') % tuple([ (x.decode('cp437') if type(x) == bytes else x) for x in args ])).encode('cp437') def _print(s, *args): """Writes a binary string to stdout. Args: s: The binary format string to write. args: The args for the format string. """ if hasattr(sys.stdout, 'buffer'): # Python 3. sys.stdout.buffer.write(_sprintf(s, *args) + b'\n') sys.stdout.buffer.flush() else: # Python 2. sys.stdout.write(_sprintf(s, *args) + b'\n') class AdbFileSystem(object): """Mimics os's file interface but uses the adb utility.""" def __init__(self, adb): self.stat_cache = {} self.adb = adb # Regarding parsing stat results, we only care for the following fields: # - st_size # - st_mtime # - st_mode (but only about S_ISDIR and S_ISREG properties) # Therefore, we only capture parts of 'ls -l' output that we actually use. # The other fields will be filled with dummy values. LS_TO_STAT_RE = re.compile(br'''^ (?: (?P -) | (?P b) | (?P c) | (?P d) | (?P l) | (?P p) | (?P s)) [-r][-w][-xsS] [-r][-w][-xsS] [-r][-w][-xtT] # Mode string. [ ]+ (?: [0-9]+ # number of hard links [ ]+ )? [^ ]+ # User name/ID. [ ]+ [^ ]+ # Group name/ID. [ ]+ (?(S_IFBLK) [^ ]+[ ]+[^ ]+[ ]+) # Device numbers. (?(S_IFCHR) [^ ]+[ ]+[^ ]+[ ]+) # Device numbers. (?(S_IFDIR) [0-9]+ [ ]+)? # directory Size. (?(S_IFREG) (?P [0-9]+) # Size. [ ]+) (?P [0-9]{4}-[0-9]{2}-[0-9]{2} # Date. [ ] [0-9]{2}:[0-9]{2}) # Time. [ ] # Don't capture filename for symlinks (ambiguous). (?(S_IFLNK) .* | (?P .*)) $''', re.DOTALL | re.VERBOSE) def LsToStat(self, line): """Convert a line from 'ls -l' output to a stat result. Args: line: Output line of 'ls -l' on Android. Returns: os.stat_result for the line. Raises: OSError: if the given string is not a 'ls -l' output line (but maybe an error message instead). """ match = self.LS_TO_STAT_RE.match(line) if match is None: _print(b'Warning: could not parse %r.', line) raise OSError('Unparseable ls -al result.') groups = match.groupdict() # Get the values we're interested in. st_mode = ( # 0755 stat.S_IRWXU | stat.S_IRGRP | stat.S_IXGRP | stat.S_IROTH | stat.S_IXOTH) if groups['S_IFREG']: st_mode |= stat.S_IFREG if groups['S_IFBLK']: st_mode |= stat.S_IFBLK if groups['S_IFCHR']: st_mode |= stat.S_IFCHR if groups['S_IFDIR']: st_mode |= stat.S_IFDIR if groups['S_IFIFO']: st_mode |= stat.S_IFIFO if groups['S_IFLNK']: st_mode |= stat.S_IFLNK if groups['S_IFSOCK']: st_mode |= stat.S_IFSOCK st_size = groups['st_size'] if st_size is not None: st_size = int(st_size) st_mtime = time.mktime(time.strptime(match.group('st_mtime').decode('utf-8'), '%Y-%m-%d %H:%M')) # Fill the rest with dummy values. st_ino = 1 st_rdev = 0 st_nlink = 1 st_uid = -2 # Nobody. st_gid = -2 # Nobody. st_atime = st_ctime = st_mtime stbuf = os.stat_result((st_mode, st_ino, st_rdev, st_nlink, st_uid, st_gid, st_size, st_atime, st_mtime, st_ctime)) filename = groups['filename'] return stbuf, filename def Stdout(self, *popen_args): """Closes the process's stdout when done. Usage: with Stdout(...) as stdout: DoSomething(stdout) Args: popen_args: Arguments for subprocess.Popen; stdout=PIPE is implicitly added. Returns: An object for use by 'with'. """ class Stdout(object): def __init__(self, popen): self.popen = popen def __enter__(self): return self.popen.stdout def __exit__(self, exc_type, exc_value, traceback): self.popen.stdout.close() if self.popen.wait() != 0: raise OSError('Subprocess exited with nonzero status.') return False return Stdout(subprocess.Popen(*popen_args, stdout=subprocess.PIPE)) def QuoteArgument(self, arg): # Quotes an argument for use by adb shell. # Usually, arguments in 'adb shell' use are put in double quotes by adb, # but not in any way escaped. arg = arg.replace(b'\\', b'\\\\') arg = arg.replace(b'"', b'\\"') arg = arg.replace(b'$', b'\\$') arg = arg.replace(b'`', b'\\`') arg = b'"' + arg + b'"' return arg def IsWorking(self): """Tests the adb connection.""" # This string should contain all possible evil, but no percent signs. # Note this code uses 'date' and not 'echo', as date just calls strftime # while echo does its own backslash escape handling additionally to the # shell's. Too bad printf "%s\n" is not available. test_strings = [ b'(', b'(; #`ls`$PATH\'"(\\\\\\\\){};!\xc0\xaf\xff\xc2\xbf' ] for test_string in test_strings: good = False with self.Stdout(self.adb + [b'shell', _sprintf(b'date +%s', self.QuoteArgument(test_string))]) as stdout: for line in stdout: line = line.rstrip(b'\r\n') if line == test_string: good = True if not good: return False return True def listdir(self, path): # os's name, so pylint: disable=g-bad-name """List the contents of a directory, caching them for later lstat calls.""" with self.Stdout(self.adb + [b'shell', _sprintf(b'ls -al %s', self.QuoteArgument(path + b'/'))]) as stdout: for line in stdout: if line.startswith(b'total '): continue line = line.rstrip(b'\r\n') try: statdata, filename = self.LsToStat(line) except OSError: continue if filename is None: _print(b'Warning: could not parse %s', line) else: self.stat_cache[path + b'/' + filename] = statdata yield filename def lstat(self, path): # os's name, so pylint: disable=g-bad-name """Stat a file.""" if path in self.stat_cache: return self.stat_cache[path] with self.Stdout(self.adb + [b'shell', _sprintf(b'ls -ald %s', self.QuoteArgument(path))]) as stdout: for line in stdout: if line.startswith(b'total '): continue line = line.rstrip(b'\r\n') statdata, filename = self.LsToStat(line) self.stat_cache[path] = statdata return statdata raise OSError('No such file or directory') def unlink(self, path): # os's name, so pylint: disable=g-bad-name """Delete a file.""" if subprocess.call(self.adb + [b'shell', _sprintf(b'rm %s', self.QuoteArgument(path))]) != 0: raise OSError('unlink failed') def rmdir(self, path): # os's name, so pylint: disable=g-bad-name """Delete a directory.""" if subprocess.call(self.adb + [b'shell', _sprintf(b'rmdir %s', self.QuoteArgument(path))]) != 0: raise OSError('rmdir failed') def makedirs(self, path): # os's name, so pylint: disable=g-bad-name """Create a directory.""" if subprocess.call(self.adb + [b'shell', _sprintf(b'mkdir -p %s', self.QuoteArgument(path))]) != 0: raise OSError('mkdir failed') def utime(self, path, times): # TODO(rpolzer): Find out why this does not work (returns status 255). """Set the time of a file to a specified unix time.""" atime, mtime = times timestr = time.strftime(b'%Y%m%d.%H%M%S', time.localtime(mtime)) if subprocess.call(self.adb + [b'shell', _sprintf(b'touch -mt %s %s', timestr, self.QuoteArgument(path))]) != 0: raise OSError('touch failed') timestr = time.strftime(b'%Y%m%d.%H%M%S', time.localtime(atime)) if subprocess.call(self.adb + [b'shell',_sprintf( b'touch -at %s %s', timestr, self.QuoteArgument(path))]) != 0: raise OSError('touch failed') def glob(self, path): with self.Stdout(self.adb + [b'shell', _sprintf(b'for p in %s; do echo "$p"; done', path)]) as stdout: for line in stdout: yield line.rstrip(b'\r\n') def Push(self, src, dst): """Push a file from the local file system to the Android device.""" if subprocess.call(self.adb + [b'push', src, dst]) != 0: raise OSError('push failed') def Pull(self, src, dst): """Pull a file from the Android device to the local file system.""" if subprocess.call(self.adb + [b'pull', src, dst]) != 0: raise OSError('pull failed') def BuildFileList(fs, path, exclude = [], prefix=b''): """Builds a file list. Args: fs: File system provider (can be os or AdbFileSystem()). path: Initial path. prefix: Path prefix for output file names. Yields: File names from path (prefixed by prefix). Directories are yielded before their contents. """ if not exclude: exclude = [] try: statresult = fs.lstat(path) except OSError: return if stat.S_ISDIR(statresult.st_mode): yield prefix, statresult try: files = [f for f in fs.listdir(path) if not any([fnmatch.fnmatch(f, pattern) for pattern in exclude])] except OSError: return files.sort() for n in files: if n == b'.' or n == b'..': continue for t in BuildFileList(fs, path + b'/' + n, exclude, prefix + b'/' + n): yield t elif stat.S_ISREG(statresult.st_mode): yield prefix, statresult else: _print(b'Note: unsupported file: %s', path) def DiffLists(a, b): """Compares two lists. Args: a: the first list. b: the second list. Returns: a_only: the items from list a. both: the items from both list, with the remaining tuple items combined. b_only: the items from list b. """ a_only = [] b_only = [] both = [] a_iter = iter(a) b_iter = iter(b) a_active = True b_active = True a_available = False b_available = False a_item = None b_item = None while a_active and b_active: if not a_available: try: a_item = next(a_iter) a_available = True except StopIteration: a_active = False break if not b_available: try: b_item = next(b_iter) b_available = True except StopIteration: b_active = False break if a_item[0] == b_item[0]: both.append(tuple([a_item[0]] + list(a_item[1:]) + list(b_item[1:]))) a_available = False b_available = False elif a_item[0] < b_item[0]: a_only.append(a_item) a_available = False elif a_item[0] > b_item[0]: b_only.append(b_item) b_available = False else: raise if a_active: if a_available: a_only.append(a_item) for item in a_iter: a_only.append(item) if b_active: if b_available: b_only.append(b_item) for item in b_iter: b_only.append(item) return a_only, both, b_only class FileSyncer(object): """File synchronizer.""" def __init__(self, adb, local_path, remote_path, exclude, local_to_remote, remote_to_local, preserve_times, delete_missing, allow_overwrite, allow_replace, dry_run): self.local = local_path self.remote = remote_path self.exclude = exclude self.adb = adb self.local_to_remote = local_to_remote self.remote_to_local = remote_to_local self.preserve_times = preserve_times self.delete_missing = delete_missing self.allow_overwrite = allow_overwrite self.allow_replace = allow_replace self.dry_run = dry_run self.local_only = None self.both = None self.remote_only = None self.num_bytes = 0 self.start_time = time.time() def IsWorking(self): """Tests the adb connection.""" return self.adb.IsWorking() def ScanAndDiff(self): """Scans the local and remote locations and identifies differences.""" _print(b'Scanning and diffing...') locallist = BuildFileList(os, self.local, self.exclude) remotelist = BuildFileList(self.adb, self.remote, self.exclude) self.local_only, self.both, self.remote_only = DiffLists(locallist, remotelist) if not self.local_only and not self.both and not self.remote_only: _print(b'No files seen. User error?') self.src_to_dst = (self.local_to_remote, self.remote_to_local) self.dst_to_src = (self.remote_to_local, self.local_to_remote) self.src_only = (self.local_only, self.remote_only) self.dst_only = (self.remote_only, self.local_only) self.src = (self.local, self.remote) self.dst = (self.remote, self.local) self.dst_fs = (self.adb, os) self.push = (b'Push', b'Pull') self.copy = (self.adb.Push, self.adb.Pull) def InterruptProtection(self, fs, name): """Sets up interrupt protection. Usage: with self.InterruptProtection(fs, name): DoSomething() If DoSomething() should get interrupted, the file 'name' will be deleted. The exception otherwise will be passed on. Args: fs: File system object. name: File name to delete. Returns: An object for use by 'with'. """ dry_run = self.dry_run class DeleteInterruptedFile(object): def __enter__(self): pass def __exit__(self, exc_type, exc_value, traceback): if exc_type is not None: _print(b'Interrupted-%s-Delete: %s', b'Pull' if fs == os else b'Push', name) if not dry_run: fs.unlink(name) return False return DeleteInterruptedFile() def PerformDeletions(self): """Perform all deleting necessary for the file sync operation.""" if not self.delete_missing: return for i in [0, 1]: if self.src_to_dst[i] and not self.dst_to_src[i]: if not self.src_only[i] and not self.both: _print(b'Cowardly refusing to delete everything.') else: for name, s in reversed(self.dst_only[i]): dst_name = self.dst[i] + name _print(b'%s-Delete: %s', self.push[i], dst_name) if stat.S_ISDIR(s.st_mode): if not self.dry_run: self.dst_fs[i].rmdir(dst_name) else: if not self.dry_run: self.dst_fs[i].unlink(dst_name) del self.dst_only[i][:] def PerformOverwrites(self): """Delete files/directories that are in the way for overwriting.""" src_only_prepend = ([], []) for name, localstat, remotestat in self.both: if stat.S_ISDIR(localstat.st_mode) and stat.S_ISDIR(remotestat.st_mode): # A dir is a dir is a dir. continue elif stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): # Dir vs file? Nothing to do here yet. pass else: # File vs file? Compare sizes. if localstat.st_size == remotestat.st_size: continue l2r = self.local_to_remote r2l = self.remote_to_local if l2r and r2l: # Truncate times to full minutes, as Android's "ls" only outputs minute # accuracy. localminute = int(localstat.st_mtime / 60) remoteminute = int(remotestat.st_mtime / 60) if localminute > remoteminute: r2l = False elif localminute < remoteminute: l2r = False if l2r and r2l: _print(b'Unresolvable: %s', name) continue if l2r: i = 0 # Local to remote operation. src_stat = localstat dst_stat = remotestat else: i = 1 # Remote to local operation. src_stat = remotestat dst_stat = localstat dst_name = self.dst[i] + name _print(b'%s-Delete-Conflicting: %s', self.push[i], dst_name) if stat.S_ISDIR(localstat.st_mode) or stat.S_ISDIR(remotestat.st_mode): if not self.allow_replace: _print(b'Would have to replace to do this. ' b'Use --force to allow this.') continue if not self.allow_overwrite: _print(b'Would have to overwrite to do this, ' b'which --no-clobber forbids.') continue if stat.S_ISDIR(dst_stat.st_mode): kill_files = [x for x in self.dst_only[i] if x[0][:len(name) + 1] == name + b'/'] self.dst_only[i][:] = [x for x in self.dst_only[i] if x[0][:len(name) + 1] != name + b'/'] for l, s in reversed(kill_files): if stat.S_ISDIR(s.st_mode): if not self.dry_run: self.dst_fs[i].rmdir(self.dst[i] + l) else: if not self.dry_run: self.dst_fs[i].unlink(self.dst[i] + l) if not self.dry_run: self.dst_fs[i].rmdir(dst_name) elif stat.S_ISDIR(src_stat.st_mode): if not self.dry_run: self.dst_fs[i].unlink(dst_name) else: if not self.dry_run: self.dst_fs[i].unlink(dst_name) src_only_prepend[i].append((name, src_stat)) for i in [0, 1]: self.src_only[i][:0] = src_only_prepend[i] def PerformCopies(self): """Perform all copying necessary for the file sync operation.""" for i in [0, 1]: if self.src_to_dst[i]: for name, s in self.src_only[i]: src_name = self.src[i] + name dst_name = self.dst[i] + name _print(b'%s: %s', self.push[i], dst_name) if stat.S_ISDIR(s.st_mode): if not self.dry_run: self.dst_fs[i].makedirs(dst_name) else: with self.InterruptProtection(self.dst_fs[i], dst_name): if not self.dry_run: self.copy[i](src_name, dst_name) self.num_bytes += s.st_size if not self.dry_run: if self.preserve_times: _print(b'%s-Times: accessed %s, modified %s', self.push[i], time.asctime(time.localtime(s.st_atime)).encode('utf-8'), time.asctime(time.localtime(s.st_mtime)).encode('utf-8')) self.dst_fs[i].utime(dst_name, (s.st_atime, s.st_mtime)) def TimeReport(self): """Report time and amount of data transferred.""" if self.dry_run: _print(b'Total: %d bytes', self.num_bytes) else: end_time = time.time() dt = end_time - self.start_time rate = self.num_bytes / 1024.0 / dt _print(b'Total: %d KB/s (%d bytes in %.3fs)', rate, self.num_bytes, dt) def ExpandWildcards(globber, path): if path.find(b'?') == -1 and path.find(b'*') == -1 and path.find(b'[') == -1: return [path] return globber.glob(path) def FixPath(src, dst): # rsync-like path munging to make remote specifications shorter. append = b'' pos = src.rfind(b'/') if pos >= 0: if src.endswith(b'/'): # Final slash: copy to the destination "as is". src = src[:-1] else: # No final slash: destination name == source name. append = src[pos:] else: # No slash at all - use same name at destination. append = b'/' + src # Append the destination file name if any. # BUT: do not append "." or ".." components! if append != b'/.' and append != b'/..': dst += append return (src, dst) def main(*args): parser = argparse.ArgumentParser( description='Synchronize a directory between an Android device and the '+ 'local file system') parser.add_argument('source', metavar='SRC', type=str, nargs='+', help='The directory to read files/directories from. '+ 'This must be a local path if -R is not specified, '+ 'and an Android path if -R is specified. If SRC does '+ 'not end with a final slash, its last path component '+ 'is appended to DST (like rsync does).') parser.add_argument('destination', metavar='DST', type=str, help='The directory to write files/directories to. '+ 'This must be an Android path if -R is not specified, '+ 'and a local path if -R is specified.') parser.add_argument('-e', '--adb', metavar='COMMAND', default='adb', type=str, help='Use the given adb binary and arguments.') parser.add_argument('--device', action='store_true', help='Directs command to the only connected USB device; '+ 'returns an error if more than one USB device is '+ 'present. '+ 'Corresponds to the "-d" option of adb.') parser.add_argument('--emulator', action='store_true', help='Directs command to the only running emulator; '+ 'returns an error if more than one emulator is running. '+ 'Corresponds to the "-e" option of adb.') parser.add_argument('-s', '--serial', metavar='DEVICE', type=str, help='Directs command to the device or emulator with '+ 'the given serial number or qualifier. Overrides '+ 'ANDROID_SERIAL environment variable. Use "adb devices" '+ 'to list all connected devices with their respective '+ 'serial number. '+ 'Corresponds to the "-s" option of adb.') parser.add_argument('-H', '--host', metavar='HOST', type=str, help='Name of adb server host (default: localhost). '+ 'Corresponds to the "-H" option of adb.') parser.add_argument('-P', '--port', metavar='PORT', type=str, help='Port of adb server (default: 5037). '+ 'Corresponds to the "-P" option of adb.') parser.add_argument('-R', '--reverse', action='store_true', help='Reverse sync (pull, not push).') parser.add_argument('-2', '--two-way', action='store_true', help='Two-way sync (compare modification time; after '+ 'the sync, both sides will have all files in the '+ 'respective newest version. This relies on the clocks '+ 'of your system and the device to match.') #parser.add_argument('-t', '--times', action='store_true', # help='Preserve modification times when copying.') parser.add_argument('-d', '--delete', action='store_true', help='Delete files from DST that are not present on '+ 'SRC. Mutually exclusive with -2.') parser.add_argument('--exclude', metavar='PATTERN', action='append', help='Exclude files matching PATTERN') parser.add_argument('-f', '--force', action='store_true', help='Allow deleting files/directories when having to '+ 'replace a file by a directory or vice versa. This is '+ 'disabled by default to prevent large scale accidents.') parser.add_argument('-n', '--no-clobber', action='store_true', help='Do not ever overwrite any '+ 'existing files. Mutually exclusive with -f.') parser.add_argument('--dry-run',action='store_true', help='Do not do anything - just show what would '+ 'be done.') args = parser.parse_args() args_encoding = locale.getdefaultlocale()[1] localpatterns = [x.encode(args_encoding) for x in args.source] remotepath = args.destination.encode(args_encoding) adb = args.adb.encode(args_encoding).split(b' ') if args.device: adb += [b'-d'] if args.emulator: adb += [b'-e'] if args.serial != None: adb += [b'-s', args.serial.encode(args_encoding)] if args.host != None: adb += [b'-H', args.host.encode(args_encoding)] if args.port != None: adb += [b'-P', args.port.encode(args_encoding)] adb = AdbFileSystem(adb) # Expand wildcards. localpaths = [] remotepaths = [] if args.reverse: for pattern in localpatterns: for src in ExpandWildcards(adb, pattern): src, dst = FixPath(src, remotepath) localpaths.append(src) remotepaths.append(dst) else: for src in localpatterns: src, dst = FixPath(src, remotepath) localpaths.append(src) remotepaths.append(dst) preserve_times = False # args.times delete_missing = args.delete allow_replace = args.force allow_overwrite = not args.no_clobber dry_run = args.dry_run local_to_remote = True remote_to_local = False if args.two_way: local_to_remote = True remote_to_local = True if args.reverse: local_to_remote, remote_to_local = remote_to_local, local_to_remote localpaths, remotepaths = remotepaths, localpaths if allow_replace and not allow_overwrite: _print(b'--no-clobber and --force are mutually exclusive.') parser.print_help() return if delete_missing and local_to_remote and remote_to_local: _print(b'--delete and --two-way are mutually exclusive.') parser.print_help() return # Two-way sync is only allowed with disjoint remote and local path sets. if (remote_to_local and local_to_remote) or delete_missing: if ((remote_to_local and len(localpaths) != len(set(localpaths))) or (local_to_remote and len(remotepaths) != len(set(remotepaths)))): _print(b'--two-way and --delete are only supported for disjoint sets of ' b'source and destination paths (in other words, all SRC must ' b'differ in basename).') parser.print_help() return for i in range(len(localpaths)): _print(b'Sync: local %s, remote %s', localpaths[i], remotepaths[i]) syncer = FileSyncer(adb, localpaths[i], remotepaths[i], args.exclude, local_to_remote, remote_to_local, preserve_times, delete_missing, allow_overwrite, allow_replace, dry_run) if not syncer.IsWorking(): _print(b'Device not connected or not working.') return try: syncer.ScanAndDiff() syncer.PerformDeletions() syncer.PerformOverwrites() syncer.PerformCopies() finally: syncer.TimeReport() if __name__ == '__main__': main(*sys.argv)