commit -- why not

Demo/pdist/FSProxy.py

@@ -0,0 +1,322 @@
+"""File System Proxy.
+
+Provide an OS-neutral view on a file system, locally or remotely.
+The functionality is geared towards implementing some sort of
+rdist-like utility between a Mac and a UNIX system.
+
+The module defines three classes:
+
+FSProxyLocal  -- used for local access
+FSProxyServer -- used on the server side of remote access
+FSProxyClient -- used on the client side of remote access
+
+The remote classes are instantiated with an IP address and an optional
+verbosity flag.
+"""
+
+import server
+import client
+import md5
+import os
+import fnmatch
+from stat import *
+import time
+import fnmatch
+
+if os.name == 'mac':
+	import macfs
+	maxnamelen = 31
+else:
+	macfs = None
+	maxnamelen = 255
+
+skipnames = (os.curdir, os.pardir)
+
+
+class FSProxyLocal:
+	
+	def __init__(self):
+		self._dirstack = []
+		self._ignore = ['*.pyc'] + self._readignore()
+	
+	def _close(self):
+		while self._dirstack:
+			self.back()
+	
+	def _readignore(self):
+		file = self._hide('ignore')
+		try:
+			f = open(file)
+		except IOError:
+			file = self._hide('synctree.ignorefiles')
+			try:
+				f = open(file)
+			except IOError:
+				return []
+		ignore = []
+		while 1:
+			line = f.readline()
+			if not line: break
+			if line[-1] == '\n': line = line[:-1]
+			ignore.append(line)
+		f.close()
+		return ignore
+	
+	def _hidden(self, name):
+		if os.name == 'mac':
+			return name[0] == '(' and name[-1] == ')'
+		else:
+			return name[0] == '.'
+	
+	def _hide(self, name):
+		if os.name == 'mac':
+			return '(%s)' % name
+		else:
+			return '.%s' % name
+	
+	def visible(self, name):
+		if len(name) > maxnamelen: return 0
+		if name[-1] == '~': return 0
+		if name in skipnames: return 0
+		if self._hidden(name): return 0
+		head, tail = os.path.split(name)
+		if head or not tail: return 0
+		if macfs:
+			if os.path.exists(name) and not os.path.isdir(name):
+				try:
+					fs = macfs.FSSpec(name)
+					c, t = fs.GetCreatorType()
+					if t != 'TEXT': return 0
+				except macfs.error, msg:
+					print "***", name, msg
+					return 0
+		else:
+			if os.path.islink(name): return 0
+			if '\0' in open(name, 'rb').read(512): return 0
+		for ign in self._ignore:
+			if fnmatch.fnmatch(name, ign): return 0
+		return 1
+	
+	def check(self, name):
+		if not self.visible(name):
+			raise os.error, "protected name %s" % repr(name)
+	
+	def checkfile(self, name):
+		self.check(name)
+		if not os.path.isfile(name):
+			raise os.error, "not a plain file %s" % repr(name)
+	
+	def pwd(self):
+		return os.getcwd()
+	
+	def cd(self, name):
+		self.check(name)
+		save = os.getcwd(), self._ignore
+		os.chdir(name)
+		self._dirstack.append(save)
+		self._ignore = self._ignore + self._readignore()
+	
+	def back(self):
+		if not self._dirstack:
+			raise os.error, "empty directory stack"
+		dir, ignore = self._dirstack[-1]
+		os.chdir(dir)
+		del self._dirstack[-1]
+		self._ignore = ignore
+	
+	def _filter(self, files, pat = None):
+		if pat:
+			def keep(name, pat = pat):
+				return fnmatch.fnmatch(name, pat)
+			files = filter(keep, files)
+		files = filter(self.visible, files)
+		files.sort()
+		return files
+	
+	def list(self, pat = None):
+		files = os.listdir(os.curdir)
+		return self._filter(files, pat)
+	
+	def listfiles(self, pat = None):
+		files = os.listdir(os.curdir)
+		files = filter(os.path.isfile, files)
+		return self._filter(files, pat)
+	
+	def listsubdirs(self, pat = None):
+		files = os.listdir(os.curdir)
+		files = filter(os.path.isdir, files)
+		return self._filter(files, pat)
+	
+	def exists(self, name):
+		return self.visible(name) and os.path.exists(name)
+	
+	def isdir(self, name):
+		return self.visible(name) and os.path.isdir(name)
+	
+	def islink(self, name):
+		return self.visible(name) and os.path.islink(name)
+	
+	def isfile(self, name):
+		return self.visible(name) and os.path.isfile(name)
+	
+	def sum(self, name):
+		self.checkfile(name)
+		BUFFERSIZE = 1024*8
+		f = open(name)
+		sum = md5.new()
+		while 1:
+			buffer = f.read(BUFFERSIZE)
+			if not buffer:
+				break
+			sum.update(buffer)
+		return sum.digest()
+	
+	def size(self, name):
+		self.checkfile(name)
+		return os.stat(name)[ST_SIZE]
+	
+	def mtime(self, name):
+		self.checkfile(name)
+		return time.localtime(os.stat(name)[ST_MTIME])
+	
+	def stat(self, name):
+		self.checkfile(name)
+		size = os.stat(name)[ST_SIZE]
+		mtime = time.localtime(os.stat(name)[ST_MTIME])
+		return size, mtime
+	
+	def info(self, name):
+		sum = self.sum(name)
+		size = os.stat(name)[ST_SIZE]
+		mtime = time.localtime(os.stat(name)[ST_MTIME])
+		return sum, size, mtime
+	
+	def _list(self, function, list):
+		if list is None:
+			list = self.listfiles()
+		res = []
+		for name in list:
+			try:
+				res.append((name, function(name)))
+			except (os.error, IOError):
+				res.append((name, None))
+		return res
+	
+	def sumlist(self, list = None):
+		return self._list(self.sum, list)
+	
+	def statlist(self, list = None):
+		return self._list(self.stat, list)
+	
+	def mtimelist(self, list = None):
+		return self._list(self.mtime, list)
+	
+	def sizelist(self, list = None):
+		return self._list(self.size, list)
+	
+	def infolist(self, list = None):
+		return self._list(self.info, list)
+	
+	def _dict(self, function, list):
+		if list is None:
+			list = self.listfiles()
+		dict = {}
+		for name in list:
+			try:
+				dict[name] = function(name)
+			except (os.error, IOError):
+				pass
+		return dict
+	
+	def sumdict(self, list = None):
+		return self.dict(self.sum, list)
+	
+	def sizedict(self, list = None):
+		return self.dict(self.size, list)
+	
+	def mtimedict(self, list = None):
+		return self.dict(self.mtime, list)
+	
+	def statdict(self, list = None):
+		return self.dict(self.stat, list)
+	
+	def infodict(self, list = None):
+		return self._dict(self.info, list)
+	
+	def read(self, name, offset = 0, length = -1):
+		self.checkfile(name)
+		f = open(name)
+		f.seek(offset)
+		if length == 0:
+			data = ''
+		elif length < 0:
+			data = f.read()
+		else:
+			data = f.read(length)
+		f.close()
+		return data
+	
+	def create(self, name):
+		self.check(name)
+		if os.path.exists(name):
+			self.checkfile(name)
+			bname = name + '~'
+			try:
+				os.unlink(bname)
+			except os.error:
+				pass
+			os.rename(name, bname)
+		f = open(name, 'w')
+		f.close()
+	
+	def write(self, name, data, offset = 0):
+		self.checkfile(name)
+		f = open(name, 'r+')
+		f.seek(offset)
+		f.write(data)
+		f.close()
+	
+	def mkdir(self, name):
+		self.check(name)
+		os.mkdir(name, 0777)
+	
+	def rmdir(self, name):
+		self.check(name)
+		os.rmdir(name)
+
+
+class FSProxyServer(FSProxyLocal, server.Server):
+	
+	def __init__(self, address, verbose = server.VERBOSE):
+		FSProxyLocal.__init__(self)
+		server.Server.__init__(self, address, verbose)
+	
+	def _close(self):
+		server.Server._close(self)
+		FSProxyLocal._close(self)
+	
+	def _serve(self):
+		server.Server._serve(self)
+		# Retreat into start directory
+		while self._dirstack: self.back()
+
+
+class FSProxyClient(client.Client):
+	
+	def __init__(self, address, verbose = client.VERBOSE):
+		client.Client.__init__(self, address, verbose)
+
+
+def test():
+	import string
+	import sys
+	if sys.argv[1:]:
+		port = string.atoi(sys.argv[1])
+	else:
+		port = 4127
+	proxy = FSProxyServer(('', port))
+	proxy._serverloop()
+
+
+if __name__ == '__main__':
+	test()

Demo/pdist/client.py

@@ -0,0 +1,133 @@
+"""RPC Client module."""
+
+import sys
+import socket
+import pickle
+import __builtin__
+import os
+
+
+# Default verbosity (0 = silent, 1 = print connections, 2 = print requests too)
+VERBOSE = 1
+
+
+class Client:
+	
+	"""RPC Client class.  No need to derive a class -- it's fully generic."""
+	
+	def __init__(self, address, verbose = VERBOSE):
+		if type(address) == type(0):
+			address = ('', address)
+		self._address = address
+		self._verbose = verbose
+		if self._verbose: print "Connecting to %s ..." % repr(address)
+		self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+		self._socket.connect(address)
+		if self._verbose: print "Connected."
+		self._lastid = 0 # Last id for which a reply has been received
+		self._nextid = 1 # Id of next request
+		self._replies = {} # Unprocessed replies
+		self._rf = self._socket.makefile('r')
+		self._wf = self._socket.makefile('w')
+		self._methods = self._call('.methods')
+	
+	def __del__(self):
+		self._close()
+	
+	def _close(self):
+		if self._rf: self._rf.close()
+		self._rf = None
+		if self._wf: self._wf.close()
+		self._wf = None
+		if self._socket: self._socket.close()
+		self._socket = None
+	
+	def __getattr__(self, name):
+		if name in self._methods:
+			method = _stub(self, name)
+			setattr(self, name, method) # XXX circular reference
+			return method
+		raise AttributeError, name
+	
+	def _setverbose(self, verbose):
+		self._verbose = verbose
+	
+	def _call(self, name, *args):
+		return self._vcall(name, args)
+	
+	def _vcall(self, name, args):
+		return self._recv(self._vsend(name, args))
+	
+	def _send(self, name, *args):
+		return self._vsend(name, args)
+	
+	def _send_noreply(self, name, *args):
+		return self._vsend(name, args, 0)
+	
+	def _vsend_noreply(self, name, args):
+		return self._vsend(name, args, 0)
+	
+	def _vsend(self, name, args, wantreply = 1):
+		id = self._nextid
+		self._nextid = id+1
+		if not wantreply: id = -id
+		request = (name, args, id)
+		if self._verbose > 1: print "sending request: %s" % repr(request)
+		wp = pickle.Pickler(self._wf)
+		wp.dump(request)
+		return id
+	
+	def _recv(self, id):
+		exception, value, rid = self._vrecv(id)
+		if rid != id:
+			raise RuntimeError, "request/reply id mismatch: %d/%d" % (id, rid)
+		if exception is None:
+			return value
+		x = exception
+		if hasattr(__builtin__, exception):
+			x = getattr(__builtin__, exception)
+		elif exception in ('posix.error', 'mac.error'):
+			x = os.error
+		if x == exception:
+			exception = x
+		raise exception, value		
+	
+	def _vrecv(self, id):
+		self._flush()
+		if self._replies.has_key(id):
+			if self._verbose > 1: print "retrieving previous reply, id = %d" % id
+			reply = self._replies[id]
+			del self._replies[id]
+			return reply
+		aid = abs(id)
+		while 1:
+			if self._verbose > 1: print "waiting for reply, id = %d" % id
+			rp = pickle.Unpickler(self._rf)
+			reply = rp.load()
+			del rp
+			if self._verbose > 1: print "got reply: %s" % repr(reply)
+			rid = reply[2]
+			arid = abs(rid)
+			if arid == aid:
+				if self._verbose > 1: print "got it"
+				return reply
+			self._replies[rid] = reply
+			if arid > aid:
+				if self._verbose > 1: print "got higher id, assume all ok"
+				return (None, None, id)
+	
+	def _flush(self):
+		self._wf.flush()
+
+
+class _stub:
+	
+	"""Helper class for Client -- each instance serves as a method of the client."""
+	
+	def __init__(self, client, name):
+		self._client = client
+		self._name = name
+	
+	def __call__(self, *args):
+		return self._client._vcall(self._name, args)
+

Demo/pdist/cmptree.py

@@ -0,0 +1,185 @@
+"""Compare local and remote dictionaries and transfer differing files -- like rdist."""
+
+import sys
+from repr import repr
+import FSProxy
+import time
+import os
+
+def main():
+	pwd = os.getcwd()
+	s = raw_input("chdir [%s] " % pwd)
+	if s:
+		os.chdir(s)
+		pwd = os.getcwd()
+	host = ask("host", 'voorn.cwi.nl')
+	port = 4127
+	verbose = 1
+	mode = ''
+	print """\
+Mode should be a string of characters, indicating what to do with differences.
+r - read different files to local file system
+w - write different files to remote file system
+c - create new files, either remote or local
+d - delete disappearing files, either remote or local
+"""
+	s = raw_input("mode [%s] " % mode)
+	if s: mode = s
+	address = (host, port)
+	t1 = time.time()
+	local = FSProxy.FSProxyLocal()
+	remote = FSProxy.FSProxyClient(address, verbose)
+	compare(local, remote, mode)
+	remote._close()
+	local._close()
+	t2 = time.time()
+	dt = t2-t1
+	mins, secs = divmod(dt, 60)
+	print mins, "minutes and", secs, "seconds"
+	raw_input("[Return to exit] ")
+
+def ask(prompt, default):
+	s = raw_input("%s [%s] " % (prompt, default))
+	return s or default
+
+def askint(prompt, default):
+	s = raw_input("%s [%s] " % (prompt, str(default)))
+	if s: return string.atoi(s)
+	return default
+
+def compare(local, remote, mode):
+	print
+	print "PWD =", `os.getcwd()`
+	sums_id = remote._send('sumlist')
+	subdirs_id = remote._send('listsubdirs')
+	remote._flush()
+	print "calculating local sums ..."
+	lsumdict = {}
+	for name, info in local.sumlist():
+		lsumdict[name] = info
+	print "getting remote sums ..."
+	sums = remote._recv(sums_id)
+	print "got", len(sums)
+	rsumdict = {}
+	for name, rsum in sums:
+		rsumdict[name] = rsum
+		if not lsumdict.has_key(name):
+			print `name`, "only remote"
+			if 'r' in mode and 'c' in mode:
+				recvfile(local, remote, name)
+		else:
+			lsum = lsumdict[name]
+			if lsum != rsum:
+				print `name`,
+				rmtime = remote.mtime(name)
+				lmtime = local.mtime(name)
+				if rmtime > lmtime:
+					print "remote newer",
+					if 'r' in mode:
+						recvfile(local, remote, name)
+				elif lmtime > rmtime:
+					print "local newer",
+					if 'w' in mode:
+						sendfile(local, remote, name)
+				else:
+					print "same mtime but different sum?!?!",
+				print
+	for name in lsumdict.keys():
+		if not rsumdict.keys():
+			print `name`, "only locally",
+			fl()
+			if 'w' in mode and 'c' in mode:
+				sendfile(local, remote, name)
+			elif 'r' in mode and 'd' in mode:
+				os.unlink(name)
+				print "removed."
+			print
+	print "gettin subdirs ..."
+	subdirs = remote._recv(subdirs_id)
+	common = []
+	for name in subdirs:
+		if local.isdir(name):
+			print "Common subdirectory", repr(name)
+			common.append(name)
+		else:
+			print "Remote subdirectory", repr(name), "not found locally"
+	lsubdirs = local.listsubdirs()
+	for name in lsubdirs:
+		if name not in subdirs:
+			print "Local subdirectory", repr(name), "not found remotely"
+	for name in common:
+		print "Entering subdirectory", repr(name)
+		local.cd(name)
+		remote.cd(name)
+		compare(local, remote, mode)
+		remote.back()
+		local.back()
+
+def sendfile(local, remote, name):
+	try:
+		remote.create(name)
+	except (IOError, os.error), msg:
+		print "cannot create:", msg
+		return
+	
+	print "sending ...",
+	fl()
+	
+	data = open(name).read()
+	
+	t1 = time.time()
+	
+	remote._send_noreply('write', name, data)
+	remote._flush()
+	
+	t2 = time.time()
+	
+	dt = t2-t1
+	print len(data), "bytes in", t2-t1, "seconds",
+	if dt:
+		print "i.e.", len(data)/dt, "bytes/sec",
+	print
+
+def recvfile(local, remote, name):
+	try:
+		local.create(name)
+	except (IOError, os.error), msg:
+		print "cannot create:", msg
+		return
+	
+	print "receiving ...",
+	fl()
+	
+	f = open(name, 'w')
+	t1 = time.time()
+	
+	length = 4*1024
+	offset = 0
+	id = remote._send('read', name, offset, length)
+	remote._flush()
+	while 1:
+		newoffset = offset + length
+		newid = remote._send('read', name, newoffset, length)
+		data = remote._recv(id)
+		id = newid
+		if not data: break
+		f.seek(offset)
+		f.write(data)
+		offset = newoffset
+	size = f.tell()
+	
+	t2 = time.time()
+	f.close()
+	
+	dt = t2-t1
+	print size, "bytes in", dt, "seconds",
+	if dt:
+		print "i.e.", int(size/dt), "bytes/sec",
+	print
+	remote._recv(id) # ignored
+
+def fl():
+	sys.stdout.flush()
+
+if __name__ == '__main__':
+	main()

Demo/pdist/server.py

@@ -0,0 +1,111 @@
+"""RPC Server module."""
+
+import sys
+import socket
+import pickle
+from fnmatch import fnmatch
+from repr import repr
+
+
+# Default verbosity (0 = silent, 1 = print connections, 2 = print requests too)
+VERBOSE = 1
+
+
+class Server:
+	
+	"""RPC Server class.  Derive a class to implement a particular service."""
+	
+	def __init__(self, address, verbose = VERBOSE):
+		if type(address) == type(0):
+			address = ('', address)
+		self._address = address
+		self._verbose = verbose
+		self._socket = None
+		self._socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+		self._socket.bind(address)
+		self._socket.listen(1)
+		self._listening = 1
+	
+	def _setverbose(self, verbose):
+		self._verbose = verbose
+	
+	def __del__(self):
+		self._close()
+	
+	def _close(self):
+		self._listening = 0
+		if self._socket:
+			self._socket.close()
+		self._socket = None
+	
+	def _serverloop(self):
+		while self._listening:
+			self._serve()
+	
+	def _serve(self):
+		if self._verbose: print "Wait for connection ..."
+		conn, address = self._socket.accept()
+		if self._verbose: print "Accepted connection from %s" % repr(address)
+		if not self._verify(conn, address):
+			print "*** Connection from %s refused" % repr(address)
+			conn.close()
+			return
+		rf = conn.makefile('r')
+		wf = conn.makefile('w')
+		ok = 1
+		while ok:
+			wf.flush()
+			if self._verbose > 1: print "Wait for next request ..."
+			ok = self._dorequest(rf, wf)
+	
+	_valid = ['192.16.201.*', '192.16.197.*']
+	
+	def _verify(self, conn, address):
+		host, port = address
+		for pat in self._valid:
+			if fnmatch(host, pat): return 1
+		return 0
+	
+	def _dorequest(self, rf, wf):
+		rp = pickle.Unpickler(rf)
+		try:
+			request = rp.load()
+		except EOFError:
+			return 0
+		if self._verbose > 1: print "Got request: %s" % repr(request)
+		try:
+			methodname, args, id = request
+			if '.' in methodname:
+				reply = (None, self._special(methodname, args), id)
+			elif methodname[0] == '_':
+				raise NameError, "illegal method name %s" % repr(methodname)
+			else:
+				method = getattr(self, methodname)
+				reply = (None, apply(method, args), id)
+		except:
+			reply = (sys.exc_type, sys.exc_value, id)
+		if id < 0 and reply[:2] == (None, None):
+			if self._verbose > 1: print "Suppress reply"
+			return 1
+		if self._verbose > 1: print "Send reply: %s" % repr(reply)
+		wp = pickle.Pickler(wf)
+		wp.dump(reply)
+		return 1
+	
+	def _special(self, methodname, args):
+		if methodname == '.methods':
+			if not hasattr(self, '_methods'):
+				self._methods = tuple(self._listmethods())
+			return self._methods
+		raise NameError, "unrecognized special method name %s" % repr(methodname)
+	
+	def _listmethods(self, cl=None):
+		if not cl: cl = self.__class__
+		names = cl.__dict__.keys()
+		names = filter(lambda x: x[0] != '_', names)
+		names.sort()
+		for base in cl.__bases__:
+			basenames = self._listmethods(base)
+			basenames = filter(lambda x, names=names: x not in names, basenames)
+			names[len(names):] = basenames
+		return names

Demo/pdist/sumtree.py

@@ -0,0 +1,24 @@
+import time
+import FSProxy
+
+def main():
+	t1 = time.time()
+	#proxy = FSProxy.FSProxyClient(('voorn.cwi.nl', 4127))
+	proxy = FSProxy.FSProxyLocal()
+	sumtree(proxy)
+	proxy._close()
+	t2 = time.time()
+	print t2-t1, "seconds"
+	raw_input("[Return to exit] ")
+
+def sumtree(proxy):
+	print "PWD =", proxy.pwd()
+	files = proxy.listfiles()
+	proxy.infolist(files)
+	subdirs = proxy.listsubdirs()
+	for name in subdirs:
+		proxy.cd(name)
+		sumtree(proxy)
+		proxy.back()
+
+main()