git » abk » commit 8d02c36

Reduce memory usage by moving methods to functions.

author Alberto Bertogli
2005-04-12 21:29:23 UTC
committer Alberto Bertogli
2005-04-12 21:29:23 UTC
parent 005d9d5762898ea46b8867521fbdb28343a1532e

Reduce memory usage by moving methods to functions.
This patch moves file_info methods to external functions which are referenced
indirectly. It reduces memory usage almost by half, leaving aprox. 2kbyte per
indexed file.

abk +205 -183

diff --git a/abk b/abk
index 7818603..56e928b 100644
--- a/abk
+++ b/abk
@@ -24,6 +24,184 @@ VERSION = "0.03"
 # classes
 #
 
+# file_info functions are not included directly in the class to avoid memory
+# waste, which is about 1k per file.
+
+def finfo_load(finfo):
+	"Loads data from the file."
+	s = os.lstat(finfo.name)
+	finfo.stat = s
+	if S_ISREG(s.st_mode):
+		finfo.type = 'r'
+	elif S_ISLNK(s.st_mode):
+		finfo.type = 'l'
+	elif S_ISCHR(s.st_mode):
+		finfo.type = 'c'
+		finfo.rdev = s.st_rdev
+	elif S_ISBLK(s.st_mode):
+		finfo.type = 'b'
+		finfo.rdev = s.st_rdev
+	elif S_ISFIFO(s.st_mode):
+		finfo.type = 'f'
+		finfo.linkto = os.readlink(finfo.name)
+	elif S_ISDIR(s.st_mode):
+		finfo.type = 'd'
+	else:
+		finfo.type = 'u'
+
+	finfo.mtime = s.st_mtime
+	finfo.atime = s.st_atime
+	finfo.size = s.st_size
+	finfo.mode = s.st_mode
+	finfo.uid = s.st_uid
+	finfo.gid = s.st_gid
+
+	if finfo.type == 'r':
+		finfo.hash = finfo.hash_file()
+
+def finfo_cmp_mdata(finfo, other):
+	"Compares metadata to other."
+	if finfo.mtime != other.mtime: return 0
+	if finfo.mode != other.mode: return 0
+	if finfo.uid != other.uid: return 0
+	if finfo.gid != other.gid: return 0
+	return 1
+
+def finfo_cmp_data(finfo, other):
+	"Compares data to other."
+	if finfo.size != other.size: return 0
+	if finfo.hash != other.hash: return 0
+	if finfo.type != other.type: return 0
+	if finfo.type == 'b' or finfo.type == 'c':
+		if finfo.rdev != other.rdev:
+			return 0
+	if finfo.type == 'l':
+		if finfo.linkto != other.linkto:
+			return 0
+	return 1
+
+def finfo_copy_file_reg_raw(finfo, dst):
+	"Copy a regular file."
+	sfile = open(finfo.name, 'r')
+	dfile = open(dst, 'w')
+
+	# the data
+	data = sfile.read(PSIZE)
+	while data:
+		dfile.write(data)
+		data = sfile.read(PSIZE)
+
+	sfile.close()
+	dfile.close()
+
+def finfo_copy_file_reg_bzip2(finfo, dst):
+	"Copy a regular file, destination is bz2 compressed."
+	import bz2
+	sfile = open(finfo.name)
+	dfile = open(dst, 'w')
+
+	bcomp = bz2.BZ2Compressor()
+	data = sfile.read(PSIZE)
+	while data:
+		dfile.write(bcomp.compress(data))
+		data = sfile.read(PSIZE)
+	dfile.write(bcomp.flush())
+	sfile.close()
+	dfile.close()
+
+def finfo_copy_file_reg_gzip(finfo, dst):
+	"Copy a regular file, destination is gzip compressed."
+	import gzip
+	sfile = open(finfo.name)
+	dfile = gzip.open(dst, 'w')
+
+	data = sfile.read(PSIZE)
+	while data:
+		dfile.write(data)
+		data = sfile.read(PSIZE)
+
+	sfile.close()
+	dfile.close()
+
+
+def finfo_copy_file_link(finfo, dst):
+	"Copy a symbolic link."
+	linkto = os.readlink(finfo.name)
+	os.symlink(linkto, dst)
+
+
+def finfo_copy_file_dev(finfo, dst):
+	"Copy a device file."
+	major = os.major(finfo.rdev)
+	minor = os.minor(finfo.rdev)
+	dev = os.makedev(major, minor)
+	os.mknod(dst, finfo.mode, dev)
+
+def finfo_update_mdata(finfo, dst):
+	"Updates a file's metadata."
+	os.lchown(dst, finfo.uid, finfo.gid)
+	if finfo.type != 'l':
+		# these don't really like symlinks
+		os.utime(dst, (finfo.atime, finfo.mtime))
+		os.chmod(dst, finfo.mode & 07777)
+
+def finfo_copy_file(finfo, dst):
+	"Copies a file, along with its permissions and ownership."
+	# create the path to dst if it doesn't exist
+	make_path(dst)
+
+	# copy accordingly to the file type
+	if finfo.type == 'r':
+		finfo.copy_file_reg(dst)
+	elif finfo.type == 'l':
+		finfo.copy_file_link(dst)
+	elif finfo.type == 'b' or finfo.type == 'c':
+		finfo.copy_file_dev(dst)
+	elif finfo.type == 'f':
+		# we just create fifos
+		os.mkfifo(dst, finfo.mode & 07777)
+	elif finfo.type == 'd':
+		# we just create directories
+		try:
+			os.makedirs(dst, finfo.mode & 07777)
+		except OSError:
+			# ignore if the dir already exists, it could
+			# happen because the walker doesn't do it in
+			# any kind of order, so a subdirectory might
+			# be created before the parent.
+			pass
+	else:
+		raise 'Unk type: 0x%x %d' % (finfo.mode, finfo.name)
+
+def finfo_hash_file_sha(finfo):
+	"Returns the sha1sum of a file."
+	import sha
+	hash = sha.new()
+	f = open(finfo.name)
+	data = f.read(PSIZE)
+	while data:
+		hash.update(data)
+		data = f.read(PSIZE)
+	f.close()
+	return hash.hexdigest()
+
+def finfo_hash_file_md5(finfo):
+	"Returns the md5sum of a file."
+	import md5
+	hash = md5.new()
+	f = open(finfo.name)
+	data = f.read(PSIZE)
+	while data:
+		hash.update(data)
+		data = f.read(PSIZE)
+	f.close()
+	return hash.hexdigest()
+
+def finfo_hash_file_none(finfo):
+	"Empty hash."
+	return '-'
+
+
 class file_info:
 	"Represents a file"
 	def __init__(self, name):
@@ -40,63 +218,31 @@ class file_info:
 		self.hash = None
 		self.stat = None
 
+	# to save memory, reference external functions
+	load = finfo_load
+	cmp_mdata = finfo_cmp_mdata
+	cmp_data = finfo_cmp_data
+	copy_file_reg_raw = finfo_copy_file_reg_raw
+	copy_file_reg_bzip2 = finfo_copy_file_reg_bzip2
+	copy_file_reg_gzip = finfo_copy_file_reg_gzip
+	copy_file_link = finfo_copy_file_link
+	copy_file_dev = finfo_copy_file_dev
+	update_mdata = finfo_update_mdata
+	copy_file = finfo_copy_file
+	hash_file_sha = finfo_hash_file_sha
+	hash_file_md5 = finfo_hash_file_md5
+	hash_file_none = finfo_hash_file_none
+
+	# the following functions are modified by configuration
+	# hashing function
+	hash_file = finfo_hash_file_sha
+
+	# copy function
+	copy_file_reg = finfo_copy_file_reg_gzip
 
 	def __repr__(self):
 		return "<%s: %s %d>" % (self.name, self.type, self.size)
 
-	def load(self):
-		"Loads data from the file."
-		s = os.lstat(self.name)
-		self.stat = s
-		if S_ISREG(s.st_mode):
-			self.type = 'r'
-		elif S_ISLNK(s.st_mode):
-			self.type = 'l'
-		elif S_ISCHR(s.st_mode):
-			self.type = 'c'
-			self.rdev = s.st_rdev
-		elif S_ISBLK(s.st_mode):
-			self.type = 'b'
-			self.rdev = s.st_rdev
-		elif S_ISFIFO(s.st_mode):
-			self.type = 'f'
-			self.linkto = os.readlink(self.name)
-		elif S_ISDIR(s.st_mode):
-			self.type = 'd'
-		else:
-			self.type = 'u'
-
-		self.mtime = s.st_mtime
-		self.atime = s.st_atime
-		self.size = s.st_size
-		self.mode = s.st_mode
-		self.uid = s.st_uid
-		self.gid = s.st_gid
-
-		if self.type == 'r':
-			self.hash = self.hash_file()
-
-	def cmp_mdata(self, other):
-		"Compares metadata to other."
-		if self.mtime != other.mtime: return 0
-		if self.mode != other.mode: return 0
-		if self.uid != other.uid: return 0
-		if self.gid != other.gid: return 0
-		return 1
-
-	def cmp_data(self, other):
-		"Compares data to other."
-		if self.size != other.size: return 0
-		if self.hash != other.hash: return 0
-		if self.type != other.type: return 0
-		if self.type == 'b' or self.type == 'c':
-			if self.rdev != other.rdev:
-				return 0
-		if self.type == 'l':
-			if self.linkto != other.linkto:
-				return 0
-		return 1
-
 	def __eq__(self, other):
 		"Compares to other file_info object."
 		if self.name != other.name: return 0
@@ -108,130 +254,6 @@ class file_info:
 	def __ne__(self, other):
 		return not (self == other)
 
-	def copy_file_reg_raw(self, dst):
-		"Copy a regular file."
-		sfile = open(self.name, 'r')
-		dfile = open(dst, 'w')
-
-		# the data
-		data = sfile.read(PSIZE)
-		while data:
-			dfile.write(data)
-			data = sfile.read(PSIZE)
-
-		sfile.close()
-		dfile.close()
-
-	def copy_file_reg_bzip2(self, dst):
-		"Copy a regular file, destination is bz2 compressed."
-		import bz2
-		sfile = open(self.name)
-		dfile = open(dst, 'w')
-
-		bcomp = bz2.BZ2Compressor()
-		data = sfile.read(PSIZE)
-		while data:
-			dfile.write(bcomp.compress(data))
-			data = sfile.read(PSIZE)
-		dfile.write(bcomp.flush())
-		sfile.close()
-		dfile.close()
-
-	def copy_file_reg_gzip(self, dst):
-		"Copy a regular file, destination is gzip compressed."
-		import gzip
-		sfile = open(self.name)
-		dfile = gzip.open(dst, 'w')
-
-		data = sfile.read(PSIZE)
-		while data:
-			dfile.write(data)
-			data = sfile.read(PSIZE)
-
-		sfile.close()
-		dfile.close()
-
-	copy_file_reg = copy_file_reg_gzip
-
-	def copy_file_link(self, dst):
-		"Copy a symbolic link."
-		linkto = os.readlink(self.name)
-		os.symlink(linkto, dst)
-
-
-	def copy_file_dev(self, dst):
-		"Copy a device file."
-		major = os.major(self.rdev)
-		minor = os.minor(self.rdev)
-		dev = os.makedev(major, minor)
-		os.mknod(dst, self.mode, dev)
-
-	def update_mdata(self, dst):
-		"Updates a file's metadata."
-		os.lchown(dst, self.uid, self.gid)
-		if self.type != 'l':
-			# these don't really like symlinks
-			os.utime(dst, (self.atime, self.mtime))
-			os.chmod(dst, self.mode & 07777)
-
-	def copy_file(self, dst):
-		"Copies a file, along with its permissions and ownership."
-		# create the path to dst if it doesn't exist
-		make_path(dst)
-
-		# copy accordingly to the file type
-		if self.type == 'r':
-			self.copy_file_reg(dst)
-		elif self.type == 'l':
-			self.copy_file_link(dst)
-		elif self.type == 'b' or self.type == 'c':
-			self.copy_file_dev(dst)
-		elif self.type == 'f':
-			# we just create fifos
-			os.mkfifo(dst, self.mode & 07777)
-		elif self.type == 'd':
-			# we just create directories
-			try:
-				os.makedirs(dst, self.mode & 07777)
-			except OSError:
-				# ignore if the dir already exists, it could
-				# happen because the walker doesn't do it in
-				# any kind of order, so a subdirectory might
-				# be created before the parent.
-				pass
-		else:
-			raise 'Unk type: 0x%x %d' % (self.mode, self.name)
-
-	def hash_file_sha(self):
-		"Returns the sha1sum of a file."
-		import sha
-		hash = sha.new()
-		f = open(self.name)
-		data = f.read(PSIZE)
-		while data:
-			hash.update(data)
-			data = f.read(PSIZE)
-		f.close()
-		return hash.hexdigest()
-
-	def hash_file_md5(self):
-		"Returns the md5sum of a file."
-		import md5
-		hash = md5.new()
-		f = open(self.name)
-		data = f.read(PSIZE)
-		while data:
-			hash.update(data)
-			data = f.read(PSIZE)
-		f.close()
-		return hash.hexdigest()
-
-	def hash_file_none(self):
-		"Empty hash."
-		return '-'
-
-	hash_file = hash_file_sha
-
 
 class index_file:
 	"Represents the index file."
@@ -496,20 +518,20 @@ verbose = opts.verbose
 
 # configuration
 if opts.copy_mode == 'raw':
-	file_info.copy_file_reg = file_info.copy_file_reg_raw
+	file_info.copy_file_reg = finfo_copy_file_reg_raw
 elif opts.copy_mode == 'gzip':
-	file_info.copy_file_reg = file_info.copy_file_reg_gzip
+	file_info.copy_file_reg = finfo_copy_file_reg_gzip
 elif opts.copy_mode == 'bzip2':
-	file_info.copy_file_reg = file_info.copy_file_reg_bzip2
+	file_info.copy_file_reg = finfo_copy_file_reg_bzip2
 else:
 	parser.error("Invalid copy mode (%s)." % opts.copy_mode)
 
 if opts.hash_mode == 'none':
-	file_info.hash_file = file_info.hash_file_none
+	file_info.hash_file = finfo_hash_file_none
 elif opts.hash_mode == 'md5':
-	file_info.hash_file = file_info.hash_file_md5
+	file_info.hash_file = finfo_hash_file_md5
 elif opts.hash_mode == 'sha':
-	file_info.hash_file = file_info.hash_file_sha
+	file_info.hash_file = finfo_hash_file_sha
 else:
 	parser.error("Invalid hash mode (%s)." % opts.hash_mode)