Minor enhancements

- efficient ntfs_get_free_clusters
- Boot records directly sizes of Cluster, Record, Index and MFT LCN
- more internal documentation
- Security_Descriptor attribute

Author: maxpat78

FATtools/NTFS/Attribute.py

@@ -8,7 +8,8 @@
 DEBUG=int(os.getenv('FATTOOLS_DEBUG', '0'))
 
 __all__ = ['Attribute', 'Standard_Information', 'Attribute_List', 'Attribute_Listed', 'File_Name', 'Data',
-'Index_Root', 'Index_Allocation', 'Bitmap', 'Volume_Name', 'Volume_Information', 'attributes_by_id', 'attributes_by_name']
+'Index_Root', 'Index_Allocation', 'Bitmap', 'Volume_Name', 'Volume_Information', 'Security_Descriptor',
+'attributes_by_id', 'attributes_by_name']
 
 
 attributes_by_id ={
@@ -49,15 +50,16 @@ class Attribute:
 	0x16: ('uchFlags', 'B'),
 	0x17: ('uchPadding', 'B') } # Size = 0x18 (24) bytes (total)
 
-	layout_nonresident = {  # additional layout (non resident content)
-	0x10: ('u64StartVCN', '<Q'),
-	0x18: ('u64EndVCN', '<Q'),
+	layout_nonresident = {  # additional layout (non resident contents)
+	0x10: ('u64StartVCN', '<Q'), # first Virtual Cluster Number of contents
+	0x18: ('u64EndVCN', '<Q'), # last VCN of contents
 	0x20: ('wDatarunOffset', '<H'),
 	0x22: ('wCompressionSize', '<H'), 
 	0x24: ('uchPadding', '4s'),
-	0x28: ('u64AllocSize', '<Q'),
-	0x30: ('u64RealSize', '<Q'),
-	0x38: ('u64StreamSize', '<Q') } # Size = 0x40 (64) bytes (total)
+	0x28: ('u64AllocSize', '<Q'), # bytes occupied by allocated clusters
+	0x30: ('u64RealSize', '<Q'), # bytes occupied by true contents
+	0x38: ('u64StreamSize', '<Q') # always equal to u64RealSize?
+	} # Size = 0x40 (64) bytes (total)
 
 	def __init__(self, parent, offset):
 		self._parent = parent # parent Record
@@ -91,10 +93,10 @@ class Standard_Information(Attribute):
 	0x24: ('dwMaxVerNum', '<I'),
 	0x28: ('dwVerNum', '<I'),
 	0x2C: ('dwClassId', '<I'),
-	0x30: ('dwOwnerId', '<I'),  # these 4 since NTFS 3.0 (Windows 2000)
+	0x30: ('dwOwnerId', '<I'),  # next 4 since NTFS 3.0 (Windows 2000)
 	0x34: ('dwSecurityId', '<I'),
 	0x38: ('u64QuotaCharged', '<Q'),
-	0x40: ('u64USN', '<Q') } # 0x48 (72) bytes
+	0x40: ('u64USN', '<Q') } # 0x30 (48) - 0x48 (72) bytes
 
 	def __init__(self, parent, offset):
 		Attribute.__init__(self, parent, offset)
@@ -104,6 +106,8 @@ def __str__ (self):
 		s = ''
 		L1 = utils.class2str(self, "$STANDARD_INFORMATION @%x\n" % self._i).split('\n')
 		L2 = []
+		if self.dwLength == 0x30: # NTFS <3.0
+			del L1[-5:-1]
 		for key in (0x18, 0x20, 0x28, 0x30):
 			o = self._kv[key][0]
 			v = getattr(self, o)
@@ -151,7 +155,7 @@ def __init__(self, parent, offset):
 		self.Name = ''
 		if self.ucbNameLen:
 			i = self._i+self.ucbNameOffs
-			self.Name = (b'\xFF\xFE' + self._buf[i:i+self.ucbNameLen*2]).decode('utf16')
+			self.Name = (self._buf[i:i+self.ucbNameLen*2]).decode('utf-16le')
 
 	__getattr__ = utils.common_getattr
 
@@ -169,7 +173,7 @@ class File_Name(Attribute):
 	0x28: ('u64AllocatedSize', '<Q'),
 	0x30: ('u64RealSize', '<Q'),
 	0x38: ('dwFlags', '<I'),
-	0x3C: ('dwEA', '<I'),  # These 3 since NTFS 3.0 (Windows 2000)
+	0x3C: ('dwEA', '<I'),
 	0x40: ('ucbFileName', 'B'),
 	0x41: ('uFileNameNamespace', 'B') } # 0x42 (66) bytes
 
@@ -178,7 +182,7 @@ def __init__(self, parent, offset):
 		common_update_and_swap(self)
 		# Always resident - so name is at (24+66) bytes from start
 		i = self._i+90
-		self.FileName = (b'\xFF\xFE' + self._buf[i:i+self.ucbFileName*2]).decode('utf16')
+		self.FileName = (self._buf[i:i+self.ucbFileName*2]).decode('utf-16le')
 
 	def __str__ (self):
 		s = ''
@@ -260,10 +264,10 @@ def __str__ (self):
 
 	decode = common_dataruns_decode
 
-""" A Bitmap is typically stored:
+""" A Bitmap is typically found:
 - as $MFT Record Attribute (tracking used FILE records)
-- as $INDEX_ALLOCATION contents (tracking FILE records *not* used)
-- as $Bitmap record contents (tracking free volume clusters) """
+- as directory record attribute (tracking INDX blocks used in an $INDEX_ALLOCATION)
+- as $Bitmap file contents (tracking free/used volume clusters)"""
 class Bitmap(Attribute):
 	specific_layout = {}
 
@@ -298,7 +302,7 @@ def __init__(self, parent, offset):
 		Attribute.__init__(self, parent, offset)
 		common_update_and_swap(self)
 		i = self._i + self.wAttrOffset
-		self.VolumeName = (b'\xFF\xFE' + self._buf[i:i+self.dwLength]).decode('utf16')
+		self.VolumeName = (self._buf[i:i+self.dwLength]).decode('utf-16le')
 
 	def __str__ (self):
 		L1 = utils.class2str(self, "$VOLUME_NAME @%x\n" % self._i).split('\n')
@@ -308,7 +312,7 @@ class Volume_Information(Attribute):
 	# Always resident
 	specific_layout = {
 	0x00: ('u64Reserved', '<Q'),
-	0x08: ('bMajorVersion', 'B'), # 3.1=XP+, 3.0=2K, 1.2=NT
+	0x08: ('bMajorVersion', 'B'), #  1.1=NT 3.51, 1.2=NT4, 3.0=2K, 3.1=XP+
 	0x09: ('bMinorVersion', 'B'),
 	0x0A: ('wFlags', '<H'), # 1=dirty, 2=log resize, 4=to upgrade, 8=mounted on NT4, 10h=del USN, 20h=repair OIDS, 8000h=modified by CHKDSK
 	0x0C: ('dwReserved', '<I') } # 0x10 (16) bytes
@@ -318,3 +322,15 @@ def __init__(self, parent, offset):
 		common_update_and_swap(self)
 
 	def __str__ (self): return utils.class2str(self, "$VOLUME_INFORMATION @%x\n" % self._i)
+
+	def is_dirty(self):
+		return self.wFlags & 1
+
+class Security_Descriptor(Attribute):
+	specific_layout = {}
+
+	def __init__(self, parent, offset):
+		Attribute.__init__(self, parent, offset)
+		common_update_and_swap(self)
+
+	def __str__ (self): return utils.class2str(self, "$SECURITY_DESCRIPTOR @%x\n" % self._i)

FATtools/NTFS/Boot.py

@@ -3,26 +3,26 @@
 
 class Bootsector:
 	layout = { # { offset: (name, unpack string) }
-	0x00: ('chJumpInstruction', '3s'),
-	0x03: ('chOemID', '8s'),
-	0x0B: ('wBytesPerSec', '<H'),
-	0x0D: ('uchSecPerClust', 'B'),
-	0x0E: ('u64Unused1', '<Q'),
-	0x15: ('uchMediaDescriptor', 'B'),
+	0x00: ('chJumpInstruction', '3s'), # CHKDSK likes this
+	0x03: ('chOemID', '8s'), # "NTFS    "
+	0x0B: ('wBytesPerSec', '<H'), # typically, 512
+	0x0D: ('uchSecPerClust', 'B'), # typically, 8 (4K cluster)
+	0x0E: ('u64Unused1', '<Q'), # 0xf800000000000000
+	0x15: ('uchMediaDescriptor', 'B'), # typically 0xF8 (HDD)
 	0x16: ('wUnused2', '<H'),
 	0x18: ('wSecPerTrack', '<H'),
 	0x1A: ('wNumberOfHeads', '<H'),
 	0x1C: ('u64Unused3', '<Q'),
-	0x24: ('dwUnknown', '<I'),
+	0x24: ('dwUnknown', '<I'), # typically 0x800080
 	0x25: ('dwUnused4', '<I'),
-	0x28: ('u64TotalSectors', '<Q'),
-	0x30: ('u64MFTLogicalClustNum', '<Q'),
-	0x38: ('u64MFTMirrLogicalClustNum', '<Q'),
-	0x40: ('nClustPerMFTRecord', '<b'), # if <0, 2^-n
+	0x28: ('u64TotalSectors', '<Q'), # count of volume sectors
+	0x30: ('u64MFTLogicalClustNum', '<Q'), # cluster of the $MFT Record (and Master File Table start)
+	0x38: ('u64MFTMirrLogicalClustNum', '<Q'), # $MFTMirr cluster (backup of first 4 $MFT Record)
+	0x40: ('nClustPerMFTRecord', '<b'), # if n<0 then 2^-n. Typically, 0xF6 (-10) or 1K. 
 	0x44: ('nClustPerIndexRecord', '<b'),
 	0x48: ('u64VolumeSerialNum', '<Q'),
-	#~ 0x50: ('dwChecksum', '<I'),
-	#~ 0x54: ('chBootstrapCode', '426s'),
+	0x50: ('dwChecksum', '<I'), # typically zero
+	0x54: ('chBootstrapCode', '426s'),
 	0x1FE: ('wSecMark', '<H') } # Size = 0x100 (512 byte)
 
 	def __init__ (self, stream):
@@ -35,6 +35,11 @@ def __init__ (self, stream):
 		self._vk = {} # { name: offset}
 		for k, v in self._kv.items():
 			self._vk[v[0]] = k
+		# calculates and stores immediately some vital numbers
+		self.LcnMFT = self.mft()
+		self.cbCluster = self.cluster()
+		self.cbRecord = self.record()
+		self.cbIndx = self.index()
 
 	__getattr__ = utils.common_getattr
 

FATtools/NTFS/Commons.py

@@ -68,7 +68,7 @@ def common_dataruns_decode(self):
 		# computates and stores run length and offset according to cluster size
 		if DEBUG&8: log("length=%d offset=%d prevoffset=%d", length, offset, self.dataruns[-1])
 		# CAVE! EFFECTIVE cluster size MUST be used!
-		self.dataruns += (length* self.boot.cluster(), (offset* self.boot.cluster()+self.dataruns[-1]))
+		self.dataruns += (length* self.boot.cbCluster, (offset* self.boot.cbCluster+self.dataruns[-1]))
 		i += n_offset
 	if DEBUG&8: log("decoded dataruns @%d:\n%s", self._i, self.dataruns)
 

FATtools/NTFS/Index.py

@@ -189,7 +189,7 @@ def __init__ (self, buffer, index):
 		self.FileName = ''
 		if self.wfilenameOffset:
 			j = index + 82
-			self.FileName = (b'\xFF\xFE' + self._buf[j: j+self.ucbFileName*2]).decode('utf16')
+			self.FileName = (self._buf[j: j+self.ucbFileName*2]).decode('utf-16le')
 		if DEBUG & 8: log('Decoded INDEX_ENTRY @%x\n%s', index, self)
 
 	__getattr__ = utils.common_getattr

FATtools/NTFS/Record.py

@@ -17,19 +17,20 @@ class Record:
 	0x10: ('wSequence', '<H'),
 	0x12: ('wHardLinks', '<H'),
 	0x14: ('wAttribOffset', '<H'),
-	0x16: ('wFlags', '<H'),
+	0x16: ('wFlags', '<H'), # 1=Record in use
 	0x18: ('dwRecLength', '<I'),
 	0x1C: ('dwAllLength', '<I'),
 	0x20: ('u64BaseMftRec', '<Q'),
 	0x28: ('wNextAttrID', '<H'),
 	0x2A: ('wFixupPattern', '<H'),
-	0x2C: ('dwMFTRecNumber', '<I') } # Size = 0x30 (48 byte)
-	
+	0x2C: ('dwMFTRecNumber', '<I') #  NTFS v3.0+
+	} # Size = 0x2C (44 byte)
+
 	def __init__ (self, boot, mftstream):
 		#~ mftrecord.boot.stream --> NTFS Volume stream
 		#~ mftrecord._stream --> $MFT stream
 		self.boot = boot
-		record_size = boot.record()
+		record_size = boot.cbRecord
 		self._i = 0 # buffer offset
 		self._pos = mftstream.tell() # MFT offset
 		self._buf = mftstream.read(record_size)
@@ -62,6 +63,8 @@ def __init__ (self, boot, mftstream):
 				self._expand_attribute_list(a)
 			elif dwType == 0x30:
 				a = File_Name(self, offset)
+			elif dwType == 0x50:
+				a = Security_Descriptor(self, offset)
 			elif dwType == 0x60:
 				a = Volume_Name(self, offset)
 			elif dwType == 0x70:
@@ -94,11 +97,11 @@ def __init__ (self, boot, mftstream):
 	__getattr__ = utils.common_getattr
 	fixup = common_fixup
 
-	def __str__ (self): return utils.class2str(self, "MFT Record 0x%X @%x\n" % (self._pos//self.boot.record(), self._pos))
+	def __str__ (self): return utils.class2str(self, "MFT Record 0x%X @%x\n" % (self._pos//self.boot.cbRecord, self._pos))
 
 	def next(self, index=1):
 		"Parses the next or n-th Record"
-		off = index * self.boot.record()
+		off = index * self.boot.cbRecord
 		if index > 1:
 			if off > self._stream.size:
 				raise NTFSException("MFT Record #%d does not exist!" % index)
@@ -131,7 +134,7 @@ def _expand_attribute_list(self, al):
 		for a in expanded:
 			n = a.u64BaseMFTFileRef & 0x0000FFFFFFFFFFFF
 			# this listed attributed resides in this same Record
-			if n*self.boot.record() == self._pos:
+			if n*self.boot.cbRecord == self._pos:
 				continue
 			if DEBUG&8: log("loading attributes in Record %08X", n)
 			# loads the referenced Record and updates with contained attributes

FATtools/NTFS/Utils.py

@@ -149,9 +149,9 @@ def iterator(p):
 
 	def getdiskspace(p):
 		"Returns the disk free space in a tuple (clusters, bytes)"
-		return (0, 0)
-		#~ freec = ntfs_get_free_clusters(p.mft)
-		#~ return (freec, freec*p.mft.boot.cluster())
+		#~ return (0, 0)
+		freec = ntfs_get_free_clusters(p.mft)
+		return (freec, freec*p.mft.boot.cbCluster)
 
 
 
@@ -170,46 +170,44 @@ def ntfs_open_volume(stream):
 	boot = Bootsector(stream)
 	assert boot.chOemID == b'NTFS    '
 	assert boot.wSecMark == 0xaa55
-	stream.seek(boot.mft())
+	stream.seek(boot.LcnMFT)
 	mft = Record(boot, stream)
 	if mft.find_attribute("$FILE_NAME")[0].FileName == '$MFT':
 		mft = Record(boot, mft.find_attribute(0x80)[0].file)
 	else:
 		raise NTFSException("The NTFS Master File Table $MFT was not found!")
 	return mft
 
-# Terribly slow! Avoid using!
 def ntfs_get_free_clusters(mft):
-	"Computates free clusters given the MFT Record"
+	# Precalcola il numero di zero per ogni byte
+	bit_zero_table = [8 - bin(i).count('1') for i in range(256)]
 	bmp = ntfs_open_record(mft, "$Bitmap") # NTFS volume bitmap
 	f = bmp.find_attribute("$DATA")[-1].file
 	totc = mft.boot.u64TotalSectors // mft.boot.uchSecPerClust # volume clusters
-	#~ print('DBG: total clusters', totc)
-	#~ print('DBG: $Bitmap size', f.size)
 	freec = 0
+	excb = 8 - totc%8 # excedent bits
 	while totc > 0:
-		B = f.read(1) # process 8 clusters at once
-		totc -= 8
-		if B == b'\x00':
-			freec += 8
-			continue
-		if B == b'\xFF':
-			continue
-		for i in range(8):
-			if not ((B[0] & (1 << i)) != 0): freec += 1
+		cb = min(totc//8 or 1, 1<<20) # process min 1b, max 1MB bitmap
+		totc -= cb*8
+		buf = f.read(cb) 
+		zeros = sum(bit_zero_table[b] for b in buf)
+		freec += zeros
+		if totc < 0:
+			# excedent bits are 1 already?
+			pass
 	return freec
 
 def ntfs_open_dir(rcrd):
 	"Opens a directory record returning its associated Index"
 	# Entries can be both resident and not resident
 	ir = rcrd.find_attribute("$INDEX_ROOT")[0]
-	ixr = Index(ir.file, None, rcrd.boot.index(), 1)
+	ixr = Index(ir.file, None, rcrd.boot.cbIndx, 1)
 	ia = rcrd.find_attribute("$INDEX_ALLOCATION")
 	# An allocation, if present, extends the root index
 	if ia:
 		ia = ia[0]
 		bm = rcrd.find_attribute("$BITMAP")[0]
-		ixa = Index(ia.file, bm, rcrd.boot.index(), 0)
+		ixa = Index(ia.file, bm, rcrd.boot.cbIndx, 0)
 		return IndexGroup(ixr, ixa)
 	return ixr
 
@@ -221,7 +219,7 @@ def ntfs_open_root(rcrd):
 def ntfs_open_record(mftrecord, record):
 	"Opens a MFT Record by name or number, directly searching the MFT"
 	if type(record) == int:
-		mftrecord._stream.seek(record*mftrecord.boot.record())
+		mftrecord._stream.seek(record*mftrecord.boot.cbRecord)
 		return Record(mftrecord.boot, mftrecord._stream)
 	elif type(record) == str:
 		mftrecord._stream.seek(0)

FATtools/NTFS/__init__.py

@@ -4,4 +4,4 @@
 COPYRIGHT = '''Copyright (C)2012-2025, by maxpat78. GNU GPL v3 applies.
 This free software reads NTFS file systems WITH ABSOLUTELY NO WARRANTY!'''
 
-VERSION = '0.9.2b'
+VERSION = '0.9.3b'

FATtools/version.py

@@ -1 +1 @@
-__version__ = '1.1.2'
+__version__ = '1.1.3'