Coverage for /root/GitHubProjects/impacket/impacket/ese.py: 20%

# Coverage for /root/GitHubProjects/impacket/impacket/ese.py : 20%

## 506 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1# Impacket - Collection of Python classes for working with network protocols.

2#

3# SECUREAUTH LABS. Copyright (C) 2018 SecureAuth Corporation. All rights reserved.

4#

5# This software is provided under a slightly modified version

6# of the Apache Software License. See the accompanying LICENSE file

7# for more information.

8#

9# Description:

10# Microsoft Extensive Storage Engine parser, just focused on trying

11# to parse NTDS.dit files (not meant as a full parser, although it might work)

12#

13# Author:

14# Alberto Solino (@agsolino)

15#

16# Reference for:

17# Structure

18#

19# Excellent reference done by Joachim Metz

20# - http://forensic-proof.com/wp-content/uploads/2011/07/Extensible-Storage-Engine-ESE-Database-File-EDB-format.pdf

21#

22# ToDo:

23# [ ] Parse multi-values properly

24# [ ] Support long values properly

25#

26

27 from __future__ import division

28 from __future__ import print_function

29 from impacket import LOG

30 try:

31 from collections import OrderedDict

32 except:

33 try:

34 from ordereddict.ordereddict import OrderedDict

35 except:

36 from ordereddict import OrderedDict

37 from impacket.structure import Structure, hexdump

38 from struct import unpack

39 from binascii import hexlify

40 from six import b

41

42# Constants

43

44 FILE_TYPE_DATABASE = 0

45 FILE_TYPE_STREAMING_FILE = 1

46

47# Database state

48 JET_dbstateJustCreated = 1

49 JET_dbstateDirtyShutdown = 2

50 JET_dbstateCleanShutdown = 3

51 JET_dbstateBeingConverted = 4

52 JET_dbstateForceDetach = 5

53

54# Page Flags

55 FLAGS_ROOT = 1

56 FLAGS_LEAF = 2

57 FLAGS_PARENT = 4

58 FLAGS_EMPTY = 8

59 FLAGS_SPACE_TREE = 0x20

60 FLAGS_INDEX = 0x40

61 FLAGS_LONG_VALUE = 0x80

62 FLAGS_NEW_FORMAT = 0x2000

63 FLAGS_NEW_CHECKSUM = 0x2000

64

65# Tag Flags

66 TAG_UNKNOWN = 0x1

67 TAG_DEFUNCT = 0x2

68 TAG_COMMON = 0x4

69

70# Fixed Page Numbers

71 DATABASE_PAGE_NUMBER = 1

72 CATALOG_PAGE_NUMBER = 4

73 CATALOG_BACKUP_PAGE_NUMBER = 24

74

75# Fixed FatherDataPages

76 DATABASE_FDP = 1

77 CATALOG_FDP = 2

78 CATALOG_BACKUP_FDP = 3

79

80# Catalog Types

81 CATALOG_TYPE_TABLE = 1

82 CATALOG_TYPE_COLUMN = 2

83 CATALOG_TYPE_INDEX = 3

84 CATALOG_TYPE_LONG_VALUE = 4

85 CATALOG_TYPE_CALLBACK = 5

86

87# Column Types

88 JET_coltypNil = 0

89 JET_coltypBit = 1

90 JET_coltypUnsignedByte = 2

91 JET_coltypShort = 3

92 JET_coltypLong = 4

93 JET_coltypCurrency = 5

94 JET_coltypIEEESingle = 6

95 JET_coltypIEEEDouble = 7

96 JET_coltypDateTime = 8

97 JET_coltypBinary = 9

98 JET_coltypText = 10

99 JET_coltypLongBinary = 11

100 JET_coltypLongText = 12

101 JET_coltypSLV = 13

102 JET_coltypUnsignedLong = 14

103 JET_coltypLongLong = 15

104 JET_coltypGUID = 16

105 JET_coltypUnsignedShort= 17

106 JET_coltypMax = 18

107

108 ColumnTypeToName = {

109 JET_coltypNil : 'NULL',

110 JET_coltypBit : 'Boolean',

111 JET_coltypUnsignedByte : 'Signed byte',

112 JET_coltypShort : 'Signed short',

113 JET_coltypLong : 'Signed long',

114 JET_coltypCurrency : 'Currency',

115 JET_coltypIEEESingle : 'Single precision FP',

116 JET_coltypIEEEDouble : 'Double precision FP',

117 JET_coltypDateTime : 'DateTime',

118 JET_coltypBinary : 'Binary',

119 JET_coltypText : 'Text',

120 JET_coltypLongBinary : 'Long Binary',

121 JET_coltypLongText : 'Long Text',

122 JET_coltypSLV : 'Obsolete',

123 JET_coltypUnsignedLong : 'Unsigned long',

124 JET_coltypLongLong : 'Long long',

125 JET_coltypGUID : 'GUID',

126 JET_coltypUnsignedShort: 'Unsigned short',

127 JET_coltypMax : 'Max',

128}

129

130 ColumnTypeSize = {

131 JET_coltypNil : None,

132 JET_coltypBit : (1,'B'),

133 JET_coltypUnsignedByte : (1,'B'),

134 JET_coltypShort : (2,'<h'),

135 JET_coltypLong : (4,'<l'),

136 JET_coltypCurrency : (8,'<Q'),

137 JET_coltypIEEESingle : (4,'<f'),

138 JET_coltypIEEEDouble : (8,'<d'),

139 JET_coltypDateTime : (8,'<Q'),

140 JET_coltypBinary : None,

141 JET_coltypText : None,

142 JET_coltypLongBinary : None,

143 JET_coltypLongText : None,

144 JET_coltypSLV : None,

145 JET_coltypUnsignedLong : (4,'<L'),

146 JET_coltypLongLong : (8,'<Q'),

147 JET_coltypGUID : (16,'16s'),

148 JET_coltypUnsignedShort: (2,'<H'),

149 JET_coltypMax : None,

150}

151

152# Tagged Data Type Flags

153 TAGGED_DATA_TYPE_VARIABLE_SIZE = 1

154 TAGGED_DATA_TYPE_COMPRESSED = 2

155 TAGGED_DATA_TYPE_STORED = 4

156 TAGGED_DATA_TYPE_MULTI_VALUE = 8

157 TAGGED_DATA_TYPE_WHO_KNOWS = 10

158

159# Code pages

160 CODEPAGE_UNICODE = 1200

161 CODEPAGE_ASCII = 20127

162 CODEPAGE_WESTERN = 1252

163

164 StringCodePages = {

165 CODEPAGE_UNICODE : 'utf-16le',

166 CODEPAGE_ASCII : 'ascii',

167 CODEPAGE_WESTERN : 'cp1252',

168}

169

170# Structures

171

172 TABLE_CURSOR = {

173 'TableData' : b'',

174 'FatherDataPageNumber': 0,

175 'CurrentPageData' : b'',

176 'CurrentTag' : 0,

177}

178

179 class ESENT_JET_SIGNATURE(Structure):

180 structure = (

181 ('Random','<L=0'),

182 ('CreationTime','<Q=0'),

183 ('NetBiosName','16s=b""'),

184 )

185

186 class ESENT_DB_HEADER(Structure):

187 structure = (

188 ('CheckSum','<L=0'),

189 ('Signature','"\xef\xcd\xab\x89'),

190 ('Version','<L=0'),

191 ('FileType','<L=0'),

192 ('DBTime','<Q=0'),

193 ('DBSignature',':',ESENT_JET_SIGNATURE),

194 ('DBState','<L=0'),

195 ('ConsistentPosition','<Q=0'),

196 ('ConsistentTime','<Q=0'),

197 ('AttachTime','<Q=0'),

198 ('AttachPosition','<Q=0'),

199 ('DetachTime','<Q=0'),

200 ('DetachPosition','<Q=0'),

201 ('LogSignature',':',ESENT_JET_SIGNATURE),

202 ('Unknown','<L=0'),

203 ('PreviousBackup','24s=b""'),

204 ('PreviousIncBackup','24s=b""'),

205 ('CurrentFullBackup','24s=b""'),

206 ('ShadowingDisables','<L=0'),

207 ('LastObjectID','<L=0'),

208 ('WindowsMajorVersion','<L=0'),

209 ('WindowsMinorVersion','<L=0'),

210 ('WindowsBuildNumber','<L=0'),

211 ('WindowsServicePackNumber','<L=0'),

212 ('FileFormatRevision','<L=0'),

213 ('PageSize','<L=0'),

214 ('RepairCount','<L=0'),

215 ('RepairTime','<Q=0'),

216 ('Unknown2','28s=b""'),

217 ('ScrubTime','<Q=0'),

218 ('RequiredLog','<Q=0'),

219 ('UpgradeExchangeFormat','<L=0'),

220 ('UpgradeFreePages','<L=0'),

221 ('UpgradeSpaceMapPages','<L=0'),

222 ('CurrentShadowBackup','24s=b""'),

223 ('CreationFileFormatVersion','<L=0'),

224 ('CreationFileFormatRevision','<L=0'),

225 ('Unknown3','16s=b""'),

226 ('OldRepairCount','<L=0'),

227 ('ECCCount','<L=0'),

228 ('LastECCTime','<Q=0'),

229 ('OldECCFixSuccessCount','<L=0'),

230 ('ECCFixErrorCount','<L=0'),

231 ('LastECCFixErrorTime','<Q=0'),

232 ('OldECCFixErrorCount','<L=0'),

233 ('BadCheckSumErrorCount','<L=0'),

234 ('LastBadCheckSumTime','<Q=0'),

235 ('OldCheckSumErrorCount','<L=0'),

236 ('CommittedLog','<L=0'),

237 ('PreviousShadowCopy','24s=b""'),

238 ('PreviousDifferentialBackup','24s=b""'),

239 ('Unknown4','40s=b""'),

240 ('NLSMajorVersion','<L=0'),

241 ('NLSMinorVersion','<L=0'),

242 ('Unknown5','148s=b""'),

243 ('UnknownFlags','<L=0'),

244 )

245

246 class ESENT_PAGE_HEADER(Structure):

247 structure_2003_SP0 = (

248 ('CheckSum','<L=0'),

249 ('PageNumber','<L=0'),

250 )

251 structure_0x620_0x0b = (

252 ('CheckSum','<L=0'),

253 ('ECCCheckSum','<L=0'),

254 )

255 structure_win7 = (

256 ('CheckSum','<Q=0'),

257 )

258 common = (

259 ('LastModificationTime','<Q=0'),

260 ('PreviousPageNumber','<L=0'),

261 ('NextPageNumber','<L=0'),

262 ('FatherDataPage','<L=0'),

263 ('AvailableDataSize','<H=0'),

264 ('AvailableUncommittedDataSize','<H=0'),

265 ('FirstAvailableDataOffset','<H=0'),

266 ('FirstAvailablePageTag','<H=0'),

267 ('PageFlags','<L=0'),

268 )

269 extended_win7 = (

270 ('ExtendedCheckSum1','<Q=0'),

271 ('ExtendedCheckSum2','<Q=0'),

272 ('ExtendedCheckSum3','<Q=0'),

273 ('PageNumber','<Q=0'),

274 ('Unknown','<Q=0'),

275 )

276 def __init__(self, version, revision, pageSize=8192, data=None):

277 if (version < 0x620) or (version == 0x620 and revision < 0x0b):

278 # For sure the old format

279 self.structure = self.structure_2003_SP0 + self.common

280 elif version == 0x620 and revision < 0x11:

281 # Exchange 2003 SP1 and Windows Vista and later

282 self.structure = self.structure_0x620_0x0b + self.common

283 else:

284 # Windows 7 and later

285 self.structure = self.structure_win7 + self.common

286 if pageSize > 8192:

287 self.structure += self.extended_win7

288

289 Structure.__init__(self,data)

290

291 class ESENT_ROOT_HEADER(Structure):

292 structure = (

293 ('InitialNumberOfPages','<L=0'),

294 ('ParentFatherDataPage','<L=0'),

295 ('ExtentSpace','<L=0'),

296 ('SpaceTreePageNumber','<L=0'),

297 )

298

299 class ESENT_BRANCH_HEADER(Structure):

300 structure = (

301 ('CommonPageKey',':'),

302 )

303

304 class ESENT_BRANCH_ENTRY(Structure):

305 common = (

306 ('CommonPageKeySize','<H=0'),

307 )

308 structure = (

309 ('LocalPageKeySize','<H=0'),

310 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'),

311 ('LocalPageKey',':'),

312 ('ChildPageNumber','<L=0'),

313 )

314 def __init__(self, flags, data=None):

315 if flags & TAG_COMMON > 0:

316 # Include the common header

317 self.structure = self.common + self.structure

318 Structure.__init__(self,data)

319

320 class ESENT_LEAF_HEADER(Structure):

321 structure = (

322 ('CommonPageKey',':'),

323 )

324

325 class ESENT_LEAF_ENTRY(Structure):

326 common = (

327 ('CommonPageKeySize','<H=0'),

328 )

329 structure = (

330 ('LocalPageKeySize','<H=0'),

331 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'),

332 ('LocalPageKey',':'),

333 ('EntryData',':'),

334 )

335 def __init__(self, flags, data=None):

336 if flags & TAG_COMMON > 0:

337 # Include the common header

338 self.structure = self.common + self.structure

339 Structure.__init__(self,data)

340

341 class ESENT_SPACE_TREE_HEADER(Structure):

342 structure = (

343 ('Unknown','<Q=0'),

344 )

345

346 class ESENT_SPACE_TREE_ENTRY(Structure):

347 structure = (

348 ('PageKeySize','<H=0'),

349 ('LastPageNumber','<L=0'),

350 ('NumberOfPages','<L=0'),

351 )

352

353 class ESENT_INDEX_ENTRY(Structure):

354 structure = (

355 ('RecordPageKey',':'),

356 )

357

358 class ESENT_DATA_DEFINITION_HEADER(Structure):

359 structure = (

360 ('LastFixedSize','<B=0'),

361 ('LastVariableDataType','<B=0'),

362 ('VariableSizeOffset','<H=0'),

363 )

364

365 class ESENT_CATALOG_DATA_DEFINITION_ENTRY(Structure):

366 fixed = (

367 ('FatherDataPageID','<L=0'),

368 ('Type','<H=0'),

369 ('Identifier','<L=0'),

370 )

371

372 column_stuff = (

373 ('ColumnType','<L=0'),

374 ('SpaceUsage','<L=0'),

375 ('ColumnFlags','<L=0'),

376 ('CodePage','<L=0'),

377 )

378

379 other = (

380 ('FatherDataPageNumber','<L=0'),

381 )

382

383 table_stuff = (

384 ('SpaceUsage','<L=0'),

385# ('TableFlags','<L=0'),

386# ('InitialNumberOfPages','<L=0'),

387 )

388

389 index_stuff = (

390 ('SpaceUsage','<L=0'),

391 ('IndexFlags','<L=0'),

392 ('Locale','<L=0'),

393 )

394

395 lv_stuff = (

396 ('SpaceUsage','<L=0'),

397# ('LVFlags','<L=0'),

398# ('InitialNumberOfPages','<L=0'),

399 )

400 common = (

401# ('RootFlag','<B=0'),

402# ('RecordOffset','<H=0'),

403# ('LCMapFlags','<L=0'),

404# ('KeyMost','<H=0'),

405 ('Trailing',':'),

406 )

407

408 def __init__(self,data):

409 # Depending on the type of data we'll end up building a different struct

410 dataType = unpack('<H', data[4:][:2])[0]

411 self.structure = self.fixed

412

413 if dataType == CATALOG_TYPE_TABLE:

414 self.structure += self.other + self.table_stuff

415 elif dataType == CATALOG_TYPE_COLUMN:

416 self.structure += self.column_stuff

417 elif dataType == CATALOG_TYPE_INDEX:

418 self.structure += self.other + self.index_stuff

419 elif dataType == CATALOG_TYPE_LONG_VALUE:

420 self.structure += self.other + self.lv_stuff

421 elif dataType == CATALOG_TYPE_CALLBACK:

422 raise Exception('CallBack types not supported!')

423 else:

424 LOG.error('Unknown catalog type 0x%x' % dataType)

425 self.structure = ()

426 Structure.__init__(self,data)

427

428 self.structure += self.common

429

430 Structure.__init__(self,data)

431

432

433#def pretty_print(x):

434# if x in '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ':

435# return x

436# else:

437# return '.'

438#

439#def hexdump(data):

440# x=str(data)

441# strLen = len(x)

442# i = 0

443# while i < strLen:

444# print "%04x " % i,

445# for j in range(16):

446# if i+j < strLen:

447# print "%02X" % ord(x[i+j]),

448#

449# else:

450# print " ",

451# if j%16 == 7:

452# print "",

453# print " ",

454# print ''.join(pretty_print(x) for x in x[i:i+16] )

455# i += 16

456

457 def getUnixTime(t):

458 t -= 116444736000000000

459 t //= 10000000

460 return t

461

462 class ESENT_PAGE:

463 def __init__(self, db, data=None):

464 self.__DBHeader = db

465 self.data = data

466 self.record = None

467 if data is not None:

468 self.record = ESENT_PAGE_HEADER(self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'], self.__DBHeader['PageSize'], data)

469

470 def printFlags(self):

471 flags = self.record['PageFlags']

472 if flags & FLAGS_EMPTY:

473 print("\tEmpty")

474 if flags & FLAGS_INDEX:

475 print("\tIndex")

476 if flags & FLAGS_LEAF:

477 print("\tLeaf")

478 else:

479 print("\tBranch")

480 if flags & FLAGS_LONG_VALUE:

481 print("\tLong Value")

482 if flags & FLAGS_NEW_CHECKSUM:

483 print("\tNew Checksum")

484 if flags & FLAGS_NEW_FORMAT:

485 print("\tNew Format")

486 if flags & FLAGS_PARENT:

487 print("\tParent")

488 if flags & FLAGS_ROOT:

489 print("\tRoot")

490 if flags & FLAGS_SPACE_TREE:

491 print("\tSpace Tree")

492

493 def dump(self):

494 baseOffset = len(self.record)

495 self.record.dump()

496 tags = self.data[-4*self.record['FirstAvailablePageTag']:]

497

498 print("FLAGS: ")

499 self.printFlags()

500

501 print()

502

503 for i in range(self.record['FirstAvailablePageTag']):

504 tag = tags[-4:]

505 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] > 11 and self.__DBHeader['PageSize'] > 8192:

506 valueSize = unpack('<H', tag[:2])[0] & 0x7fff

507 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff

508 hexdump((self.data[baseOffset+valueOffset:][:6]))

509 pageFlags = ord(self.data[baseOffset+valueOffset:][1]) >> 5

510 #print "TAG FLAG: 0x%x " % (unpack('<L', self.data[baseOffset+valueOffset:][:4]) ) >> 5

511 #print "TAG FLAG: 0x " , ord(self.data[baseOffset+valueOffset:][0])

512 else:

513 valueSize = unpack('<H', tag[:2])[0] & 0x1fff

514 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13

515 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff

516

517 print("TAG %-8d offset:0x%-6x flags:0x%-4x valueSize:0x%x" % (i,valueOffset,pageFlags,valueSize))

518 #hexdump(self.getTag(i)[1])

519 tags = tags[:-4]

520

521 if self.record['PageFlags'] & FLAGS_ROOT > 0:

522 rootHeader = ESENT_ROOT_HEADER(self.getTag(0)[1])

523 rootHeader.dump()

524 elif self.record['PageFlags'] & FLAGS_LEAF == 0:

525 # Branch Header

526 flags, data = self.getTag(0)

527 branchHeader = ESENT_BRANCH_HEADER(data)

528 branchHeader.dump()

529 else:

530 # Leaf Header

531 flags, data = self.getTag(0)

532 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0:

533 # Space Tree

534 spaceTreeHeader = ESENT_SPACE_TREE_HEADER(data)

535 spaceTreeHeader.dump()

536 else:

537 leafHeader = ESENT_LEAF_HEADER(data)

538 leafHeader.dump()

539

540 # Print the leaf/branch tags

541 for tagNum in range(1,self.record['FirstAvailablePageTag']):

542 flags, data = self.getTag(tagNum)

543 if self.record['PageFlags'] & FLAGS_LEAF == 0:

544 # Branch page

545 branchEntry = ESENT_BRANCH_ENTRY(flags, data)

546 branchEntry.dump()

547 elif self.record['PageFlags'] & FLAGS_LEAF > 0:

548 # Leaf page

549 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0:

550 # Space Tree

551 spaceTreeEntry = ESENT_SPACE_TREE_ENTRY(data)

552 #spaceTreeEntry.dump()

553

554 elif self.record['PageFlags'] & FLAGS_INDEX > 0:

555 # Index Entry

556 indexEntry = ESENT_INDEX_ENTRY(data)

557 #indexEntry.dump()

558 elif self.record['PageFlags'] & FLAGS_LONG_VALUE > 0:

559 # Long Page Value

560 raise Exception('Long value still not supported')

561 else:

562 # Table Value

563 leafEntry = ESENT_LEAF_ENTRY(flags, data)

564 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(leafEntry['EntryData'])

565 dataDefinitionHeader.dump()

566 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(leafEntry['EntryData'][len(dataDefinitionHeader):])

567 catalogEntry.dump()

568 hexdump(leafEntry['EntryData'])

569

570 def getTag(self, tagNum):

571 if self.record['FirstAvailablePageTag'] < tagNum:

572 raise Exception('Trying to grab an unknown tag 0x%x' % tagNum)

573

574 tags = self.data[-4*self.record['FirstAvailablePageTag']:]

575 baseOffset = len(self.record)

576 for i in range(tagNum):

577 tags = tags[:-4]

578

579 tag = tags[-4:]

580

581 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192:

582 valueSize = unpack('<H', tag[:2])[0] & 0x7fff

583 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff

584 tmpData = list(self.data[baseOffset+valueOffset:][:valueSize])

585 pageFlags = ord(tmpData[1]) >> 5

586 tmpData[1] = chr(ord(tmpData[1:2]) & 0x1f)

587 tagData = "".join(tmpData)

588 else:

589 valueSize = unpack('<H', tag[:2])[0] & 0x1fff

590 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13

591 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff

592 tagData = self.data[baseOffset+valueOffset:][:valueSize]

593

594 #return pageFlags, self.data[baseOffset+valueOffset:][:valueSize]

595 return pageFlags, tagData

596

597 class ESENT_DB:

598 def __init__(self, fileName, pageSize = 8192, isRemote = False):

599 self.__fileName = fileName

600 self.__pageSize = pageSize

601 self.__DB = None

602 self.__DBHeader = None

603 self.__totalPages = None

604 self.__tables = OrderedDict()

605 self.__currentTable = None

606 self.__isRemote = isRemote

607 self.mountDB()

608

609 def mountDB(self):

610 LOG.debug("Mounting DB...")

611 if self.__isRemote is True:

612 self.__DB = self.__fileName

613 self.__DB.open()

614 else:

615 self.__DB = open(self.__fileName,"rb")

616 mainHeader = self.getPage(-1)

617 self.__DBHeader = ESENT_DB_HEADER(mainHeader)

618 self.__pageSize = self.__DBHeader['PageSize']

619 self.__DB.seek(0,2)

620 self.__totalPages = (self.__DB.tell() // self.__pageSize) -2

621 LOG.debug("Database Version:0x%x, Revision:0x%x"% (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision']))

622 LOG.debug("Page Size: %d" % self.__pageSize)

623 LOG.debug("Total Pages in file: %d" % self.__totalPages)

624 self.parseCatalog(CATALOG_PAGE_NUMBER)

625

626 def printCatalog(self):

627 indent = ' '

628

629 print("Database version: 0x%x, 0x%x" % (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'] ))

630 print("Page size: %d " % self.__pageSize)

631 print("Number of pages: %d" % self.__totalPages)

632 print()

633 print("Catalog for %s" % self.__fileName)

634 for table in list(self.__tables.keys()):

635 print("[%s]" % table.decode('utf8'))

636 print("%sColumns " % indent)

637 for column in list(self.__tables[table]['Columns'].keys()):

638 record = self.__tables[table]['Columns'][column]['Record']

639 print("%s%-5d%-30s%s" % (indent*2, record['Identifier'], column.decode('utf-8'),ColumnTypeToName[record['ColumnType']]))

640 print("%sIndexes"% indent)

641 for index in list(self.__tables[table]['Indexes'].keys()):

642 print("%s%s" % (indent*2, index.decode('utf-8')))

643 print("")

644

645 def __addItem(self, entry):

646 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])

647 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):])

648 itemName = self.__parseItemName(entry)

649

650 if catalogEntry['Type'] == CATALOG_TYPE_TABLE:

651 self.__tables[itemName] = OrderedDict()

652 self.__tables[itemName]['TableEntry'] = entry

653 self.__tables[itemName]['Columns'] = OrderedDict()

654 self.__tables[itemName]['Indexes'] = OrderedDict()

655 self.__tables[itemName]['LongValues'] = OrderedDict()

656 self.__currentTable = itemName

657 elif catalogEntry['Type'] == CATALOG_TYPE_COLUMN:

658 self.__tables[self.__currentTable]['Columns'][itemName] = entry

659 self.__tables[self.__currentTable]['Columns'][itemName]['Header'] = dataDefinitionHeader

660 self.__tables[self.__currentTable]['Columns'][itemName]['Record'] = catalogEntry

661 elif catalogEntry['Type'] == CATALOG_TYPE_INDEX:

662 self.__tables[self.__currentTable]['Indexes'][itemName] = entry

663 elif catalogEntry['Type'] == CATALOG_TYPE_LONG_VALUE:

664 self.__addLongValue(entry)

665 else:

666 raise Exception('Unknown type 0x%x' % catalogEntry['Type'])

667

668 def __parseItemName(self,entry):

669 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])

670

671 if dataDefinitionHeader['LastVariableDataType'] > 127:

672 numEntries = dataDefinitionHeader['LastVariableDataType'] - 127

673 else:

674 numEntries = dataDefinitionHeader['LastVariableDataType']

675

676 itemLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0]

677 itemName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][2*numEntries:][:itemLen]

678 return itemName

679

680 def __addLongValue(self, entry):

681 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])

682 lvLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0]

683 lvName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][7:][:lvLen]

684 self.__tables[self.__currentTable]['LongValues'][lvName] = entry

685

686 def parsePage(self, page):

687 # Print the leaf/branch tags

688 for tagNum in range(1,page.record['FirstAvailablePageTag']):

689 flags, data = page.getTag(tagNum)

690 if page.record['PageFlags'] & FLAGS_LEAF > 0:

691 # Leaf page

692 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0:

693 pass

694 elif page.record['PageFlags'] & FLAGS_INDEX > 0:

695 pass

696 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0:

697 pass

698 else:

699 # Table Value

700 leafEntry = ESENT_LEAF_ENTRY(flags, data)

701 self.__addItem(leafEntry)

702

703 def parseCatalog(self, pageNum):

704 # Parse all the pages starting at pageNum and commit table data

705 page = self.getPage(pageNum)

706 self.parsePage(page)

707

708 for i in range(1, page.record['FirstAvailablePageTag']):

709 flags, data = page.getTag(i)

710 if page.record['PageFlags'] & FLAGS_LEAF == 0:

711 # Branch page

712 branchEntry = ESENT_BRANCH_ENTRY(flags, data)

713 self.parseCatalog(branchEntry['ChildPageNumber'])

714

715

716 def readHeader(self):

717 LOG.debug("Reading Boot Sector for %s" % self.__volumeName)

718

719 def getPage(self, pageNum):

720 LOG.debug("Trying to fetch page %d (0x%x)" % (pageNum, (pageNum+1)*self.__pageSize))

721 self.__DB.seek((pageNum+1)*self.__pageSize, 0)

722 data = self.__DB.read(self.__pageSize)

723 while len(data) < self.__pageSize:

724 remaining = self.__pageSize - len(data)

725 data += self.__DB.read(remaining)

726 # Special case for the first page

727 if pageNum <= 0:

728 return data

729 else:

730 return ESENT_PAGE(self.__DBHeader, data)

731

732 def close(self):

733 self.__DB.close()

734

735 def openTable(self, tableName):

736 # Returns a cursos for later use

737

738 if isinstance(tableName, bytes) is not True:

739 tableName = b(tableName)

740

741 if tableName in self.__tables:

742 entry = self.__tables[tableName]['TableEntry']

743 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])

744 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):])

745

746 # Let's position the cursor at the leaf levels for fast reading

747 pageNum = catalogEntry['FatherDataPageNumber']

748 done = False

749 while done is False:

750 page = self.getPage(pageNum)

751 if page.record['FirstAvailablePageTag'] <= 1:

752 # There are no records

753 done = True

754 for i in range(1, page.record['FirstAvailablePageTag']):

755 flags, data = page.getTag(i)

756 if page.record['PageFlags'] & FLAGS_LEAF == 0:

757 # Branch page, move on to the next page

758 branchEntry = ESENT_BRANCH_ENTRY(flags, data)

759 pageNum = branchEntry['ChildPageNumber']

760 break

761 else:

762 done = True

763 break

764

765 cursor = TABLE_CURSOR

766 cursor['TableData'] = self.__tables[tableName]

767 cursor['FatherDataPageNumber'] = catalogEntry['FatherDataPageNumber']

768 cursor['CurrentPageData'] = page

769 cursor['CurrentTag'] = 0

770 return cursor

771 else:

772 return None

773

774 def __getNextTag(self, cursor):

775 page = cursor['CurrentPageData']

776

777 if cursor['CurrentTag'] >= page.record['FirstAvailablePageTag']:

778 # No more data in this page, chau

779 return None

780

781 flags, data = page.getTag(cursor['CurrentTag'])

782 if page.record['PageFlags'] & FLAGS_LEAF > 0:

783 # Leaf page

784 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0:

785 raise Exception('FLAGS_SPACE_TREE > 0')

786 elif page.record['PageFlags'] & FLAGS_INDEX > 0:

787 raise Exception('FLAGS_INDEX > 0')

788 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0:

789 raise Exception('FLAGS_LONG_VALUE > 0')

790 else:

791 # Table Value

792 leafEntry = ESENT_LEAF_ENTRY(flags, data)

793 return leafEntry

794

795 return None

796

797 def getNextRow(self, cursor, filter_tables = None):

798 cursor['CurrentTag'] += 1

799

800 tag = self.__getNextTag(cursor)

801 #hexdump(tag)

802

803 if tag is None:

804 # No more tags in this page, search for the next one on the right

805 page = cursor['CurrentPageData']

806 if page.record['NextPageNumber'] == 0:

807 # No more pages, chau

808 return None

809 else:

810 cursor['CurrentPageData'] = self.getPage(page.record['NextPageNumber'])

811 cursor['CurrentTag'] = 0

812 return self.getNextRow(cursor, filter_tables = filter_tables)

813 else:

814 return self.__tagToRecord(cursor, tag['EntryData'], filter_tables = filter_tables)

815

816 def __tagToRecord(self, cursor, tag, filter_tables = None):

817 # So my brain doesn't forget, the data record is composed of:

818 # Header

819 # Fixed Size Data (ID < 127)

820 # The easiest to parse. Their size is fixed in the record. You can get its size

821 # from the Column Record, field SpaceUsage

822 # Variable Size Data (127 < ID < 255)

823 # At VariableSizeOffset you get an array of two bytes per variable entry, pointing

824 # to the length of the value. Values start at:

825 # numEntries = LastVariableDataType - 127

826 # VariableSizeOffset + numEntries * 2 (bytes)

827 # Tagged Data ( > 255 )

828 # After the Variable Size Value, there's more data for the tagged values.

829 # Right at the beginning there's another array (taggedItems), pointing to the

830 # values, size.

831 #

832 # The interesting thing about this DB records is there's no need for all the columns to be there, hence

833 # saving space. That's why I got over all the columns, and if I find data (of any type), i assign it. If

834 # not, the column's empty.

835 #

836 # There are a lot of caveats in the code, so take your time to explore it.

837 #

838 # ToDo: Better complete this description

839 #

840

841 record = OrderedDict()

842 taggedItems = OrderedDict()

843 taggedItemsParsed = False

844

845 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(tag)

846 #dataDefinitionHeader.dump()

847 variableDataBytesProcessed = (dataDefinitionHeader['LastVariableDataType'] - 127) * 2

848 prevItemLen = 0

849 tagLen = len(tag)

850 fixedSizeOffset = len(dataDefinitionHeader)

851 variableSizeOffset = dataDefinitionHeader['VariableSizeOffset']

852

853 columns = cursor['TableData']['Columns']

854

855 for column in list(columns.keys()):

856 if filter_tables is not None:

857 if column not in filter_tables:

858 continue

859 columnRecord = columns[column]['Record']

860 #columnRecord.dump()

861 if columnRecord['Identifier'] <= dataDefinitionHeader['LastFixedSize']:

862 # Fixed Size column data type, still available data

863 record[column] = tag[fixedSizeOffset:][:columnRecord['SpaceUsage']]

864 fixedSizeOffset += columnRecord['SpaceUsage']

865

866 elif 127 < columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']:

867 # Variable data type

868 index = columnRecord['Identifier'] - 127 - 1

869 itemLen = unpack('<H',tag[variableSizeOffset+index*2:][:2])[0]

870

871 if itemLen & 0x8000:

872 # Empty item

873 itemLen = prevItemLen

874 record[column] = None

875 else:

876 itemValue = tag[variableSizeOffset+variableDataBytesProcessed:][:itemLen-prevItemLen]

877 record[column] = itemValue

878

879 #if columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']:

880 variableDataBytesProcessed +=itemLen-prevItemLen

881

882 prevItemLen = itemLen

883

884 elif columnRecord['Identifier'] > 255:

885 # Have we parsed the tagged items already?

886 if taggedItemsParsed is False and (variableDataBytesProcessed+variableSizeOffset) < tagLen:

887 index = variableDataBytesProcessed+variableSizeOffset

888 #hexdump(tag[index:])

889 endOfVS = self.__pageSize

890 firstOffsetTag = (unpack('<H', tag[index+2:][:2])[0] & 0x3fff) + variableDataBytesProcessed+variableSizeOffset

891 while True:

892 taggedIdentifier = unpack('<H', tag[index:][:2])[0]

893 index += 2

894 taggedOffset = (unpack('<H', tag[index:][:2])[0] & 0x3fff)

895 # As of Windows 7 and later ( version 0x620 revision 0x11) the

896 # tagged data type flags are always present

897 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192:

898 flagsPresent = 1

899 else:

900 flagsPresent = (unpack('<H', tag[index:][:2])[0] & 0x4000)

901 index += 2

902 if taggedOffset < endOfVS:

903 endOfVS = taggedOffset

904 taggedItems[taggedIdentifier] = (taggedOffset, tagLen, flagsPresent)

905 #print "ID: %d, Offset:%d, firstOffset:%d, index:%d, flag: 0x%x" % (taggedIdentifier, taggedOffset,firstOffsetTag,index, flagsPresent)

906 if index >= firstOffsetTag:

907 # We reached the end of the variable size array

908 break

909

910 # Calculate length of variable items

911 # Ugly.. should be redone

912 prevKey = list(taggedItems.keys())[0]

913 for i in range(1,len(taggedItems)):

914 offset0, length, flags = taggedItems[prevKey]

915 offset, _, _ = list(taggedItems.items())[i][1]

916 taggedItems[prevKey] = (offset0, offset-offset0, flags)

917 #print "ID: %d, Offset: %d, Len: %d, flags: %d" % (prevKey, offset0, offset-offset0, flags)

918 prevKey = list(taggedItems.keys())[i]

919 taggedItemsParsed = True

920

921 # Tagged data type

922 if columnRecord['Identifier'] in taggedItems:

923 offsetItem = variableDataBytesProcessed + variableSizeOffset + taggedItems[columnRecord['Identifier']][0]

924 itemSize = taggedItems[columnRecord['Identifier']][1]

925 # If item have flags, we should skip them

926 if taggedItems[columnRecord['Identifier']][2] > 0:

927 itemFlag = ord(tag[offsetItem: offsetItem+1])

928 offsetItem += 1

929 itemSize -= 1

930 else:

931 itemFlag = 0

932

933 #print "ID: %d, itemFlag: 0x%x" %( columnRecord['Identifier'], itemFlag)

934 if itemFlag & (TAGGED_DATA_TYPE_COMPRESSED ):

935 LOG.error('Unsupported tag column: %s, flag:0x%x' % (column, itemFlag))

936 record[column] = None

937 elif itemFlag & TAGGED_DATA_TYPE_MULTI_VALUE:

938 # ToDo: Parse multi-values properly

939 LOG.debug('Multivalue detected in column %s, returning raw results' % (column))

940 record[column] = (hexlify(tag[offsetItem:][:itemSize]),)

941 else:

942 record[column] = tag[offsetItem:][:itemSize]

943

944 else:

945 record[column] = None

946 else:

947 record[column] = None

948

949 # If we understand the data type, we unpack it and cast it accordingly

950 # otherwise, we just encode it in hex

951 if type(record[column]) is tuple:

952 # A multi value data, we won't decode it, just leave it this way

953 record[column] = record[column][0]

954 elif columnRecord['ColumnType'] == JET_coltypText or columnRecord['ColumnType'] == JET_coltypLongText:

955 # Let's handle strings

956 if record[column] is not None:

957 if columnRecord['CodePage'] not in StringCodePages:

958 raise Exception('Unknown codepage 0x%x'% columnRecord['CodePage'])

959 stringDecoder = StringCodePages[columnRecord['CodePage']]

960

961 try:

962 record[column] = record[column].decode(stringDecoder)

963 except Exception:

964 LOG.debug("Exception:", exc_info=True)

965 LOG.debug('Fixing Record[%r][%d]: %r' % (column, columnRecord['ColumnType'], record[column]))

966 record[column] = record[column].decode(stringDecoder, "replace")

967 pass

968 else:

969 unpackData = ColumnTypeSize[columnRecord['ColumnType']]

970 if record[column] is not None:

971 if unpackData is None:

972 record[column] = hexlify(record[column])

973 else:

974 unpackStr = unpackData[1]

975 record[column] = unpack(unpackStr, record[column])[0]

976

977 return record