Coverage for /root/GitHubProjects/impacket/impacket/ese.py: 68%

# Coverage for /root/GitHubProjects/impacket/impacket/ese.py : 68%

## 502 statements

Hot-keys on this page

r m x p toggle line displays

j k next/prev highlighted chunk

0 (zero) top of page

1 (one) first highlighted chunk

1# SECUREAUTH LABS. Copyright 2018 SecureAuth Corporation. All rights reserved.

2#

3# This software is provided under under a slightly modified version

4# of the Apache Software License. See the accompanying LICENSE file

5# for more information.

6#

7# Description:

8# Microsoft Extensive Storage Engine parser, just focused on trying

9# to parse NTDS.dit files (not meant as a full parser, although it might work)

10#

11# Author:

12# Alberto Solino (@agsolino)

13#

14# Reference for:

15# Structure.

16#

17# Excellent reference done by Joachim Metz

18# http://forensic-proof.com/wp-content/uploads/2011/07/Extensible-Storage-Engine-ESE-Database-File-EDB-format.pdf

19#

20# ToDo:

21# [ ] Parse multi-values properly

22# [ ] Support long values properly

23 from __future__ import division

24 from __future__ import print_function

25 from impacket import LOG

26 try:

27 from collections import OrderedDict

28 except:

29 try:

30 from ordereddict.ordereddict import OrderedDict

31 except:

32 from ordereddict import OrderedDict

33 from impacket.structure import Structure, hexdump

34 from struct import unpack

35 from binascii import hexlify

36 from six import b

37

38# Constants

39

40 FILE_TYPE_DATABASE = 0

41 FILE_TYPE_STREAMING_FILE = 1

42

43# Database state

44 JET_dbstateJustCreated = 1

45 JET_dbstateDirtyShutdown = 2

46 JET_dbstateCleanShutdown = 3

47 JET_dbstateBeingConverted = 4

48 JET_dbstateForceDetach = 5

49

50# Page Flags

51 FLAGS_ROOT = 1

52 FLAGS_LEAF = 2

53 FLAGS_PARENT = 4

54 FLAGS_EMPTY = 8

55 FLAGS_SPACE_TREE = 0x20

56 FLAGS_INDEX = 0x40

57 FLAGS_LONG_VALUE = 0x80

58 FLAGS_NEW_FORMAT = 0x2000

59 FLAGS_NEW_CHECKSUM = 0x2000

60

61# Tag Flags

62 TAG_UNKNOWN = 0x1

63 TAG_DEFUNCT = 0x2

64 TAG_COMMON = 0x4

65

66# Fixed Page Numbers

67 DATABASE_PAGE_NUMBER = 1

68 CATALOG_PAGE_NUMBER = 4

69 CATALOG_BACKUP_PAGE_NUMBER = 24

70

71# Fixed FatherDataPages

72 DATABASE_FDP = 1

73 CATALOG_FDP = 2

74 CATALOG_BACKUP_FDP = 3

75

76# Catalog Types

77 CATALOG_TYPE_TABLE = 1

78 CATALOG_TYPE_COLUMN = 2

79 CATALOG_TYPE_INDEX = 3

80 CATALOG_TYPE_LONG_VALUE = 4

81 CATALOG_TYPE_CALLBACK = 5

82

83# Column Types

84 JET_coltypNil = 0

85 JET_coltypBit = 1

86 JET_coltypUnsignedByte = 2

87 JET_coltypShort = 3

88 JET_coltypLong = 4

89 JET_coltypCurrency = 5

90 JET_coltypIEEESingle = 6

91 JET_coltypIEEEDouble = 7

92 JET_coltypDateTime = 8

93 JET_coltypBinary = 9

94 JET_coltypText = 10

95 JET_coltypLongBinary = 11

96 JET_coltypLongText = 12

97 JET_coltypSLV = 13

98 JET_coltypUnsignedLong = 14

99 JET_coltypLongLong = 15

100 JET_coltypGUID = 16

101 JET_coltypUnsignedShort= 17

102 JET_coltypMax = 18

103

104 ColumnTypeToName = {

105 JET_coltypNil : 'NULL',

106 JET_coltypBit : 'Boolean',

107 JET_coltypUnsignedByte : 'Signed byte',

108 JET_coltypShort : 'Signed short',

109 JET_coltypLong : 'Signed long',

110 JET_coltypCurrency : 'Currency',

111 JET_coltypIEEESingle : 'Single precision FP',

112 JET_coltypIEEEDouble : 'Double precision FP',

113 JET_coltypDateTime : 'DateTime',

114 JET_coltypBinary : 'Binary',

115 JET_coltypText : 'Text',

116 JET_coltypLongBinary : 'Long Binary',

117 JET_coltypLongText : 'Long Text',

118 JET_coltypSLV : 'Obsolete',

119 JET_coltypUnsignedLong : 'Unsigned long',

120 JET_coltypLongLong : 'Long long',

121 JET_coltypGUID : 'GUID',

122 JET_coltypUnsignedShort: 'Unsigned short',

123 JET_coltypMax : 'Max',

124}

125

126 ColumnTypeSize = {

127 JET_coltypNil : None,

128 JET_coltypBit : (1,'B'),

129 JET_coltypUnsignedByte : (1,'B'),

130 JET_coltypShort : (2,'<h'),

131 JET_coltypLong : (4,'<l'),

132 JET_coltypCurrency : (8,'<Q'),

133 JET_coltypIEEESingle : (4,'<f'),

134 JET_coltypIEEEDouble : (8,'<d'),

135 JET_coltypDateTime : (8,'<Q'),

136 JET_coltypBinary : None,

137 JET_coltypText : None,

138 JET_coltypLongBinary : None,

139 JET_coltypLongText : None,

140 JET_coltypSLV : None,

141 JET_coltypUnsignedLong : (4,'<L'),

142 JET_coltypLongLong : (8,'<Q'),

143 JET_coltypGUID : (16,'16s'),

144 JET_coltypUnsignedShort: (2,'<H'),

145 JET_coltypMax : None,

146}

147

148# Tagged Data Type Flags

149 TAGGED_DATA_TYPE_VARIABLE_SIZE = 1

150 TAGGED_DATA_TYPE_COMPRESSED = 2

151 TAGGED_DATA_TYPE_STORED = 4

152 TAGGED_DATA_TYPE_MULTI_VALUE = 8

153 TAGGED_DATA_TYPE_WHO_KNOWS = 10

154

155# Code pages

156 CODEPAGE_UNICODE = 1200

157 CODEPAGE_ASCII = 20127

158 CODEPAGE_WESTERN = 1252

159

160 StringCodePages = {

161 CODEPAGE_UNICODE : 'utf-16le',

162 CODEPAGE_ASCII : 'ascii',

163 CODEPAGE_WESTERN : 'cp1252',

164}

165

166# Structures

167

168 TABLE_CURSOR = {

169 'TableData' : b'',

170 'FatherDataPageNumber': 0,

171 'CurrentPageData' : b'',

172 'CurrentTag' : 0,

173}

174

175 class ESENT_JET_SIGNATURE(Structure):

176 structure = (

177 ('Random','<L=0'),

178 ('CreationTime','<Q=0'),

179 ('NetBiosName','16s=b""'),

180 )

181

182 class ESENT_DB_HEADER(Structure):

183 structure = (

184 ('CheckSum','<L=0'),

185 ('Signature','"\xef\xcd\xab\x89'),

186 ('Version','<L=0'),

187 ('FileType','<L=0'),

188 ('DBTime','<Q=0'),

189 ('DBSignature',':',ESENT_JET_SIGNATURE),

190 ('DBState','<L=0'),

191 ('ConsistentPosition','<Q=0'),

192 ('ConsistentTime','<Q=0'),

193 ('AttachTime','<Q=0'),

194 ('AttachPosition','<Q=0'),

195 ('DetachTime','<Q=0'),

196 ('DetachPosition','<Q=0'),

197 ('LogSignature',':',ESENT_JET_SIGNATURE),

198 ('Unknown','<L=0'),

199 ('PreviousBackup','24s=b""'),

200 ('PreviousIncBackup','24s=b""'),

201 ('CurrentFullBackup','24s=b""'),

202 ('ShadowingDisables','<L=0'),

203 ('LastObjectID','<L=0'),

204 ('WindowsMajorVersion','<L=0'),

205 ('WindowsMinorVersion','<L=0'),

206 ('WindowsBuildNumber','<L=0'),

207 ('WindowsServicePackNumber','<L=0'),

208 ('FileFormatRevision','<L=0'),

209 ('PageSize','<L=0'),

210 ('RepairCount','<L=0'),

211 ('RepairTime','<Q=0'),

212 ('Unknown2','28s=b""'),

213 ('ScrubTime','<Q=0'),

214 ('RequiredLog','<Q=0'),

215 ('UpgradeExchangeFormat','<L=0'),

216 ('UpgradeFreePages','<L=0'),

217 ('UpgradeSpaceMapPages','<L=0'),

218 ('CurrentShadowBackup','24s=b""'),

219 ('CreationFileFormatVersion','<L=0'),

220 ('CreationFileFormatRevision','<L=0'),

221 ('Unknown3','16s=b""'),

222 ('OldRepairCount','<L=0'),

223 ('ECCCount','<L=0'),

224 ('LastECCTime','<Q=0'),

225 ('OldECCFixSuccessCount','<L=0'),

226 ('ECCFixErrorCount','<L=0'),

227 ('LastECCFixErrorTime','<Q=0'),

228 ('OldECCFixErrorCount','<L=0'),

229 ('BadCheckSumErrorCount','<L=0'),

230 ('LastBadCheckSumTime','<Q=0'),

231 ('OldCheckSumErrorCount','<L=0'),

232 ('CommittedLog','<L=0'),

233 ('PreviousShadowCopy','24s=b""'),

234 ('PreviousDifferentialBackup','24s=b""'),

235 ('Unknown4','40s=b""'),

236 ('NLSMajorVersion','<L=0'),

237 ('NLSMinorVersion','<L=0'),

238 ('Unknown5','148s=b""'),

239 ('UnknownFlags','<L=0'),

240 )

241

242 class ESENT_PAGE_HEADER(Structure):

243 structure_2003_SP0 = (

244 ('CheckSum','<L=0'),

245 ('PageNumber','<L=0'),

246 )

247 structure_0x620_0x0b = (

248 ('CheckSum','<L=0'),

249 ('ECCCheckSum','<L=0'),

250 )

251 structure_win7 = (

252 ('CheckSum','<Q=0'),

253 )

254 common = (

255 ('LastModificationTime','<Q=0'),

256 ('PreviousPageNumber','<L=0'),

257 ('NextPageNumber','<L=0'),

258 ('FatherDataPage','<L=0'),

259 ('AvailableDataSize','<H=0'),

260 ('AvailableUncommittedDataSize','<H=0'),

261 ('FirstAvailableDataOffset','<H=0'),

262 ('FirstAvailablePageTag','<H=0'),

263 ('PageFlags','<L=0'),

264 )

265 extended_win7 = (

266 ('ExtendedCheckSum1','<Q=0'),

267 ('ExtendedCheckSum2','<Q=0'),

268 ('ExtendedCheckSum3','<Q=0'),

269 ('PageNumber','<Q=0'),

270 ('Unknown','<Q=0'),

271 )

272 def __init__(self, version, revision, pageSize=8192, data=None):

273 if (version < 0x620) or (version == 0x620 and revision < 0x0b): 273 ↛ 275line 273 didn't jump to line 275, because the condition on line 273 was never true

274 # For sure the old format

275 self.structure = self.structure_2003_SP0 + self.common

276 elif version == 0x620 and revision < 0x11: 276 ↛ 278line 276 didn't jump to line 278, because the condition on line 276 was never true

277 # Exchange 2003 SP1 and Windows Vista and later

278 self.structure = self.structure_0x620_0x0b + self.common

279 else:

280 # Windows 7 and later

281 self.structure = self.structure_win7 + self.common

282 if pageSize > 8192: 282 ↛ 283line 282 didn't jump to line 283, because the condition on line 282 was never true

283 self.structure += self.extended_win7

284

285 Structure.__init__(self,data)

286

287 class ESENT_ROOT_HEADER(Structure):

288 structure = (

289 ('InitialNumberOfPages','<L=0'),

290 ('ParentFatherDataPage','<L=0'),

291 ('ExtentSpace','<L=0'),

292 ('SpaceTreePageNumber','<L=0'),

293 )

294

295 class ESENT_BRANCH_HEADER(Structure):

296 structure = (

297 ('CommonPageKey',':'),

298 )

299

300 class ESENT_BRANCH_ENTRY(Structure):

301 common = (

302 ('CommonPageKeySize','<H=0'),

303 )

304 structure = (

305 ('LocalPageKeySize','<H=0'),

306 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'),

307 ('LocalPageKey',':'),

308 ('ChildPageNumber','<L=0'),

309 )

310 def __init__(self, flags, data=None):

311 if flags & TAG_COMMON > 0:

312 # Include the common header

313 self.structure = self.common + self.structure

314 Structure.__init__(self,data)

315

316 class ESENT_LEAF_HEADER(Structure):

317 structure = (

318 ('CommonPageKey',':'),

319 )

320

321 class ESENT_LEAF_ENTRY(Structure):

322 common = (

323 ('CommonPageKeySize','<H=0'),

324 )

325 structure = (

326 ('LocalPageKeySize','<H=0'),

327 ('_LocalPageKey','_-LocalPageKey','self["LocalPageKeySize"]'),

328 ('LocalPageKey',':'),

329 ('EntryData',':'),

330 )

331 def __init__(self, flags, data=None):

332 if flags & TAG_COMMON > 0:

333 # Include the common header

334 self.structure = self.common + self.structure

335 Structure.__init__(self,data)

336

337 class ESENT_SPACE_TREE_HEADER(Structure):

338 structure = (

339 ('Unknown','<Q=0'),

340 )

341

342 class ESENT_SPACE_TREE_ENTRY(Structure):

343 structure = (

344 ('PageKeySize','<H=0'),

345 ('LastPageNumber','<L=0'),

346 ('NumberOfPages','<L=0'),

347 )

348

349 class ESENT_INDEX_ENTRY(Structure):

350 structure = (

351 ('RecordPageKey',':'),

352 )

353

354 class ESENT_DATA_DEFINITION_HEADER(Structure):

355 structure = (

356 ('LastFixedSize','<B=0'),

357 ('LastVariableDataType','<B=0'),

358 ('VariableSizeOffset','<H=0'),

359 )

360

361 class ESENT_CATALOG_DATA_DEFINITION_ENTRY(Structure):

362 fixed = (

363 ('FatherDataPageID','<L=0'),

364 ('Type','<H=0'),

365 ('Identifier','<L=0'),

366 )

367

368 column_stuff = (

369 ('ColumnType','<L=0'),

370 ('SpaceUsage','<L=0'),

371 ('ColumnFlags','<L=0'),

372 ('CodePage','<L=0'),

373 )

374

375 other = (

376 ('FatherDataPageNumber','<L=0'),

377 )

378

379 table_stuff = (

380 ('SpaceUsage','<L=0'),

381# ('TableFlags','<L=0'),

382# ('InitialNumberOfPages','<L=0'),

383 )

384

385 index_stuff = (

386 ('SpaceUsage','<L=0'),

387 ('IndexFlags','<L=0'),

388 ('Locale','<L=0'),

389 )

390

391 lv_stuff = (

392 ('SpaceUsage','<L=0'),

393# ('LVFlags','<L=0'),

394# ('InitialNumberOfPages','<L=0'),

395 )

396 common = (

397# ('RootFlag','<B=0'),

398# ('RecordOffset','<H=0'),

399# ('LCMapFlags','<L=0'),

400# ('KeyMost','<H=0'),

401 ('Trailing',':'),

402 )

403

404 def __init__(self,data):

405 # Depending on the type of data we'll end up building a different struct

406 dataType = unpack('<H', data[4:][:2])[0]

407 self.structure = self.fixed

408

409 if dataType == CATALOG_TYPE_TABLE:

410 self.structure += self.other + self.table_stuff

411 elif dataType == CATALOG_TYPE_COLUMN:

412 self.structure += self.column_stuff

413 elif dataType == CATALOG_TYPE_INDEX:

414 self.structure += self.other + self.index_stuff

415 elif dataType == CATALOG_TYPE_LONG_VALUE: 415 ↛ 417line 415 didn't jump to line 417, because the condition on line 415 was never false

416 self.structure += self.other + self.lv_stuff

417 elif dataType == CATALOG_TYPE_CALLBACK:

418 raise Exception('CallBack types not supported!')

419 else:

420 LOG.error('Unknown catalog type 0x%x' % dataType)

421 self.structure = ()

422 Structure.__init__(self,data)

423

424 self.structure += self.common

425

426 Structure.__init__(self,data)

427

428

429#def pretty_print(x):

430# if x in '0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ!"#$%&\'()*+,-./:;<=>?@[\\]^_`{|}~ ':

431# return x

432# else:

433# return '.'

434#

435#def hexdump(data):

436# x=str(data)

437# strLen = len(x)

438# i = 0

439# while i < strLen:

440# print "%04x " % i,

441# for j in range(16):

442# if i+j < strLen:

443# print "%02X" % ord(x[i+j]),

444#

445# else:

446# print " ",

447# if j%16 == 7:

448# print "",

449# print " ",

450# print ''.join(pretty_print(x) for x in x[i:i+16] )

451# i += 16

452

453 def getUnixTime(t):

454 t -= 116444736000000000

455 t //= 10000000

456 return t

457

458 class ESENT_PAGE:

459 def __init__(self, db, data=None):

460 self.__DBHeader = db

461 self.data = data

462 self.record = None

463 if data is not None: 463 ↛ exitline 463 didn't return from function '__init__', because the condition on line 463 was never false

464 self.record = ESENT_PAGE_HEADER(self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'], self.__DBHeader['PageSize'], data)

465

466 def printFlags(self):

467 flags = self.record['PageFlags']

468 if flags & FLAGS_EMPTY:

469 print("\tEmpty")

470 if flags & FLAGS_INDEX:

471 print("\tIndex")

472 if flags & FLAGS_LEAF:

473 print("\tLeaf")

474 else:

475 print("\tBranch")

476 if flags & FLAGS_LONG_VALUE:

477 print("\tLong Value")

478 if flags & FLAGS_NEW_CHECKSUM:

479 print("\tNew Checksum")

480 if flags & FLAGS_NEW_FORMAT:

481 print("\tNew Format")

482 if flags & FLAGS_PARENT:

483 print("\tParent")

484 if flags & FLAGS_ROOT:

485 print("\tRoot")

486 if flags & FLAGS_SPACE_TREE:

487 print("\tSpace Tree")

488

489 def dump(self):

490 baseOffset = len(self.record)

491 self.record.dump()

492 tags = self.data[-4*self.record['FirstAvailablePageTag']:]

493

494 print("FLAGS: ")

495 self.printFlags()

496

497 print()

498

499 for i in range(self.record['FirstAvailablePageTag']):

500 tag = tags[-4:]

501 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] > 11 and self.__DBHeader['PageSize'] > 8192:

502 valueSize = unpack('<H', tag[:2])[0] & 0x7fff

503 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff

504 hexdump((self.data[baseOffset+valueOffset:][:6]))

505 pageFlags = ord(self.data[baseOffset+valueOffset:][1]) >> 5

506 #print "TAG FLAG: 0x%x " % (unpack('<L', self.data[baseOffset+valueOffset:][:4]) ) >> 5

507 #print "TAG FLAG: 0x " , ord(self.data[baseOffset+valueOffset:][0])

508 else:

509 valueSize = unpack('<H', tag[:2])[0] & 0x1fff

510 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13

511 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff

512

513 print("TAG %-8d offset:0x%-6x flags:0x%-4x valueSize:0x%x" % (i,valueOffset,pageFlags,valueSize))

514 #hexdump(self.getTag(i)[1])

515 tags = tags[:-4]

516

517 if self.record['PageFlags'] & FLAGS_ROOT > 0:

518 rootHeader = ESENT_ROOT_HEADER(self.getTag(0)[1])

519 rootHeader.dump()

520 elif self.record['PageFlags'] & FLAGS_LEAF == 0:

521 # Branch Header

522 flags, data = self.getTag(0)

523 branchHeader = ESENT_BRANCH_HEADER(data)

524 branchHeader.dump()

525 else:

526 # Leaf Header

527 flags, data = self.getTag(0)

528 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0:

529 # Space Tree

530 spaceTreeHeader = ESENT_SPACE_TREE_HEADER(data)

531 spaceTreeHeader.dump()

532 else:

533 leafHeader = ESENT_LEAF_HEADER(data)

534 leafHeader.dump()

535

536 # Print the leaf/branch tags

537 for tagNum in range(1,self.record['FirstAvailablePageTag']):

538 flags, data = self.getTag(tagNum)

539 if self.record['PageFlags'] & FLAGS_LEAF == 0:

540 # Branch page

541 branchEntry = ESENT_BRANCH_ENTRY(flags, data)

542 branchEntry.dump()

543 elif self.record['PageFlags'] & FLAGS_LEAF > 0:

544 # Leaf page

545 if self.record['PageFlags'] & FLAGS_SPACE_TREE > 0:

546 # Space Tree

547 spaceTreeEntry = ESENT_SPACE_TREE_ENTRY(data)

548 #spaceTreeEntry.dump()

549

550 elif self.record['PageFlags'] & FLAGS_INDEX > 0:

551 # Index Entry

552 indexEntry = ESENT_INDEX_ENTRY(data)

553 #indexEntry.dump()

554 elif self.record['PageFlags'] & FLAGS_LONG_VALUE > 0:

555 # Long Page Value

556 raise Exception('Long value still not supported')

557 else:

558 # Table Value

559 leafEntry = ESENT_LEAF_ENTRY(flags, data)

560 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(leafEntry['EntryData'])

561 dataDefinitionHeader.dump()

562 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(leafEntry['EntryData'][len(dataDefinitionHeader):])

563 catalogEntry.dump()

564 hexdump(leafEntry['EntryData'])

565

566 def getTag(self, tagNum):

567 if self.record['FirstAvailablePageTag'] < tagNum: 567 ↛ 568line 567 didn't jump to line 568, because the condition on line 567 was never true

568 raise Exception('Trying to grab an unknown tag 0x%x' % tagNum)

569

570 tags = self.data[-4*self.record['FirstAvailablePageTag']:]

571 baseOffset = len(self.record)

572 for i in range(tagNum):

573 tags = tags[:-4]

574

575 tag = tags[-4:]

576

577 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 577 ↛ 578line 577 didn't jump to line 578, because the condition on line 577 was never true

578 valueSize = unpack('<H', tag[:2])[0] & 0x7fff

579 valueOffset = unpack('<H',tag[2:])[0] & 0x7fff

580 tmpData = list(self.data[baseOffset+valueOffset:][:valueSize])

581 pageFlags = ord(tmpData[1]) >> 5

582 tmpData[1] = chr(ord(tmpData[1:2]) & 0x1f)

583 tagData = "".join(tmpData)

584 else:

585 valueSize = unpack('<H', tag[:2])[0] & 0x1fff

586 pageFlags = (unpack('<H', tag[2:])[0] & 0xe000) >> 13

587 valueOffset = unpack('<H',tag[2:])[0] & 0x1fff

588 tagData = self.data[baseOffset+valueOffset:][:valueSize]

589

590 #return pageFlags, self.data[baseOffset+valueOffset:][:valueSize]

591 return pageFlags, tagData

592

593 class ESENT_DB:

594 def __init__(self, fileName, pageSize = 8192, isRemote = False):

595 self.__fileName = fileName

596 self.__pageSize = pageSize

597 self.__DB = None

598 self.__DBHeader = None

599 self.__totalPages = None

600 self.__tables = OrderedDict()

601 self.__currentTable = None

602 self.__isRemote = isRemote

603 self.mountDB()

604

605 def mountDB(self):

606 LOG.debug("Mounting DB...")

607 if self.__isRemote is True: 607 ↛ 611line 607 didn't jump to line 611, because the condition on line 607 was never false

608 self.__DB = self.__fileName

609 self.__DB.open()

610 else:

611 self.__DB = open(self.__fileName,"rb")

612 mainHeader = self.getPage(-1)

613 self.__DBHeader = ESENT_DB_HEADER(mainHeader)

614 self.__pageSize = self.__DBHeader['PageSize']

615 self.__DB.seek(0,2)

616 self.__totalPages = (self.__DB.tell() // self.__pageSize) -2

617 LOG.debug("Database Version:0x%x, Revision:0x%x"% (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision']))

618 LOG.debug("Page Size: %d" % self.__pageSize)

619 LOG.debug("Total Pages in file: %d" % self.__totalPages)

620 self.parseCatalog(CATALOG_PAGE_NUMBER)

621

622 def printCatalog(self):

623 indent = ' '

624

625 print("Database version: 0x%x, 0x%x" % (self.__DBHeader['Version'], self.__DBHeader['FileFormatRevision'] ))

626 print("Page size: %d " % self.__pageSize)

627 print("Number of pages: %d" % self.__totalPages)

628 print()

629 print("Catalog for %s" % self.__fileName)

630 for table in list(self.__tables.keys()):

631 print("[%s]" % table.decode('utf8'))

632 print("%sColumns " % indent)

633 for column in list(self.__tables[table]['Columns'].keys()):

634 record = self.__tables[table]['Columns'][column]['Record']

635 print("%s%-5d%-30s%s" % (indent*2, record['Identifier'], column.decode('utf-8'),ColumnTypeToName[record['ColumnType']]))

636 print("%sIndexes"% indent)

637 for index in list(self.__tables[table]['Indexes'].keys()):

638 print("%s%s" % (indent*2, index.decode('utf-8')))

639 print("")

640

641 def __addItem(self, entry):

642 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])

643 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):])

644 itemName = self.__parseItemName(entry)

645

646 if catalogEntry['Type'] == CATALOG_TYPE_TABLE:

647 self.__tables[itemName] = OrderedDict()

648 self.__tables[itemName]['TableEntry'] = entry

649 self.__tables[itemName]['Columns'] = OrderedDict()

650 self.__tables[itemName]['Indexes'] = OrderedDict()

651 self.__tables[itemName]['LongValues'] = OrderedDict()

652 self.__currentTable = itemName

653 elif catalogEntry['Type'] == CATALOG_TYPE_COLUMN:

654 self.__tables[self.__currentTable]['Columns'][itemName] = entry

655 self.__tables[self.__currentTable]['Columns'][itemName]['Header'] = dataDefinitionHeader

656 self.__tables[self.__currentTable]['Columns'][itemName]['Record'] = catalogEntry

657 elif catalogEntry['Type'] == CATALOG_TYPE_INDEX:

658 self.__tables[self.__currentTable]['Indexes'][itemName] = entry

659 elif catalogEntry['Type'] == CATALOG_TYPE_LONG_VALUE: 659 ↛ 662line 659 didn't jump to line 662, because the condition on line 659 was never false

660 self.__addLongValue(entry)

661 else:

662 raise Exception('Unknown type 0x%x' % catalogEntry['Type'])

663

664 def __parseItemName(self,entry):

665 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])

666

667 if dataDefinitionHeader['LastVariableDataType'] > 127: 667 ↛ 670line 667 didn't jump to line 670, because the condition on line 667 was never false

668 numEntries = dataDefinitionHeader['LastVariableDataType'] - 127

669 else:

670 numEntries = dataDefinitionHeader['LastVariableDataType']

671

672 itemLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0]

673 itemName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][2*numEntries:][:itemLen]

674 return itemName

675

676 def __addLongValue(self, entry):

677 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])

678 lvLen = unpack('<H',entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][:2])[0]

679 lvName = entry['EntryData'][dataDefinitionHeader['VariableSizeOffset']:][7:][:lvLen]

680 self.__tables[self.__currentTable]['LongValues'][lvName] = entry

681

682 def parsePage(self, page):

683 # Print the leaf/branch tags

684 for tagNum in range(1,page.record['FirstAvailablePageTag']):

685 flags, data = page.getTag(tagNum)

686 if page.record['PageFlags'] & FLAGS_LEAF > 0:

687 # Leaf page

688 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 688 ↛ 689line 688 didn't jump to line 689, because the condition on line 688 was never true

689 pass

690 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 690 ↛ 691line 690 didn't jump to line 691, because the condition on line 690 was never true

691 pass

692 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 692 ↛ 693line 692 didn't jump to line 693, because the condition on line 692 was never true

693 pass

694 else:

695 # Table Value

696 leafEntry = ESENT_LEAF_ENTRY(flags, data)

697 self.__addItem(leafEntry)

698

699 def parseCatalog(self, pageNum):

700 # Parse all the pages starting at pageNum and commit table data

701 page = self.getPage(pageNum)

702 self.parsePage(page)

703

704 for i in range(1, page.record['FirstAvailablePageTag']):

705 flags, data = page.getTag(i)

706 if page.record['PageFlags'] & FLAGS_LEAF == 0:

707 # Branch page

708 branchEntry = ESENT_BRANCH_ENTRY(flags, data)

709 self.parseCatalog(branchEntry['ChildPageNumber'])

710

711

712 def readHeader(self):

713 LOG.debug("Reading Boot Sector for %s" % self.__volumeName)

714

715 def getPage(self, pageNum):

716 LOG.debug("Trying to fetch page %d (0x%x)" % (pageNum, (pageNum+1)*self.__pageSize))

717 self.__DB.seek((pageNum+1)*self.__pageSize, 0)

718 data = self.__DB.read(self.__pageSize)

719 while len(data) < self.__pageSize: 719 ↛ 720line 719 didn't jump to line 720, because the condition on line 719 was never true

720 remaining = self.__pageSize - len(data)

721 data += self.__DB.read(remaining)

722 # Special case for the first page

723 if pageNum <= 0:

724 return data

725 else:

726 return ESENT_PAGE(self.__DBHeader, data)

727

728 def close(self):

729 self.__DB.close()

730

731 def openTable(self, tableName):

732 # Returns a cursos for later use

733

734 if isinstance(tableName, bytes) is not True: 734 ↛ 737line 734 didn't jump to line 737, because the condition on line 734 was never false

735 tableName = b(tableName)

736

737 if tableName in self.__tables: 737 ↛ 768line 737 didn't jump to line 768, because the condition on line 737 was never false

738 entry = self.__tables[tableName]['TableEntry']

739 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(entry['EntryData'])

740 catalogEntry = ESENT_CATALOG_DATA_DEFINITION_ENTRY(entry['EntryData'][len(dataDefinitionHeader):])

741

742 # Let's position the cursor at the leaf levels for fast reading

743 pageNum = catalogEntry['FatherDataPageNumber']

744 done = False

745 while done is False:

746 page = self.getPage(pageNum)

747 if page.record['FirstAvailablePageTag'] <= 1: 747 ↛ 749line 747 didn't jump to line 749, because the condition on line 747 was never true

748 # There are no records

749 done = True

750 for i in range(1, page.record['FirstAvailablePageTag']): 750 ↛ 745line 750 didn't jump to line 745, because the loop on line 750 didn't complete

751 flags, data = page.getTag(i)

752 if page.record['PageFlags'] & FLAGS_LEAF == 0:

753 # Branch page, move on to the next page

754 branchEntry = ESENT_BRANCH_ENTRY(flags, data)

755 pageNum = branchEntry['ChildPageNumber']

756 break

757 else:

758 done = True

759 break

760

761 cursor = TABLE_CURSOR

762 cursor['TableData'] = self.__tables[tableName]

763 cursor['FatherDataPageNumber'] = catalogEntry['FatherDataPageNumber']

764 cursor['CurrentPageData'] = page

765 cursor['CurrentTag'] = 0

766 return cursor

767 else:

768 return None

769

770 def __getNextTag(self, cursor):

771 page = cursor['CurrentPageData']

772

773 if cursor['CurrentTag'] >= page.record['FirstAvailablePageTag']:

774 # No more data in this page, chau

775 return None

776

777 flags, data = page.getTag(cursor['CurrentTag'])

778 if page.record['PageFlags'] & FLAGS_LEAF > 0: 778 ↛ 791line 778 didn't jump to line 791, because the condition on line 778 was never false

779 # Leaf page

780 if page.record['PageFlags'] & FLAGS_SPACE_TREE > 0: 780 ↛ 781line 780 didn't jump to line 781, because the condition on line 780 was never true

781 raise Exception('FLAGS_SPACE_TREE > 0')

782 elif page.record['PageFlags'] & FLAGS_INDEX > 0: 782 ↛ 783line 782 didn't jump to line 783, because the condition on line 782 was never true

783 raise Exception('FLAGS_INDEX > 0')

784 elif page.record['PageFlags'] & FLAGS_LONG_VALUE > 0: 784 ↛ 785line 784 didn't jump to line 785, because the condition on line 784 was never true

785 raise Exception('FLAGS_LONG_VALUE > 0')

786 else:

787 # Table Value

788 leafEntry = ESENT_LEAF_ENTRY(flags, data)

789 return leafEntry

790

791 return None

792

793 def getNextRow(self, cursor):

794 cursor['CurrentTag'] += 1

795

796 tag = self.__getNextTag(cursor)

797 #hexdump(tag)

798

799 if tag is None:

800 # No more tags in this page, search for the next one on the right

801 page = cursor['CurrentPageData']

802 if page.record['NextPageNumber'] == 0:

803 # No more pages, chau

804 return None

805 else:

806 cursor['CurrentPageData'] = self.getPage(page.record['NextPageNumber'])

807 cursor['CurrentTag'] = 0

808 return self.getNextRow(cursor)

809 else:

810 return self.__tagToRecord(cursor, tag['EntryData'])

811

812 def __tagToRecord(self, cursor, tag):

813 # So my brain doesn't forget, the data record is composed of:

814 # Header

815 # Fixed Size Data (ID < 127)

816 # The easiest to parse. Their size is fixed in the record. You can get its size

817 # from the Column Record, field SpaceUsage

818 # Variable Size Data (127 < ID < 255)

819 # At VariableSizeOffset you get an array of two bytes per variable entry, pointing

820 # to the length of the value. Values start at:

821 # numEntries = LastVariableDataType - 127

822 # VariableSizeOffset + numEntries * 2 (bytes)

823 # Tagged Data ( > 255 )

824 # After the Variable Size Value, there's more data for the tagged values.

825 # Right at the beginning there's another array (taggedItems), pointing to the

826 # values, size.

827 #

828 # The interesting thing about this DB records is there's no need for all the columns to be there, hence

829 # saving space. That's why I got over all the columns, and if I find data (of any type), i assign it. If

830 # not, the column's empty.

831 #

832 # There are a lot of caveats in the code, so take your time to explore it.

833 #

834 # ToDo: Better complete this description

835 #

836

837 record = OrderedDict()

838 taggedItems = OrderedDict()

839 taggedItemsParsed = False

840

841 dataDefinitionHeader = ESENT_DATA_DEFINITION_HEADER(tag)

842 #dataDefinitionHeader.dump()

843 variableDataBytesProcessed = (dataDefinitionHeader['LastVariableDataType'] - 127) * 2

844 prevItemLen = 0

845 tagLen = len(tag)

846 fixedSizeOffset = len(dataDefinitionHeader)

847 variableSizeOffset = dataDefinitionHeader['VariableSizeOffset']

848

849 columns = cursor['TableData']['Columns']

850

851 for column in list(columns.keys()):

852 columnRecord = columns[column]['Record']

853 #columnRecord.dump()

854 if columnRecord['Identifier'] <= dataDefinitionHeader['LastFixedSize']:

855 # Fixed Size column data type, still available data

856 record[column] = tag[fixedSizeOffset:][:columnRecord['SpaceUsage']]

857 fixedSizeOffset += columnRecord['SpaceUsage']

858

859 elif 127 < columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']: 859 ↛ 861line 859 didn't jump to line 861, because the condition on line 859 was never true

860 # Variable data type

861 index = columnRecord['Identifier'] - 127 - 1

862 itemLen = unpack('<H',tag[variableSizeOffset+index*2:][:2])[0]

863

864 if itemLen & 0x8000:

865 # Empty item

866 itemLen = prevItemLen

867 record[column] = None

868 else:

869 itemValue = tag[variableSizeOffset+variableDataBytesProcessed:][:itemLen-prevItemLen]

870 record[column] = itemValue

871

872 #if columnRecord['Identifier'] <= dataDefinitionHeader['LastVariableDataType']:

873 variableDataBytesProcessed +=itemLen-prevItemLen

874

875 prevItemLen = itemLen

876

877 elif columnRecord['Identifier'] > 255:

878 # Have we parsed the tagged items already?

879 if taggedItemsParsed is False and (variableDataBytesProcessed+variableSizeOffset) < tagLen:

880 index = variableDataBytesProcessed+variableSizeOffset

881 #hexdump(tag[index:])

882 endOfVS = self.__pageSize

883 firstOffsetTag = (unpack('<H', tag[index+2:][:2])[0] & 0x3fff) + variableDataBytesProcessed+variableSizeOffset

884 while True:

885 taggedIdentifier = unpack('<H', tag[index:][:2])[0]

886 index += 2

887 taggedOffset = (unpack('<H', tag[index:][:2])[0] & 0x3fff)

888 # As of Windows 7 and later ( version 0x620 revision 0x11) the

889 # tagged data type flags are always present

890 if self.__DBHeader['Version'] == 0x620 and self.__DBHeader['FileFormatRevision'] >= 17 and self.__DBHeader['PageSize'] > 8192: 890 ↛ 891line 890 didn't jump to line 891, because the condition on line 890 was never true

891 flagsPresent = 1

892 else:

893 flagsPresent = (unpack('<H', tag[index:][:2])[0] & 0x4000)

894 index += 2

895 if taggedOffset < endOfVS:

896 endOfVS = taggedOffset

897 taggedItems[taggedIdentifier] = (taggedOffset, tagLen, flagsPresent)

898 #print "ID: %d, Offset:%d, firstOffset:%d, index:%d, flag: 0x%x" % (taggedIdentifier, taggedOffset,firstOffsetTag,index, flagsPresent)

899 if index >= firstOffsetTag:

900 # We reached the end of the variable size array

901 break

902

903 # Calculate length of variable items

904 # Ugly.. should be redone

905 prevKey = list(taggedItems.keys())[0]

906 for i in range(1,len(taggedItems)):

907 offset0, length, flags = taggedItems[prevKey]

908 offset, _, _ = list(taggedItems.items())[i][1]

909 taggedItems[prevKey] = (offset0, offset-offset0, flags)

910 #print "ID: %d, Offset: %d, Len: %d, flags: %d" % (prevKey, offset0, offset-offset0, flags)

911 prevKey = list(taggedItems.keys())[i]

912 taggedItemsParsed = True

913

914 # Tagged data type

915 if columnRecord['Identifier'] in taggedItems:

916 offsetItem = variableDataBytesProcessed + variableSizeOffset + taggedItems[columnRecord['Identifier']][0]

917 itemSize = taggedItems[columnRecord['Identifier']][1]

918 # If item have flags, we should skip them

919 if taggedItems[columnRecord['Identifier']][2] > 0:

920 itemFlag = ord(tag[offsetItem: offsetItem+1])

921 offsetItem += 1

922 itemSize -= 1

923 else:

924 itemFlag = 0

925

926 #print "ID: %d, itemFlag: 0x%x" %( columnRecord['Identifier'], itemFlag)

927 if itemFlag & (TAGGED_DATA_TYPE_COMPRESSED ): 927 ↛ 928line 927 didn't jump to line 928, because the condition on line 927 was never true

928 LOG.error('Unsupported tag column: %s, flag:0x%x' % (column, itemFlag))

929 record[column] = None

930 elif itemFlag & TAGGED_DATA_TYPE_MULTI_VALUE:

931 # ToDo: Parse multi-values properly

932 LOG.debug('Multivalue detected in column %s, returning raw results' % (column))

933 record[column] = (hexlify(tag[offsetItem:][:itemSize]),)

934 else:

935 record[column] = tag[offsetItem:][:itemSize]

936

937 else:

938 record[column] = None

939 else:

940 record[column] = None

941

942 # If we understand the data type, we unpack it and cast it accordingly

943 # otherwise, we just encode it in hex

944 if type(record[column]) is tuple:

945 # A multi value data, we won't decode it, just leave it this way

946 record[column] = record[column][0]

947 elif columnRecord['ColumnType'] == JET_coltypText or columnRecord['ColumnType'] == JET_coltypLongText:

948 # Let's handle strings

949 if record[column] is not None:

950 if columnRecord['CodePage'] not in StringCodePages: 950 ↛ 951line 950 didn't jump to line 951, because the condition on line 950 was never true

951 raise Exception('Unknown codepage 0x%x'% columnRecord['CodePage'])

952 stringDecoder = StringCodePages[columnRecord['CodePage']]

953

954 try:

955 record[column] = record[column].decode(stringDecoder)

956 except Exception:

957 LOG.debug("Exception:", exc_info=True)

958 LOG.debug('Fixing Record[%r][%d]: %r' % (column, columnRecord['ColumnType'], record[column]))

959 record[column] = record[column].decode(stringDecoder, "replace")

960 pass

961 else:

962 unpackData = ColumnTypeSize[columnRecord['ColumnType']]

963 if record[column] is not None:

964 if unpackData is None:

965 record[column] = hexlify(record[column])

966 else:

967 unpackStr = unpackData[1]

968 record[column] = unpack(unpackStr, record[column])[0]

969

970 return record