Skip to content

EXT4: read sparse files and unwritten extents as zeros #960

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
sax wants to merge 1 commit into
base: main
Choose a base branch
from
Open

EXT4: read sparse files and unwritten extents as zeros #960

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 33 additions & 6 deletions Sources/ContainerizationEXT4/EXT4+Reader.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,11 +190,16 @@ extension EXT4 {
}
}

/// Physical block ranges of an inode, without their file offsets. Use `getFileExtents(inode:)` to read file contents.
func getExtents(inode: InodeNumber) throws -> [(start: UInt32, end: UInt32)]? {
try getFileExtents(inode: inode)?.map { ($0.physicalBlock, $0.physicalBlock + $0.length) }
}

func getFileExtents(inode: InodeNumber) throws -> [FileExtent]? {
let inode = try self.getInode(number: inode)
let inodeBlock = Data(tupleToArray(inode.block))
var offset = 0
var extents: [(start: UInt32, end: UInt32)] = []
var extents: [FileExtent] = []

let extentHeaderSize = MemoryLayout<ExtentHeader>.size
let extentIndexSize = MemoryLayout<ExtentIndex>.size
Expand All @@ -204,7 +209,10 @@ extension EXT4 {
$0.loadLittleEndian(as: ExtentHeader.self)
}
guard header.magic == EXT4.ExtentHeaderMagic else {
return []
return nil
}
guard extentHeaderSize + Int(header.entries) * extentLeafSize <= inodeBlock.count else {
throw Error.invalidExtents
}
offset += extentHeaderSize // Jump to entries
switch header.depth {
Expand All @@ -214,7 +222,7 @@ extension EXT4 {
let leaf = inodeBlock.subdata(in: offset..<offset + extentLeafSize).withUnsafeBytes {
$0.loadLittleEndian(as: ExtentLeaf.self)
}
extents.append((leaf.startLow, leaf.startLow + UInt32(leaf.length)))
extents.append(try Self.fileExtent(leaf))
offset += extentLeafSize
}
case 1:
Expand All @@ -223,23 +231,28 @@ extension EXT4 {
let indexNode = inodeBlock.subdata(in: offset..<offset + extentIndexSize).withUnsafeBytes {
$0.loadLittleEndian(as: ExtentIndex.self)
}
guard indexNode.leafHigh == 0 else {
throw Error.invalidExtents
}
try self.seek(block: indexNode.leafLow)
guard let block = try self.handle.read(upToCount: Int(self.blockSize)) else {
guard let block = try self.handle.read(upToCount: Int(self.blockSize)), block.count == Int(self.blockSize) else {
throw EXT4.Error.couldNotReadBlock(indexNode.leafLow)
}
var blockOffset = 0
let leafHeader = block.subdata(in: blockOffset..<extentHeaderSize).withUnsafeBytes {
$0.loadLittleEndian(as: ExtentHeader.self)
}
guard leafHeader.magic == EXT4.ExtentHeaderMagic else {
guard leafHeader.magic == EXT4.ExtentHeaderMagic, leafHeader.depth == 0,
extentHeaderSize + Int(leafHeader.entries) * extentLeafSize <= block.count
else {
throw Error.invalidExtents
}
blockOffset += extentHeaderSize
for _ in 0..<leafHeader.entries {
let leaf = block.subdata(in: blockOffset..<blockOffset + extentLeafSize).withUnsafeBytes {
$0.loadLittleEndian(as: ExtentLeaf.self)
}
extents.append((leaf.startLow, leaf.startLow + UInt32(leaf.length)))
extents.append(try Self.fileExtent(leaf))
blockOffset += extentLeafSize
}
offset += extentIndexSize
Expand All @@ -250,6 +263,20 @@ extension EXT4 {
return extents
}

private static func fileExtent(_ leaf: ExtentLeaf) throws -> FileExtent {
// This reader only supports 32-bit block numbers.
guard leaf.startHigh == 0 else {
throw Error.invalidExtents
}
// A length above 32768 marks an unwritten extent of (length - 32768) blocks.
let initialized = UInt32(leaf.length) <= EXT4.MaxBlocksPerExtent
let length = initialized ? UInt32(leaf.length) : UInt32(leaf.length) - EXT4.MaxBlocksPerExtent
guard leaf.startLow <= UInt32.max - length, leaf.block <= UInt32.max - length else {
throw Error.invalidExtents
}
return FileExtent(logicalBlock: leaf.block, physicalBlock: leaf.startLow, length: length, initialized: initialized)
}

// MARK: Internal functions
func getInode(number: UInt32) throws -> Inode {
if let inode = self.inodes[number] {
Expand Down
7 changes: 7 additions & 0 deletions Sources/ContainerizationEXT4/EXT4+Types.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -522,6 +522,13 @@ extension EXT4 {
var leaves: [ExtentLeaf]
}

struct FileExtent {
let logicalBlock: UInt32
let physicalBlock: UInt32
let length: UInt32
let initialized: Bool
}

struct DirectoryEntry {
let inode: InodeNumber
let recordLength: UInt16
Expand Down
3 changes: 3 additions & 0 deletions Sources/ContainerizationEXT4/EXT4.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -326,6 +326,7 @@ extension EXT4 {
case invalidPathEncoding(_ path: String)
case couldNotReadInode(_ inode: UInt32)
case couldNotReadGroup(_ group: UInt32)
case fileSizeMismatch(_ path: String, _ expected: UInt64, _ actual: UInt64)
public var description: String {
switch self {
case .notFound(let path):
Expand All @@ -348,6 +349,8 @@ extension EXT4 {
return "could not read inode \(inode)"
case .couldNotReadGroup(let group):
return "could not read group descriptor \(group)"
case .fileSizeMismatch(let path, let expected, let actual):
return "read \(actual) bytes of \(path), expected \(expected)"
}
}
}
Expand Down
98 changes: 61 additions & 37 deletions Sources/ContainerizationEXT4/EXT4Reader+Export.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -88,43 +88,10 @@ extension EXT4.EXT4Reader {
try writer.writeEntry(entry: entry, data: nil)
} else if mode.isReg() {
entry.fileType = .regular
var data = Data()
var remaining: UInt64 = size
if let block = item.blocks {
for dataBlock in block.start..<block.end {
try self.seek(block: dataBlock)
var count: UInt64
if remaining > self.blockSize {
count = self.blockSize
} else {
count = remaining
}
guard let dataBytes = try self.handle.read(upToCount: Int(count)) else {
throw EXT4.Error.couldNotReadBlock(dataBlock)
}
data.append(dataBytes)
remaining -= UInt64(dataBytes.count)
}
}
if let additionalBlocks = item.additionalBlocks {
for block in additionalBlocks {
for dataBlock in block.start..<block.end {
try self.seek(block: dataBlock)
var count: UInt64
if remaining > self.blockSize {
count = self.blockSize
} else {
count = remaining
}
guard let dataBytes = try self.handle.read(upToCount: Int(count)) else {
throw EXT4.Error.couldNotReadBlock(dataBlock)
}
data.append(dataBytes)
remaining -= UInt64(dataBytes.count)
}
}
}
try writer.writeEntry(entry: entry, data: data)
let transaction = writer.makeTransactionWriter()
try transaction.writeHeader(entry: entry)
try self.writeContents(of: item.inode, size: size, path: pathStr, to: transaction)
try transaction.finish()
} else if mode.isLink() {
entry.fileType = .symbolicLink
if size < 60 {
Expand Down Expand Up @@ -163,6 +130,63 @@ extension EXT4.EXT4Reader {
try writer.finishEncoding()
}

private func writeContents(of inode: EXT4.InodeNumber, size: UInt64, path: String, to writer: ArchiveWriterTransaction) throws {
guard let extents = try self.getFileExtents(inode: inode) else {
// Inline-data and block-mapped files aren't supported.
guard size == 0 else {
throw EXT4.Error.invalidExtents
}
return
}

let chunkSize = UInt64(1 << 20)
var zeros = Data()
var written: UInt64 = 0

func writeZeros(upTo end: UInt64) throws {
if written < end && zeros.isEmpty {
zeros = Data(count: Int(min(size, chunkSize)))
}
while written < end {
let count = Int(min(end - written, chunkSize))
try zeros.withUnsafeBytes { try writer.writeChunk(data: UnsafeRawBufferPointer(rebasing: $0[0..<count])) }
written += UInt64(count)
}
}

for extent in extents {
let extentStart = UInt64(extent.logicalBlock) * self.blockSize
guard extentStart >= written else {
// Extents must be sorted and non-overlapping.
throw EXT4.Error.invalidExtents
}
if extentStart >= size {
break
}
let extentEnd = min(extentStart + UInt64(extent.length) * self.blockSize, size)
try writeZeros(upTo: extentStart)
guard extent.initialized else {
try writeZeros(upTo: extentEnd)
continue
}
try self.seek(block: extent.physicalBlock)
while written < extentEnd {
let count = Int(min(extentEnd - written, chunkSize))
guard let data = try self.handle.read(upToCount: count), data.count == count else {
let block = UInt64(extent.physicalBlock) + (written - extentStart) / self.blockSize
throw EXT4.Error.couldNotReadBlock(UInt32(truncatingIfNeeded: block))
}
try data.withUnsafeBytes { try writer.writeChunk(data: $0) }
written += UInt64(count)
}
}
try writeZeros(upTo: size)

guard written == size else {
throw EXT4.Error.fileSizeMismatch(path, size, written)
}
}

@available(*, deprecated, renamed: "readInlineExtendedAttributes(from:)")
public static func readInlineExtenedAttributes(from buffer: [UInt8]) throws -> [EXT4.ExtendedAttribute] {
try readInlineExtendedAttributes(from: buffer)
Expand Down
Loading