From b957619ac399188fd8658980d6eb038e17247f68 Mon Sep 17 00:00:00 2001 From: starainrt Date: Tue, 16 Nov 2021 15:58:13 +0800 Subject: [PATCH] add mft read method --- filestats_windows.go | 131 ++++++++++- mft.go | 218 ++++++++++++++++++ ntfs/binutil/binutil.go | 111 +++++++++ ntfs/bootsect/bootsect.go | 72 ++++++ ntfs/cmd/example/main.go | 141 ++++++++++++ ntfs/cmd/mftdump/main.go | 251 +++++++++++++++++++++ ntfs/fragment/reader.go | 83 +++++++ ntfs/mft/attributes.go | 343 ++++++++++++++++++++++++++++ ntfs/mft/mft.go | 462 ++++++++++++++++++++++++++++++++++++++ ntfs/mft/output.go | 187 +++++++++++++++ ntfs/utf16/decode.go | 18 ++ os.go => usn.go | 91 +++++--- 12 files changed, 2072 insertions(+), 36 deletions(-) create mode 100644 mft.go create mode 100644 ntfs/binutil/binutil.go create mode 100644 ntfs/bootsect/bootsect.go create mode 100644 ntfs/cmd/example/main.go create mode 100644 ntfs/cmd/mftdump/main.go create mode 100644 ntfs/fragment/reader.go create mode 100644 ntfs/mft/attributes.go create mode 100644 ntfs/mft/mft.go create mode 100644 ntfs/mft/output.go create mode 100644 ntfs/utf16/decode.go rename os.go => usn.go (86%) diff --git a/filestats_windows.go b/filestats_windows.go index 0bee999..36e1208 100644 --- a/filestats_windows.go +++ b/filestats_windows.go @@ -40,8 +40,8 @@ type FileStat struct { // newFileStatFromWin32finddata copies all required information // from syscall.Win32finddata d into the newly created fileStat. -func newFileStatFromInformation(d *syscall.ByHandleFileInformation, name string, path string) *FileStat { - return &FileStat{ +func newFileStatFromInformation(d *syscall.ByHandleFileInformation, name string, path string) FileStat { + return FileStat{ name: name, path: path, FileAttributes: d.FileAttributes, @@ -127,3 +127,130 @@ var devNullStat = FileStat{ idxhi: 0, idxlo: 0, } + +func fixLongPath(path string) string { + // Do nothing (and don't allocate) if the path is "short". + // Empirically (at least on the Windows Server 2013 builder), + // the kernel is arbitrarily okay with < 248 bytes. That + // matches what the docs above say: + // "When using an API to create a directory, the specified + // path cannot be so long that you cannot append an 8.3 file + // name (that is, the directory name cannot exceed MAX_PATH + // minus 12)." Since MAX_PATH is 260, 260 - 12 = 248. + // + // The MSDN docs appear to say that a normal path that is 248 bytes long + // will work; empirically the path must be less then 248 bytes long. + if len(path) < 248 { + // Don't fix. (This is how Go 1.7 and earlier worked, + // not automatically generating the \\?\ form) + return path + } + + // The extended form begins with \\?\, as in + // \\?\c:\windows\foo.txt or \\?\UNC\server\share\foo.txt. + // The extended form disables evaluation of . and .. path + // elements and disables the interpretation of / as equivalent + // to \. The conversion here rewrites / to \ and elides + // . elements as well as trailing or duplicate separators. For + // simplicity it avoids the conversion entirely for relative + // paths or paths containing .. elements. For now, + // \\server\share paths are not converted to + // \\?\UNC\server\share paths because the rules for doing so + // are less well-specified. + if len(path) >= 2 && path[:2] == `\\` { + // Don't canonicalize UNC paths. + return path + } + if !isAbs(path) { + // Relative path + return path + } + + const prefix = `\\?` + + pathbuf := make([]byte, len(prefix)+len(path)+len(`\`)) + copy(pathbuf, prefix) + n := len(path) + r, w := 0, len(prefix) + for r < n { + switch { + case IsPathSeparator(path[r]): + // empty block + r++ + case path[r] == '.' && (r+1 == n || IsPathSeparator(path[r+1])): + // /./ + r++ + case r+1 < n && path[r] == '.' && path[r+1] == '.' && (r+2 == n || IsPathSeparator(path[r+2])): + // /../ is currently unhandled + return path + default: + pathbuf[w] = '\\' + w++ + for ; r < n && !IsPathSeparator(path[r]); r++ { + pathbuf[w] = path[r] + w++ + } + } + } + // A drive's root directory needs a trailing \ + if w == len(`\\?\c:`) { + pathbuf[w] = '\\' + w++ + } + return string(pathbuf[:w]) +} + +func IsPathSeparator(c uint8) bool { + // NOTE: Windows accept / as path separator. + return c == '\\' || c == '/' +} + +func isAbs(path string) (b bool) { + v := volumeName(path) + if v == "" { + return false + } + path = path[len(v):] + if path == "" { + return false + } + return IsPathSeparator(path[0]) +} + +func volumeName(path string) (v string) { + if len(path) < 2 { + return "" + } + // with drive letter + c := path[0] + if path[1] == ':' && + ('0' <= c && c <= '9' || 'a' <= c && c <= 'z' || + 'A' <= c && c <= 'Z') { + return path[:2] + } + // is it UNC + if l := len(path); l >= 5 && IsPathSeparator(path[0]) && IsPathSeparator(path[1]) && + !IsPathSeparator(path[2]) && path[2] != '.' { + // first, leading `\\` and next shouldn't be `\`. its server name. + for n := 3; n < l-1; n++ { + // second, next '\' shouldn't be repeated. + if IsPathSeparator(path[n]) { + n++ + // third, following something characters. its share name. + if !IsPathSeparator(path[n]) { + if path[n] == '.' { + break + } + for ; n < l; n++ { + if IsPathSeparator(path[n]) { + break + } + } + return path[:n] + } + break + } + } + } + return "" +} diff --git a/mft.go b/mft.go new file mode 100644 index 0000000..c214b3b --- /dev/null +++ b/mft.go @@ -0,0 +1,218 @@ +package wincmd + +import ( + "b612.me/win32api" + "b612.me/wincmd/ntfs/binutil" + "b612.me/wincmd/ntfs/mft" + "b612.me/wincmd/ntfs/utf16" + "encoding/binary" + "errors" + "io" + "os" + "reflect" + "runtime" + "time" + "unsafe" +) + +type MFTFile struct { + Name string + Path string + ModTime time.Time + Size uint64 + IsDir bool + Node uint64 +} + +func GetFileListsByMftFn(driver string, fn func(string, bool) bool) ([]MFTFile, error) { + var result []MFTFile + fileMap := make(map[win32api.DWORDLONG]FileEntry) + f, size, err := mft.GetMFTFile(driver) + if err != nil { + return []MFTFile{}, err + } + recordSize := int64(1024) + alreadyGot := int64(0) + maxRecordSize := size / recordSize + if maxRecordSize > 1024 { + maxRecordSize = 1024 + } + for { + for { + if (size - alreadyGot) < maxRecordSize*recordSize { + maxRecordSize-- + } else { + break + } + } + if maxRecordSize < 10 { + maxRecordSize = 1 + } + buf := make([]byte, maxRecordSize*recordSize) + got, err := io.ReadFull(f, buf) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return []MFTFile{}, err + } + alreadyGot += int64(got) + for j := int64(0); j < 1024*maxRecordSize; j += 1024 { + record, err := mft.ParseRecord(buf[j : j+1024]) + if err != nil { + continue + } + if record.Flags&mft.RecordFlagInUse == 1 && record.Flags&mft.RecordFlagIsIndex == 0 { + var file MFTFile + file.IsDir = record.Flags&mft.RecordFlagIsDirectory != 0 + file.Node = record.FileReference.ToUint64() + parent := uint64(0) + for _, v := range record.Attributes { + if v.Type == mft.AttributeTypeFileName { + if file.Name != "" { + continue + } + file.Name = utf16.DecodeString(v.Data[66:], binary.LittleEndian) + if file.Name != "" { + parent = binutil.NewLittleEndianReader(v.Data[:8]).Uint64(0) + } + } + if v.Type == mft.AttributeTypeData { + file.Size = v.ActualSize + } + if v.Type == mft.AttributeTypeStandardInformation { + if len(v.Data) < 48 { + continue + } + r := binutil.NewLittleEndianReader(v.Data) + file.ModTime = mft.ConvertFileTime(r.Uint64(0x08)) + } + } + if file.Name != "" { + canAdd := fn(file.Name, file.IsDir) + if canAdd { + result = append(result, file) + } + if canAdd || file.IsDir { + fileMap[win32api.DWORDLONG(file.Node)] = FileEntry{ + Name: file.Name, + Parent: win32api.DWORDLONG(parent), + Type: 0, + } + } + } + } + } + } + + (*reflect.SliceHeader)(unsafe.Pointer(&result)).Cap = len(result) + for k, v := range result { + result[k].Path = GetFullUsnPath(driver, fileMap, win32api.DWORDLONG(v.Node)) + } + fileMap = nil + runtime.GC() + return result, nil +} + +func GetFileListsByMft(driver string) ([]MFTFile, error) { + return GetFileListsByMftFn(driver, func(string, bool) bool { return true }) +} + +func GetFileListsFromMftFileFn(filepath string, fn func(string, bool) bool) ([]MFTFile, error) { + var result []MFTFile + fileMap := make(map[win32api.DWORDLONG]FileEntry) + f, err := os.Open(filepath) + if err != nil { + return []MFTFile{}, err + } + stat, err := f.Stat() + if err != nil { + return []MFTFile{}, err + } + size := stat.Size() + recordSize := int64(1024) + alreadyGot := int64(0) + maxRecordSize := size / recordSize + if maxRecordSize > 1024 { + maxRecordSize = 1024 + } + for { + for { + if (size - alreadyGot) < maxRecordSize*recordSize { + maxRecordSize-- + } else { + break + } + } + if maxRecordSize < 10 { + maxRecordSize = 1 + } + buf := make([]byte, maxRecordSize*recordSize) + got, err := io.ReadFull(f, buf) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + return []MFTFile{}, err + } + alreadyGot += int64(got) + for j := int64(0); j < 1024*maxRecordSize; j += 1024 { + record, err := mft.ParseRecord(buf[j : j+1024]) + if err != nil { + continue + } + if record.Flags&mft.RecordFlagInUse == 1 && record.Flags&mft.RecordFlagIsIndex == 0 { + var file MFTFile + file.IsDir = record.Flags&mft.RecordFlagIsDirectory != 0 + file.Node = record.FileReference.ToUint64() + parent := uint64(0) + for _, v := range record.Attributes { + if v.Type == mft.AttributeTypeFileName { + if file.Name != "" { + continue + } + file.Name = utf16.DecodeString(v.Data[66:], binary.LittleEndian) + if file.Name != "" { + parent = binutil.NewLittleEndianReader(v.Data[:8]).Uint64(0) + } + } + if v.Type == mft.AttributeTypeData { + file.Size = v.ActualSize + } + if v.Type == mft.AttributeTypeStandardInformation { + if len(v.Data) < 48 { + continue + } + r := binutil.NewLittleEndianReader(v.Data) + file.ModTime = mft.ConvertFileTime(r.Uint64(0x08)) + } + } + if file.Name != "" { + canAdd := fn(file.Name, file.IsDir) + if canAdd { + result = append(result, file) + } + if canAdd || file.IsDir { + fileMap[win32api.DWORDLONG(file.Node)] = FileEntry{ + Name: file.Name, + Parent: win32api.DWORDLONG(parent), + Type: 0, + } + } + } + } + } + } + + (*reflect.SliceHeader)(unsafe.Pointer(&result)).Cap = len(result) + for k, v := range result { + result[k].Path = GetFullUsnPath(" ", fileMap, win32api.DWORDLONG(v.Node)) + } + fileMap = nil + runtime.GC() + return result, nil +} + +func GetFileListsFromMftFile(filepath string) ([]MFTFile, error) { + return GetFileListsFromMftFileFn(filepath, func(string, bool) bool { return true }) +} diff --git a/ntfs/binutil/binutil.go b/ntfs/binutil/binutil.go new file mode 100644 index 0000000..08afb02 --- /dev/null +++ b/ntfs/binutil/binutil.go @@ -0,0 +1,111 @@ +// Package binutil contains some helpful utilities for reading binary data from byte slices. +package binutil + +import "encoding/binary" + +// Duplicate creates a full copy of the input byte slice. +func Duplicate(in []byte) []byte { + out := make([]byte, len(in)) + copy(out, in) + return out +} + +// IsOnlyZeroes return true when the input data is all bytes of zero value and false if any of the bytes has a nonzero +// value. +func IsOnlyZeroes(data []byte) bool { + for _, b := range data { + if b != 0 { + return false + } + } + return true +} + +// BinReader helps to read data from a byte slice using an offset and a data length (instead two offsets when using +// a slice expression). For example b[2:4] yields the same as Read(2, 2) using a BinReader over b. Also some convenient +// methods are provided to read integer values using a binary.ByteOrder from the slice directly. +// +// Note that methods that return a []byte may not necessarily copy the data, so modifying the returned slice may also +// affect the data in the BinReader. +// +// Methods will panic when any offset or length is outside of the bounds of the original data. +type BinReader struct { + data []byte + bo binary.ByteOrder +} + +// NewBinReader creates a BinReader over data using the specified binary.ByteOrder. The data slice is stored directly, +// no copy is made, so modifying the original slice will also affect the returned BinReader. +func NewBinReader(data []byte, bo binary.ByteOrder) *BinReader { + return &BinReader{data: data, bo: bo} +} + +// NewLittleEndianReader creates a BinReader over data using binary.LittleEndian. The data slice is stored directly, +// no copy is made, so modifying the original slice will also affect the returned BinReader. +func NewLittleEndianReader(data []byte) *BinReader { + return NewBinReader(data, binary.LittleEndian) +} + +// NewLittleEndianReader creates a BinReader over data using binary.BigEndian. The data slice is stored directly, +// no copy is made, so modifying the original slice will also affect the returned BinReader. +func NewBigEndianReader(data []byte, bo binary.ByteOrder) *BinReader { + return NewBinReader(data, binary.BigEndian) +} + +// Data returns all data inside this BinReader. +func (r *BinReader) Data() []byte { + return r.data +} + +// ByteOrder returns the ByteOrder for this BinReader. +func (r *BinReader) ByteOrder() binary.ByteOrder { + return r.bo +} + +// Length returns the length of the contained data. +func (r *BinReader) Length() int { + return len(r.data) +} + +// Read reads an amount of bytes as specified by length from the provided offset. The returned slice's length is the +// same as the specified length. +func (r *BinReader) Read(offset int, length int) []byte { + return r.data[offset : offset+length] +} + +// Reader returns a new BinReader over the data read by Read(offset, length) using the same ByteOrder as this reader. +// There is no guarantee a copy of the data is made, so modifying the new reader's data may affect the original. +func (r *BinReader) Reader(offset int, length int) *BinReader { + return &BinReader{data: r.data[offset : offset+length], bo: r.bo} +} + +// Byte returns the byte at the position indicated by the offset. +func (r *BinReader) Byte(offset int) byte { + return r.Read(offset, 1)[0] +} + +// ReadFrom returns all data starting at the specified offset. +func (r *BinReader) ReadFrom(offset int) []byte { + return r.data[offset:] +} + +// ReaderFrom returns a BinReader over the data read by ReadFrom(offset) using the same ByteOrder as this reader. +// There is no guarantee a copy of the data is made, so modifying the new reader's data may affect the original. +func (r *BinReader) ReaderFrom(offset int) *BinReader { + return &BinReader{data: r.data[offset:], bo: r.bo} +} + +// Uint16 reads 2 bytes from the provided offset and parses them into a uint16 using the provided ByteOrder. +func (r *BinReader) Uint16(offset int) uint16 { + return r.bo.Uint16(r.Read(offset, 2)) +} + +// Uint32 reads 4 bytes from the provided offset and parses them into a uint32 using the provided ByteOrder. +func (r *BinReader) Uint32(offset int) uint32 { + return r.bo.Uint32(r.Read(offset, 4)) +} + +// Uint64 reads 8 bytes from the provided offset and parses them into a uint64 using the provided ByteOrder. +func (r *BinReader) Uint64(offset int) uint64 { + return r.bo.Uint64(r.Read(offset, 8)) +} diff --git a/ntfs/bootsect/bootsect.go b/ntfs/bootsect/bootsect.go new file mode 100644 index 0000000..b52a98f --- /dev/null +++ b/ntfs/bootsect/bootsect.go @@ -0,0 +1,72 @@ +/* + Package bootsect provides functions to parse the boot sector (also sometimes called Volume Boot Record, VBR, or + $Boot file) of an NTFS volume. +*/ +package bootsect + +import ( + "fmt" + + "b612.me/wincmd/ntfs/binutil" +) + +// BootSector represents the parsed data of an NTFS boot sector. The OemId should typically be "NTFS " ("NTFS" +// followed by 4 trailing spaces) for a valid NTFS boot sector. +type BootSector struct { + OemId string + BytesPerSector int + SectorsPerCluster int + MediaDescriptor byte + SectorsPerTrack int + NumberofHeads int + HiddenSectors int + TotalSectors uint64 + MftClusterNumber uint64 + MftMirrorClusterNumber uint64 + FileRecordSegmentSizeInBytes int + IndexBufferSizeInBytes int + VolumeSerialNumber []byte +} + +// Parse parses the data of an NTFS boot sector into a BootSector structure. +func Parse(data []byte) (BootSector, error) { + if len(data) < 80 { + return BootSector{}, fmt.Errorf("boot sector data should be at least 80 bytes but is %d", len(data)) + } + r := binutil.NewLittleEndianReader(data) + bytesPerSector := int(r.Uint16(0x0B)) + sectorsPerCluster := int(int8(r.Byte(0x0D))) + if sectorsPerCluster < 0 { + // Quoth Wikipedia: The number of sectors in a cluster. If the value is negative, the amount of sectors is 2 + // to the power of the absolute value of this field. + sectorsPerCluster = 1 << -sectorsPerCluster + } + bytesPerCluster := bytesPerSector * sectorsPerCluster + return BootSector{ + OemId: string(r.Read(0x03, 8)), + BytesPerSector: bytesPerSector, + SectorsPerCluster: sectorsPerCluster, + MediaDescriptor: r.Byte(0x15), + SectorsPerTrack: int(r.Uint16(0x18)), + NumberofHeads: int(r.Uint16(0x1A)), + HiddenSectors: int(r.Uint16(0x1C)), + TotalSectors: r.Uint64(0x28), + MftClusterNumber: r.Uint64(0x30), + MftMirrorClusterNumber: r.Uint64(0x38), + FileRecordSegmentSizeInBytes: bytesOrClustersToBytes(r.Byte(0x40), bytesPerCluster), + IndexBufferSizeInBytes: bytesOrClustersToBytes(r.Byte(0x44), bytesPerCluster), + VolumeSerialNumber: binutil.Duplicate(r.Read(0x48, 8)), + }, nil +} + +func bytesOrClustersToBytes(b byte, bytesPerCluster int) int { + // From Wikipedia: + // A positive value denotes the number of clusters in a File Record Segment. A negative value denotes the amount of + // bytes in a File Record Segment, in which case the size is 2 to the power of the absolute value. + // (0xF6 = -10 → 210 = 1024). + i := int(int8(b)) + if i < 0 { + return 1 << -i + } + return i * bytesPerCluster +} diff --git a/ntfs/cmd/example/main.go b/ntfs/cmd/example/main.go new file mode 100644 index 0000000..2c82a21 --- /dev/null +++ b/ntfs/cmd/example/main.go @@ -0,0 +1,141 @@ +package main + +import ( + "b612.me/starlog" + "b612.me/wincmd" + "b612.me/wincmd/ntfs/mft" + "b612.me/wincmd/ntfs/utf16" + "encoding/binary" + "errors" + "fmt" + "io" + "log" + _ "net/http/pprof" + "regexp" + "runtime" + "time" +) + +func main() { + f, size, err := mft.GetMFTFile(`C:\`) + if err != nil { + panic(err) + } + recordSize := int64(1024) + i := int64(0) + starlog.Infoln("start size is", size) + alreadyGot := int64(0) + maxRecordSize := size / recordSize + if maxRecordSize > 1024 { + maxRecordSize = 1024 + } + for { + for { + if (size - alreadyGot) < maxRecordSize*recordSize { + maxRecordSize-- + } else { + break + } + } + if maxRecordSize < 10 { + maxRecordSize = 1 + } + buf := make([]byte, maxRecordSize*recordSize) + got, err := io.ReadFull(f, buf) + if err != nil { + if errors.Is(err, io.EOF) { + break + } + log.Fatalln("Unable to read record data", err) + } + alreadyGot += int64(got) + for j := int64(0); j < 1024*maxRecordSize; j += 1024 { + record, err := mft.ParseRecord(buf[j : j+1024]) + if err != nil { + // fmt.Println("Unable to parse MFT record", err) + continue + } + + fname := record.FindAttributes(mft.AttributeTypeFileName) + + for _, v := range fname { + if (record.Flags&mft.RecordFlagIsIndex)>>3 == 0 && record.Flags&mft.RecordFlagInUse == 1 { + filename := utf16.DecodeString(v.Data[66:], binary.LittleEndian) + //data:=utf16.DecodeString(v.Data, binary.LittleEndian) + if filename != "" { + attbr := record.FindAttributes(mft.AttributeTypeStandardInformation) + if len(attbr) == 0 { + break + } + stand, _ := mft.ParseStandardInformation(attbr[0].Data) + if stand.FileAttributes&mft.FileAttributeNormal == 1 { + i++ + } + //oo:=record.FindAttributes(mft.AttributeTypeData) + //if len(oo)!=0 { + // fmt.Println(oo[0].ActualSize) + //} + //r := binutil.NewLittleEndianReader(v.Data[:8]) + //fmt.Println(record.FileReference.ToUint64(), r.Uint64(0)) + //fmt.Println(filename) + //log.Println("Read MFT record") + + } + } + } + } + } + starlog.Infoln(i) + + //go http.ListenAndServe("0.0.0.0:8888", nil) + starlog.Debugln("开始获取Windows磁盘列表") + lists, err := wincmd.ListDrivers() + if err != nil { + panic(err) + } + info := []wincmd.DiskInfo{} + for _, v := range lists { + data, err := wincmd.GetDiskInfo(v) + if err != nil { + continue + } + fmt.Println("获取到磁盘信息:", data) + info = append(info, data) + } + starlog.Warningln("忽略非NTFS磁盘") + var m runtime.MemStats + for _, v := range info { + if v.Format == "NTFS" && v.Driver == `C:\` { + + starlog.Infoln("开始获取NTFS USN日志，磁盘：", v.Driver) + fileLists, err := wincmd.ListUsnFileFn(v.Driver, func(name string, typed uint8) bool { + return true + if ok, _ := regexp.MatchString(`\.exe$`, name); ok { + return true + } + return false + }) + if err != nil { + starlog.Panicln("获取失败", err) + } + fmt.Println(len(fileLists)) + return + for k, _ := range fileLists { + fmt.Println(k) + } + runtime.ReadMemStats(&m) + log.Printf("Alloc = %v TotalAlloc = %v Sys = %v NumGC = %v\n", m.Alloc/1024, m.TotalAlloc/1024, m.Sys/1024, m.NumGC) + runtime.GC() + p := time.After(time.Second * 86400) + for { + select { + case <-p: + return + case <-time.After(time.Second * 10): + runtime.ReadMemStats(&m) + log.Printf("Alloc = %v TotalAlloc = %v Sys = %v NumGC = %v\n", m.Alloc/1024, m.TotalAlloc/1024, m.Sys/1024, m.NumGC) + } + } + } + } +} diff --git a/ntfs/cmd/mftdump/main.go b/ntfs/cmd/mftdump/main.go new file mode 100644 index 0000000..8a635ed --- /dev/null +++ b/ntfs/cmd/mftdump/main.go @@ -0,0 +1,251 @@ +package main + +import ( + "b612.me/wincmd/ntfs/bootsect" + "b612.me/wincmd/ntfs/fragment" + "b612.me/wincmd/ntfs/mft" + "flag" + "fmt" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "time" + +) + +const supportedOemId = "NTFS " + +const ( + exitCodeUserError int = iota + 2 + exitCodeFunctionalError + exitCodeTechnicalError +) + +const isWin = runtime.GOOS == "windows" + +var ( + // flags + verbose = false + overwriteOutputIfExists = false + showProgress = false +) + +func main() { + start := time.Now() + verboseFlag := flag.Bool("v", false, "verbose; print details about what's going on") + forceFlag := flag.Bool("f", false, "force; overwrite the output file if it already exists") + progressFlag := flag.Bool("p", false, "progress; show progress during dumping") + + flag.Usage = printUsage + flag.Parse() + + verbose = *verboseFlag + overwriteOutputIfExists = *forceFlag + showProgress = *progressFlag + args := flag.Args() + + if len(args) != 2 { + printUsage() + os.Exit(exitCodeUserError) + return + } + + volume := args[0] + if isWin { + volume = `\\.\` + volume + } + //fmt.Println(volume) + outfile := args[1] + + in, err := os.Open(volume) + if err != nil { + fatalf(exitCodeTechnicalError, "Unable to open volume using path %s: %v\n", volume, err) + } + defer in.Close() + + printVerbose("Reading boot sector\n") + bootSectorData := make([]byte, 512) + _, err = io.ReadFull(in, bootSectorData) + if err != nil { + fatalf(exitCodeTechnicalError, "Unable to read boot sector: %v\n", err) + } + + printVerbose("Read %d bytes of boot sector, parsing boot sector\n", len(bootSectorData)) + bootSector, err := bootsect.Parse(bootSectorData) + if err != nil { + fatalf(exitCodeTechnicalError, "Unable to parse boot sector data: %v\n", err) + } + + if bootSector.OemId != supportedOemId { + fatalf(exitCodeFunctionalError, "Unknown OemId (file system type) %q (expected %q)\n", bootSector.OemId, supportedOemId) + } + + bytesPerCluster := bootSector.BytesPerSector * bootSector.SectorsPerCluster + mftPosInBytes := int64(bootSector.MftClusterNumber) * int64(bytesPerCluster) + + _, err = in.Seek(mftPosInBytes, 0) + if err != nil { + fatalf(exitCodeTechnicalError, "Unable to seek to MFT position: %v\n", err) + } + + mftSizeInBytes := bootSector.FileRecordSegmentSizeInBytes + printVerbose("Reading $MFT file record at position %d (size: %d bytes)\n", mftPosInBytes, mftSizeInBytes) + mftData := make([]byte, mftSizeInBytes) + _, err = io.ReadFull(in, mftData) + if err != nil { + fatalf(exitCodeTechnicalError, "Unable to read $MFT record: %v\n", err) + } + + printVerbose("Parsing $MFT file record\n") + record, err := mft.ParseRecord(mftData) + if err != nil { + fatalf(exitCodeTechnicalError, "Unable to parse $MFT record: %v\n", err) + } + + printVerbose("Reading $DATA attribute in $MFT file record\n") + dataAttributes := record.FindAttributes(mft.AttributeTypeData) + if len(dataAttributes) == 0 { + fatalf(exitCodeTechnicalError, "No $DATA attribute found in $MFT record\n") + } + + if len(dataAttributes) > 1 { + fatalf(exitCodeTechnicalError, "More than 1 $DATA attribute found in $MFT record\n") + } + + dataAttribute := dataAttributes[0] + if dataAttribute.Resident { + fatalf(exitCodeTechnicalError, "Don't know how to handle resident $DATA attribute in $MFT record\n") + } + + dataRuns, err := mft.ParseDataRuns(dataAttribute.Data) + if err != nil { + fatalf(exitCodeTechnicalError, "Unable to parse dataruns in $MFT $DATA record: %v\n", err) + } + + if len(dataRuns) == 0 { + fatalf(exitCodeTechnicalError, "No dataruns found in $MFT $DATA record\n") + } + + fragments := mft.DataRunsToFragments(dataRuns, bytesPerCluster) + totalLength := int64(0) + for _, frag := range fragments { + totalLength += int64(frag.Length) + } + + out, err := openOutputFile(outfile) + if err != nil { + fatalf(exitCodeFunctionalError, "Unable to open output file: %v\n", err) + } + defer out.Close() + + printVerbose("Copying %d bytes (%s) of data to %s\n", totalLength, formatBytes(totalLength), outfile) + n, err := copy(out, fragment.NewReader(in, fragments), totalLength) + if err != nil { + fatalf(exitCodeTechnicalError, "Error copying data to output file: %v\n", err) + } + + if n != totalLength { + fatalf(exitCodeTechnicalError, "Expected to copy %d bytes, but copied only %d\n", totalLength, n) + } + end := time.Now() + dur := end.Sub(start) + printVerbose("Finished in %v\n", dur) +} + +func copy(dst io.Writer, src io.Reader, totalLength int64) (written int64, err error) { + buf := make([]byte, 1024*1024) + if !showProgress { + return io.CopyBuffer(dst, src, buf) + } + + onePercent := float64(totalLength) / float64(100.0) + totalSize := formatBytes(totalLength) + + // Below copied from io.copyBuffer (https://golang.org/src/io/io.go?s=12796:12856#L380) + for { + printProgress(written, totalSize, onePercent) + + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + printProgress(written, totalSize, onePercent) + fmt.Println() + return written, err +} + +func printProgress(n int64, totalSize string, onePercent float64) { + percentage := float64(n) / onePercent + barCount := int(percentage / 2.0) + spaceCount := 50 - barCount + fmt.Printf("\r[%s%s] %.2f%% (%s / %s) ", strings.Repeat("|", barCount), strings.Repeat(" ", spaceCount), percentage, formatBytes(n), totalSize) +} + +func openOutputFile(outfile string) (*os.File, error) { + if overwriteOutputIfExists { + return os.Create(outfile) + } else { + return os.OpenFile(outfile, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0666) + } +} + +func printUsage() { + out := os.Stderr + exe := filepath.Base(os.Args[0]) + fmt.Fprintf(out, "\nusage: %s [flags] \n\n", exe) + fmt.Fprintln(out, "Dump the MFT of a volume to a file. The volume should be NTFS formatted.") + fmt.Fprintln(out, "\nFlags:") + + flag.PrintDefaults() + + fmt.Fprintf(out, "\nFor example: ") + if isWin { + fmt.Fprintf(out, "%s -v -f C: D:\\c.mft\n", exe) + } else { + fmt.Fprintf(out, "%s -v -f /dev/sdb1 ~/sdb1.mft\n", exe) + } +} + +func fatalf(exitCode int, format string, v ...interface{}) { + fmt.Printf(format, v...) + os.Exit(exitCode) +} + +func printVerbose(format string, v ...interface{}) { + if verbose { + fmt.Printf(format, v...) + } +} + +func formatBytes(b int64) string { + if b < 1024 { + return fmt.Sprintf("%dB", b) + } + if b < 1048576 { + return fmt.Sprintf("%.2fKiB", float32(b)/float32(1024)) + } + if b < 1073741824 { + return fmt.Sprintf("%.2fMiB", float32(b)/float32(1048576)) + } + return fmt.Sprintf("%.2fGiB", float32(b)/float32(1073741824)) +} diff --git a/ntfs/fragment/reader.go b/ntfs/fragment/reader.go new file mode 100644 index 0000000..a0eacb5 --- /dev/null +++ b/ntfs/fragment/reader.go @@ -0,0 +1,83 @@ +/* + Package fragment contains a Reader which can read Fragments which may be scattered around a volume (and perhaps even + not in sequence). Typically these could be translated from MFT attribute DataRuns. To convert MFT attribute DataRuns + to Fragments for use in the fragment Reader, use mft.DataRunsToFragments(). + + Implementation notes + + When the fragment Reader is near the end of a fragment and a Read() call requests more data than what is left in + the current fragment, the Reader will exhaust only the current fragment and return that data (which could be less + than len(p)). A next Read() call will then seek to the next fragment and continue reading there. When the last + fragment is exhausted by a Read(), it will return the remaining bytes read and a nil error. Any subsequent Read() + calls after that will return 0, io.EOF. + + When accessing a new fragment, the Reader will seek using the absolute Length in the fragment from the start + of the contained io.ReadSeeker (using io.SeekStart). +*/ +package fragment + +import ( + "fmt" + "io" + "os" +) + +// Fragment contains an absolute Offset in bytes from the start of a volume and a Length of the fragment, also in bytes. +type Fragment struct { + Offset int64 + Length int64 +} + +// A fragment Reader will read data from the fragments in order. When one fragment is depleted, it will seek to the +// position of the next fragment and continue reading from there, until all fragments have been exhausted. When the last +// fragment has been exhaused, each subsequent Read() will return io.EOF. +type Reader struct { + src io.ReadSeeker + fragments []Fragment + idx int + remaining int64 + file *os.File +} + +// NewReader initializes a new Reader from the io.ReaderSeeker and fragments and returns a pointer to. Note that +// fragments may not be sequential in order, so the io.ReadSeeker should support seeking backwards (or rather, from the +// start). +func NewReader(src io.ReadSeeker, fragments []Fragment) *Reader { + return &Reader{src: src, fragments: fragments, idx: -1, remaining: 0} +} + +func (r *Reader) Read(p []byte) (n int, err error) { + if r.idx >= len(r.fragments) { + r.src.(*os.File).Close() + return 0, io.EOF + } + + if len(p) == 0 { + return 0, nil + } + + if r.remaining == 0 { + r.idx++ + if r.idx >= len(r.fragments) { + return 0, io.EOF + } + next := r.fragments[r.idx] + r.remaining = next.Length + seeked, err := r.src.Seek(next.Offset, io.SeekStart) + if err != nil { + return 0, fmt.Errorf("unable to seek to next offset %d: %v", next.Offset, err) + } + if seeked != next.Offset { + return 0, fmt.Errorf("wanted to seek to %d but reached %d", next.Offset, seeked) + } + } + + target := p + if int64(len(p)) > r.remaining { + target = p[:r.remaining] + } + + n, err = io.ReadFull(r.src, target) + r.remaining -= int64(n) + return n, err +} diff --git a/ntfs/mft/attributes.go b/ntfs/mft/attributes.go new file mode 100644 index 0000000..1b40373 --- /dev/null +++ b/ntfs/mft/attributes.go @@ -0,0 +1,343 @@ +package mft + +import ( + "encoding/binary" + "fmt" + "time" + + "b612.me/wincmd/ntfs/binutil" + "b612.me/wincmd/ntfs/utf16" +) + +var ( + reallyStrangeEpoch = time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC) +) + +// StandardInformation represents the data contained in a $STANDARD_INFORMATION attribute. +type StandardInformation struct { + Creation time.Time + FileLastModified time.Time + MftLastModified time.Time + LastAccess time.Time + FileAttributes FileAttribute + MaximumNumberOfVersions uint32 + VersionNumber uint32 + ClassId uint32 + OwnerId uint32 + SecurityId uint32 + QuotaCharged uint64 + UpdateSequenceNumber uint64 +} + +// ParseStandardInformation parses the data of a $STANDARD_INFORMATION attribute's data (type +// AttributeTypeStandardInformation) into StandardInformation. Note that no additional correctness checks are done, so +// it's up to the caller to ensure the passed data actually represents a $STANDARD_INFORMATION attribute's data. +func ParseStandardInformation(b []byte) (StandardInformation, error) { + if len(b) < 48 { + return StandardInformation{}, fmt.Errorf("expected at least %d bytes but got %d", 48, len(b)) + } + + r := binutil.NewLittleEndianReader(b) + ownerId := uint32(0) + securityId := uint32(0) + quotaCharged := uint64(0) + updateSequenceNumber := uint64(0) + if len(b) >= 0x30+4 { + ownerId = r.Uint32(0x30) + } + if len(b) >= 0x34+4 { + securityId = r.Uint32(0x34) + } + if len(b) >= 0x38+8 { + quotaCharged = r.Uint64(0x38) + } + if len(b) >= 0x40+8 { + updateSequenceNumber = r.Uint64(0x40) + } + return StandardInformation{ + Creation: ConvertFileTime(r.Uint64(0x00)), + FileLastModified: ConvertFileTime(r.Uint64(0x08)), + MftLastModified: ConvertFileTime(r.Uint64(0x10)), + LastAccess: ConvertFileTime(r.Uint64(0x18)), + FileAttributes: FileAttribute(r.Uint32(0x20)), + MaximumNumberOfVersions: r.Uint32(0x24), + VersionNumber: r.Uint32(0x28), + ClassId: r.Uint32(0x2C), + OwnerId: ownerId, + SecurityId: securityId, + QuotaCharged: quotaCharged, + UpdateSequenceNumber: updateSequenceNumber, + }, nil +} + +// FileAttribute represents a bit mask of various file attributes. +type FileAttribute uint32 + +// Bit values for FileAttribute. For example, a normal, hidden file has value 0x0082. +const ( + FileAttributeReadOnly FileAttribute = 0x0001 + FileAttributeHidden FileAttribute = 0x0002 + FileAttributeSystem FileAttribute = 0x0004 + FileAttributeArchive FileAttribute = 0x0020 + FileAttributeDevice FileAttribute = 0x0040 + FileAttributeNormal FileAttribute = 0x0080 + FileAttributeTemporary FileAttribute = 0x0100 + FileAttributeSparseFile FileAttribute = 0x0200 + FileAttributeReparsePoint FileAttribute = 0x0400 + FileAttributeCompressed FileAttribute = 0x1000 + FileAttributeOffline FileAttribute = 0x1000 + FileAttributeNotContentIndexed FileAttribute = 0x2000 + FileAttributeEncrypted FileAttribute = 0x4000 +) + +// Is checks if this FileAttribute's bit mask contains the specified attribute value. +func (a *FileAttribute) Is(c FileAttribute) bool { + return *a&c == c +} + +// FileNameNamespace indicates the namespace of a $FILE_NAME attribute's file name. +type FileNameNamespace byte + +const ( + FileNameNamespacePosix FileNameNamespace = 0 + FileNameNamespaceWin32 FileNameNamespace = 1 + FileNameNamespaceDos FileNameNamespace = 2 + FileNameNamespaceWin32Dos FileNameNamespace = 3 +) + +// FileName represents the data of a $FILE_NAME attribute. ParentFileReference points to the MFT record that is the +// parent (ie. containing directory of this file). The AllocatedSize and ActualSize may be zero, in which case the file +// size may be found in a $DATA attribute instead (it could also be the ActualSize is zero, while the AllocatedSize does +// contain a value). +type FileName struct { + ParentFileReference FileReference + Creation time.Time + FileLastModified time.Time + MftLastModified time.Time + LastAccess time.Time + AllocatedSize uint64 + ActualSize uint64 + Flags FileAttribute + ExtendedData uint32 + Namespace FileNameNamespace + Name string +} + +// ParseFileName parses the data of a $FILE_NAME attribute's data (type AttributeTypeFileName) into FileName. Note that +// no additional correctness checks are done, so it's up to the caller to ensure the passed data actually represents a +// $FILE_NAME attribute's data. +func ParseFileName(b []byte) (FileName, error) { + if len(b) < 66 { + return FileName{}, fmt.Errorf("expected at least %d bytes but got %d", 66, len(b)) + } + + fileNameLength := int(b[0x40 : 0x40+1][0]) * 2 + minExpectedSize := 66 + fileNameLength + if len(b) < minExpectedSize { + return FileName{}, fmt.Errorf("expected at least %d bytes but got %d", minExpectedSize, len(b)) + } + + r := binutil.NewLittleEndianReader(b) + parentRef, err := ParseFileReference(r.Read(0x00, 8)) + if err != nil { + return FileName{}, fmt.Errorf("unable to parse file reference: %v", err) + } + return FileName{ + ParentFileReference: parentRef, + Creation: ConvertFileTime(r.Uint64(0x08)), + FileLastModified: ConvertFileTime(r.Uint64(0x10)), + MftLastModified: ConvertFileTime(r.Uint64(0x18)), + LastAccess: ConvertFileTime(r.Uint64(0x20)), + AllocatedSize: r.Uint64(0x28), + ActualSize: r.Uint64(0x30), + Flags: FileAttribute(r.Uint32(0x38)), + ExtendedData: r.Uint32(0x3c), + Namespace: FileNameNamespace(r.Byte(0x41)), + Name: utf16.DecodeString(r.Read(0x42, fileNameLength), binary.LittleEndian), + }, nil +} + +// AttributeListEntry represents an entry in an $ATTRIBUTE_LIST attribute. The Type indicates the attribute type, while +// the BaseRecordReference indicates which MFT record the attribute is located in (ie. an "extension record", if it is +// not the same as the one where the $ATTRIBUTE_LIST is located). +type AttributeListEntry struct { + Type AttributeType + Name string + StartingVCN uint64 + BaseRecordReference FileReference + AttributeId uint16 +} + +// ParseAttributeList parses the data of a $ATTRIBUTE_LIST attribute's data (type AttributeTypeAttributeList) into a +// list of AttributeListEntry. Note that no additional correctness checks are done, so it's up to the caller to ensure +// the passed data actually represents a $ATTRIBUTE_LIST attribute's data. +func ParseAttributeList(b []byte) ([]AttributeListEntry, error) { + if len(b) < 26 { + return []AttributeListEntry{}, fmt.Errorf("expected at least %d bytes but got %d", 26, len(b)) + } + + entries := make([]AttributeListEntry, 0) + + for len(b) > 0 { + r := binutil.NewLittleEndianReader(b) + entryLength := int(r.Uint16(0x04)) + if len(b) < entryLength { + return entries, fmt.Errorf("expected at least %d bytes remaining for AttributeList entry but is %d", entryLength, len(b)) + } + nameLength := int(r.Byte(0x06)) + name := "" + if nameLength != 0 { + nameOffset := int(r.Byte(0x07)) + name = utf16.DecodeString(r.Read(nameOffset, nameLength*2), binary.LittleEndian) + } + baseRef, err := ParseFileReference(r.Read(0x10, 8)) + if err != nil { + return entries, fmt.Errorf("unable to parse base record reference: %v", err) + } + entry := AttributeListEntry{ + Type: AttributeType(r.Uint32(0)), + Name: name, + StartingVCN: r.Uint64(0x08), + BaseRecordReference: baseRef, + AttributeId: r.Uint16(0x18), + } + entries = append(entries, entry) + b = r.ReadFrom(entryLength) + } + return entries, nil +} + +// CollationType indicates how the entries in an index should be ordered. +type CollationType uint32 + +const ( + CollationTypeBinary CollationType = 0x00000000 + CollationTypeFileName CollationType = 0x00000001 + CollationTypeUnicodeString CollationType = 0x00000002 + CollationTypeNtofsULong CollationType = 0x00000010 + CollationTypeNtofsSid CollationType = 0x00000011 + CollationTypeNtofsSecurityHash CollationType = 0x00000012 + CollationTypeNtofsUlongs CollationType = 0x00000013 +) + +// IndexRoot represents the data (header and entries) of an $INDEX_ROOT attribute, which typically is the root of a +// directory's B+tree index containing file names of the directory (but could be use for other types of indices, too). +// The AttributeType is the type of attributes that are contained in the entries (currently only $FILE_NAME attributes +// are supported). +type IndexRoot struct { + AttributeType AttributeType + CollationType CollationType + BytesPerRecord uint32 + ClustersPerRecord uint32 + Flags uint32 + Entries []IndexEntry +} + +// IndexEntry represents an entry in an B+tree index. Currently only $FILE_NAME attribute entries are supported. The +// FileReference points to the MFT record of the indexed file. +type IndexEntry struct { + FileReference FileReference + Flags uint32 + FileName FileName + SubNodeVCN uint64 +} + +// ParseIndexRoot parses the data of a $INDEX_ROOT attribute's data (type AttributeTypeIndexRoot) into +// IndexRoot. Note that no additional correctness checks are done, so it's up to the caller to ensure the passed data +// actually represents a $INDEX_ROOT attribute's data. +func ParseIndexRoot(b []byte) (IndexRoot, error) { + if len(b) < 32 { + return IndexRoot{}, fmt.Errorf("expected at least %d bytes but got %d", 32, len(b)) + } + r := binutil.NewLittleEndianReader(b) + attributeType := AttributeType(r.Uint32(0x00)) + if attributeType != AttributeTypeFileName { + return IndexRoot{}, fmt.Errorf("unable to handle attribute type %d (%s) in $INDEX_ROOT", attributeType, attributeType.Name()) + } + + uTotalSize := r.Uint32(0x14) + if int64(uTotalSize) > maxInt { + return IndexRoot{}, fmt.Errorf("index root size %d overflows maximum int value %d", uTotalSize, maxInt) + } + totalSize := int(uTotalSize) + expectedSize := totalSize + 16 + if len(b) < expectedSize { + return IndexRoot{}, fmt.Errorf("expected %d bytes in $INDEX_ROOT but is %d", expectedSize, len(b)) + } + entries := []IndexEntry{} + if totalSize >= 16 { + parsed, err := parseIndexEntries(r.Read(0x20, totalSize-16)) + if err != nil { + return IndexRoot{}, fmt.Errorf("error parsing index entries: %v", err) + } + entries = parsed + } + + return IndexRoot{ + AttributeType: attributeType, + CollationType: CollationType(r.Uint32(0x04)), + BytesPerRecord: r.Uint32(0x08), + ClustersPerRecord: r.Uint32(0x0C), + Flags: r.Uint32(0x1C), + Entries: entries, + }, nil +} + +func parseIndexEntries(b []byte) ([]IndexEntry, error) { + if len(b) < 13 { + return []IndexEntry{}, fmt.Errorf("expected at least %d bytes but got %d", 13, len(b)) + } + entries := make([]IndexEntry, 0) + for len(b) > 0 { + r := binutil.NewLittleEndianReader(b) + entryLength := int(r.Uint16(0x08)) + + if len(b) < entryLength { + return entries, fmt.Errorf("index entry length indicates %d bytes but got %d", entryLength, len(b)) + } + + flags := r.Uint32(0x0C) + pointsToSubNode := flags&0b1 != 0 + isLastEntryInNode := flags&0b10 != 0 + contentLength := int(r.Uint16(0x0A)) + + fileName := FileName{} + if contentLength != 0 && !isLastEntryInNode { + parsedFileName, err := ParseFileName(r.Read(0x10, contentLength)) + if err != nil { + return entries, fmt.Errorf("error parsing $FILE_NAME record in index entry: %v", err) + } + fileName = parsedFileName + } + subNodeVcn := uint64(0) + if pointsToSubNode { + subNodeVcn = r.Uint64(entryLength - 8) + } + + fileReference, err := ParseFileReference(r.Read(0x00, 8)) + if err != nil { + return entries, fmt.Errorf("unable to file reference: %v", err) + } + entry := IndexEntry{ + FileReference: fileReference, + Flags: flags, + FileName: fileName, + SubNodeVCN: subNodeVcn, + } + entries = append(entries, entry) + b = r.ReadFrom(entryLength) + } + return entries, nil +} + +// ConvertFileTime converts a Windows "file time" to a time.Time. A "file time" is a 64-bit value that represents the +// number of 100-nanosecond intervals that have elapsed since 12:00 A.M. January 1, 1601 Coordinated Universal Time +// (UTC). See also: https://docs.microsoft.com/en-us/windows/win32/sysinfo/file-times +func ConvertFileTime(timeValue uint64) time.Time { + dur := time.Duration(int64(timeValue)) + r := time.Date(1601, time.January, 1, 0, 0, 0, 0, time.UTC) + for i := 0; i < 100; i++ { + r = r.Add(dur) + } + return r +} diff --git a/ntfs/mft/mft.go b/ntfs/mft/mft.go new file mode 100644 index 0000000..f57cd22 --- /dev/null +++ b/ntfs/mft/mft.go @@ -0,0 +1,462 @@ +/* + Package mft provides functions to parse records and their attributes in an NTFS Master File Table ("MFT" for short). + + Basic usage + + First parse a record using mft.ParseRecord(), which parses the record header and the attribute headers. Then parse + each attribute's data individually using the various mft.Parse...() functions. + // Error handling left out for brevity + record, err := mft.ParseRecord() + attrs, err := record.FindAttributes(mft.AttributeTypeFileName) + fileName, err := mft.ParseFileName(attrs[0]) +*/ +package mft + +import ( + "bytes" + "encoding/binary" + "fmt" + + "b612.me/wincmd/ntfs/binutil" + "b612.me/wincmd/ntfs/fragment" + "b612.me/wincmd/ntfs/utf16" +) + +var ( + fileSignature = []byte{0x46, 0x49, 0x4c, 0x45} +) + +const maxInt = int64(^uint(0) >> 1) + +// A Record represents an MFT entry, excluding all technical data (such as "offset to first attribute"). The Attributes +// list only contains the attribute headers and raw data; the attribute data has to be parsed separately. When this is a +// base record, the BaseRecordReference will be zero. When it is an extension record, the BaseRecordReference points to +// the record's base record. +type Record struct { + Signature []byte + FileReference FileReference + BaseRecordReference FileReference + LogFileSequenceNumber uint64 + HardLinkCount int + Flags RecordFlag + ActualSize uint32 + AllocatedSize uint32 + NextAttributeId int + Attributes []Attribute +} + +// ParseRecord parses bytes into a Record after applying fixup. The data is assumed to be in Little Endian order. Only +// the attribute headers are parsed, not the actual attribute data. +func ParseRecord(b []byte) (Record, error) { + if len(b) < 42 { + return Record{}, fmt.Errorf("record data length should be at least 42 but is %d", len(b)) + } + sig := b[:4] + if bytes.Compare(sig, fileSignature) != 0 { + return Record{}, fmt.Errorf("unknown record signature: %# x", sig) + } + + b = binutil.Duplicate(b) + r := binutil.NewLittleEndianReader(b) + baseRecordRef, err := ParseFileReference(r.Read(0x20, 8)) + if err != nil { + return Record{}, fmt.Errorf("unable to parse base record reference: %v", err) + } + + firstAttributeOffset := int(r.Uint16(0x14)) + if firstAttributeOffset < 0 || firstAttributeOffset >= len(b) { + return Record{}, fmt.Errorf("invalid first attribute offset %d (data length: %d)", firstAttributeOffset, len(b)) + } + + updateSequenceOffset := int(r.Uint16(0x04)) + updateSequenceSize := int(r.Uint16(0x06)) + b, err = applyFixUp(b, updateSequenceOffset, updateSequenceSize) + if err != nil { + return Record{}, fmt.Errorf("unable to apply fixup: %v", err) + } + + attributes, err := ParseAttributes(b[firstAttributeOffset:]) + if err != nil { + return Record{}, err + } + return Record{ + Signature: binutil.Duplicate(sig), + FileReference: FileReference{RecordNumber: uint64(r.Uint32(0x2C)), SequenceNumber: r.Uint16(0x10)}, + BaseRecordReference: baseRecordRef, + LogFileSequenceNumber: r.Uint64(0x08), + HardLinkCount: int(r.Uint16(0x12)), + Flags: RecordFlag(r.Uint16(0x16)), + ActualSize: r.Uint32(0x18), + AllocatedSize: r.Uint32(0x1C), + NextAttributeId: int(r.Uint16(0x28)), + Attributes: attributes, + }, nil +} + +// A FileReference represents a reference to an MFT record. Since the FileReference in a Record is only 4 bytes, the +// RecordNumber will probably not exceed 32 bits. +type FileReference struct { + RecordNumber uint64 + SequenceNumber uint16 +} + +func (f FileReference) ToUint64() uint64 { + origin := make([]byte, 8) + binary.LittleEndian.PutUint16(origin, f.SequenceNumber) + origin[6] = origin[0] + origin[7] = origin[1] + binary.LittleEndian.PutUint32(origin, uint32(f.RecordNumber)) + return binary.LittleEndian.Uint64(origin) +} + +// ParseFileReference parses a Little Endian ordered 8-byte slice into a FileReference. The first 6 bytes indicate the +// record number, while the final 2 bytes indicate the sequence number. +func ParseFileReference(b []byte) (FileReference, error) { + if len(b) != 8 { + return FileReference{}, fmt.Errorf("expected 8 bytes but got %d", len(b)) + } + + return FileReference{ + RecordNumber: binary.LittleEndian.Uint64(padTo(b[:6], 8)), + SequenceNumber: binary.LittleEndian.Uint16(b[6:]), + }, nil +} + +// RecordFlag represents a bit mask flag indicating the status of the MFT record. +type RecordFlag uint16 + +// Bit values for the RecordFlag. For example, an in-use directory has value 0x0003. +const ( + RecordFlagInUse RecordFlag = 0x0001 + RecordFlagIsDirectory RecordFlag = 0x0002 + RecordFlagInExtend RecordFlag = 0x0004 + RecordFlagIsIndex RecordFlag = 0x0008 +) + +// Is checks if this RecordFlag's bit mask contains the specified flag. +func (f *RecordFlag) Is(c RecordFlag) bool { + return *f&c == c +} + +func applyFixUp(b []byte, offset int, length int) ([]byte, error) { + r := binutil.NewLittleEndianReader(b) + + updateSequence := r.Read(offset, length*2) // length is in pairs, not bytes + updateSequenceNumber := updateSequence[:2] + updateSequenceArray := updateSequence[2:] + + sectorCount := len(updateSequenceArray) / 2 + sectorSize := len(b) / sectorCount + + for i := 1; i <= sectorCount; i++ { + offset := sectorSize*i - 2 + if bytes.Compare(updateSequenceNumber, b[offset:offset+2]) != 0 { + return nil, fmt.Errorf("update sequence mismatch at pos %d", offset) + } + } + + for i := 0; i < sectorCount; i++ { + offset := sectorSize*(i+1) - 2 + num := i * 2 + copy(b[offset:offset+2], updateSequenceArray[num:num+2]) + } + + return b, nil +} + +// FindAttributes returns all attributes of the specified type contained in this record. When no matches are found an +// empty slice is returned. +func (r *Record) FindAttributes(attrType AttributeType) []Attribute { + ret := make([]Attribute, 0) + for _, a := range r.Attributes { + if a.Type == attrType { + ret = append(ret, a) + } + } + return ret +} + +// Attribute represents an MFT record attribute header and its corresponding raw attribute Data (excluding header data). +// When the attribute is Resident, the Data contains the actual attribute's data. When the attribute is non-resident, +// the Data contains DataRuns pointing to the actual data. DataRun data can be parsed using ParseDataRuns(). +type Attribute struct { + Type AttributeType + Resident bool + Name string + Flags AttributeFlags + AttributeId int + AllocatedSize uint64 + ActualSize uint64 + Data []byte +} + +// AttributeType represents the type of an Attribute. Use Name() to get the attribute type's name. +type AttributeType uint32 + +// Known values for AttributeType. Note that other values might occur too. +const ( + AttributeTypeStandardInformation AttributeType = 0x10 // $STANDARD_INFORMATION; always resident + AttributeTypeAttributeList AttributeType = 0x20 // $ATTRIBUTE_LIST; mixed residency + AttributeTypeFileName AttributeType = 0x30 // $FILE_NAME; always resident + AttributeTypeObjectId AttributeType = 0x40 // $OBJECT_ID; always resident + AttributeTypeSecurityDescriptor AttributeType = 0x50 // $SECURITY_DESCRIPTOR; always resident? + AttributeTypeVolumeName AttributeType = 0x60 // $VOLUME_NAME; always resident? + AttributeTypeVolumeInformation AttributeType = 0x70 // $VOLUME_INFORMATION; never resident? + AttributeTypeData AttributeType = 0x80 // $DATA; mixed residency + AttributeTypeIndexRoot AttributeType = 0x90 // $INDEX_ROOT; always resident + AttributeTypeIndexAllocation AttributeType = 0xa0 // $INDEX_ALLOCATION; never resident? + AttributeTypeBitmap AttributeType = 0xb0 // $BITMAP; nearly always resident? + AttributeTypeReparsePoint AttributeType = 0xc0 // $REPARSE_POINT; always resident? + AttributeTypeEAInformation AttributeType = 0xd0 // $EA_INFORMATION; always resident + AttributeTypeEA AttributeType = 0xe0 // $EA; nearly always resident? + AttributeTypePropertySet AttributeType = 0xf0 // $PROPERTY_SET + AttributeTypeLoggedUtilityStream AttributeType = 0x100 // $LOGGED_UTILITY_STREAM; always resident + AttributeTypeTerminator AttributeType = 0xFFFFFFFF // Indicates the last attribute in a list; will not actually be returned by ParseAttributes +) + +// AttributeFlags represents a bit mask flag indicating various properties of an attribute's data. +type AttributeFlags uint16 + +// Bit values for the AttributeFlags. For example, an encrypted, compressed attribute has value 0x4001. +const ( + AttributeFlagsCompressed AttributeFlags = 0x0001 + AttributeFlagsEncrypted AttributeFlags = 0x4000 + AttributeFlagsSparse AttributeFlags = 0x8000 +) + +// Is checks if this AttributeFlags's bit mask contains the specified flag. +func (f *AttributeFlags) Is(c AttributeFlags) bool { + return *f&c == c +} + +// ParseAttributes parses bytes into Attributes. The data is assumed to be in Little Endian order. Only the attribute +// headers are parsed, not the actual attribute data. +func ParseAttributes(b []byte) ([]Attribute, error) { + if len(b) == 0 { + return []Attribute{}, nil + } + attributes := make([]Attribute, 0) + for len(b) > 0 { + if len(b) < 4 { + return nil, fmt.Errorf("attribute header data should be at least 4 bytes but is %d", len(b)) + } + + r := binutil.NewLittleEndianReader(b) + attrType := r.Uint32(0) + if attrType == uint32(AttributeTypeTerminator) { + break + } + + if len(b) < 8 { + return nil, fmt.Errorf("cannot read attribute header record length, data should be at least 8 bytes but is %d", len(b)) + } + + uRecordLength := r.Uint32(0x04) + if int64(uRecordLength) > maxInt { + return nil, fmt.Errorf("record length %d overflows maximum int value %d", uRecordLength, maxInt) + } + recordLength := int(uRecordLength) + if recordLength <= 0 { + return nil, fmt.Errorf("cannot handle attribute with zero or negative record length %d", recordLength) + } + + if recordLength > len(b) { + return nil, fmt.Errorf("attribute record length %d exceeds data length %d", recordLength, len(b)) + } + + recordData := r.Read(0, recordLength) + attribute, err := ParseAttribute(recordData) + if err != nil { + return nil, err + } + attributes = append(attributes, attribute) + b = r.ReadFrom(recordLength) + } + return attributes, nil +} + +// ParseAttribute parses bytes into an Attribute. The data is assumed to be in Little Endian order. Only the attribute +// headers are parsed, not the actual attribute data. +func ParseAttribute(b []byte) (Attribute, error) { + if len(b) < 22 { + return Attribute{}, fmt.Errorf("attribute data should be at least 22 bytes but is %d", len(b)) + } + + r := binutil.NewLittleEndianReader(b) + + nameLength := r.Byte(0x09) + nameOffset := r.Uint16(0x0A) + + name := "" + if nameLength != 0 { + nameBytes := r.Read(int(nameOffset), int(nameLength)*2) + name = utf16.DecodeString(nameBytes, binary.LittleEndian) + } + + resident := r.Byte(0x08) == 0x00 + var attributeData []byte + actualSize := uint64(0) + allocatedSize := uint64(0) + if resident { + dataOffset := int(r.Uint16(0x14)) + uDataLength := r.Uint32(0x10) + if int64(uDataLength) > maxInt { + return Attribute{}, fmt.Errorf("attribute data length %d overflows maximum int value %d", uDataLength, maxInt) + } + dataLength := int(uDataLength) + expectedDataLength := dataOffset + dataLength + + if len(b) < expectedDataLength { + return Attribute{}, fmt.Errorf("expected attribute data length to be at least %d but is %d", expectedDataLength, len(b)) + } + + attributeData = r.Read(dataOffset, dataLength) + } else { + dataOffset := int(r.Uint16(0x20)) + if len(b) < dataOffset { + return Attribute{}, fmt.Errorf("expected attribute data length to be at least %d but is %d", dataOffset, len(b)) + } + allocatedSize = r.Uint64(0x28) + actualSize = r.Uint64(0x30) + attributeData = r.ReadFrom(int(dataOffset)) + } + + return Attribute{ + Type: AttributeType(r.Uint32(0)), + Resident: resident, + Name: name, + Flags: AttributeFlags(r.Uint16(0x0C)), + AttributeId: int(r.Uint16(0x0E)), + AllocatedSize: allocatedSize, + ActualSize: actualSize, + Data: binutil.Duplicate(attributeData), + }, nil +} + +// A DataRun represents a fragment of data somewhere on a volume. The OffsetCluster, which can be negative, is relative +// to a previous DataRun's offset. The OffsetCluster of the first DataRun in a list is relative to the beginning of the +// volume. +type DataRun struct { + OffsetCluster int64 + LengthInClusters uint64 +} + +// ParseDataRuns parses bytes into a list of DataRuns. Each DataRun's OffsetCluster is relative to the DataRun before +// it. The first element's OffsetCluster is relative to the beginning of the volume. +func ParseDataRuns(b []byte) ([]DataRun, error) { + if len(b) == 0 { + return []DataRun{}, nil + } + + runs := make([]DataRun, 0) + for len(b) > 0 { + r := binutil.NewLittleEndianReader(b) + header := r.Byte(0) + if header == 0 { + break + } + + lengthLength := int(header &^ 0xF0) + offsetLength := int(header >> 4) + + dataRunDataLength := offsetLength + lengthLength + + headerAndDataLength := dataRunDataLength + 1 + if len(b) < headerAndDataLength { + return nil, fmt.Errorf("expected at least %d bytes of datarun data but is %d", headerAndDataLength, len(b)) + } + + dataRunData := r.Reader(1, dataRunDataLength) + + lengthBytes := dataRunData.Read(0, lengthLength) + dataLength := binary.LittleEndian.Uint64(padTo(lengthBytes, 8)) + + offsetBytes := dataRunData.Read(lengthLength, offsetLength) + dataOffset := int64(binary.LittleEndian.Uint64(padTo(offsetBytes, 8))) + + runs = append(runs, DataRun{OffsetCluster: dataOffset, LengthInClusters: dataLength}) + + b = r.ReadFrom(headerAndDataLength) + } + + return runs, nil +} + +// DataRunsToFragments transform a list of DataRuns with relative offsets and lengths specified in cluster into a list +// of fragment.Fragment elements with absolute offsets and lengths specified in bytes (for example for use in a +// fragment.Reader). Note that data will probably not align to a cluster exactly so there could be some padding at the +// end. It is up to the user of the Fragments to limit reads to actual data size (eg. by using an io.LimitedReader or +// modifying the last element in the list to limit its length). +func DataRunsToFragments(runs []DataRun, bytesPerCluster int) []fragment.Fragment { + frags := make([]fragment.Fragment, len(runs)) + previousOffsetCluster := int64(0) + for i, run := range runs { + exactClusterOffset := previousOffsetCluster + run.OffsetCluster + frags[i] = fragment.Fragment{ + Offset: exactClusterOffset * int64(bytesPerCluster), + Length: int64(run.LengthInClusters) * int64(bytesPerCluster), + } + previousOffsetCluster = exactClusterOffset + } + return frags +} + +func padTo(data []byte, length int) []byte { + if len(data) > length { + return data + } + if len(data) == length { + return data + } + result := make([]byte, length) + if len(data) == 0 { + return result + } + copy(result, data) + if data[len(data)-1]&0b10000000 == 0b10000000 { + for i := len(data); i < length; i++ { + result[i] = 0xFF + } + } + return result +} + +// Name returns a string representation of the attribute type. For example "$STANDARD_INFORMATION" or "$FILE_NAME". For +// anyte attribute type which is unknown, Name will return "unknown". +func (at AttributeType) Name() string { + switch at { + case AttributeTypeStandardInformation: + return "$STANDARD_INFORMATION" + case AttributeTypeAttributeList: + return "$ATTRIBUTE_LIST" + case AttributeTypeFileName: + return "$FILE_NAME" + case AttributeTypeObjectId: + return "$OBJECT_ID" + case AttributeTypeSecurityDescriptor: + return "$SECURITY_DESCRIPTOR" + case AttributeTypeVolumeName: + return "$VOLUME_NAME" + case AttributeTypeVolumeInformation: + return "$VOLUME_INFORMATION" + case AttributeTypeData: + return "$DATA" + case AttributeTypeIndexRoot: + return "$INDEX_ROOT" + case AttributeTypeIndexAllocation: + return "$INDEX_ALLOCATION" + case AttributeTypeBitmap: + return "$BITMAP" + case AttributeTypeReparsePoint: + return "$REPARSE_POINT" + case AttributeTypeEAInformation: + return "$EA_INFORMATION" + case AttributeTypeEA: + return "$EA" + case AttributeTypePropertySet: + return "$PROPERTY_SET" + case AttributeTypeLoggedUtilityStream: + return "$LOGGED_UTILITY_STREAM" + } + return "unknown" +} diff --git a/ntfs/mft/output.go b/ntfs/mft/output.go new file mode 100644 index 0000000..c5b27c6 --- /dev/null +++ b/ntfs/mft/output.go @@ -0,0 +1,187 @@ +package mft + +import ( + "b612.me/wincmd/ntfs/bootsect" + "b612.me/wincmd/ntfs/fragment" + "bytes" + "fmt" + "github.com/t9t/gomft/mft" + "io" + "os" + "runtime" +) + +const supportedOemId = "NTFS " + + +const isWin = runtime.GOOS == "windows" + +func GetMFTFileBytes(volume string) ([]byte, error) { + reader, length, err := GetMFTFile(volume) + if err != nil { + return nil, err + } + buf := make([]byte, length) + bfio := bytes.NewBuffer(buf) + written, err := copyBytes(bfio, reader, length) + if written != length { + return nil, fmt.Errorf("Write Not Ok,Should %d got %d", length, written) + } + return bfio.Bytes(), nil +} + +func DumpMFTFile(volume, filepath string, fn func(int64, int64, float64)) error { + reader, length, err := GetMFTFile(volume) + if err != nil { + return err + } + out, err := os.Create(filepath) + if err != nil { + return err + } + defer out.Close() + written, err := copyFiles(out, reader, length, fn) + if written != length { + return fmt.Errorf("Write Not Ok,Should %d got %d", length, written) + } + return nil +} + +func GetMFTFile(volume string) (io.Reader, int64, error) { + if isWin { + volume = `\\.\` + volume[:len(volume)-1] + } + in, err := os.Open(volume) + if err != nil { + return nil, 0, err + } + bootSectorData := make([]byte, 512) + _, err = io.ReadFull(in, bootSectorData) + if err != nil { + return nil, 0, fmt.Errorf("Unable to read boot sector: %v\n", err) + } + + bootSector, err := bootsect.Parse(bootSectorData) + if err != nil { + return nil, 0, fmt.Errorf("Unable to parse boot sector data: %v\n", err) + } + + if bootSector.OemId != supportedOemId { + return nil, 0, fmt.Errorf("Unknown OemId (file system type) %q (expected %q)\n", bootSector.OemId, supportedOemId) + } + + bytesPerCluster := bootSector.BytesPerSector * bootSector.SectorsPerCluster + mftPosInBytes := int64(bootSector.MftClusterNumber) * int64(bytesPerCluster) + + _, err = in.Seek(mftPosInBytes, 0) + if err != nil { + return nil, 0, fmt.Errorf("Unable to seek to MFT position: %v\n", err) + } + + mftSizeInBytes := bootSector.FileRecordSegmentSizeInBytes + mftData := make([]byte, mftSizeInBytes) + _, err = io.ReadFull(in, mftData) + if err != nil { + return nil, 0, fmt.Errorf("Unable to read $MFT record: %v\n", err) + } + + record, err := mft.ParseRecord(mftData) + if err != nil { + return nil, 0, fmt.Errorf("Unable to parse $MFT record: %v\n", err) + } + + dataAttributes := record.FindAttributes(mft.AttributeTypeData) + if len(dataAttributes) == 0 { + return nil, 0, fmt.Errorf("No $DATA attribute found in $MFT record\n") + } + + if len(dataAttributes) > 1 { + return nil, 0, fmt.Errorf("More than 1 $DATA attribute found in $MFT record\n") + } + + dataAttribute := dataAttributes[0] + if dataAttribute.Resident { + return nil, 0, fmt.Errorf("Don't know how to handle resident $DATA attribute in $MFT record\n") + } + + dataRuns, err := ParseDataRuns(dataAttribute.Data) + if err != nil { + return nil, 0, fmt.Errorf("Unable to parse dataruns in $MFT $DATA record: %v\n", err) + } + + if len(dataRuns) == 0 { + return nil, 0, fmt.Errorf("No dataruns found in $MFT $DATA record\n") + } + + fragments := DataRunsToFragments(dataRuns, bytesPerCluster) + totalLength := int64(0) + for _, frag := range fragments { + totalLength += int64(frag.Length) + } + + return fragment.NewReader(in, fragments), totalLength, nil +} + +func copyBytes(dst io.Writer, src io.Reader, totalLength int64) (written int64, err error) { + buf := make([]byte, 1024*1024) + + // Below copied from io.copyBuffer (https://golang.org/src/io/io.go?s=12796:12856#L380) + for { + + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + return written, err +} + +func copyFiles(dst io.Writer, src io.Reader, totalLength int64, fn func(int64, int64, float64)) (written int64, err error) { + buf := make([]byte, 1024*1024) + onePercent := float64(written) / float64(totalLength) * float64(100.0) + + // Below copied from io.copyBuffer (https://golang.org/src/io/io.go?s=12796:12856#L380) + for { + fn(written, totalLength, onePercent) + nr, er := src.Read(buf) + if nr > 0 { + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er != nil { + if er != io.EOF { + err = er + } + break + } + } + fn(written, totalLength, onePercent) + return written, err +} diff --git a/ntfs/utf16/decode.go b/ntfs/utf16/decode.go new file mode 100644 index 0000000..198ffb9 --- /dev/null +++ b/ntfs/utf16/decode.go @@ -0,0 +1,18 @@ +package utf16 + +import ( + "encoding/binary" + "unicode/utf16" +) + +// Decode the input data as UTF-16 using the provided byte order and convert the result to a string. The input data +// length must be a multiple of 2. DecodeString will panic if that is not the case. +func DecodeString(b []byte, bo binary.ByteOrder) string { + slen := len(b) / 2 + shorts := make([]uint16, slen) + for i := 0; i < slen; i++ { + bi := i * 2 + shorts[i] = bo.Uint16(b[bi : bi+2]) + } + return string(utf16.Decode(shorts)) +} diff --git a/os.go b/usn.go similarity index 86% rename from os.go rename to usn.go index 89a934d..2408ca9 100644 --- a/os.go +++ b/usn.go @@ -4,6 +4,7 @@ import ( "b612.me/stario" "b612.me/win32api" "fmt" + "os" "reflect" "runtime" "syscall" @@ -221,7 +222,7 @@ func ListUsnFile(driver string) (map[win32api.DWORDLONG]FileEntry, error) { } } -func ListUsnFileFn(driver string, searchFn func(string, uint8) bool) (map[win32api.DWORDLONG]FileEntry, error) { +func ListUsnFileFn(driver string, searchFn func(string, bool) bool) (map[win32api.DWORDLONG]FileEntry, error) { fileMap := make(map[win32api.DWORDLONG]FileEntry) pDriver := "\\\\.\\" + driver[:len(driver)-1] fd, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL) @@ -257,7 +258,7 @@ func ListUsnFileFn(driver string, searchFn func(string, uint8) bool) (map[win32a if ur.FileAttributes&win32api.FILE_ATTRIBUTE_DIRECTORY != 0 { typed = 1 } - if typed == 1 || searchFn(fn, typed) { + if typed == 1 || searchFn(fn, typed == 1) { // fmt.Println("len", ur.FileNameLength, ur.FileNameOffset, "fn", fn) fileMap[ur.FileReferenceNumber] = FileEntry{Name: fn, Parent: ur.ParentFileReferenceNumber, Type: typed} } @@ -314,37 +315,37 @@ const ( NO_FOLDER ) -func ListNTFSDriverFilesFn(diskName string, searchFn func(string, uint8) bool) ([]string, error) { +func ListNTFSUsnDriverFilesFn(diskName string, searchFn func(string, bool) bool) ([]string, error) { var result []string data, err := ListUsnFileFn(diskName, searchFn) if err != nil { return result, err } - return listNTFSDriverFiles(diskName, searchFn, data) + return listNTFSUsnDriverFiles(diskName, searchFn, data) } -func ListNTFSDriverFiles(diskName string, folder uint8) ([]string, error) { +func ListNTFSUsnDriverFiles(diskName string, folder uint8) ([]string, error) { var result []string data, err := ListUsnFile(diskName) if err != nil { return result, err } - return listNTFSDriverFiles(diskName, func(name string, tp uint8) bool { - if tp == 0 && folder == ONLY_FOLDER { + return listNTFSUsnDriverFiles(diskName, func(name string, tp bool) bool { + if !tp && folder == ONLY_FOLDER { return false } - if tp == 1 && folder == NO_FOLDER { + if tp && folder == NO_FOLDER { return false } return true }, data) } -func listNTFSDriverFiles(diskName string, fn func(string, uint8) bool, data map[win32api.DWORDLONG]FileEntry) ([]string, error) { +func listNTFSUsnDriverFiles(diskName string, fn func(string, bool) bool, data map[win32api.DWORDLONG]FileEntry) ([]string, error) { result := make([]string, len(data)) i := 0 for k, v := range data { - if !fn(v.Name, v.Type) { + if !fn(v.Name, v.Type == 1) { continue } name := GetFullUsnPath(diskName, data, k) @@ -359,70 +360,92 @@ func listNTFSDriverFiles(diskName string, fn func(string, uint8) bool, data map[ return result, nil } -func ListNTFSDriverInfoFn(diskName string, searchFn func(string, uint8) bool) ([]*FileStat, error) { +func ListNTFSUsnDriverInfoFn(diskName string, searchFn func(string, bool) bool) ([]FileStat, error) { data, err := ListUsnFileFn(diskName, searchFn) if err != nil { return nil, err } - return listNTFSDriverInfo(diskName, searchFn, data) + return listNTFSUsnDriverInfo(diskName, searchFn, data) } -func ListNTFSDriverInfo(diskName string, folder uint8) ([]*FileStat, error) { +func ListNTFSUsnDriverInfo(diskName string, folder uint8) ([]FileStat, error) { data, err := ListUsnFile(diskName) if err != nil { return nil, err } - return listNTFSDriverInfo(diskName, func(name string, tp uint8) bool { - if tp == 0 && folder == ONLY_FOLDER { + return listNTFSUsnDriverInfo(diskName, func(name string, tp bool) bool { + if !tp && folder == ONLY_FOLDER { return false } - if tp == 1 && folder == NO_FOLDER { + if tp && folder == NO_FOLDER { return false } return true }, data) } -func listNTFSDriverInfo(diskName string, fn func(string, uint8) bool, data map[win32api.DWORDLONG]FileEntry) ([]*FileStat, error) { +func listNTFSUsnDriverInfo(diskName string, fn func(string, bool) bool, data map[win32api.DWORDLONG]FileEntry) ([]FileStat, error) { + //fmt.Println("finished 1") pDriver := "\\\\.\\" + diskName[:len(diskName)-1] fd, err := CreateFile(pDriver, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL) if err != nil { return nil, err } defer syscall.Close(fd) - result := make([]*FileStat, len(data)) - i := int (0) - wg := stario.NewWaitGroup(1000) + result := make([]FileStat, len(data)) + i := int(0) + wg := stario.NewWaitGroup(100) for k, v := range data { - if !fn(v.Name, v.Type) { + if !fn(v.Name, v.Type == 1) { continue } wg.Add(1) go func(k win32api.DWORDLONG, v FileEntry, i int) { defer wg.Done() //now := time.Now().UnixNano() - fd2, err := OpenFileByIdWithfd(fd, k, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL) + /* + fd2, err := OpenFileByIdWithfd(fd, k, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL) + if err != nil { + return + } + //fmt.Println("cost", float64((time.Now().UnixNano()-now)/1000000)) + var info syscall.ByHandleFileInformation + err = syscall.GetFileInformationByHandle(fd2, &info) + syscall.Close(fd2) + //fmt.Println("cost", float64((time.Now().UnixNano()-now)/1000000)) + if err != nil { + return + } + + */ + path := GetFullUsnPath(diskName, data, k) + fileInfo, err := os.Stat(path) if err != nil { return } - //fmt.Println("cost", float64((time.Now().UnixNano()-now)/1000000)) - var info syscall.ByHandleFileInformation - err = syscall.GetFileInformationByHandle(fd2, &info) - syscall.Close(fd2) - //fmt.Println("cost", float64((time.Now().UnixNano()-now)/1000000)) - if err != nil { - return + fs := fileInfo.Sys().(*syscall.Win32FileAttributeData) + stat := FileStat{ + FileAttributes: fs.FileAttributes, + CreationTime: fs.CreationTime, + LastAccessTime: fs.LastAccessTime, + LastWriteTime: fs.LastWriteTime, + FileSizeHigh: fs.FileSizeHigh, + FileSizeLow: fs.FileSizeLow, } - path := GetFullUsnPath(diskName, data, k) - result[i] = newFileStatFromInformation(&info, v.Name, path) + stat.name = v.Name + stat.path = path + return + result[i] = stat + //result[i] = newFileStatFromInformation(&info, v.Name, path) }(k, v, i) i++ } wg.Wait() + //fmt.Println("finished 2") (*reflect.SliceHeader)(unsafe.Pointer(&result)).Cap = i (*reflect.SliceHeader)(unsafe.Pointer(&result)).Len = i data = nil - data = make(map[win32api.DWORDLONG]FileEntry, 0) + //data = make(map[win32api.DWORDLONG]FileEntry, 0) runtime.GC() return result, nil } @@ -516,12 +539,12 @@ func MonitorUsnChange(driver string, rec chan FileMonitor) error { } } -func GetUsnFileInfo(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, id win32api.DWORDLONG) (*FileStat, error) { +func GetUsnFileInfo(diskName string, fileMap map[win32api.DWORDLONG]FileEntry, id win32api.DWORDLONG) (FileStat, error) { name := fileMap[id].Name path := GetFullUsnPath(diskName, fileMap, id) fd, err := OpenFileById(diskName, id, syscall.O_RDONLY, win32api.FILE_ATTRIBUTE_NORMAL) if err != nil { - return nil, err + return FileStat{}, err } var info syscall.ByHandleFileInformation err = syscall.GetFileInformationByHandle(fd, &info)