diff --git a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs index 3745a4a..58c69f5 100644 --- a/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs +++ b/ValvePak/ValvePak.Test/MultiChunkWriteTest.cs @@ -34,13 +34,13 @@ public async Task WriteMultiChunkPackage() using (var package = new Package()) { package.SetFileName(dirPath); - package.WriteChunkSize = 1024; + package.WriteChunkSize = 800; for (var i = 0; i < 10; i++) { var entry = package.AddFile($"files/chunked_{i}.bin", CreateTestData(400, (byte)i), multiChunk: true); - // 400 byte files with a 1024 chunk size, so three files per chunk + // 400 byte files with an 800 byte chunk size: a chunk file at exactly 800 bytes is not full yet, so each one holds three files await Assert.That(entry.ArchiveIndex).IsEqualTo((ushort)(i / 3)); } @@ -274,17 +274,17 @@ public async Task AddFileThrowsWhenExceedingChunkLimit() package.SetFileName(TempPath("limit_dir.vpk")); package.WriteChunkSize = 1; - // With a chunk size of 1 every file starts a new chunk + // With a chunk size of 1 every file of 2 bytes starts a new chunk PackageEntry? entry = null; for (var i = 0; i < 0x7FFF; i++) { - entry = package.AddFile($"{i}.bin", [1], multiChunk: true); + entry = package.AddFile($"{i}.bin", [1, 2], multiChunk: true); } await Assert.That(entry!.ArchiveIndex).IsEqualTo((ushort)0x7FFE); - await Assert.That(() => package.AddFile("one too many.bin", [1], multiChunk: true)).ThrowsExactly() + await Assert.That(() => package.AddFile("one too many.bin", [1, 2], multiChunk: true)).ThrowsExactly() .WithMessageContaining("maximum amount of chunk files", StringComparison.Ordinal); } @@ -492,7 +492,7 @@ public async Task WritesAgainAfterAddingMoreFiles() public async Task WriteChunkSizeValidation() { using var package = new Package(); - await Assert.That(package.WriteChunkSize).IsEqualTo(200 * 1024 * 1024); + await Assert.That(package.WriteChunkSize).IsEqualTo(100 * 1024 * 1024); await Assert.That(() => package.WriteChunkSize = 0).ThrowsExactly(); await Assert.That(() => package.WriteChunkSize = -1).ThrowsExactly(); } diff --git a/ValvePak/ValvePak.Test/PackageTest.cs b/ValvePak/ValvePak.Test/PackageTest.cs index 4fd7063..68cc3be 100644 --- a/ValvePak/ValvePak.Test/PackageTest.cs +++ b/ValvePak/ValvePak.Test/PackageTest.cs @@ -524,6 +524,44 @@ public async Task InvalidWholeFileChecksumThrows() await Assert.That(() => package.VerifyHashes()).ThrowsExactly(); } + [Test] + [Arguments(0, "File tree")] + [Arguments(16, "Archive MD5 entries")] + public async Task SectionChecksumMismatchIsReportedWhenWholeFileChecksumMatches(int checksumOffset, string sectionName) + { + var data = CreateSmallPackageBytes(); + var otherMD5SectionOffset = GetOtherMD5SectionOffset(data); + + data.AsSpan(otherMD5SectionOffset + checksumOffset, 16).Fill(0xAB); + MD5.HashData(data.AsSpan(0, otherMD5SectionOffset + 32)).CopyTo(data, otherMD5SectionOffset + 32); + + using var package = new Package(); + package.SetFileName("a.vpk"); + package.Read(new MemoryStream(data)); + + var progress = new SynchronousProgress(); + + await Assert.That(() => package.VerifyHashes(progress)).ThrowsNothing(); + await Assert.That(progress.Reports).Count().IsEqualTo(1); + await Assert.That(progress.Reports[0]).StartsWith($"{sectionName} checksum mismatch (", StringComparison.Ordinal); + } + + [Test] + public async Task SectionChecksumMismatchThrowsWhenWholeFileChecksumMismatches() + { + var data = CreateSmallPackageBytes(); + var otherMD5SectionOffset = GetOtherMD5SectionOffset(data); + + data.AsSpan(otherMD5SectionOffset, 16).Fill(0xAB); + + using var package = new Package(); + package.SetFileName("a.vpk"); + package.Read(new MemoryStream(data)); + + await Assert.That(() => package.VerifyHashes()).ThrowsExactly() + .WithMessageContaining("File tree checksum mismatch", StringComparison.Ordinal); + } + [Test] public async Task InvalidSignatureFails() { @@ -629,7 +667,7 @@ public async Task ThrowsWhenReadTwice() } [Test] - public async Task VerifyHashesThrowsOnVersion1() + public async Task VerifyHashesSkipsVersion1() { using var package = new Package(); package.SetFileName("a.vpk"); @@ -638,8 +676,10 @@ public async Task VerifyHashesThrowsOnVersion1() using var ms = new MemoryStream([0x34, 0x12, 0xAA, 0x55, 0x01, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00]); package.Read(ms); - await Assert.That(() => package.VerifyHashes()).ThrowsExactly() - .WithMessage("Only version 2 is supported.", StringComparison.Ordinal); + var progress = new SynchronousProgress(); + + await Assert.That(() => package.VerifyHashes(progress)).ThrowsNothing(); + await Assert.That(progress.Reports).IsEquivalentTo(["Version 1 packages have no checksums to verify."]); } [Test] @@ -870,8 +910,10 @@ public async Task ThrowsOnDirectoryTreeEndingInTheMiddleOfString(string tree) } private const int TreeSizeOffset = 8; + private const int FileDataSectionSizeOffset = 12; private const int ArchiveMD5SectionSizeOffset = 16; private const int SignatureSectionSizeOffset = 24; + private const int HeaderSize = 28; private const int EmbeddedPrefixLength = 1000; private const int EmbeddedSuffixLength = 50; @@ -886,6 +928,16 @@ private static byte[] CreateSmallPackageBytes() return output.ToArray(); } + private static int GetOtherMD5SectionOffset(byte[] package) + { + var span = package.AsSpan(); + var treeSize = BinaryPrimitives.ReadInt32LittleEndian(span[TreeSizeOffset..]); + var fileDataSectionSize = BinaryPrimitives.ReadInt32LittleEndian(span[FileDataSectionSizeOffset..]); + var archiveMD5SectionSize = BinaryPrimitives.ReadInt32LittleEndian(span[ArchiveMD5SectionSizeOffset..]); + + return HeaderSize + treeSize + fileDataSectionSize + archiveMD5SectionSize; + } + /// /// Creates a package whose signature section holds the given little-endian integers. /// diff --git a/ValvePak/ValvePak.Test/WriteTest.cs b/ValvePak/ValvePak.Test/WriteTest.cs index cd9078c..48b102e 100644 --- a/ValvePak/ValvePak.Test/WriteTest.cs +++ b/ValvePak/ValvePak.Test/WriteTest.cs @@ -35,9 +35,7 @@ public async Task CreateNewPackage() output.Position = 0; // Verify - using var packageWritten = new Package(); - packageWritten.SetFileName("test.vpk"); - packageWritten.Read(output); + using var packageWritten = ReadBack(output); packageWritten.VerifyHashes(); var newEntry = packageWritten.FindEntry(newName); @@ -76,9 +74,7 @@ public async Task WriteManyFiles() output.Position = 0; // Verify - using var packageWritten = new Package(); - packageWritten.SetFileName("test.vpk"); - packageWritten.Read(output); + using var packageWritten = ReadBack(output); packageWritten.VerifyHashes(); await Assert.That(packageWritten.Entries!["txt"]).Count().IsEqualTo(1000); @@ -118,15 +114,7 @@ await Assert.That(package.RemoveFile(new PackageEntry public async Task SetsSpaces() { using var package = new Package(); - var file = package.AddFile("", []); - using (Assert.Multiple()) - { - await Assert.That(file.TypeName).IsEqualTo(" "); - await Assert.That(file.DirectoryName).IsEqualTo(" "); - await Assert.That(file.FileName).IsEqualTo(""); - await Assert.That(package.Entries!.ContainsKey(" ")).IsTrue(); - await Assert.That(package.Entries[" "][0]).IsEqualTo(file); - } + await Assert.That(() => package.AddFile("", [])).ThrowsExactly(); var file2 = package.AddFile("hello", []); using (Assert.Multiple()) @@ -134,6 +122,8 @@ public async Task SetsSpaces() await Assert.That(file2.TypeName).IsEqualTo(" "); await Assert.That(file2.DirectoryName).IsEqualTo(" "); await Assert.That(file2.FileName).IsEqualTo("hello"); + await Assert.That(package.Entries!.ContainsKey(" ")).IsTrue(); + await Assert.That(package.Entries[" "][0]).IsEqualTo(file2); } var file3 = package.AddFile("hello.txt", []); @@ -194,9 +184,7 @@ public async Task WritesToNonSeekableStream(uint version) } using var output = new MemoryStream(nonSeekable.ToArray()); - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); if (version >= 2) { @@ -259,9 +247,7 @@ public async Task WriteAndVerifyRoundTrip() output.Position = 0; - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); using (Assert.Multiple()) { @@ -292,9 +278,7 @@ public async Task WriteVersion1Package() output.Position = 0; - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); using (Assert.Multiple()) { @@ -327,18 +311,14 @@ public async Task WritePreservesVersionOfReadPackage() output.Position = 0; - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); using var output2 = new MemoryStream(); readBack.Write(output2); output2.Position = 0; - using var readBack2 = new Package(); - readBack2.SetFileName("test.vpk"); - readBack2.Read(output2); + using var readBack2 = ReadBack(output2); await Assert.That(readBack2.Version).IsEqualTo(1u); @@ -428,9 +408,7 @@ public async Task WriteIntoLongerStream(int position, uint version) output.Position = position; - using var readBack = new Package(); - readBack.SetFileName("test.vpk"); - readBack.Read(output); + using var readBack = ReadBack(output); await Assert.That(readBack.Version).IsEqualTo(version); @@ -517,6 +495,35 @@ public async Task AddedFilesCanBeFoundAfterOptimizingForBinarySearch(StringCompa await Assert.That(package.FindEntry("addons/chess/missing.vdf")).IsNull(); } + [Test] + public async Task RenameFile() + { + using var package = new Package(); + package.OptimizeEntriesForBinarySearch(StringComparison.OrdinalIgnoreCase); + var entry = package.AddFile("kitten.jpg", Encoding.UTF8.GetBytes("meow")); + package.AddFile("other.txt", []); + + package.RenameFile(entry, "/Path\\To/Cat.PNG"); + + using (Assert.Multiple()) + { + await Assert.That(entry.GetFullPath()).IsEqualTo("path/to/cat.png"); + await Assert.That(package.FindEntry("kitten.jpg")).IsNull(); + await Assert.That(package.FindEntry("path/to/cat.png")).IsEqualTo(entry); + await Assert.That(package.Entries!.ContainsKey("jpg")).IsFalse(); + await Assert.That(() => package.RenameFile(entry, "other.txt")).ThrowsExactly(); + } + } + + private static Package ReadBack(Stream stream) + { + var package = new Package(); + package.SetFileName("test.vpk"); + package.Read(stream); + + return package; + } + /// /// Write only stream that can not seek, collecting the written bytes. /// diff --git a/ValvePak/ValvePak/Archive.cs b/ValvePak/ValvePak/Archive.cs index 861447a..2500c92 100644 --- a/ValvePak/ValvePak/Archive.cs +++ b/ValvePak/ValvePak/Archive.cs @@ -25,6 +25,11 @@ protected Archive(long origin, long dataOffset) /// protected long Origin { get; } + /// + /// Gets the size in bytes of the archive. + /// + public abstract long Length { get; } + /// /// Reads data at the given offset until the output is full or the end of the archive is reached. /// This method is safe to call from multiple threads. diff --git a/ValvePak/ValvePak/FileArchive.cs b/ValvePak/ValvePak/FileArchive.cs index 5a7668f..9196f4c 100644 --- a/ValvePak/ValvePak/FileArchive.cs +++ b/ValvePak/ValvePak/FileArchive.cs @@ -25,6 +25,8 @@ public FileArchive(SafeFileHandle handle, bool ownsHandle, long origin, long dat OwnsHandle = ownsHandle; } + public override long Length => RandomAccess.GetLength(Handle) - Origin; + public override int Read(long offset, Span output) { var totalRead = 0; diff --git a/ValvePak/ValvePak/Package.Read.cs b/ValvePak/ValvePak/Package.Read.cs index d77c38c..2785af7 100644 --- a/ValvePak/ValvePak/Package.Read.cs +++ b/ValvePak/ValvePak/Package.Read.cs @@ -23,6 +23,7 @@ public partial class Package : IDisposable public void Read(string filename) { ObjectDisposedException.ThrowIf(IsDisposed, this); + ThrowIfFilesAdded(); SetFileName(filename); @@ -44,6 +45,7 @@ public void Read(Stream input) ArgumentNullException.ThrowIfNull(input); ObjectDisposedException.ThrowIf(IsDisposed, this); ThrowIfAlreadyRead(); + ThrowIfFilesAdded(); if (FileName == null) { @@ -121,6 +123,8 @@ public void Read(Stream input) public void ReadEntry(PackageEntry entry, out byte[] output, bool validateCrc = true) { ArgumentNullException.ThrowIfNull(entry); + ObjectDisposedException.ThrowIf(IsDisposed, this); + ThrowIfEntryPastEndOfArchive(entry); output = new byte[entry.TotalLength]; @@ -141,7 +145,7 @@ public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true ArgumentNullException.ThrowIfNull(output); ObjectDisposedException.ThrowIf(IsDisposed, this); - var totalLength = (int)entry.TotalLength; + var totalLength = (long)entry.Length + entry.SmallData.Length; if (output.Length < totalLength) { @@ -169,7 +173,7 @@ public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true return; } - var actualChecksum = Crc32.HashToUInt32(output.AsSpan(0, totalLength)); + var actualChecksum = Crc32.HashToUInt32(output.AsSpan(0, (int)totalLength)); if (entry.CRC32 != actualChecksum) { @@ -177,6 +181,22 @@ public void ReadEntry(PackageEntry entry, byte[] output, bool validateCrc = true } } + private void ThrowIfEntryPastEndOfArchive(PackageEntry entry) + { + if (entry.Length == 0) + { + return; + } + + var archive = GetArchive(entry.ArchiveIndex); + var available = archive.Length - archive.DataOffset - entry.Offset; + + if (entry.Length > available) + { + throw new EndOfStreamException($"Archive {entry.ArchiveIndex} ends {Math.Max(available, 0)} bytes after the offset of '{entry.GetFullPath()}', which is {entry.Length} bytes long."); + } + } + /// /// Reads exactly bytes of the directory tree and parses the entries from them. /// @@ -288,13 +308,22 @@ private Dictionary> ParseEntries(ReadOnlySpan t } } + // Types that differ only in case share a list when the dictionary is case insensitive + if (typeEntries.TryGetValue(typeName, out var existingEntries)) + { + existingEntries.AddRange(entries); + entries = existingEntries; + } + else + { + typeEntries.Add(typeName, entries); + } + if (Comparer != null) { // Sorting at the end is faster than doing BinarySearch+Insert entries.Sort(Comparer); } - - typeEntries.Add(typeName, entries); } return typeEntries; @@ -317,6 +346,13 @@ private void ReadArchiveMD5Section() throw new InvalidDataException($"Archive MD5 section size ({ArchiveMD5SectionSize}) is not a multiple of {ChunkHashFraction.SectionEntrySize}."); } + var input = Reader.BaseStream; + + if (ArchiveMD5SectionSize > input.Length - input.Position) + { + throw new InvalidDataException($"Archive MD5 section size ({ArchiveMD5SectionSize}) is larger than the rest of the file ({input.Length - input.Position})."); + } + var entries = (int)(ArchiveMD5SectionSize / ChunkHashFraction.SectionEntrySize); AccessPackFileHashes = new List(entries); @@ -344,17 +380,19 @@ private void ReadArchiveMD5Section() private void ReadOtherMD5Section() { - if (OtherMD5SectionSize != 48) + Debug.Assert(Reader != null); + + if (OtherMD5SectionSize < 48) { + Reader.BaseStream.Position += OtherMD5SectionSize; return; } - Debug.Assert(Reader != null); - TreeChecksum = Reader.ReadBytes(16); ArchiveMD5EntriesChecksum = Reader.ReadBytes(16); FileSizeBeforeWholeFileHash = (uint)(Reader.BaseStream.Position - StreamOffset); WholeFileChecksum = Reader.ReadBytes(16); + Reader.BaseStream.Position += OtherMD5SectionSize - 48; } private void ReadSignatureSection() diff --git a/ValvePak/ValvePak/Package.Save.cs b/ValvePak/ValvePak/Package.Save.cs index 1f535bb..9b91960 100644 --- a/ValvePak/ValvePak/Package.Save.cs +++ b/ValvePak/ValvePak/Package.Save.cs @@ -12,9 +12,10 @@ namespace ValvePak public partial class Package { /// - /// Gets or sets the maximum size in bytes of chunk files created when adding files with multi chunk enabled, - /// see . A new chunk file is started once the current one reaches this size. - /// Defaults to 200 MiB. + /// Gets or sets the size in bytes of chunk files created when adding files with multi chunk enabled, + /// see . A new chunk file is started once the current one is larger than this size, + /// so a chunk file can end past it by up to the size of its last file. + /// Defaults to 100 MiB. /// public int WriteChunkSize { @@ -25,7 +26,7 @@ public int WriteChunkSize field = value; } - } = 200 * 1024 * 1024; + } = 100 * 1024 * 1024; /// /// Size in bytes of the buffer that file data is copied through when it is added to a chunk file. @@ -50,6 +51,7 @@ public int WriteChunkSize public bool RemoveFile(PackageEntry entry) { ArgumentNullException.ThrowIfNull(entry); + ObjectDisposedException.ThrowIf(IsDisposed, this); if (Entries == null) { @@ -71,53 +73,115 @@ public bool RemoveFile(PackageEntry entry) return removed; } + /// + /// Moves an entry in current package to a new path. Only the directory tree changes, the file data is not touched. + /// The path is lowercased like Valve's packer does. + /// + /// The package entry to rename. + /// Full new file path for this entry. + /// The entry is not in this package, another entry already has the new path ignoring case, or the new path has an empty file name or extension or contains a null character. + public void RenameFile(PackageEntry entry, string newFilePath) + { + ArgumentNullException.ThrowIfNull(entry); + ArgumentNullException.ThrowIfNull(newFilePath); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var (directoryName, fileName, typeName) = ParseEntryPath(newFilePath, nameof(newFilePath)); + var existingEntry = FindEntryIgnoringCase(directoryName, fileName, typeName); + + if (existingEntry != null && existingEntry != entry) + { + throw new ArgumentException($"An entry already exists at '{existingEntry.GetFullPath()}'.", nameof(newFilePath)); + } + + if (!RemoveFile(entry)) + { + throw new ArgumentException("The entry is not in this package.", nameof(entry)); + } + + (entry.DirectoryName, entry.FileName, entry.TypeName) = (directoryName, fileName, typeName); + + InsertEntry(entry); + } + + private PackageEntry? FindEntryIgnoringCase(string directoryName, string fileName, string typeName) + { + foreach (var (entryTypeName, typeEntries) in Entries ?? []) + { + if (!entryTypeName.Equals(typeName, StringComparison.OrdinalIgnoreCase)) + { + continue; + } + + var match = typeEntries.Find(entry => entry.FileName.Equals(fileName, StringComparison.OrdinalIgnoreCase) + && entry.DirectoryName.Equals(directoryName, StringComparison.OrdinalIgnoreCase)); + + if (match != null) + { + return match; + } + } + + return null; + } + /// /// Add file to current package. Be careful to not add duplicate entries, because this does not check for duplicates. + /// The path is lowercased like Valve's packer does. /// /// Full file path for this entry. /// File data for this entry. /// If true, the data is written right away into a numbered chunk file (such as "example_001.vpk") instead of being kept in memory for the directory file. /// must be called first with the path of the directory file, the chunk files are created next to it. - /// A new chunk file is started whenever the current one reaches . + /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. + /// The path has an empty file name or extension, or contains a null character. public PackageEntry AddFile(string filePath, byte[] fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); ArgumentNullException.ThrowIfNull(fileData); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var entry = CreateEntry(filePath, nameof(filePath)); // Files with no data are only ever written into the directory file if (!multiChunk || fileData.Length == 0) { - return AddFileToDirectoryFile(filePath, fileData); + return AddFileToDirectoryFile(entry, fileData); } var chunkFile = GetChunkFileForNextFile(); var offset = chunkFile.Length; chunkFile.Write(fileData); - return AddChunkFileEntry(filePath, chunkFile, offset, Crc32.HashToUInt32(fileData)); + return AddChunkFileEntry(entry, chunkFile, offset, Crc32.HashToUInt32(fileData)); } /// /// Add file to current package, reading its data from the current position of the stream to its end. /// Be careful to not add duplicate entries, because this does not check for duplicates. + /// The path is lowercased like Valve's packer does. /// /// Full file path for this entry. /// Stream to read the file data from. It is not disposed. /// If true, the data is copied right away into a numbered chunk file (such as "example_001.vpk") instead of being kept in memory for the directory file. /// must be called first with the path of the directory file, the chunk files are created next to it. - /// A new chunk file is started whenever the current one reaches . + /// A new chunk file is started whenever the current one is larger than . /// Chunk files stay on disk even if the directory file is never written. /// The added entry. + /// The path has an empty file name or extension, or contains a null character. public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = false) { ArgumentNullException.ThrowIfNull(filePath); ArgumentNullException.ThrowIfNull(fileData); + ObjectDisposedException.ThrowIf(IsDisposed, this); + + var entry = CreateEntry(filePath, nameof(filePath)); if (!multiChunk) { - return AddFileToDirectoryFile(filePath, ReadToEnd(fileData)); + return AddFileToDirectoryFile(entry, ReadToEnd(fileData)); } var buffer = ArrayPool.Shared.Rent(AddFileBufferSize); @@ -129,7 +193,7 @@ public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = if (bytesRead == 0) { // Files with no data are only ever written into the directory file - return AddFileToDirectoryFile(filePath, []); + return AddFileToDirectoryFile(entry, []); } var chunkFile = GetChunkFileForNextFile(); @@ -146,7 +210,7 @@ public PackageEntry AddFile(string filePath, Stream fileData, bool multiChunk = } while (bytesRead > 0); - return AddChunkFileEntry(filePath, chunkFile, offset, crc.GetCurrentHashAsUInt32()); + return AddChunkFileEntry(entry, chunkFile, offset, crc.GetCurrentHashAsUInt32()); } finally { @@ -176,9 +240,8 @@ private static byte[] ReadToEnd(Stream stream) /// /// Adds an entry whose data is kept in memory until it is written into the directory file. /// - private PackageEntry AddFileToDirectoryFile(string filePath, byte[] fileData) + private PackageEntry AddFileToDirectoryFile(PackageEntry entry, byte[] fileData) { - var entry = CreateEntry(filePath); entry.SmallData = fileData; entry.CRC32 = Crc32.HashToUInt32(fileData); entry.ArchiveIndex = 0x7FFF; @@ -191,12 +254,11 @@ private PackageEntry AddFileToDirectoryFile(string filePath, byte[] fileData) /// /// Adds an entry for the data that was written to the chunk file since it was at the given offset. /// - private PackageEntry AddChunkFileEntry(string filePath, ChunkFileWriter chunkFile, uint offset, uint crc) + private PackageEntry AddChunkFileEntry(PackageEntry entry, ChunkFileWriter chunkFile, uint offset, uint crc) { // Hand the data to the OS so the entry can be read back before the package is written chunkFile.Flush(); - var entry = CreateEntry(filePath); entry.CRC32 = crc; entry.ArchiveIndex = chunkFile.ArchiveIndex; entry.Offset = offset; @@ -221,7 +283,7 @@ private ChunkFileWriter GetChunkFileForNextFile() if (CurrentChunkFile != null) { - if (CurrentChunkFile.Length < (uint)WriteChunkSize) + if (CurrentChunkFile.Length <= (uint)WriteChunkSize) { return CurrentChunkFile; } @@ -250,9 +312,29 @@ private ChunkFileWriter GetChunkFileForNextFile() /// /// Creates an entry with the directory, file and type names split from the given path. /// - private static PackageEntry CreateEntry(string filePath) + private static PackageEntry CreateEntry(string filePath, string paramName) + { + var (directoryName, fileName, typeName) = ParseEntryPath(filePath, paramName); + + return new PackageEntry + { + DirectoryName = directoryName, + FileName = fileName, + TypeName = typeName, + }; + } + + /// + /// Splits the given path into lowercased directory, file and type names. + /// + private static (string DirectoryName, string FileName, string TypeName) ParseEntryPath(string filePath, string paramName) { - filePath = filePath.Replace(WindowsDirectorySeparator, DirectorySeparatorChar); + if (filePath.Contains('\0', StringComparison.Ordinal)) + { + throw new ArgumentException($"'{filePath}' contains a null character.", paramName); + } + + filePath = ToLowerAscii(filePath.Replace(WindowsDirectorySeparator, DirectorySeparatorChar)); var lastSeparator = filePath.LastIndexOf(DirectorySeparatorChar); var directory = lastSeparator > -1 ? filePath[..lastSeparator] : string.Empty; @@ -272,17 +354,36 @@ private static PackageEntry CreateEntry(string filePath) extension = Space; } + if (fileName.Length == 0 || extension.Length == 0) + { + throw new ArgumentException($"'{filePath}' has an empty file name or extension.", paramName); + } + + directory = directory.Trim(DirectorySeparatorChar); + if (directory.Length == 0) { directory = Space; } - return new PackageEntry + return (directory, fileName, extension); + } + + private static string ToLowerAscii(string value) + { + if (!value.AsSpan().ContainsAnyInRange('A', 'Z')) { - FileName = fileName, - DirectoryName = directory, - TypeName = extension, - }; + return value; + } + + return string.Create(value.Length, value, static (span, source) => + { + for (var i = 0; i < source.Length; i++) + { + var c = source[i]; + span[i] = c is >= 'A' and <= 'Z' ? (char)(c | 0x20) : c; + } + }); } /// @@ -322,6 +423,7 @@ private void InsertEntry(PackageEntry entry) public void Write(string filename) { ArgumentNullException.ThrowIfNull(filename); + ObjectDisposedException.ThrowIf(IsDisposed, this); if (CurrentChunkFile != null) { @@ -337,9 +439,11 @@ public void Write(string filename) } } - using var fs = new FileStream(filename, FileMode.Create, FileAccess.Write, FileShare.None); + using var fs = new FileStream(filename, FileMode.OpenOrCreate, FileAccess.Write, FileShare.None); Write(fs); + + fs.SetLength(fs.Position); } /// @@ -362,17 +466,26 @@ public void Write(Stream stream) } ArgumentNullException.ThrowIfNull(stream); + ObjectDisposedException.ThrowIf(IsDisposed, this); var tree = new Dictionary>>(); // Group the entries by type and directory foreach (var typeEntries in Entries ?? []) { - var typeTree = new Dictionary>(); - tree[typeEntries.Key] = typeTree; - foreach (var entry in typeEntries.Value) { + if (entry.FileName.Length == 0 || entry.TypeName.Length == 0) + { + throw new InvalidOperationException($"Entry '{entry.GetFullPath()}' has an empty file name or extension, which the directory tree can not store."); + } + + if (!tree.TryGetValue(entry.TypeName, out var typeTree)) + { + typeTree = []; + tree[entry.TypeName] = typeTree; + } + var directoryName = entry.DirectoryName.Length == 0 ? Space : entry.DirectoryName; if (!typeTree.TryGetValue(directoryName, out var directoryEntries)) diff --git a/ValvePak/ValvePak/Package.Verify.cs b/ValvePak/ValvePak/Package.Verify.cs index 66ba961..ed0ce1f 100644 --- a/ValvePak/ValvePak/Package.Verify.cs +++ b/ValvePak/ValvePak/Package.Verify.cs @@ -34,9 +34,15 @@ public partial class Package : IDisposable /// /// Verify MD5 hashes provided in the VPK. /// + /// + /// Version 1 packages have no checksums, so nothing is verified for them. + /// The whole file checksum covers the file tree and archive MD5 section checksums, so when it matches, + /// a mismatching file tree or archive MD5 section checksum is reported to instead of thrown. + /// + /// If provided, will report a string for each checksum mismatch that is not thrown, or that a version 1 package has no checksums. /// The package has not been read. - /// The package has no checksums, or a checksum does not match. - public void VerifyHashes() + /// The package has no checksums, or the whole file checksum does not match. + public void VerifyHashes(IProgress? progressReporter = null) { ObjectDisposedException.ThrowIf(IsDisposed, this); @@ -45,9 +51,10 @@ public void VerifyHashes() throw new InvalidOperationException("The package has not been read."); } - if (Version != 2) + if (Version == 1) { - throw new InvalidDataException("Only version 2 is supported."); + progressReporter?.Report("Version 1 packages have no checksums to verify."); + return; } if (TreeChecksum == null || ArchiveMD5EntriesChecksum == null || WholeFileChecksum == null) @@ -55,26 +62,35 @@ public void VerifyHashes() throw new InvalidDataException("The package has no checksum section."); } - var hash = HashPackageRange(HeaderSize, TreeSize, HashAlgorithmName.MD5); + var wholeFileHash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5); + var wholeFileHashMatches = wholeFileHash.SequenceEqual(WholeFileChecksum); + + VerifySectionHash("File tree", HeaderSize, TreeSize, TreeChecksum, wholeFileHashMatches, progressReporter); + VerifySectionHash("Archive MD5 entries", FileSizeBeforeArchiveMD5Entries, ArchiveMD5SectionSize, ArchiveMD5EntriesChecksum, wholeFileHashMatches, progressReporter); - if (!hash.SequenceEqual(TreeChecksum)) + if (!wholeFileHashMatches) { - throw new InvalidDataException($"File tree checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(TreeChecksum)})"); + throw new InvalidDataException($"Package checksum mismatch ({BitConverter.ToString(wholeFileHash)} != expected {BitConverter.ToString(WholeFileChecksum)})"); } + } - hash = HashPackageRange(FileSizeBeforeArchiveMD5Entries, ArchiveMD5SectionSize, HashAlgorithmName.MD5); + private void VerifySectionHash(string sectionName, long offset, long length, byte[] expected, bool wholeFileHashMatches, IProgress? progressReporter) + { + var hash = HashPackageRange(offset, length, HashAlgorithmName.MD5); - if (!hash.SequenceEqual(ArchiveMD5EntriesChecksum)) + if (hash.SequenceEqual(expected)) { - throw new InvalidDataException($"Archive MD5 entries checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(ArchiveMD5EntriesChecksum)})"); + return; } - hash = HashPackageRange(0, FileSizeBeforeWholeFileHash, HashAlgorithmName.MD5); + var message = $"{sectionName} checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(expected)})"; - if (!hash.SequenceEqual(WholeFileChecksum)) + if (!wholeFileHashMatches) { - throw new InvalidDataException($"Package checksum mismatch ({BitConverter.ToString(hash)} != expected {BitConverter.ToString(WholeFileChecksum)})"); + throw new InvalidDataException(message); } + + progressReporter?.Report($"{message}, ignored because the package checksum matches."); } /// @@ -190,6 +206,7 @@ public void VerifyChunkHashes(IProgress? progressReporter = null) /// Verify CRC32 checksums of all files in the package. /// /// If provided, will report a string with the current verification progress. + /// An archive ends before all bytes of an entry could be read. public void VerifyFileChecksums(IProgress? progressReporter = null) { ObjectDisposedException.ThrowIf(IsDisposed, this); @@ -210,6 +227,8 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) { progressReporter?.Report($"Verifying CRC32 checksum for '{entry.GetFullPath()}' in archive {entry.ArchiveIndex}."); + ThrowIfEntryPastEndOfArchive(entry); + if (buffer.Length < entry.TotalLength) { buffer = new byte[entry.TotalLength]; @@ -224,7 +243,7 @@ public void VerifyFileChecksums(IProgress? progressReporter = null) /// /// Verifies the RSA signature. /// - /// True if signature is valid, false otherwise. + /// True if the signature is valid or the package has no signature (see ), false otherwise. public bool IsSignatureValid() { if (PublicKey == null || Signature == null) diff --git a/ValvePak/ValvePak/Package.cs b/ValvePak/ValvePak/Package.cs index 51abece..1232353 100644 --- a/ValvePak/ValvePak/Package.cs +++ b/ValvePak/ValvePak/Package.cs @@ -120,6 +120,11 @@ public uint Version /// public byte[]? Signature { get; private set; } + /// + /// Gets whether the package has both a public key and a signature. + /// + public bool IsSigned => PublicKey != null && Signature != null; + /// /// Gets the signature type. /// @@ -185,6 +190,14 @@ private void ThrowIfAlreadyRead() } } + private void ThrowIfFilesAdded() + { + if (Reader == null && Entries != null) + { + throw new InvalidOperationException("Files were added to this package, create a new instance to read a package."); + } + } + private Dictionary> CreateEntriesDictionary() { var stringComparer = Comparer == null ? null : StringComparer.FromComparison(Comparer.Comparison); diff --git a/ValvePak/ValvePak/PackageEntry.cs b/ValvePak/ValvePak/PackageEntry.cs index 8cd0cc0..881acc5 100644 --- a/ValvePak/ValvePak/PackageEntry.cs +++ b/ValvePak/ValvePak/PackageEntry.cs @@ -22,7 +22,7 @@ public class PackageEntry /// /// Gets or sets the file extension. - /// If the file has no extension, this is an empty string. + /// If the file has no extension, this is a single space. /// public required string TypeName { get; set; } diff --git a/ValvePak/ValvePak/StreamArchive.cs b/ValvePak/ValvePak/StreamArchive.cs index 95b7740..58b6f1e 100644 --- a/ValvePak/ValvePak/StreamArchive.cs +++ b/ValvePak/ValvePak/StreamArchive.cs @@ -21,6 +21,17 @@ public StreamArchive(Stream stream, long origin, long dataOffset) Stream = stream; } + public override long Length + { + get + { + lock (StreamLock) + { + return Stream.Length - Origin; + } + } + } + public override int Read(long offset, Span output) { lock (StreamLock)