Fix reading zip files with incomplete local file headers. These require proper...

Fix reading zip files with incomplete local file headers. These require proper support for the central directory, which is now implemented. Thanks to hiker for supplying such a file.

git-svn-id: svn://svn.code.sf.net/p/irrlicht/code/trunk@3544 dfc29bdd-3216-0410-991c-e03cc46cb475

source/Irrlicht/CZipReader.cpp

@@ -176,90 +176,6 @@ const IFileList* CZipReader::getFileList() const
 	return this;
 }
 
-#if 0
-#include <windows.h>
-
-const c8 *sigName( u32 sig )
-{
-	switch ( sig )
-	{
-		case 0x04034b50: return "PK0304";
-		case 0x02014b50: return "PK0102";
-		case 0x06054b50: return "PK0506";
-	}
-	return "unknown";
-}
-
-bool CZipReader::scanLocalHeader2()
-{
-	c8 buf [ 128 ];
-	c8 *c;
-
-	File->read( &temp.header.Sig, 4 );
-
-#ifdef __BIG_ENDIAN__
-	os::Byteswap::byteswap(temp.header.Sig);
-#endif
-
-	sprintf ( buf, "sig: %08x,%s,", temp.header.Sig, sigName ( temp.header.Sig ) );
-	OutputDebugStringA ( buf );
-
-	// Local File Header
-	if ( temp.header.Sig == 0x04034b50 )
-	{
-		File->read( &temp.header.VersionToExtract, sizeof( temp.header ) - 4 );
-
-		temp.zipFileName.reserve( temp.header.FilenameLength+2);
-		c = (c8*) temp.zipFileName.c_str();
-		File->read( c, temp.header.FilenameLength);
-		c [ temp.header.FilenameLength ] = 0;
-		temp.zipFileName.verify();
-
-		sprintf ( buf, "%d,'%s'\n", temp.header.CompressionMethod, c );
-		OutputDebugStringA ( buf );
-
-		if (temp.header.ExtraFieldLength)
-		{
-			File->seek( temp.header.ExtraFieldLength, true);
-		}
-
-		if (temp.header.GeneralBitFlag & ZIP_INFO_IN_DATA_DESCRIPTOR)
-		{
-			// read data descriptor
-			File->seek(sizeof(SZIPFileDataDescriptor), true );
-		}
-
-		// compressed data
-		temp.fileDataPosition = File->getPos();
-		File->seek( temp.header.DataDescriptor.CompressedSize, true);
-		FileList.push_back( temp );
-		return true;
-	}
-
-	// Central directory structure
-	if ( temp.header.Sig == 0x04034b50 )
-	{
-		//SZIPFileCentralDirFileHeader h;
-		//File->read( &h, sizeof( h ) - 4 );
-		return true;
-	}
-
-	// End of central dir
-	if ( temp.header.Sig == 0x06054b50 )
-	{
-		return true;
-	}
-
-	// eof
-	if ( temp.header.Sig == 0x02014b50 )
-	{
-		return false;
-	}
-
-	return false;
-}
-
-#endif
 
 //! scans for a local header, returns false if there is no more local file header.
 //! The gzip file format seems to think that there can be multiple files in a gzip file
@@ -372,7 +288,7 @@ bool CZipReader::scanGZipHeader()
 }
 
 //! scans for a local header, returns false if there is no more local file header.
-bool CZipReader::scanZipHeader()
+bool CZipReader::scanZipHeader(bool ignoreGPBits)
 {
 	io::path ZipFileName = "";
 	SZipFileEntry entry;
@@ -450,16 +366,57 @@ bool CZipReader::scanZipHeader()
 	if (entry.header.ExtraFieldLength)
 		File->seek(entry.header.ExtraFieldLength, true);
 
-	// if bit 3 was set, read DataDescriptor, following after the compressed data
-	if (entry.header.GeneralBitFlag & ZIP_INFO_IN_DATA_DESCRIPTOR)
+	// if bit 3 was set, use CentralDirectory for setup
+	if (!ignoreGPBits && entry.header.GeneralBitFlag & ZIP_INFO_IN_DATA_DESCRIPTOR)
 	{
-		// read data descriptor
-		File->read(&entry.header.DataDescriptor, sizeof(entry.header.DataDescriptor));
+		SZIPFileCentralDirEnd dirEnd;
+		FileInfo.clear();
+		Files.clear();
+		// First place where the end record could be stored
+		File->seek(File->getSize()-22);
+		const char endID[] = {0x50, 0x4b, 0x05, 0x06, 0x0};
+		char tmp[5]={'\0'};
+		bool found=false;
+		// search for the end record ID
+		while (!found && File->getPos()>0)
+		{
+			int seek=8;
+			File->read(tmp, 4);
+			switch (tmp[0])
+			{
+			case 0x50:
+				if (!strcmp(endID, tmp))
+				{
+					seek=4;
+					found=true;
+				}
+				break;
+			case 0x4b:
+				seek=5;
+				break;
+			case 0x05:
+				seek=6;
+				break;
+			case 0x06:
+				seek=7;
+				break;
+			}
+			File->seek(-seek, true);
+		}
+		File->read(&dirEnd, sizeof(dirEnd));
 #ifdef __BIG_ENDIAN__
-		entry.header.DataDescriptor.CRC32 = os::Byteswap::byteswap(entry.header.DataDescriptor.CRC32);
-		entry.header.DataDescriptor.CompressedSize = os::Byteswap::byteswap(entry.header.DataDescriptor.CompressedSize);
-		entry.header.DataDescriptor.UncompressedSize = os::Byteswap::byteswap(entry.header.DataDescriptor.UncompressedSize);
+		dirEnd.NumberDisk = os::Byteswap::byteswap(dirEnd.NumberDisk);
+		dirEnd.NumberStart = os::Byteswap::byteswap(dirEnd.NumberStart);
+		dirEnd.TotalDisk = os::Byteswap::byteswap(dirEnd.TotalDisk);
+		dirEnd.TotalEntries = os::Byteswap::byteswap(dirEnd.TotalEntries);
+		dirEnd.Size = os::Byteswap::byteswap(dirEnd.Size);
+		dirEnd.Offset = os::Byteswap::byteswap(dirEnd.Offset);
+		dirEnd.CommentLength = os::Byteswap::byteswap(dirEnd.CommentLength);
 #endif
+		FileInfo.reallocate(dirEnd.TotalEntries);
+		File->seek(dirEnd.Offset);
+		while (scanCentralDirectoryHeader()) { }
+		return false;
 	}
 
 	// store position in file
@@ -478,6 +435,48 @@ bool CZipReader::scanZipHeader()
 }
 
 
+//! scans for a local header, returns false if there is no more local file header.
+bool CZipReader::scanCentralDirectoryHeader()
+{
+	io::path ZipFileName = "";
+	SZIPFileCentralDirFileHeader entry;
+	File->read(&entry, sizeof(SZIPFileCentralDirFileHeader));
+
+#ifdef __BIG_ENDIAN__
+	entry.Sig = os::Byteswap::byteswap(entry.Sig);
+	entry.VersionMadeBy = os::Byteswap::byteswap(entry.VersionMadeBy);
+	entry.VersionToExtract = os::Byteswap::byteswap(entry.VersionToExtract);
+	entry.GeneralBitFlag = os::Byteswap::byteswap(entry.GeneralBitFlag);
+	entry.CompressionMethod = os::Byteswap::byteswap(entry.CompressionMethod);
+	entry.LastModFileTime = os::Byteswap::byteswap(entry.LastModFileTime);
+	entry.LastModFileDate = os::Byteswap::byteswap(entry.LastModFileDate);
+	entry.CRC32 = os::Byteswap::byteswap(entry.CRC32);
+	entry.CompressedSize = os::Byteswap::byteswap(entry.CompressedSize);
+	entry.UncompressedSize = os::Byteswap::byteswap(entry.UncompressedSize);
+	entry.FilenameLength = os::Byteswap::byteswap(entry.FilenameLength);
+	entry.ExtraFieldLength = os::Byteswap::byteswap(entry.ExtraFieldLength);
+	entry.FileCommentLength = os::Byteswap::byteswap(entry.FileCommentLength);
+	entry.DiskNumberStart = os::Byteswap::byteswap(entry.DiskNumberStart);
+	entry.InternalFileAttributes = os::Byteswap::byteswap(entry.InternalFileAttributes);
+	entry.ExternalFileAttributes = os::Byteswap::byteswap(entry.ExternalFileAttributes);
+	entry.RelativeOffsetOfLocalHeader = os::Byteswap::byteswap(entry.RelativeOffsetOfLocalHeader);
+#endif
+
+	if (entry.Sig != 0x02014b50)
+		return false; // central dir headers end here.
+
+	const long pos = File->getPos();
+	File->seek(entry.RelativeOffsetOfLocalHeader);
+	scanZipHeader(true);
+	File->seek(pos+entry.FilenameLength+entry.ExtraFieldLength+entry.FileCommentLength);
+	FileInfo.getLast().header.DataDescriptor.CompressedSize=entry.CompressedSize;
+	FileInfo.getLast().header.DataDescriptor.UncompressedSize=entry.UncompressedSize;
+	FileInfo.getLast().header.DataDescriptor.CRC32=entry.CRC32;
+	Files.getLast().Size=entry.UncompressedSize;
+	return true;
+}
+
+
 //! opens a file by file name
 IReadFile* CZipReader::createAndOpenFile(const io::path& filename)
 {

source/Irrlicht/CZipReader.h

@@ -60,23 +60,23 @@ namespace io
 
 	struct SZIPFileCentralDirFileHeader
 	{
-		u32 central_file_header_signature;	// 'PK0102' (0x02014b50)
-		u16 version_made_by;
-		u16 version_needed_to_extract;
-		u16 general_purpose_bit_flag;
-		u16 compression_method;
-		u16 last_mod_file_time;
-		u16 last_mod_file_date;
-		u32 crc_32;
-		u32 compressed_size;
-		u32 uncompressed_size;
-		u16 filename_length;
-		u16 extra_field_length;
-		u16 file_comment_length;
-		u16 disk_number_start;
-		u16 internal_file_attributes;
-		u32 external_file_attributes;
-		u32 relative_offset_of_local_header;
+		u32 Sig;	// 'PK0102' (0x02014b50)
+		u16 VersionMadeBy;
+		u16 VersionToExtract;
+		u16 GeneralBitFlag;
+		u16 CompressionMethod;
+		u16 LastModFileTime;
+		u16 LastModFileDate;
+		u32 CRC32;
+		u32 CompressedSize;
+		u32 UncompressedSize;
+		u16 FilenameLength;
+		u16 ExtraFieldLength;
+		u16 FileCommentLength;
+		u16 DiskNumberStart;
+		u16 InternalFileAttributes;
+		u32 ExternalFileAttributes;
+		u32 RelativeOffsetOfLocalHeader;
 
 		// filename (variable size)
 		// extra field (variable size)
@@ -86,14 +86,14 @@ namespace io
 
 	struct SZIPFileCentralDirEnd
 	{
-		u32 sig;			// 'PK0506' end_of central dir signature			// (0x06054b50)
-		u16 numberDisk;		// number of this disk
-		u16 numberStart;	// number of the disk with the start of the central directory
-		u16 totalDisk;		// total number of entries in the central dir on this disk
-		u16 totalEntries;	// total number of entries in the central dir
-		u32 size;			// size of the central directory
-		u32 offset;			// offset of start of centraldirectory with respect to the starting disk number
-		u16 comment_length;	// zipfile comment length
+		u32 Sig;			// 'PK0506' end_of central dir signature			// (0x06054b50)
+		u16 NumberDisk;		// number of this disk
+		u16 NumberStart;	// number of the disk with the start of the central directory
+		u16 TotalDisk;		// total number of entries in the central dir on this disk
+		u16 TotalEntries;	// total number of entries in the central dir
+		u32 Size;			// size of the central directory
+		u32 Offset;			// offset of start of centraldirectory with respect to the starting disk number
+		u16 CommentLength;	// zipfile comment length
 		// zipfile comment (variable size)
 	} PACK_STRUCT;
 
@@ -211,18 +211,23 @@ namespace io
 
 	protected:
 
-		IReadFile* File;
-
 		//! reads the next file header from a ZIP file, returns false if there are no more headers.
-		bool scanZipHeader();
+		/* if ignoreGPBits is set, the item will be read despite missing
+		file information. This is used when reading items from the central
+		directory. */
+		bool scanZipHeader(bool ignoreGPBits=false);
 
 		//! the same but for gzip files
 		bool scanGZipHeader();
 
-		bool IsGZip;
+		bool scanCentralDirectoryHeader();
+
+		IReadFile* File;
 
 		// holds extended info about files
 		core::array<SZipFileEntry> FileInfo;
+
+		bool IsGZip;
 	};
 
 

tests/archiveReader.cpp

@@ -172,6 +172,7 @@ bool testEncryptedZip(IFileSystem* fs)
 	if ( !fs->addFileArchive(archiveName, /*bool ignoreCase=*/true, /*bool ignorePaths=*/false) )
 	{
 		logTestString("Mounting a second time failed\n");
+		fs->removeFileArchive(fs->getFileArchiveCount()-1);
 		return false;
 	}
 
@@ -179,6 +180,8 @@ bool testEncryptedZip(IFileSystem* fs)
 	if ( fs->getFileArchiveCount() != 1 )
 	{
 		logTestString("Duplicate mount not recognized\n");
+		while (fs->getFileArchiveCount())
+			fs->removeFileArchive(fs->getFileArchiveCount()-1);
 		return false;
 	}
 
@@ -195,6 +198,7 @@ bool testEncryptedZip(IFileSystem* fs)
 	if (!fs->existFile(filename))
 	{
 		logTestString("existFile failed\n");
+		fs->removeFileArchive(fs->getFileArchiveCount()-1);
 		return false;
 	}
 
@@ -202,6 +206,8 @@ bool testEncryptedZip(IFileSystem* fs)
 	if ( readFile )
 	{
 		logTestString("createAndOpenFile succeeded, even though no password was set.\n");
+		readFile->drop();
+		fs->removeFileArchive(fs->getFileArchiveCount()-1);
 		return false;
 	}
 
@@ -211,6 +217,7 @@ bool testEncryptedZip(IFileSystem* fs)
 	if ( !readFile )
 	{
 		logTestString("createAndOpenFile failed\n");
+		fs->removeFileArchive(fs->getFileArchiveCount()-1);
 		return false;
 	}
 
@@ -238,6 +245,79 @@ bool testEncryptedZip(IFileSystem* fs)
 	return true;
 }
 
+bool testSpecialZip(IFileSystem* fs)
+{
+	// make sure there is no archive mounted
+	if ( fs->getFileArchiveCount() )
+	{
+		logTestString("Already mounted archives found\n");
+		return false;
+	}
+
+	const char* archiveName = "media/Monty.zip";
+	if ( !fs->addFileArchive(archiveName, /*bool ignoreCase=*/true, /*bool ignorePaths=*/false) )
+	{
+		logTestString("Mounting archive failed\n");
+		return false;
+	}
+
+	// make sure there is an archive mounted
+	if ( !fs->getFileArchiveCount() )
+	{
+		logTestString("Mounted archive not in list\n");
+		return false;
+	}
+
+	// log what we got
+	io::IFileArchive* archive = fs->getFileArchive(fs->getFileArchiveCount()-1);
+	const io::IFileList* fileList = archive->getFileList();
+	for ( u32 f=0; f < fileList->getFileCount(); ++f)
+	{
+		logTestString("File name: %s\n", fileList->getFileName(f).c_str());
+		logTestString("Full path: %s\n", fileList->getFullFileName(f).c_str());
+	}
+	
+	io::path filename("monty/license.txt");
+	if (!fs->existFile(filename))
+	{
+		logTestString("existFile failed\n");
+		fs->removeFileArchive(fs->getFileArchiveCount()-1);
+		return false;
+	}
+
+	IReadFile* readFile = fs->createAndOpenFile(filename);
+	if ( !readFile )
+	{
+		logTestString("createAndOpenFile failed\n");
+		fs->removeFileArchive(fs->getFileArchiveCount()-1);
+		return false;
+	}
+
+	char tmp[6] = {'\0'};
+	readFile->read(tmp, 5);
+	if (strcmp(tmp, "Monty"))
+	{
+		logTestString("Read bad data from archive: %s\n", tmp);
+		readFile->drop();
+		fs->removeFileArchive(fs->getFileArchiveCount()-1);
+		return false;
+	}
+
+	readFile->drop();
+
+	if (!fs->removeFileArchive(fs->getFileArchiveCount()-1))
+	{
+		logTestString("Couldn't remove archive.\n");
+		return false;
+	}
+
+	// make sure there is no archive mounted
+	if ( fs->getFileArchiveCount() )
+		return false;
+
+	return true;
+}
+
 static bool testMountFile(IFileSystem* fs)
 {
 	bool result = true;
@@ -298,6 +378,8 @@ bool archiveReader()
 	ret &= testArchive(fs, "media/file_with_path.npk");
 	logTestString("Testing encrypted zip files.\n");
 	ret &= testEncryptedZip(fs);
+	logTestString("Testing special zip files.\n");
+	ret &= testSpecialZip(fs);
 //	logTestString("Testing complex mount file.\n");
 //	ret &= testMountFile(fs);