Added method to add an archive from an IReadFile pointer. Added FileToHeader,...

Added method to add an archive from an IReadFile pointer. Added FileToHeader, a tool for embedding a collection of IReadFiles into a user app. Described a way to make single file portable applications using it.



git-svn-id: svn://svn.code.sf.net/p/irrlicht/code/trunk@3586 dfc29bdd-3216-0410-991c-e03cc46cb475

changes.txt

 Changes in 1.8 (??.??.2011)
 
+ - Added the ability to open an archive from an IReadFile*, added a FileToHeader tool with instructions of how to make a portable app that consists of a single executable file.
+
  - Added ISceneManager::createSceneNodeAnimator to create animators by name
 
  - The Makefile now creates a symlink from the soname to the binary name during install. Binary compatibility is only confirmed between minor releases, so the only useful symlink is from libIrrlicht.so.1.8 to libIrrlicht.so.1.8.0; others should rightly fail.
@@ -2524,6 +2526,7 @@ Font improvements:
   of Animation name...
 
 - changed spelling "frustrum" to "frustum"
+
   -> changed also SViewFrustrum.h to SViewFrustum.h
 
 - changed Parameter AutomaticCulling from bool to enum E_CULLING_TYPE

examples/BuildAllExamples.workspace

@@ -2,7 +2,7 @@
 <CodeBlocks_workspace_file>
 	<Workspace title="Build all examples">
 		<Project filename="01.HelloWorld/HelloWorld.cbp" />
-		<Project filename="02.Quake3Map/Quake3Map.cbp" />
+		<Project filename="02.Quake3Map/Quake3Map.cbp" active="1" />
 		<Project filename="03.CustomSceneNode/CustomSceneNode.cbp" />
 		<Project filename="04.Movement/Movement.cbp" />
 		<Project filename="05.UserInterface/UserInterface.cbp" />
@@ -15,7 +15,7 @@
 		<Project filename="12.TerrainRendering/TerrainRendering.cbp" />
 		<Project filename="13.RenderToTexture/RenderToTexture.cbp" />
 		<Project filename="14.Win32Window/Win32Window.cbp" />
-		<Project filename="15.LoadIrrFile/LoadIrrFile.cbp" active="1" />
+		<Project filename="15.LoadIrrFile/LoadIrrFile.cbp" />
 		<Project filename="16.Quake3MapShader/Quake3MapShader.cbp" />
 		<Project filename="18.SplitScreen/SplitScreen.cbp" />
 		<Project filename="19.MouseAndJoystick/MouseAndJoystick.cbp" />
@@ -26,5 +26,6 @@
 		<Project filename="../tools/GUIEditor/GUIEditor_gcc.cbp" />
 		<Project filename="../tools/MeshConverter/MeshConverter.cbp" />
 		<Project filename="../source/Irrlicht/Irrlicht-gcc.cbp" />
+		<Project filename="../tools/FileToHeader/FileToHeader.cbp" />
 	</Workspace>
 </CodeBlocks_workspace_file>

include/IFileSystem.h

@@ -117,6 +117,32 @@ public:
 			E_FILE_ARCHIVE_TYPE archiveType=EFAT_UNKNOWN,
 			const core::stringc& password="") =0;
 
+	//! Adds an archive to the file system.
+	/** After calling this, the Irrlicht Engine will also search and open
+	files directly from this archive. This is useful for hiding data from
+	the end user, speeding up file access and making it possible to access
+	for example Quake3 .pk3 files, which are just renamed .zip files. By
+	default Irrlicht supports ZIP, PAK, TAR, PNK, and directories as
+	archives. You can provide your own archive types by implementing
+	IArchiveLoader and passing an instance to addArchiveLoader.
+	Irrlicht supports AES-encrypted zip files, and the advanced compression
+	techniques lzma and bzip2.
+	\param file: Archive to add to the file system.
+	\param ignoreCase: If set to true, files in the archive can be accessed without
+	writing all letters in the right case.
+	\param ignorePaths: If set to true, files in the added archive can be accessed
+	without its complete path.
+	\param archiveType: If no specific E_FILE_ARCHIVE_TYPE is selected then
+	the type of archive will depend on the extension of the file name. If
+	you use a different extension then you can use this parameter to force
+	a specific type of archive.
+	\param password An optional password, which is used in case of encrypted archives.
+	\return True if the archive was added successfully, false if not. */
+	virtual bool addFileArchive(IReadFile* file, bool ignoreCase=true,
+			bool ignorePaths=true,
+			E_FILE_ARCHIVE_TYPE archiveType=EFAT_UNKNOWN,
+			const core::stringc& password="") =0;
+
 	//! Get the number of archives currently attached to the file system
 	virtual u32 getFileArchiveCount() const =0;
 

readme.txt

@@ -14,7 +14,6 @@ The Irrlicht Engine SDK version 1.8
   6. Contact
 
 
-
 ==========================================================================
 1. Directory Structure Overview
 ==========================================================================

source/Irrlicht/CFileSystem.cpp

@@ -172,7 +172,7 @@ void CFileSystem::addArchiveLoader(IArchiveLoader* loader)
 }
 
 //! Returns the total number of archive loaders added.
-u32 CFileSystem::getArchiveLoaderCount() const 
+u32 CFileSystem::getArchiveLoaderCount() const
 {
 	return ArchiveLoader.size();
 }
@@ -217,20 +217,9 @@ bool CFileSystem::addFileArchive(const io::path& filename, bool ignoreCase,
 	IFileArchive* archive = 0;
 	bool ret = false;
 
-	// check if the archive was already loaded
-	for (u32 idx = 0; idx < FileArchives.size(); ++idx)
-	{
-		// TODO: This should go into a path normalization method
-		// We need to check for directory names with trailing slash and without
-		const core::stringc absPath = getAbsolutePath(filename);
-		const core::stringc arcPath = FileArchives[idx]->getFileList()->getPath();
-		if ((absPath == arcPath) || ((absPath+"/") == arcPath))
-		{
-			if (password.size())
-				FileArchives[idx]->Password=password;
-			return true;
-		}
-	}
+	// see if archive is already added
+	if (changeArchivePassword(filename, password))
+		return true;
 
 	s32 i;
 
@@ -325,6 +314,105 @@ bool CFileSystem::addFileArchive(const io::path& filename, bool ignoreCase,
 	return ret;
 }
 
+// don't expose!
+bool CFileSystem::changeArchivePassword(const path& filename, const core::stringc& password)
+{
+	for (s32 idx = 0; idx < (s32)FileArchives.size(); ++idx)
+	{
+		// TODO: This should go into a path normalization method
+		// We need to check for directory names with trailing slash and without
+		const core::stringc absPath = getAbsolutePath(filename);
+		const core::stringc arcPath = FileArchives[idx]->getFileList()->getPath();
+		if ((absPath == arcPath) || ((absPath+"/") == arcPath))
+		{
+			if (password.size())
+				FileArchives[idx]->Password=password;
+			return true;
+		}
+	}
+
+	return false;
+}
+
+bool CFileSystem::addFileArchive(IReadFile* file, bool ignoreCase, bool ignorePaths,
+			E_FILE_ARCHIVE_TYPE archiveType, const core::stringc& password)
+{
+	if (!file || archiveType == EFAT_FOLDER)
+		return false;
+
+	if (file)
+	{
+		if (changeArchivePassword(file->getFileName(), password))
+			return true;
+
+		IFileArchive* archive = 0;
+		s32 i;
+
+		if (archiveType == EFAT_UNKNOWN)
+		{
+			// try to load archive based on file name
+			for (i = ArchiveLoader.size()-1; i >=0 ; --i)
+			{
+				if (ArchiveLoader[i]->isALoadableFileFormat(file->getFileName()))
+				{
+					archive = ArchiveLoader[i]->createArchive(file, ignoreCase, ignorePaths);
+					if (archive)
+						break;
+				}
+			}
+
+			// try to load archive based on content
+			if (!archive)
+			{
+				for (i = ArchiveLoader.size()-1; i >= 0; --i)
+				{
+					file->seek(0);
+					if (ArchiveLoader[i]->isALoadableFileFormat(file))
+					{
+						file->seek(0);
+						archive = ArchiveLoader[i]->createArchive(file, ignoreCase, ignorePaths);
+						if (archive)
+							break;
+					}
+				}
+			}
+		}
+		else
+		{
+			// try to open archive based on archive loader type
+			for (i = ArchiveLoader.size()-1; i >= 0; --i)
+			{
+				if (ArchiveLoader[i]->isALoadableFileFormat(archiveType))
+				{
+					// attempt to open archive
+					file->seek(0);
+					if (ArchiveLoader[i]->isALoadableFileFormat(file))
+					{
+						file->seek(0);
+						archive = ArchiveLoader[i]->createArchive(file, ignoreCase, ignorePaths);
+						if (archive)
+							break;
+					}
+				}
+			}
+		}
+
+		if (archive)
+		{
+			FileArchives.push_back(archive);
+			if (password.size())
+				archive->Password=password;
+			return true;
+		}
+		else
+		{
+			os::Printer::log("Could not create archive for", file->getFileName(), ELL_ERROR);
+		}
+	}
+
+	return false;
+}
+
 
 //! removes an archive from the file system.
 bool CFileSystem::removeFileArchive(u32 index)

source/Irrlicht/CFileSystem.h

@@ -51,6 +51,12 @@ public:
 			E_FILE_ARCHIVE_TYPE archiveType = EFAT_UNKNOWN,
 			const core::stringc& password="");
 
+	//! Adds an archive to the file system.
+	virtual bool addFileArchive(IReadFile* file, bool ignoreCase=true,
+			bool ignorePaths=true,
+			E_FILE_ARCHIVE_TYPE archiveType=EFAT_UNKNOWN,
+			const core::stringc& password="");
+
 	//! move the hirarchy of the filesystem. moves sourceIndex relative up or down
 	virtual bool moveFileArchive( u32 sourceIndex, s32 relative );
 
@@ -136,6 +142,9 @@ public:
 
 private:
 
+	// don't expose, needs refactoring
+	bool changeArchivePassword(const path& filename, const core::stringc& password);
+
 	//! Currently used FileSystemType
 	EFileSystemType FileSystemType;
 	//! WorkingDirectory for Native and Virtual filesystems

tools/FileToHeader/FileToHeader.cbp

+<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+<CodeBlocks_project_file>
+	<FileVersion major="1" minor="6" />
+	<Project>
+		<Option title="FileToHeader" />
+		<Option pch_mode="0" />
+		<Option compiler="gcc" />
+		<Build>
+			<Target title="Linux">
+				<Option platforms="Unix;" />
+				<Option output="../../bin/Linux/FileToHeader" prefix_auto="0" extension_auto="0" />
+				<Option type="1" />
+				<Option compiler="gcc" />
+				<Compiler>
+					<Add option="-g" />
+					<Add option="-W" />
+					<Add option="-D_IRR_STATIC_LIB_" />
+				</Compiler>
+				<Linker>
+					<Add directory="../../lib/Linux" />
+				</Linker>
+			</Target>
+			<Target title="Windows">
+				<Option platforms="Windows;" />
+				<Option output="../../bin/Win32-gcc/FileToHeader" prefix_auto="0" extension_auto="1" />
+				<Option type="1" />
+				<Option compiler="gcc" />
+				<Option projectResourceIncludeDirsRelation="1" />
+				<Compiler>
+					<Add option="-Wall" />
+					<Add option="-g" />
+				</Compiler>
+				<Linker>
+					<Add directory="../../lib/Win32-gcc" />
+				</Linker>
+			</Target>
+		</Build>
+		<VirtualTargets>
+			<Add alias="All" targets="Windows;" />
+		</VirtualTargets>
+		<Compiler>
+			<Add option="-g" />
+			<Add option="-W" />
+		</Compiler>
+		<Unit filename="main.cpp" />
+		<Extensions>
+			<code_completion />
+			<debugger />
+		</Extensions>
+	</Project>
+</CodeBlocks_project_file>

tools/FileToHeader/FileToHeader.layout

+<?xml version="1.0" encoding="UTF-8" standalone="yes" ?>
+<CodeBlocks_layout_file>
+	<ActiveTarget name="Linux" />
+	<File name="main.cpp" open="0" top="0" tabpos="0">
+		<Cursor position="1821" topLine="63" />
+	</File>
+</CodeBlocks_layout_file>

tools/FileToHeader/Makefile

+# Makefile for FileToHeader
+Target = FileToHeader
+Sources = main.cpp
+
+# general compiler settings
+CPPFLAGS = -I../../include
+CXXFLAGS = -O3 -ffast-math -Wall
+#CXXFLAGS = -g -Wall
+
+#default target is Linux
+all: all_linux
+
+ifeq ($(HOSTTYPE), x86_64)
+LIBSELECT=64
+endif
+
+# target specific settings
+all_linux clean_linux: SYSTEM=Linux
+all_win32: LDFLAGS = -L../../lib/Win32-gcc 
+all_win32 clean_win32: SYSTEM=Win32-gcc
+all_win32 clean_win32: SUF=.exe
+# name of the binary - only valid for targets which set SYSTEM
+DESTPATH = ../../bin/$(SYSTEM)/$(Target)$(SUF)
+
+all_linux all_win32:
+	$(warning Building...)
+	$(CXX) $(CPPFLAGS) $(CXXFLAGS) $(Sources) -o $(DESTPATH) $(LDFLAGS)
+
+clean: clean_linux clean_win32
+	$(warning Cleaning...)
+
+clean_linux clean_win32:
+	@$(RM) $(DESTPATH)
+
+.PHONY: all all_win32 clean clean_linux clean_win32

tools/FileToHeader/main.cpp

+// Copyright (C) 2011 Gaz Davidson
+// This file is part of the "Irrlicht Engine".
+// For conditions of distribution and use, see copyright notice in irrlicht.h
+
+/***
+	This tool creates a .h file from a given input file by encoding it into a C string,
+	allowing you to build your resources directly into your binaries, just like Irrlicht's
+	built-in font.
+
+	To distribute your app as a single executable file of minimal size:
+
+	1. Put all your resources into a single directory and add it to Irrlicht's filesystem
+	   as a folder through IFileSystem::addArchive. Develop and test your app as usual.
+	2. Open IrrCompileConfig.h and comment out all the options that your app does not use.
+	   This will reduce the size of the Irrlicht library considerably.
+	     * You should remove the D3D video drivers, because they rely on external DLLs.
+	     * You should keep either the TAR or ZIP archive loader, used in step 6.
+	     * If you remove the JPEG loader, you don't have to say you use JPEG code in your
+	       documentation.
+	3. Recompile Irrlicht as a static library, editing the IRR_STATIC_LIB line in
+	   IrrCompileConfig.h.
+	   The next time you compile your application it will take a while longer, but
+	   Irrlicht will be built into your binary.
+	4. TAR or ZIP your resource directory using your favourite archiving tool (ie 7zip).
+	     * If you chose ZIP but compiled without zlib, don't compress this archive or it
+	       won't open.
+	5. Run this tool to convert your resource file into a .h file, like so:
+	     FileToHeader res.zip > EmbeddedResources.h
+	6. Add the .h file to your project, create the embedded read file then mount as a
+	   ZIP or TAR archive instead of the folder, like so:
+	     io::IReadFile *f = io::createEmbeddedFile(device->getFileSystem(), "res.zip");
+	     device->getFileSystem()->addFileArchive(f);
+	     archive->drop();
+	7. Recompile your app.
+	     * If you use Microsoft's compiler, make sure your CRT (common run-time) is
+	       the static library version, otherwise you'll have a dependency on the CRT DLLs.
+	   Your binary should now be completely portable; you can distribute just the exe file.
+	8. Optional: Use UPX (upx.sf.net) to compress your binary.
+*/
+
+#include <iostream>
+#include <fstream>
+#include <sstream>
+
+using namespace std;
+
+int main(int argc, char* argv[])
+{
+	if (argc < 2)
+	{
+		// print usage
+		cerr << "You must to specify at least one input file" << endl;
+		cerr << "usage: " << argv[0] << "<file1> [file2...]" << endl;
+		cerr << "outputs a header file to stdout, so for example use";
+		return 1;
+	}
+
+	int i = 1;
+
+	// write file header
+	cout << "// File made by FileToHeader, part of the Irrlicht Engine" << endl
+	     << endl
+	     << "#ifndef _EMBEDDED_FILES_H_INCLUDED_" << endl
+	     << "#define _EMBEDDED_FILES_H_INCLUDED_" << endl
+	     << endl
+	     << "#include \"irrTypes.h\"" << endl
+	     << "#include \"IReadFile.h\"" << endl
+	     << "#include \"IFileSystem.h\"" << endl
+	     << endl
+	     << "namespace irr" << endl
+	     << "{" << endl
+	     << "namespace io" << endl
+	     << "{" << endl
+	     << endl
+	     << "	const c8* EmbeddedFileData[] = " << endl
+	     << "	{" << endl;
+
+	// store sizes and file names
+	stringstream sizes;
+	stringstream names;
+	sizes << "const u32 EmbeddedFileSizes[] = {";
+	names << "const c8* EmbeddedFileNames[] = {";
+	int fileCount = 0;
+
+	// char to hex digit table, probably doesn't help for speed due to fstream. better than using sprintf though
+	char hextable[16] = {'0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f'};
+
+	while (i < argc)
+	{
+		// open and seek to end of file
+		ifstream input;
+		input.open(argv[i], ios::in | ios::binary | ios::ate);
+
+		if (input.is_open())
+		{
+			int size = input.tellg();
+			input.seekg(0, ios::beg);
+			// read the file into RAM
+			char *entireFile = new char[size];
+			input.read(entireFile, size);
+
+			if (fileCount)
+			{
+				sizes << ", ";
+				names << ", ";
+				cout  << "," << endl;
+			}
+
+			// save file size and name
+			sizes << size;
+			names << '"' << argv[i] << '"';
+
+			// write the file data
+			cout << "		\"";
+			for (int count=0; count < size; ++count)
+			{
+				if (count && (count % 16) == 0)
+					cout << "\"" << endl << "		\"";
+
+				cout << "\\x" << hextable[(entireFile[count] >> 4) & 0xF] << hextable[entireFile[count] & 0x0F];
+			}
+			cout << "\"";
+
+			delete [] entireFile;
+			//
+			input.close();
+
+			fileCount++;
+		}
+		else
+		{
+			cerr << "Failed to open file: " << argv[i] << endl;
+		}
+
+		++i;
+	}
+
+	// close binary file list and write file metadata
+	cout << endl
+	     << "		, 0};" << endl
+	     << endl
+	     << "	const u32 EmbeddedFileCount = " << fileCount << ";" << endl
+	     << "	" << sizes.str() << "};" << endl
+	     << "	" << names.str() << "};" << endl
+	     << endl;
+
+	// write functions to create embedded IReadFiles
+	cout << "	IReadFile* createEmbeddedFile(IFileSystem *fs, u32 index)" << endl
+	     << "	{" << endl
+	     << "		if (EmbeddedFileCount < index)" << endl
+	     << "			return 0;" << endl
+	     << endl
+	     << "		return fs->createMemoryReadFile((void*)EmbeddedFileData[index], " << endl
+	     << "		                                EmbeddedFileSizes[index], EmbeddedFileNames[index]);" << endl
+	     << "	}" << endl
+	     << endl
+	     << "	IReadFile* createEmbeddedFile(IFileSystem *fs, path filename)" << endl
+	     << "	{" << endl
+	     << "		for (u32 i=0; i < EmbeddedFileCount; ++i)" << endl
+	     << "			if (filename == EmbeddedFileNames[i])" << endl
+	     << "				return createEmbeddedFile(fs, i);" << endl
+	     << endl
+	     << "		return 0;" << endl
+	     << "	}" << endl
+	     << endl;
+
+	// write footer
+	cout << "} // namespace io"  << endl
+	     << "} // namespace irr" << endl
+	     << endl
+	     << "#endif // _EMBEDDED_FILES_H_INCLUDED_" << endl;
+
+	return 0;
+}
+