FileDocCategorySizeDatePackage
CompoundFileWriter.javaAPI DocApache Lucene 2.2.08226Sat Jun 16 22:20:36 BST 2007org.apache.lucene.index

CompoundFileWriter

public final class CompoundFileWriter extends Object
Combines multiple files into a single compound file. The file format:
  • VInt fileCount
  • {Directory} fileCount entries with the following structure:
    • long dataOffset
    • String fileName
  • {File Data} fileCount entries with the raw data of the corresponding file
The fileCount integer indicates how many files are contained in this compound file. The {directory} that follows has that many entries. Each directory entry contains a long pointer to the start of this file's data section, and a String with that file's name.
author
Dmitry Serebrennikov
version
$Id: CompoundFileWriter.java 542561 2007-05-29 15:14:07Z mikemccand $

Fields Summary
private Directory
directory
private String
fileName
private HashSet
ids
private LinkedList
entries
private boolean
merged
Constructors Summary
public CompoundFileWriter(Directory dir, String name)
Create the compound stream in the specified file. The file name is the entire name (no extensions are added).

throws
NullPointerException if dir or name is null



                                      
         
        if (dir == null)
            throw new NullPointerException("directory cannot be null");
        if (name == null)
            throw new NullPointerException("name cannot be null");

        directory = dir;
        fileName = name;
        ids = new HashSet();
        entries = new LinkedList();
    
Methods Summary
public voidaddFile(java.lang.String file)
Add a source stream. file is the string by which the sub-stream will be known in the compound stream.

throws
IllegalStateException if this writer is closed
throws
NullPointerException if file is null
throws
IllegalArgumentException if a file with the same name has been added already

        if (merged)
            throw new IllegalStateException(
                "Can't add extensions after merge has been called");

        if (file == null)
            throw new NullPointerException(
                "file cannot be null");

        if (! ids.add(file))
            throw new IllegalArgumentException(
                "File " + file + " already added");

        FileEntry entry = new FileEntry();
        entry.file = file;
        entries.add(entry);
    
public voidclose()
Merge files with the extensions added up to now. All files with these extensions are combined sequentially into the compound stream. After successful merge, the source files are deleted.

throws
IllegalStateException if close() had been called before or if no file has been added to this object

        if (merged)
            throw new IllegalStateException(
                "Merge already performed");

        if (entries.isEmpty())
            throw new IllegalStateException(
                "No entries to merge have been defined");

        merged = true;

        // open the compound stream
        IndexOutput os = null;
        try {
            os = directory.createOutput(fileName);

            // Write the number of entries
            os.writeVInt(entries.size());

            // Write the directory with all offsets at 0.
            // Remember the positions of directory entries so that we can
            // adjust the offsets later
            Iterator it = entries.iterator();
            while(it.hasNext()) {
                FileEntry fe = (FileEntry) it.next();
                fe.directoryOffset = os.getFilePointer();
                os.writeLong(0);    // for now
                os.writeString(fe.file);
            }

            // Open the files and copy their data into the stream.
            // Remember the locations of each file's data section.
            byte buffer[] = new byte[16384];
            it = entries.iterator();
            while(it.hasNext()) {
                FileEntry fe = (FileEntry) it.next();
                fe.dataOffset = os.getFilePointer();
                copyFile(fe, os, buffer);
            }

            // Write the data offsets into the directory of the compound stream
            it = entries.iterator();
            while(it.hasNext()) {
                FileEntry fe = (FileEntry) it.next();
                os.seek(fe.directoryOffset);
                os.writeLong(fe.dataOffset);
            }

            // Close the output stream. Set the os to null before trying to
            // close so that if an exception occurs during the close, the
            // finally clause below will not attempt to close the stream
            // the second time.
            IndexOutput tmp = os;
            os = null;
            tmp.close();

        } finally {
            if (os != null) try { os.close(); } catch (IOException e) { }
        }
    
private voidcopyFile(org.apache.lucene.index.CompoundFileWriter$FileEntry source, org.apache.lucene.store.IndexOutput os, byte[] buffer)
Copy the contents of the file with specified extension into the provided output stream. Use the provided buffer for moving data to reduce memory allocation.

        IndexInput is = null;
        try {
            long startPtr = os.getFilePointer();

            is = directory.openInput(source.file);
            long length = is.length();
            long remainder = length;
            int chunk = buffer.length;

            while(remainder > 0) {
                int len = (int) Math.min(chunk, remainder);
                is.readBytes(buffer, 0, len);
                os.writeBytes(buffer, len);
                remainder -= len;
            }

            // Verify that remainder is 0
            if (remainder != 0)
                throw new IOException(
                    "Non-zero remainder length after copying: " + remainder
                    + " (id: " + source.file + ", length: " + length
                    + ", buffer size: " + chunk + ")");

            // Verify that the output length diff is equal to original file
            long endPtr = os.getFilePointer();
            long diff = endPtr - startPtr;
            if (diff != length)
                throw new IOException(
                    "Difference in the output file offsets " + diff
                    + " does not match the original file length " + length);

        } finally {
            if (is != null) is.close();
        }
    
public org.apache.lucene.store.DirectorygetDirectory()
Returns the directory of the compound file.

        return directory;
    
public java.lang.StringgetName()
Returns the name of the compound file.

        return fileName;