FileDocCategorySizeDatePackage
TestDeletionPolicy.javaAPI DocApache Lucene 2.2.019811Sat Jun 16 22:20:28 BST 2007org.apache.lucene.index

TestDeletionPolicy

public class TestDeletionPolicy extends TestCase
Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file to You under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

Fields Summary
Constructors Summary
Methods Summary
private voidaddDoc(org.apache.lucene.index.IndexWriter writer)

    Document doc = new Document();
    doc.add(new Field("content", "aaa", Field.Store.NO, Field.Index.TOKENIZED));
    writer.addDocument(doc);
public voidtestExpirationTimeDeletionPolicy()


    final double SECONDS = 2.0;

    boolean autoCommit = false;
    boolean useCompoundFile = true;

    Directory dir = new RAMDirectory();
    ExpirationTimeDeletionPolicy policy = new ExpirationTimeDeletionPolicy(dir, SECONDS);
    IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
    writer.setUseCompoundFile(useCompoundFile);
    writer.close();

    long lastDeleteTime = 0;
    for(int i=0;i<7;i++) {
      // Record last time when writer performed deletes of
      // past commits
      lastDeleteTime = System.currentTimeMillis();
      writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
      writer.setUseCompoundFile(useCompoundFile);
      for(int j=0;j<17;j++) {
        addDoc(writer);
      }
      writer.close();

      // Make sure to sleep long enough so that some commit
      // points will be deleted:
      Thread.sleep((int) (1000.0*(SECONDS/5.0)));
    }

    // First, make sure the policy in fact deleted something:
    assertTrue("no commits were deleted", policy.numDelete > 0);

    // Then simplistic check: just verify that the
    // segments_N's that still exist are in fact within SECONDS
    // seconds of the last one's mod time, and, that I can
    // open a reader on each:
    long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
    
    String fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                            "",
                                                            gen);
    while(gen > 0) {
      try {
        IndexReader reader = IndexReader.open(dir);
        reader.close();
        fileName = IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS,
                                                         "",
                                                         gen);
        long modTime = dir.fileModified(fileName);
        assertTrue("commit point was older than " + SECONDS + " seconds but did not get deleted", lastDeleteTime - modTime < (SECONDS*1000));
      } catch (IOException e) {
        // OK
        break;
      }
      
      dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
      gen--;
    }

    dir.close();
public voidtestKeepAllDeletionPolicy()


    for(int pass=0;pass<4;pass++) {

      boolean autoCommit = pass < 2;
      boolean useCompoundFile = (pass % 2) > 0;

      KeepAllDeletionPolicy policy = new KeepAllDeletionPolicy();

      Directory dir = new RAMDirectory();

      IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
      writer.setUseCompoundFile(useCompoundFile);
      for(int i=0;i<107;i++) {
        addDoc(writer);
      }
      writer.close();

      writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
      writer.setUseCompoundFile(useCompoundFile);
      writer.optimize();
      writer.close();

      assertEquals(2, policy.numOnInit);
      if (autoCommit) {
        assertTrue(policy.numOnCommit > 2);
      } else {
        // If we are not auto committing then there should
        // be exactly 2 commits (one per close above):
        assertEquals(2, policy.numOnCommit);
      }

      // Simplistic check: just verify all segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
      while(gen > 0) {
        IndexReader reader = IndexReader.open(dir);
        reader.close();
        dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        gen--;

        if (gen > 0) {
          // Now that we've removed a commit point, which
          // should have orphan'd at least one index file.
          // Open & close a writer and assert that it
          // actually removed something:
          int preCount = dir.list().length;
          writer = new IndexWriter(dir, false, new WhitespaceAnalyzer(), false, policy);
          writer.close();
          int postCount = dir.list().length;
          assertTrue(postCount < preCount);
        }
      }

      dir.close();
    }
public voidtestKeepLastNDeletionPolicy()


    final int N = 5;

    for(int pass=0;pass<4;pass++) {

      boolean autoCommit = pass < 2;
      boolean useCompoundFile = (pass % 2) > 0;

      Directory dir = new RAMDirectory();

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      for(int j=0;j<N+1;j++) {
        IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
        writer.setUseCompoundFile(useCompoundFile);
        for(int i=0;i<17;i++) {
          addDoc(writer);
        }
        writer.optimize();
        writer.close();
      }

      assertTrue(policy.numDelete > 0);
      assertEquals(N+1, policy.numOnInit);
      if (autoCommit) {
        assertTrue(policy.numOnCommit > 1);
      } else {
        assertEquals(N+1, policy.numOnCommit);
      }

      // Simplistic check: just verify only the past N segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);
      for(int i=0;i<N+1;i++) {
        try {
          IndexReader reader = IndexReader.open(dir);
          reader.close();
          if (i == N) {
            fail("should have failed on commits prior to last " + N);
          }
        } catch (IOException e) {
          if (i != N) {
            throw e;
          }
        }
        if (i < N) {
          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        }
        gen--;
      }

      dir.close();
    }
public voidtestKeepLastNDeletionPolicyWithCreates()


    final int N = 10;

    for(int pass=0;pass<4;pass++) {

      boolean autoCommit = pass < 2;
      boolean useCompoundFile = (pass % 2) > 0;

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      Directory dir = new RAMDirectory();
      IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
      writer.setUseCompoundFile(useCompoundFile);
      writer.close();
      Term searchTerm = new Term("content", "aaa");        
      Query query = new TermQuery(searchTerm);

      for(int i=0;i<N+1;i++) {

        writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
        writer.setUseCompoundFile(useCompoundFile);
        for(int j=0;j<17;j++) {
          addDoc(writer);
        }
        // this is a commit when autoCommit=false:
        writer.close();
        IndexReader reader = IndexReader.open(dir, policy);
        reader.deleteDocument(3);
        reader.setNorm(5, "content", 2.0F);
        IndexSearcher searcher = new IndexSearcher(reader);
        Hits hits = searcher.search(query);
        assertEquals(16, hits.length());
        // this is a commit when autoCommit=false:
        reader.close();
        searcher.close();

        writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
        // This will not commit: there are no changes
        // pending because we opened for "create":
        writer.close();
      }

      assertEquals(1+3*(N+1), policy.numOnInit);
      if (autoCommit) {
        assertTrue(policy.numOnCommit > 3*(N+1)-1);
      } else {
        assertEquals(2*(N+1), policy.numOnCommit);
      }

      IndexSearcher searcher = new IndexSearcher(dir);
      Hits hits = searcher.search(query);
      assertEquals(0, hits.length());

      // Simplistic check: just verify only the past N segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);

      int expectedCount = 0;

      for(int i=0;i<N+1;i++) {
        try {
          IndexReader reader = IndexReader.open(dir);

          // Work backwards in commits on what the expected
          // count should be.  Only check this in the
          // autoCommit false case:
          if (!autoCommit) {
            searcher = new IndexSearcher(reader);
            hits = searcher.search(query);
            assertEquals(expectedCount, hits.length());
            searcher.close();
            if (expectedCount == 0) {
              expectedCount = 16;
            } else if (expectedCount == 16) {
              expectedCount = 17;
            } else if (expectedCount == 17) {
              expectedCount = 0;
            }
          }
          reader.close();
          if (i == N) {
            fail("should have failed on commits before last " + N);
          }
        } catch (IOException e) {
          if (i != N) {
            throw e;
          }
        }
        if (i < N) {
          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        }
        gen--;
      }
      
      dir.close();
    }
public voidtestKeepLastNDeletionPolicyWithReader()


    final int N = 10;

    for(int pass=0;pass<4;pass++) {

      boolean autoCommit = pass < 2;
      boolean useCompoundFile = (pass % 2) > 0;

      KeepLastNDeletionPolicy policy = new KeepLastNDeletionPolicy(N);

      Directory dir = new RAMDirectory();
      IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
      writer.setUseCompoundFile(useCompoundFile);
      writer.close();
      Term searchTerm = new Term("content", "aaa");        
      Query query = new TermQuery(searchTerm);

      for(int i=0;i<N+1;i++) {
        writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
        writer.setUseCompoundFile(useCompoundFile);
        for(int j=0;j<17;j++) {
          addDoc(writer);
        }
        // this is a commit when autoCommit=false:
        writer.close();
        IndexReader reader = IndexReader.open(dir, policy);
        reader.deleteDocument(3*i+1);
        reader.setNorm(4*i+1, "content", 2.0F);
        IndexSearcher searcher = new IndexSearcher(reader);
        Hits hits = searcher.search(query);
        assertEquals(16*(1+i), hits.length());
        // this is a commit when autoCommit=false:
        reader.close();
        searcher.close();
      }
      writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
      writer.setUseCompoundFile(useCompoundFile);
      writer.optimize();
      // this is a commit when autoCommit=false:
      writer.close();

      assertEquals(2*(N+2), policy.numOnInit);
      if (autoCommit) {
        assertTrue(policy.numOnCommit > 2*(N+2)-1);
      } else {
        assertEquals(2*(N+2)-1, policy.numOnCommit);
      }

      IndexSearcher searcher = new IndexSearcher(dir);
      Hits hits = searcher.search(query);
      assertEquals(176, hits.length());

      // Simplistic check: just verify only the past N segments_N's still
      // exist, and, I can open a reader on each:
      long gen = SegmentInfos.getCurrentSegmentGeneration(dir);

      int expectedCount = 176;

      for(int i=0;i<N+1;i++) {
        try {
          IndexReader reader = IndexReader.open(dir);

          // Work backwards in commits on what the expected
          // count should be.  Only check this in the
          // autoCommit false case:
          if (!autoCommit) {
            searcher = new IndexSearcher(reader);
            hits = searcher.search(query);
            if (i > 1) {
              if (i % 2 == 0) {
                expectedCount += 1;
              } else {
                expectedCount -= 17;
              }
            }
            assertEquals(expectedCount, hits.length());
            searcher.close();
          }
          reader.close();
          if (i == N) {
            fail("should have failed on commits before last 5");
          }
        } catch (IOException e) {
          if (i != N) {
            throw e;
          }
        }
        if (i < N) {
          dir.deleteFile(IndexFileNames.fileNameFromGeneration(IndexFileNames.SEGMENTS, "", gen));
        }
        gen--;
      }
      
      dir.close();
    }
public voidtestKeepNoneOnInitDeletionPolicy()


    for(int pass=0;pass<4;pass++) {

      boolean autoCommit = pass < 2;
      boolean useCompoundFile = (pass % 2) > 0;

      KeepNoneOnInitDeletionPolicy policy = new KeepNoneOnInitDeletionPolicy();

      Directory dir = new RAMDirectory();

      IndexWriter writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), true, policy);
      writer.setUseCompoundFile(useCompoundFile);
      for(int i=0;i<107;i++) {
        addDoc(writer);
      }
      writer.close();

      writer = new IndexWriter(dir, autoCommit, new WhitespaceAnalyzer(), false, policy);
      writer.setUseCompoundFile(useCompoundFile);
      writer.optimize();
      writer.close();

      assertEquals(2, policy.numOnInit);
      if (autoCommit) {
        assertTrue(policy.numOnCommit > 2);
      } else {
        // If we are not auto committing then there should
        // be exactly 2 commits (one per close above):
        assertEquals(2, policy.numOnCommit);
      }

      // Simplistic check: just verify the index is in fact
      // readable:
      IndexReader reader = IndexReader.open(dir);
      reader.close();

      dir.close();
    }
private voidverifyCommitOrder(java.util.List commits)

    long last = SegmentInfos.generationFromSegmentsFileName(((IndexCommitPoint) commits.get(0)).getSegmentsFileName());
    for(int i=1;i<commits.size();i++) {
      long now = SegmentInfos.generationFromSegmentsFileName(((IndexCommitPoint) commits.get(i)).getSegmentsFileName());
      assertTrue("SegmentInfos commits are out-of-order", now > last);
      last = now;
    }