FileDocCategorySizeDatePackage
TrecDocMaker.javaAPI DocApache Lucene 2.2.06781Sat Jun 16 22:20:58 BST 2007org.apache.lucene.benchmark.byTask.feeds

TrecDocMaker

public class TrecDocMaker extends BasicDocMaker
A DocMaker using the (compressed) Trec collection for its input.

Fields Summary
private static final String
newline
private DateFormat[]
dateFormat
private File
dataDir
private ArrayList
inputFiles
private int
nextFile
private int
iteration
private BufferedReader
reader
private GZIPInputStream
zis
private static final String[]
DATE_FORMATS
Constructors Summary
Methods Summary
private voidcloseInputs()

    if (zis!=null) {
      try {
        zis.close();
      } catch (IOException e) {
        System.out.println("closeInputs(): Ingnoring error: "+e);
        e.printStackTrace();
      }
      zis = null;
    }
    if (reader!=null) { 
      try {
        reader.close();
      } catch (IOException e) {
        System.out.println("closeInputs(): Ingnoring error: "+e);
        e.printStackTrace();
      }
      reader = null;
    }
protected DocDatagetNextDocData()

    if (reader==null) {
      openNextFile();
    }
    // 1. skip until doc start
    read("<DOC>",null,false,false); 
    // 2. name
    StringBuffer sb = read("<DOCNO>",null,true,false);
    String name = sb.substring("<DOCNO>".length());
    name = name.substring(0,name.indexOf("</DOCNO>"))+"_"+iteration;
    // 3. skip until doc header
    read("<DOCHDR>",null,false,false); 
    // 4. date
    sb = read("Date: ",null,true,false);
    String dateStr = sb.substring("Date: ".length());
    // 5. skip until end of doc header
    read("</DOCHDR>",null,false,false); 
    // 6. collect until end of doc
    sb = read("</DOC>",null,false,true);
    // this is the next document, so parse it 
    Date date = parseDate(dateStr);
    HTMLParser p = getHtmlParser();
    DocData docData = p.parse(name, date, sb, dateFormat[0]);
    addBytes(sb.length()); // count char length of parsed html text (larger than the plain doc body text). 
    
    return docData;
public intnumUniqueTexts()

    return inputFiles.size();
private voidopenNextFile()

    closeInputs();
    int retries = 0;
    while (true) {
      File f = null;
      synchronized (this) {
        if (nextFile >= inputFiles.size()) { 
          // exhausted files, start a new round, unless forever set to false.
          if (!forever) {
            throw new NoMoreDataException();
          }
          nextFile = 0;
          iteration++;
        }
        f = (File) inputFiles.get(nextFile++);
      }
      System.out.println("opening: "+f+" length: "+f.length());
      try {
        zis = new GZIPInputStream(new BufferedInputStream(new FileInputStream(f)));
        reader = new BufferedReader(new InputStreamReader(zis));
        return;
      } catch (Exception e) {
        retries++;
        if (retries<20) {
          System.out.println("Skipping 'bad' file "+f.getAbsolutePath()+"  #retries="+retries);
          continue;
        } else {
          throw new NoMoreDataException();
        }
      }
    }
private java.util.DateparseDate(java.lang.String dateStr)

    Date date = null;
    for (int i=0; i<dateFormat.length; i++) {
      try {
        date = dateFormat[i].parse(dateStr.trim());
        return date;
      } catch (ParseException e) {
      }
    }
    // do not fail test just because a date could not be parsed
    System.out.println("ignoring date parse exception (assigning 'now') for: "+dateStr);
    date = new Date(); // now 
    return date;
private java.lang.StringBufferread(java.lang.String prefix, java.lang.StringBuffer sb, boolean collectMatchLine, boolean collectAll)

    sb = (sb==null ? new StringBuffer() : sb);
    String sep = "";
    while (true) {
      String line = reader.readLine();
      if (line==null) {
        openNextFile();
        continue;
      }
      if (line.startsWith(prefix)) {
        if (collectMatchLine) {
          sb.append(sep+line);
          sep = newline;
        }
        break;
      }
      if (collectAll) {
        sb.append(sep+line);
        sep = newline;
      }
    }
    //System.out.println("read: "+sb);
    return sb;
public synchronized voidresetInputs()

    super.resetInputs();
    closeInputs();
    nextFile = 0;
    iteration = 0;
public voidsetConfig(org.apache.lucene.benchmark.byTask.utils.Config config)

  
  /* (non-Javadoc)
   * @see SimpleDocMaker#setConfig(java.util.Properties)
   */
      
    super.setConfig(config);
    String d = config.get("docs.dir","trec");
    dataDir = new File(new File("work"),d);
    collectFiles(dataDir,inputFiles);
    if (inputFiles.size()==0) {
      throw new RuntimeException("No txt files in dataDir: "+dataDir.getAbsolutePath());
    }
    // date format: 30-MAR-1987 14:22:36.87
    dateFormat = new SimpleDateFormat[DATE_FORMATS.length];
    for (int i = 0; i < dateFormat.length; i++) {
      dateFormat[i] = new SimpleDateFormat(DATE_FORMATS[i],Locale.US);
      dateFormat[i].setLenient(true);
    }