package com.develop.ss;
import junit.framework.Test;
import junit.framework.TestSuite;
import junit.framework.TestResult;
import junit.textui.TestRunner;
import com.meterware.httpunit.WebConversation;
import com.meterware.httpunit.WebLink;
import java.util.HashSet;
import java.util.Set;
import java.util.Iterator;
import java.util.StringTokenizer;
import java.util.ArrayList;
import java.net.MalformedURLException;
import java.net.URL;
import java.io.FileOutputStream;
import java.io.FileNotFoundException;
import java.io.PrintStream;
import java.io.IOException;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.analysis.standard.StandardAnalyzer;
import org.apache.lucene.analysis.Analyzer;
public class LinkTestSuite extends TestSuite {
WebConversation conversation = new WebConversation();
Set linksAlreadyFollowed = new HashSet();
Set linksNotFollowed = new HashSet();
Set linkPrefixesToFollow = new HashSet();
HashSet linkPrefixesToAvoid = new HashSet();
private String linksNotFollowedOutputFileName;
private int maxLinks = Integer.MAX_VALUE;
private IndexWriter writer;
private String initialLink;
public String getInitialLink() {
return initialLink;
}
public void setInitialLink(String initialLink) throws MalformedURLException {
if ((initialLink == null) || (initialLink.length() == 0)) {
throw new Error("Must specify a non-null initialLink");
}
linkPrefixesToFollow.add(new URL(initialLink));
this.initialLink = initialLink;
LinkTest.log.info("Initial link is " + initialLink);
addTest(new LinkTest(initialLink,conversation,this));
}
public IndexWriter getWriter() {
return writer;
}
public LinkTestSuite(String indexPath, int maxLinks, String skippedLinksOutputFileName) throws IOException {
this.maxLinks = maxLinks;
this.linksNotFollowedOutputFileName = skippedLinksOutputFileName;
writer = new IndexWriter(indexPath, new StandardAnalyzer(), true);
}
public void followPrefixes(String[] prefixesToFollow) throws MalformedURLException {
for (int i = 0; i < prefixesToFollow.length; i++) {
String s = prefixesToFollow[i];
LinkTest.log.info("Following links prefixed with " + s);
linkPrefixesToFollow.add(new URL(s));
}
}
public void avoidPrefixes(String[] prefixesToAvoid) throws MalformedURLException {
for (int i = 0; i < prefixesToAvoid.length; i++) {
String s = prefixesToAvoid[i];
LinkTest.log.info("Avoid links prefixed with " + s);
linkPrefixesToAvoid.add(new URL(s));
}
}
private static String[] EMPTY_STRINGARRAY = new String[0];
public static Test suite() throws IOException {
String indexPath = System.getProperty("com.develop.ss.Index", "index.1");
String initialLink = System.getProperty("com.develop.ss.InitialLink");
int maxLinks = Integer.MAX_VALUE;
String maxLinksString = System.getProperty("com.develop.ss.MaxLinks");
if (maxLinksString != null) {
maxLinks = Integer.parseInt(maxLinksString);
}
LinkTestSuite suite = new LinkTestSuite(indexPath, maxLinks, "linksNotFollowed.txt");
suite.setInitialLink(initialLink);
suite.initFollowPrefixesFromSystemProperties();
suite.initAvoidPrefixesFromSystemProperties();
return suite;
}
public void initFollowPrefixesFromSystemProperties() throws MalformedURLException {
String followPrefixes = System.getProperty("com.develop.ss.FollowLinks");
if (followPrefixes == null || followPrefixes.length() == 0) return;
String[] prefixes = followPrefixes.split(" ");
if (prefixes != null && prefixes.length != 0) {
followPrefixes(prefixes);
}
}
public void initAvoidPrefixesFromSystemProperties() throws MalformedURLException {
String avoidPrefixes = System.getProperty("com.develop.ss.AvoidLinks");
if (avoidPrefixes == null || avoidPrefixes.length() == 0) return;
String[] prefixes = avoidPrefixes.split(" ");
if (prefixes != null && prefixes.length != 0) {
avoidPrefixes(prefixes);
}
}
public void run(TestResult result) {
super.run(result);
try {
writeStringSet(linksNotFollowedOutputFileName, linksNotFollowed);
} catch (Exception e) {
e.printStackTrace();
}
try {
writer.optimize();
writer.close();
} catch (Exception e) {
e.printStackTrace();
}
}
private void writeStringSet(String s, Set skippedPaths) throws IOException {
FileOutputStream fos = new FileOutputStream(s);
PrintStream ps = new PrintStream(fos);
Iterator it = skippedPaths.iterator();
while (it.hasNext()) {
ps.println(it.next());
}
fos.flush();
fos.close();
}
boolean shouldFollowLink(URL newLink) {
for (Iterator iterator = linkPrefixesToFollow.iterator(); iterator.hasNext();) {
URL u = (URL) iterator.next();
if (matchesDownToPathPrefix(u, newLink)) {
return true;
}
}
return false;
}
boolean shouldNotFollowLink(URL newLink) {
for (Iterator iterator = linkPrefixesToAvoid.iterator(); iterator.hasNext();) {
URL u = (URL) iterator.next();
if (matchesDownToPathPrefix(u, newLink)) {
return true;
}
}
return false;
}
private boolean matchesDownToPathPrefix(URL matchBase, URL newLink) {
return matchBase.getHost().equals(newLink.getHost()) &&
matchBase.getPort() == newLink.getPort() &&
matchBase.getProtocol().equals(newLink.getProtocol()) &&
newLink.getPath().startsWith(matchBase.getPath());
}
void considerNewLink(String linkFrom, WebLink newLink) throws MalformedURLException {
URL url = null;
url = newLink.getRequest().getURL();
if (shouldFollowLink(url)) {
if (linksAlreadyFollowed.add(url.toExternalForm())) {
if (linksAlreadyFollowed.size() > maxLinks) {
linksAlreadyFollowed.remove(url.toExternalForm());
throw new Error("Max links exceeded " + maxLinks);
}
if (shouldNotFollowLink(url)) {
LinkTest.log.info("Not following " + url.toExternalForm() + " from " + linkFrom);
} else {
LinkTest.log.info("Following " + url.toExternalForm() + " from " + linkFrom);
addTest(new LinkTest(url.toString(),conversation, this));
}
}
} else {
ignoreLink(url, linkFrom);
}
}
private void ignoreLink(URL url, String linkFrom) {
String status = "Ignoring " + url.toExternalForm() + " from " + linkFrom;
linksNotFollowed.add(status);
LinkTest.log.fine(status);
}
public void execute()
{
junit.textui.TestRunner.run(this);
}
}
|