org.gcube.resource.discovery.crawler
Class Configs
java.lang.Object
org.gcube.resource.discovery.crawler.Configs
public class Configs
- extends java.lang.Object
Methods inherited from class java.lang.Object |
equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait |
agentName
public static java.lang.String agentName
ignoreRobotsTxt
public static boolean ignoreRobotsTxt
sleepTime
public static int sleepTime
maxDepth
public static int maxDepth
walkToOtherHosts
public static boolean walkToOtherHosts
allowWholeHost
public static boolean allowWholeHost
allowWholeDomain
public static boolean allowWholeDomain
flexibleHostCheck
public static boolean flexibleHostCheck
localizeLinks
public static boolean localizeLinks
enableCookies
public static boolean enableCookies
startReferer
public static java.lang.String startReferer
maxDocumentAge
public static int maxDocumentAge
allowedUrl
public static java.lang.String[] allowedUrl
visitMany
public static java.lang.String[] visitMany
proxy
public static java.lang.String proxy
bandwidth
public static int bandwidth
Configs
public Configs()
configureCrawlerForValidation
public static void configureCrawlerForValidation(WebRobot crawler)
throws java.io.IOException
- Throws:
java.io.IOException
configureCrawler
public static void configureCrawler(WebRobot crawler)
throws java.io.IOException,
HttpException
- Throws:
java.io.IOException
HttpException
main
public static void main(java.lang.String[] args)