org.gcube.resource.discovery.crawler
Class Configs

java.lang.Object
  extended by org.gcube.resource.discovery.crawler.Configs

public class Configs
extends java.lang.Object


Field Summary
static java.lang.String agentName
           
static java.lang.String[] allowedUrl
           
static boolean allowWholeDomain
           
static boolean allowWholeHost
           
static int bandwidth
           
static boolean enableCookies
           
static boolean flexibleHostCheck
           
static boolean ignoreRobotsTxt
           
static boolean localizeLinks
           
static int maxDepth
           
static int maxDocumentAge
           
static java.lang.String proxy
           
static int sleepTime
           
static java.lang.String startReferer
           
static java.lang.String[] visitMany
           
static boolean walkToOtherHosts
           
 
Constructor Summary
Configs()
           
 
Method Summary
static void configureCrawler(WebRobot crawler)
           
static void configureCrawlerForValidation(WebRobot crawler)
           
static void main(java.lang.String[] args)
           
 
Methods inherited from class java.lang.Object
equals, getClass, hashCode, notify, notifyAll, toString, wait, wait, wait
 

Field Detail

agentName

public static java.lang.String agentName

ignoreRobotsTxt

public static boolean ignoreRobotsTxt

sleepTime

public static int sleepTime

maxDepth

public static int maxDepth

walkToOtherHosts

public static boolean walkToOtherHosts

allowWholeHost

public static boolean allowWholeHost

allowWholeDomain

public static boolean allowWholeDomain

flexibleHostCheck

public static boolean flexibleHostCheck

localizeLinks

public static boolean localizeLinks

enableCookies

public static boolean enableCookies

startReferer

public static java.lang.String startReferer

maxDocumentAge

public static int maxDocumentAge

allowedUrl

public static java.lang.String[] allowedUrl

visitMany

public static java.lang.String[] visitMany

proxy

public static java.lang.String proxy

bandwidth

public static int bandwidth
Constructor Detail

Configs

public Configs()
Method Detail

configureCrawlerForValidation

public static void configureCrawlerForValidation(WebRobot crawler)
                                          throws java.io.IOException
Throws:
java.io.IOException

configureCrawler

public static void configureCrawler(WebRobot crawler)
                             throws java.io.IOException,
                                    HttpException
Throws:
java.io.IOException
HttpException

main

public static void main(java.lang.String[] args)