Results 1 to 2 of 2
  1. #1
    rahulraj is offline Member
    Join Date
    Mar 2011
    Posts
    9
    Rep Power
    0

    Smile What is the error?

    I just tried to run the java web crawler program that got from java.sun.com

    Java Code:
    import java.applet.Applet;
    import java.text.*;
    import java.awt.*;
    import java.awt.event.*;
    import java.util.*;
    import java.net.*;
    import java.io.*;
    
    public class WebCrawler extends Applet implements ActionListener, Runnable {
        public static final String SEARCH = "Search";
        public static final String STOP = "Stop";
        public static final String DISALLOW = "Disallow:";
        public static final int    SEARCH_LIMIT = 50;
    
        Panel   panelMain;
        List    listMatches;
        Label   labelStatus;
    
        // URLs to be searched
        Vector vectorToSearch;
        // URLs already searched
        Vector vectorSearched;
        // URLs which match
        Vector vectorMatches;
    
        Thread searchThread;
    
        TextField textURL;
        Choice    choiceType;
    
        public void init() {
    
    	// set up the main UI panel
    	panelMain = new Panel();
    	panelMain.setLayout(new BorderLayout(5, 5));
    
    	// text entry components
    	Panel panelEntry = new Panel();
    	panelEntry.setLayout(new BorderLayout(5, 5));
    
    	Panel panelURL = new Panel();
    	panelURL.setLayout(new FlowLayout(FlowLayout.LEFT, 5, 5));
    	Label labelURL = new Label("Starting URL: ", Label.RIGHT);
    	panelURL.add(labelURL);
    	textURL = new TextField("", 40);
    	panelURL.add(textURL);
    	panelEntry.add("North", panelURL);
    
    	Panel panelType = new Panel();
    	panelType.setLayout(new FlowLayout(FlowLayout.LEFT, 5, 5));
    	Label labelType = new Label("Content type: ", Label.RIGHT);
    	panelType.add(labelType);
    	choiceType = new Choice();
    	choiceType.addItem("text/html");
    	choiceType.addItem("audio/basic");
    	choiceType.addItem("audio/au");
    	choiceType.addItem("audio/aiff");
    	choiceType.addItem("audio/wav");
    	choiceType.addItem("video/mpeg");
    	choiceType.addItem("video/x-avi");
    	panelType.add(choiceType);
    	panelEntry.add("South", panelType);
    
    	panelMain.add("North", panelEntry);
    
    	// list of result URLs
    	Panel panelListButtons = new Panel();
    	panelListButtons.setLayout(new BorderLayout(5, 5));
    
    	Panel panelList = new Panel();
    	panelList.setLayout(new BorderLayout(5, 5));
    	Label labelResults = new Label("Search results");
    	panelList.add("North", labelResults);
    	Panel panelListCurrent = new Panel();
    	panelListCurrent.setLayout(new BorderLayout(5, 5));
    	listMatches = new List(10);
    	panelListCurrent.add("North", listMatches);
    	labelStatus = new Label("");
    	panelListCurrent.add("South", labelStatus);
    	panelList.add("South", panelListCurrent);
    
    	panelListButtons.add("North", panelList);
    
    	// control buttons
    	Panel panelButtons = new Panel();
    	Button buttonSearch = new Button(SEARCH);
    	buttonSearch.addActionListener(this);
    	panelButtons.add(buttonSearch);
    	Button buttonStop = new Button(STOP);
    	buttonStop.addActionListener(this);
    	panelButtons.add(buttonStop);
    
    	panelListButtons.add("South", panelButtons);
    
    	panelMain.add("South", panelListButtons);
    
    	add(panelMain);
    	setVisible(true);
    
    	repaint(); 
    
    	// initialize search data structures
    	vectorToSearch = new Vector();
    	vectorSearched = new Vector();
    	vectorMatches = new Vector();
    
    	// set default for URL access
    	URLConnection.setDefaultAllowUserInteraction(false);
        }
    
        public void start() {
        }
    
        public void stop() {
    	if (searchThread != null) {
    	    setStatus("stopping...");
    	    searchThread = null;
    	}
        }
    
        public void destroy() {
        }
    
        boolean robotSafe(URL url) {
    	String strHost = url.getHost();
    
    	// form URL of the robots.txt file
    	String strRobot = "http://" + strHost + "/robots.txt";
    	URL urlRobot;
    	try { 
    	    urlRobot = new URL(strRobot);
    	} catch (MalformedURLException e) {
    	    // something weird is happening, so don't trust it
    	    return false;
    	}
    
    	String strCommands;
    	try {
    	    InputStream urlRobotStream = urlRobot.openStream();
    
    	    // read in entire file
    	    byte b[] = new byte[1000];
    	    int numRead = urlRobotStream.read(b);
    	    strCommands = new String(b, 0, numRead);
    	    while (numRead != -1) {
    		if (Thread.currentThread() != searchThread)
    		    break;
    		numRead = urlRobotStream.read(b);
    		if (numRead != -1) {
    		    String newCommands = new String(b, 0, numRead);
    		    strCommands += newCommands;
    		}
    	    }
    	    urlRobotStream.close();
    	} catch (IOException e) {
    	    // if there is no robots.txt file, it is OK to search
    	    return true;
    	}
    
    	// assume that this robots.txt refers to us and 
    	// search for "Disallow:" commands.
    	String strURL = url.getFile();
    	int index = 0;
    	while ((index = strCommands.indexOf(DISALLOW, index)) != -1) {
    	    index += DISALLOW.length();
    	    String strPath = strCommands.substring(index);
    	    StringTokenizer st = new StringTokenizer(strPath);
    
    	    if (!st.hasMoreTokens())
    		break;
    	    
    	    String strBadPath = st.nextToken();
    
    	    // if the URL starts with a disallowed path, it is not safe
    	    if (strURL.indexOf(strBadPath) == 0)
    		return false;
    	}
    
    	return true;
        }
    
        public void paint(Graphics g) {
          	//Draw a Rectangle around the applet's display area.
          	g.drawRect(0, 0, getSize().width - 1, getSize().height - 1);
    
    	panelMain.paint(g);
    	panelMain.paintComponents(g);
    	// update(g);
    	// panelMain.update(g);
        }
    
        public void run() {
    	String strURL = textURL.getText();
    	String strTargetType = choiceType.getSelectedItem();
    	int numberSearched = 0;
    	int numberFound = 0;
    
    	if (strURL.length() == 0) {
    	    setStatus("ERROR: must enter a starting URL");
    	    return;
    	}
    
    	// initialize search data structures
    	vectorToSearch.removeAllElements();
    	vectorSearched.removeAllElements();
    	vectorMatches.removeAllElements();
    	listMatches.removeAll();
    
    	vectorToSearch.addElement(strURL);
    
    	while ((vectorToSearch.size() > 0) 
    	  && (Thread.currentThread() == searchThread)) {
    	    // get the first element from the to be searched list
    	    strURL = (String) vectorToSearch.elementAt(0);
    
    	    setStatus("searching " + strURL);
    
    	    URL url;
    	    try { 
    		url = new URL(strURL);
    	    } catch (MalformedURLException e) {
    		setStatus("ERROR: invalid URL " + strURL);
    		break;
    	    }
    
    	    // mark the URL as searched (we want this one way or the other)
    	    vectorToSearch.removeElementAt(0);
    	    vectorSearched.addElement(strURL);
    
    	    // can only search http: protocol URLs
    	    if (url.getProtocol().compareTo("http") != 0) 
    		break;
    
    	    // test to make sure it is before searching
    	    if (!robotSafe(url))
    		break;
    
    	    try {
    		// try opening the URL
    		URLConnection urlConnection = url.openConnection();
    
    		urlConnection.setAllowUserInteraction(false);
    
    		InputStream urlStream = url.openStream();
    		String type 
    		  = urlConnection.guessContentTypeFromStream(urlStream);
    		if (type == null)
    		    break;
    		if (type.compareTo("text/html") != 0) 
    		    break;
    
    		// search the input stream for links
    		// first, read in the entire URL
    		byte b[] = new byte[1000];
    		int numRead = urlStream.read(b);
    		String content = new String(b, 0, numRead);
    		while (numRead != -1) {
    		    if (Thread.currentThread() != searchThread)
    			break;
    		    numRead = urlStream.read(b);
    		    if (numRead != -1) {
    			String newContent = new String(b, 0, numRead);
    			content += newContent;
    		    }
    		}
    		urlStream.close();
    
    		if (Thread.currentThread() != searchThread)
    		    break;
    
    		String lowerCaseContent = content.toLowerCase();
    
    		int index = 0;
    		while ((index = lowerCaseContent.indexOf("<a", index)) != -1)
    		{
    		    if ((index = lowerCaseContent.indexOf("href", index)) == -1) 
    			break;
    		    if ((index = lowerCaseContent.indexOf("=", index)) == -1) 
    			break;
    		    
    		    if (Thread.currentThread() != searchThread)
    			break;
    
    		    index++;
    		    String remaining = content.substring(index);
    
    		    StringTokenizer st 
    		      = new StringTokenizer(remaining, "\t\n\r\">#");
    		    String strLink = st.nextToken();
    
    		    URL urlLink;
    		    try {
    			urlLink = new URL(url, strLink);
    			strLink = urlLink.toString();
    		    } catch (MalformedURLException e) {
    			setStatus("ERROR: bad URL " + strLink);
    			continue;
    		    }
    
    		    // only look at http links
    		    if (urlLink.getProtocol().compareTo("http") != 0)
    			break;
    
    		    if (Thread.currentThread() != searchThread)
    			break;
    
    		    try {
    			// try opening the URL
    			URLConnection urlLinkConnection 
    			  = urlLink.openConnection();
    			urlLinkConnection.setAllowUserInteraction(false);
    			InputStream linkStream = urlLink.openStream();
    			String strType 
    			  = urlLinkConnection.guessContentTypeFromStream(linkStream);
    			linkStream.close();
    
    			// if another page, add to the end of search list
    			if (strType == null)
    			    break;
    			if (strType.compareTo("text/html") == 0) {
    			    // check to see if this URL has already been 
    			    // searched or is going to be searched
    			    if ((!vectorSearched.contains(strLink)) 
    			      && (!vectorToSearch.contains(strLink))) {
    
    				// test to make sure it is robot-safe!
    				if (robotSafe(urlLink))
    				    vectorToSearch.addElement(strLink);
    			    }
    			}
    
    			// if the proper type, add it to the results list
    			// unless we have already seen it
    			if (strType.compareTo(strTargetType) == 0) {
    			    if (vectorMatches.contains(strLink) == false) {
    				listMatches.add(strLink);
    				vectorMatches.addElement(strLink);
    				numberFound++;
    				if (numberFound >= SEARCH_LIMIT)
    				    break;
    			    }
    			}
    		    } catch (IOException e) {
    			setStatus("ERROR: couldn't open URL " + strLink);
    			continue;
    		    }
    		}
    	    } catch (IOException e) {
    		setStatus("ERROR: couldn't open URL " + strURL);
    		break;
    	    }
    
    	    numberSearched++;
    	    if (numberSearched >= SEARCH_LIMIT)
    		break;
    	}
    
    	if (numberSearched >= SEARCH_LIMIT || numberFound >= SEARCH_LIMIT)
    	    setStatus("reached search limit of " + SEARCH_LIMIT);
    	else
    	    setStatus("done");
    	searchThread = null;
    	// searchThread.stop();
        }
    
        void setStatus(String status) {
    	labelStatus.setText(status);
        }
    
        public void actionPerformed(ActionEvent event) {
    	String command = event.getActionCommand();
    
    	if (command.compareTo(SEARCH) == 0) {
    	    setStatus("searching...");
    
    	    // launch a thread to do the search
    	    if (searchThread == null) {
    		searchThread = new Thread(this);
    	    }
    	    searchThread.start();
    	}
    	else if (command.compareTo(STOP) == 0) {
    	    stop();
    	}
        }
            public static void main (String argv[])
            {
                    Frame f = new Frame("WebFrame");
                    WebCrawler applet = new WebCrawler();
    		f.add("Center", applet);
    
    /*		Behind a firewall set your proxy and port here!
    */
                    Properties props= new Properties(System.getProperties());
                    props.put("http.proxySet", "true");
            	props.put("http.proxyHost", "webcache-cup");
            	props.put("http.proxyPort", "8080");
    
                    Properties newprops = new Properties(props);
                    System.setProperties(newprops);
    /**/
    
    		
                    applet.init();
                    applet.start();
                    f.pack();
                    f.show();
            }
    
    }
    I got 2 errors when I compiled..



    Can anyone help me to solve this?

  2. #2
    pbrockway2 is offline Moderator
    Join Date
    Feb 2009
    Location
    New Zealand
    Posts
    4,585
    Rep Power
    12

    Default

    This is very old code. (Writing a Web Crawler in the Java Programming Language is dated January 1998)

    As the compiler says, the class List is ambiguous: there is more than one List class referred to in the import statements at the start of the code. The point is that there may not have been two such classes back in 1998, but there are now.

    I suggest you remove all the import lines and replace them with statements that import just the classes you are using. This is good practice anyway (the code becomes more self documenting) and, if you are studying the code, it forces you to actually think about exactly which classes are being used.

Similar Threads

  1. java out of memory error-heap space error
    By elsanthosh in forum NetBeans
    Replies: 4
    Last Post: 06-15-2010, 10:31 AM
  2. Replies: 1
    Last Post: 03-10-2010, 02:53 PM
  3. Replies: 1
    Last Post: 12-05-2009, 08:28 AM
  4. Replies: 1
    Last Post: 10-26-2009, 05:38 PM
  5. Replies: 3
    Last Post: 01-23-2009, 09:53 AM

Tags for this Thread

Posting Permissions

  • You may not post new threads
  • You may not post replies
  • You may not post attachments
  • You may not edit your posts
  •