C# 0 The Complete Reference

href="http

 hypertext command are found. Relative links are not used. Second, there is 

no way to go back to an earlier link. Third, it displays only the links and no surrounding 

content. Despite these limitations, the skeleton is fully functional, and you will have no 

trouble enhancing MiniCrawler to perform other tasks. In fact, adding features to MiniCrawler 

is a good way to learn more about the networking classes and networking in general.

Here is the entire code for MiniCrawler:

/* MiniCrawler: A skeletal Web crawler.

   Usage:

     To start crawling, specify a starting

     URI on the command line. For example,

     to start at McGraw-Hill.com, use this

     command line:

       MiniCrawler http://McGraw-Hill.com

*/

using System;

using System.Net;

using System.IO;

class MiniCrawler {

  // Find a link in a content string.

  static string FindLink(string htmlstr,

                         ref int startloc) {

    int i;

    int start, end;

    string uri = null;

    string lowcasestr = htmlstr.ToLower();

    i = lowcasestr.IndexOf("href=\"http", startloc);

    if(i != -1) {

      start = htmlstr.IndexOf(‘"’, i) + 1;

      end = htmlstr.IndexOf(‘"’, start);

      uri = htmlstr.Substring(start, end-start);

      startloc = end;

    }

www.freepdf-books.com

---

840

 

P a r t   I I :  

E x p l o r i n g   t h e   C #   L i b r a r y

    return uri;

  }

  static void Main(string[] args) {

    string link = null;

    string str;

    string answer;

    int curloc; // holds current location in response

    if(args.Length != 1) {

      Console.WriteLine("Usage: MiniCrawler ");

      return ;

    }

    string uristr = args[0]; // holds current URI

    try {

      do {

        Console.WriteLine("Linking to " + uristr);

        // Create a WebRequest to the specified URI.

        HttpWebRequest req = (HttpWebRequest)

               WebRequest.Create(uristr);

        uristr = null; // disallow further use of this URI

        // Send that request and return the response.

        HttpWebResponse resp = (HttpWebResponse)

               req.GetResponse();

        // From the response, obtain an input stream.

        Stream istrm = resp.GetResponseStream();

        // Wrap the input stream in a StreamReader.

        StreamReader rdr = new StreamReader(istrm);

        // Read in the entire page.

        str = rdr.ReadToEnd();

        curloc = 0;

        do {

          // Find the next URI to link to.

          link = FindLink(str, ref curloc);

          if(link != null) {

            Console.WriteLine("Link found: " + link);

            Console.Write("Link, More, Quit?");

            answer = Console.ReadLine();

            if(string.Compare(answer, "L", true) == 0) {

              uristr = string.Copy(link);

www.freepdf-books.com

---