利用C#实现web信息自动抓取

背景

随着Internet的普及网络信息正以极高的速度增长在这么多数据中找到自己需要的信息是一件很繁琐的事情找到需要的信息后如何获取也是件麻烦的事这就需要Internet信息抓取程序来代替人工的操作

所谓Internet信息抓取程序就是程序会按照用户的关键词或关键网站来收集相应的信息并提供给用户想要的信息格式

信息量的增加会带来信息网站发布人员工作量的剧增为实现信息发布系统实现信息自

动发布减少工作人员工作量即时跟蹤最新信息就需要自动信息提供程序因此Internet信息抓取程序应运而生

目标

实现自定义网站信息分类抓取存入本地数据库生成静态页面或其它用户定义的信息结构并下载与信息相关的多媒体文件

开发

目标站点结构分析

本步骤是准确抓取信息个关键

首先要选择更新频率高的页面做为抓取地址然后分析要抓取内容页面url特点

然后分析要抓取信息页面的元素特性比如标题位置内容位置 等得到定位标记点

将以上信息写成自己的配置文件或存到数据库中

每个网站都需要分析写出单独的配置文件供抓取程序使用

信息提取

根据配置文件取得要抓取页面url使用HttpWebRequest类获取内容

//获取http页面函数

  public string Get_Http(string a_strUrlint timeout)

{

  string strResult ;  

  try

{

HttpWebRequest myReq = (HttpWebRequest)HttpWebRequestCreate(a_strUrl) ;

  myReqTimeout = timeout;

  HttpWebResponse HttpWResp = (HttpWebResponse)myReqGetResponse();

  Stream myStream = HttpWRespGetResponseStream () ;

  StreamReader sr = new StreamReader(myStream EncodingDefault);

  StringBuilder strBuilder = new StringBuilder();

  while ( != srPeek())

  {

   strBuilderAppend(srReadLine()+\r\n);

  }

  strResult = strBuilderToString();

}

  catch(Exception exp)

{

  strResult = 错误 + expMessage ;

}

  return strResult ;

}

获取页面内容后分析页面中连接地址取到要抓取的url

//处理页面标题和链接

  public string SniffWebUrl( string urlStrstring blockBstring blockE )

{  

  string urlch = ;

  string urlch = ;  

  int end_n = ;

  int end_nums = ;

  int end_nums = ;

  int end_nums = ;

  int end_nums = ;  

  string reUTStr = ;

  string reTitle = ;

  string ret = ;  

  try

{

  int pos = urlStrIndexOf( );

  int pos = urlStrLastIndexOf( / );

  if( pos < )

  {

   return ;

  }

  if( pos < )

  {

   return ;

  }

  int pos = urlStrIndexOf( /pos );

  if ( pos < )

  {

   urlch = urlStr;

   urlch = urlStr;

  }

  else

  {

   urlch = urlStrSubstring( pos );

   urlch = urlStrSubstring( pos );

  }

  string tmpAllStr = new PublicFun()Get_Http( urlStr time);

  int pos = tmpAllStrIndexOf( blockB );

  int pos = tmpAllStrIndexOf( blockEpos + blockBLength );

  if ( pos> && pos> && pos>pos )

  {

  ret = tmpAllStrSubstring( pos + blockBLengthpos pos blockBLength );

  ret = retSubstring( retIndexOf( < ));

   while( retIndexOf( <A ) >= )

  {

   ret = retSubstring( retIndexOf( <A ) ) + <a + retSubstring( retIndexOf( <A ) + );

  }

   while( retIndexOf( </A ) >= )

  {

   ret = retSubstring( retIndexOf( </A ) ) + </a + retSubstring( retIndexOf( </A ) + );

  }

   while( retIndexOf( Href= ) >= )

  {

   ret = retSubstring( retIndexOf( Href= )) + + retSubstring( retIndexOf( Href= ) + );

  }

   while( retIndexOf( HREF= ) >= )

  {

   ret = retSubstring( retIndexOf( HREF= )) + + retSubstring( retIndexOf( HREF= ) + );

  }

   while( retIndexOf( ) >= )

  {

   ret = retSubstring( retIndexOf( )) + href=\ + retSubstring( retIndexOf( ) + );

  }

  }

  tmpAllStr = ret;

  int begin_nums = tmpAllStrIndexOf( );

  while ( begin_nums >= )

  {  

   string tmpStrA = ;

   string tmpStrB = tmpAllStrSubstring( begin_nums + );

  if ( tmpStrB == \ )

  {

   end_n = begin_nums + ;

   if ( ( end_n + ) > tmpAllStrLength )

  {

   return ;

  }

   tmpStrA = tmpAllStrSubstring( begin_nums+ );

  }

  else

  {

   end_n = begin_nums + ;

   tmpStrA = tmpStrB;

  }

  if ( tmpStrA == # )

  {

   tmpAllStr = tmpAllStrSubstring( end_n );

   begin_nums = tmpAllStrIndexOf( );

  }

  else

  {   

   end_nums = tmpAllStrIndexOf( end_n );

   end_nums = tmpAllStrIndexOf( >end_n );

   end_nums = tmpAllStrIndexOf( </aend_nums );

   if ( ( end_nums >= ) && ( end_nums >= ) )

  {

   reTitle = tmpAllStrSubstring( end_nums + end_nums end_nums );

   if ( end_nums > end_nums )

   {

    end_nums = end_nums;

   }

   else

   {

    if ( end_nums < )

   {

    end_nums = end_nums;

   }

   else

   {

    end_nums = end_nums;

   }

   }

   string str = tmpAllStrSubstring( end_nums end_nums end_nums + );

   if ( str ==\|| str == )

   {

    end_nums = end_nums ;

   }

   string sTotalOne = tmpAllStrSubstring( end_nend_nums end_n );

   if ( sTotalOneIndexOf( // ) < )

   {

   if ( sTotalOneIndexOf( / ) == )

   {

     sTotalOne = urlch + sTotalOne;

   }

   else

   {   

    int linshiIntNum = ;

    int flags = ;

    string urlChange = urlStr;;

    while( sTotalOneIndexOf( / ) >= )

   {

    sTotalOne = sTotalOneSubstring( sTotalOneIndexOf( / ) + );

    linshiIntNum = linshiIntNum + ;

    flags = flags +;

   }

    while( ( urlChangeLastIndexOf( / ) >= ) && ( linshiIntNum >= ) )

   {

    urlChange = urlChangeSubstring( urlChangeLastIndexOf( / ) );

    linshiIntNum = linshiIntNum ;

   }

    if ( flags == )

   {

    sTotalOne = urlch + / + sTotalOne;

   }

    else

   {

    sTotalOne = urlChange + / + sTotalOne;

   }

   }

   }

   reUTStr = reUTStr + new PublicFun()RemoveHtmlCode( reTitle ) + sTotalOne;

   tmpAllStr = tmpAllStrSubstring( end_nums + );

   begin_nums = tmpAllStrIndexOf( );

  }

   else

  {

   begin_nums = ;

  }  

  }

  }

  return reUTStr;

}

  catch( Exception e)

{

  return ;

}

}

得到要抓取内容的url后处理该页面

//获取链接内容并分类处理

  public string GetWebContent( string gatherUrlstring subUrlstring subTitlestring b_Contentstring e_Contentstring b_Filterstring e_Filterstring root )

{

  string tmpAllStr = ;

  string dfStrB = ;

  string dfStrE = ;  

  string rePicStr = ;//图片返回路径

  string reContentStr = ;

  string picHtml = images; //本地图片路径

  string urlch =;

  string urlch =;

  int pos = gatherUrlIndexOf( );

  int pos = gatherUrlLastIndexOf( / );

  if( pos < )

{

  return ;

}

  if( pos < )

{  

  return ;

}

  int pos = gatherUrlIndexOf( /pos );

  if ( pos < )

{

  urlch = gatherUrl;

  urlch = gatherUrl;

}

  else

{

  urlch = gatherUrlSubstring( pos );

  urlch = gatherUrlSubstring( pos );

}

  tmpAllStr = new PublicFun()Get_Http( subUrltime );

  //取稿源

  string docFromStr = ;

  if ( dfStrB != && dfStrE != )

{

  if ( tmpAllStr != )

  {

  int b_docF = tmpAllStrIndexOf( dfStrB );

  if ( b_docF > )

  {

   int e_docF = tmpAllStrIndexOf( dfStrEb_docF + dfStrBLength );

   if ( e_docF > && e_docF > b_docF && e_docF b_docF < )

  {

   docFromStr = tmpAllStrSubstring( b_docF + dfStrBLength e_docF b_docF dfStrBLength );

  }

  }

  }

}

  //取内容

  if ( tmpAllStr != )

{  

  int begin_strnum = tmpAllStrIndexOf( b_Content );

  if ( begin_strnum < )

  {  

   return ;

  }

  int end_strnum = tmpAllStrIndexOf( e_Contentbegin_strnum + b_ContentLength );

  if ( end_strnum < )

  {  

   return ;

  }

  string sTotalSubM = ;

  if ( end_strnum > begin_strnum )

  {

   sTotalSubM = tmpAllStrSubstring ( begin_strnumend_strnum begin_strnum );

  }

  

  if ( sTotalSubM == )

  {  

   return ;

  }  

  //过滤无用信息

  int bfnum = sTotalSubMIndexOf( b_Filter );

  if ( bfnum > )

  {

  int efnum = sTotalSubMIndexOf( e_Filterbfnum );

  if ( efnum > )

  {

   if ( efnum > bfnum )

  {

   sTotalSubM = sTotalSubMSubstring( bfnum ) + sTotalSubMSubstring( efnum + e_FilterLength );

  }

  }

  }

  //格式化图片标记

  

  while( sTotalSubMIndexOf( Src= ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( Src= ) ) + src= + sTotalSubMSubstring( sTotalSubMIndexOf( Src= ) + );

  }

  while( sTotalSubMIndexOf( SRC= ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( SRC= ) ) + src= + sTotalSubMSubstring( sTotalSubMIndexOf( SRC= ) + );

  }

  while( sTotalSubMIndexOf( src= ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( src= ) ) + src=\ + sTotalSubMSubstring( sTotalSubMIndexOf( src= ) + );

  }

  //取图片地址

  int end_n = ;

  int end_nums = ;

  int begin_nums = sTotalSubMIndexOf( src= );

  while( begin_nums >= )

  {

   String tmpStr = sTotalSubMSubstring( begin_nums + );

  if ( tmpStr == \ )

  {

   end_n = begin_nums + ;

  }

  else

  {

   end_n = begin_nums + ;

  }

  int end_numsa = sTotalSubMIndexOf( end_n );

  int end_numsb = sTotalSubMIndexOf( >end_n );

  if ( end_numsb < )

  {

   break;

  }

  if ( end_numsa > end_numsb )

  {

   end_nums = end_numsb;

  }

  else

  {

   if (end_numsa<)

  {

   end_nums = end_numsb;

  }

   else

  {

   end_nums = end_numsa;

  }

  }

   tmpStr = sTotalSubMSubstring( end_nums );

  if ( tmpStr == \ || tmpStr == )

  {

   end_nums = end_nums ;

  }

   string tmpPicStr = sTotalSubMSubstring( end_nend_nums end_n );

  if ( tmpPicStrIndexOf( // ) < )

  {

   if ( tmpPicStrIndexOf( / ) == )

  {

   tmpPicStr = urlch + tmpPicStr;

  }

   else

  {   

   int linshiIntNum = ;

   int flags = ;

   string urlChange = subUrl;

   while( tmpPicStrIndexOf( / ) >= )

   {

     tmpPicStr = tmpPicStrSubstring( tmpPicStrIndexOf(/) + );

    linshiIntNum = linshiIntNum + ;

   flags = flags + ;

   }

   while( ( urlChangeLastIndexOf( / ) >= ) && ( linshiIntNum >= ) )

   {

    urlChange = urlChangeSubstring( urlChangeLastIndexOf( / ) );

    linshiIntNum = linshiIntNum ;

   }

   if ( flags == )

   {

    tmpPicStr = urlch + / + tmpPicStr;

   }

   else

   {

    tmpPicStr = urlChange + / + tmpPicStr;

   }

  }

  }

   //tmpPicStr = tmpPicStrToLower();

   string tmpPicStrTmp = tmpPicStrToLower();

  //if ( tmpPicStrIndexOf( jpg ) > || tmpPicStrIndexOf( gif ) > || tmpPicStrIndexOf( bmp ) > )

  if ( tmpPicStrTmpIndexOf( jpg ) > || tmpPicStrTmpIndexOf( gif ) > || tmpPicStrTmpIndexOf( bmp ) > )

  {

   rePicStr = rePicStr + || + tmpPicStr ;

   int flagN = tmpPicStrLastIndexOf( / );

   string fileN = picHtml + tmpPicStrSubstring( flagN );

   sTotalSubM = sTotalSubMSubstring( end_nums ) + >****** + fileN + ******< + sTotalSubMSubstring( end_nums );

   begin_nums = sTotalSubMIndexOf( src= end_nums + fileNLength + );

  }

  else

  {

   begin_nums = sTotalSubMIndexOf( src= end_nums + );  

  }   

  }

  if ( rePicStrLength > )

   rePicStr =rePicStrSubstring();  

  //内容处理 格式化关键标记

  while( sTotalSubMIndexOf( <P ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( <P ) ) + |****|< + sTotalSubMSubstring( sTotalSubMIndexOf( <P ) + );

  }

  while( sTotalSubMIndexOf( <p ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( <p ) ) + |****|< + sTotalSubMSubstring( sTotalSubMIndexOf( <p ) + );

  }

  while( sTotalSubMIndexOf( </P ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( </P ) ) + |****|< + sTotalSubMSubstring( sTotalSubMIndexOf( </P ) + );

  }

  while( sTotalSubMIndexOf( </p ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( </p ) ) + |****|< + sTotalSubMSubstring( sTotalSubMIndexOf( </p ) + );

  }

  while( sTotalSubMIndexOf( <br ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( <br ) ) + +****+< + sTotalSubMSubstring( sTotalSubMIndexOf( <br ) + );

  }

  while( sTotalSubMIndexOf( <BR ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( <BR ) ) + +****+< + sTotalSubMSubstring( sTotalSubMIndexOf( <BR ) + );

  }

  while( sTotalSubMIndexOf( <Br ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( <Br ) ) + +****+< + sTotalSubMSubstring( sTotalSubMIndexOf( <Br ) + );

  }

  while( sTotalSubMIndexOf( <bR ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( <bR ) ) + +****+< + sTotalSubMSubstring( sTotalSubMIndexOf( <bR ) + );

  }

  //去除html标记

  int linshiInt = sTotalSubMIndexOf( < );

  int linshiInt = sTotalSubMIndexOf( > );

  if ( linshiInt < linshiInt )

  {

   sTotalSubM = sTotalSubMSubstring( linshiInt + );

  }

  int linshiInt = sTotalSubMLastIndexOf( < );

  int linshiInt = sTotalSubMLastIndexOf( > );

  if ( linshiInt < linshiInt )

  {

   sTotalSubM = sTotalSubMSubstring( linshiInt + );

  }

  linshiInt = sTotalSubMIndexOf( < );

  while ( linshiInt >= )

  {

    linshiInt = sTotalSubMIndexOf( >linshiInt );

  if ( linshiInt >= )

  {  

   sTotalSubM = sTotalSubMSubstring( linshiInt ) + sTotalSubMSubstring( linshiInt + );

  }

  else

  {

   sTotalSubM = sTotalSubMSubstring( linshiInt );

  }

   linshiInt = sTotalSubMIndexOf(<);

  }

  //还原关键标记

  int linshiInt = ;

  int linshiInt = ;

  while( sTotalSubMIndexOf( +****+ ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( +****+ ) ) + <br>\n + sTotalSubMSubstring( sTotalSubMIndexOf( +****+ ) + );

  }

  while( sTotalSubMIndexOf( |****| ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( |****| ) ) + <br>\n + sTotalSubMSubstring( sTotalSubMIndexOf( |****| ) + );

  }

  while( sTotalSubMIndexOf( ****** ) >= )

  {

   linshiInt = sTotalSubMIndexOf( ****** ) + ;

   linshiInt = sTotalSubMIndexOf( ******linshiInt );

  if ( linshiInt >= )

  {

   int tmpPos = sTotalSubMIndexOf( ****** );

   string tmpStr = sTotalSubMSubstring( tmpPos );

   string tmpStr = sTotalSubMSubstring( linshiIntlinshiInt linshiInt );

   string tmpStr = sTotalSubMSubstring( linshiInt + );

   sTotalSubM = tmpStr + <img src= + tmpStr + > + tmpStr;

  }

  else

  {

   break;

  }

  }

  //去除内容中的标题

  if ( sTotalSubMIndexOf( subTitle ) >= )

  {

   sTotalSubM = sTotalSubMSubstring( sTotalSubMIndexOf( subTitle ) ) + sTotalSubMSubstring( sTotalSubMIndexOf( subTitle ) + subTitleLength );

  }

  reContentStr = sTotalSubM;

  //调用下载图片功能

  //下载图片到指定目录

  string[] img_Url = new PublicFun()split( rePicStr|| );

  for ( int i=;i<img_UrlLength;i++ )

  {

  if ( img_Url[i] != )

  {

   new PublicFun()Get_Img( img_Url[i]root + \\images\\ + img_Url[i]Substring( img_Url[i]LastIndexOf(/)+ ) );

  }

  }

}

  return reContentStr;

}

以上方法返回要取得的信息包括标题内容图片地址等

下载页面中图片

//下载图片

  public void Get_Img(string a_strUrlint timeoutstring filepath)

{

  try

{

HttpWebRequest myReq = (HttpWebRequest)HttpWebRequestCreate(a_strUrl) ;

  myReqTimeout = timeout;

  HttpWebResponse HttpWResp = (HttpWebResponse)myReqGetResponse();  

  Stream myStream = HttpWRespGetResponseStream () ;  

  Bitmap map = new Bitmap( myStream );

  PictureBox picB = new PictureBox();

  picBImage = (Image)map;

  string path = filepathSubstring( filepathLastIndexOf( \\ ) );

  if (!DirectoryExists(path))

  {

   CreateDir( path );

  }  

  picBImageSave(filepath);  

}

  catch(Exception exp)

{

  string ss = expMessage;

WriteLog( filepathSubstring(filepathLastIndexOf(\\)) + \\errorloga_strUrl + + ss + \r\n);

}

}

保存文件或入库

上面取得的信息可以按自己的要求保存

****设计的时候没有使用url按层次循环抓取这样定义抓取url效率更高速度更快

测试程序下载

如有建议请发送EMail 或msn

注此版本只提供静态文件存储功能不提供数据库接口不提供自定义网站功能

本程序运行需要先安 框架