Golang通过http代理抓取页面代码

jopen 11年前

网络编程中抓取开发的话一般要用到代理服务器,Python的代码很多了,现在用GoLang来实现的话也是很简洁的,GetByProxy函数封装了代理功能。

package main    import (   "fmt"   "io/ioutil"   "net/http"   "net/url"  )    // http get by proxy  func GetByProxy(url_addr, proxy_addr string) (*http.Response, error) {   request, _ := http.NewRequest("GET", url_addr, nil)   proxy, err := url.Parse(proxy_addr)   if err != nil {    return nil, err   }   client := &http.Client{    Transport: &http.Transport{     Proxy: http.ProxyURL(proxy),    },   }   return client.Do(request)  }    func main() {   proxy := "http://58.252.56.149:9000/"   url := "http://www.baidu.com/"   resp, _ := GetByProxy(url, proxy)   fmt.Println(resp)   defer resp.Body.Close()   body, _ := ioutil.ReadAll(resp.Body)   fmt.Println(string(body))  }