使用 Colly 实现 豆瓣电影Top250爬取

package main

import (
	"encoding/csv"
	"github/PuerkitoBio/goquery"
	"github/gocolly/colly"
	"log"
	"os"
	"strings"
	"time"
)

type Movie struct {
	idx    string
	title  string
	year   string
	info   string
	rating string
	url    string
}

func main() {

	// 存储文件名
	fName := "douban_movie_top250.csv"
	file, err := os.Create(fName)
	if err != nil {
		log.Fatalf("创建文件失败 %q: %s\n", fName, err)
		return
	}
	defer file.Close()
	writer := csv.NewWriter(file)
	defer writer.Flush()
	// 写CSV头部
	writer.Write([]string{"Idx", "Title", "Year", "Info", "Rating", "URL"})

	// 起始Url
	startUrl := "https://movie.douban/top250"

	// 创建Collector
	collector := colly.NewCollector(
		// 设置用户代理
		colly.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit

使用 Colly 实现 豆瓣电影Top250爬取

package main

import (
	"encoding/csv"
	"github/PuerkitoBio/goquery"
	"github/gocolly/colly"
	"log"
	"os"
	"strings"
	"time"
)

type Movie struct {
	idx    string
	title  string
	year   string
	info   string
	rating string
	url    string
}

func main() {

	// 存储文件名
	fName := "douban_movie_top250.csv"
	file, err := os.Create(fName)
	if err != nil {
		log.Fatalf("创建文件失败 %q: %s\n", fName, err)
		return
	}
	defer file.Close()
	writer := csv.NewWriter(file)
	defer writer.Flush()
	// 写CSV头部
	writer.Write([]string{"Idx", "Title", "Year", "Info", "Rating", "URL"})

	// 起始Url
	startUrl := "https://movie.douban/top250"

	// 创建Collector
	collector := colly.NewCollector(
		// 设置用户代理
		colly.UserAgent("Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_2) AppleWebKit