首页 > Golang > Go问答

无法在 Golang Colly 中同时打印 JSON 数组中的数据

来源：stackoverflow

时间：2024-02-16 10:45:24 242浏览收藏

推广推荐

免费电影APP ➜

支持 PC / 移动端，安全直达

小伙伴们有没有觉得学习Golang很有意思？有意思就对了！今天就给大家带来《无法在 Golang Colly 中同时打印 JSON 数组中的数据》，以下内容将会涉及到，若是在学习中对其中部分知识点有疑问，或许看了本文就能帮到你！

问题内容

我正在使用 amazon 的 golang colly 拍照，我想将这些图片以 json 格式放入单个数组中（每个产品图像只有 1 个数组）。我抓取了我需要的图片，只是 json 文件有问题。预先非常感谢您。

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "time"

    "github.com/gocolly/colly"
)

type info struct {
    id     int      `json:"id"`
    images []string `json:"images"`
}

func main() {
    allinfos := make([]info, 0)

    start := time.now()

    co := colly.newcollector(
        colly.alloweddomains("www.amazon.com", "amazon.com"),
    )

    // get images
    counter := 0
    var info info
    var thearray [10]string
    co.onhtml("img[src]", func(e *colly.htmlelement) {
        imgsrc := e.attr("src")
        imgclass := e.attr("class")

        if imgsrc[0:49] == "https://images-na.ssl-images-amazon.com/images/i/" && imgclass == "" {
            thearray[counter] = imgsrc
            fmt.printf("the array %d %v", counter, thearray[counter]+"\n")
            counter = counter + 1
            co.visit(e.request.absoluteurl(imgsrc))

            info = info{
                images: []string{
                    thearray[0],
                    thearray[1],
                    thearray[2],
                    thearray[3],
                    thearray[4],
                    thearray[5],
                    thearray[6],
                },
            }
            allinfos = append(allinfos, info)
        }
    })

    co.visit("https://www.amazon.com/bluetooth-fm-transmitter-compatible-smartphones/dp/b088tcsh8t/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=a2xmghkvcx40wb&spla=zw5jcnlwdgvkuxvhbglmawvypufgvegxtjjlrfdlskumzw5jcnlwdgvkswq9qtaynde0odkyrlndwlazuktpqzjsjmvuy3j5chrlzefkswq9qta5mtkwnjgywvuzq0rumtbcteffjndpzgdlde5hbwu9c3bfyxrmjmfjdglvbj1jbglja1jlzglyzwn0jmrvtm90tg9nq2xpy2s9dhj1zq==wdy4vktwqlimd2lkz2v0tmftzt1zcf9kzxrhawwmywn0aw9upwnsawnrumvkaxjly3qmzg9ob3rmb2ddbgljaz10cnvl")

    co.onrequest(func(r *colly.request) {
        fmt.println("visiting: ", r.url.string())
    })

    enc := json.newencoder(os.stdout)
    enc.setindent("", " ")
    enc.encode(allinfos)

    writejson(allinfos)

    duration := time.since(start)

    fmt.println(duration.seconds())
}

func writejson(data []info) {
    datafile, err := json.marshalindent(data, "", " ")
    if err != nil {
        log.println("could not create json", err)
    }

    ioutil.writefile("stocky.json", datafile, 0666)
}

我的代码的 json 结果；

[
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/i/41ekksqcu-l._ac_us40_.jpg",
   "",
   "",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/i/41ekksqcu-l._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/511l3hyccll._ac_us40_.jpg",
   "",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/i/41ekksqcu-l._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/511l3hyccll._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/51kso5k364l._ac_us40_.jpg",
   "",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/i/41ekksqcu-l._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/511l3hyccll._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/51kso5k364l._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/61nvwkbuxul._ac_us40_.jpg",
   "",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/i/41ekksqcu-l._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/511l3hyccll._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/51kso5k364l._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/61nvwkbuxul._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/51hwjpj4ogl._ac_us40_.jpg",
   "",
   ""
  ]
 },
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/i/41ekksqcu-l._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/511l3hyccll._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/51kso5k364l._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/61nvwkbuxul._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/51hwjpj4ogl._ac_us40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/i/51dz9pnwvrl._ac_us40_.jpg",
   ""
  ]
 }
]

我需要这样的 json 结果；

[
 {
  "id": 0,
  "images": [
   "https://images-na.ssl-images-amazon.com/images/I/41EKkSQCU-L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/511L3hYCClL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51kSO5K364L._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/61NvwkbuXUL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51hwJpj4OgL._AC_US40_.jpg",
   "https://images-na.ssl-images-amazon.com/images/I/51dz9PNWVrL._AC_US40_.jpg",
  ]
 }
]

正确答案

我很难理解你想做什么，但这就是我的想法：

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "strings"
    "time"

    "github.com/gocolly/colly"
)

type info struct {
    id     int      `json:"id"`
    images []string `json:"images"`
}

func main() {
    allinfos := new(info)

    start := time.now()

    co := colly.newcollector(
        colly.alloweddomains("www.amazon.com", "amazon.com"),
    )

    // get images
    co.onhtml("img[src]", func(e *colly.htmlelement) {
        imgsrc := e.attr("src")
        imgclass := e.attr("class")

        if strings.hasprefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/i/") && imgclass == "" {
            fmt.printf("the array %d %v", len(allinfos.images), imgsrc+"\n")
            allinfos.images = append(allinfos.images, imgsrc)
            co.visit(e.request.absoluteurl(imgsrc))
        }
    })

    co.visit("https://www.amazon.com/bluetooth-fm-transmitter-compatible-smartphones/dp/b088tcsh8t/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=a2xmghkvcx40wb&spla=zw5jcnlwdgvkuxvhbglmawvypufgvegxtjjlrfdlskumzw5jcnlwdgvkswq9qtaynde0odkyrlndwlazuktpqzjsjmvuy3j5chrlzefkswq9qta5mtkwnjgywvuzq0rumtbcteffjndpzgdlde5hbwu9c3bfyxrmjmfjdglvbj1jbglja1jlzglyzwn0jmrvtm90tg9nq2xpy2s9dhj1zq==wdy4vktwqlimd2lkz2v0tmftzt1zcf9kzxrhawwmywn0aw9upwnsawnrumvkaxjly3qmzg9ob3rmb2ddbgljaz10cnvl")

    co.onrequest(func(r *colly.request) {
        fmt.println("visiting: ", r.url.string())
    })

    enc := json.newencoder(os.stdout)
    enc.setindent("", " ")
    enc.encode(allinfos)

    writejson(allinfos)

    duration := time.since(start)

    fmt.println(duration.seconds())
}

func writejson(data *info) {
    datafile, err := json.marshalindent(data, "", " ")
    if err != nil {
        log.println("could not create json", err)
    }

    ioutil.writefile("stocky.json", datafile, 0666)
}

正如 @flimzy 所建议的，我仅将新的 imagesrc 附加到信息结构的since实例中。看来您的预期输出是包含所有图像的长度为 1 的切片。

我还使用了 strings.hasprefix 来清理切片检查。这是等效的，但我认为它更干净，所以这是一个可选的更改。

如果您还有其他问题，请告诉我。

编辑，如果您需要继续使用切片，如下所示的内容可能会起作用。从您的问题很难判断何时需要清除新信息：

package main

import (
    "encoding/json"
    "fmt"
    "io/ioutil"
    "log"
    "os"
    "strings"
    "time"

    "github.com/gocolly/colly"
)

type Info struct {
    ID     int      `json:"id"`
    Images []string `json:"images"`
}

func main() {
    allInfos := make([]Info, 1)

    start := time.Now()

    co := colly.NewCollector(
        colly.AllowedDomains("www.amazon.com", "amazon.com"),
    )

    // GET Images
    co.OnHTML("img[src]", func(e *colly.HTMLElement) {
        imgsrc := e.Attr("src")
        imgclass := e.Attr("class")

        if strings.HasPrefix(imgsrc, "https://images-na.ssl-images-amazon.com/images/I/") && imgclass == "" {
            fmt.Printf("The Array %d %v", len(allInfos[0].Images), imgsrc+"\n")
            allInfos[0].Images = append(allInfos[0].Images, imgsrc)
            co.Visit(e.Request.AbsoluteURL(imgsrc))
        }
    })

    co.Visit("https://www.amazon.com/Bluetooth-FM-Transmitter-Compatible-Smartphones/dp/B088TCSH8T/ref=sr_1_1_sspa?dchild=1&keywords=transmitter&qid=1623860482&sr=8-1-spons&psc=1&smid=A2XMGHKVCX40WB&spLa=ZW5jcnlwdGVkUXVhbGlmaWVyPUFGVEgxTjJLRFdLSkUmZW5jcnlwdGVkSWQ9QTAyNDE0ODkyRlNDWlAzUktPQzJSJmVuY3J5cHRlZEFkSWQ9QTA5MTkwNjgyWVUzQ0RUMTBCTEFFJndpZGdldE5hbWU9c3BfYXRmJmFjdGlvbj1jbGlja1JlZGlyZWN0JmRvTm90TG9nQ2xpY2s9dHJ1ZQ==WDY4VktWQlImd2lkZ2V0TmFtZT1zcF9kZXRhaWwmYWN0aW9uPWNsaWNrUmVkaXJlY3QmZG9Ob3RMb2dDbGljaz10cnVl")

    co.OnRequest(func(r *colly.Request) {
        fmt.Println("Visiting: ", r.URL.String())
    })

    enc := json.NewEncoder(os.Stdout)
    enc.SetIndent("", " ")
    enc.Encode(allInfos)

    writeJson(allInfos)

    duration := time.Since(start)

    fmt.Println(duration.Seconds())
}

func writeJson(data []Info) {
    dataFile, err := json.MarshalIndent(data, "", " ")
    if err != nil {
        log.Println("Could not create JSON", err)
    }

    ioutil.WriteFile("stocky.json", dataFile, 0666)
}

今天关于《无法在 Golang Colly 中同时打印 JSON 数组中的数据》的内容就介绍到这里了，是不是学起来一目了然！想要了解更多关于的内容请关注golang学习网公众号！

声明：本文转载于：stackoverflow 如有侵犯，请联系study_golang@163.com删除