AWS Lambda 内存不足：如何在使用 Go AWS SDK 时下载 460MB 文件到 Lambda 中解决内存问题

本篇文章向大家介绍《AWS Lambda 内存不足：如何在使用 Go AWS SDK 时下载 460MB 文件到 Lambda 中解决内存问题》，主要包括，具有一定的参考价值，需要的朋友可以参考一下。

我正在使用 aws-sdk-go-v2 运行 lambda，但在下载较大文件时遇到内存问题。我已经尝试了partsize和并发的各种组合，但我要么在将并发设置为较小的数字时遇到超时，要么遇到内存不足问题。

有人知道如何解决这个问题，或者知道如何使用 go 从 s3 下载文件的更好或其他方法吗？

package main

import (
    "context"
    "fmt"
    awsevents "github.com/aws/aws-lambda-go/events"
    "github.com/aws/aws-lambda-go/lambda"
    "github.com/aws/aws-sdk-go-v2/aws"
    "github.com/aws/aws-sdk-go-v2/aws/external"
    "github.com/aws/aws-sdk-go-v2/service/s3"
    "github.com/aws/aws-sdk-go-v2/service/s3/s3manager"
)

func handledemoevent(_ context.context, s3event awsevents.s3event) error {
    for _, record := range s3event.records {
        return handledemorecord(record)
    }

    return nil
}

func handledemorecord(record awsevents.s3eventrecord) error {
    fmt.println(record.responseelements)
    fmt.printf("[%s - %s] bucket = %s, key = %s \n", record.eventsource, record.eventtime, record.s3.bucket.name, record.s3.object.key)

    bucket := record.s3.bucket.name
    item := record.s3.object.key

    config, err := external.loaddefaultawsconfig()

    if err != nil {
        return err
    }

    headobject := getheadobject(bucket, item, config)

    fmt.printf("%+v", headobject)

    downloader := s3manager.newdownloader(config, func(d *s3manager.downloader) {
        d.partsize = 2 * 1024 * 1024
        d.concurrency = 70
    })

    var buf []byte
    buffer := aws.newwriteatbuffer(buf)

    fmt.println("starting download of file")
    numbytesdownloaded, err := downloader.download(buffer, &s3.getobjectinput{
        bucket: aws.string(bucket),
        key:    aws.string(item),
    })

    if err != nil {
        return err
    }

    fmt.println("downloaded ", numbytesdownloaded, " bytes")
    return nil
}


func getheadobject(bucket string, key string, config aws.config) *s3.headobjectresponse {
    client := s3.new(config)
    input := &s3.headobjectinput{
        bucket: aws.string(bucket),
        key:    aws.string(key),
    }

    request := client.headobjectrequest(input)

    headobjectresponse, err := request.send(context.todo())

    if err != nil {
        panic(err)
    }

    fmt.printf("downloaded headobject: %v\n", request)

    return headobjectresponse
}

func main() {
    lambda.start(handledemoevent)
}

这是来自日志流的错误，没有包含所有内容，因为它很长。

START RequestId: 8f5bc332-b4ad-4749-97ac-ee7c65a4404a Version: $LATEST
map[x-amz-request-id:AA3E4F41C6FCA9B9 x-amz-id-2:HEoJcflBdxLbSLkOOUD19kI7TboN8hDfkBf8vqQlFy1FQni2wUUi09cRwliZeDn6nFiNx73V2v9V5jreSlgOr1bpPDl9Lywe]
[aws:s3 - 2020-01-24 08:25:11.271 +0000 UTC] Bucket = 5kstats-demos, Key = to-be-parsed/vitality-vs-mousesports-m1-inferno.dem 
Downloaded HeadObject: {0xc000105b80 {
  Bucket: "5kstats-demos",
  Key: "to-be-parsed/vitality-vs-mousesports-m1-inferno.dem",
  RequestPayer: 
} 0x908e80}
{
  AcceptRanges: "bytes",
  ContentLength: 491103546,
  ContentType: "application/octet-stream",
  ETag: "\"f3870e5b519ea52dbd249f69f6072f45-30\"",
  LastModified: 2020-01-24 08:25:12 +0000 UTC,
  Metadata: {
    Event-Id: "4979",
    Event-Name: "EPICENTER 2019",
    Event-End-Sec: "1577012400000",
    Match-Id: "2338373",
    Event-Start-Sec: "1576580400000"
  },
  ObjectLockLegalHoldStatus: ,
  ObjectLockMode: ,
  ReplicationStatus: ,
  RequestCharged: ,
  ServerSideEncryption: ,
  StorageClass: 
}Starting download of file
fatal error: runtime: out of memory

runtime stack:
runtime.throw(0xac163b, 0x16)
    /usr/local/go/src/runtime/panic.go:608 +0x72
runtime.sysMap(0xc0b8000000, 0x1c000000, 0x1112d18)
    /usr/local/go/src/runtime/mem_linux.go:156 +0xc7
runtime.(*mheap).sysAlloc(0x10f96c0, 0x1c000000, 0x80, 0x0)
    /usr/local/go/src/runtime/malloc.go:619 +0x1c7
runtime.(*mheap).grow(0x10f96c0, 0xd202, 0x0)
    /usr/local/go/src/runtime/mheap.go:920 +0x42
runtime.(*mheap).allocSpanLocked(0x10f96c0, 0xd202, 0x1112d28, 0x432c55)
    /usr/local/go/src/runtime/mheap.go:848 +0x337
runtime.(*mheap).alloc_m(0x10f96c0, 0xd202, 0xffffffffffff0101, 0x7ffe2e1d0780)
    /usr/local/go/src/runtime/mheap.go:692 +0x119
runtime.(*mheap).alloc.func1()
    /usr/local/go/src/runtime/mheap.go:759 +0x4c
runtime.(*mheap).alloc(0x10f96c0, 0xd202, 0x7ffe2e010101, 0x7ffe2e1d0820)
    /usr/local/go/src/runtime/mheap.go:758 +0x8a
runtime.largeAlloc(0x1a403fe8, 0x101, 0xc00ba5bbb0)
    /usr/local/go/src/runtime/malloc.go:1019 +0x97
runtime.mallocgc.func1()
    /usr/local/go/src/runtime/malloc.go:914 +0x46
runtime.systemstack(0x456509)
    /usr/local/go/src/runtime/asm_amd64.s:351 +0x66
runtime.mstart()
    /usr/local/go/src/runtime/proc.go:1229

goroutine 10 [running]:
runtime.systemstack_switch()
    /usr/local/go/src/runtime/asm_amd64.s:311 fp=0xc00ba5bb40 sp=0xc00ba5bb38 pc=0x456600
runtime.mallocgc(0x1a403fe8, 0x9c9780, 0x41829684901, 0xc00ba5bc18)
    /usr/local/go/src/runtime/malloc.go:913 +0x896 fp=0xc00ba5bbe0 sp=0xc00ba5bb40 pc=0x40d646
runtime.makeslice(0x9c9780, 0x1a403fe8, 0x1a403fe8, 0x8000, 0x3fe8, 0x0)
    /usr/local/go/src/runtime/slice.go:70 +0x77 fp=0xc00ba5bc10 sp=0xc00ba5bbe0 pc=0x441877
5kparser/vendor/github.com/aws/aws-sdk-go-v2/aws.(*WriteAtBuffer).WriteAt(0xc0002c8210, 0xc001ade000, 0x3fe8, 0x8000, 0x1a400000, 0x0, 0x0, 0x0)
    /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/aws/types.go:190 +0xf9 fp=0xc00ba5bc78 sp=0xc00ba5bc10 pc=0x76bb69
5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*dlchunk).Write(0xc000068740, 0xc001ade000, 0x3fe8, 0x8000, 0x3fe8, 0x0, 0x0)
    /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:590 +0x75 fp=0xc00ba5bcc8 sp=0xc00ba5bc78 pc=0x970c65
io.copyBuffer(0xc20f20, 0xc000068740, 0xc215e0, 0xc0007da880, 0xc001ade000, 0x8000, 0x8000, 0xa162e0, 0xc000293600, 0xc215e0)
    /usr/local/go/src/io/io.go:404 +0x201 fp=0xc00ba5bd38 sp=0xc00ba5bcc8 pc=0x46f701
io.Copy(0xc20f20, 0xc000068740, 0xc215e0, 0xc0007da880, 0xc0007da880, 0xc000419970, 0x0)
    /usr/local/go/src/io/io.go:364 +0x5a fp=0xc00ba5bd98 sp=0xc00ba5bd38 pc=0x46f4ba
5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).tryDownloadChunk(0xc00008c750, 0xc0005cd0e0, 0xc20f20, 0xc000068740, 0x0, 0x0, 0x0)
    /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:470 +0x1e0 fp=0xc00ba5be28 sp=0xc00ba5bd98 pc=0x970430
5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).downloadChunk(0xc00008c750, 0xc20280, 0xc0002c8210, 0x1a400000, 0x200000, 0x0, 0x0, 0x0, 0x0, 0x0)
    /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:429 +0x2a5 fp=0xc00ba5bf00 sp=0xc00ba5be28 pc=0x9700d5
5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).downloadPart(0xc00008c750, 0xc00006c2a0)
    /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:378 +0x142 fp=0xc00ba5bfd0 sp=0xc00ba5bf00 pc=0x96fb42
runtime.goexit()
    /usr/local/go/src/runtime/asm_amd64.s:1333 +0x1 fp=0xc00ba5bfd8 sp=0xc00ba5bfd0 pc=0x4586e1
created by 5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager.(*downloader).download
    /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-sdk-go-v2/service/s3/s3manager/download.go:322 +0x1ed

goroutine 1 [IO wait]:
internal/poll.runtime_pollWait(0x7f018740cf00, 0x72, 0x0)
    /usr/local/go/src/runtime/netpoll.go:173 +0x66
internal/poll.(*pollDesc).wait(0xc000120218, 0x72, 0xc00009a000, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:85 +0x9a
internal/poll.(*pollDesc).waitRead(0xc000120218, 0xffffffffffffff00, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_poll_runtime.go:90 +0x3d
internal/poll.(*FD).Accept(0xc000120200, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0)
    /usr/local/go/src/internal/poll/fd_unix.go:384 +0x1a0
net.(*netFD).accept(0xc000120200, 0x7f0187415200, 0x0, 0x0)
    /usr/local/go/src/net/fd_unix.go:238 +0x42
net.(*TCPListener).accept(0xc0000965a8, 0xc000171e30, 0xc000171e38, 0x18)
    /usr/local/go/src/net/tcpsock_posix.go:139 +0x2e
net.(*TCPListener).Accept(0xc0000965a8, 0xada0f8, 0xc0000ba320, 0x7f0187415200, 0xc0000965f8)
    /usr/local/go/src/net/tcpsock.go:260 +0x47
net/rpc.(*Server).Accept(0xc0000ba320, 0xc26c00, 0xc0000965a8)
    /usr/local/go/src/net/rpc/server.go:632 +0x69
net/rpc.Accept(0xc26c00, 0xc0000965a8)
    /usr/local/go/src/net/rpc/server.go:692 +0x41
5kparser/vendor/github.com/aws/aws-lambda-go/lambda.StartHandler(0xc21a20, 0xc00009bb80)
    /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-lambda-go/lambda/entry.go:60 +0x174
5kparser/vendor/github.com/aws/aws-lambda-go/lambda.Start(0x9e9140, 0xad9dd8)
    /home/softarn/go/src/5kparser/vendor/github.com/aws/aws-lambda-go/lambda/entry.go:41 +0x4d
main.main()
    /home/softarn/go/src/5kparser/aws-lambda-runner.go:209 +0x39

解决方案

尝试设置切片长度/上限：

buf := make([]byte, int(*headobject.contentlength))

这适用于我的代码和 s3 上的 425.9mb 文件。

Duration: 6652.14 ms    Billed Duration: 6700 ms    Memory Size: 3008 MB    Max Memory Used: 523 MB

根据https://docs.aws.amazon.com/lambda/latest/dg/limits.html默认mem：

128 mb 到 3,008 mb，增量为 64 mb。

您可以使用文件编写器，/tmp 目录存储中最多可使用 512mb，如此处所示的示例

https://github.com/aws/aws-sdk-go/blob/master/service/s3/s3manager/download.go#L222-L249

此外，检查并增加您的函数可用的内存 $ aws lambda 更新函数配置 \ --函数名称我的函数 --内存大小 512

今天关于《AWS Lambda 内存不足：如何在使用 Go AWS SDK 时下载 460MB 文件到 Lambda 中解决内存问题》的内容就介绍到这里了，是不是学起来一目了然！想要了解更多关于的内容请关注golang学习网公众号！