运行时/cgo：pthread_create错误：资源暂时无法获取

最近发现不少小伙伴都对Golang很感兴趣，所以今天继续给大家介绍Golang相关的知识，本文《运行时/cgo：pthread_create错误：资源暂时无法获取》主要内容涉及到等等知识点，希望能帮到你！当然如果阅读本文时存在不同想法，可以在评论中表达，但是请勿使用过激的措辞~

我目前有这个函数，它通过 ffmpeg 管道传输多个 youtubedl 命令，然后将 ffmpeg 的输出管道传输到 http 客户端。

func pipethruffmpegtomp4(vi *videoinfo, rw web.responsewriter) error {
    var ffmpeg *exec.cmd
    ffmpeg = exec.command(
        "ffmpeg",
        "-i", "-",
        "-i", "pipe:3",
        "-c:v", "copy", "-c:a", "copy",
        "-preset", "veryfast",
        "-metadata", fmt.sprintf(`title=%s`, vi.gettitle()),
        "-movflags", "frag_keyframe+empty_moov",
        "-f", "mp4",
        "-")


    youtubevideo := exec.command(youtubedlpath, "-c", "-f", fmt.sprintf("%s/bestvideo[ext=mp4]/bestvideo/best", vi.getformat()), "--no-cache-dir", "--restrict-filenames", "--hls-prefer-native", "-o", "-", fmt.sprintf("%s", vi.getvideourl()))
    fmt.println(youtubevideo)

    youtube := exec.command(youtubedlpath, "-c", "-f", "bestaudio[ext=m4a]/bestaudio/best", "--no-cache-dir", "--restrict-filenames", "--hls-prefer-native", "-o", "-", fmt.sprintf("%s", vi.getvideourl()))
    fmt.println(youtube)

    var ytvbuf, ytbuf, ffbuf bytes.buffer
    youtubevideo.stderr = &ytvbuf
    youtube.stderr = &ytbuf
    ffmpeg.stderr = &ffbuf

    video, err := youtubevideo.stdoutpipe()
    if err != nil {
        log.printf("pipethruffmpegtomp4: %v\n", err)
        return err
    }

    pipe3, err := youtube.stdoutpipe()
    if err != nil {
        log.printf("pipethruffmpegtomp4: %v\n", err)
        return err
    }

    ffmpeg.stdin = video
    ffmpeg.extrafiles = []*os.file{pipe3.(*os.file)}
    ffmpeg.stdout = rw

    // headers sent, no turning back now
    rw.header().set("content-type", "video/mp4")
    rw.header().set("content-disposition", fmt.sprintf("attachment;filename=\"%s.mp4\"", vi.getslug()))
    rw.flush()

    ffmpeg.start()
    youtubevideo.start()
    youtube.start()
    ffmpeg.wait()
    youtubevideo.wait()
    youtube.wait()

    // check ytvbuf, ytbuf, ffbuf for stderr errors

    if ffbuf.len() != 0 {
        rollbar.error(rollbar.err, err, &rollbar.field{"stderr", ffbuf.string()})
        log.printf("pipethruffmpegtomp4: %v\n", ffbuf.string())
    }
    if ytvbuf.len() != 0 {
        rollbar.error(rollbar.err, err, &rollbar.field{"stderr", ytvbuf.string()})
        log.printf("pipethruyoutubevdltomp4: %v\n", ytvbuf.string())
    }
    if ytbuf.len() != 0 {
        rollbar.error(rollbar.err, err, &rollbar.field{"stderr", ytbuf.string()})
        log.printf("pipethruyoutubedltomp4: %v\n", ytbuf.string())
    }

    return nil
}

问题是一切都运行良好，但过了一会儿，服务器上的内存开始填满，直到出现 runtime/cgo 错误：pthread_create 失败：资源暂时不可用

我不确定这是否是内存泄漏，或者 youtube-dl 的任一实例未正确关闭，或者 ffmpeg 是否未正确关闭，并且随着程序运行更多而消耗越来越多的内存，直到程序崩溃出现此错误

runtime/cgo: pthread_create failed: resource temporarily unavailable
sigabrt: abort
pc=0x7f083501fe97 m=128 sigcode=18446744073709551610

goroutine 0 [idle]:
runtime: unknown pc 0x7f083501fe97
stack: frame={sp:0x7f05ff7fd7d0, fp:0x0} stack=[0x7f05feffe288,0x7f05ff7fde88)
00007f05ff7fd6d0:  00007f05ff7fd700  0000ffff00001fa0
00007f05ff7fd6e0:  00007f05ff7fdbe0  00007f05f8000da0
00007f05ff7fd6f0:  0000000000000000  000000000093032c
00007f05ff7fd700:  0000000000000000  00007f0835600ec3
00007f05ff7fd710:  0000000000000005  0000000000000000
00007f05ff7fd720:  000000c0000ce120  00007f0834ff1ce0
00007f05ff7fd730:  00007f05ff7fdaf0  00007f083560870a
00007f05ff7fd740:  0000000000000000  0000000000000000
00007f05ff7fd750:  0000000000000000  00007f05ff7fdbe0
00007f05ff7fd760:  2525252525252525  2525252525252525
00007f05ff7fd770:  000000ffffffffff  0000000000000000
00007f05ff7fd780:  000000ffffffffff  0000000000000000
00007f05ff7fd790:  000000c00010d1a0  000000c000953740
00007f05ff7fd7a0:  000000c0000ce120  000000c000cf2300
00007f05ff7fd7b0:  000000c00010d260  000000c001f4e180
00007f05ff7fd7c0:  000000c001f4e000  000000c00169f680
00007f05ff7fd7d0: <0000000000000000  000000c001f34180
00007f05ff7fd7e0:  6e75720000000000  6f67632f656d6974
00007f05ff7fd7f0:  0000000000000000  0000000000000000
00007f05ff7fd800:  000000c000cf2300  000000c00010d260
00007f05ff7fd810:  000000c001f4e180  000000c001f4e000
00007f05ff7fd820:  000000c00169f680  000000c00169f500
00007f05ff7fd830:  000000c001f34180  000000c001f34000
00007f05ff7fd840:  000000c000c92780  000000c001ec2600
00007f05ff7fd850:  fffffffe7fffffff  ffffffffffffffff
00007f05ff7fd860:  ffffffffffffffff  ffffffffffffffff
00007f05ff7fd870:  ffffffffffffffff  ffffffffffffffff
00007f05ff7fd880:  ffffffffffffffff  ffffffffffffffff
00007f05ff7fd890:  ffffffffffffffff  ffffffffffffffff
00007f05ff7fd8a0:  ffffffffffffffff  ffffffffffffffff
00007f05ff7fd8b0:  ffffffffffffffff  ffffffffffffffff
00007f05ff7fd8c0:  ffffffffffffffff  ffffffffffffffff
runtime: unknown pc 0x7f083501fe97
stack: frame={sp:0x7f05ff7fd7d0, fp:0x0} stack=[0x7f05feffe288,0x7f05ff7fde88)
00007f05ff7fd6d0:  00007f05ff7fd700  0000ffff00001fa0
00007f05ff7fd6e0:  00007f05ff7fdbe0  00007f05f8000da0
00007f05ff7fd6f0:  0000000000000000  000000000093032c
00007f05ff7fd700:  0000000000000000  00007f0835600ec3
00007f05ff7fd710:  0000000000000005  0000000000000000

我还尝试使用 cgo_enabled=0 构建二进制文件，即使我什至没有 import "c" 但最终也会出错

runtime: failed to create new os thread (have 21 already; errno=11)
runtime: may need to increase max user processes (ulimit -u)
fatal error: newosproc

我的限制已经非常慷慨了，这也许也是一个问题？或者我是否必须增加管道尺寸？

$ ulimit -a
core file size          (blocks, -c) 0
data seg size           (kbytes, -d) unlimited
scheduling priority             (-e) 0
file size               (blocks, -f) unlimited
pending signals                 (-i) 192907
max locked memory       (kbytes, -l) 16384
max memory size         (kbytes, -m) unlimited
open files                      (-n) 100000
pipe size            (512 bytes, -p) 8
POSIX message queues     (bytes, -q) 819200
real-time priority              (-r) 0
stack size              (kbytes, -s) 65536
cpu time               (seconds, -t) unlimited
max user processes              (-u) 63883
virtual memory          (kbytes, -v) unlimited

无论哪种方式，我都很感谢我能得到的所有帮助。这是整个程序的链接，这是 golang-nuts 论坛中关于此问题的帖子

解决方案

经过一些调试并在凯文回答的帮助下。看来 youtube-dl 成功后并没有关闭。

鉴于如果 ffmpeg 完成，那么运行 youtube-dl 实例就没有任何意义，并且由于 wait() 函数正在等待 youtube-dl 完成，但它似乎从未发送完成信号，我继续并且必须添加一个kill来杀死子子进程。

所以我必须添加更改

ffmpeg.start()
    youtubevideo.start()
    youtube.start()
    ffmpeg.wait()
    youtubevideo.wait()
    youtube.wait()

至此

youtube.sysprocattr = &syscall.sysprocattr{setpgid: true}
    youtubevideo.sysprocattr = &syscall.sysprocattr{setpgid: true}
    ffmpeg.start()
    youtubevideo.start()
    youtube.start()
    ffmpeg.wait()
    syscall.kill(-youtube.process.pid, syscall.sigkill)
    syscall.kill(-youtubevideo.process.pid, syscall.sigkill)
    youtubevideo.wait()
    youtube.wait()

请注意，因为它似乎会产生某种子子进程，所以 youtube.process.kill() 实际上并没有杀死该进程，这就是为什么我必须使用 syscall 以及 setpgid: true

我有时会遇到问题，您的代码不会杀死 youtube-dl 进程。视频已转换，但过程保留。作为第一个指标，观察进程计数，如下所示：

ps | grep youtube-dl | wc -l

可以让子进程超时，对于一个子进程来说，这看起来像这样：

package main

import (
    "bytes"
    "fmt"
    "os/exec"
    "time"
)

func main() {
    // sleep is used, so we can control how long it runs.
    cmd := exec.command("sleep", "2")

    // use a bytes.buffer to get the output
    var buf bytes.buffer
    cmd.stdout = &buf

    cmd.start()

    // use a channel to signal completion so we can use a select statement
    done := make(chan error)
    go func() { done <- cmd.wait() }()

    // start a timer, should be higher for your video conversion, 
    // i suggest you use a value that fits both your videos and server capabilities
    timeout := time.after(2 * time.second)

    // the select statement allows us to execute based on which channel
    // we get a message from first.
    select {
    case <-timeout:
        // timeout happened first, kill the process and print a message.
        cmd.process.kill()
        fmt.println("command timed out")
    case err := <-done:
        // command completed before timeout. print output and error if it exists.
        fmt.println("output:", buf.string())
        if err != nil {
            fmt.println("non-zero exit code:", err)
        }
    }
}

根据您的情况，您可以替换

    ffmpeg.start()
    youtubevideo.start()
    youtube.start()
    ffmpeg.wait()
    youtubevideo.wait()
    youtube.wait()

与

    ffmpeg.Start()
    youtubevideo.Start()
    youtube.Start()
    commands := 3

    done := make(chan error)
    go func() { done <- ffmpeg.Wait() }()
    go func() { done <- youtubevideo.Wait() }()
    go func() { done <- youtube.Wait() }()

    timeout := time.After(1 * time.Hour)

Loop:
    for {
        select {
        case <-timeout:
            ffmpeg.Process.Kill()
            youtubevideo.Process.Kill()
            youtube.Process.Kill()
            log.Println("Conversion timed out")
            break Loop
        case err := <-done:
            if err != nil {
                log.Println("Non-zero exit code:", err)
            }
            commands = commands - 1
            if commands == 0 {
                break Loop
            }
        }
    }

终于介绍完啦！小伙伴们，这篇关于《运行时/cgo：pthread_create错误：资源暂时无法获取》的介绍应该让你收获多多了吧！欢迎大家收藏或分享给更多需要学习的朋友吧~golang学习网公众号也会发布Golang相关知识，快来关注吧！