有效的方式使用Golang从文本文件的末尾读取特定内容

golang学习网今天将给大家带来《有效的方式使用Golang从文本文件的末尾读取特定内容》，感兴趣的朋友请继续看下去吧！以下内容将会涉及到等等知识点，如果你是正在学习Golang或者已经是大佬级别了，都非常欢迎也希望大家都能给我建议评论哈~希望能帮助到大家！

我有一个很大的文本日志文件，其中包含由一些特殊字符分隔的两个部分，就像

...
this is the very large 
part, contains lots lines.

#special chars start#

...
this is the small part the the end, 
contain several lines, but 
we do not know how many lines
this part contains

我的要求是获取#special chars start#之后到最后的小部分文本内容，如何用golang有效获取它？

更新： 我当前的解决方案是从文件末尾逐行获取并记住光标，如果该行包含特殊字符，则中断循环并已经获取光标

func getBackwardLine(file *os.File, start int64) (string, int64) {
    line := ""
    cursor :=start
    stat, _ := file.Stat()
    filesize := stat.Size()

    for { 
        cursor--
        file.Seek(cursor, io.SeekEnd)

        char := make([]byte, 1)
        file.Read(char)

        if cursor != -1 && (char[0] == 10 || char[0] == 13) { 
            break
        }

        line = fmt.Sprintf("%s%s", string(char), line) 

        if cursor == -filesize { 
            break
        }
    }
    return line, cursor

}

func main() {
    file, err := os.Open("some.log")
    if err != nil {
        os.Exit(1)
    }
    defer file.Close()

    var cursor int64 = 0
    var line = ""

    for {  
        line, cursor = getBackwardLine(file, cursor)
        fmt.Println(line)
        if(strings.Contains(line, "#SPECIAL CHARS START#")) {
            break
        }
    }


    fmt.Println(cursor)  //now we get the cursor for the start of special characters
}

解决方案

该解决方案实现了向后读取器。

它从 b.len 字节块的末尾开始读取文件，然后向前查找分隔符，当前为块内的 \n，然后将起始偏移量前进 sepindex （这是为了防止出现搜索字符串被分割为两个连续读取）。在继续读取下一个块之前，它会在读取的块中查找 search 字符串，如果找到，它将返回其在文件中的起始位置并停止。 否则，它将起始偏移量减少 b.len，然后读取下一个块。

只要您的搜索字符串位于文件的最后 40%，您就应该获得更好的性能，但是这还需要经过实战测试。

如果您的搜索字符串在最后 10% 之内，我相信您会获胜。

main.go

package main

import (
    "bytes"
    "flag"
    "fmt"
    "io"
    "log"
    "os"
    "time"

    "github.com/mattetti/filebuffer"
)

func main() {

    var search string
    var sep string
    var verbose bool
    flag.stringvar(&search, "search", "findme", "search word")
    flag.stringvar(&sep, "sep", "\n", "separator for the search detection")
    flag.boolvar(&verbose, "v", false, "verbosity")
    flag.parse()

    d := make(chan struct{})
    b := &bytes.buffer{}
    go func() {
        io.copy(b, os.stdin)
        d <- struct{}{}
    }()
    <-time.after(time.millisecond)
    select {
    case <-d:
    default:
        os.stdin.close()
    }

    readsize := 1024
    if b.len() < 1 {
        input := fmt.sprintf("%s%stail content", bytes.repeat([]byte(" "), readsize-5), search)
        input += input
        b.writestring(input)
    }

    bsearch := []byte(search)
    s, err := bytessearch(b.bytes())
    if err != nil {
        log.fatal(err)
    }
    if verbose {
        s.logger = log.new(os.stderr, "", log.lstdflags)
    }
    s.buffer = make([]byte, readsize)
    s.sep = []byte(sep)
    got, err := s.index(bsearch)
    if err != nil {
        log.fatal(err)
    }
    fmt.println("index ", got)
    got, err = s.index2(bsearch)
    if err != nil {
        log.fatal(err)
    }
    fmt.println("index ", got)

}

type tailsearch struct {
    f      io.readseeker
    buffer []byte
    sep    []byte
    start  int64
    logger interface {
        println(...interface{})
    }
}

func filesearch(f *os.file) (ret tailsearch, err error) {
    ret.f = f
    st, err := f.stat()
    if err != nil {
        return
    }
    ret.start = st.size()
    ret.sep = []byte("\n")
    return ret, nil
}

func bytessearch(b []byte) (ret tailsearch, err error) {
    ret.f = filebuffer.new(b)
    ret.start = int64(len(b))
    ret.sep = []byte("\n")
    return
}

func (b tailsearch) index(search []byte) (int64, error) {

    if b.buffer == nil {
        b.buffer = make([]byte, 1024, 1024)
    }
    buf := b.buffer
    blen := len(b.buffer)

    hasended := false
    for !hasended {
        if b.logger != nil {
            b.logger.println("a start", b.start)
        }
        offset := b.start - int64(blen)
        if offset < 0 {
            offset = 0
            hasended = true
        }
        _, err := b.f.seek(offset, 0)
        if err != nil {
            hasended = true
        }
        n, err := b.f.read(buf)
        if b.logger != nil {
            b.logger.println("f n", n, "err", err)
        }
        if err != nil {
            hasended = true
        }
        buf = buf[:n]
        b.start -= int64(n)
        if b.logger != nil {
            b.logger.println("g start", b.start)
        }
        if b.start > 0 {
            i := bytes.index(buf, b.sep)
            if b.logger != nil {
                b.logger.println("h sep", i)
            }
            if i > -1 {
                b.start += int64(i)
                buf = buf[i:]
                if b.logger != nil {
                    b.logger.println("i start", b.start)
                }
            }
        }
        if e := bytes.lastindex(buf, search); e > -1 {
            return b.start + int64(e), nil
        }
    }

    return -1, nil
}

func (b tailsearch) index2(search []byte) (int64, error) {

    if b.buffer == nil {
        b.buffer = make([]byte, 1024, 1024)
    }
    buf := b.buffer
    blen := len(b.buffer)

    hasended := false
    for !hasended {
        if b.logger != nil {
            b.logger.println("a start", b.start)
        }
        offset := b.start - int64(blen)
        if offset < 0 {
            offset = 0
            hasended = true
        }
        _, err := b.f.seek(offset, 0)
        if err != nil {
            hasended = true
        }

        n, err := b.f.read(buf)
        if b.logger != nil {
            b.logger.println("f n", n, "err", err)
        }
        if err != nil {
            hasended = true
        }
        buf = buf[:n]
        b.start -= int64(n)

        if b.logger != nil {
            b.logger.println("g start", b.start)
        }

        for i := 1; i < len(search); i++ {
            if bytes.hasprefix(buf, search[i:]) {
                e := i - len(search)
                b.start += int64(e)
                buf = buf[e:]
            }
        }
        if b.logger != nil {
            b.logger.println("g start", b.start)
        }

        if e := bytes.lastindex(buf, search); e > -1 {
            return b.start + int64(e), nil
        }
    }

    return -1, nil
}

main_test.go

package main

import (
    "bytes"
    "fmt"
    "strings"
    "testing"
)

func testone(t *testing.t) {

    type test struct {
        search  []byte
        readlen int
        input   string
        sep     []byte
        want    int64
    }

    search := []byte("find me")
    blocklen := 1024
    tests := []test{
        test{
            search:  search,
            sep:     []byte("\n"),
            readlen: blocklen,
            input:   fmt.sprintf("%stail content", search),
            want:    0,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readlen: blocklen,
            input:   fmt.sprintf(""),
            want:    -1,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readlen: blocklen,
            input:   strings.repeat("nop\n", 10000),
            want:    -1,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readlen: blocklen,
            input:   fmt.sprintf("%s%stail content", bytes.repeat([]byte(" "), blocklen-5), search),
            want:    1019,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readlen: blocklen,
            input:   fmt.sprintf("%s%stail content", bytes.repeat([]byte(" "), blocklen), search),
            want:    1024,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readlen: blocklen,
            input:   fmt.sprintf("%s%stail content", bytes.repeat([]byte(" "), blocklen+10), search),
            want:    1034,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readlen: blocklen,
            input:   fmt.sprintf("%s%stail content", bytes.repeat([]byte(" "), (blocklen*2)+10), search),
            want:    2058,
        },
        test{
            search:  search,
            sep:     []byte("\n"),
            readlen: blocklen,
            input:   fmt.sprintf("%s%s%stail content", bytes.repeat([]byte(" "), (blocklen*2)+10), search, search),
            want:    2065,
        },
    }

    for i, test := range tests {
        s, err := bytessearch([]byte(test.input))
        if err != nil {
            t.fatal(err)
        }
        s.buffer = make([]byte, test.readlen)
        s.sep = test.sep
        got, err := s.index(test.search)
        if err != nil {
            t.fatal(err)
        }
        if got != test.want {
            t.fatalf("invalid index at %v got %v wanted %v", i, got, test.want)
        }
        got, err = s.index2(test.search)
        if err != nil {
            t.fatal(err)
        }
        if got != test.want {
            t.fatalf("invalid index at %v got %v wanted %v", i, got, test.want)
        }
    }

}

bench_test.go

package main

import (
    "bytes"
    "fmt"
    "testing"

    "github.com/mattetti/filebuffer"
)

func benchmarkindex(b *testing.b) {
    search := []byte("find me")
    blocklen := 1024
    input := fmt.sprintf("%s%stail content", bytes.repeat([]byte(" "), blocklen-5), search)
    input += input
    s := tailsearch{}
    s.f = filebuffer.new([]byte(input))
    s.buffer = make([]byte, blocklen)
    for i := 0; i < b.n; i++ {
        s.start = int64(len(input))
        _, err := s.index(search)
        if err != nil {
            b.fatal(err)
        }
    }
}

func benchmarkindex2(b *testing.b) {
    search := []byte("find me")
    blocklen := 1024
    input := fmt.sprintf("%s%stail content", bytes.repeat([]byte(" "), blocklen-5), search)
    input += input
    s := tailsearch{}
    s.f = filebuffer.new([]byte(input))
    s.buffer = make([]byte, blocklen)
    for i := 0; i < b.n; i++ {
        s.start = int64(len(input))
        _, err := s.index2(search)
        if err != nil {
            b.fatal(err)
        }
    }
}

测试

$ go test -v
=== RUN   TestOne
--- PASS: TestOne (0.00s)
PASS
ok      test/backwardsearch 0.002s
$ go test -bench=. -benchmem -v
=== RUN   TestOne
--- PASS: TestOne (0.00s)
goos: linux
goarch: amd64
pkg: test/backwardsearch
BenchmarkIndex-4        20000000           108 ns/op           0 B/op          0 allocs/op
BenchmarkIndex2-4       10000000           167 ns/op           0 B/op          0 allocs/op
PASS
ok      test/backwardsearch 4.129s
$ echo "rrrrfindme" | go run main.go -v
2019/10/17 12:17:04 a start 11
2019/10/17 12:17:04 f n 11 err <nil>
2019/10/17 12:17:04 g start 0
Index  4
2019/10/17 12:17:04 a start 11
2019/10/17 12:17:04 f n 11 err <nil>
2019/10/17 12:17:04 g start 0
2019/10/17 12:17:04 g start 0
Index  4
$ cat bench_test.go | go run main.go -search main
Index  8
Index  8
$ go run main.go 
Index  2056
Index  2056

以上就是本文的全部内容了，是否有顺利帮助你解决问题？若是能给你带来学习上的帮助，请大家多多支持golang学习网！更多关于Golang的相关知识，也可关注golang学习网公众号。