perf(compress): optimize short logs by bypass compressing under 128 bytes
This commit is contained in:
@@ -260,6 +260,22 @@ func (l *TinyLog) CompressAndCleanup() (string, error) {
|
||||
os.Remove(l.path) // Cleanup
|
||||
}()
|
||||
|
||||
// 获取文件大小
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
size := stat.Size()
|
||||
|
||||
// 如果日志极短,免去压缩和 Base64 编码,直接以 raw: 明文形式返回
|
||||
if size <= 128 {
|
||||
content, err := io.ReadAll(f)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return "raw:" + string(content), nil
|
||||
}
|
||||
|
||||
// 创建压缩输出缓冲区
|
||||
var buf bytes.Buffer
|
||||
b64Writer := base64.NewEncoder(base64.StdEncoding, &buf)
|
||||
@@ -268,12 +284,6 @@ func (l *TinyLog) CompressAndCleanup() (string, error) {
|
||||
zw := utils.GetZstdWriter(b64Writer)
|
||||
defer utils.PutZstdWriter(zw)
|
||||
|
||||
// 获取文件大小
|
||||
stat, err := f.Stat()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
size := stat.Size()
|
||||
maxSize := int64(constant.MaxLogSize)
|
||||
if maxSize < 1024*1024 {
|
||||
maxSize = 1024 * 1024
|
||||
|
||||
@@ -11,7 +11,10 @@ import (
|
||||
"github.com/klauspost/compress/zstd"
|
||||
)
|
||||
|
||||
const zstdPrefix = "zstd:"
|
||||
const (
|
||||
zstdPrefix = "zstd:"
|
||||
rawPrefix = "raw:"
|
||||
)
|
||||
|
||||
var zlibWriterPool = sync.Pool{
|
||||
New: func() interface{} {
|
||||
@@ -71,23 +74,32 @@ func initZstd() {
|
||||
})
|
||||
}
|
||||
|
||||
// CompressToBase64 compresses data using zstd and encodes to base64 with a prefix
|
||||
// CompressToBase64 compresses data using zstd and encodes to base64 with a prefix (uses raw for data under 128 bytes)
|
||||
func CompressToBase64(data string) (string, error) {
|
||||
if data == "" {
|
||||
return "", nil
|
||||
}
|
||||
initZstd()
|
||||
|
||||
// 小于等于 128 字节,不需要压缩,直接前缀明文保存
|
||||
if len(data) <= 128 {
|
||||
return rawPrefix + data, nil
|
||||
}
|
||||
|
||||
initZstd()
|
||||
compressed := zstdEncoder.EncodeAll([]byte(data), nil)
|
||||
return zstdPrefix + base64.StdEncoding.EncodeToString(compressed), nil
|
||||
}
|
||||
|
||||
// DecompressFromBase64 decodes base64 and decompresses data (supports zstd prefix and falls back to zlib)
|
||||
// DecompressFromBase64 decodes base64 and decompresses data (supports raw/zstd prefix and falls back to zlib)
|
||||
func DecompressFromBase64(data string) (string, error) {
|
||||
if data == "" {
|
||||
return "", nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(data, rawPrefix) {
|
||||
return data[len(rawPrefix):], nil
|
||||
}
|
||||
|
||||
if strings.HasPrefix(data, zstdPrefix) {
|
||||
initZstd()
|
||||
encoded := data[len(zstdPrefix):]
|
||||
|
||||
@@ -91,3 +91,31 @@ func TestEmptyString(t *testing.T) {
|
||||
t.Errorf("Expected empty string for empty input, got: %q", decompressed)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCompressShortTextRaw(t *testing.T) {
|
||||
shortText := "python secret.py" // 16 bytes
|
||||
|
||||
// 1. 压缩(应转为 raw 前缀)
|
||||
output, err := CompressToBase64(shortText)
|
||||
if err != nil {
|
||||
t.Fatalf("CompressToBase64 short text failed: %v", err)
|
||||
}
|
||||
|
||||
if !strings.HasPrefix(output, "raw:") {
|
||||
t.Errorf("Expected short text output to have 'raw:' prefix, got: %q", output)
|
||||
}
|
||||
if output != "raw:python secret.py" {
|
||||
t.Errorf("Expected output raw:python secret.py, got: %q", output)
|
||||
}
|
||||
|
||||
// 2. 解码解密
|
||||
decompressed, err := DecompressFromBase64(output)
|
||||
if err != nil {
|
||||
t.Fatalf("DecompressFromBase64 short text failed: %v", err)
|
||||
}
|
||||
|
||||
if decompressed != shortText {
|
||||
t.Errorf("Decompressed short text mismatch.\nExpected: %q\nGot: %q", shortText, decompressed)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -4,6 +4,10 @@ import { decompress } from 'fzstd'
|
||||
export function decompressFromBase64(compressed: string): string {
|
||||
if (!compressed) return ''
|
||||
try {
|
||||
if (compressed.startsWith('raw:')) {
|
||||
return compressed.slice(4)
|
||||
}
|
||||
|
||||
let isZstd = false
|
||||
let dataToDecode = compressed
|
||||
|
||||
|
||||
Reference in New Issue
Block a user