perf(compress): optimize short logs by bypass compressing under 128 bytes

This commit is contained in:
duorameng
2026-07-09 20:36:47 +08:00
parent 516a91acc2
commit b67fa5bd58
4 changed files with 64 additions and 10 deletions
+16 -6
View File
@@ -260,6 +260,22 @@ func (l *TinyLog) CompressAndCleanup() (string, error) {
os.Remove(l.path) // Cleanup
}()
// 获取文件大小
stat, err := f.Stat()
if err != nil {
return "", err
}
size := stat.Size()
// 如果日志极短,免去压缩和 Base64 编码,直接以 raw: 明文形式返回
if size <= 128 {
content, err := io.ReadAll(f)
if err != nil {
return "", err
}
return "raw:" + string(content), nil
}
// 创建压缩输出缓冲区
var buf bytes.Buffer
b64Writer := base64.NewEncoder(base64.StdEncoding, &buf)
@@ -268,12 +284,6 @@ func (l *TinyLog) CompressAndCleanup() (string, error) {
zw := utils.GetZstdWriter(b64Writer)
defer utils.PutZstdWriter(zw)
// 获取文件大小
stat, err := f.Stat()
if err != nil {
return "", err
}
size := stat.Size()
maxSize := int64(constant.MaxLogSize)
if maxSize < 1024*1024 {
maxSize = 1024 * 1024
+16 -4
View File
@@ -11,7 +11,10 @@ import (
"github.com/klauspost/compress/zstd"
)
const zstdPrefix = "zstd:"
const (
zstdPrefix = "zstd:"
rawPrefix = "raw:"
)
var zlibWriterPool = sync.Pool{
New: func() interface{} {
@@ -71,23 +74,32 @@ func initZstd() {
})
}
// CompressToBase64 compresses data using zstd and encodes to base64 with a prefix
// CompressToBase64 compresses data using zstd and encodes to base64 with a prefix (uses raw for data under 128 bytes)
func CompressToBase64(data string) (string, error) {
if data == "" {
return "", nil
}
initZstd()
// 小于等于 128 字节,不需要压缩,直接前缀明文保存
if len(data) <= 128 {
return rawPrefix + data, nil
}
initZstd()
compressed := zstdEncoder.EncodeAll([]byte(data), nil)
return zstdPrefix + base64.StdEncoding.EncodeToString(compressed), nil
}
// DecompressFromBase64 decodes base64 and decompresses data (supports zstd prefix and falls back to zlib)
// DecompressFromBase64 decodes base64 and decompresses data (supports raw/zstd prefix and falls back to zlib)
func DecompressFromBase64(data string) (string, error) {
if data == "" {
return "", nil
}
if strings.HasPrefix(data, rawPrefix) {
return data[len(rawPrefix):], nil
}
if strings.HasPrefix(data, zstdPrefix) {
initZstd()
encoded := data[len(zstdPrefix):]
+28
View File
@@ -91,3 +91,31 @@ func TestEmptyString(t *testing.T) {
t.Errorf("Expected empty string for empty input, got: %q", decompressed)
}
}
func TestCompressShortTextRaw(t *testing.T) {
shortText := "python secret.py" // 16 bytes
// 1. 压缩(应转为 raw 前缀)
output, err := CompressToBase64(shortText)
if err != nil {
t.Fatalf("CompressToBase64 short text failed: %v", err)
}
if !strings.HasPrefix(output, "raw:") {
t.Errorf("Expected short text output to have 'raw:' prefix, got: %q", output)
}
if output != "raw:python secret.py" {
t.Errorf("Expected output raw:python secret.py, got: %q", output)
}
// 2. 解码解密
decompressed, err := DecompressFromBase64(output)
if err != nil {
t.Fatalf("DecompressFromBase64 short text failed: %v", err)
}
if decompressed != shortText {
t.Errorf("Decompressed short text mismatch.\nExpected: %q\nGot: %q", shortText, decompressed)
}
}
+4
View File
@@ -4,6 +4,10 @@ import { decompress } from 'fzstd'
export function decompressFromBase64(compressed: string): string {
if (!compressed) return ''
try {
if (compressed.startsWith('raw:')) {
return compressed.slice(4)
}
let isZstd = false
let dataToDecode = compressed