perf(compress): optimize short logs by bypass compressing under 128 bytes
This commit is contained in:
@@ -260,6 +260,22 @@ func (l *TinyLog) CompressAndCleanup() (string, error) {
|
|||||||
os.Remove(l.path) // Cleanup
|
os.Remove(l.path) // Cleanup
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
// 获取文件大小
|
||||||
|
stat, err := f.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
size := stat.Size()
|
||||||
|
|
||||||
|
// 如果日志极短,免去压缩和 Base64 编码,直接以 raw: 明文形式返回
|
||||||
|
if size <= 128 {
|
||||||
|
content, err := io.ReadAll(f)
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return "raw:" + string(content), nil
|
||||||
|
}
|
||||||
|
|
||||||
// 创建压缩输出缓冲区
|
// 创建压缩输出缓冲区
|
||||||
var buf bytes.Buffer
|
var buf bytes.Buffer
|
||||||
b64Writer := base64.NewEncoder(base64.StdEncoding, &buf)
|
b64Writer := base64.NewEncoder(base64.StdEncoding, &buf)
|
||||||
@@ -268,12 +284,6 @@ func (l *TinyLog) CompressAndCleanup() (string, error) {
|
|||||||
zw := utils.GetZstdWriter(b64Writer)
|
zw := utils.GetZstdWriter(b64Writer)
|
||||||
defer utils.PutZstdWriter(zw)
|
defer utils.PutZstdWriter(zw)
|
||||||
|
|
||||||
// 获取文件大小
|
|
||||||
stat, err := f.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return "", err
|
|
||||||
}
|
|
||||||
size := stat.Size()
|
|
||||||
maxSize := int64(constant.MaxLogSize)
|
maxSize := int64(constant.MaxLogSize)
|
||||||
if maxSize < 1024*1024 {
|
if maxSize < 1024*1024 {
|
||||||
maxSize = 1024 * 1024
|
maxSize = 1024 * 1024
|
||||||
|
|||||||
@@ -11,7 +11,10 @@ import (
|
|||||||
"github.com/klauspost/compress/zstd"
|
"github.com/klauspost/compress/zstd"
|
||||||
)
|
)
|
||||||
|
|
||||||
const zstdPrefix = "zstd:"
|
const (
|
||||||
|
zstdPrefix = "zstd:"
|
||||||
|
rawPrefix = "raw:"
|
||||||
|
)
|
||||||
|
|
||||||
var zlibWriterPool = sync.Pool{
|
var zlibWriterPool = sync.Pool{
|
||||||
New: func() interface{} {
|
New: func() interface{} {
|
||||||
@@ -71,23 +74,32 @@ func initZstd() {
|
|||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
// CompressToBase64 compresses data using zstd and encodes to base64 with a prefix
|
// CompressToBase64 compresses data using zstd and encodes to base64 with a prefix (uses raw for data under 128 bytes)
|
||||||
func CompressToBase64(data string) (string, error) {
|
func CompressToBase64(data string) (string, error) {
|
||||||
if data == "" {
|
if data == "" {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
initZstd()
|
|
||||||
|
|
||||||
|
// 小于等于 128 字节,不需要压缩,直接前缀明文保存
|
||||||
|
if len(data) <= 128 {
|
||||||
|
return rawPrefix + data, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
initZstd()
|
||||||
compressed := zstdEncoder.EncodeAll([]byte(data), nil)
|
compressed := zstdEncoder.EncodeAll([]byte(data), nil)
|
||||||
return zstdPrefix + base64.StdEncoding.EncodeToString(compressed), nil
|
return zstdPrefix + base64.StdEncoding.EncodeToString(compressed), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DecompressFromBase64 decodes base64 and decompresses data (supports zstd prefix and falls back to zlib)
|
// DecompressFromBase64 decodes base64 and decompresses data (supports raw/zstd prefix and falls back to zlib)
|
||||||
func DecompressFromBase64(data string) (string, error) {
|
func DecompressFromBase64(data string) (string, error) {
|
||||||
if data == "" {
|
if data == "" {
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if strings.HasPrefix(data, rawPrefix) {
|
||||||
|
return data[len(rawPrefix):], nil
|
||||||
|
}
|
||||||
|
|
||||||
if strings.HasPrefix(data, zstdPrefix) {
|
if strings.HasPrefix(data, zstdPrefix) {
|
||||||
initZstd()
|
initZstd()
|
||||||
encoded := data[len(zstdPrefix):]
|
encoded := data[len(zstdPrefix):]
|
||||||
|
|||||||
@@ -91,3 +91,31 @@ func TestEmptyString(t *testing.T) {
|
|||||||
t.Errorf("Expected empty string for empty input, got: %q", decompressed)
|
t.Errorf("Expected empty string for empty input, got: %q", decompressed)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestCompressShortTextRaw(t *testing.T) {
|
||||||
|
shortText := "python secret.py" // 16 bytes
|
||||||
|
|
||||||
|
// 1. 压缩(应转为 raw 前缀)
|
||||||
|
output, err := CompressToBase64(shortText)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("CompressToBase64 short text failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !strings.HasPrefix(output, "raw:") {
|
||||||
|
t.Errorf("Expected short text output to have 'raw:' prefix, got: %q", output)
|
||||||
|
}
|
||||||
|
if output != "raw:python secret.py" {
|
||||||
|
t.Errorf("Expected output raw:python secret.py, got: %q", output)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. 解码解密
|
||||||
|
decompressed, err := DecompressFromBase64(output)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DecompressFromBase64 short text failed: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if decompressed != shortText {
|
||||||
|
t.Errorf("Decompressed short text mismatch.\nExpected: %q\nGot: %q", shortText, decompressed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -4,6 +4,10 @@ import { decompress } from 'fzstd'
|
|||||||
export function decompressFromBase64(compressed: string): string {
|
export function decompressFromBase64(compressed: string): string {
|
||||||
if (!compressed) return ''
|
if (!compressed) return ''
|
||||||
try {
|
try {
|
||||||
|
if (compressed.startsWith('raw:')) {
|
||||||
|
return compressed.slice(4)
|
||||||
|
}
|
||||||
|
|
||||||
let isZstd = false
|
let isZstd = false
|
||||||
let dataToDecode = compressed
|
let dataToDecode = compressed
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user