feat: complete remaining Go backend modules
This commit is contained in:
zn-admin committed
1 parent
cea2751dc5
commit
aef5a97165
145 files changed
+18376
-199
No files matched your search
@@ -0,0 +1,318 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/rand"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"reflect"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
)
|
||||
|
||||
type Level string
|
||||
|
||||
const (
|
||||
Info Level = "info"
|
||||
Warning Level = "warning"
|
||||
Error Level = "error"
|
||||
)
|
||||
|
||||
type Entry struct {
|
||||
ID string `json:"id"`
|
||||
CreatedAt string `json:"createdAt"`
|
||||
Level Level `json:"level"`
|
||||
Source string `json:"source"`
|
||||
Message string `json:"message"`
|
||||
Status int `json:"status,omitempty"`
|
||||
Method string `json:"method,omitempty"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Stack string `json:"stack,omitempty"`
|
||||
Details any `json:"details,omitempty"`
|
||||
}
|
||||
type Input struct {
|
||||
Level Level
|
||||
Source string
|
||||
Message string
|
||||
Error error
|
||||
Status int
|
||||
Method string
|
||||
Path string
|
||||
Stack string
|
||||
Details any
|
||||
}
|
||||
type Filters struct {
|
||||
Level, Q, Source string
|
||||
Limit int
|
||||
}
|
||||
type Service struct {
|
||||
mu sync.Mutex
|
||||
path string
|
||||
maxBytes int64
|
||||
now func() time.Time
|
||||
newID func() string
|
||||
}
|
||||
|
||||
func New(path string, maxBytes int64, now func() time.Time, newID func() string) *Service {
|
||||
if maxBytes <= 0 {
|
||||
maxBytes = 5 * 1024 * 1024
|
||||
}
|
||||
if now == nil {
|
||||
now = time.Now
|
||||
}
|
||||
if newID == nil {
|
||||
newID = randomID
|
||||
}
|
||||
return &Service{path: path, maxBytes: maxBytes, now: now, newID: newID}
|
||||
}
|
||||
func (s *Service) Append(ctx context.Context, input Input) (Entry, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return Entry{}, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
level := input.Level
|
||||
if !validLevel(level) {
|
||||
if input.Status >= 500 {
|
||||
level = Error
|
||||
} else {
|
||||
level = Info
|
||||
}
|
||||
}
|
||||
message := input.Message
|
||||
if message == "" && input.Error != nil {
|
||||
message = input.Error.Error()
|
||||
}
|
||||
if message == "" {
|
||||
message = "Unknown log event"
|
||||
}
|
||||
stack := input.Stack
|
||||
if stack == "" && input.Error != nil {
|
||||
stack = input.Error.Error()
|
||||
}
|
||||
entry := Entry{ID: "log_" + s.newID(), CreatedAt: s.now().UTC().Format(time.RFC3339Nano), Level: level, Source: sanitizeText(defaultString(input.Source, "server")), Message: sanitizeText(message), Status: input.Status, Method: sanitizeText(input.Method), Path: sanitizeText(input.Path), Stack: sanitizeText(stack), Details: sanitize(input.Details, 0, map[visit]bool{})}
|
||||
line, err := json.Marshal(entry)
|
||||
if err != nil {
|
||||
return Entry{}, err
|
||||
}
|
||||
if err = os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
|
||||
return Entry{}, err
|
||||
}
|
||||
if info, statErr := os.Stat(s.path); statErr == nil && info.Size() >= s.maxBytes {
|
||||
if err = os.Rename(s.path, s.path+".1"); err != nil {
|
||||
return Entry{}, err
|
||||
}
|
||||
} else if statErr != nil && !errors.Is(statErr, os.ErrNotExist) {
|
||||
return Entry{}, statErr
|
||||
}
|
||||
file, err := os.OpenFile(s.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0o600)
|
||||
if err != nil {
|
||||
return Entry{}, err
|
||||
}
|
||||
_, writeErr := file.Write(append(line, '\n'))
|
||||
closeErr := file.Close()
|
||||
if writeErr != nil {
|
||||
return Entry{}, writeErr
|
||||
}
|
||||
if closeErr != nil {
|
||||
return Entry{}, closeErr
|
||||
}
|
||||
return entry, nil
|
||||
}
|
||||
func (s *Service) List(ctx context.Context, filters Filters) ([]Entry, error) {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
data, err := os.ReadFile(s.path)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return []Entry{}, nil
|
||||
}
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
limit := filters.Limit
|
||||
if limit == 0 {
|
||||
limit = 100
|
||||
}
|
||||
if limit < 1 {
|
||||
limit = 1
|
||||
}
|
||||
if limit > 500 {
|
||||
limit = 500
|
||||
}
|
||||
q := strings.ToLower(strings.TrimSpace(filters.Q))
|
||||
source := strings.ToLower(strings.TrimSpace(filters.Source))
|
||||
level := strings.TrimSpace(filters.Level)
|
||||
lines := strings.Split(string(data), "\n")
|
||||
result := make([]Entry, 0, limit)
|
||||
for i := len(lines) - 1; i >= 0 && len(result) < limit; i-- {
|
||||
if strings.TrimSpace(lines[i]) == "" {
|
||||
continue
|
||||
}
|
||||
var entry Entry
|
||||
if json.Unmarshal([]byte(lines[i]), &entry) != nil || entry.ID == "" || entry.CreatedAt == "" || entry.Source == "" || entry.Message == "" || !validLevel(entry.Level) {
|
||||
continue
|
||||
}
|
||||
if level != "" && level != "all" && string(entry.Level) != level {
|
||||
continue
|
||||
}
|
||||
if source != "" && !strings.Contains(strings.ToLower(entry.Source), source) {
|
||||
continue
|
||||
}
|
||||
if q != "" && !strings.Contains(strings.ToLower(search(entry)), q) {
|
||||
continue
|
||||
}
|
||||
result = append(result, entry)
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
func (s *Service) Clear(ctx context.Context) error {
|
||||
if err := ctx.Err(); err != nil {
|
||||
return err
|
||||
}
|
||||
s.mu.Lock()
|
||||
defer s.mu.Unlock()
|
||||
if err := os.MkdirAll(filepath.Dir(s.path), 0o755); err != nil {
|
||||
return err
|
||||
}
|
||||
return os.WriteFile(s.path, []byte{}, 0o600)
|
||||
}
|
||||
|
||||
const maxText = 20000
|
||||
|
||||
var sensitiveKey = regexp.MustCompile(`(?i)api[_-]?key|access[_-]?key|secret|token|password|authorization|credential`)
|
||||
var authorizationText = regexp.MustCompile(`(?i)(authorization\s*[:=]\s*)[^\s,;}]+`)
|
||||
var authorizationBearerText = regexp.MustCompile(`(?i)(authorization\s*[:=]\s*bearer\s+)[^\s,;}]+`)
|
||||
var bearerText = regexp.MustCompile(`(?i)(bearer\s+)[A-Za-z0-9._~+/=-]+`)
|
||||
var sensitiveText = regexp.MustCompile(`(?i)((?:api[_-]?key|access[_-]?key|secret|token|password|client[_-]?secret)\s*[:=]\s*)[^\s,;}]+`)
|
||||
var jwtText = regexp.MustCompile(`\beyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\b`)
|
||||
|
||||
func sanitizeText(input string) string {
|
||||
input = truncate(input)
|
||||
input = authorizationBearerText.ReplaceAllString(input, "${1}[redacted]")
|
||||
input = authorizationText.ReplaceAllString(input, "${1}[redacted]")
|
||||
input = bearerText.ReplaceAllString(input, "${1}[redacted]")
|
||||
input = sensitiveText.ReplaceAllString(input, "${1}[redacted]")
|
||||
return jwtText.ReplaceAllString(input, "[jwt-redacted]")
|
||||
}
|
||||
func truncate(input string) string {
|
||||
runes := []rune(input)
|
||||
if len(runes) <= maxText {
|
||||
return input
|
||||
}
|
||||
return string(runes[:maxText]) + "...[truncated]"
|
||||
}
|
||||
|
||||
type visit struct {
|
||||
kind reflect.Kind
|
||||
pointer uintptr
|
||||
}
|
||||
|
||||
func sanitize(value any, depth int, seen map[visit]bool) any {
|
||||
if value == nil {
|
||||
return nil
|
||||
}
|
||||
if text, ok := value.(string); ok {
|
||||
return sanitizeText(text)
|
||||
}
|
||||
v := reflect.ValueOf(value)
|
||||
for v.Kind() == reflect.Interface || v.Kind() == reflect.Pointer {
|
||||
if v.IsNil() {
|
||||
return nil
|
||||
}
|
||||
key := visit{v.Kind(), v.Pointer()}
|
||||
if seen[key] {
|
||||
return "[circular]"
|
||||
}
|
||||
seen[key] = true
|
||||
v = v.Elem()
|
||||
}
|
||||
if depth >= 5 {
|
||||
return "[truncated]"
|
||||
}
|
||||
switch v.Kind() {
|
||||
case reflect.Bool:
|
||||
return v.Bool()
|
||||
case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
|
||||
return v.Int()
|
||||
case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64:
|
||||
return v.Uint()
|
||||
case reflect.Float32, reflect.Float64:
|
||||
return v.Float()
|
||||
case reflect.String:
|
||||
return sanitizeText(v.String())
|
||||
case reflect.Slice, reflect.Array:
|
||||
length := v.Len()
|
||||
if length > 50 {
|
||||
length = 50
|
||||
}
|
||||
out := make([]any, length)
|
||||
for i := 0; i < length; i++ {
|
||||
out[i] = sanitize(v.Index(i).Interface(), depth+1, seen)
|
||||
}
|
||||
return out
|
||||
case reflect.Map:
|
||||
if v.Type().Key().Kind() != reflect.String {
|
||||
return sanitizeText(fmt.Sprint(value))
|
||||
}
|
||||
keys := v.MapKeys()
|
||||
if len(keys) > 80 {
|
||||
keys = keys[:80]
|
||||
}
|
||||
out := map[string]any{}
|
||||
for _, key := range keys {
|
||||
name := key.String()
|
||||
if sensitiveKey.MatchString(name) {
|
||||
out[name] = "[redacted]"
|
||||
} else {
|
||||
out[name] = sanitize(v.MapIndex(key).Interface(), depth+1, seen)
|
||||
}
|
||||
}
|
||||
return out
|
||||
case reflect.Struct:
|
||||
out := map[string]any{}
|
||||
typeOfValue := v.Type()
|
||||
for index := 0; index < v.NumField() && index < 80; index++ {
|
||||
fieldValue := v.Field(index)
|
||||
if !fieldValue.CanInterface() {
|
||||
continue
|
||||
}
|
||||
name := typeOfValue.Field(index).Name
|
||||
if sensitiveKey.MatchString(name) {
|
||||
out[name] = "[redacted]"
|
||||
} else {
|
||||
out[name] = sanitize(fieldValue.Interface(), depth+1, seen)
|
||||
}
|
||||
}
|
||||
return out
|
||||
default:
|
||||
return sanitizeText(fmt.Sprint(value))
|
||||
}
|
||||
}
|
||||
func validLevel(level Level) bool { return level == Info || level == Warning || level == Error }
|
||||
func search(entry Entry) string {
|
||||
details, _ := json.Marshal(entry.Details)
|
||||
return strings.Join([]string{entry.Message, entry.Source, entry.Path, entry.Method, strconv.Itoa(entry.Status), entry.Stack, string(details)}, " ")
|
||||
}
|
||||
func defaultString(value, fallback string) string {
|
||||
if value == "" {
|
||||
return fallback
|
||||
}
|
||||
return value
|
||||
}
|
||||
func randomID() string {
|
||||
buffer := make([]byte, 9)
|
||||
if _, err := rand.Read(buffer); err != nil {
|
||||
return strconv.FormatInt(time.Now().UnixNano(), 36)
|
||||
}
|
||||
return hex.EncodeToString(buffer)
|
||||
}
|
||||
@@ -0,0 +1,123 @@
|
||||
package logging
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestFixtureIsConsumedByGo(t *testing.T) {
|
||||
data, err := os.ReadFile("../../../contracts/logs/runtime-v1.json")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var fixture struct {
|
||||
Version, DefaultLimit, MinimumLimit, MaximumLimit, MaximumTextLength, MaximumDepth int
|
||||
FileName, RotatedSuffix string
|
||||
}
|
||||
if err := json.Unmarshal(data, &fixture); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if fixture.Version != 1 || fixture.FileName != "server-events.jsonl" || fixture.MaximumLimit != 500 || fixture.MaximumDepth != 5 {
|
||||
t.Fatalf("fixture = %#v", fixture)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceAppendsListsFiltersAndClears(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "server-events.jsonl")
|
||||
now := time.Date(2026, 8, 13, 8, 0, 0, 0, time.UTC)
|
||||
service := New(path, 1<<20, func() time.Time { now = now.Add(time.Second); return now }, func() string { return "fixed-id" })
|
||||
first, err := service.Append(context.Background(), Input{Level: Error, Source: "api.test", Message: "Authorization: Bearer top-secret", Status: 500, Method: "POST", Path: "/api/test?token=abc", Stack: "password=hunter2", Details: map[string]any{"Authorization": "Bearer hidden", "nested": map[string]any{"api_key": "key", "prompt": "EvoLink"}}})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if first.ID != "log_fixed-id" {
|
||||
t.Fatalf("entry = %#v", first)
|
||||
}
|
||||
if _, err := service.Append(context.Background(), Input{Level: Warning, Source: "worker.test", Message: "retry scheduled"}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(path, append(mustRead(t, path), []byte("not-json\n")...), 0o600); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
entries, err := service.List(context.Background(), Filters{Level: "error", Q: "evolink", Source: "API", Limit: 900})
|
||||
if err != nil || len(entries) != 1 || entries[0].Source != "api.test" {
|
||||
t.Fatalf("entries = %#v, err = %v", entries, err)
|
||||
}
|
||||
raw := string(mustRead(t, path))
|
||||
for _, secret := range []string{"top-secret", "abc", "hunter2", "hidden", "\"key\""} {
|
||||
if strings.Contains(raw, secret) {
|
||||
t.Fatalf("raw log leaked %q: %s", secret, raw)
|
||||
}
|
||||
}
|
||||
all, err := service.List(context.Background(), Filters{})
|
||||
if err != nil || len(all) != 2 || all[0].Message != "retry scheduled" {
|
||||
t.Fatalf("all = %#v, err = %v", all, err)
|
||||
}
|
||||
if err := service.Clear(context.Background()); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if entries, err := service.List(context.Background(), Filters{}); err != nil || len(entries) != 0 {
|
||||
t.Fatalf("after clear = %#v, %v", entries, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceRotatesAndBoundsNestedDetails(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
path := filepath.Join(dir, "server-events.jsonl")
|
||||
service := New(path, 200, time.Now, func() string { return "id" })
|
||||
if _, err := service.Append(context.Background(), Input{Source: "one", Message: strings.Repeat("x", 250)}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := service.Append(context.Background(), Input{Source: "two", Message: "next", Details: map[string]any{"jwt": "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiIxIn0.signature", "deep": nested(8)}}); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, err := os.Stat(path + ".1"); err != nil {
|
||||
t.Fatalf("rotated log: %v", err)
|
||||
}
|
||||
raw := string(mustRead(t, path))
|
||||
if strings.Contains(raw, "eyJhbGci") || !strings.Contains(raw, "[truncated]") {
|
||||
t.Fatalf("bounds/redaction missing: %s", raw)
|
||||
}
|
||||
}
|
||||
|
||||
func TestServiceSerializesConcurrentAppends(t *testing.T) {
|
||||
path := filepath.Join(t.TempDir(), "server-events.jsonl")
|
||||
service := New(path, 1<<20, time.Now, nil)
|
||||
var wait sync.WaitGroup
|
||||
for index := 0; index < 40; index++ {
|
||||
wait.Add(1)
|
||||
go func(index int) {
|
||||
defer wait.Done()
|
||||
if _, err := service.Append(context.Background(), Input{Source: "parallel", Message: fmt.Sprintf("entry-%d", index)}); err != nil {
|
||||
t.Errorf("append %d: %v", index, err)
|
||||
}
|
||||
}(index)
|
||||
}
|
||||
wait.Wait()
|
||||
entries, err := service.List(context.Background(), Filters{Limit: 100})
|
||||
if err != nil || len(entries) != 40 {
|
||||
t.Fatalf("entries = %d, err = %v", len(entries), err)
|
||||
}
|
||||
}
|
||||
|
||||
func nested(depth int) any {
|
||||
if depth == 0 {
|
||||
return "leaf"
|
||||
}
|
||||
return map[string]any{"child": nested(depth - 1)}
|
||||
}
|
||||
func mustRead(t *testing.T, path string) []byte {
|
||||
t.Helper()
|
||||
data, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return data
|
||||
}
|
||||
Reference in new issue
Block a user