优化结构

This commit is contained in:
Yvan
2026-08-21 21:37:30 +08:00
parent a2ce3ae218
commit e4ba0dced9
41 changed files with 3646 additions and 364 deletions
+6
View File
@@ -65,6 +65,12 @@ func NewGinEngineWithRuntime(runtime *conf.Runtime, access *service.AccessContro
}
registerSwagger(engine, prefix, version, logger)
registerLocalStorage(engine, runtime)
if logger != nil {
for _, route := range engine.Routes() {
logger.Info("router registered", "method", route.Method, "path", route.Path)
}
logger.Info("router register success", "route_count", len(engine.Routes()))
}
engine.NoRoute(func(c *gin.Context) {
if ws != nil && ws.Enabled() && c.Request.Method == http.MethodGet && c.Request.URL.Path == ws.Path() {
+95 -12
View File
@@ -2,10 +2,14 @@ package middleware
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"errors"
"io"
"log/slog"
"mime"
"net/http"
"strconv"
"strings"
"time"
@@ -67,14 +71,26 @@ func AccessLog(runtime *conf.Runtime, logger *slog.Logger, version string) gin.H
if config != nil && config.Zap != nil && config.Zap.AccessLogMaxBytes > 0 {
logLimit = int(config.Zap.AccessLogMaxBytes)
}
paymentCallback := isPaymentCallbackPath(c.Request.URL.Path)
paymentConfigWrite := isPaymentIntegrationConfigWrite(c.Request.Method, c.Request.URL.Path)
requestText := ""
if multipart {
if paymentCallback {
requestText = paymentCallbackSummary(requestBody, c.GetHeader("Content-Type"))
} else if paymentConfigWrite {
requestText = paymentConfigSummary(requestBody)
} else if multipart {
requestText = "[文件]"
} else {
requestText = redactJSON(requestBody, c.GetHeader("Content-Type"), logLimit)
}
c.Set(ctxReqBodyKey, requestText)
c.Set(ctxRespBufferKey, &writer.body)
if paymentCallback {
// Callback acknowledgements and provider payloads must not flow into
// the generic response/error audit pipeline.
c.Set(ctxRespBufferKey, &bytes.Buffer{})
} else {
c.Set(ctxRespBufferKey, &writer.body)
}
if !requestReadFailed {
c.Next()
}
@@ -82,6 +98,9 @@ func AccessLog(runtime *conf.Runtime, logger *slog.Logger, version string) gin.H
return
}
responseText := redactJSON(writer.body.Bytes(), c.Writer.Header().Get("Content-Type"), logLimit)
if paymentCallback {
responseText = "[支付回调响应已省略]"
}
userID, authorityID := uint(0), uint(0)
if claims := Claims(c); claims != nil {
userID, authorityID = claims.ID, claims.AuthorityID
@@ -95,22 +114,36 @@ func AccessLog(runtime *conf.Runtime, logger *slog.Logger, version string) gin.H
bytesOut = 0
}
privateErrors := strings.TrimRight(c.Errors.ByType(gin.ErrorTypePrivate).String(), "\n")
attributes := []any{
"mod", "http", "ip", c.ClientIP(), "method", c.Request.Method, "http_path", c.Request.URL.Path, "http_route", route,
"http_status", c.Writer.Status(), "latency_ms", time.Since(started).Milliseconds(),
"request_id", stringValueFromContext(c, "request_id"), "trace_id", stringValueFromContext(c, "trace_id"),
"bytes_in", bytesIn, "bytes_out", bytesOut, "user_id", userID, "authority_id", authorityID,
"error", c.Writer.Status() >= http.StatusInternalServerError || privateErrors != "", "ua", c.Request.UserAgent(), "req_query", c.Request.URL.RawQuery}
if config != nil && config.Zap != nil && config.Zap.AccessReqHeaders {
var attributes []any
if paymentCallback {
attributes = []any{
"mod", "payment-callback", "payment_provider", paymentCallbackProvider(c.Request.URL.Path),
"http_status", c.Writer.Status(), "latency_ms", time.Since(started).Milliseconds(),
"request_id", stringValueFromContext(c, "request_id"), "trace_id", stringValueFromContext(c, "trace_id"),
"bytes_in", bytesIn, "bytes_out", bytesOut,
"error", c.Writer.Status() >= http.StatusInternalServerError || privateErrors != "",
"payment_callback", true, "payment_callback_summary", requestText,
}
} else {
attributes = []any{
"mod", "http", "ip", c.ClientIP(), "method", c.Request.Method, "http_path", c.Request.URL.Path, "http_route", route,
"http_status", c.Writer.Status(), "latency_ms", time.Since(started).Milliseconds(),
"request_id", stringValueFromContext(c, "request_id"), "trace_id", stringValueFromContext(c, "trace_id"),
"bytes_in", bytesIn, "bytes_out", bytesOut, "user_id", userID, "authority_id", authorityID,
"error", c.Writer.Status() >= http.StatusInternalServerError || privateErrors != "", "ua", c.Request.UserAgent(),
"req_query", c.Request.URL.RawQuery,
}
}
if !paymentCallback && config != nil && config.Zap != nil && config.Zap.AccessReqHeaders {
attributes = append(attributes, "req_headers", redactHeaders(c.Request.Header))
}
if config != nil && config.Zap != nil && config.Zap.AccessReqBody {
if !paymentCallback && config != nil && config.Zap != nil && config.Zap.AccessReqBody {
attributes = append(attributes, "req_body", requestText)
}
if config != nil && config.Zap != nil && config.Zap.AccessRespData {
if !paymentCallback && config != nil && config.Zap != nil && config.Zap.AccessRespData {
attributes = append(attributes, "resp_data", responseText)
}
if privateErrors != "" {
if !paymentCallback && privateErrors != "" {
attributes = append(attributes, "error_msg", privateErrors)
}
logger.InfoContext(c.Request.Context(), "请求完成", attributes...)
@@ -122,6 +155,56 @@ func isMediaUploadRoute(route string) bool {
strings.HasSuffix(route, "/mediaUpload/chunk")
}
func isPaymentCallbackPath(path string) bool {
parts := strings.Split(strings.Trim(path, "/"), "/")
for index := 0; index+1 < len(parts); index++ {
if parts[index] == "payment" && parts[index+1] == "callback" {
return true
}
}
return false
}
func paymentCallbackProvider(path string) string {
parts := strings.Split(strings.Trim(path, "/"), "/")
for index := 0; index+2 < len(parts); index++ {
if parts[index] == "payment" && parts[index+1] == "callback" {
return parts[index+2]
}
}
return "unknown"
}
func paymentCallbackSummary(body []byte, contentType string) string {
mediaType, _, err := mime.ParseMediaType(contentType)
if err != nil || mediaType == "" {
mediaType = strings.TrimSpace(strings.SplitN(contentType, ";", 2)[0])
}
if mediaType == "" {
mediaType = "unknown"
}
digest := sha256.Sum256(body)
return "[支付回调正文已省略 body_bytes=" + strconv.Itoa(len(body)) + " body_sha256=" + hex.EncodeToString(digest[:]) + " content_type=" + mediaType + "]"
}
func isPaymentIntegrationConfigWrite(method, path string) bool {
if method != http.MethodPut {
return false
}
parts := strings.Split(strings.Trim(path, "/"), "/")
for index := 0; index+3 < len(parts); index++ {
if parts[index] == "integration" && parts[index+1] == "configs" && parts[index+2] == "payment" && parts[index+3] != "" {
return true
}
}
return false
}
func paymentConfigSummary(body []byte) string {
digest := sha256.Sum256(body)
return "[支付配置正文已省略 body_bytes=" + strconv.Itoa(len(body)) + " body_sha256=" + hex.EncodeToString(digest[:]) + "]"
}
func redactHeaders(headers map[string][]string) map[string]string {
out := make(map[string]string, len(headers))
for key, values := range headers {
@@ -1,7 +1,9 @@
package middleware
import (
"bytes"
"io"
"log/slog"
"net/http"
"net/http/httptest"
"strings"
@@ -63,3 +65,60 @@ func TestAccessLogAllowsMediaLimitOnlyOnUploadRoute(t *testing.T) {
t.Fatalf("status=%d called=%v body=%s", response.Code, called, response.Body.String())
}
}
func TestAccessLogRedactsPaymentCallbackPayloadAndHeaders(t *testing.T) {
gin.SetMode(gin.TestMode)
var logs bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo}))
runtime := conf.NewRuntime(nil, &conf.AdminBackend{Zap: &conf.AdminBackend_Zap{AccessReqBody: true, AccessReqHeaders: true, AccessRespData: true}})
engine := gin.New()
engine.Use(AccessLog(runtime, logger, "test"))
engine.POST("/api/payment/callback/:provider", func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"body": "callback-response-secret"})
})
request := httptest.NewRequest(http.MethodPost, "/api/payment/callback/alipay?signature=query-secret", strings.NewReader("payment-body-secret"))
request.Header.Set("Content-Type", "application/json; boundary=credential-secret")
request.Header.Set("Authorization", "Bearer header-secret")
request.Header.Set("X-Alipay-Signature", "signature-secret")
response := httptest.NewRecorder()
engine.ServeHTTP(response, request)
logText := logs.String()
for _, secret := range []string{"payment-body-secret", "query-secret", "header-secret", "signature-secret", "callback-response-secret", "credential-secret"} {
if strings.Contains(logText, secret) {
t.Fatalf("payment callback secret leaked into access log: %q in %s", secret, logText)
}
}
for _, marker := range []string{"payment_callback=true", "payment_provider=alipay", "body_sha256=", "http_status=200"} {
if !strings.Contains(logText, marker) {
t.Fatalf("payment callback access summary missing %q: %s", marker, logText)
}
}
}
func TestAccessLogOmitsPaymentIntegrationConfigBody(t *testing.T) {
gin.SetMode(gin.TestMode)
var logs bytes.Buffer
logger := slog.New(slog.NewTextHandler(&logs, &slog.HandlerOptions{Level: slog.LevelInfo}))
runtime := conf.NewRuntime(nil, &conf.AdminBackend{Zap: &conf.AdminBackend_Zap{AccessReqBody: true, AccessReqHeaders: true, AccessRespData: true}})
engine := gin.New()
engine.Use(AccessLog(runtime, logger, "test"))
engine.PUT("/api/integration/configs/:kind/:provider", func(c *gin.Context) { c.JSON(http.StatusOK, gin.H{"code": 0}) })
body := `{"enabled":true,"config":{"key":"saobei-secret","certificate_blob":"certificate-secret","unknown_credential":"credential-secret"}}`
request := httptest.NewRequest(http.MethodPut, "/api/integration/configs/payment/saobei", strings.NewReader(body))
request.Header.Set("Content-Type", "application/json")
response := httptest.NewRecorder()
engine.ServeHTTP(response, request)
logText := logs.String()
for _, secret := range []string{"saobei-secret", "certificate-secret", "credential-secret"} {
if strings.Contains(logText, secret) {
t.Fatalf("payment configuration secret leaked into access log: %q in %s", secret, logText)
}
}
if !strings.Contains(logText, "支付配置正文已省略") || !strings.Contains(logText, "body_sha256=") {
t.Fatalf("payment configuration summary missing: %s", logText)
}
}
+45 -7
View File
@@ -81,7 +81,11 @@ func OperationAudit(runtime *conf.Runtime, recorder *service.AuditRecorder) gin.
responseBody = "[超出记录长度]"
}
errorMessage := c.Errors.ByType(gin.ErrorTypePrivate).String()
if err := recorder.RecordOperationRequest(c.Request.Context(), &dto.OperationRecordRequest{IP: c.ClientIP(), Method: c.Request.Method, Path: path, Status: status, LatencyMS: time.Since(started).Milliseconds(), Agent: c.Request.UserAgent(), ErrorMessage: errorMessage, Body: operationRequestBody(requestBody, c.GetHeader("Content-Type"), maxBytes), Response: responseBody, UserID: userID, RequestID: stringValue(requestID), TraceID: stringValueFromContext(c, "trace_id"), DeviceID: c.GetHeader("X-Device-Id")}); err != nil {
operationBody := operationRequestBody(requestBody, c.GetHeader("Content-Type"), maxBytes)
if isPaymentIntegrationConfigWrite(c.Request.Method, path) {
operationBody = paymentConfigSummary(requestBody)
}
if err := recorder.RecordOperationRequest(c.Request.Context(), &dto.OperationRecordRequest{IP: c.ClientIP(), Method: c.Request.Method, Path: path, Status: status, LatencyMS: time.Since(started).Milliseconds(), Agent: c.Request.UserAgent(), ErrorMessage: errorMessage, Body: operationBody, Response: responseBody, UserID: userID, RequestID: stringValue(requestID), TraceID: stringValueFromContext(c, "trace_id"), DeviceID: c.GetHeader("X-Device-Id")}); err != nil {
// Preserve the business response, but expose audit persistence failures
// to the global access/error logging pipeline.
c.Set(ctxOperationAuditPersistFailedKey, true)
@@ -128,7 +132,7 @@ func maskOperationBody(value any) {
case map[string]any:
for key, item := range current {
normalized := strings.ToLower(strings.ReplaceAll(strings.ReplaceAll(key, "_", ""), "-", ""))
if normalized == "password" || normalized == "newpassword" || normalized == "oldpassword" || normalized == "confirmpassword" || normalized == "passwd" || normalized == "pwd" || normalized == "token" || normalized == "accesstoken" || normalized == "refreshtoken" || normalized == "secret" || normalized == "clientsecret" || normalized == "apikey" || normalized == "privatekey" || normalized == "idcard" {
if normalized == "password" || normalized == "newpassword" || normalized == "oldpassword" || normalized == "confirmpassword" || normalized == "passwd" || normalized == "pwd" || normalized == "token" || normalized == "accesstoken" || normalized == "refreshtoken" || normalized == "secret" || normalized == "clientsecret" || normalized == "apikey" || normalized == "privatekey" || normalized == "idcard" || normalized == "appkey" || normalized == "mchkey" || normalized == "apiv3key" || normalized == "clientcert" || normalized == "clientkey" || normalized == "platformcert" || normalized == "platformserialno" || normalized == "credentialcode" || normalized == "certfile" || normalized == "keyfile" || normalized == "publickey" || normalized == "rootcert" || normalized == "appcert" || normalized == "webhookid" {
current[key] = "***"
continue
}
@@ -157,17 +161,49 @@ func isDownloadResponse(c *gin.Context) bool {
// recordsOperation mirrors the routes on which operation records are enabled.
// Matching by suffix keeps the behavior stable when router-prefix is configured.
func recordsOperation(method, path string) bool {
_, ok := operationRoutes[method+" "+routeSuffix(path)]
return ok
for route := range operationRoutes {
parts := strings.SplitN(route, " ", 2)
if len(parts) != 2 || parts[0] != method || !operationPathMatches(parts[1], path) {
continue
}
return true
}
return false
}
func operationPathMatches(pattern, path string) bool {
patternParts := strings.Split(strings.Trim(pattern, "/"), "/")
pathParts := strings.Split(strings.Trim(path, "/"), "/")
if len(pathParts) < len(patternParts) {
return false
}
pathParts = pathParts[len(pathParts)-len(patternParts):]
for index, patternPart := range patternParts {
if strings.HasPrefix(patternPart, "*") {
return index <= len(pathParts)
}
if index >= len(pathParts) || (strings.HasPrefix(patternPart, ":") == false && patternPart != pathParts[index]) {
return false
}
}
return len(patternParts) == len(pathParts)
}
func routeSuffix(path string) string {
for _, marker := range []string{"/user/", "/api/", "/casbin/", "/authority/", "/menu/", "/department/", "/position/", "/sysDictionary/", "/sysDictionaryDetail/", "/sysParams/", "/securityConfig/", "/system/", "/sysApiToken/", "/sysVersion/", "/sysExportTemplate/", "/sysError/", "/sysLoginLog/", "/sysOperationRecord/", "/dataAccessLog/", "/timedTask/", "/info/", "/email/"} {
bestIndex := -1
bestPath := path
for _, marker := range []string{"/user/", "/api/", "/casbin/", "/authority/", "/menu/", "/department/", "/position/", "/sysDictionary/", "/sysDictionaryDetail/", "/sysParams/", "/securityConfig/", "/system/", "/sysApiToken/", "/sysVersion/", "/sysExportTemplate/", "/sysError/", "/sysLoginLog/", "/sysOperationRecord/", "/dataAccessLog/", "/timedTask/", "/info/", "/email/", "/integration/", "/payment/"} {
if index := strings.Index(path, marker); index >= 0 {
return path[index:]
// Router prefixes may themselves contain a registered route marker
// (for example /api/integration/...). Keep the deepest match so the
// policy and audit route remain the actual application endpoint.
if index > bestIndex {
bestIndex = index
bestPath = path[index:]
}
}
}
return path
return bestPath
}
var operationRoutes = func() map[string]struct{} {
@@ -189,6 +225,8 @@ var operationRoutes = func() map[string]struct{} {
"DELETE /sysLoginLog/deleteLoginLog", "DELETE /sysLoginLog/deleteLoginLogByIds", "DELETE /dataAccessLog/deleteDataAccessLogByIds",
"POST /timedTask/createTimedTask", "PUT /timedTask/updateTimedTask", "DELETE /timedTask/deleteTimedTask", "POST /timedTask/toggleTimedTask", "POST /timedTask/triggerTimedTask",
"POST /info/createInfo", "DELETE /info/deleteInfo", "DELETE /info/deleteInfoByIds", "PUT /info/updateInfo", "POST /email/emailTest", "POST /email/sendEmail",
"PUT /integration/configs/:kind/:provider", "DELETE /integration/configs/:kind/:provider",
"POST /payment/create", "POST /payment/query", "POST /payment/refund", "POST /payment/orders/:provider/:tradeNo/refund", "POST /payment/fulfill", "POST /payment/orders/:provider/:tradeNo/fulfill", "POST /payment/providers/:provider/test",
}
out := make(map[string]struct{}, len(values))
for _, value := range values {
@@ -0,0 +1,56 @@
package middleware
import (
"encoding/json"
"strings"
"testing"
)
func TestPaymentIntegrationSecretsAreRedacted(t *testing.T) {
raw := []byte(`{"enabled":true,"config":{"app_id":"app","mch_key":"merchant-secret","api_v3_key":"v3-secret","client_cert":"certificate","client_key":"private-key","platform_cert":"platform-certificate","credential_code":"credential","webhook_id":"webhook"}}`)
redacted := redactJSON(raw, "application/json", 4096)
var payload struct {
Config map[string]string `json:"config"`
}
if err := json.Unmarshal([]byte(redacted), &payload); err != nil {
t.Fatalf("decode redacted payload: %v", err)
}
for _, key := range []string{"mch_key", "api_v3_key", "client_cert", "client_key", "platform_cert", "credential_code", "webhook_id"} {
if payload.Config[key] != "***" {
t.Fatalf("payment secret %q was not redacted: %s", key, redacted)
}
}
if !strings.Contains(redacted, `"app_id":"app"`) {
t.Fatalf("non-secret integration field was removed: %s", redacted)
}
}
func TestPaymentOperationsAreAuditedWithRouterPrefix(t *testing.T) {
for _, route := range []struct {
method string
path string
}{
{method: "PUT", path: "/api/integration/configs/payment/alipay"},
{method: "POST", path: "/api/payment/refund"},
{method: "POST", path: "/api/payment/fulfill"},
{method: "POST", path: "/api/payment/providers/alipay/test"},
} {
if !recordsOperation(route.method, route.path) {
t.Fatalf("payment operation was not audited: %s %s", route.method, route.path)
}
}
}
func TestPaymentIntegrationConfigUsesRouteLevelSummary(t *testing.T) {
raw := []byte(`{"enabled":true,"config":{"key":"secret","custom_certificate":"certificate"}}`)
summary := paymentConfigSummary(raw)
if strings.Contains(summary, "secret") || strings.Contains(summary, "certificate") {
t.Fatalf("payment configuration summary leaked payload: %s", summary)
}
if !isPaymentIntegrationConfigWrite("PUT", "/api/integration/configs/payment/saobei") {
t.Fatal("payment configuration write route was not recognized")
}
if isPaymentIntegrationConfigWrite("PUT", "/api/integration/configs/mq/emqx") {
t.Fatal("non-payment integration was treated as payment configuration")
}
}