From a9e1f9e6f85083588cf6f355f6065c4f646ffaf8 Mon Sep 17 00:00:00 2001 From: yvan <8574526@qq.com> Date: Sun, 16 Aug 2026 20:58:57 +0800 Subject: [PATCH] =?UTF-8?q?=E4=BC=98=E5=8C=96?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- cmd/kratos-admin/main.go | 40 ++- cmd/kratos-admin/main_test.go | 27 ++ cmd/kratos-admin/wire.go | 3 +- cmd/kratos-admin/wire_gen.go | 7 +- go.mod | 16 +- go.sum | 38 ++- internal/biz/api.go | 5 +- internal/biz/security.go | 27 +- internal/data/authority.go | 4 +- internal/data/casbin.go | 2 +- internal/data/config_store.go | 14 +- internal/data/config_watch.go | 16 +- internal/data/data.go | 71 +++-- internal/data/data_scope.go | 34 ++- internal/data/data_scope_audit.go | 121 ++++++++ internal/data/data_scope_audit_test.go | 72 +++++ internal/data/database.go | 29 +- internal/data/dictionary.go | 10 +- internal/data/gorm_logger.go | 65 ++++ internal/data/gorm_logger_test.go | 31 ++ internal/data/log_file_test.go | 50 ++++ internal/data/migrations.go | 52 ++-- internal/data/runtime_clients.go | 8 +- internal/data/system_init.go | 3 +- internal/data/system_init_ignore_test.go | 12 + internal/data/transactions_test.go | 2 +- internal/server/gin.go | 3 +- internal/server/gin_test.go | 279 +++++++++++++++++- internal/server/handler/menu.go | 2 +- internal/server/handler/user.go | 14 +- internal/server/middleware/error_audit.go | 36 ++- .../server/middleware/error_audit_test.go | 75 +++++ internal/server/middleware/rate_limit.go | 2 +- internal/server/middleware/rate_limit_test.go | 49 +++ internal/server/middleware/recovery.go | 11 +- internal/server/swagger.go | 167 +++++++++++ internal/service/api_metadata.go | 6 + internal/service/security.go | 16 +- internal/worker/task_scheduler.go | 13 + pkg/logging/source.go | 98 ++++++ pkg/logging/zap.go | 129 +++++++- pkg/logging/zap_test.go | 66 ++++- 42 files changed, 1569 insertions(+), 156 deletions(-) create mode 100644 cmd/kratos-admin/main_test.go create mode 100644 internal/data/data_scope_audit.go create mode 100644 internal/data/data_scope_audit_test.go create mode 100644 internal/data/gorm_logger.go create mode 100644 internal/data/gorm_logger_test.go create mode 100644 internal/data/log_file_test.go create mode 100644 internal/data/system_init_ignore_test.go create mode 100644 internal/server/middleware/error_audit_test.go create mode 100644 internal/server/middleware/rate_limit_test.go create mode 100644 internal/server/swagger.go create mode 100644 pkg/logging/source.go diff --git a/cmd/kratos-admin/main.go b/cmd/kratos-admin/main.go index 179a149..00d0fc1 100644 --- a/cmd/kratos-admin/main.go +++ b/cmd/kratos-admin/main.go @@ -1,12 +1,17 @@ package main import ( + "context" "flag" "log/slog" + "net" "os" "path/filepath" + "strings" "kra/internal/conf" + "kra/internal/service" + "kra/internal/service/dto" "kra/internal/worker" "kra/pkg/logging" @@ -35,7 +40,12 @@ func init() { flag.StringVar(&flagconf, "conf", "../../configs", "config path, eg: -conf config.yaml") } -func newApp(logger *slog.Logger, hs *kratoshttp.Server, scheduler *worker.TaskScheduler) *kratos.App { +func newApp(logger *slog.Logger, hs *kratoshttp.Server, scheduler *worker.TaskScheduler, audit *service.AuditRecorder, loggerControl *logging.ReloadableLogger) *kratos.App { + if audit != nil && loggerControl != nil { + loggerControl.SetErrorSink(logging.ErrorSinkFunc(func(ctx context.Context, entry logging.ErrorEntry) error { + return audit.CreateErrorRequest(ctx, &dto.ErrorRecordRequest{Form: entry.Form, Info: entry.Info, Level: entry.Level, RequestID: entry.RequestID, TraceID: entry.TraceID}) + })) + } return kratos.New( kratos.ID(id), kratos.Name(Name), @@ -63,6 +73,30 @@ func zapSettings(admin *conf.AdminBackend) (string, logging.Options) { return root, options } +func httpAddress(server *conf.Server) string { + if server != nil && server.Http != nil && server.Http.Addr != "" { + return server.Http.Addr + } + return ":8000" +} + +func swaggerAddress(server *conf.Server, admin *conf.AdminBackend) string { + address := httpAddress(server) + if strings.HasPrefix(address, ":") { + address = "127.0.0.1" + address + } else if host, port, err := net.SplitHostPort(address); err == nil && (host == "" || host == "0.0.0.0" || host == "::") { + address = net.JoinHostPort("127.0.0.1", port) + } + prefix := "" + if admin != nil { + prefix = strings.Trim(admin.RouterPrefix, "/") + } + if prefix != "" { + prefix = "/" + prefix + } + return "http://" + address + prefix + "/swagger/index.html" +} + func main() { flag.Parse() c := config.New( @@ -87,6 +121,7 @@ func main() { } logger, loggerControl := logging.NewReloadableZapLogger(logRoot, "application.log", logOptions, loggerAttrs...) defer loggerControl.Close() + slog.SetDefault(logger) log.SetDefault(logger) if bc.Admin != nil { bc.Admin.ConfigPath = flagconf @@ -102,11 +137,12 @@ func main() { }) defer unsubscribeLogger() - app, cleanup, err := wireApp(bc.Server, runtime, logger, Version) + app, cleanup, err := wireApp(bc.Server, runtime, logger, loggerControl, Version) if err != nil { panic(err) } defer cleanup() + logger.Info("Kra administration service initialized", "mod", "system", "version", Version, "http_address", httpAddress(bc.Server), "swagger", swaggerAddress(bc.Server, bc.Admin), "admin_frontend", "http://127.0.0.1:8080") // start and wait for stop signal if err := app.Run(); err != nil { diff --git a/cmd/kratos-admin/main_test.go b/cmd/kratos-admin/main_test.go new file mode 100644 index 0000000..135d2b5 --- /dev/null +++ b/cmd/kratos-admin/main_test.go @@ -0,0 +1,27 @@ +package main + +import ( + "testing" + + "kra/internal/conf" +) + +func TestSwaggerAddress(t *testing.T) { + tests := []struct { + name string + server *conf.Server + admin *conf.AdminBackend + want string + }{ + {name: "default", want: "http://127.0.0.1:8000/swagger/index.html"}, + {name: "wildcard with prefix", server: &conf.Server{Http: &conf.Server_HTTP{Addr: "0.0.0.0:9000"}}, admin: &conf.AdminBackend{RouterPrefix: "/admin/"}, want: "http://127.0.0.1:9000/admin/swagger/index.html"}, + {name: "host", server: &conf.Server{Http: &conf.Server_HTTP{Addr: "example.test:8080"}}, want: "http://example.test:8080/swagger/index.html"}, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + if got := swaggerAddress(test.server, test.admin); got != test.want { + t.Fatalf("swaggerAddress() = %q, want %q", got, test.want) + } + }) + } +} diff --git a/cmd/kratos-admin/wire.go b/cmd/kratos-admin/wire.go index 6923583..f4e352a 100644 --- a/cmd/kratos-admin/wire.go +++ b/cmd/kratos-admin/wire.go @@ -14,12 +14,13 @@ import ( "kra/internal/server" "kra/internal/service" "kra/internal/worker" + "kra/pkg/logging" "github.com/go-kratos/kratos/v3" "github.com/google/wire" ) // wireApp init kratos application. -func wireApp(*conf.Server, *conf.Runtime, *slog.Logger, string) (*kratos.App, func(), error) { +func wireApp(*conf.Server, *conf.Runtime, *slog.Logger, *logging.ReloadableLogger, string) (*kratos.App, func(), error) { panic(wire.Build(server.ProviderSet, worker.ProviderSet, data.ProviderSet, biz.ProviderSet, service.ProviderSet, newApp)) } diff --git a/cmd/kratos-admin/wire_gen.go b/cmd/kratos-admin/wire_gen.go index ed88aac..79aacfb 100644 --- a/cmd/kratos-admin/wire_gen.go +++ b/cmd/kratos-admin/wire_gen.go @@ -15,6 +15,7 @@ import ( "kra/internal/server/handler" "kra/internal/service" "kra/internal/worker" + "kra/pkg/logging" "log/slog" ) @@ -25,8 +26,8 @@ import ( // Injectors from wire.go: // wireApp init kratos application. -func wireApp(confServer *conf.Server, runtime *conf.Runtime, logger *slog.Logger, string2 string) (*kratos.App, func(), error) { - dataData, cleanup, err := data.NewData(runtime) +func wireApp(confServer *conf.Server, runtime *conf.Runtime, logger *slog.Logger, reloadableLogger *logging.ReloadableLogger, string2 string) (*kratos.App, func(), error) { + dataData, cleanup, err := data.NewData(runtime, logger) if err != nil { return nil, nil, err } @@ -132,7 +133,7 @@ func wireApp(confServer *conf.Server, runtime *conf.Runtime, logger *slog.Logger set := handler.NewSet(authority, menu, api, permission, organization, announcement, email, task, media, audit, export, version, dictionary, parameter, apiToken, systemConfig, public, user, navigation, session) engine := server.NewGinEngine(runtime, accessControlService, set, authService, securityService, auditRecorder, logger, string2) httpServer := server.NewGinServer(confServer, engine) - app := newApp(logger, httpServer, taskScheduler) + app := newApp(logger, httpServer, taskScheduler, auditRecorder, reloadableLogger) return app, func() { cleanup() }, nil diff --git a/go.mod b/go.mod index 553b8a2..f3923b3 100644 --- a/go.mod +++ b/go.mod @@ -26,6 +26,9 @@ require ( github.com/redis/go-redis/v9 v9.7.0 github.com/robfig/cron/v3 v3.0.1 github.com/shirou/gopsutil/v4 v4.25.7 + github.com/swaggo/files v1.0.1 + github.com/swaggo/gin-swagger v1.6.0 + github.com/swaggo/swag v1.16.4 github.com/tencentyun/cos-go-sdk-v5 v0.7.60 github.com/xuri/excelize/v2 v2.9.0 go.einride.tech/aip v0.86.3 @@ -34,6 +37,7 @@ require ( go.uber.org/zap v1.27.0 go.uber.org/zap/exp v0.3.0 golang.org/x/crypto v0.53.0 + golang.org/x/sync v0.21.0 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/mysql v1.6.0 @@ -47,6 +51,9 @@ replace github.com/go-kratos/kratos/v3 v3.0.0 => github.com/go-kratos/kratos/v3 require ( filippo.io/edwards25519 v1.2.0 // indirect github.com/BurntSushi/toml v1.3.2 // indirect + github.com/KyleBanks/depth v1.2.1 // indirect + github.com/PuerkitoBio/purell v1.1.1 // indirect + github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.5 // indirect github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.18 // indirect @@ -83,6 +90,10 @@ require ( github.com/go-logr/logr v1.4.3 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect + github.com/go-openapi/jsonpointer v0.19.5 // indirect + github.com/go-openapi/jsonreference v0.19.6 // indirect + github.com/go-openapi/spec v0.20.4 // indirect + github.com/go-openapi/swag v0.19.15 // indirect github.com/go-playground/form/v4 v4.3.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect @@ -103,11 +114,13 @@ require ( github.com/jackc/puddle/v2 v2.2.2 // indirect github.com/jinzhu/inflection v1.0.0 // indirect github.com/jinzhu/now v1.1.5 // indirect + github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.18.0 // indirect github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect + github.com/mailru/easyjson v0.7.6 // indirect github.com/mattn/go-isatty v0.0.20 // indirect github.com/microsoft/go-mssqldb v1.8.2 // indirect github.com/minio/crc64nvme v1.0.1 // indirect @@ -147,13 +160,14 @@ require ( golang.org/x/arch v0.8.0 // indirect golang.org/x/image v0.23.0 // indirect golang.org/x/net v0.56.0 // indirect - golang.org/x/sync v0.21.0 // indirect golang.org/x/sys v0.46.0 // indirect golang.org/x/text v0.38.0 // indirect golang.org/x/time v0.15.0 // indirect + golang.org/x/tools v0.45.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260615183401-62b3387ff324 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260615183401-62b3387ff324 // indirect google.golang.org/grpc v1.81.1 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect modernc.org/fileutil v1.0.0 // indirect modernc.org/libc v1.22.5 // indirect modernc.org/mathutil v1.5.0 // indirect diff --git a/go.sum b/go.sum index 9eb7458..4d40196 100644 --- a/go.sum +++ b/go.sum @@ -20,6 +20,12 @@ github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 h1:XHOnouVk1mx github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= +github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= +github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= +github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 h1:d+Bc7a5rLufV/sSk/8dngufqelfh6jnri85riMAaF/M= +github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= github.com/QcloudApi/qcloud_sign_golang v0.0.0-20141224014652-e4130a326409/go.mod h1:1pk82RBxDY/JZnPQrtqHlUFfCctgdorsd9M06fMynOM= github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82 h1:7dONQ3WNZ1zy960TmkxJPuwoolZwL7xKtpcM04MBnt4= github.com/alex-ant/gomath v0.0.0-20160516115720-89013a210a82/go.mod h1:nLnM0KdK1CmygvjpDUO6m1TjSsiQtL61juhNsvV/JVI= @@ -110,6 +116,8 @@ github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uq github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk= github.com/gammazero/toposort v0.1.1 h1:OivGxsWxF3U3+U80VoLJ+f50HcPU1MIqE1JlKzoJ2Eg= github.com/gammazero/toposort v0.1.1/go.mod h1:H2cozTnNpMw0hg2VHAYsAxmkHXBYroNangj2NTBQDvw= +github.com/gin-contrib/gzip v0.0.6 h1:NjcunTcGAj5CO1gn4N8jHOSIeRFHIbn51z6K+xaN4d4= +github.com/gin-contrib/gzip v0.0.6/go.mod h1:QOJlmV2xmayAjkNS2Y8NQsMneuRShOU/kjovCXNuzzk= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= @@ -131,6 +139,16 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= +github.com/go-openapi/jsonpointer v0.19.3/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonpointer v0.19.5 h1:gZr+CIYByUqjcgeLXnQu2gHYQC9o73G2XUeOFYEICuY= +github.com/go-openapi/jsonpointer v0.19.5/go.mod h1:Pl9vOtqEWErmShwVjC8pYs9cog34VGT37dQOVbmoatg= +github.com/go-openapi/jsonreference v0.19.6 h1:UBIxjkht+AWIgYzCDSv2GN+E/togfwXUJFRTWhl2Jjs= +github.com/go-openapi/jsonreference v0.19.6/go.mod h1:diGHMEHg2IqXZGKxqyvWdfWU/aim5Dprw5bqpKkTvns= +github.com/go-openapi/spec v0.20.4 h1:O8hJrt0UMnhHcluhIdUgCLRWyM2x7QkBXRvOs7m+O1M= +github.com/go-openapi/spec v0.20.4/go.mod h1:faYFR1CvsJZ0mNsmsphTMSoRrNV3TEDoAM7FOEWeq8I= +github.com/go-openapi/swag v0.19.5/go.mod h1:POnQmlKehdgb5mhVOsnJFsivZCEZ/vjK9gh66Z9tfKk= +github.com/go-openapi/swag v0.19.15 h1:D2NRCBzS9/pEY3gP9Nl8aDqGUcPFrwG2p+CNFrLyrCM= +github.com/go-openapi/swag v0.19.15/go.mod h1:QYRuS/SOXUCsnplDa677K7+DxSOj6IPNl/eQntq43wQ= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s= github.com/go-playground/assert/v2 v2.2.0/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= @@ -177,8 +195,6 @@ github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26 h1:Xim43kblpZXfIBQsbuBVKCudVG457BR2GZFIz3uw3hQ= github.com/google/pprof v0.0.0-20221118152302-e6195bd50e26/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/subcommands v1.2.0 h1:vWQspBTo2nEqTUFita5/KeEWlUL8kQObDFbub/EN9oE= -github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -216,6 +232,8 @@ github.com/jinzhu/inflection v1.0.0/go.mod h1:h+uFLlag+Qp1Va5pdKtLDYj+kHp5pxUVkr github.com/jinzhu/now v1.1.4/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= github.com/jinzhu/now v1.1.5 h1:/o9tlHleP7gOFmsnYNz3RGnqzefHA47wQpKrrdTIwXQ= github.com/jinzhu/now v1.1.5/go.mod h1:d3SSVoowX0Lcu0IBviAWJpolVfI5UJVZZ7cO71lE/z8= +github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo= @@ -242,6 +260,10 @@ github.com/leodido/go-urn v1.4.0 h1:WT9HwE9SGECu3lg4d/dIA+jxlljEa1/ffXKmRjqdmIQ= github.com/leodido/go-urn v1.4.0/go.mod h1:bvxc+MVxLKB4z00jd1z+Dvzr47oO32F/QSNjSBOlFxI= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 h1:6E+4a0GO5zZEnZ81pIr0yLvtUWk2if982qA3F3QD6H4= github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0/go.mod h1:zJYVVT2jmtg6P3p1VtQj7WsuWi/y4VnjVBn7F8KPB3I= +github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= +github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= +github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/microsoft/go-mssqldb v1.8.2 h1:236sewazvC8FvG6Dr3bszrVhMkAl4KYImryLkRMCd0I= @@ -270,6 +292,7 @@ github.com/montanaflynn/stats v0.7.1 h1:etflOAAHORrCC44V+aR6Ftzort912ZU+YLiSTuV8 github.com/montanaflynn/stats v0.7.1/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mozillazg/go-httpheader v0.2.1 h1:geV7TrjbL8KXSyvghnFm+NyTux/hxwueTSrwhe88TQQ= github.com/mozillazg/go-httpheader v0.2.1/go.mod h1:jJ8xECTlalr6ValeXYdOF8fFUISeBAdw6E61aqQma60= +github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/philhofer/fwd v1.1.3-0.20240916144458-20a13a1f6b7c h1:dAMKvw0MlJT1GshSTtih8C2gDs04w8dReiOGXrGLNoY= @@ -327,6 +350,12 @@ github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXl github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/swaggo/files v1.0.1 h1:J1bVJ4XHZNq0I46UU90611i9/YzdrF7x92oX1ig5IdE= +github.com/swaggo/files v1.0.1/go.mod h1:0qXmMNH6sXNf+73t65aKeB+ApmgxdnkQzVTAj2uaMUg= +github.com/swaggo/gin-swagger v1.6.0 h1:y8sxvQ3E20/RCyrXeFfg60r6H0Z+SwpTjMYsMm+zy8M= +github.com/swaggo/gin-swagger v1.6.0/go.mod h1:BG00cCEy294xtVpyIAHG6+e2Qzj/xKlRdOqDkvq0uzo= +github.com/swaggo/swag v1.16.4 h1:clWJtd9LStiG3VeijiCfOVODP6VpHtKdQy9ELFG3s1A= +github.com/swaggo/swag v1.16.4/go.mod h1:VBsHJRsDvfYvqoiMKnsdwhNV9LEMHgEDZcyVYX0sxPg= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common v1.0.563/go.mod h1:7sCQWVkxcsR38nffDW057DRGk8mUjK1Ing/EFOK8s8Y= github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/kms v1.0.563/go.mod h1:uom4Nvi9W+Qkom0exYiJ9VWJjXwyxtPYTkKkaLMlfE0= github.com/tencentyun/cos-go-sdk-v5 v0.7.60 h1:/e/tmvRmfKexr/QQIBzWhOkZWsmY3EK72NrI6G/Tv0o= @@ -418,6 +447,7 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210421230115-4e50805a0758/go.mod h1:72T/g9IO56b78aLF+1Kcs5dz7/ng1VjMUvfKvpfy+jM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= @@ -449,6 +479,7 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210420072515-93ed5bcd2bfe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616045830-e2b7044e8c71/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -528,14 +559,17 @@ google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBN google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/internal/biz/api.go b/internal/biz/api.go index f5db27c..c144a13 100644 --- a/internal/biz/api.go +++ b/internal/biz/api.go @@ -93,7 +93,10 @@ func (uc *APIUsecase) SyncAPIs(ctx context.Context, routes []*API) (*APISyncDiff } for _, item := range stored { storedKey := key(item) - if routeSet[storedKey] == nil && !ignoreSet[storedKey] { + // Ignoring an API removes it from the in-memory route comparison. If it + // already exists in sys_apis, the reference implementation therefore + // returns it in deleteApis while also returning it in ignoreApis. + if routeSet[storedKey] == nil { diff.Deleted = append(diff.Deleted, item) } } diff --git a/internal/biz/security.go b/internal/biz/security.go index e46f341..c1c0bb1 100644 --- a/internal/biz/security.go +++ b/internal/biz/security.go @@ -3,10 +3,13 @@ package biz import ( "context" "errors" + "fmt" "strconv" "strings" "sync" "time" + "unicode" + "unicode/utf8" ) type SecurityConfig struct { @@ -90,33 +93,37 @@ func (uc *SecurityUsecase) Current(ctx context.Context) (*SecurityConfig, error) } func (uc *SecurityUsecase) ValidatePassword(value *SecurityConfig, password string) error { - if len([]rune(password)) < value.PwdMinLength { - return errors.New("密码长度不足") + if value.PwdMinLength > 0 && utf8.RuneCountInString(password) < value.PwdMinLength { + return fmt.Errorf("密码长度不能少于 %d 位", value.PwdMinLength) } hasUpper, hasLower, hasDigit, hasSpecial := false, false, false, false for _, ch := range password { switch { - case ch >= 'A' && ch <= 'Z': + case unicode.IsUpper(ch): hasUpper = true - case ch >= 'a' && ch <= 'z': + case unicode.IsLower(ch): hasLower = true - case ch >= '0' && ch <= '9': + case unicode.IsDigit(ch): hasDigit = true - default: + case unicode.IsPunct(ch) || unicode.IsSymbol(ch): hasSpecial = true } } + missing := make([]string, 0, 4) if value.PwdRequireUpper && !hasUpper { - return errors.New("密码必须包含大写字母") + missing = append(missing, "大写字母") } if value.PwdRequireLower && !hasLower { - return errors.New("密码必须包含小写字母") + missing = append(missing, "小写字母") } if value.PwdRequireDigit && !hasDigit { - return errors.New("密码必须包含数字") + missing = append(missing, "数字") } if value.PwdRequireSpecial && !hasSpecial { - return errors.New("密码必须包含特殊字符") + missing = append(missing, "特殊字符") + } + if len(missing) > 0 { + return fmt.Errorf("密码必须包含%s", strings.Join(missing, "、")) } return nil } diff --git a/internal/data/authority.go b/internal/data/authority.go index 96332f1..1ca49d1 100644 --- a/internal/data/authority.go +++ b/internal/data/authority.go @@ -125,7 +125,7 @@ func (r *authorityAccessRepo) CopyAuthority(ctx context.Context, sourceID uint, } parentAllowed := value.ParentID != nil && (allowed[*value.ParentID] || *value.ParentID == actor.AuthorityID) if !parentAllowed { - return errors.New("严格角色模式下只能在自己的角色树中复制角色") + return errors.New("您提交的角色ID不合法") } } var count int64 @@ -168,7 +168,7 @@ func (r *authorityAccessRepo) CopyAuthority(ctx context.Context, sourceID uint, return err } if !allowed { - return errors.New("严格角色模式下不能复制当前角色未拥有的 API") + return errors.New("存在api不在权限列表中") } } key := api.V1 + "\x00" + api.V2 diff --git a/internal/data/casbin.go b/internal/data/casbin.go index ab02c65..baef4fc 100644 --- a/internal/data/casbin.go +++ b/internal/data/casbin.go @@ -34,7 +34,7 @@ func newPolicyRule(authorityID uint, path, method string) casbinRulePO { } func policyScope(db *gorm.DB) *gorm.DB { - return db.Where("ptype = ?", "p") + return db.Session(&gorm.Session{NewDB: true}).Model(&casbinRulePO{}).Where("ptype = ?", "p") } func deletePoliciesForAuthority(db *gorm.DB, authorityID uint) error { diff --git a/internal/data/config_store.go b/internal/data/config_store.go index 8931ab6..2809ac2 100644 --- a/internal/data/config_store.go +++ b/internal/data/config_store.go @@ -260,7 +260,7 @@ func (d *Data) reloadConfig(ctx context.Context) error { return fmt.Errorf("data.database and admin configuration are required") } next.Admin.ConfigPath = configPath - candidateDB, err := openDatabase(next.Data.Database, false, "") + candidateDB, err := openDatabase(next.Data.Database, false, "", d.logger()) if err != nil { return fmt.Errorf("reload database: %w", err) } @@ -287,21 +287,27 @@ func (d *Data) reloadConfig(ctx context.Context) error { return fmt.Errorf("reload storage: %w", err) } useRedis := next.Admin.System != nil && next.Admin.System.UseRedis - candidateRedis := openRedis(next.Data.Redis, useRedis) + candidateRedis := openRedis(next.Data.Redis, useRedis, d.logger()) useMongo := next.Admin.System != nil && next.Admin.System.UseMongo candidateMongo, mongoErr := openMongo(next.Data.Mongo, useMongo) + if mongoErr != nil { + d.logger().Error("mongo unavailable during configuration reload", "mod", "mongo", "error", mongoErr) + } mongoAccepted := false defer func() { if !mongoAccepted && candidateMongo != nil { _ = candidateMongo.Disconnect(context.Background()) } }() - candidateDBList, err := openDatabaseList(next.Data.DatabaseList) + candidateDBList, err := openDatabaseList(next.Data.DatabaseList, d.logger()) if err != nil { return err } - d.gormDB.replace(candidateDB) + d.gormDB.replace(candidateDB, d.enqueueDataScopeAudit) + for _, item := range candidateDBList { + registerDataScopeCallbacks(item, d.enqueueDataScopeAudit) + } d.replaceDatabaseList(candidateDBList) d.redis.replace(candidateRedis) if mongoErr == nil { diff --git a/internal/data/config_watch.go b/internal/data/config_watch.go index 0230401..f7130a1 100644 --- a/internal/data/config_watch.go +++ b/internal/data/config_watch.go @@ -1,7 +1,6 @@ package data import ( - "log" "path/filepath" "sync" "time" @@ -13,22 +12,23 @@ import ( // /system/reloadSystem remains responsible // for rebuilding database, Redis, storage, and scheduled tasks. func (d *Data) watchConfig() func() { + logger := d.logger() configPath := d.runtime.ConfigPath() if configPath == "" { return func() {} } absolute, err := filepath.Abs(configPath) if err != nil { - log.Printf("resolve config watch path: %v", err) + logger.Error("resolve config watch path", "mod", "system", "error", err) return func() {} } watcher, err := fsnotify.NewWatcher() if err != nil { - log.Printf("create config watcher: %v", err) + logger.Error("create config watcher", "mod", "system", "error", err) return func() {} } if err = watcher.Add(filepath.Dir(absolute)); err != nil { - log.Printf("watch config directory: %v", err) + logger.Error("watch config directory", "mod", "system", "error", err) _ = watcher.Close() return func() {} } @@ -48,20 +48,20 @@ func (d *Data) watchConfig() func() { timer = time.AfterFunc(100*time.Millisecond, func() { next, loadErr := readBootstrap(absolute) if loadErr != nil { - log.Printf("reload changed config: %v", loadErr) + logger.Error("reload changed config", "mod", "system", "error", loadErr) return } if next.Data == nil || next.Admin == nil { - log.Printf("reload changed config: data and admin configuration are required") + logger.Error("reload changed config: data and admin configuration are required", "mod", "system") return } next.Admin.ConfigPath = absolute d.runtime.Replace(next.Data, next.Admin) - log.Printf("config file changed: %s", absolute) + logger.Info("config file changed", "mod", "system", "path", absolute) }) case watchErr, ok := <-watcher.Errors: if ok { - log.Printf("config watcher error: %v", watchErr) + logger.Error("config watcher error", "mod", "system", "error", watchErr) } case <-done: if timer != nil { diff --git a/internal/data/data.go b/internal/data/data.go index 75f50ef..ff75630 100644 --- a/internal/data/data.go +++ b/internal/data/data.go @@ -3,7 +3,7 @@ package data import ( "context" "fmt" - "log" + "log/slog" "sync" "time" @@ -16,24 +16,33 @@ import ( var ProviderSet = wire.NewSet(NewData, NewRuntimeSettings, NewTokenIssuer, NewUserRepo, NewInitializationRepo, NewAuthorityAccessRepo, NewAPIRepo, NewPermissionRepo, NewMenuRepo, NewDepartmentRepo, NewPositionRepo, NewDictionaryRepo, NewParameterRepo, NewAPITokenRepo, NewSecurityRepo, NewVersionRepo, NewExportRepo, NewAuditRepo, NewAuditRecorderRepo, NewLogFileRepo, NewTaskRepo, NewMediaRepo, NewAnnouncementRepo, NewEmailRepo, NewCache, NewFileStorage) type Data struct { - initMu sync.Mutex - configMu sync.Mutex - gormDB *reloadableDB - redis *reloadableRedis - mongo *reloadableMongo - runtime *conf.Runtime - storage *reloadableStorage - dbListMu sync.RWMutex - dbList map[string]*gorm.DB + initMu sync.Mutex + configMu sync.Mutex + gormDB *reloadableDB + redis *reloadableRedis + mongo *reloadableMongo + runtime *conf.Runtime + storage *reloadableStorage + dbListMu sync.RWMutex + dbList map[string]*gorm.DB + appLogger *slog.Logger + auditLog *dataScopeAuditWriter } -func openDatabaseList(configs []*conf.Data_Database) (map[string]*gorm.DB, error) { +func (d *Data) logger() *slog.Logger { + if d != nil && d.appLogger != nil { + return d.appLogger + } + return slog.Default() +} + +func openDatabaseList(configs []*conf.Data_Database, appLogger ...*slog.Logger) (map[string]*gorm.DB, error) { items := make(map[string]*gorm.DB) for _, config := range configs { if config == nil || config.Disable || config.AliasName == "" { continue } - db, err := openDatabase(config, false, "") + db, err := openDatabase(config, false, "", appLogger...) if err != nil { for _, opened := range items { if sqlDB, dbErr := opened.DB(); dbErr == nil { @@ -42,7 +51,6 @@ func openDatabaseList(configs []*conf.Data_Database) (map[string]*gorm.DB, error } return nil, fmt.Errorf("open database %q: %w", config.AliasName, err) } - registerDataScopeCallbacks(db) items[config.AliasName] = db } return items, nil @@ -77,30 +85,38 @@ func (d *Data) database(name string) (*gorm.DB, error) { return db, nil } -func NewData(runtime *conf.Runtime) (*Data, func(), error) { +func NewData(runtime *conf.Runtime, appLogger *slog.Logger) (*Data, func(), error) { + if appLogger == nil { + appLogger = slog.Default() + } c := runtime.Data() if c == nil || c.Database == nil { return nil, nil, fmt.Errorf("database configuration is required") } - d := &Data{runtime: runtime} - db, err := openDatabase(c.Database, false, "") + d := &Data{runtime: runtime, appLogger: appLogger} + db, err := openDatabase(c.Database, false, "", appLogger) usingFallback := false if err != nil { // The initialization endpoint must remain available when the configured // target database has not been created yet. - log.Printf("configured database unavailable before initialization: %v", err) - db, err = openFallbackDatabase() + appLogger.Warn("configured database unavailable before initialization", "mod", "system", "error", err) + db, err = openFallbackDatabase(appLogger) if err != nil { return nil, nil, fmt.Errorf("open bootstrap database: %w", err) } usingFallback = true } - d.gormDB = newReloadableDB(db) - d.dbList, err = openDatabaseList(c.DatabaseList) + d.gormDB = newReloadableDB(db, d.enqueueDataScopeAudit) + d.auditLog = newDataScopeAuditWriter(d, appLogger) + d.dbList, err = openDatabaseList(c.DatabaseList, appLogger) if err != nil { + d.auditLog.Close() d.gormDB.close() return nil, nil, err } + for _, item := range d.dbList { + registerDataScopeCallbacks(item, d.enqueueDataScopeAudit) + } admin := runtime.Admin() disableAutoMigrate := admin != nil && admin.System != nil && admin.System.DisableAutoMigrate if !usingFallback && !disableAutoMigrate { @@ -109,17 +125,18 @@ func NewData(runtime *conf.Runtime) (*Data, func(), error) { } } useRedis := admin != nil && admin.System != nil && admin.System.UseRedis - d.redis = newReloadableRedis(openRedis(c.Redis, useRedis)) + d.redis = newReloadableRedis(openRedis(c.Redis, useRedis, appLogger)) useMongo := admin != nil && admin.System != nil && admin.System.UseMongo mongoClient, err := openMongo(c.Mongo, useMongo) if err != nil { - log.Printf("mongo unavailable: %v", err) + appLogger.Error("mongo unavailable", "mod", "mongo", "error", err) mongoClient = nil } d.mongo = newReloadableMongo(mongoClient) stopConfigWatcher := d.watchConfig() cleanup := func() { stopConfigWatcher() + d.auditLog.Close() d.gormDB.close() closeDatabaseList(d.dbList) d.redis.close() @@ -128,7 +145,7 @@ func NewData(runtime *conf.Runtime) (*Data, func(), error) { return d, cleanup, nil } -func openRedis(config *conf.Data_Redis, enabled bool) redis.UniversalClient { +func openRedis(config *conf.Data_Redis, enabled bool, appLogger ...*slog.Logger) redis.UniversalClient { if !enabled || config == nil || (config.Addr == "" && len(config.ClusterAddrs) == 0) { return nil } @@ -152,7 +169,11 @@ func openRedis(config *conf.Data_Redis, enabled bool) redis.UniversalClient { pingCtx, cancel := context.WithTimeout(context.Background(), 800*time.Millisecond) defer cancel() if err := candidate.Ping(pingCtx).Err(); err != nil { - log.Printf("redis unavailable, using in-memory cache: %v", err) + log := slog.Default() + if len(appLogger) > 0 && appLogger[0] != nil { + log = appLogger[0] + } + log.Warn("redis unavailable, using in-memory cache", "mod", "redis", "error", err) _ = candidate.Close() return nil } @@ -160,6 +181,6 @@ func openRedis(config *conf.Data_Redis, enabled bool) redis.UniversalClient { } func (d *Data) activateDatabase(db *gorm.DB, config *conf.Data_Database) { - d.gormDB.replace(db) + d.gormDB.replace(db, d.enqueueDataScopeAudit) d.runtime.UpdateDatabase(config) } diff --git a/internal/data/data_scope.go b/internal/data/data_scope.go index 9cf2c9c..a147f86 100644 --- a/internal/data/data_scope.go +++ b/internal/data/data_scope.go @@ -2,7 +2,7 @@ package data import ( "database/sql" - "log" + "log/slog" "reflect" "strings" @@ -16,25 +16,27 @@ import ( // registerDataScopeCallbacks installs the global GORM data-scope engine. // System tables are deliberately excluded: their access is controlled by // Casbin, while ownership columns on business tables are row-level scope. -func registerDataScopeCallbacks(db *gorm.DB) { +type dataScopeAuditEnqueue func(dataAccessLogPO) + +func registerDataScopeCallbacks(db *gorm.DB, enqueue dataScopeAuditEnqueue) { if db == nil { return } q := db.Callback().Query() if q.Get("data_scope:query") == nil { - _ = q.Before("gorm:query").Register("data_scope:query", applyDataScope("query")) + _ = q.Before("gorm:query").Register("data_scope:query", applyDataScope("query", enqueue)) } u := db.Callback().Update() if u.Get("data_scope:update") == nil { - _ = u.Before("gorm:update").Register("data_scope:update", applyDataScope("update")) + _ = u.Before("gorm:update").Register("data_scope:update", applyDataScope("update", enqueue)) _ = u.Before("gorm:update").Register("data_scope:stamp_update", stampUpdatedBy) - _ = u.After("gorm:update").Register("data_scope:audit_update", auditBlockedWrite("update")) + _ = u.After("gorm:update").Register("data_scope:audit_update", auditBlockedWrite("update", enqueue)) } d := db.Callback().Delete() if d.Get("data_scope:delete") == nil { - _ = d.Before("gorm:delete").Register("data_scope:delete", applyDataScope("delete")) + _ = d.Before("gorm:delete").Register("data_scope:delete", applyDataScope("delete", enqueue)) _ = d.Before("gorm:delete").After("data_scope:delete").Register("data_scope:stamp_delete", stampDeletedBy) - _ = d.After("gorm:delete").Register("data_scope:audit_delete", auditBlockedWrite("delete")) + _ = d.After("gorm:delete").Register("data_scope:audit_delete", auditBlockedWrite("delete", enqueue)) } c := db.Callback().Create() if c.Get("data_scope:stamp") == nil { @@ -50,7 +52,7 @@ func isControlledTable(db *gorm.DB) bool { return db.Statement.Schema != nil && !strings.HasPrefix(db.Statement.Table, "sys_") && (hasScopeField(db, "dept_id") || hasScopeField(db, "created_by")) } -func applyDataScope(operation string) func(*gorm.DB) { +func applyDataScope(operation string, enqueue dataScopeAuditEnqueue) func(*gorm.DB) { return func(db *gorm.DB) { if !isControlledTable(db) { return @@ -65,8 +67,8 @@ func applyDataScope(operation string) func(*gorm.DB) { } scope, ok := biz.DataScopeFromContext(db.Statement.Context) if !ok { - log.Printf("数据权限: 业务表[%s]访问无身份上下文, 已放行(待补 ctx / 或使用系统上下文)", db.Statement.Table) - recordDataScopeEvent(db, "no_identity", operation, "无身份上下文访问受控表, 已放行", biz.DataScope{}) + slog.WarnContext(db.Statement.Context, "数据权限: 业务表访问无身份上下文, 已放行(待补 ctx / 或使用系统上下文)", "mod", "data-scope", "table", db.Statement.Table) + recordDataScopeEvent(db, enqueue, "no_identity", operation, "无身份上下文访问受控表, 已放行", biz.DataScope{}) return } if (operation == "update" || operation == "delete") && !db.AllowGlobalUpdate && !hasWriteConditions(db) { @@ -91,18 +93,20 @@ func applyDataScope(operation string) func(*gorm.DB) { } } -func recordDataScopeEvent(db *gorm.DB, eventType, operation, detail string, scope biz.DataScope) { - record := &dataAccessLogPO{EventType: eventType, TargetTable: db.Statement.Table, Operation: operation, UserID: scope.UserID, AuthorityID: scope.AuthorityID, Scope: scope.Scope, RequestID: scope.RequestID, Method: scope.Method, Path: scope.Path, Detail: detail} - _ = db.Session(&gorm.Session{NewDB: true, SkipHooks: true}).Create(record).Error +func recordDataScopeEvent(db *gorm.DB, enqueue dataScopeAuditEnqueue, eventType, operation, detail string, scope biz.DataScope) { + if enqueue == nil { + return + } + enqueue(dataAccessLogPO{EventType: eventType, TargetTable: db.Statement.Table, Operation: operation, UserID: scope.UserID, AuthorityID: scope.AuthorityID, Scope: scope.Scope, RequestID: scope.RequestID, Method: scope.Method, Path: scope.Path, Detail: detail}) } -func auditBlockedWrite(operation string) func(*gorm.DB) { +func auditBlockedWrite(operation string, enqueue dataScopeAuditEnqueue) func(*gorm.DB) { return func(db *gorm.DB) { if _, applied := db.Statement.Clauses["data_scope:applied"]; !applied || db.Error != nil || db.RowsAffected != 0 { return } if scope, ok := biz.DataScopeFromContext(db.Statement.Context); ok && !scope.All { - recordDataScopeEvent(db, "blocked_write", operation, "数据范围过滤后写操作影响 0 行(疑似越权尝试)", scope) + recordDataScopeEvent(db, enqueue, "blocked_write", operation, "数据范围过滤后写操作影响 0 行(疑似越权尝试)", scope) } } } diff --git a/internal/data/data_scope_audit.go b/internal/data/data_scope_audit.go new file mode 100644 index 0000000..c3c8bcc --- /dev/null +++ b/internal/data/data_scope_audit.go @@ -0,0 +1,121 @@ +package data + +import ( + "context" + "log/slog" + "sync" + "time" + + "gorm.io/gorm" +) + +const ( + dataScopeAuditQueueSize = 1024 + dataScopeAuditBatchSize = 100 + dataScopeAuditInterval = 2 * time.Second +) + +// dataScopeAuditWriter keeps row-scope auditing best-effort: callbacks only +// enqueue, while a background worker writes batches to the active database. +type dataScopeAuditWriter struct { + data *Data + logger *slog.Logger + queue chan dataAccessLogPO + batchSize int + interval time.Duration + stop chan struct{} + done chan struct{} + closeOnce sync.Once +} + +func newDataScopeAuditWriter(data *Data, logger *slog.Logger) *dataScopeAuditWriter { + return newDataScopeAuditWriterWithOptions(data, logger, dataScopeAuditQueueSize, dataScopeAuditBatchSize, dataScopeAuditInterval) +} + +func newDataScopeAuditWriterWithOptions(data *Data, logger *slog.Logger, queueSize, batchSize int, interval time.Duration) *dataScopeAuditWriter { + w := &dataScopeAuditWriter{ + data: data, + logger: logger, + queue: make(chan dataAccessLogPO, queueSize), + batchSize: batchSize, + interval: interval, + stop: make(chan struct{}), + done: make(chan struct{}), + } + go w.run() + return w +} + +func (d *Data) enqueueDataScopeAudit(record dataAccessLogPO) { + if d == nil || d.auditLog == nil { + return + } + d.auditLog.Enqueue(record) +} + +func (w *dataScopeAuditWriter) Enqueue(record dataAccessLogPO) { + if w == nil { + return + } + select { + case w.queue <- record: + default: + w.log().Warn("数据权限审计缓冲已满, 事件被丢弃", "mod", "data-scope", "event_type", record.EventType, "table", record.TargetTable) + } +} + +func (w *dataScopeAuditWriter) Close() { + if w == nil { + return + } + w.closeOnce.Do(func() { close(w.stop) }) + <-w.done +} + +func (w *dataScopeAuditWriter) run() { + defer close(w.done) + ticker := time.NewTicker(w.interval) + defer ticker.Stop() + batch := make([]dataAccessLogPO, 0, w.batchSize) + flush := func() { + if len(batch) == 0 || w.data == nil || w.data.gormDB == nil { + return + } + db := w.data.gormDB.DB() + if db == nil { + return + } + if err := db.WithContext(context.Background()).Session(&gorm.Session{NewDB: true, SkipHooks: true}).Create(&batch).Error; err != nil { + w.log().Warn("数据权限审计批量写入失败", "mod", "data-scope", "error", err) + } + batch = batch[:0] + } + for { + select { + case record := <-w.queue: + batch = append(batch, record) + if len(batch) >= w.batchSize { + flush() + } + case <-ticker.C: + flush() + case <-w.stop: + for { + select { + case record := <-w.queue: + batch = append(batch, record) + default: + flush() + return + } + } + } + } +} + +func (w *dataScopeAuditWriter) log() *slog.Logger { + if w != nil && w.logger != nil { + return w.logger + } + return slog.Default() +} diff --git a/internal/data/data_scope_audit_test.go b/internal/data/data_scope_audit_test.go new file mode 100644 index 0000000..84d08e4 --- /dev/null +++ b/internal/data/data_scope_audit_test.go @@ -0,0 +1,72 @@ +package data + +import ( + "io" + "log/slog" + "testing" + "time" + + "github.com/glebarez/sqlite" + "gorm.io/gorm" +) + +func openDataScopeAuditTestDB(t *testing.T, name string) *gorm.DB { + t.Helper() + db, err := gorm.Open(sqlite.Open("file:"+name+"?mode=memory&cache=shared"), &gorm.Config{}) + if err != nil { + t.Fatal(err) + } + if err = db.AutoMigrate(&dataAccessLogPO{}); err != nil { + t.Fatal(err) + } + return db +} + +func waitForDataScopeAuditCount(t *testing.T, db *gorm.DB, want int64) { + t.Helper() + deadline := time.Now().Add(time.Second) + for time.Now().Before(deadline) { + var count int64 + if err := db.Model(&dataAccessLogPO{}).Count(&count).Error; err != nil { + t.Fatal(err) + } + if count == want { + return + } + time.Sleep(5 * time.Millisecond) + } + t.Fatalf("data access log count did not reach %d", want) +} + +func TestDataScopeAuditWriterFlushesBatch(t *testing.T) { + db := openDataScopeAuditTestDB(t, "data-scope-audit-batch") + d := &Data{gormDB: newReloadableDB(db, nil)} + w := newDataScopeAuditWriterWithOptions(d, slog.New(slog.NewTextHandler(io.Discard, nil)), 8, 2, time.Hour) + d.auditLog = w + t.Cleanup(w.Close) + + d.enqueueDataScopeAudit(dataAccessLogPO{EventType: "no_identity", TargetTable: "example"}) + d.enqueueDataScopeAudit(dataAccessLogPO{EventType: "blocked_write", TargetTable: "example"}) + waitForDataScopeAuditCount(t, db, 2) +} + +func TestDataScopeAuditWriterUsesReloadedDatabase(t *testing.T) { + first := openDataScopeAuditTestDB(t, "data-scope-audit-first") + second := openDataScopeAuditTestDB(t, "data-scope-audit-second") + d := &Data{gormDB: newReloadableDB(first, nil)} + w := newDataScopeAuditWriterWithOptions(d, slog.New(slog.NewTextHandler(io.Discard, nil)), 8, 100, 10*time.Millisecond) + d.auditLog = w + t.Cleanup(w.Close) + + d.enqueueDataScopeAudit(dataAccessLogPO{EventType: "no_identity", TargetTable: "example"}) + d.gormDB.replace(second, nil) + waitForDataScopeAuditCount(t, second, 1) + + var firstCount int64 + if err := first.Model(&dataAccessLogPO{}).Count(&firstCount).Error; err != nil { + t.Fatal(err) + } + if firstCount != 0 { + t.Fatalf("audit log was written to retired database: %d", firstCount) + } +} diff --git a/internal/data/database.go b/internal/data/database.go index c363ea5..d8bc829 100644 --- a/internal/data/database.go +++ b/internal/data/database.go @@ -2,6 +2,7 @@ package data import ( "fmt" + "log/slog" "net" "net/url" "os" @@ -102,7 +103,7 @@ func databaseDSN(c *conf.Data_Database, name string) (string, error) { return "", fmt.Errorf("unsupported database driver %q", c.Driver) } -func gormConfig(config *conf.Data_Database) *gorm.Config { +func gormConfig(config *conf.Data_Database, appLogger ...*slog.Logger) *gorm.Config { level := logger.Info switch strings.ToLower(config.LogMode) { case "silent": @@ -112,15 +113,19 @@ func gormConfig(config *conf.Data_Database) *gorm.Config { case "warn": level = logger.Warn } - return &gorm.Config{Logger: logger.Default.LogMode(level), NamingStrategy: schema.NamingStrategy{TablePrefix: config.Prefix, SingularTable: config.Singular}} + var log *slog.Logger + if len(appLogger) > 0 { + log = appLogger[0] + } + return &gorm.Config{Logger: newGormLogger(log, level), NamingStrategy: schema.NamingStrategy{TablePrefix: config.Prefix, SingularTable: config.Singular}} } -func openWithDriver(driver, dsn string) (*gorm.DB, error) { - return openWithDriverConfig(driver, dsn, &conf.Data_Database{Driver: driver}) +func openWithDriver(driver, dsn string, appLogger ...*slog.Logger) (*gorm.DB, error) { + return openWithDriverConfig(driver, dsn, &conf.Data_Database{Driver: driver}, appLogger...) } -func openWithDriverConfig(driver, dsn string, config *conf.Data_Database) (*gorm.DB, error) { - gormConfig := gormConfig(config) +func openWithDriverConfig(driver, dsn string, config *conf.Data_Database, appLogger ...*slog.Logger) (*gorm.DB, error) { + gormConfig := gormConfig(config, appLogger...) var db *gorm.DB var err error switch normalizedDriver(driver) { @@ -159,7 +164,7 @@ func openWithDriverConfig(driver, dsn string, config *conf.Data_Database) (*gorm return db, nil } -func openDatabase(c *conf.Data_Database, create bool, template string) (*gorm.DB, error) { +func openDatabase(c *conf.Data_Database, create bool, template string, appLogger ...*slog.Logger) (*gorm.DB, error) { driver := normalizedDriver(c.Driver) if driver == "" { return nil, fmt.Errorf("unsupported database driver %q", c.Driver) @@ -172,7 +177,7 @@ func openDatabase(c *conf.Data_Database, create bool, template string) (*gorm.DB if err = os.MkdirAll(filepath.Dir(dsn), 0o755); err != nil { return nil, err } - return openWithDriverConfig(driver, dsn, c) + return openWithDriverConfig(driver, dsn, c, appLogger...) } if create && driver != "oracle" { if !databaseNamePattern.MatchString(c.Name) { @@ -189,7 +194,7 @@ func openDatabase(c *conf.Data_Database, create bool, template string) (*gorm.DB if err != nil { return nil, err } - adminDB, err := openWithDriverConfig(driver, dsn, c) + adminDB, err := openWithDriverConfig(driver, dsn, c, appLogger...) if err != nil { return nil, fmt.Errorf("connect database server: %w", err) } @@ -225,9 +230,9 @@ func openDatabase(c *conf.Data_Database, create bool, template string) (*gorm.DB if err != nil { return nil, err } - return openWithDriverConfig(driver, dsn, c) + return openWithDriverConfig(driver, dsn, c, appLogger...) } -func openFallbackDatabase() (*gorm.DB, error) { - return openWithDriver("sqlite", "file:kra-bootstrap?mode=memory&cache=shared") +func openFallbackDatabase(appLogger ...*slog.Logger) (*gorm.DB, error) { + return openWithDriver("sqlite", "file:kra-bootstrap?mode=memory&cache=shared", appLogger...) } diff --git a/internal/data/dictionary.go b/internal/data/dictionary.go index cf824b7..2bbee16 100644 --- a/internal/data/dictionary.go +++ b/internal/data/dictionary.go @@ -389,9 +389,11 @@ func (r *dictionaryRepo) ListDictionaryDetails(ctx context.Context, page, size i } func (r *dictionaryRepo) DictionaryDetailTree(ctx context.Context, dictionaryID uint, typ string) ([]*biz.DictionaryDetail, error) { if dictionaryID == 0 { - active := true - dictionary, err := r.FindDictionary(ctx, 0, typ, &active, false) - if err != nil { + // The tree-by-type endpoint resolves only by dictionary type. Unlike the + // public dictionary lookup, it does not require the dictionary itself to + // be enabled. + var dictionary dictionaryPO + if err := r.data.gormDB.WithContext(ctx).Where("type = ?", typ).First(&dictionary).Error; err != nil { return nil, err } dictionaryID = dictionary.ID @@ -409,7 +411,7 @@ func (r *dictionaryRepo) DictionaryDetailTree(ctx context.Context, dictionaryID for _, item := range items { if item.ParentID != nil && byID[*item.ParentID] != nil { byID[*item.ParentID].Children = append(byID[*item.ParentID].Children, item) - } else { + } else if item.ParentID == nil { roots = append(roots, item) } } diff --git a/internal/data/gorm_logger.go b/internal/data/gorm_logger.go new file mode 100644 index 0000000..6f92644 --- /dev/null +++ b/internal/data/gorm_logger.go @@ -0,0 +1,65 @@ +package data + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "time" + + "gorm.io/gorm/logger" +) + +// gormLogger forwards GORM diagnostics through the application logger so SQL +// entries participate in the same daily/category files and error sink as the +// rest of the service. Slow queries use the administration contract's 200ms +// threshold. +type gormLogger struct { + logger *slog.Logger + slowThreshold time.Duration + level logger.LogLevel +} + +func newGormLogger(log *slog.Logger, level logger.LogLevel) *gormLogger { + if log == nil { + log = slog.New(slog.NewTextHandler(io.Discard, nil)) + } + return &gormLogger{logger: log, slowThreshold: 200 * time.Millisecond, level: level} +} + +func (g *gormLogger) LogMode(level logger.LogLevel) logger.Interface { + next := *g + next.level = level + return &next +} + +func (g *gormLogger) Info(ctx context.Context, message string, args ...any) { + g.logger.InfoContext(ctx, fmt.Sprintf(message, args...), "mod", "sql", "gorm_logger", true) +} + +func (g *gormLogger) Warn(ctx context.Context, message string, args ...any) { + g.logger.WarnContext(ctx, fmt.Sprintf(message, args...), "mod", "sql", "gorm_logger", true) +} + +func (g *gormLogger) Error(ctx context.Context, message string, args ...any) { + g.logger.ErrorContext(ctx, fmt.Sprintf(message, args...), "mod", "sql", "gorm_logger", true) +} + +func (g *gormLogger) Trace(ctx context.Context, begin time.Time, query func() (string, int64), queryErr error) { + if g.level <= logger.Silent { + return + } + elapsed := time.Since(begin) + sql, rows := query() + fields := []any{"mod", "sql", "gorm_logger", true, "sql", sql, "rows", rows, "elapsed_ms", elapsed.Milliseconds()} + switch { + case queryErr != nil && g.level >= logger.Error && !errors.Is(queryErr, logger.ErrRecordNotFound): + fields = append(fields, "error", queryErr) + g.logger.ErrorContext(ctx, "SQL 执行错误", fields...) + case elapsed > g.slowThreshold && g.level >= logger.Warn: + g.logger.WarnContext(ctx, "SQL 慢查询", fields...) + case g.level >= logger.Info: + g.logger.InfoContext(ctx, "SQL", fields...) + } +} diff --git a/internal/data/gorm_logger_test.go b/internal/data/gorm_logger_test.go new file mode 100644 index 0000000..171684a --- /dev/null +++ b/internal/data/gorm_logger_test.go @@ -0,0 +1,31 @@ +package data + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + "time" + + "kra/pkg/logging" + + "gorm.io/gorm/logger" +) + +func TestGORMLoggerUsesApplicationCategories(t *testing.T) { + root := t.TempDir() + appLogger, cleanup := logging.NewZapLogger(root, "application.log", logging.Options{Level: "info", Format: "json"}) + databaseLogger := newGormLogger(appLogger, logger.Info) + databaseLogger.Trace(context.Background(), time.Now(), func() (string, int64) { return "SELECT 1", 1 }, nil) + databaseLogger.Trace(context.Background(), time.Now(), func() (string, int64) { return "SELECT missing", 0 }, errors.New("database failure")) + cleanup() + + date := time.Now().Format("2006-01-02") + for _, name := range []string{"application.log", filepath.Join("sql", "application.log"), filepath.Join("error", "error.log")} { + info, err := os.Stat(filepath.Join(root, date, name)) + if err != nil || info.Size() == 0 { + t.Fatalf("expected non-empty GORM log %s: info=%v err=%v", name, info, err) + } + } +} diff --git a/internal/data/log_file_test.go b/internal/data/log_file_test.go new file mode 100644 index 0000000..b17792e --- /dev/null +++ b/internal/data/log_file_test.go @@ -0,0 +1,50 @@ +package data + +import ( + "context" + "errors" + "os" + "path/filepath" + "testing" + + "kra/internal/biz" + "kra/internal/conf" +) + +func TestLogViewerReadsNestedCategoryFiles(t *testing.T) { + root := t.TempDir() + date := "2026-08-16" + path := filepath.Join(root, date, "http", "access.log") + if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(path, []byte("first\nsecond\n"), 0o644); err != nil { + t.Fatal(err) + } + + runtime := conf.NewRuntime(nil, &conf.AdminBackend{Zap: &conf.AdminBackend_Zap{Director: root}}) + repo := &logFileRepo{data: &Data{runtime: runtime}} + files, err := repo.LogFiles(context.Background(), date) + if err != nil { + t.Fatal(err) + } + if len(files) != 1 || files[0].Path != filepath.Join("http", "access.log") { + t.Fatalf("unexpected nested log files: %+v", files) + } + content, err := repo.LogContent(context.Background(), date, "http/access.log", nil) + if err != nil { + t.Fatal(err) + } + if content.Content != "first\nsecond\n" || content.LineCount != 2 || content.HasMore { + t.Fatalf("unexpected log content: %+v", content) + } +} + +func TestLogViewerRejectsPathTraversal(t *testing.T) { + runtime := conf.NewRuntime(nil, &conf.AdminBackend{Zap: &conf.AdminBackend_Zap{Director: t.TempDir()}}) + repo := &logFileRepo{data: &Data{runtime: runtime}} + _, err := repo.LogContent(context.Background(), "2026-08-16", "../application.log", nil) + if !errors.Is(err, biz.ErrInvalidLogPath) { + t.Fatalf("expected invalid path error, got %v", err) + } +} diff --git a/internal/data/migrations.go b/internal/data/migrations.go index 5625ad6..d0fbbd9 100644 --- a/internal/data/migrations.go +++ b/internal/data/migrations.go @@ -38,7 +38,7 @@ func migrateAll(db *gorm.DB) error { // Older builds used a status label outside the administration page's supported // state set, so normalize existing rows during migration. func normalizeErrorRecordStatuses(db *gorm.DB) error { - return db.Model(&errorRecordPO{}).Where("status = ?", "未解决").Update("status", "未处理").Error + return db.Session(&gorm.Session{NewDB: true}).Model(&errorRecordPO{}).Where("status = ?", "未解决").Update("status", "未处理").Error } // migrateLegacyAuthorityAPIsToCasbinRules upgrades the early Kra join-table @@ -46,7 +46,8 @@ func normalizeErrorRecordStatuses(db *gorm.DB) error { // table in place for backwards compatibility, but make casbin_rule the sole // live policy source. Existing policy rows are not duplicated. func migrateLegacyAuthorityAPIsToCasbinRules(db *gorm.DB) error { - if !db.Migrator().HasTable(&authorityAPIPO{}) || !db.Migrator().HasTable(&casbinRulePO{}) { + clean := db.Session(&gorm.Session{NewDB: true}) + if !clean.Migrator().HasTable(&authorityAPIPO{}) || !clean.Migrator().HasTable(&casbinRulePO{}) { return nil } type legacyPolicy struct { @@ -54,22 +55,28 @@ func migrateLegacyAuthorityAPIsToCasbinRules(db *gorm.DB) error { Path string Method string } - var rows []legacyPolicy - if err := db.Table("sys_authority_apis sa"). + query := clean.Table("sys_authority_apis sa"). Select("sa.authority_id, a.path, a.method"). - Joins("JOIN sys_apis a ON a.id = sa.api_id"). - Where("a.deleted_at IS NULL").Find(&rows).Error; err != nil { + Joins("JOIN sys_apis a ON a.id = sa.api_id") + // Early Kra schemas stored sys_apis without soft-delete timestamps. The + // legacy-policy migration must run before assuming that column exists; + // otherwise an upgrade from those schemas cannot start on MySQL. + if clean.Migrator().HasColumn(&apiPO{}, "deleted_at") { + query = query.Where("a.deleted_at IS NULL") + } + var rows []legacyPolicy + if err := query.Find(&rows).Error; err != nil { return err } for _, row := range rows { - exists, err := policyExists(db, row.AuthorityID, row.Path, row.Method) + exists, err := policyExists(clean, row.AuthorityID, row.Path, row.Method) if err != nil { return err } if exists { continue } - if err := db.Create(&casbinRulePO{Ptype: "p", V0: fmt.Sprint(row.AuthorityID), V1: row.Path, V2: row.Method}).Error; err != nil { + if err := clean.Create(&casbinRulePO{Ptype: "p", V0: fmt.Sprint(row.AuthorityID), V1: row.Path, V2: row.Method}).Error; err != nil { return err } } @@ -82,7 +89,8 @@ func migrateLegacyAuthorityAPIsToCasbinRules(db *gorm.DB) error { // composite primary key portably, so rebuild the small table once while // preserving every existing ignore rule. func migrateLegacyIgnoreAPITable(db *gorm.DB) error { - if !db.Migrator().HasTable(&ignoredAPIPO{}) || db.Migrator().HasColumn(&ignoredAPIPO{}, "id") { + clean := db.Session(&gorm.Session{NewDB: true}) + if !clean.Migrator().HasTable(&ignoredAPIPO{}) || clean.Migrator().HasColumn(&ignoredAPIPO{}, "id") { return nil } legacyTable := fmt.Sprintf("sys_ignore_apis_legacy_%d", time.Now().UnixNano()) @@ -90,7 +98,7 @@ func migrateLegacyIgnoreAPITable(db *gorm.DB) error { Path string Method string } - return db.Transaction(func(tx *gorm.DB) error { + return clean.Transaction(func(tx *gorm.DB) error { if err := tx.Migrator().RenameTable(ignoredAPIPO{}.TableName(), legacyTable); err != nil { return fmt.Errorf("rename legacy ignore API table: %w", err) } @@ -123,16 +131,17 @@ func migrateLegacyIgnoreAPITable(db *gorm.DB) error { // database has the root role but no stored API links, materialize the same // policy set and let normal authorization read it thereafter. func reconcileRootAuthorityAPIs(db *gorm.DB) error { + clean := db.Session(&gorm.Session{NewDB: true}) var authorityCount int64 - if err := db.Model(&authorityPO{}).Where("authority_id = ?", 888).Count(&authorityCount).Error; err != nil || authorityCount == 0 { + if err := clean.Session(&gorm.Session{NewDB: true}).Model(&authorityPO{}).Where("authority_id = ?", 888).Count(&authorityCount).Error; err != nil || authorityCount == 0 { return err } var policyCount int64 - if err := policyScope(db).Where("v0 = ?", "888").Count(&policyCount).Error; err != nil || policyCount != 0 { + if err := policyScope(clean).Where("v0 = ?", "888").Count(&policyCount).Error; err != nil || policyCount != 0 { return err } var ignored []ignoredAPIPO - if err := db.Find(&ignored).Error; err != nil { + if err := clean.Session(&gorm.Session{NewDB: true}).Find(&ignored).Error; err != nil { return err } ignoreSet := make(map[string]struct{}, len(ignored)) @@ -140,7 +149,7 @@ func reconcileRootAuthorityAPIs(db *gorm.DB) error { ignoreSet[item.Method+"\x00"+item.Path] = struct{}{} } var apis []apiPO - if err := db.Find(&apis).Error; err != nil { + if err := clean.Session(&gorm.Session{NewDB: true}).Find(&apis).Error; err != nil { return err } rules := make([]casbinRulePO, 0, len(apis)) @@ -153,13 +162,14 @@ func reconcileRootAuthorityAPIs(db *gorm.DB) error { if len(rules) == 0 { return nil } - return db.Create(&rules).Error + return clean.Session(&gorm.Session{NewDB: true}).Create(&rules).Error } // reconcileReferenceIndexes removes constraints created by older Kra builds // that are not part of the administration data model. Business services own // duplicate checks and their user-facing error messages. func reconcileReferenceIndexes(db *gorm.DB) error { + clean := db.Session(&gorm.Session{NewDB: true}) obsolete := []struct { model any name string @@ -171,8 +181,9 @@ func reconcileReferenceIndexes(db *gorm.DB) error { {&exportTemplatePO{}, "idx_sys_export_templates_template_id"}, } for _, item := range obsolete { - if db.Migrator().HasIndex(item.model, item.name) { - if err := db.Migrator().DropIndex(item.model, item.name); err != nil { + migrator := clean.Session(&gorm.Session{NewDB: true}).Migrator() + if migrator.HasIndex(item.model, item.name) { + if err := migrator.DropIndex(item.model, item.name); err != nil { return fmt.Errorf("drop obsolete index %s: %w", item.name, err) } } @@ -181,17 +192,18 @@ func reconcileReferenceIndexes(db *gorm.DB) error { name string field string }{{"idx_sys_users_uuid", "UUID"}, {"idx_sys_users_username", "Username"}} { - unique, err := indexIsUnique(db, &userPO{}, item.name) + unique, err := indexIsUnique(clean.Session(&gorm.Session{NewDB: true}), &userPO{}, item.name) if err != nil { return err } if !unique { continue } - if err = db.Migrator().DropIndex(&userPO{}, item.name); err != nil { + migrator := clean.Session(&gorm.Session{NewDB: true}).Migrator() + if err = migrator.DropIndex(&userPO{}, item.name); err != nil { return fmt.Errorf("drop legacy unique index %s: %w", item.name, err) } - if err = db.Migrator().CreateIndex(&userPO{}, item.field); err != nil { + if err = migrator.CreateIndex(&userPO{}, item.field); err != nil { return fmt.Errorf("create reference index %s: %w", item.name, err) } } diff --git a/internal/data/runtime_clients.go b/internal/data/runtime_clients.go index 7891df1..0eac0fc 100644 --- a/internal/data/runtime_clients.go +++ b/internal/data/runtime_clients.go @@ -49,9 +49,9 @@ func (r *reloadableMongo) close() { } } -func newReloadableDB(db *gorm.DB) *reloadableDB { +func newReloadableDB(db *gorm.DB, enqueue dataScopeAuditEnqueue) *reloadableDB { r := &reloadableDB{} - registerDataScopeCallbacks(db) + registerDataScopeCallbacks(db, enqueue) r.current.Store(db) return r } @@ -62,8 +62,8 @@ func (r *reloadableDB) WithContext(ctx context.Context) *gorm.DB { func (r *reloadableDB) DB() *gorm.DB { return r.current.Load() } -func (r *reloadableDB) replace(db *gorm.DB) { - registerDataScopeCallbacks(db) +func (r *reloadableDB) replace(db *gorm.DB, enqueue dataScopeAuditEnqueue) { + registerDataScopeCallbacks(db, enqueue) old := r.current.Swap(db) if old != nil && old != db { r.mu.Lock() diff --git a/internal/data/system_init.go b/internal/data/system_init.go index 5dcf90d..c29c6ea 100644 --- a/internal/data/system_init.go +++ b/internal/data/system_init.go @@ -95,7 +95,7 @@ func (r *initializationRepo) Initialize(ctx context.Context, input *biz.Database if initialized { return errors.New("数据库已初始化，无需重复初始化") } - candidate, err := openDatabase(config, true, input.Template) + candidate, err := openDatabase(config, true, input.Template, r.data.logger()) if err != nil { return err } @@ -251,6 +251,7 @@ func (r *initializationRepo) Initialize(ctx context.Context, input *biz.Database func defaultIgnoredAPIs() []ignoredAPIPO { return []ignoredAPIPO{ {Method: "GET", Path: "/api/freshCasbin"}, {Method: "GET", Path: "/health"}, + {Method: "GET", Path: "/swagger/*any"}, {Method: "POST", Path: "/system/reloadSystem"}, {Method: "POST", Path: "/base/login"}, {Method: "POST", Path: "/base/captcha"}, {Method: "POST", Path: "/init/initdb"}, {Method: "POST", Path: "/init/checkdb"}, {Method: "GET", Path: "/info/getInfoDataSource"}, diff --git a/internal/data/system_init_ignore_test.go b/internal/data/system_init_ignore_test.go new file mode 100644 index 0000000..44fc192 --- /dev/null +++ b/internal/data/system_init_ignore_test.go @@ -0,0 +1,12 @@ +package data + +import "testing" + +func TestDefaultIgnoredAPIsIncludeSwagger(t *testing.T) { + for _, api := range defaultIgnoredAPIs() { + if api.Method == "GET" && api.Path == "/swagger/*any" { + return + } + } + t.Fatal("default ignored APIs do not include the Swagger handler") +} diff --git a/internal/data/transactions_test.go b/internal/data/transactions_test.go index 72271fd..7f0ea0a 100644 --- a/internal/data/transactions_test.go +++ b/internal/data/transactions_test.go @@ -19,7 +19,7 @@ func newTransactionTestData(t *testing.T) *Data { if err = migrateAll(db); err != nil { t.Fatal(err) } - data := &Data{gormDB: newReloadableDB(db), redis: newReloadableRedis(nil), runtime: conf.NewRuntime(&conf.Data{Database: &conf.Data_Database{Driver: "sqlite"}}, &conf.AdminBackend{})} + data := &Data{gormDB: newReloadableDB(db, nil), redis: newReloadableRedis(nil), runtime: conf.NewRuntime(&conf.Data{Database: &conf.Data_Database{Driver: "sqlite"}}, &conf.AdminBackend{})} t.Cleanup(func() { data.gormDB.close() }) return data } diff --git a/internal/server/gin.go b/internal/server/gin.go index 40a0c22..8bd9026 100644 --- a/internal/server/gin.go +++ b/internal/server/gin.go @@ -24,7 +24,7 @@ import ( func NewGinEngine(runtime *conf.Runtime, access *service.AccessControlService, handlers *handler.Set, auth *service.AuthService, security *service.SecurityService, audit *service.AuditRecorder, logger *slog.Logger, version string) *gin.Engine { gin.SetMode(gin.ReleaseMode) engine := gin.New() - engine.Use(servermiddleware.RequestMeta(), servermiddleware.Recovery(audit, logger), servermiddleware.AccessLog(runtime, logger, version), servermiddleware.CORS(runtime), servermiddleware.ErrorAudit(audit), servermiddleware.SecurityRateLimit(security)) + engine.Use(servermiddleware.RequestMeta(), servermiddleware.Recovery(logger), servermiddleware.AccessLog(runtime, logger, version), servermiddleware.CORS(runtime), servermiddleware.ErrorAudit(logger), servermiddleware.SecurityRateLimit(security)) prefix := "" config := runtime.Admin() @@ -60,6 +60,7 @@ func NewGinEngine(runtime *conf.Runtime, access *service.AccessControlService, h serverrouter.RegisterMedia(private, handlers.Media) serverrouter.RegisterAnnouncement(private, public, handlers.Announcement) serverrouter.RegisterEmail(private, handlers.Email) + registerSwagger(engine, prefix, version, logger) engine.NoRoute(func(c *gin.Context) { if serveLocalStorage(c, runtime) { diff --git a/internal/server/gin_test.go b/internal/server/gin_test.go index b8d0d40..6502946 100644 --- a/internal/server/gin_test.go +++ b/internal/server/gin_test.go @@ -1,14 +1,22 @@ package server import ( + "bytes" + "log/slog" + "net/http" + "net/http/httptest" + "os" + "path/filepath" + "sort" + "strings" "testing" "kra/internal/conf" "kra/internal/server/handler" ) -func TestGinRouteContract(t *testing.T) { - handlers := &handler.Set{ +func emptyHandlers() *handler.Set { + return &handler.Set{ Authority: &handler.Authority{}, Menu: &handler.Menu{}, API: &handler.API{}, Permission: &handler.Permission{}, Organization: &handler.Organization{}, Announcement: &handler.Announcement{}, Email: &handler.Email{}, Task: &handler.Task{}, @@ -17,11 +25,12 @@ func TestGinRouteContract(t *testing.T) { APIToken: &handler.APIToken{}, SystemConfig: &handler.SystemConfig{}, Public: &handler.Public{}, User: &handler.User{}, Navigation: &handler.Navigation{}, Session: &handler.Session{}, } - engine := NewGinEngine(conf.NewRuntime(nil, &conf.AdminBackend{}), nil, handlers, nil, nil, nil, nil, "test") +} + +func TestGinRouteContract(t *testing.T) { + engine := NewGinEngine(conf.NewRuntime(nil, &conf.AdminBackend{}), nil, emptyHandlers(), nil, nil, nil, nil, "test") routes := engine.Routes() - if len(routes) != 177 { - t.Fatalf("route contract changed: got %d routes, want 177", len(routes)) - } + actual := make([]string, 0, len(routes)) seen := make(map[string]struct{}, len(routes)) for _, route := range routes { key := route.Method + " " + route.Path @@ -29,5 +38,263 @@ func TestGinRouteContract(t *testing.T) { t.Fatalf("duplicate route %s", key) } seen[key] = struct{}{} + actual = append(actual, key) + } + sort.Strings(actual) + if value := strings.Join(actual, "\n"); value != expectedGinRouteContract { + t.Fatalf("route contract changed:\n%s", value) } } + +func TestGinStartupLogsEveryRegisteredRoute(t *testing.T) { + var output bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&output, nil)) + engine := NewGinEngine(conf.NewRuntime(nil, &conf.AdminBackend{}), nil, emptyHandlers(), nil, nil, nil, logger, "test") + text := output.String() + if got, want := strings.Count(text, `"msg":"router registered"`), len(engine.Routes()); got != want { + t.Fatalf("registered route log count = %d, want %d", got, want) + } + if !strings.Contains(text, `"msg":"router register success"`) || !strings.Contains(text, `"route_count":178`) { + t.Fatalf("startup route summary is missing: %s", text) + } +} + +func TestSwaggerUsesRegisteredGinRoutes(t *testing.T) { + engine := NewGinEngine(conf.NewRuntime(nil, &conf.AdminBackend{}), nil, emptyHandlers(), nil, nil, nil, nil, "v1.0.0") + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, "/swagger/doc.json", nil) + engine.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("swagger document status = %d, body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + for _, expected := range []string{`"swagger":"2.0"`, `"version":"v1.0.0"`, `"/base/login"`, `"/timedTask/triggerTimedTask"`, `"/mediaUpload/{uploadId}"`} { + if !strings.Contains(body, expected) { + t.Fatalf("swagger document missing %s", expected) + } + } +} + +func TestSwaggerSupportsRouterPrefix(t *testing.T) { + engine := NewGinEngine(conf.NewRuntime(nil, &conf.AdminBackend{RouterPrefix: "/admin"}), nil, emptyHandlers(), nil, nil, nil, nil, "v1.0.0") + for _, path := range []string{"/admin/swagger/index.html", "/admin/swagger/doc.json"} { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, path, nil) + engine.ServeHTTP(response, request) + if response.Code != http.StatusOK { + t.Fatalf("swagger path %s status = %d, body=%s", path, response.Code, response.Body.String()) + } + if strings.HasSuffix(path, "/doc.json") && !strings.Contains(response.Body.String(), `"basePath":"/admin"`) { + t.Fatalf("swagger document does not use router prefix: %s", response.Body.String()) + } + } +} + +func TestLocalStorageResponseHeaders(t *testing.T) { + root := t.TempDir() + for name, body := range map[string]string{"script.html": "", "image.png": "png"} { + if err := os.WriteFile(filepath.Join(root, name), []byte(body), 0o600); err != nil { + t.Fatal(err) + } + } + runtime := conf.NewRuntime(nil, &conf.AdminBackend{Local: &conf.AdminBackend_Local{StorePath: root, PathPrefix: "uploads/file"}, Storage: &conf.AdminBackend_Storage{Type: "local"}}) + engine := NewGinEngine(runtime, nil, emptyHandlers(), nil, nil, nil, nil, "test") + + for _, test := range []struct { + path string + attachment bool + }{ + {path: "/uploads/file/script.html", attachment: true}, + {path: "/uploads/file/image.png", attachment: false}, + } { + response := httptest.NewRecorder() + request := httptest.NewRequest(http.MethodGet, test.path, nil) + engine.ServeHTTP(response, request) + if response.Code != http.StatusOK || response.Header().Get("X-Content-Type-Options") != "nosniff" { + t.Fatalf("unexpected static response for %s: status=%d headers=%v", test.path, response.Code, response.Header()) + } + hasAttachment := strings.Contains(response.Header().Get("Content-Disposition"), "attachment") + if hasAttachment != test.attachment { + t.Fatalf("attachment header for %s = %v, want %v", test.path, hasAttachment, test.attachment) + } + } +} + +const expectedGinRouteContract = `DELETE /api/deleteApisByIds +DELETE /dataAccessLog/deleteDataAccessLogByIds +DELETE /department/deleteDepartment +DELETE /info/deleteInfo +DELETE /info/deleteInfoByIds +DELETE /mediaUpload/:uploadId +DELETE /position/deletePosition +DELETE /sysDictionary/deleteSysDictionary +DELETE /sysDictionaryDetail/deleteSysDictionaryDetail +DELETE /sysError/deleteSysError +DELETE /sysError/deleteSysErrorByIds +DELETE /sysExportTemplate/deleteSysExportTemplate +DELETE /sysExportTemplate/deleteSysExportTemplateByIds +DELETE /sysLoginLog/deleteLoginLog +DELETE /sysLoginLog/deleteLoginLogByIds +DELETE /sysOperationRecord/deleteSysOperationRecord +DELETE /sysOperationRecord/deleteSysOperationRecordByIds +DELETE /sysParams/deleteSysParams +DELETE /sysParams/deleteSysParamsByIds +DELETE /sysVersion/deleteSysVersion +DELETE /sysVersion/deleteSysVersionByIds +DELETE /timedTask/deleteTimedTask +DELETE /user/deleteUser +GET /api/freshCasbin +GET /api/getApiGroups +GET /api/getApiRoles +GET /api/syncApi +GET /attachmentCategory/getCategoryList +GET /authority/getDataScopeDepts +GET /authority/getUsersByAuthority +GET /department/findDepartment +GET /department/getDepartmentUsers +GET /fileUploadAndDownload/findFile +GET /health +GET /info/findInfo +GET /info/getInfoDataSource +GET /info/getInfoList +GET /info/getInfoPublic +GET /logViewer/content +GET /logViewer/dates +GET /logViewer/files +GET /menu/getMenuRoles +GET /position/findPosition +GET /position/getPositionUsers +GET /securityConfig/getSecurityConfig +GET /swagger/*any +GET /sysDictionary/exportSysDictionary +GET /sysDictionary/findSysDictionary +GET /sysDictionary/getSysDictionaryList +GET /sysDictionary/getSysDictionaryListWithDetails +GET /sysDictionaryDetail/findSysDictionaryDetail +GET /sysDictionaryDetail/getDictionaryDetailsByParent +GET /sysDictionaryDetail/getDictionaryPath +GET /sysDictionaryDetail/getDictionaryTreeList +GET /sysDictionaryDetail/getDictionaryTreeListByType +GET /sysDictionaryDetail/getSysDictionaryDetailList +GET /sysError/findSysError +GET /sysError/getSysErrorList +GET /sysExportTemplate/exportExcel +GET /sysExportTemplate/exportExcelByToken +GET /sysExportTemplate/exportTemplate +GET /sysExportTemplate/exportTemplateByToken +GET /sysExportTemplate/findSysExportTemplate +GET /sysExportTemplate/getSysExportTemplateList +GET /sysExportTemplate/previewSQL +GET /sysLoginLog/findLoginLog +GET /sysLoginLog/getLoginLogList +GET /sysOperationRecord/findSysOperationRecord +GET /sysOperationRecord/getSysOperationRecordList +GET /sysParams/findSysParams +GET /sysParams/getSysParam +GET /sysParams/getSysParamsList +GET /sysVersion/downloadVersionJson +GET /sysVersion/findSysVersion +GET /sysVersion/getSysVersionList +GET /timedTask/alertStream +GET /timedTask/getRegisteredMethods +GET /timedTask/getTimedTaskList +GET /timedTask/getTimedTaskLogList +GET /user/getUserInfo +POST /api/createApi +POST /api/deleteApi +POST /api/enterSyncApi +POST /api/getAllApis +POST /api/getApiById +POST /api/getApiList +POST /api/ignoreApi +POST /api/setApiRoles +POST /api/updateApi +POST /attachmentCategory/addCategory +POST /attachmentCategory/deleteCategory +POST /authority/copyAuthority +POST /authority/createAuthority +POST /authority/deleteAuthority +POST /authority/getAuthorityList +POST /authority/setDataScope +POST /authority/setRoleUsers +POST /authorityBtn/canRemoveAuthorityBtn +POST /authorityBtn/getAuthorityBtn +POST /authorityBtn/setAuthorityBtn +POST /base/captcha +POST /base/login +POST /casbin/getPolicyPathByAuthorityId +POST /casbin/updateCasbin +POST /dataAccessLog/getDataAccessLogList +POST /department/createDepartment +POST /department/getDepartmentList +POST /department/setDepartmentUsers +POST /email/emailTest +POST /email/sendEmail +POST /fileUploadAndDownload/deleteFile +POST /fileUploadAndDownload/deleteFiles +POST /fileUploadAndDownload/editFileName +POST /fileUploadAndDownload/getFileList +POST /fileUploadAndDownload/importURL +POST /fileUploadAndDownload/listOssFiles +POST /fileUploadAndDownload/upload +POST /info/createInfo +POST /init/checkdb +POST /init/initdb +POST /jwt/jsonInBlacklist +POST /mediaUpload/chunk +POST /mediaUpload/complete +POST /mediaUpload/init +POST /menu/addBaseMenu +POST /menu/addMenuAuthority +POST /menu/deleteBaseMenu +POST /menu/getBaseMenuById +POST /menu/getBaseMenuTree +POST /menu/getMenu +POST /menu/getMenuAuthority +POST /menu/getMenuList +POST /menu/setMenuRoles +POST /menu/updateBaseMenu +POST /position/createPosition +POST /position/getPositionList +POST /position/setPositionUsers +POST /securityConfig/setSecurityConfig +POST /sysApiToken/createApiToken +POST /sysApiToken/deleteApiToken +POST /sysApiToken/getApiTokenList +POST /sysDictionary/createSysDictionary +POST /sysDictionary/importSysDictionary +POST /sysDictionaryDetail/createSysDictionaryDetail +POST /sysError/createSysError +POST /sysExportTemplate/createSysExportTemplate +POST /sysExportTemplate/importExcel +POST /sysParams/createSysParams +POST /sysVersion/exportVersion +POST /sysVersion/importVersion +POST /system/getServerInfo +POST /system/getSystemConfig +POST /system/reloadSystem +POST /system/setSystemConfig +POST /timedTask/createTimedTask +POST /timedTask/toggleTimedTask +POST /timedTask/triggerTimedTask +POST /user/admin_register +POST /user/changePassword +POST /user/getUserList +POST /user/resetPassword +POST /user/setUserAuthorities +POST /user/setUserAuthority +POST /user/setUserDepartments +POST /user/setUserPositions +PUT /authority/updateAuthority +PUT /department/updateDepartment +PUT /info/updateInfo +PUT /position/updatePosition +PUT /sysDictionary/updateSysDictionary +PUT /sysDictionaryDetail/updateSysDictionaryDetail +PUT /sysError/updateSysError +PUT /sysExportTemplate/updateSysExportTemplate +PUT /sysParams/updateSysParams +PUT /timedTask/updateTimedTask +PUT /user/setSelfInfo +PUT /user/setSelfSetting +PUT /user/setUserInfo` diff --git a/internal/server/handler/menu.go b/internal/server/handler/menu.go index e4e44bc..f21f255 100644 --- a/internal/server/handler/menu.go +++ b/internal/server/handler/menu.go @@ -160,7 +160,7 @@ func (h *Menu) AuthorityMenus(c *gin.Context) { } menus, err := h.service.AuthorityMenus(c.Request.Context(), req.AuthorityID) if err != nil { - httpx.Fail(c, "获取失败") + httpx.Write(c, httpx.CodeError, gin.H{"menus": menus}, "获取失败") return } httpx.Write(c, httpx.CodeSuccess, gin.H{"menus": menus}, "获取成功") diff --git a/internal/server/handler/user.go b/internal/server/handler/user.go index e6a3335..d5d4a28 100644 --- a/internal/server/handler/user.go +++ b/internal/server/handler/user.go @@ -67,7 +67,11 @@ func (h *User) Create(c *gin.Context) { } result, err := h.service.CreateUserRequest(c.Request.Context(), &req) if err != nil { - httpx.Fail(c, "注册失败") + if service.IsPasswordPolicyError(err) { + httpx.Fail(c, err.Error()) + return + } + httpx.Write(c, httpx.CodeError, gin.H{"user": dto.UserResponse{}}, "注册失败") return } httpx.Write(c, httpx.CodeSuccess, gin.H{"user": result}, "注册成功") @@ -138,6 +142,10 @@ func (h *User) ResetPassword(c *gin.Context) { return } if err := h.service.ResetPassword(c.Request.Context(), req.ID, req.Password); err != nil { + if service.IsPasswordPolicyError(err) { + httpx.Fail(c, err.Error()) + return + } httpx.Fail(c, "重置失败"+err.Error()) return } @@ -164,6 +172,10 @@ func (h *User) ChangePassword(c *gin.Context) { return } if err := h.service.ChangePassword(c.Request.Context(), claims.ID, req.Password, req.NewPassword); err != nil { + if service.IsPasswordPolicyError(err) { + httpx.Fail(c, err.Error()) + return + } httpx.Fail(c, "修改失败，原密码与当前账户不符") return } diff --git a/internal/server/middleware/error_audit.go b/internal/server/middleware/error_audit.go index b51080b..62ef39f 100644 --- a/internal/server/middleware/error_audit.go +++ b/internal/server/middleware/error_audit.go @@ -3,22 +3,25 @@ package middleware import ( "bytes" "encoding/json" + "log/slog" "strings" "kra/internal/server/httpx" - "kra/internal/service" - "kra/internal/service/dto" "github.com/gin-gonic/gin" ) -// ErrorAudit supplies the database sink that the Error-level logging core -// provides. Expected authentication, permission and input failures are not -// system errors and therefore are not inserted into sys_error. -func ErrorAudit(audit *service.AuditRecorder) gin.HandlerFunc { +// ErrorAudit emits a structured Error-level log for unexpected HTTP failures. +// The logging core is the single persistence path for sys_error, matching the +// reference behavior and avoiding duplicate rows for HTTP failures. +func ErrorAudit(logger *slog.Logger) gin.HandlerFunc { return func(c *gin.Context) { c.Next() - if strings.Contains(c.Request.URL.Path, "/sysError/") || c.Writer.Status() >= 300 && c.Writer.Status() < 500 { + // sysError writes must never audit themselves. Log-viewer failures are + // already recorded by the handler with the underlying filesystem error; + // emitting again from the response envelope would duplicate both the + // classified error file and the sys_error row. + if strings.Contains(c.Request.URL.Path, "/sysError/") || strings.Contains(c.Request.URL.Path, "/logViewer/") || c.Writer.Status() >= 300 && c.Writer.Status() < 500 { return } var response httpx.Response @@ -32,10 +35,27 @@ func ErrorAudit(audit *service.AuditRecorder) gin.HandlerFunc { return } requestID, _ := c.Get("request_id") - _ = audit.CreateErrorRequest(c.Request.Context(), &dto.ErrorRecordRequest{Form: c.Request.URL.Path, Info: response.Msg, Level: "error", RequestID: stringValue(requestID), TraceID: stringValueFromContext(c, "trace_id")}) + if logger != nil { + logger.ErrorContext(c.Request.Context(), "请求处理失败", "mod", failureLogModule(c.Request.URL.Path), "path", c.Request.URL.Path, "method", c.Request.Method, "status", c.Writer.Status(), "error", response.Msg, "request_id", stringValue(requestID), "trace_id", stringValueFromContext(c, "trace_id")) + } } } +func failureLogModule(path string) string { + for _, marker := range []string{"/fileUploadAndDownload/", "/mediaUpload/", "/attachmentCategory/"} { + if strings.Contains(path, marker) { + return "upload" + } + } + if strings.Contains(path, "/timedTask/") { + return "timedTask" + } + if strings.Contains(path, "/logViewer/") { + return "log-viewer" + } + return "biz" +} + func expectedClientFailure(message string) bool { for _, value := range []string{"参数错误", "请输入用户名和密码", "验证码错误", "用户名不存在或者密码错误", "用户被禁止登录", "账号已锁定", "权限不足", "密码已过期", "未登录", "token", "令牌失效"} { if strings.Contains(message, value) { diff --git a/internal/server/middleware/error_audit_test.go b/internal/server/middleware/error_audit_test.go new file mode 100644 index 0000000..c884501 --- /dev/null +++ b/internal/server/middleware/error_audit_test.go @@ -0,0 +1,75 @@ +package middleware + +import ( + "bytes" + "log/slog" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/gin-gonic/gin" +) + +func runErrorAudit(t *testing.T, path, response string, logger *slog.Logger) { + t.Helper() + gin.SetMode(gin.TestMode) + engine := gin.New() + engine.Use(ErrorAudit(logger)) + engine.GET(path, func(c *gin.Context) { + c.Set(ctxRespBufferKey, bytes.NewBufferString(response)) + c.Set("request_id", "request-1") + c.Set("trace_id", "trace-1") + c.Status(http.StatusOK) + }) + request := httptest.NewRequest(http.MethodGet, path, nil) + responseRecorder := httptest.NewRecorder() + engine.ServeHTTP(responseRecorder, request) +} + +func TestErrorAuditEmitsUnexpectedBusinessFailure(t *testing.T) { + var output bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&output, nil)) + + runErrorAudit(t, "/test", `{"code":7,"data":{},"msg":"数据库写入失败"}`, logger) + + if text := output.String(); !strings.Contains(text, `"mod":"biz"`) || !strings.Contains(text, `"error":"数据库写入失败"`) { + t.Fatalf("unexpected error log: %s", text) + } +} + +func TestErrorAuditUsesFeatureModule(t *testing.T) { + var output bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&output, nil)) + + runErrorAudit(t, "/timedTask/triggerTimedTask", `{"code":7,"data":{},"msg":"任务执行失败"}`, logger) + if text := output.String(); !strings.Contains(text, `"mod":"timedTask"`) { + t.Fatalf("unexpected feature log module: %s", text) + } +} + +func TestErrorAuditSkipsExpectedClientFailure(t *testing.T) { + var output bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&output, nil)) + + runErrorAudit(t, "/test", `{"code":7,"data":{},"msg":"参数错误"}`, logger) + + if output.Len() != 0 { + t.Fatalf("expected no error log, got %s", output.String()) + } +} + +func TestErrorAuditAllowsNilLogger(t *testing.T) { + runErrorAudit(t, "/test", `{"code":7,"data":{},"msg":"数据库写入失败"}`, nil) +} + +func TestErrorAuditSkipsLogViewerFailureAlreadyLoggedByHandler(t *testing.T) { + var output bytes.Buffer + logger := slog.New(slog.NewJSONHandler(&output, nil)) + + runErrorAudit(t, "/logViewer/content", `{"code":7,"data":{},"msg":"日志文件不可读取"}`, logger) + + if output.Len() != 0 { + t.Fatalf("log viewer failure must not be emitted twice, got %s", output.String()) + } +} diff --git a/internal/server/middleware/rate_limit.go b/internal/server/middleware/rate_limit.go index 3f14a96..0eea341 100644 --- a/internal/server/middleware/rate_limit.go +++ b/internal/server/middleware/rate_limit.go @@ -29,7 +29,7 @@ func SecurityRateLimit(settings *service.SecurityService) gin.HandlerFunc { key := "KRA_SecLimit" + c.ClientIP() + c.FullPath() count, cacheErr := settings.IncrementRateLimit(c.Request.Context(), key, time.Duration(window)*time.Second) if cacheErr == nil && int(count) > config.LimitCount { - httpx.Fail(c, "请求太过频繁，请稍后再试") + c.JSON(200, gin.H{"code": httpx.CodeError, "msg": "请求太过频繁，请稍后再试"}) c.Abort() return } diff --git a/internal/server/middleware/rate_limit_test.go b/internal/server/middleware/rate_limit_test.go new file mode 100644 index 0000000..ea6789c --- /dev/null +++ b/internal/server/middleware/rate_limit_test.go @@ -0,0 +1,49 @@ +package middleware + +import ( + "context" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" + + "kra/internal/biz" + "kra/internal/service" + + "github.com/gin-gonic/gin" +) + +type rateLimitSecurityRepo struct{} + +func (rateLimitSecurityRepo) SecurityConfig(context.Context) (*biz.SecurityConfig, error) { + return &biz.SecurityConfig{LimitEnable: true, LimitWindow: 60, LimitCount: 1}, nil +} +func (rateLimitSecurityRepo) SaveSecurityConfig(context.Context, *biz.SecurityConfig) error { + return nil +} + +type rateLimitCache struct{} + +func (rateLimitCache) Get(context.Context, string) (string, bool, error) { return "", false, nil } +func (rateLimitCache) Set(context.Context, string, string, time.Duration) error { return nil } +func (rateLimitCache) Delete(context.Context, string) error { return nil } +func (rateLimitCache) Increment(context.Context, string, time.Duration) (int64, error) { return 2, nil } + +func TestSecurityRateLimitMatchesResponseContract(t *testing.T) { + gin.SetMode(gin.TestMode) + settings := service.NewSecurityService(biz.NewSecurityUsecase(rateLimitSecurityRepo{}, rateLimitCache{}, nil, nil)) + engine := gin.New() + engine.Use(SecurityRateLimit(settings)) + engine.POST("/base/login", func(c *gin.Context) { c.Status(http.StatusNoContent) }) + + response := httptest.NewRecorder() + engine.ServeHTTP(response, httptest.NewRequest(http.MethodPost, "/base/login", nil)) + if response.Code != http.StatusOK { + t.Fatalf("status = %d, body=%s", response.Code, response.Body.String()) + } + body := response.Body.String() + if !strings.Contains(body, `"code":7`) || !strings.Contains(body, `"msg":"请求太过频繁，请稍后再试"`) || strings.Contains(body, `"data"`) { + t.Fatalf("unexpected rate-limit response: %s", body) + } +} diff --git a/internal/server/middleware/recovery.go b/internal/server/middleware/recovery.go index fa5d953..2373962 100644 --- a/internal/server/middleware/recovery.go +++ b/internal/server/middleware/recovery.go @@ -1,7 +1,6 @@ package middleware import ( - "fmt" "log/slog" "net" "net/http" @@ -10,13 +9,10 @@ import ( "runtime/debug" "strings" - "kra/internal/service" - "kra/internal/service/dto" - "github.com/gin-gonic/gin" ) -func Recovery(audit *service.AuditRecorder, logger *slog.Logger) gin.HandlerFunc { +func Recovery(logger *slog.Logger) gin.HandlerFunc { return func(c *gin.Context) { defer func() { panicValue := recover() @@ -31,12 +27,9 @@ func Recovery(audit *service.AuditRecorder, logger *slog.Logger) gin.HandlerFunc } } request, _ := httputil.DumpRequest(c.Request, false) - info := fmt.Sprintf("error=%v request=%s stack=%s", panicValue, request, debug.Stack()) if logger != nil { - logger.ErrorContext(c.Request.Context(), "recovery from panic", "mod", "error", "error", panicValue, "request", string(request), "stack", string(debug.Stack())) + logger.ErrorContext(c.Request.Context(), "recovery from panic", "mod", "http", "error", panicValue, "request", string(request), "stack", string(debug.Stack())) } - requestID, _ := c.Get("request_id") - _ = audit.CreateErrorRequest(c.Request.Context(), &dto.ErrorRecordRequest{Form: c.Request.URL.Path, Info: info, Level: "error", RequestID: stringValue(requestID), TraceID: stringValueFromContext(c, "trace_id")}) if brokenPipe { if err, ok := panicValue.(error); ok { _ = c.Error(err) diff --git a/internal/server/swagger.go b/internal/server/swagger.go new file mode 100644 index 0000000..2c4b6c6 --- /dev/null +++ b/internal/server/swagger.go @@ -0,0 +1,167 @@ +package server + +import ( + "encoding/json" + "log/slog" + "regexp" + "sort" + "strings" + "sync" + + "kra/internal/service" + + "github.com/gin-gonic/gin" + swaggerFiles "github.com/swaggo/files" + ginSwagger "github.com/swaggo/gin-swagger" + "github.com/swaggo/swag" +) + +const swaggerInstanceName = "kra-admin" + +var ( + swaggerPathParameter = regexp.MustCompile(`:([A-Za-z0-9_]+)`) + swaggerRegistration sync.Once + swaggerDocument runtimeSwaggerDocument +) + +type runtimeSwaggerDocument struct { + mu sync.RWMutex + doc string +} + +func (d *runtimeSwaggerDocument) ReadDoc() string { + d.mu.RLock() + defer d.mu.RUnlock() + return d.doc +} + +func (d *runtimeSwaggerDocument) replace(doc string) { + d.mu.Lock() + d.doc = doc + d.mu.Unlock() +} + +func registerSwagger(engine *gin.Engine, prefix, version string, logger *slog.Logger) { + swaggerRegistration.Do(func() { swag.Register(swaggerInstanceName, &swaggerDocument) }) + swaggerDocument.replace(buildSwaggerDocument(engine.Routes(), prefix, version)) + path := strings.TrimSuffix(prefix, "/") + "/swagger/*any" + engine.GET(path, ginSwagger.WrapHandler( + swaggerFiles.Handler, + ginSwagger.InstanceName(swaggerInstanceName), + ginSwagger.URL("doc.json"), + ginSwagger.PersistAuthorization(true), + )) + if logger != nil { + logger.Info("register swagger handler", "mod", "system", "path", path) + } +} + +func buildSwaggerDocument(routes []gin.RouteInfo, prefix, version string) string { + basePath := strings.TrimSuffix(prefix, "/") + if basePath == "" { + basePath = "/" + } + sort.Slice(routes, func(i, j int) bool { + if routes[i].Path == routes[j].Path { + return routes[i].Method < routes[j].Method + } + return routes[i].Path < routes[j].Path + }) + paths := make(map[string]map[string]any, len(routes)) + for _, route := range routes { + method := strings.ToLower(route.Method) + switch method { + case "get", "post", "put", "delete", "patch": + default: + continue + } + apiPath := route.Path + if prefix != "" { + apiPath = strings.TrimPrefix(apiPath, strings.TrimSuffix(prefix, "/")) + } + if apiPath == "" { + apiPath = "/" + } + documentPath := swaggerPathParameter.ReplaceAllString(apiPath, `{$1}`) + group, description := service.RouteMetadata(route.Method, apiPath) + if description == "" { + description = route.Method + " " + apiPath + } + operation := map[string]any{ + "tags": []string{group}, + "summary": description, + "operationId": swaggerOperationID(route.Method, apiPath), + "produces": []string{"application/json"}, + "responses": map[string]any{ + "200": map[string]any{"description": "OK", "schema": map[string]any{"$ref": "#/definitions/Response"}}, + }, + } + if parameters := swaggerPathParameters(apiPath); len(parameters) > 0 { + operation["parameters"] = parameters + } + if !swaggerPublicPath(apiPath) { + operation["security"] = []map[string][]string{{"ApiKeyAuth": {}}} + } + if paths[documentPath] == nil { + paths[documentPath] = map[string]any{} + } + paths[documentPath][method] = operation + } + document := map[string]any{ + "swagger": "2.0", + "info": map[string]any{"title": "Kra Administration API", "version": version}, + "basePath": basePath, + "schemes": []string{"http", "https"}, + "consumes": []string{"application/json"}, + "produces": []string{"application/json"}, + "securityDefinitions": map[string]any{ + "ApiKeyAuth": map[string]any{"type": "apiKey", "name": "x-token", "in": "header"}, + }, + "paths": paths, + "definitions": map[string]any{ + "Response": map[string]any{ + "type": "object", + "properties": map[string]any{ + "code": map[string]any{"type": "integer"}, + "data": map[string]any{"type": "object"}, + "msg": map[string]any{"type": "string"}, + }, + }, + }, + } + raw, _ := json.Marshal(document) + return string(raw) +} + +func swaggerOperationID(method, path string) string { + value := strings.ToLower(method) + "_" + strings.Trim(path, "/") + value = swaggerPathParameter.ReplaceAllString(value, "$1") + return strings.Map(func(r rune) rune { + if r >= 'a' && r <= 'z' || r >= '0' && r <= '9' || r == '_' { + return r + } + return '_' + }, value) +} + +func swaggerPathParameters(path string) []map[string]any { + matches := swaggerPathParameter.FindAllStringSubmatch(path, -1) + parameters := make([]map[string]any, 0, len(matches)) + for _, match := range matches { + parameters = append(parameters, map[string]any{"name": match[1], "in": "path", "required": true, "type": "string"}) + } + return parameters +} + +func swaggerPublicPath(path string) bool { + for _, marker := range []string{ + "/health", "/base/login", "/base/captcha", "/init/checkdb", "/init/initdb", + "/api/freshCasbin", "/sysExportTemplate/exportExcelByToken", "/sysExportTemplate/exportTemplateByToken", + "/sysError/createSysError", "/info/getInfoDataSource", "/info/getInfoPublic", + } { + if path == marker { + return true + } + } + return false +} diff --git a/internal/service/api_metadata.go b/internal/service/api_metadata.go index 2c9456c..b883794 100644 --- a/internal/service/api_metadata.go +++ b/internal/service/api_metadata.go @@ -177,3 +177,9 @@ func routeMetadata(method, path string) (string, string) { } return routeGroup(path), "" } + +// RouteMetadata exposes the administration API grouping and description to +// transport-level documentation without leaking the metadata table itself. +func RouteMetadata(method, path string) (string, string) { + return routeMetadata(method, path) +} diff --git a/internal/service/security.go b/internal/service/security.go index 734fede..cf2b189 100644 --- a/internal/service/security.go +++ b/internal/service/security.go @@ -2,11 +2,22 @@ package service import ( "context" + "errors" "kra/internal/biz" "kra/internal/service/dto" ) +type PasswordPolicyError struct{ Err error } + +func (e *PasswordPolicyError) Error() string { return e.Err.Error() } +func (e *PasswordPolicyError) Unwrap() error { return e.Err } + +func IsPasswordPolicyError(err error) bool { + var policyErr *PasswordPolicyError + return errors.As(err, &policyErr) +} + func securityDTO(v *biz.SecurityConfig) *dto.SecurityConfigResponse { return &dto.SecurityConfigResponse{ID: v.ID, CreatedAt: v.CreatedAt, UpdatedAt: v.UpdatedAt, DeletedAt: nil, CaptchaOpen: v.CaptchaOpen, CaptchaTimeout: v.CaptchaTimeout, KeyLong: v.KeyLong, ImgWidth: v.ImgWidth, ImgHeight: v.ImgHeight, PwdMinLength: v.PwdMinLength, PwdRequireUpper: v.PwdRequireUpper, PwdRequireLower: v.PwdRequireLower, PwdRequireDigit: v.PwdRequireDigit, PwdRequireSpecial: v.PwdRequireSpecial, LimitEnable: v.LimitEnable, LimitWindow: v.LimitWindow, LimitCount: v.LimitCount, LockEnable: v.LockEnable, LockThreshold: v.LockThreshold, LockDuration: v.LockDuration, PwdExpireEnable: v.PwdExpireEnable, PwdExpireDays: v.PwdExpireDays, ForceNewUserChangePassword: v.ForceNewUserChangePassword} } @@ -44,5 +55,8 @@ func (s *SecurityService) ValidatePassword(ctx context.Context, password string) if err != nil { return err } - return s.uc.ValidatePassword(cfg, password) + if err := s.uc.ValidatePassword(cfg, password); err != nil { + return &PasswordPolicyError{Err: err} + } + return nil } diff --git a/internal/worker/task_scheduler.go b/internal/worker/task_scheduler.go index b1bebc7..9744a9d 100644 --- a/internal/worker/task_scheduler.go +++ b/internal/worker/task_scheduler.go @@ -67,6 +67,7 @@ func (s *TaskScheduler) Start(ctx context.Context) error { } func (s *TaskScheduler) Stop(ctx context.Context) error { + s.closeSubscribers() s.ctxMu.Lock() if s.cancel != nil { s.cancel() @@ -241,6 +242,18 @@ func (s *TaskScheduler) Unsubscribe(userID uint, ch chan []byte) { } s.subMu.Unlock() } + +func (s *TaskScheduler) closeSubscribers() { + s.subMu.Lock() + defer s.subMu.Unlock() + for userID, subscribers := range s.subscribers { + for ch := range subscribers { + close(ch) + } + delete(s.subscribers, userID) + } +} + func (s *TaskScheduler) PublishToUsers(userIDs []uint, value any) { raw, err := json.Marshal(value) if err != nil { diff --git a/pkg/logging/source.go b/pkg/logging/source.go new file mode 100644 index 0000000..dc7eb6b --- /dev/null +++ b/pkg/logging/source.go @@ -0,0 +1,98 @@ +package logging + +import ( + "fmt" + "go/ast" + "go/parser" + "go/token" + "os" + "regexp" + "strconv" + "strings" +) + +type stackFrame struct { + File string + Line int + Func string +} + +var stackFileLinePattern = regexp.MustCompile(`\s*(.+\.go):(\d+)\s*$`) + +func finalApplicationCaller(stack string) (stackFrame, bool) { + if stack == "" { + return stackFrame{}, false + } + functionName := "" + for _, raw := range strings.Split(stack, "\n") { + line := strings.TrimSpace(raw) + if line == "" { + continue + } + matches := stackFileLinePattern.FindStringSubmatch(line) + if matches == nil { + functionName = line + continue + } + lineNumber, _ := strconv.Atoi(matches[2]) + if skipStackFile(matches[1]) { + functionName = "" + continue + } + return stackFrame{File: matches[1], Line: lineNumber, Func: functionName}, true + } + return stackFrame{}, false +} + +func skipStackFile(filename string) bool { + normalized := strings.ReplaceAll(filename, "\\", "/") + for _, marker := range []string{ + "/go/pkg/mod/", + "/go.uber.org/", + "/gorm.io/", + "/pkg/logging/", + "/internal/server/middleware/", + "/internal/server/router/", + } { + if strings.Contains(normalized, marker) { + return true + } + } + return strings.Contains(normalized, "/src/") && + (strings.Contains(normalized, "/go/go") || strings.Contains(normalized, "/go/src/")) +} + +func functionSourceAt(filename string, line int) (name, source string, startLine, endLine int, err error) { + content, err := os.ReadFile(filename) + if err != nil { + return "", "", 0, 0, fmt.Errorf("read file failed: %w", err) + } + files := token.NewFileSet() + parsed, err := parser.ParseFile(files, filename, content, parser.ParseComments) + if err != nil { + return "", "", 0, 0, fmt.Errorf("parse file failed: %w", err) + } + var target *ast.FuncDecl + ast.Inspect(parsed, func(node ast.Node) bool { + declaration, ok := node.(*ast.FuncDecl) + if !ok { + return true + } + start := files.Position(declaration.Pos()).Line + end := files.Position(declaration.End()).Line + if line >= start && line <= end { + target, startLine, endLine = declaration, start, end + return false + } + return true + }) + if target == nil { + return "", "", 0, 0, fmt.Errorf("no function encloses line %d in %s", line, filename) + } + start := files.Position(target.Pos()).Offset + end := files.Position(target.End()).Offset + if start < 0 || end > len(content) || start >= end { + return "", "", 0, 0, fmt.Errorf("invalid offsets for function: start=%d end=%d len=%d", start, end, len(content)) + } + return target.Name.Name, string(content[start:end]), startLine, endLine, nil +} diff --git a/pkg/logging/zap.go b/pkg/logging/zap.go index 04fa92e..6613d55 100644 --- a/pkg/logging/zap.go +++ b/pkg/logging/zap.go @@ -24,6 +24,42 @@ type Options struct { FileOnlyModules []string } +// ErrorEntry is the storage-neutral representation of an Error-level log. +// Keeping it in pkg/logging lets the log core report failures without taking a +// dependency on the application service or persistence layers. +type ErrorEntry struct { + Form, Info, Level, RequestID, TraceID string +} + +// ErrorSink receives Error-level log entries. The sink must not log failures +// through the same logger, otherwise a storage failure could recurse forever. +type ErrorSink interface { + RecordLogError(context.Context, ErrorEntry) error +} + +type ErrorSinkFunc func(context.Context, ErrorEntry) error + +func (f ErrorSinkFunc) RecordLogError(ctx context.Context, entry ErrorEntry) error { + return f(ctx, entry) +} + +type errorSinkState struct { + mu sync.RWMutex + sink ErrorSink +} + +func (s *errorSinkState) record(entry ErrorEntry) { + if s == nil { + return + } + s.mu.RLock() + sink := s.sink + s.mu.RUnlock() + if sink != nil { + _ = sink.RecordLogError(context.Background(), entry) + } +} + type handlerOperation struct { attrs []slog.Attr group string @@ -77,12 +113,13 @@ func (h *reloadableHandler) WithGroup(name string) slog.Handler { } type ReloadableLogger struct { - state *reloadableHandlerState - filename string + state *reloadableHandlerState + filename string + errorSink *errorSinkState } func (l *ReloadableLogger) Reload(root string, options Options) { - handler, cleanup := newZapHandler(root, l.filename, options) + handler, cleanup := newZapHandler(root, l.filename, options, l.errorSink) l.state.mu.Lock() previous := l.state.cleanup l.state.handler = handler @@ -93,6 +130,17 @@ func (l *ReloadableLogger) Reload(root string, options Options) { } } +// SetErrorSink changes the database/audit target while keeping the current +// logger and its hot-reload state intact. +func (l *ReloadableLogger) SetErrorSink(sink ErrorSink) { + if l == nil || l.errorSink == nil { + return + } + l.errorSink.mu.Lock() + l.errorSink.sink = sink + l.errorSink.mu.Unlock() +} + func (l *ReloadableLogger) Close() { l.state.mu.Lock() cleanup := l.state.cleanup @@ -162,6 +210,7 @@ type routedFileCore struct { retentionDay int state *routedFileState fields []zapcore.Field + errorSink *errorSinkState } func (c *routedFileCore) Enabled(level zapcore.Level) bool { return c.level.Enabled(level) } @@ -169,7 +218,7 @@ func (c *routedFileCore) Enabled(level zapcore.Level) bool { return c.level.Enab func (c *routedFileCore) With(fields []zapcore.Field) zapcore.Core { inherited := append([]zapcore.Field(nil), c.fields...) inherited = append(inherited, fields...) - return &routedFileCore{base: c.base.With(fields), encoder: c.encoder, level: c.level, root: c.root, retentionDay: c.retentionDay, state: c.state, fields: inherited} + return &routedFileCore{base: c.base.With(fields), encoder: c.encoder, level: c.level, root: c.root, retentionDay: c.retentionDay, state: c.state, fields: inherited, errorSink: c.errorSink} } func (c *routedFileCore) Check(entry zapcore.Entry, checked *zapcore.CheckedEntry) *zapcore.CheckedEntry { @@ -183,6 +232,9 @@ func (c *routedFileCore) Write(entry zapcore.Entry, fields []zapcore.Field) erro baseErr := c.base.Write(entry, fields) allFields := append([]zapcore.Field(nil), c.fields...) allFields = append(allFields, fields...) + if entry.Level >= zapcore.ErrorLevel && !isGORMLoggerEntry(entry.Caller.File, allFields) { + c.errorSink.record(errorEntryFromZap(entry, allFields)) + } paths := routedLogPaths(moduleField(allFields), entry.Level) if len(paths) == 0 { return baseErr @@ -204,6 +256,66 @@ func (c *routedFileCore) Write(entry zapcore.Entry, fields []zapcore.Field) erro return baseErr } +func isGORMLoggerEntry(filename string, fields []zapcore.Field) bool { + for _, field := range fields { + if field.Key == "gorm_logger" { + return true + } + } + normalized := strings.ReplaceAll(filename, "\\", "/") + return strings.HasSuffix(normalized, "/gorm_logger_writer.go") || + strings.HasSuffix(normalized, "/internal/data/gorm_logger.go") +} + +func errorEntryFromZap(entry zapcore.Entry, fields []zapcore.Field) ErrorEntry { + requestID, traceID, errorText := "", "", "" + for _, field := range fields { + switch field.Key { + case "request_id": + if requestID == "" { + requestID = zapFieldString(field) + } + case "trace_id": + if traceID == "" { + traceID = zapFieldString(field) + } + case "error", "err": + if errorText == "" { + errorText = zapFieldString(field) + } + } + } + info := entry.Message + if errorText != "" { + info += " | 错误: " + errorText + } + if entry.Caller.File != "" { + info += fmt.Sprintf(" \n 源文件:%s:%d", entry.Caller.File, entry.Caller.Line) + } + if entry.Stack != "" { + info += " \n 调用栈：" + entry.Stack + if frame, ok := finalApplicationCaller(entry.Stack); ok { + functionName, source, startLine, endLine, err := functionSourceAt(frame.File, frame.Line) + if err == nil { + info += fmt.Sprintf(" \n 最终调用方法:%s:%d (%s lines %d-%d)\n----- 产生日志的方法代码如下 -----\n%s", frame.File, frame.Line, functionName, startLine, endLine, source) + } else { + info += fmt.Sprintf(" \n 最终调用方法:%s:%d (%s) | extract_err=%v", frame.File, frame.Line, functionName, err) + } + } + } + return ErrorEntry{Form: "后端", Info: info, Level: entry.Level.String(), RequestID: requestID, TraceID: traceID} +} + +func zapFieldString(field zapcore.Field) string { + if field.String != "" { + return field.String + } + if field.Interface != nil { + return fmt.Sprint(field.Interface) + } + return "" +} + func (c *routedFileCore) Sync() error { result := c.base.Sync() c.state.mu.Lock() @@ -274,7 +386,7 @@ func safeModuleName(value string) string { // NewZapLogger adapts a Zap core to the slog logger used by Kratos v3. // The file layout remains compatible with the administration log viewer. -func newZapHandler(root, filename string, options Options) (slog.Handler, func()) { +func newZapHandler(root, filename string, options Options, errorSink *errorSinkState) (slog.Handler, func()) { file := NewDailyWriter(root, filename, options.RetentionDay) encoder := zap.NewProductionEncoderConfig() encoder.EncodeTime = zapcore.RFC3339NanoTimeEncoder @@ -324,7 +436,7 @@ func newZapHandler(root, filename string, options Options) (slog.Handler, func() consoleCore := zapcore.NewCore(outputEncoder.Clone(), zapcore.AddSync(os.Stdout), levelEnabler) core = zapcore.NewTee(fileCore, &moduleFilterCore{Core: consoleCore, fileOnly: fileOnly}) } - routed := &routedFileCore{base: core, encoder: outputEncoder.Clone(), level: levelEnabler, root: root, retentionDay: options.RetentionDay, state: &routedFileState{writers: map[string]*DailyWriter{}}} + routed := &routedFileCore{base: core, encoder: outputEncoder.Clone(), level: levelEnabler, root: root, retentionDay: options.RetentionDay, state: &routedFileState{writers: map[string]*DailyWriter{}}, errorSink: errorSink} zapLogger := zap.New(routed) handlerOptions := []zapslog.HandlerOption{zapslog.AddStacktraceAt(slog.LevelError)} if options.ShowLine { @@ -342,9 +454,10 @@ func newZapHandler(root, filename string, options Options) (slog.Handler, func() // NewReloadableZapLogger keeps the slog/Kratos adapter stable while replacing // the underlying Zap core when the runtime configuration changes. func NewReloadableZapLogger(root, filename string, options Options, attrs ...any) (*slog.Logger, *ReloadableLogger) { - baseHandler, cleanup := newZapHandler(root, filename, options) + errorSink := &errorSinkState{} + baseHandler, cleanup := newZapHandler(root, filename, options, errorSink) state := &reloadableHandlerState{handler: baseHandler, cleanup: cleanup} - control := &ReloadableLogger{state: state, filename: filename} + control := &ReloadableLogger{state: state, filename: filename, errorSink: errorSink} handler := &contextHandler{handler: &reloadableHandler{state: state}} logger := kratoslog.NewLogger(handler, kratoslog.WithExtractor(tracing.TraceAttrs)).With(attrs...) return logger, control diff --git a/pkg/logging/zap_test.go b/pkg/logging/zap_test.go index 60edacf..d132db1 100644 --- a/pkg/logging/zap_test.go +++ b/pkg/logging/zap_test.go @@ -5,12 +5,16 @@ import ( "log/slog" "os" "path/filepath" + "strings" "testing" + + "go.uber.org/zap" + "go.uber.org/zap/zapcore" ) func TestZapHandlerHonorsConfiguredLevel(t *testing.T) { root := t.TempDir() - handler, cleanup := newZapHandler(root, "application.log", Options{Level: "error", Format: "json"}) + handler, cleanup := newZapHandler(root, "application.log", Options{Level: "error", Format: "json"}, nil) defer cleanup() if handler.Enabled(context.Background(), slog.LevelInfo) { @@ -23,7 +27,7 @@ func TestZapHandlerHonorsConfiguredLevel(t *testing.T) { func TestZapHandlerUsesDebugFallbackForInvalidLevel(t *testing.T) { root := t.TempDir() - handler, cleanup := newZapHandler(root, "application.log", Options{Level: "not-a-level", Format: "json"}) + handler, cleanup := newZapHandler(root, "application.log", Options{Level: "not-a-level", Format: "json"}, nil) defer cleanup() if !handler.Enabled(context.Background(), slog.LevelDebug) { @@ -33,7 +37,7 @@ func TestZapHandlerUsesDebugFallbackForInvalidLevel(t *testing.T) { func TestZapHandlerRoutesHTTPAndErrorLogs(t *testing.T) { root := t.TempDir() - handler, cleanup := newZapHandler(root, "application.log", Options{Level: "info", Format: "json"}) + handler, cleanup := newZapHandler(root, "application.log", Options{Level: "info", Format: "json"}, nil) logger := slog.New(handler) logger.Info("request", "mod", "http", "request_id", "req-1") logger.Error("failed", "mod", "users") @@ -56,3 +60,59 @@ func TestZapHandlerRoutesHTTPAndErrorLogs(t *testing.T) { } } } + +func TestZapHandlerRecordsEveryErrorThroughSink(t *testing.T) { + root := t.TempDir() + var entries []ErrorEntry + logger, control := NewReloadableZapLogger(root, "application.log", Options{Level: "info", Format: "json"}) + defer control.Close() + control.SetErrorSink(ErrorSinkFunc(func(_ context.Context, entry ErrorEntry) error { + entries = append(entries, entry) + return nil + })) + logger.Error("task execution failed", "mod", "timedTask", "request_id", "request-1", "trace_id", "trace-1", "error", os.ErrPermission) + control.Reload(t.TempDir(), Options{Level: "info", Format: "json"}) + logger.Error("reloaded logger failure", "mod", "system") + + if len(entries) != 2 { + t.Fatalf("expected two error entries across reload, got %d", len(entries)) + } + entry := entries[0] + if entry.Form != "后端" || entry.Level != "error" || entry.RequestID != "request-1" || entry.TraceID != "trace-1" || !strings.Contains(entry.Info, "task execution failed") || !strings.Contains(entry.Info, os.ErrPermission.Error()) { + t.Fatalf("unexpected error entry: %+v", entry) + } +} + +func TestErrorEntryIncludesFinalApplicationSource(t *testing.T) { + filename := filepath.Join(t.TempDir(), "worker.go") + content := "package sample\n\nfunc execute() {\n\tprintln(\"failed\")\n}\n" + if err := os.WriteFile(filename, []byte(content), 0o600); err != nil { + t.Fatal(err) + } + entry := errorEntryFromZap(zapcore.Entry{Message: "task failed", Level: zapcore.ErrorLevel, Stack: "kra/internal/worker.execute\n" + filename + ":4"}, nil) + if !strings.Contains(entry.Info, "最终调用方法:"+filename+":4 (execute lines 3-5)") || !strings.Contains(entry.Info, "func execute()") { + t.Fatalf("expected final caller source in error entry: %s", entry.Info) + } +} + +func TestErrorSinkSkipsGORMBridge(t *testing.T) { + root := t.TempDir() + var entries []ErrorEntry + state := &errorSinkState{sink: ErrorSinkFunc(func(_ context.Context, entry ErrorEntry) error { + entries = append(entries, entry) + return nil + })} + base := zapcore.NewNopCore() + core := &routedFileCore{base: base, encoder: zapcore.NewJSONEncoder(zap.NewProductionEncoderConfig()), level: zapcore.ErrorLevel, root: root, state: &routedFileState{writers: map[string]*DailyWriter{}}, errorSink: state} + for _, filename := range []string{"/tmp/gorm_logger_writer.go", "/workspace/internal/data/gorm_logger.go"} { + if err := core.Write(zapcore.Entry{Level: zapcore.ErrorLevel, Message: "database failed", Caller: zapcore.EntryCaller{Defined: true, File: filename, Line: 10}}, nil); err != nil { + t.Fatal(err) + } + } + if err := core.Write(zapcore.Entry{Level: zapcore.ErrorLevel, Message: "database failed"}, []zapcore.Field{zap.Bool("gorm_logger", true)}); err != nil { + t.Fatal(err) + } + if len(entries) != 0 { + t.Fatalf("gorm bridge error must not recurse into sys_error: %+v", entries) + } +}