452 lines
14 KiB
Go
452 lines
14 KiB
Go
package system
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"errors"
|
|
"fmt"
|
|
"kra/internal/biz/system"
|
|
"strconv"
|
|
"strings"
|
|
"time"
|
|
|
|
"kra/pkg/database/pagination"
|
|
|
|
"gorm.io/gorm"
|
|
)
|
|
|
|
type exportTemplatePO struct {
|
|
ID uint `gorm:"primaryKey"`
|
|
CreatedAt time.Time
|
|
UpdatedAt time.Time
|
|
DeletedAt gorm.DeletedAt `gorm:"index"`
|
|
DBName string
|
|
Name string
|
|
DBTableName string `gorm:"column:table_name"`
|
|
TemplateID string
|
|
TemplateInfo string `gorm:"type:text"`
|
|
SQL string `gorm:"type:text"`
|
|
ImportSQL string `gorm:"type:text"`
|
|
Limit *int
|
|
Order string
|
|
}
|
|
|
|
func (exportTemplatePO) TableName() string { return "sys_export_templates" }
|
|
|
|
type exportConditionPO struct {
|
|
ID uint `gorm:"primaryKey"`
|
|
CreatedAt time.Time
|
|
UpdatedAt time.Time
|
|
DeletedAt gorm.DeletedAt `gorm:"index"`
|
|
TemplateID string `gorm:"index"`
|
|
From string
|
|
Column string
|
|
Operator string
|
|
}
|
|
|
|
func (exportConditionPO) TableName() string { return "sys_export_template_condition" }
|
|
|
|
type exportJoinPO struct {
|
|
ID uint `gorm:"primaryKey"`
|
|
CreatedAt time.Time
|
|
UpdatedAt time.Time
|
|
DeletedAt gorm.DeletedAt `gorm:"index"`
|
|
TemplateID string `gorm:"index"`
|
|
Join string `gorm:"column:joins"`
|
|
Table string
|
|
On string `gorm:"column:on"`
|
|
}
|
|
|
|
func (exportJoinPO) TableName() string { return "sys_export_template_join" }
|
|
|
|
type exportRepo struct{ data Provider }
|
|
|
|
func NewExportRepo(data Provider) system.ExportRepo { return &exportRepo{data: data} }
|
|
func exportFromPO(po exportTemplatePO, conditions []exportConditionPO, joins []exportJoinPO) *system.ExportTemplate {
|
|
v := &system.ExportTemplate{ID: po.ID, CreatedAt: po.CreatedAt, UpdatedAt: po.UpdatedAt, DBName: po.DBName, Name: po.Name, TableName: po.DBTableName, TemplateID: po.TemplateID, TemplateInfo: po.TemplateInfo, SQL: po.SQL, ImportSQL: po.ImportSQL, Limit: po.Limit, Order: po.Order}
|
|
if conditions != nil {
|
|
v.Conditions = make([]system.ExportCondition, 0, len(conditions))
|
|
}
|
|
for _, x := range conditions {
|
|
v.Conditions = append(v.Conditions, system.ExportCondition{ID: x.ID, CreatedAt: x.CreatedAt, UpdatedAt: x.UpdatedAt, TemplateID: x.TemplateID, From: x.From, Column: x.Column, Operator: x.Operator})
|
|
}
|
|
if joins != nil {
|
|
v.Joins = make([]system.ExportJoin, 0, len(joins))
|
|
}
|
|
for _, x := range joins {
|
|
v.Joins = append(v.Joins, system.ExportJoin{ID: x.ID, CreatedAt: x.CreatedAt, UpdatedAt: x.UpdatedAt, TemplateID: x.TemplateID, Join: x.Join, Table: x.Table, On: x.On})
|
|
}
|
|
return v
|
|
}
|
|
func exportToPO(v *system.ExportTemplate) exportTemplatePO {
|
|
return exportTemplatePO{DBName: v.DBName, Name: v.Name, DBTableName: v.TableName, TemplateID: v.TemplateID, TemplateInfo: v.TemplateInfo, SQL: v.SQL, ImportSQL: v.ImportSQL, Limit: v.Limit, Order: v.Order}
|
|
}
|
|
func (r *exportRepo) saveRelations(tx *gorm.DB, v *system.ExportTemplate, replace bool) error {
|
|
if replace {
|
|
if err := tx.Where("template_id = ?", v.TemplateID).Delete(&exportConditionPO{}).Error; err != nil {
|
|
return err
|
|
}
|
|
if err := tx.Where("template_id = ?", v.TemplateID).Delete(&exportJoinPO{}).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
conditions := make([]exportConditionPO, 0, len(v.Conditions))
|
|
for _, x := range v.Conditions {
|
|
conditions = append(conditions, exportConditionPO{TemplateID: v.TemplateID, From: x.From, Column: x.Column, Operator: x.Operator})
|
|
}
|
|
if len(conditions) > 0 {
|
|
if err := tx.Create(&conditions).Error; err != nil {
|
|
return err
|
|
}
|
|
}
|
|
joins := make([]exportJoinPO, 0, len(v.Joins))
|
|
for _, x := range v.Joins {
|
|
joins = append(joins, exportJoinPO{TemplateID: v.TemplateID, Join: x.Join, Table: x.Table, On: x.On})
|
|
}
|
|
if len(joins) > 0 {
|
|
return tx.Create(&joins).Error
|
|
}
|
|
return nil
|
|
}
|
|
func (r *exportRepo) CreateExportTemplate(ctx context.Context, v *system.ExportTemplate) error {
|
|
return r.data.DB().WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
|
po := exportToPO(v)
|
|
if err := tx.Create(&po).Error; err != nil {
|
|
return err
|
|
}
|
|
v.ID = po.ID
|
|
return r.saveRelations(tx, v, false)
|
|
})
|
|
}
|
|
func (r *exportRepo) UpdateExportTemplate(ctx context.Context, v *system.ExportTemplate) error {
|
|
return r.data.DB().WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
|
po := exportToPO(v)
|
|
if err := tx.Model(&exportTemplatePO{}).Where("id = ?", v.ID).Updates(&po).Error; err != nil {
|
|
return err
|
|
}
|
|
return r.saveRelations(tx, v, true)
|
|
})
|
|
}
|
|
func (r *exportRepo) DeleteExportTemplates(ctx context.Context, ids []uint) error {
|
|
return r.data.DB().WithContext(ctx).Delete(&[]exportTemplatePO{}, "id IN ?", ids).Error
|
|
}
|
|
func (r *exportRepo) FindExportTemplate(ctx context.Context, id uint, tid string) (*system.ExportTemplate, error) {
|
|
var po exportTemplatePO
|
|
db := r.data.DB().WithContext(ctx)
|
|
var err error
|
|
if tid != "" {
|
|
err = db.Where("template_id = ?", tid).First(&po).Error
|
|
} else {
|
|
err = db.Where("id = ?", id).First(&po).Error
|
|
}
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
var conditions []exportConditionPO
|
|
var joins []exportJoinPO
|
|
if err = db.Where("template_id = ?", po.TemplateID).Find(&conditions).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
if err = db.Where("template_id = ?", po.TemplateID).Find(&joins).Error; err != nil {
|
|
return nil, err
|
|
}
|
|
return exportFromPO(po, conditions, joins), nil
|
|
}
|
|
func (r *exportRepo) ListExportTemplates(ctx context.Context, page, size int, q *system.ExportTemplateFilter) ([]*system.ExportTemplate, int64, error) {
|
|
db := r.data.DB().WithContext(ctx).Model(&exportTemplatePO{})
|
|
if q != nil {
|
|
if q.StartCreatedAt != nil && q.EndCreatedAt != nil {
|
|
db = db.Where("created_at BETWEEN ? AND ?", q.StartCreatedAt, q.EndCreatedAt)
|
|
}
|
|
if q.Name != "" {
|
|
db = db.Where("name LIKE ?", "%"+q.Name+"%")
|
|
}
|
|
if q.TableName != "" {
|
|
db = db.Where("table_name = ?", q.TableName)
|
|
}
|
|
if q.TemplateID != "" {
|
|
db = db.Where("template_id = ?", q.TemplateID)
|
|
}
|
|
}
|
|
var total int64
|
|
if err := db.Count(&total).Error; err != nil {
|
|
return nil, 0, err
|
|
}
|
|
var pos []exportTemplatePO
|
|
db = pagination.Apply(db, page, size, 100)
|
|
if err := db.Find(&pos).Error; err != nil {
|
|
return nil, 0, err
|
|
}
|
|
out := make([]*system.ExportTemplate, 0, len(pos))
|
|
for _, po := range pos {
|
|
out = append(out, exportFromPO(po, nil, nil))
|
|
}
|
|
return out, total, nil
|
|
}
|
|
|
|
func (r *exportRepo) QueryExport(ctx context.Context, t *system.ExportTemplate, params map[string]string) ([]map[string]any, string, error) {
|
|
query, err := r.buildExportQuery(ctx, t, params)
|
|
if err != nil {
|
|
return nil, "", err
|
|
}
|
|
preview := query.ToSQL(func(tx *gorm.DB) *gorm.DB { return tx.Find(&[]map[string]any{}) })
|
|
rows, err := queryExportRows(query)
|
|
return rows, preview, err
|
|
}
|
|
|
|
func queryExportRows(query *gorm.DB) ([]map[string]any, error) {
|
|
rows, err := query.Rows()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
defer rows.Close()
|
|
columns, err := rows.Columns()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
columnTypes, err := rows.ColumnTypes()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
result := make([]map[string]any, 0)
|
|
for rows.Next() {
|
|
values := make([]any, len(columns))
|
|
destinations := make([]any, len(values))
|
|
for index := range values {
|
|
destinations[index] = &values[index]
|
|
}
|
|
if err := rows.Scan(destinations...); err != nil {
|
|
return nil, err
|
|
}
|
|
item := make(map[string]any, len(columns))
|
|
for index, column := range columns {
|
|
typeName := ""
|
|
if index < len(columnTypes) {
|
|
typeName = columnTypes[index].DatabaseTypeName()
|
|
}
|
|
item[column] = exportQueryValue(values[index], typeName)
|
|
}
|
|
result = append(result, item)
|
|
}
|
|
if err := rows.Err(); err != nil {
|
|
return nil, err
|
|
}
|
|
return result, nil
|
|
}
|
|
|
|
func exportQueryValue(value any, databaseType string) any {
|
|
bytesValue, ok := value.([]byte)
|
|
if !ok {
|
|
return value
|
|
}
|
|
raw := string(bytesValue)
|
|
typeName := strings.ToUpper(databaseType)
|
|
if strings.Contains(typeName, "INT") || strings.Contains(typeName, "SERIAL") {
|
|
if integer, err := strconv.ParseInt(raw, 10, 64); err == nil {
|
|
return integer
|
|
}
|
|
}
|
|
if strings.Contains(typeName, "DECIMAL") || strings.Contains(typeName, "NUMERIC") || strings.Contains(typeName, "NUMBER") || strings.Contains(typeName, "REAL") || strings.Contains(typeName, "DOUBLE") || strings.Contains(typeName, "FLOAT") {
|
|
if decimal, ok := exactExportFloat(raw); ok {
|
|
return decimal
|
|
}
|
|
}
|
|
return raw
|
|
}
|
|
|
|
func exactExportFloat(value string) (float64, bool) {
|
|
decimal, err := strconv.ParseFloat(value, 64)
|
|
if err != nil {
|
|
return 0, false
|
|
}
|
|
return decimal, normalizeDecimal(value) == normalizeDecimal(strconv.FormatFloat(decimal, 'f', -1, 64))
|
|
}
|
|
|
|
func normalizeDecimal(value string) string {
|
|
value = strings.TrimSpace(value)
|
|
negative := strings.HasPrefix(value, "-")
|
|
value = strings.TrimPrefix(value, "+")
|
|
value = strings.TrimPrefix(value, "-")
|
|
parts := strings.SplitN(value, ".", 2)
|
|
whole := strings.TrimLeft(parts[0], "0")
|
|
if whole == "" {
|
|
whole = "0"
|
|
}
|
|
fraction := ""
|
|
if len(parts) == 2 {
|
|
fraction = strings.TrimRight(parts[1], "0")
|
|
}
|
|
result := whole
|
|
if fraction != "" {
|
|
result += "." + fraction
|
|
}
|
|
if negative && result != "0" {
|
|
result = "-" + result
|
|
}
|
|
return result
|
|
}
|
|
|
|
func exportColumns(raw string) ([]string, error) {
|
|
columns := make([]string, 0)
|
|
decoder := json.NewDecoder(strings.NewReader(raw))
|
|
if token, err := decoder.Token(); err != nil || token != json.Delim('{') {
|
|
return nil, errors.New("导出模板列定义必须是 JSON 对象")
|
|
}
|
|
for decoder.More() {
|
|
key, err := decoder.Token()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
column, ok := key.(string)
|
|
if !ok {
|
|
return nil, errors.New("导出模板列名无效")
|
|
}
|
|
columns = append(columns, column)
|
|
var ignored any
|
|
if err = decoder.Decode(&ignored); err != nil {
|
|
return nil, err
|
|
}
|
|
}
|
|
if len(columns) == 0 {
|
|
return nil, errors.New("模板列为空")
|
|
}
|
|
return columns, nil
|
|
}
|
|
|
|
func (r *exportRepo) buildExportQuery(ctx context.Context, t *system.ExportTemplate, params map[string]string) (*gorm.DB, error) {
|
|
if err := system.ValidateExportTemplate(t); err != nil {
|
|
return nil, err
|
|
}
|
|
selected, err := r.data.Database(t.DBName)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
db := selected.WithContext(ctx)
|
|
columns, err := exportColumns(t.TemplateInfo)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
query := db.Table(t.TableName).Select(strings.Join(columns, ","))
|
|
for _, j := range t.Joins {
|
|
query = query.Joins(strings.ToUpper(strings.Join(strings.Fields(j.Join), " ")) + " " + j.Table + " ON " + j.On)
|
|
}
|
|
if params["filterDeleted"] == "true" {
|
|
query = query.Where(t.TableName + ".deleted_at IS NULL")
|
|
for _, join := range t.Joins {
|
|
if db.Migrator().HasColumn(join.Table, "deleted_at") {
|
|
query = query.Where(join.Table + ".deleted_at IS NULL")
|
|
}
|
|
}
|
|
}
|
|
for _, condition := range t.Conditions {
|
|
operator := strings.ToUpper(strings.Join(strings.Fields(condition.Operator), " "))
|
|
value := params[condition.From]
|
|
switch operator {
|
|
case "LIKE":
|
|
if value != "" {
|
|
query = query.Where(condition.Column+" LIKE ?", "%"+value+"%")
|
|
}
|
|
case "IN", "NOT IN":
|
|
if value != "" {
|
|
values := strings.Split(value, ",")
|
|
for index := range values {
|
|
values[index] = strings.TrimSpace(values[index])
|
|
}
|
|
query = query.Where(condition.Column+" "+operator+" ?", values)
|
|
}
|
|
case "BETWEEN":
|
|
start, end := params["start"+condition.From], params["end"+condition.From]
|
|
if start != "" && end != "" {
|
|
query = query.Where(condition.Column+" BETWEEN ? AND ?", start, end)
|
|
}
|
|
default:
|
|
if value != "" {
|
|
query = query.Where(condition.Column+" "+operator+" ?", value)
|
|
}
|
|
}
|
|
}
|
|
limit := 0
|
|
if raw := params["limit"]; raw != "" {
|
|
limit, err = strconv.Atoi(raw)
|
|
if err != nil || limit < 0 || limit > 100000 {
|
|
return nil, errors.New("导出行数限制不合法")
|
|
}
|
|
} else if t.Limit != nil {
|
|
limit = *t.Limit
|
|
}
|
|
if limit > 0 {
|
|
query = query.Limit(limit)
|
|
}
|
|
if raw := params["offset"]; raw != "" {
|
|
offset, parseErr := strconv.Atoi(raw)
|
|
if parseErr != nil || offset < 0 {
|
|
return nil, errors.New("导出偏移量不合法")
|
|
}
|
|
query = query.Offset(offset)
|
|
}
|
|
order := params["order"]
|
|
if order == "" {
|
|
order = t.Order
|
|
}
|
|
if order != "" {
|
|
parts := strings.Fields(order)
|
|
if len(parts) > 2 {
|
|
return nil, errors.New("导出排序不合法")
|
|
}
|
|
fields := map[string]bool{}
|
|
columnTypes, columnErr := db.Migrator().ColumnTypes(t.TableName)
|
|
if columnErr != nil {
|
|
return nil, columnErr
|
|
}
|
|
for _, column := range columnTypes {
|
|
fields[column.Name()] = true
|
|
}
|
|
columnName := parts[0]
|
|
if dot := strings.LastIndex(columnName, "."); dot >= 0 {
|
|
columnName = columnName[dot+1:]
|
|
}
|
|
if !fields[columnName] {
|
|
return nil, fmt.Errorf("order by %s is not in the fields", order)
|
|
}
|
|
orderSQL := parts[0]
|
|
if len(parts) > 1 {
|
|
if !strings.EqualFold(parts[1], "asc") && !strings.EqualFold(parts[1], "desc") {
|
|
return nil, fmt.Errorf("order by %s is not secure", order)
|
|
}
|
|
orderSQL += " " + strings.ToUpper(parts[1])
|
|
}
|
|
query = query.Order(orderSQL)
|
|
}
|
|
return query, nil
|
|
}
|
|
|
|
func (r *exportRepo) PreviewExport(ctx context.Context, t *system.ExportTemplate, params map[string]string) (string, error) {
|
|
query, err := r.buildExportQuery(ctx, t, params)
|
|
if err != nil {
|
|
return "", err
|
|
}
|
|
return query.ToSQL(func(tx *gorm.DB) *gorm.DB { return tx.Find(&[]map[string]any{}) }), nil
|
|
}
|
|
func (r *exportRepo) ImportExportRows(ctx context.Context, t *system.ExportTemplate, rows []map[string]any) error {
|
|
if err := system.ValidateExportTemplate(t); err != nil {
|
|
return err
|
|
}
|
|
selected, err := r.data.Database(t.DBName)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
return selected.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
|
|
needCreated := tx.Migrator().HasColumn(t.TableName, "created_at")
|
|
needUpdated := tx.Migrator().HasColumn(t.TableName, "updated_at")
|
|
for _, row := range rows {
|
|
if row["created_at"] == nil && needCreated {
|
|
row["created_at"] = time.Now()
|
|
}
|
|
if row["updated_at"] == nil && needUpdated {
|
|
row["updated_at"] = time.Now()
|
|
}
|
|
}
|
|
return tx.Table(t.TableName).CreateInBatches(&rows, 1000).Error
|
|
})
|
|
}
|