feat(service): implement file cleanup and deletion functionality

Author: LinkinStars

cmd/wire_gen.go

@@ -33,6 +33,9 @@ swaggerui:
   address: ':80'
 service_config:
   upload_path: "/data/uploads"
+  clean_up_uploads: true
+  clean_orphan_uploads_period_hours: 48
+  purge_deleted_files_period_days: 30
 ui:
   public_url: '/'
   api_url: '/'

configs/config.yaml

@@ -60,6 +60,7 @@ require (
 	golang.org/x/crypto v0.27.0
 	golang.org/x/image v0.20.0
 	golang.org/x/net v0.29.0
+	golang.org/x/text v0.18.0
 	gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df
 	gopkg.in/yaml.v3 v3.0.1
 	modernc.org/sqlite v1.33.0
@@ -160,7 +161,6 @@ require (
 	golang.org/x/arch v0.10.0 // indirect
 	golang.org/x/exp v0.0.0-20240909161429-701f63a606c0 // indirect
 	golang.org/x/sys v0.25.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
 	golang.org/x/tools v0.25.0 // indirect
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc // indirect

go.mod

@@ -25,4 +25,5 @@ const (
 	PostSubPath        = "post"
 	BrandingSubPath    = "branding"
 	FilesPostSubPath   = "files/post"
+	DeletedSubPath     = "deleted"
 )

internal/base/constant/upload.go

@@ -24,36 +24,45 @@ import (
 	"fmt"
 
 	"github.com/apache/answer/internal/service/content"
+	"github.com/apache/answer/internal/service/file_record"
+	"github.com/apache/answer/internal/service/service_config"
 	"github.com/apache/answer/internal/service/siteinfo_common"
 	"github.com/robfig/cron/v3"
 	"github.com/segmentfault/pacman/log"
 )
 
 // ScheduledTaskManager scheduled task manager
 type ScheduledTaskManager struct {
-	siteInfoService siteinfo_common.SiteInfoCommonService
-	questionService *content.QuestionService
+	siteInfoService   siteinfo_common.SiteInfoCommonService
+	questionService   *content.QuestionService
+	fileRecordService *file_record.FileRecordService
+	serviceConfig     *service_config.ServiceConfig
 }
 
 // NewScheduledTaskManager new scheduled task manager
 func NewScheduledTaskManager(
 	siteInfoService siteinfo_common.SiteInfoCommonService,
 	questionService *content.QuestionService,
+	fileRecordService *file_record.FileRecordService,
+	serviceConfig *service_config.ServiceConfig,
 ) *ScheduledTaskManager {
 	manager := &ScheduledTaskManager{
-		siteInfoService: siteInfoService,
-		questionService: questionService,
+		siteInfoService:   siteInfoService,
+		questionService:   questionService,
+		fileRecordService: fileRecordService,
+		serviceConfig:     serviceConfig,
 	}
 	return manager
 }
 
 func (s *ScheduledTaskManager) Run() {
-	fmt.Println("start cron")
+	log.Infof("cron job manager start")
+
 	s.questionService.SitemapCron(context.Background())
 	c := cron.New()
 	_, err := c.AddFunc("0 */1 * * *", func() {
 		ctx := context.Background()
-		fmt.Println("sitemap cron execution")
+		log.Infof("sitemap cron execution")
 		s.questionService.SitemapCron(ctx)
 	})
 	if err != nil {
@@ -62,12 +71,32 @@ func (s *ScheduledTaskManager) Run() {
 
 	_, err = c.AddFunc("0 */1 * * *", func() {
 		ctx := context.Background()
-		fmt.Println("refresh hottest cron execution")
+		log.Infof("refresh hottest cron execution")
 		s.questionService.RefreshHottestCron(ctx)
 	})
 	if err != nil {
 		log.Error(err)
 	}
 
+	if s.serviceConfig.CleanUpUploads {
+		log.Infof("clean up uploads cron enabled")
+
+		conf := s.serviceConfig
+		_, err = c.AddFunc(fmt.Sprintf("0 */%d * * *", conf.CleanOrphanUploadsPeriodHours), func() {
+			log.Infof("clean orphan upload files cron execution")
+			s.fileRecordService.CleanOrphanUploadFiles(context.Background())
+		})
+		if err != nil {
+			log.Error(err)
+		}
+
+		_, err = c.AddFunc(fmt.Sprintf("0 0 */%d * *", conf.PurgeDeletedFilesPeriodDays), func() {
+			log.Infof("purge deleted files cron execution")
+			s.fileRecordService.PurgeDeletedFiles(context.Background())
+		})
+		if err != nil {
+			log.Error(err)
+		}
+	}
 	c.Start()
 }

internal/base/cron/cron.go

@@ -70,19 +70,20 @@ func (uc *UploadController) UploadFile(ctx *gin.Context) {
 	)
 
 	source := ctx.PostForm("source")
+	userID := middleware.GetLoginUserIDFromContext(ctx)
 	switch source {
 	case fileFromAvatar:
-		url, err = uc.uploaderService.UploadAvatarFile(ctx)
+		url, err = uc.uploaderService.UploadAvatarFile(ctx, userID)
 	case fileFromPost:
-		url, err = uc.uploaderService.UploadPostFile(ctx)
+		url, err = uc.uploaderService.UploadPostFile(ctx, userID)
 	case fileFromBranding:
 		if !middleware.GetIsAdminFromContext(ctx) {
 			handler.HandleResponse(ctx, errors.Forbidden(reason.ForbiddenError), nil)
 			return
 		}
-		url, err = uc.uploaderService.UploadBrandingFile(ctx)
+		url, err = uc.uploaderService.UploadBrandingFile(ctx, userID)
 	case fileFromPostAttachment:
-		url, err = uc.uploaderService.UploadPostAttachment(ctx)
+		url, err = uc.uploaderService.UploadPostAttachment(ctx, userID)
 	default:
 		handler.HandleResponse(ctx, errors.BadRequest(reason.UploadFileSourceUnsupported), nil)
 		return

internal/controller/upload_controller.go

@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package entity
+
+import "time"
+
+const (
+	FileRecordStatusAvailable = 1
+	FileRecordStatusDeleted   = 10
+)
+
+// FileRecord file record
+type FileRecord struct {
+	ID        int       `xorm:"not null pk autoincr INT(10) id"`
+	CreatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP created TIMESTAMP created_at"`
+	UpdatedAt time.Time `xorm:"not null default CURRENT_TIMESTAMP updated TIMESTAMP updated_at"`
+	UserID    string    `xorm:"not null default 0 BIGINT(20) user_id"`
+	FilePath  string    `xorm:"not null VARCHAR(256) file_path"`
+	FileURL   string    `xorm:"not null VARCHAR(1024) file_url"`
+	ObjectID  string    `xorm:"not null default 0 INDEX BIGINT(20) object_id"`
+	Source    string    `xorm:"not null VARCHAR(128) source"`
+	Status    int       `xorm:"not null default 0 TINYINT(4) status"`
+}
+
+// TableName file record table name
+func (FileRecord) TableName() string {
+	return "file_record"
+}

internal/entity/file_record_entity.go

@@ -100,6 +100,7 @@ var migrations = []Migration{
 	NewMigration("v1.4.0", "add badge/badge_group/badge_award table", addBadges, true),
 	NewMigration("v1.4.1", "add question link", addQuestionLink, true),
 	NewMigration("v1.4.2", "add the number of question links", addQuestionLinkedCount, true),
+	NewMigration("v1.4.5", "add file record", addFileRecord, true),
 }
 
 func GetMigrations() []Migration {

internal/migrations/migrations.go

@@ -0,0 +1,31 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package migrations
+
+import (
+	"context"
+
+	"github.com/apache/answer/internal/entity"
+	"xorm.io/xorm"
+)
+
+func addFileRecord(ctx context.Context, x *xorm.Engine) error {
+	return x.Context(ctx).Sync(new(entity.FileRecord))
+}

internal/migrations/v25.go

@@ -529,7 +529,24 @@ func (ar *answerRepo) updateSearch(ctx context.Context, answerID string) (err er
 }
 
 func (ar *answerRepo) DeletePermanentlyAnswers(ctx context.Context) error {
-	_, err := ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{})
+	// get all deleted answers ids
+	ids := make([]string, 0)
+	err := ar.data.DB.Context(ctx).Select("id").Table(new(entity.Answer).TableName()).
+		Where("status = ?", entity.AnswerStatusDeleted).Find(&ids)
+	if err != nil {
+		return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+	}
+	if len(ids) == 0 {
+		return nil
+	}
+
+	// delete all revisions permanently
+	_, err = ar.data.DB.Context(ctx).In("object_id", ids).Delete(&entity.Revision{})
+	if err != nil {
+		return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
+	}
+
+	_, err = ar.data.DB.Context(ctx).Where("status = ?", entity.AnswerStatusDeleted).Delete(&entity.Answer{})
 	if err != nil {
 		return errors.InternalServer(reason.DatabaseError).WithError(err).WithStack()
 	}