Something went wrong. Try again.
Self-hosted file management and sharing system, supports multiple storage providers. forked from tdvorak.tngl.sh/cloudreve
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568package queue
import ( "context" "encoding/gob" "errors" "fmt" "sync" "time"
"github.com/cloudreve/Cloudreve/v4/ent" "github.com/cloudreve/Cloudreve/v4/ent/task" "github.com/cloudreve/Cloudreve/v4/inventory" "github.com/cloudreve/Cloudreve/v4/inventory/types" "github.com/cloudreve/Cloudreve/v4/pkg/hashid" "github.com/cloudreve/Cloudreve/v4/pkg/logging" "github.com/gofrs/uuid" "github.com/samber/lo")
type ( Task interface { Do(ctx context.Context) (task.Status, error)
// ID returns the Task ID ID() int // Type returns the Task type Type() string // Status returns the Task status Status() task.Status // Owner returns the Task owner Owner() *ent.User // State returns the internal Task state State() string // ShouldPersist returns true if the Task should be persisted into DB ShouldPersist() bool // Persisted returns true if the Task is persisted in DB Persisted() bool // Executed returns the duration of the Task execution Executed() time.Duration // Retried returns the number of times the Task has been retried Retried() int // Error returns the error of the Task Error() error // ErrorHistory returns the error history of the Task ErrorHistory() []error // Model returns the ent model of the Task Model() *ent.Task // CorrelationID returns the correlation ID of the Task CorrelationID() uuid.UUID // ResumeTime returns the time when the Task is resumed ResumeTime() int64 // ResumeAfter sets the time when the Task should be resumed ResumeAfter(next time.Duration) Progress(ctx context.Context) Progresses // Summarize returns the Task summary for UI display Summarize(hasher hashid.Encoder) *Summary // OnSuspend is called when queue decides to suspend the Task OnSuspend(time int64) // OnPersisted is called when the Task is persisted or updated in DB OnPersisted(task *ent.Task) // OnError is called when the Task encounters an error OnError(err error, d time.Duration) // OnRetry is called when the iteration returns error and before retry OnRetry(err error) // OnIterationComplete is called when the one iteration is completed OnIterationComplete(executed time.Duration) // OnStatusTransition is called when the Task status is changed OnStatusTransition(newStatus task.Status)
// Cleanup is called when the Task is done or error. Cleanup(ctx context.Context) error
Lock() Unlock() } ResumableTaskFactory func(model *ent.Task) Task Progress struct { Total int64 `json:"total"` Current int64 `json:"current"` Identifier string `json:"identifier"` } Progresses map[string]*Progress Summary struct { NodeID int `json:"-"` Phase string `json:"phase,omitempty"` Props map[string]any `json:"props,omitempty"` }
stateTransition func(ctx context.Context, task Task, newStatus task.Status, q *queue) error)
var ( taskFactories sync.Map)
const ( MediaMetaTaskType = "media_meta" EntityRecycleRoutineTaskType = "entity_recycle_routine" ExplicitEntityRecycleTaskType = "explicit_entity_recycle" UploadSentinelCheckTaskType = "upload_sentinel_check" CreateArchiveTaskType = "create_archive" ExtractArchiveTaskType = "extract_archive" RelocateTaskType = "relocate" RemoteDownloadTaskType = "remote_download" ImportTaskType = "import" BlobAuditTaskType = "blob_audit"
FullTextIndexTaskType = "full_text_index" FullTextCopyTaskType = "full_text_copy" FullTextChangeOwnerTaskType = "full_text_change_owner" FullTextDeleteTaskType = "full_text_delete" FullTextRebuildTaskType = "full_text_rebuild"
SlaveCreateArchiveTaskType = "slave_create_archive" SlaveUploadTaskType = "slave_upload" SlaveExtractArchiveType = "slave_extract_archive")
func init() { gob.Register(Progresses{})}
// RegisterResumableTaskFactory registers a resumable Task factoryfunc RegisterResumableTaskFactory(taskType string, factory ResumableTaskFactory) { taskFactories.Store(taskType, factory)}
// NewTaskFromModel creates a Task from ent.Task modelfunc NewTaskFromModel(model *ent.Task) (Task, error) { if factory, ok := taskFactories.Load(model.Type); ok { return factory.(ResumableTaskFactory)(model), nil }
return nil, fmt.Errorf("unknown Task type: %s", model.Type)}
// InMemoryTask implements part Task interface using in-memory data.type InMemoryTask struct { *DBTask}
func (i *InMemoryTask) ShouldPersist() bool { return false}
func (t *InMemoryTask) OnStatusTransition(newStatus task.Status) { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { t.Task.Status = newStatus }}
// DBTask implements Task interface related to DB schematype DBTask struct { DirectOwner *ent.User Task *ent.Task
mu sync.Mutex}
func (t *DBTask) ID() int { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { return t.Task.ID } return 0}
func (t *DBTask) Status() task.Status { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { return t.Task.Status } return ""}
func (t *DBTask) Type() string { t.mu.Lock() defer t.mu.Unlock() return t.Task.Type}
func (t *DBTask) Owner() *ent.User { t.mu.Lock() defer t.mu.Unlock()
if t.DirectOwner != nil { return t.DirectOwner } if t.Task != nil { return t.Task.Edges.User } return nil}
func (t *DBTask) State() string { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { return t.Task.PrivateState } return ""}
func (t *DBTask) Persisted() bool { t.mu.Lock() defer t.mu.Unlock()
return t.Task != nil && t.Task.ID != 0}
func (t *DBTask) Executed() time.Duration { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { return t.Task.PublicState.ExecutedDuration } return 0}
func (t *DBTask) Retried() int { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { return t.Task.PublicState.RetryCount } return 0}
func (t *DBTask) Error() error { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil && t.Task.PublicState.Error != "" { return errors.New(t.Task.PublicState.Error) }
return nil}
func (t *DBTask) ErrorHistory() []error { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { return lo.Map(t.Task.PublicState.ErrorHistory, func(err string, index int) error { return errors.New(err) }) }
return nil}
func (t *DBTask) Model() *ent.Task { t.mu.Lock() defer t.mu.Unlock() return t.Task}
func (t *DBTask) Cleanup(ctx context.Context) error { return nil}
func (t *DBTask) CorrelationID() uuid.UUID { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { return t.Task.CorrelationID } return uuid.Nil}
func (t *DBTask) ShouldPersist() bool { return true}
func (t *DBTask) OnPersisted(task *ent.Task) { t.mu.Lock() defer t.mu.Unlock()
t.Task = task}
func (t *DBTask) OnError(err error, d time.Duration) { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { t.Task.PublicState.Error = err.Error() t.Task.PublicState.ExecutedDuration += d }}
func (t *DBTask) OnRetry(err error) { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { if t.Task.PublicState.ErrorHistory == nil { t.Task.PublicState.ErrorHistory = make([]string, 0) }
t.Task.PublicState.ErrorHistory = append(t.Task.PublicState.ErrorHistory, err.Error()) t.Task.PublicState.RetryCount++ }}
func (t *DBTask) OnIterationComplete(d time.Duration) { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { t.Task.PublicState.ExecutedDuration += d }}
func (t *DBTask) ResumeTime() int64 { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { return t.Task.PublicState.ResumeTime } return 0}
func (t *DBTask) OnSuspend(time int64) { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { t.Task.PublicState.ResumeTime = time }}
func (t *DBTask) Progress(ctx context.Context) Progresses { return nil}
func (t *DBTask) OnStatusTransition(newStatus task.Status) { // Nop}
func (t *DBTask) Lock() { t.mu.Lock()}
func (t *DBTask) Unlock() { t.mu.Unlock()}
func (t *DBTask) Summarize(hasher hashid.Encoder) *Summary { return &Summary{}}
func (t *DBTask) ResumeAfter(next time.Duration) { t.mu.Lock() defer t.mu.Unlock()
if t.Task != nil { t.Task.PublicState.ResumeTime = time.Now().Add(next).Unix() }}
var stateTransitions map[task.Status]map[task.Status]stateTransition
func init() { stateTransitions = map[task.Status]map[task.Status]stateTransition{ "": { task.StatusQueued: persistTask, }, task.StatusQueued: { task.StatusProcessing: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { if err := persistTask(ctx, task, newStatus, q); err != nil { return err } return nil }, task.StatusQueued: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { return nil }, task.StatusError: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { q.metric.IncFailureTask() return persistTask(ctx, task, newStatus, q) }, task.StatusCanceled: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { if q.registry != nil { q.registry.Delete(task.ID()) } return persistTask(ctx, task, newStatus, q) }, }, task.StatusProcessing: { task.StatusQueued: persistTask, task.StatusCompleted: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { q.logger.Info("Execution completed in %s with %d retries, clean up...", task.Executed(), task.Retried()) q.metric.IncSuccessTask()
if err := task.Cleanup(ctx); err != nil { q.logger.Error("Task cleanup failed: %s", err.Error()) }
if q.registry != nil { q.registry.Delete(task.ID()) }
if err := persistTask(ctx, task, newStatus, q); err != nil { return err } return nil }, task.StatusError: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { q.logger.Error("Execution failed with error in %s with %d retries, clean up...", task.Executed(), task.Retried()) q.metric.IncFailureTask()
if err := task.Cleanup(ctx); err != nil { q.logger.Error("Task cleanup failed: %s", err.Error()) }
if q.registry != nil { q.registry.Delete(task.ID()) }
if err := persistTask(ctx, task, newStatus, q); err != nil { return err }
return nil }, task.StatusCanceled: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { q.logger.Info("Execution canceled, clean up...", task.Executed(), task.Retried()) q.metric.IncFailureTask()
if err := task.Cleanup(ctx); err != nil { q.logger.Error("Task cleanup failed: %s", err.Error()) }
if q.registry != nil { q.registry.Delete(task.ID()) }
if err := persistTask(ctx, task, newStatus, q); err != nil { return err }
return nil }, task.StatusProcessing: persistTask, task.StatusSuspending: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { q.metric.IncSuspendingTask() if err := persistTask(ctx, task, newStatus, q); err != nil { return err } q.logger.Info("Task %d suspended, resume time: %d", task.ID(), task.ResumeTime()) return q.QueueTask(ctx, task) }, }, task.StatusSuspending: { task.StatusProcessing: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { q.metric.DecSuspendingTask() return persistTask(ctx, task, newStatus, q) }, task.StatusError: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { q.metric.IncFailureTask() return persistTask(ctx, task, newStatus, q) }, task.StatusCanceled: func(ctx context.Context, task Task, newStatus task.Status, q *queue) error { q.metric.DecSuspendingTask() if q.registry != nil { q.registry.Delete(task.ID()) } return persistTask(ctx, task, newStatus, q) }, }, task.StatusError: { // Manual retry re-queues a failed task with its original args // (#2823). RetryCount history is preserved in PublicState. task.StatusQueued: persistTask, }, }
}
func persistTask(ctx context.Context, task Task, newState task.Status, q *queue) error { // Persist Task into inventory if task.ShouldPersist() { if err := saveTaskToInventory(ctx, task, newState, q); err != nil { return err } } else { task.OnStatusTransition(newState) }
return nil}
func saveTaskToInventory(ctx context.Context, task Task, newStatus task.Status, q *queue) error { var ( errStr string errHistory []string ) if err := task.Error(); err != nil { errStr = err.Error() }
errHistory = lo.Map(task.ErrorHistory(), func(err error, index int) string { return err.Error() })
args := &inventory.TaskArgs{ Status: newStatus, Type: task.Type(), PublicState: &types.TaskPublicState{ RetryCount: task.Retried(), ExecutedDuration: task.Executed(), ErrorHistory: errHistory, Error: errStr, ResumeTime: task.ResumeTime(), }, PrivateState: task.State(), OwnerID: task.Owner().ID, CorrelationID: logging.CorrelationID(ctx), // The first persistence happens on the request's context when the task // is submitted from an HTTP handler; gin.Context satisfies ClientIP. CreatorIP: clientIPFromContext(ctx), }
var ( res *ent.Task err error )
if !task.Persisted() { res, err = q.taskClient.New(ctx, args) } else { res, err = q.taskClient.Update(ctx, task.Model(), args) } if err != nil { return fmt.Errorf("failed to persist Task into DB: %w", err) }
task.OnPersisted(res) return nil}
// clientIPFromContext extracts the client IP when ctx is an HTTP request// context (gin.Context implements ClientIP()). Worker contexts return "".func clientIPFromContext(ctx context.Context) string { type clientIPer interface { ClientIP() string } if c, ok := ctx.(clientIPer); ok && c != nil { return c.ClientIP() } return ""}