Something went wrong. Try again.
Self-hosted file management and sharing system, supports multiple storage providers. forked from tdvorak.tngl.sh/cloudreve
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465package queue
import ( "context" "errors" "fmt" "runtime/debug" "sync" "sync/atomic" "time"
"github.com/cloudreve/Cloudreve/v4/ent/task" "github.com/cloudreve/Cloudreve/v4/inventory" "github.com/cloudreve/Cloudreve/v4/pkg/logging" "github.com/jpillora/backoff")
type ( Queue interface { // Start resume tasks and starts all workers. Start() // Shutdown stops all workers. Shutdown() // SubmitTask submits a Task to the queue. QueueTask(ctx context.Context, t Task) error // CancelTask marks a queued or suspending task as canceled. The // scheduler entry is left in place; workers skip canceled tasks at // pickup. Returns false if the task is not in this queue's registry. CancelTask(ctx context.Context, taskID int) bool // BusyWorkers returns the numbers of workers in the running process. BusyWorkers() int // BusyWorkers returns the numbers of success tasks. SuccessTasks() int // FailureTasks returns the numbers of failure tasks. FailureTasks() int // SubmittedTasks returns the numbers of submitted tasks. SubmittedTasks() int // SuspendingTasks returns the numbers of suspending tasks. SuspendingTasks() int } queue struct { sync.Mutex routineGroup *routineGroup metric *metric quit chan struct{} ready chan struct{} scheduler Scheduler stopOnce sync.Once stopFlag int32 rootCtx context.Context cancel context.CancelFunc
// Dependencies logger logging.Logger taskClient inventory.TaskClient dep Dep registry TaskRegistry
// Options *options }
Dep interface { ForkWithLogger(ctx context.Context, l logging.Logger) context.Context })
var ( CriticalErr = errors.New("non-retryable error"))
func New(l logging.Logger, taskClient inventory.TaskClient, registry TaskRegistry, dep Dep, opts ...Option) Queue { o := newDefaultOptions() for _, opt := range opts { opt.apply(o) }
ctx, cancel := context.WithCancel(context.Background())
return &queue{ routineGroup: newRoutineGroup(), scheduler: NewFifoScheduler(0, l), quit: make(chan struct{}), ready: make(chan struct{}, 1), metric: &metric{}, options: o, logger: l, registry: registry, taskClient: taskClient, dep: dep, rootCtx: ctx, cancel: cancel, }}
// Start to enable all workerfunc (q *queue) Start() { q.routineGroup.Run(func() { // Resume tasks in DB if len(q.options.resumeTaskType) > 0 && q.taskClient != nil {
ctx := context.TODO() ctx = context.WithValue(ctx, inventory.LoadTaskUser{}, true) ctx = context.WithValue(ctx, inventory.LoadUserGroup{}, true) tasks, err := q.taskClient.GetPendingTasks(ctx, q.resumeTaskType...) if err != nil { q.logger.Warning("Failed to get pending tasks from DB for given type %v: %s", q.resumeTaskType, err) }
resumed := 0 for _, t := range tasks { resumedTask, err := NewTaskFromModel(t) if err != nil { q.logger.Warning("Failed to resume task %d: %s", t.ID, err) continue }
if resumedTask.Status() == task.StatusSuspending { q.metric.IncSuspendingTask() q.metric.IncSubmittedTask() }
if err := q.QueueTask(ctx, resumedTask); err != nil { q.logger.Warning("Failed to resume task %d: %s", t.ID, err) } resumed++ }
q.logger.Info("Resumed %d tasks from DB.", resumed) }
q.start() }) q.logger.Info("Queue %q started with %d workers.", q.name, q.workerCount)}
// Shutdown stops all queues.func (q *queue) Shutdown() { q.logger.Info("Shutting down queue %q...", q.name) defer func() { q.routineGroup.Wait() }()
if !atomic.CompareAndSwapInt32(&q.stopFlag, 0, 1) { return }
q.stopOnce.Do(func() { q.cancel() if q.metric.BusyWorkers() > 0 { q.logger.Info("shutdown all tasks in queue %q: %d workers", q.name, q.metric.BusyWorkers()) }
if err := q.scheduler.Shutdown(); err != nil { q.logger.Error("failed to shutdown scheduler in queue %q: %w", q.name, err) } close(q.quit) })
}
// BusyWorkers returns the numbers of workers in the running process.func (q *queue) BusyWorkers() int { return int(q.metric.BusyWorkers())}
// BusyWorkers returns the numbers of success tasks.func (q *queue) SuccessTasks() int { return int(q.metric.SuccessTasks())}
// BusyWorkers returns the numbers of failure tasks.func (q *queue) FailureTasks() int { return int(q.metric.FailureTasks())}
// BusyWorkers returns the numbers of submitted tasks.func (q *queue) SubmittedTasks() int { return int(q.metric.SubmittedTasks())}
// SuspendingTasks returns the numbers of suspending tasks.func (q *queue) SuspendingTasks() int { return int(q.metric.SuspendingTasks())}
// QueueTask to queue single Taskfunc (q *queue) QueueTask(ctx context.Context, t Task) error { if atomic.LoadInt32(&q.stopFlag) == 1 { return ErrQueueShutdown }
if t.Status() != task.StatusSuspending { q.metric.IncSubmittedTask() if err := q.transitStatus(ctx, t, task.StatusQueued); err != nil { return err } }
if err := q.scheduler.Queue(t); err != nil { return err } owner := "" if t.Owner() != nil { owner = t.Owner().Email } q.logger.Info("New Task with type %q submitted to queue %q by %q", t.Type(), q.name, owner) if q.registry != nil { q.registry.Set(t.ID(), t) }
return nil}
// newContext creates a new context for a new Task iteration.func (q *queue) newContext(t Task) context.Context { l := q.logger.CopyWithPrefix(fmt.Sprintf("[Cid: %s TaskID: %d Queue: %s]", t.CorrelationID(), t.ID(), q.name)) ctx := q.dep.ForkWithLogger(q.rootCtx, l) ctx = context.WithValue(ctx, logging.CorrelationIDCtx{}, t.CorrelationID()) ctx = context.WithValue(ctx, logging.LoggerCtx{}, l) ctx = context.WithValue(ctx, inventory.UserCtx{}, t.Owner()) return ctx}
// CancelTask marks a queued or suspending task as canceled. The scheduler// heap entry is not removed; workers skip canceled tasks at pickup.func (q *queue) CancelTask(ctx context.Context, taskID int) bool { if q.registry == nil { return false } t, found := q.registry.Get(taskID) if !found { return false } if t.Status() != task.StatusQueued && t.Status() != task.StatusSuspending { return false } return q.transitStatus(ctx, t, task.StatusCanceled) == nil}
func (q *queue) work(t Task) { ctx := q.newContext(t) l := logging.FromContext(ctx) timeIterationStart := time.Now()
var err error // to handle panic cases from inside the worker // in such case, we start a new goroutine defer func() { q.metric.DecBusyWorker() e := recover() if e != nil { l.Error("Panic error in queue %q: %v", q.name, e) t.OnError(fmt.Errorf("panic error: %v", e), time.Since(timeIterationStart))
_ = q.transitStatus(ctx, t, task.StatusError) } q.schedule() }()
// Canceled while waiting in the scheduler; nothing to run. if t.Status() == task.StatusCanceled { return }
err = q.transitStatus(ctx, t, task.StatusProcessing) if err != nil { l.Error("failed to transit task %d to processing: %s", t.ID(), err.Error()) panic(err) }
for { timeIterationStart = time.Now() var next task.Status next, err = q.run(ctx, t) if err != nil { t.OnError(err, time.Since(timeIterationStart)) l.Error("runtime error in queue %q: %s", q.name, err.Error())
_ = q.transitStatus(ctx, t, task.StatusError) break }
// iteration completes t.OnIterationComplete(time.Since(timeIterationStart)) _ = q.transitStatus(ctx, t, next) if next != task.StatusProcessing { break } }}
func (q *queue) run(ctx context.Context, t Task) (task.Status, error) { l := logging.FromContext(ctx)
// create channel with buffer size 1 to avoid goroutine leak done := make(chan struct { err error next task.Status }, 1) panicChan := make(chan interface{}, 1) startTime := time.Now() ctx, cancel := context.WithTimeout(ctx, q.maxTaskExecution-t.Executed()) defer func() { cancel() }()
// run the job go func() { // handle panic issue defer func() { if p := recover(); p != nil { q.logger.Error("panic in queue %q: %s", q.name, debug.Stack()) panicChan <- p } }()
l.Debug("Iteration started.") next, err := t.Do(ctx) l.Debug("Iteration ended with err=%s", err) if err != nil && q.maxRetry-t.Retried() > 0 && !errors.Is(err, CriticalErr) && atomic.LoadInt32(&q.stopFlag) != 1 { // Retry needed t.OnRetry(err) b := &backoff.Backoff{ Max: q.backoffMaxDuration, Factor: q.backoffFactor, } delay := q.retryDelay if q.retryDelay == 0 { delay = b.ForAttempt(float64(t.Retried())) }
// Resume after to retry l.Info("Will be retried in %s", delay) t.OnSuspend(time.Now().Add(delay).Unix()) err = nil next = task.StatusSuspending }
done <- struct { err error next task.Status }{err: err, next: next} }()
select { case p := <-panicChan: panic(p) case <-ctx.Done(): // timeout reached return task.StatusError, ctx.Err() case <-q.quit: // shutdown service // cancel job cancel()
leftTime := q.maxTaskExecution - t.Executed() - time.Since(startTime) // wait job select { case <-time.After(leftTime): return task.StatusError, context.DeadlineExceeded case r := <-done: // job finish return r.next, r.err case p := <-panicChan: panic(p) } case r := <-done: // job finish return r.next, r.err }}
// beforeTaskStart updates Task status from queued to processingfunc (q *queue) transitStatus(ctx context.Context, task Task, to task.Status) (err error) { old := task.Status() transition, ok := stateTransitions[task.Status()][to] if !ok { err = fmt.Errorf("invalid state transition from %s to %s", old, to) } else { if innerErr := transition(ctx, task, to, q); innerErr != nil { err = fmt.Errorf("failed to transit Task status from %s to %s: %w", old, to, innerErr) } }
l := logging.FromContext(ctx) if err != nil { l.Error(err.Error()) }
l.Info("Task %d status changed from %q to %q.", task.ID(), old, to) return}
// schedule to check worker numberfunc (q *queue) schedule() { q.Lock() defer q.Unlock() if q.BusyWorkers() >= q.workerCount { return }
select { case q.ready <- struct{}{}: default: }}
// start to start all workerfunc (q *queue) start() { tasks := make(chan Task, 1)
for { // check worker number q.schedule()
select { // wait worker ready case <-q.ready: case <-q.quit: return }
// request Task from queue in background q.routineGroup.Run(func() { for { t, err := q.scheduler.Request() if t == nil || err != nil { if err != nil { select { case <-q.quit: if !errors.Is(err, ErrNoTaskInQueue) { close(tasks) return } case <-time.After(q.taskPullInterval): // sleep to fetch new Task } } } if t != nil { tasks <- t return }
select { case <-q.quit: if !errors.Is(err, ErrNoTaskInQueue) { close(tasks) return } default: } } })
t, ok := <-tasks if !ok { return }
// start new Task q.metric.IncBusyWorker() q.routineGroup.Run(func() { q.work(t) }) }}