tjh.dev
Linux kernel mirror (for testing)
git.kernel.org/pub/scm/linux/kernel/git/torvalds/linux.git
kernel
os
linux
drm/i915 Implement LMEM backup and restore for suspend / resume
Just evict unpinned objects to system. For pinned LMEM objects,
make a backup system object and blit the contents to that.
Backup is performed in three steps,
1: Opportunistically evict evictable objects using the gpu blitter.
2: After gt idle, evict evictable objects using the gpu blitter. This will
be modified in an upcoming patch to backup pinned objects that are not used
by the blitter itself.
3: Backup remaining pinned objects using memcpy.
Also move uC suspend to after 2) to make sure we have a functional GuC
during 2) if using GuC submission.
v2:
- Major refactor to make sure gem_exec_suspend@hang-SX subtests work, and
suspend / resume works with a slightly modified GuC submission enabling
patch series.
v3:
- Fix a potential use-after-free (Matthew Auld)
- Use i915_gem_object_create_shmem() instead of
i915_gem_object_create_region (Matthew Auld)
- Minor simplifications (Matthew Auld)
- Fix up kerneldoc for i195_ttm_restore_region().
- Final lmem_suspend() call moved to i915_gem_backup_suspend from
i915_gem_suspend_late, since the latter gets called at driver unload
and we don't unnecessarily want to run it at that time.
v4:
- Interface change of ttm- & lmem suspend / resume functions to use
flags rather than bools. (Matthew Auld)
- Completely drop the i915_gem_backup_suspend change (Matthew Auld)
Signed-off-by: Thomas Hellström <thomas.hellstrom@linux.intel.com>
Reviewed-by: Matthew Auld <matthew.auld@intel.com>
Link: https://patchwork.freedesktop.org/patch/msgid/20210922062527.865433-5-thomas.hellstrom@linux.intel.com
+353
-13
+1
drivers/gpu/drm/i915/Makefile
+1
drivers/gpu/drm/i915/Makefile
+1
drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+1
drivers/gpu/drm/i915/gem/i915_gem_object_types.h
+87
drivers/gpu/drm/i915/gem/i915_gem_pm.c
+87
drivers/gpu/drm/i915/gem/i915_gem_pm.c
···
5
5
*/
6
6
7
7
#include "gem/i915_gem_pm.h"
8
+
#include "gem/i915_gem_ttm_pm.h"
8
9
#include "gt/intel_gt.h"
9
10
#include "gt/intel_gt_pm.h"
10
11
#include "gt/intel_gt_requests.h"
···
38
37
intel_gt_suspend_prepare(&i915->gt);
39
38
40
39
i915_gem_drain_freed_objects(i915);
40
+
}
41
+
42
+
static int lmem_restore(struct drm_i915_private *i915, u32 flags)
43
+
{
44
+
struct intel_memory_region *mr;
45
+
int ret = 0, id;
46
+
47
+
for_each_memory_region(mr, i915, id) {
48
+
if (mr->type == INTEL_MEMORY_LOCAL) {
49
+
ret = i915_ttm_restore_region(mr, flags);
50
+
if (ret)
51
+
break;
52
+
}
53
+
}
54
+
55
+
return ret;
56
+
}
57
+
58
+
static int lmem_suspend(struct drm_i915_private *i915, u32 flags)
59
+
{
60
+
struct intel_memory_region *mr;
61
+
int ret = 0, id;
62
+
63
+
for_each_memory_region(mr, i915, id) {
64
+
if (mr->type == INTEL_MEMORY_LOCAL) {
65
+
ret = i915_ttm_backup_region(mr, flags);
66
+
if (ret)
67
+
break;
68
+
}
69
+
}
70
+
71
+
return ret;
72
+
}
73
+
74
+
static void lmem_recover(struct drm_i915_private *i915)
75
+
{
76
+
struct intel_memory_region *mr;
77
+
int id;
78
+
79
+
for_each_memory_region(mr, i915, id)
80
+
if (mr->type == INTEL_MEMORY_LOCAL)
81
+
i915_ttm_recover_region(mr);
82
+
}
83
+
84
+
int i915_gem_backup_suspend(struct drm_i915_private *i915)
85
+
{
86
+
int ret;
87
+
88
+
/* Opportunistically try to evict unpinned objects */
89
+
ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
90
+
if (ret)
91
+
goto out_recover;
92
+
93
+
i915_gem_suspend(i915);
94
+
95
+
/*
96
+
* More objects may have become unpinned as requests were
97
+
* retired. Now try to evict again. The gt may be wedged here
98
+
* in which case we automatically fall back to memcpy.
99
+
*/
100
+
ret = lmem_suspend(i915, I915_TTM_BACKUP_ALLOW_GPU);
101
+
if (ret)
102
+
goto out_recover;
103
+
104
+
/*
105
+
* Remaining objects are backed up using memcpy once we've stopped
106
+
* using the migrate context.
107
+
*/
108
+
ret = lmem_suspend(i915, I915_TTM_BACKUP_PINNED);
109
+
if (ret)
110
+
goto out_recover;
111
+
112
+
return 0;
113
+
114
+
out_recover:
115
+
lmem_recover(i915);
116
+
117
+
return ret;
41
118
}
42
119
43
120
void i915_gem_suspend_late(struct drm_i915_private *i915)
···
207
128
208
129
void i915_gem_resume(struct drm_i915_private *i915)
209
130
{
131
+
int ret;
132
+
210
133
GEM_TRACE("%s\n", dev_name(i915->drm.dev));
134
+
135
+
ret = lmem_restore(i915, 0);
136
+
GEM_WARN_ON(ret);
211
137
212
138
/*
213
139
* As we didn't flush the kernel context before suspend, we cannot
···
220
136
* it and start again.
221
137
*/
222
138
intel_gt_resume(&i915->gt);
139
+
140
+
ret = lmem_restore(i915, I915_TTM_BACKUP_ALLOW_GPU);
141
+
GEM_WARN_ON(ret);
223
142
}
+1
drivers/gpu/drm/i915/gem/i915_gem_pm.h
+1
drivers/gpu/drm/i915/gem/i915_gem_pm.h
···
18
18
19
19
void i915_gem_suspend(struct drm_i915_private *i915);
20
20
void i915_gem_suspend_late(struct drm_i915_private *i915);
21
+
int i915_gem_backup_suspend(struct drm_i915_private *i915);
21
22
22
23
int i915_gem_freeze(struct drm_i915_private *i915);
23
24
int i915_gem_freeze_late(struct drm_i915_private *i915);
+22
-8
drivers/gpu/drm/i915/gem/i915_gem_ttm.c
+22
-8
drivers/gpu/drm/i915/gem/i915_gem_ttm.c
···
10
10
#include "intel_memory_region.h"
11
11
#include "intel_region_ttm.h"
12
12
13
+
#include "gem/i915_gem_mman.h"
13
14
#include "gem/i915_gem_object.h"
14
15
#include "gem/i915_gem_region.h"
15
16
#include "gem/i915_gem_ttm.h"
16
-
#include "gem/i915_gem_mman.h"
17
+
#include "gem/i915_gem_ttm_pm.h"
17
18
18
-
#include "gt/intel_migrate.h"
19
+
19
20
#include "gt/intel_engine_pm.h"
20
-
21
-
#define I915_PL_LMEM0 TTM_PL_PRIV
22
-
#define I915_PL_SYSTEM TTM_PL_SYSTEM
23
-
#define I915_PL_STOLEN TTM_PL_VRAM
24
-
#define I915_PL_GGTT TTM_PL_TT
21
+
#include "gt/intel_gt.h"
22
+
#include "gt/intel_migrate.h"
25
23
26
24
#define I915_TTM_PRIO_PURGE 0
27
25
#define I915_TTM_PRIO_NO_PAGES 1
···
61
63
.num_busy_placement = 1,
62
64
.busy_placement = &sys_placement_flags,
63
65
};
66
+
67
+
/**
68
+
* i915_ttm_sys_placement - Return the struct ttm_placement to be
69
+
* used for an object in system memory.
70
+
*
71
+
* Rather than making the struct extern, use this
72
+
* function.
73
+
*
74
+
* Return: A pointer to a static variable for sys placement.
75
+
*/
76
+
struct ttm_placement *i915_ttm_sys_placement(void)
77
+
{
78
+
return &i915_sys_placement;
79
+
}
64
80
65
81
static int i915_ttm_err_to_gem(int err)
66
82
{
···
455
443
enum i915_cache_level src_level, dst_level;
456
444
int ret;
457
445
458
-
if (!i915->gt.migrate.context)
446
+
if (!i915->gt.migrate.context || intel_gt_is_wedged(&i915->gt))
459
447
return -EINVAL;
460
448
461
449
dst_level = i915_ttm_cache_level(i915, dst_mem, dst_ttm);
···
898
886
void i915_ttm_bo_destroy(struct ttm_buffer_object *bo)
899
887
{
900
888
struct drm_i915_gem_object *obj = i915_ttm_to_gem(bo);
889
+
890
+
i915_ttm_backup_free(obj);
901
891
902
892
/* This releases all gem object bindings to the backend. */
903
893
__i915_gem_free_object(obj);
+10
drivers/gpu/drm/i915/gem/i915_gem_ttm.h
+10
drivers/gpu/drm/i915/gem/i915_gem_ttm.h
···
50
50
int i915_gem_obj_copy_ttm(struct drm_i915_gem_object *dst,
51
51
struct drm_i915_gem_object *src,
52
52
bool allow_accel, bool intr);
53
+
54
+
/* Internal I915 TTM declarations and definitions below. */
55
+
56
+
#define I915_PL_LMEM0 TTM_PL_PRIV
57
+
#define I915_PL_SYSTEM TTM_PL_SYSTEM
58
+
#define I915_PL_STOLEN TTM_PL_VRAM
59
+
#define I915_PL_GGTT TTM_PL_TT
60
+
61
+
struct ttm_placement *i915_ttm_sys_placement(void);
62
+
53
63
#endif
+202
drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c
+202
drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.c
···
1
+
// SPDX-License-Identifier: MIT
2
+
/*
3
+
* Copyright © 2021 Intel Corporation
4
+
*/
5
+
6
+
#include <drm/ttm/ttm_placement.h>
7
+
#include <drm/ttm/ttm_tt.h>
8
+
9
+
#include "i915_drv.h"
10
+
#include "intel_memory_region.h"
11
+
#include "intel_region_ttm.h"
12
+
13
+
#include "gem/i915_gem_region.h"
14
+
#include "gem/i915_gem_ttm.h"
15
+
#include "gem/i915_gem_ttm_pm.h"
16
+
17
+
/**
18
+
* i915_ttm_backup_free - Free any backup attached to this object
19
+
* @obj: The object whose backup is to be freed.
20
+
*/
21
+
void i915_ttm_backup_free(struct drm_i915_gem_object *obj)
22
+
{
23
+
if (obj->ttm.backup) {
24
+
i915_gem_object_put(obj->ttm.backup);
25
+
obj->ttm.backup = NULL;
26
+
}
27
+
}
28
+
29
+
/**
30
+
* struct i915_gem_ttm_pm_apply - Apply-to-region subclass for restore
31
+
* @base: The i915_gem_apply_to_region we derive from.
32
+
* @allow_gpu: Whether using the gpu blitter is allowed.
33
+
* @backup_pinned: On backup, backup also pinned objects.
34
+
*/
35
+
struct i915_gem_ttm_pm_apply {
36
+
struct i915_gem_apply_to_region base;
37
+
bool allow_gpu : 1;
38
+
bool backup_pinned : 1;
39
+
};
40
+
41
+
static int i915_ttm_backup(struct i915_gem_apply_to_region *apply,
42
+
struct drm_i915_gem_object *obj)
43
+
{
44
+
struct i915_gem_ttm_pm_apply *pm_apply =
45
+
container_of(apply, typeof(*pm_apply), base);
46
+
struct ttm_buffer_object *bo = i915_gem_to_ttm(obj);
47
+
struct ttm_buffer_object *backup_bo;
48
+
struct drm_i915_private *i915 =
49
+
container_of(bo->bdev, typeof(*i915), bdev);
50
+
struct drm_i915_gem_object *backup;
51
+
struct ttm_operation_ctx ctx = {};
52
+
int err = 0;
53
+
54
+
if (bo->resource->mem_type == I915_PL_SYSTEM || obj->ttm.backup)
55
+
return 0;
56
+
57
+
if (pm_apply->allow_gpu && i915_gem_object_evictable(obj))
58
+
return ttm_bo_validate(bo, i915_ttm_sys_placement(), &ctx);
59
+
60
+
if (!pm_apply->backup_pinned)
61
+
return 0;
62
+
63
+
backup = i915_gem_object_create_shmem(i915, obj->base.size);
64
+
if (IS_ERR(backup))
65
+
return PTR_ERR(backup);
66
+
67
+
err = i915_gem_object_lock(backup, apply->ww);
68
+
if (err)
69
+
goto out_no_lock;
70
+
71
+
backup_bo = i915_gem_to_ttm(backup);
72
+
err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx);
73
+
if (err)
74
+
goto out_no_populate;
75
+
76
+
err = i915_gem_obj_copy_ttm(backup, obj, pm_apply->allow_gpu, false);
77
+
GEM_WARN_ON(err);
78
+
79
+
obj->ttm.backup = backup;
80
+
return 0;
81
+
82
+
out_no_populate:
83
+
i915_gem_ww_unlock_single(backup);
84
+
out_no_lock:
85
+
i915_gem_object_put(backup);
86
+
87
+
return err;
88
+
}
89
+
90
+
static int i915_ttm_recover(struct i915_gem_apply_to_region *apply,
91
+
struct drm_i915_gem_object *obj)
92
+
{
93
+
i915_ttm_backup_free(obj);
94
+
return 0;
95
+
}
96
+
97
+
/**
98
+
* i915_ttm_recover_region - Free the backup of all objects of a region
99
+
* @mr: The memory region
100
+
*
101
+
* Checks all objects of a region if there is backup attached and if so
102
+
* frees that backup. Typically this is called to recover after a partially
103
+
* performed backup.
104
+
*/
105
+
void i915_ttm_recover_region(struct intel_memory_region *mr)
106
+
{
107
+
static const struct i915_gem_apply_to_region_ops recover_ops = {
108
+
.process_obj = i915_ttm_recover,
109
+
};
110
+
struct i915_gem_apply_to_region apply = {.ops = &recover_ops};
111
+
int ret;
112
+
113
+
ret = i915_gem_process_region(mr, &apply);
114
+
GEM_WARN_ON(ret);
115
+
}
116
+
117
+
/**
118
+
* i915_ttm_backup_region - Back up all objects of a region to smem.
119
+
* @mr: The memory region
120
+
* @allow_gpu: Whether to allow the gpu blitter for this backup.
121
+
* @backup_pinned: Backup also pinned objects.
122
+
*
123
+
* Loops over all objects of a region and either evicts them if they are
124
+
* evictable or backs them up using a backup object if they are pinned.
125
+
*
126
+
* Return: Zero on success. Negative error code on error.
127
+
*/
128
+
int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags)
129
+
{
130
+
static const struct i915_gem_apply_to_region_ops backup_ops = {
131
+
.process_obj = i915_ttm_backup,
132
+
};
133
+
struct i915_gem_ttm_pm_apply pm_apply = {
134
+
.base = {.ops = &backup_ops},
135
+
.allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
136
+
.backup_pinned = flags & I915_TTM_BACKUP_PINNED,
137
+
};
138
+
139
+
return i915_gem_process_region(mr, &pm_apply.base);
140
+
}
141
+
142
+
static int i915_ttm_restore(struct i915_gem_apply_to_region *apply,
143
+
struct drm_i915_gem_object *obj)
144
+
{
145
+
struct i915_gem_ttm_pm_apply *pm_apply =
146
+
container_of(apply, typeof(*pm_apply), base);
147
+
struct drm_i915_gem_object *backup = obj->ttm.backup;
148
+
struct ttm_buffer_object *backup_bo = i915_gem_to_ttm(backup);
149
+
struct ttm_operation_ctx ctx = {};
150
+
int err;
151
+
152
+
if (!backup)
153
+
return 0;
154
+
155
+
if (!pm_apply->allow_gpu && (obj->flags & I915_BO_ALLOC_USER))
156
+
return 0;
157
+
158
+
err = i915_gem_object_lock(backup, apply->ww);
159
+
if (err)
160
+
return err;
161
+
162
+
/* Content may have been swapped. */
163
+
err = ttm_tt_populate(backup_bo->bdev, backup_bo->ttm, &ctx);
164
+
if (!err) {
165
+
err = i915_gem_obj_copy_ttm(obj, backup, pm_apply->allow_gpu,
166
+
false);
167
+
GEM_WARN_ON(err);
168
+
169
+
obj->ttm.backup = NULL;
170
+
err = 0;
171
+
}
172
+
173
+
i915_gem_ww_unlock_single(backup);
174
+
175
+
if (!err)
176
+
i915_gem_object_put(backup);
177
+
178
+
return err;
179
+
}
180
+
181
+
/**
182
+
* i915_ttm_restore_region - Restore backed-up objects of a region from smem.
183
+
* @mr: The memory region
184
+
* @allow_gpu: Whether to allow the gpu blitter to recover.
185
+
*
186
+
* Loops over all objects of a region and if they are backed-up, restores
187
+
* them from smem.
188
+
*
189
+
* Return: Zero on success. Negative error code on error.
190
+
*/
191
+
int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags)
192
+
{
193
+
static const struct i915_gem_apply_to_region_ops restore_ops = {
194
+
.process_obj = i915_ttm_restore,
195
+
};
196
+
struct i915_gem_ttm_pm_apply pm_apply = {
197
+
.base = {.ops = &restore_ops},
198
+
.allow_gpu = flags & I915_TTM_BACKUP_ALLOW_GPU,
199
+
};
200
+
201
+
return i915_gem_process_region(mr, &pm_apply.base);
202
+
}
+26
drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.h
+26
drivers/gpu/drm/i915/gem/i915_gem_ttm_pm.h
···
1
+
/* SPDX-License-Identifier: MIT */
2
+
/*
3
+
* Copyright © 2021 Intel Corporation
4
+
*/
5
+
6
+
#ifndef _I915_GEM_TTM_PM_H_
7
+
#define _I915_GEM_TTM_PM_H_
8
+
9
+
#include <linux/types.h>
10
+
11
+
struct intel_memory_region;
12
+
struct drm_i915_gem_object;
13
+
14
+
#define I915_TTM_BACKUP_ALLOW_GPU BIT(0)
15
+
#define I915_TTM_BACKUP_PINNED BIT(1)
16
+
17
+
int i915_ttm_backup_region(struct intel_memory_region *mr, u32 flags);
18
+
19
+
void i915_ttm_recover_region(struct intel_memory_region *mr);
20
+
21
+
int i915_ttm_restore_region(struct intel_memory_region *mr, u32 flags);
22
+
23
+
/* Internal I915 TTM functions below. */
24
+
void i915_ttm_backup_free(struct drm_i915_gem_object *obj);
25
+
26
+
#endif
+2
-2
drivers/gpu/drm/i915/gt/intel_gt_pm.c
+2
-2
drivers/gpu/drm/i915/gt/intel_gt_pm.c
···
297
297
{
298
298
user_forcewake(gt, true);
299
299
wait_for_suspend(gt);
300
-
301
-
intel_uc_suspend(>->uc);
302
300
}
303
301
304
302
static suspend_state_t pm_suspend_target(void)
···
319
321
return;
320
322
321
323
GEM_BUG_ON(gt->awake);
324
+
325
+
intel_uc_suspend(>->uc);
322
326
323
327
/*
324
328
* On disabling the device, we want to turn off HW access to memory
+1
-3
drivers/gpu/drm/i915/i915_drv.c
+1
-3
drivers/gpu/drm/i915/i915_drv.c
···
1094
1094
* split out that work and pull it forward so that after point,
1095
1095
* the GPU is not woken again.
1096
1096
*/
1097
-
i915_gem_suspend(i915);
1098
-
1099
-
return 0;
1097
+
return i915_gem_backup_suspend(i915);
1100
1098
}
1101
1099
1102
1100
static int i915_drm_suspend(struct drm_device *dev)