A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
1// SiYuan - Refactor your thinking
2// Copyright (c) 2020-present, b3log.org
3//
4// This program is free software: you can redistribute it and/or modify
5// it under the terms of the GNU Affero General Public License as published by
6// the Free Software Foundation, either version 3 of the License, or
7// (at your option) any later version.
8//
9// This program is distributed in the hope that it will be useful,
10// but WITHOUT ANY WARRANTY; without even the implied warranty of
11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12// GNU Affero General Public License for more details.
13//
14// You should have received a copy of the GNU Affero General Public License
15// along with this program. If not, see <https://www.gnu.org/licenses/>.
16
17package model
18
19import (
20 "bytes"
21 "errors"
22 "fmt"
23 "io/fs"
24 "mime"
25 "net/http"
26 "net/url"
27 "os"
28 "path"
29 "path/filepath"
30 "sort"
31 "strings"
32 "time"
33
34 "github.com/88250/go-humanize"
35 "github.com/88250/gulu"
36 "github.com/88250/lute/ast"
37 "github.com/88250/lute/editor"
38 "github.com/88250/lute/html"
39 "github.com/88250/lute/parse"
40 "github.com/disintegration/imaging"
41 "github.com/gabriel-vasile/mimetype"
42 "github.com/siyuan-note/filelock"
43 "github.com/siyuan-note/httpclient"
44 "github.com/siyuan-note/logging"
45 "github.com/siyuan-note/siyuan/kernel/av"
46 "github.com/siyuan-note/siyuan/kernel/cache"
47 "github.com/siyuan-note/siyuan/kernel/filesys"
48 "github.com/siyuan-note/siyuan/kernel/search"
49 "github.com/siyuan-note/siyuan/kernel/sql"
50 "github.com/siyuan-note/siyuan/kernel/treenode"
51 "github.com/siyuan-note/siyuan/kernel/util"
52)
53
54func GetAssetPathByHash(hash string) string {
55 assetHash := cache.GetAssetHash(hash)
56 if nil == assetHash {
57 sqlAsset := sql.QueryAssetByHash(hash)
58 if nil == sqlAsset {
59 return ""
60 }
61 cache.SetAssetHash(sqlAsset.Hash, sqlAsset.Path)
62 return sqlAsset.Path
63 }
64 return assetHash.Path
65}
66
67func HandleAssetsRemoveEvent(assetAbsPath string) {
68 removeIndexAssetContent(assetAbsPath)
69 removeAssetThumbnail(assetAbsPath)
70}
71
72func HandleAssetsChangeEvent(assetAbsPath string) {
73 indexAssetContent(assetAbsPath)
74 removeAssetThumbnail(assetAbsPath)
75}
76
77func removeAssetThumbnail(assetAbsPath string) {
78 if util.IsCompressibleAssetImage(assetAbsPath) {
79 p := filepath.ToSlash(assetAbsPath)
80 idx := strings.Index(p, "assets/")
81 if -1 == idx {
82 return
83 }
84 thumbnailPath := filepath.Join(util.TempDir, "thumbnails", "assets", p[idx+7:])
85 os.RemoveAll(thumbnailPath)
86 }
87}
88
89func NeedGenerateAssetsThumbnail(sourceImgPath string) bool {
90 info, err := os.Stat(sourceImgPath)
91 if err != nil {
92 return false
93 }
94 if info.IsDir() {
95 return false
96 }
97 return info.Size() > 1024*1024
98}
99
100func GenerateAssetsThumbnail(sourceImgPath, resizedImgPath string) (err error) {
101 start := time.Now()
102 img, err := imaging.Open(sourceImgPath)
103 if err != nil {
104 return
105 }
106
107 // 获取原图宽高
108 originalWidth := img.Bounds().Dx()
109 originalHeight := img.Bounds().Dy()
110
111 // 固定最大宽度为 520,计算缩放比例
112 maxWidth := 520
113 scale := float64(maxWidth) / float64(originalWidth)
114
115 // 按比例计算新的宽高
116 newWidth := maxWidth
117 newHeight := int(float64(originalHeight) * scale)
118
119 // 缩放图片
120 resizedImg := imaging.Resize(img, newWidth, newHeight, imaging.Lanczos)
121
122 // 保存缩放后的图片
123 err = os.MkdirAll(filepath.Dir(resizedImgPath), 0755)
124 if err != nil {
125 return
126 }
127 err = imaging.Save(resizedImg, resizedImgPath)
128 if err != nil {
129 return
130 }
131 logging.LogDebugf("generated thumbnail image [%s] to [%s], cost [%d]ms", sourceImgPath, resizedImgPath, time.Since(start).Milliseconds())
132 return
133}
134
135func DocImageAssets(rootID string) (ret []string, err error) {
136 tree, err := LoadTreeByBlockID(rootID)
137 if err != nil {
138 return
139 }
140
141 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
142 if !entering {
143 return ast.WalkContinue
144 }
145 if ast.NodeImage == n.Type {
146 linkDest := n.ChildByType(ast.NodeLinkDest)
147 dest := linkDest.Tokens
148 if 1 > len(dest) { // 双击打开图片不对 https://github.com/siyuan-note/siyuan/issues/5876
149 return ast.WalkContinue
150 }
151 ret = append(ret, gulu.Str.FromBytes(dest))
152 }
153 return ast.WalkContinue
154 })
155 return
156}
157
158func DocAssets(rootID string) (ret []string, err error) {
159 tree, err := LoadTreeByBlockID(rootID)
160 if err != nil {
161 return
162 }
163
164 ret = getAssetsLinkDests(tree.Root)
165 return
166}
167
168func NetAssets2LocalAssets(rootID string, onlyImg bool, originalURL string) (err error) {
169 tree, err := LoadTreeByBlockID(rootID)
170 if err != nil {
171 return
172 }
173
174 var files int
175 msgId := gulu.Rand.String(7)
176
177 docDirLocalPath := filepath.Join(util.DataDir, tree.Box, path.Dir(tree.Path))
178 assetsDirPath := getAssetsDir(filepath.Join(util.DataDir, tree.Box), docDirLocalPath)
179 if !gulu.File.IsExist(assetsDirPath) {
180 if err = os.MkdirAll(assetsDirPath, 0755); err != nil {
181 return
182 }
183 }
184
185 browserClient := util.NewCustomReqClient() // 自定义了 TLS 指纹,增加下载成功率
186
187 forbiddenCount := 0
188 destNodes := getRemoteAssetsLinkDestsInTree(tree, onlyImg)
189 for _, destNode := range destNodes {
190 dests := getRemoteAssetsLinkDests(destNode, onlyImg)
191 if 1 > len(dests) {
192 continue
193 }
194
195 for _, dest := range dests {
196 if strings.HasPrefix(strings.ToLower(dest), "file://") { // 处理本地文件链接
197 u := dest[7:]
198 unescaped, _ := url.PathUnescape(u)
199 if unescaped != u {
200 // `Convert network images/assets to local` supports URL-encoded local file names https://github.com/siyuan-note/siyuan/issues/9929
201 u = unescaped
202 }
203 if strings.Contains(u, ":") {
204 u = strings.TrimPrefix(u, "/")
205 }
206 if strings.Contains(u, "?") {
207 u = u[:strings.Index(u, "?")]
208 }
209
210 if !gulu.File.IsExist(u) || gulu.File.IsDir(u) {
211 continue
212 }
213
214 name := filepath.Base(u)
215 name = util.FilterUploadFileName(name)
216 name = "network-asset-" + name
217 name = util.AssetName(name, ast.NewNodeID())
218 writePath := filepath.Join(assetsDirPath, name)
219 if err = filelock.Copy(u, writePath); err != nil {
220 logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err)
221 continue
222 }
223
224 setAssetsLinkDest(destNode, dest, "assets/"+name)
225 files++
226 continue
227 }
228
229 if strings.HasPrefix(strings.ToLower(dest), "https://") || strings.HasPrefix(strings.ToLower(dest), "http://") || strings.HasPrefix(dest, "//") {
230 if strings.HasPrefix(dest, "//") {
231 // `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598
232 dest = "https:" + dest
233 }
234
235 u := dest
236 if strings.Contains(u, "qpic.cn") {
237 // 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052
238 if strings.Contains(u, "http://") {
239 u = strings.Replace(u, "http://", "https://", 1)
240 }
241
242 // 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431
243 // 下面这部分需要注释掉,否则会导致响应 400
244 //if strings.HasSuffix(u, "/0") {
245 // u = strings.Replace(u, "/0", "/640", 1)
246 //} else if strings.Contains(u, "/0?") {
247 // u = strings.Replace(u, "/0?", "/640?", 1)
248 //}
249 }
250
251 displayU := u
252 if 64 < len(displayU) {
253 displayU = displayU[:64] + "..."
254 }
255 util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), displayU), 15000)
256 request := browserClient.R()
257 request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second)
258 if "" != originalURL {
259 request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464
260 }
261 resp, reqErr := request.Get(u)
262 if nil != reqErr {
263 logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr)
264 continue
265 }
266 if http.StatusForbidden == resp.StatusCode || http.StatusUnauthorized == resp.StatusCode {
267 forbiddenCount++
268 }
269 if strings.Contains(strings.ToLower(resp.GetContentType()), "text/html") {
270 // 忽略超链接网页 `Convert network assets to local` no longer process webpage https://github.com/siyuan-note/siyuan/issues/9965
271 continue
272 }
273 if 200 != resp.StatusCode {
274 logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode)
275 continue
276 }
277
278 if 1024*1024*96 < resp.ContentLength {
279 logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength)))
280 continue
281 }
282
283 data, repErr := resp.ToBytes()
284 if nil != repErr {
285 logging.LogErrorf("download network asset [%s] failed: %s", u, repErr)
286 continue
287 }
288 var name string
289 if strings.Contains(u, "?") {
290 name = u[:strings.Index(u, "?")]
291 name = path.Base(name)
292 } else {
293 name = path.Base(u)
294 }
295 if strings.Contains(name, "#") {
296 name = name[:strings.Index(name, "#")]
297 }
298 name, _ = url.PathUnescape(name)
299 name = util.FilterUploadFileName(name)
300 ext := util.Ext(name)
301 if !util.IsCommonExt(ext) {
302 if mtype := mimetype.Detect(data); nil != mtype {
303 ext = mtype.Extension()
304 name += ext
305 }
306 }
307 if "" == ext && bytes.HasPrefix(data, []byte("<svg ")) && bytes.HasSuffix(data, []byte("</svg>")) {
308 ext = ".svg"
309 name += ext
310 }
311 if "" == ext {
312 contentType := resp.Header.Get("Content-Type")
313 exts, _ := mime.ExtensionsByType(contentType)
314 if 0 < len(exts) {
315 ext = exts[0]
316 name += ext
317 }
318 }
319 name = util.AssetName(name, ast.NewNodeID())
320 name = "network-asset-" + name
321 writePath := filepath.Join(assetsDirPath, name)
322 if err = filelock.WriteFile(writePath, data); err != nil {
323 logging.LogErrorf("write downloaded network asset [%s] to local asset [%s] failed: %s", u, writePath, err)
324 continue
325 }
326
327 setAssetsLinkDest(destNode, dest, "assets/"+name)
328 files++
329 continue
330 }
331 }
332 }
333
334 util.PushClearMsg(msgId)
335 if 0 < files {
336 msgId = util.PushMsg(Conf.Language(113), 7000)
337 if err = writeTreeUpsertQueue(tree); err != nil {
338 return
339 }
340 util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files), 5000)
341
342 if 0 < forbiddenCount {
343 util.PushErrMsg(fmt.Sprintf(Conf.Language(255), forbiddenCount), 5000)
344 }
345 } else {
346 if 0 < forbiddenCount {
347 util.PushErrMsg(fmt.Sprintf(Conf.Language(255), forbiddenCount), 5000)
348 } else {
349 util.PushMsg(Conf.Language(121), 3000)
350 }
351 }
352 return
353}
354
355func SearchAssetsByName(keyword string, exts []string) (ret []*cache.Asset) {
356 ret = []*cache.Asset{}
357 var keywords []string
358 keywords = append(keywords, keyword)
359 if "" != keyword {
360 keywords = append(keywords, strings.Split(keyword, " ")...)
361 }
362 pathHitCount := map[string]int{}
363 filterByExt := 0 < len(exts)
364 for _, asset := range cache.GetAssets() {
365 if filterByExt {
366 ext := filepath.Ext(asset.HName)
367 includeExt := false
368 for _, e := range exts {
369 if strings.ToLower(ext) == strings.ToLower(e) {
370 includeExt = true
371 break
372 }
373 }
374 if !includeExt {
375 continue
376 }
377 }
378
379 lowerHName := strings.ToLower(asset.HName)
380 lowerPath := strings.ToLower(asset.Path)
381 var hitNameCount, hitPathCount int
382 for i, k := range keywords {
383 lowerKeyword := strings.ToLower(k)
384 if 0 == i {
385 // 第一个是完全匹配,权重最高
386 if strings.Contains(lowerHName, lowerKeyword) {
387 hitNameCount += 64
388 }
389 if strings.Contains(lowerPath, lowerKeyword) {
390 hitPathCount += 64
391 }
392 }
393
394 hitNameCount += strings.Count(lowerHName, lowerKeyword)
395 hitPathCount += strings.Count(lowerPath, lowerKeyword)
396 if 1 > hitNameCount && 1 > hitPathCount {
397 continue
398 }
399 }
400
401 if 1 > hitNameCount+hitPathCount {
402 continue
403 }
404 pathHitCount[asset.Path] += hitNameCount + hitPathCount
405
406 hName := asset.HName
407 if 0 < hitNameCount {
408 _, hName = search.MarkText(asset.HName, strings.Join(keywords, search.TermSep), 64, Conf.Search.CaseSensitive)
409 }
410 ret = append(ret, &cache.Asset{
411 HName: hName,
412 Path: asset.Path,
413 Updated: asset.Updated,
414 })
415 }
416
417 if 0 < len(pathHitCount) {
418 sort.Slice(ret, func(i, j int) bool {
419 return pathHitCount[ret[i].Path] > pathHitCount[ret[j].Path]
420 })
421 } else {
422 sort.Slice(ret, func(i, j int) bool {
423 return ret[i].Updated > ret[j].Updated
424 })
425 }
426
427 if Conf.Search.Limit <= len(ret) {
428 ret = ret[:Conf.Search.Limit]
429 }
430 return
431}
432
433func GetAssetAbsPath(relativePath string) (ret string, err error) {
434 relativePath = strings.TrimSpace(relativePath)
435 if strings.Contains(relativePath, "?") {
436 relativePath = relativePath[:strings.Index(relativePath, "?")]
437 }
438
439 // 在全局 assets 路径下搜索
440 p := filepath.Join(util.DataDir, relativePath)
441 if gulu.File.IsExist(p) {
442 ret = p
443 if !util.IsSubPath(util.WorkspaceDir, ret) {
444 err = fmt.Errorf("[%s] is not sub path of workspace", ret)
445 return
446 }
447 return
448 }
449
450 // 在笔记本下搜索
451 notebooks, err := ListNotebooks()
452 if err != nil {
453 err = errors.New(Conf.Language(0))
454 return
455 }
456 for _, notebook := range notebooks {
457 notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
458 filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error {
459 if isSkipFile(d.Name()) {
460 if d.IsDir() {
461 return filepath.SkipDir
462 }
463 return nil
464 }
465 if p := filepath.ToSlash(path); strings.HasSuffix(p, relativePath) {
466 if gulu.File.IsExist(path) {
467 ret = path
468 return fs.SkipAll
469 }
470 }
471 return nil
472 })
473
474 if "" != ret {
475 if !util.IsSubPath(util.WorkspaceDir, ret) {
476 err = fmt.Errorf("[%s] is not sub path of workspace", ret)
477 return
478 }
479 return
480 }
481 }
482 return "", errors.New(fmt.Sprintf(Conf.Language(12), relativePath))
483}
484
485func UploadAssets2Cloud(id string) (count int, err error) {
486 if !IsSubscriber() {
487 return
488 }
489
490 tree, err := LoadTreeByBlockID(id)
491 if err != nil {
492 return
493 }
494
495 node := treenode.GetNodeInTree(tree, id)
496 if nil == node {
497 err = ErrBlockNotFound
498 return
499 }
500
501 nodes := []*ast.Node{node}
502 if ast.NodeHeading == node.Type {
503 nodes = append(nodes, treenode.HeadingChildren(node)...)
504 }
505
506 var assets []string
507 for _, n := range nodes {
508 assets = append(assets, getAssetsLinkDests(n)...)
509 assets = append(assets, getQueryEmbedNodesAssetsLinkDests(n)...)
510 }
511 assets = gulu.Str.RemoveDuplicatedElem(assets)
512 count, err = uploadAssets2Cloud(assets, bizTypeUploadAssets)
513 if err != nil {
514 return
515 }
516 return
517}
518
519const (
520 bizTypeUploadAssets = "upload-assets"
521 bizTypeExport2Liandi = "export-liandi"
522)
523
524// uploadAssets2Cloud 将资源文件上传到云端图床。
525func uploadAssets2Cloud(assetPaths []string, bizType string) (count int, err error) {
526 var uploadAbsAssets []string
527 for _, assetPath := range assetPaths {
528 var absPath string
529 absPath, err = GetAssetAbsPath(assetPath)
530 if err != nil {
531 logging.LogWarnf("get asset [%s] abs path failed: %s", assetPath, err)
532 return
533 }
534 if "" == absPath {
535 logging.LogErrorf("not found asset [%s]", assetPath)
536 continue
537 }
538
539 uploadAbsAssets = append(uploadAbsAssets, absPath)
540 }
541
542 uploadAbsAssets = gulu.Str.RemoveDuplicatedElem(uploadAbsAssets)
543 if 1 > len(uploadAbsAssets) {
544 return
545 }
546
547 logging.LogInfof("uploading [%d] assets", len(uploadAbsAssets))
548 msgId := util.PushMsg(fmt.Sprintf(Conf.Language(27), len(uploadAbsAssets)), 3000)
549 if loadErr := LoadUploadToken(); nil != loadErr {
550 util.PushMsg(loadErr.Error(), 5000)
551 return
552 }
553
554 limitSize := uint64(3 * 1024 * 1024) // 3MB
555 if IsSubscriber() {
556 limitSize = 10 * 1024 * 1024 // 10MB
557 }
558
559 // metaType 为服务端 Filemeta.FILEMETA_TYPE,这里只有两个值:
560 //
561 // 5: SiYuan,表示为 SiYuan 上传图床
562 // 4: Client,表示作为客户端分享发布帖子时上传的文件
563 var metaType = "5"
564 if bizTypeUploadAssets == bizType {
565 metaType = "5"
566 } else if bizTypeExport2Liandi == bizType {
567 metaType = "4"
568 }
569
570 pushErrMsgCount := 0
571 var completedUploadAssets []string
572 for _, absAsset := range uploadAbsAssets {
573 fi, statErr := os.Stat(absAsset)
574 if nil != statErr {
575 logging.LogErrorf("stat file [%s] failed: %s", absAsset, statErr)
576 return count, statErr
577 }
578
579 if limitSize < uint64(fi.Size()) {
580 logging.LogWarnf("file [%s] larger than limit size [%s], ignore uploading it", absAsset, humanize.IBytes(limitSize))
581 if 3 > pushErrMsgCount {
582 msg := fmt.Sprintf(Conf.Language(247), filepath.Base(absAsset), humanize.IBytes(limitSize))
583 util.PushErrMsg(msg, 30000)
584 }
585 pushErrMsgCount++
586 continue
587 }
588
589 msg := fmt.Sprintf(Conf.Language(27), html.EscapeString(absAsset))
590 util.PushStatusBar(msg)
591 util.PushUpdateMsg(msgId, msg, 3000)
592
593 requestResult := gulu.Ret.NewResult()
594 request := httpclient.NewCloudFileRequest2m()
595 resp, reqErr := request.
596 SetSuccessResult(requestResult).
597 SetFile("file[]", absAsset).
598 SetCookies(&http.Cookie{Name: "symphony", Value: uploadToken}).
599 SetHeader("meta-type", metaType).
600 SetHeader("biz-type", bizType).
601 Post(util.GetCloudServer() + "/apis/siyuan/upload?ver=" + util.Ver)
602 if nil != reqErr {
603 logging.LogErrorf("upload assets failed: %s", reqErr)
604 return count, ErrFailedToConnectCloudServer
605 }
606
607 if 401 == resp.StatusCode {
608 err = errors.New(Conf.Language(31))
609 return
610 }
611
612 if 0 != requestResult.Code {
613 logging.LogErrorf("upload assets failed: %s", requestResult.Msg)
614 err = errors.New(fmt.Sprintf(Conf.Language(94), requestResult.Msg))
615 return
616 }
617
618 absAsset = filepath.ToSlash(absAsset)
619 relAsset := absAsset[strings.Index(absAsset, "assets/"):]
620 completedUploadAssets = append(completedUploadAssets, relAsset)
621 logging.LogInfof("uploaded asset [%s]", relAsset)
622 count++
623 }
624 util.PushClearMsg(msgId)
625
626 if 0 < len(completedUploadAssets) {
627 logging.LogInfof("uploaded [%d] assets", len(completedUploadAssets))
628 }
629 return
630}
631
632func RemoveUnusedAssets() (ret []string) {
633 ret = []string{}
634 var size int64
635
636 msgId := util.PushMsg(Conf.Language(100), 30*1000)
637 defer func() {
638 msg := fmt.Sprintf(Conf.Language(91), len(ret), humanize.BytesCustomCeil(uint64(size), 2))
639 util.PushUpdateMsg(msgId, msg, 7000)
640 }()
641
642 unusedAssets := UnusedAssets()
643
644 historyDir, err := GetHistoryDir(HistoryOpClean)
645 if err != nil {
646 logging.LogErrorf("get history dir failed: %s", err)
647 return
648 }
649
650 var hashes []string
651 for _, p := range unusedAssets {
652 historyPath := filepath.Join(historyDir, p)
653 if p = filepath.Join(util.DataDir, p); filelock.IsExist(p) {
654 if filelock.IsHidden(p) {
655 continue
656 }
657
658 if err = filelock.Copy(p, historyPath); err != nil {
659 return
660 }
661
662 hash, _ := util.GetEtag(p)
663 hashes = append(hashes, hash)
664 cache.RemoveAssetHash(hash)
665 }
666 }
667
668 sql.BatchRemoveAssetsQueue(hashes)
669
670 for _, unusedAsset := range unusedAssets {
671 absPath := filepath.Join(util.DataDir, unusedAsset)
672 if filelock.IsExist(absPath) {
673 info, statErr := os.Stat(absPath)
674 if statErr == nil {
675 if info.IsDir() {
676 dirSize, _ := util.SizeOfDirectory(absPath)
677 size += dirSize
678 } else {
679 size += info.Size()
680 }
681 }
682
683 if err := filelock.Remove(absPath); err != nil {
684 logging.LogErrorf("remove unused asset [%s] failed: %s", absPath, err)
685 }
686 util.RemoveAssetText(unusedAsset)
687 }
688 ret = append(ret, absPath)
689 }
690 if 0 < len(ret) {
691 IncSync()
692 }
693
694 indexHistoryDir(filepath.Base(historyDir), util.NewLute())
695 cache.LoadAssets()
696 return
697}
698
699func RemoveUnusedAsset(p string) (ret string) {
700 absPath := filepath.Join(util.DataDir, p)
701 if !filelock.IsExist(absPath) {
702 return absPath
703 }
704
705 historyDir, err := GetHistoryDir(HistoryOpClean)
706 if err != nil {
707 logging.LogErrorf("get history dir failed: %s", err)
708 return
709 }
710
711 newP := strings.TrimPrefix(absPath, util.DataDir)
712 historyPath := filepath.Join(historyDir, newP)
713 if filelock.IsExist(absPath) {
714 if err = filelock.Copy(absPath, historyPath); err != nil {
715 return
716 }
717
718 hash, _ := util.GetEtag(absPath)
719 sql.BatchRemoveAssetsQueue([]string{hash})
720 cache.RemoveAssetHash(hash)
721 }
722
723 if err = filelock.Remove(absPath); err != nil {
724 logging.LogErrorf("remove unused asset [%s] failed: %s", absPath, err)
725 }
726 ret = absPath
727
728 util.RemoveAssetText(p)
729
730 IncSync()
731
732 indexHistoryDir(filepath.Base(historyDir), util.NewLute())
733 cache.RemoveAsset(p)
734 return
735}
736
737func RenameAsset(oldPath, newName string) (newPath string, err error) {
738 util.PushEndlessProgress(Conf.Language(110))
739 defer util.PushClearProgress()
740
741 newName = strings.TrimSpace(newName)
742 newName = util.FilterUploadFileName(newName)
743 if path.Base(oldPath) == newName {
744 return
745 }
746 if "" == newName {
747 return
748 }
749
750 if !gulu.File.IsValidFilename(newName) {
751 err = errors.New(Conf.Language(151))
752 return
753 }
754
755 newName = util.AssetName(newName+filepath.Ext(oldPath), ast.NewNodeID())
756 parentDir := path.Dir(oldPath)
757 newPath = path.Join(parentDir, newName)
758 oldAbsPath, getErr := GetAssetAbsPath(oldPath)
759 if getErr != nil {
760 logging.LogErrorf("get asset [%s] abs path failed: %s", oldPath, getErr)
761 return
762 }
763 newAbsPath := filepath.Join(filepath.Dir(oldAbsPath), newName)
764 if err = filelock.Copy(oldAbsPath, newAbsPath); err != nil {
765 logging.LogErrorf("copy asset [%s] failed: %s", oldAbsPath, err)
766 return
767 }
768
769 if filelock.IsExist(filepath.Join(util.DataDir, oldPath+".sya")) {
770 // Rename the .sya annotation file when renaming a PDF asset https://github.com/siyuan-note/siyuan/issues/9390
771 if err = filelock.Copy(filepath.Join(util.DataDir, oldPath+".sya"), filepath.Join(util.DataDir, newPath+".sya")); err != nil {
772 logging.LogErrorf("copy PDF annotation [%s] failed: %s", oldPath+".sya", err)
773 return
774 }
775 }
776
777 oldName := path.Base(oldPath)
778
779 notebooks, err := ListNotebooks()
780 if err != nil {
781 return
782 }
783
784 luteEngine := util.NewLute()
785 for _, notebook := range notebooks {
786 pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
787
788 for _, paths := range pages {
789 for _, treeAbsPath := range paths {
790 data, readErr := filelock.ReadFile(treeAbsPath)
791 if nil != readErr {
792 logging.LogErrorf("get data [path=%s] failed: %s", treeAbsPath, readErr)
793 err = readErr
794 return
795 }
796
797 if !bytes.Contains(data, []byte(oldName)) {
798 continue
799 }
800
801 data = bytes.Replace(data, []byte(oldName), []byte(newName), -1)
802 if writeErr := filelock.WriteFile(treeAbsPath, data); nil != writeErr {
803 logging.LogErrorf("write data [path=%s] failed: %s", treeAbsPath, writeErr)
804 err = writeErr
805 return
806 }
807
808 p := filepath.ToSlash(strings.TrimPrefix(treeAbsPath, filepath.Join(util.DataDir, notebook.ID)))
809 tree, parseErr := filesys.LoadTreeByData(data, notebook.ID, p, luteEngine)
810 if nil != parseErr {
811 logging.LogWarnf("parse json to tree [%s] failed: %s", treeAbsPath, parseErr)
812 continue
813 }
814
815 treenode.UpsertBlockTree(tree)
816 sql.UpsertTreeQueue(tree)
817
818 util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title"))))
819 }
820 }
821 }
822
823 storageAvDir := filepath.Join(util.DataDir, "storage", "av")
824 if gulu.File.IsDir(storageAvDir) {
825 entries, readErr := os.ReadDir(storageAvDir)
826 if nil != readErr {
827 logging.LogErrorf("read dir [%s] failed: %s", storageAvDir, readErr)
828 err = readErr
829 return
830 }
831
832 for _, entry := range entries {
833 if !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(entry.Name(), ".json")) {
834 continue
835 }
836
837 data, readDataErr := filelock.ReadFile(filepath.Join(util.DataDir, "storage", "av", entry.Name()))
838 if nil != readDataErr {
839 logging.LogErrorf("read file [%s] failed: %s", entry.Name(), readDataErr)
840 err = readDataErr
841 return
842 }
843
844 if bytes.Contains(data, []byte(oldPath)) {
845 data = bytes.ReplaceAll(data, []byte(oldPath), []byte(newPath))
846 if writeDataErr := filelock.WriteFile(filepath.Join(util.DataDir, "storage", "av", entry.Name()), data); nil != writeDataErr {
847 logging.LogErrorf("write file [%s] failed: %s", entry.Name(), writeDataErr)
848 err = writeDataErr
849 return
850 }
851 }
852
853 util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(entry.Name())))
854 }
855 }
856
857 if ocrText := util.GetAssetText(oldPath); "" != ocrText {
858 // 图片重命名后 ocr-texts.json 需要更新 https://github.com/siyuan-note/siyuan/issues/12974
859 util.SetAssetText(newPath, ocrText)
860 }
861
862 IncSync()
863 return
864}
865
866func UnusedAssets() (ret []string) {
867 defer logging.Recover()
868 ret = []string{}
869
870 assetsPathMap, err := allAssetAbsPaths()
871 if err != nil {
872 return
873 }
874 linkDestMap := map[string]bool{}
875 notebooks, err := ListNotebooks()
876 if err != nil {
877 return
878 }
879 luteEngine := util.NewLute()
880 for _, notebook := range notebooks {
881 dests := map[string]bool{}
882
883 // 分页加载,优化清理未引用资源内存占用 https://github.com/siyuan-note/siyuan/issues/5200
884 pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
885 for _, paths := range pages {
886 var trees []*parse.Tree
887 for _, localPath := range paths {
888 tree, loadTreeErr := loadTree(localPath, luteEngine)
889 if nil != loadTreeErr {
890 continue
891 }
892 trees = append(trees, tree)
893 }
894 for _, tree := range trees {
895 for _, d := range getAssetsLinkDests(tree.Root) {
896 dests[d] = true
897 }
898
899 if titleImgPath := treenode.GetDocTitleImgPath(tree.Root); "" != titleImgPath {
900 // 题头图计入
901 if !util.IsAssetLinkDest([]byte(titleImgPath)) {
902 continue
903 }
904 dests[titleImgPath] = true
905 }
906 }
907 }
908
909 var linkDestFolderPaths, linkDestFilePaths []string
910 for dest := range dests {
911 if !strings.HasPrefix(dest, "assets/") {
912 continue
913 }
914
915 if idx := strings.Index(dest, "?"); 0 < idx {
916 // `pdf?page` 资源文件链接会被判定为未引用资源 https://github.com/siyuan-note/siyuan/issues/5649
917 dest = dest[:idx]
918 }
919
920 if "" == assetsPathMap[dest] {
921 continue
922 }
923 if strings.HasSuffix(dest, "/") {
924 linkDestFolderPaths = append(linkDestFolderPaths, dest)
925 } else {
926 linkDestFilePaths = append(linkDestFilePaths, dest)
927 }
928 }
929
930 // 排除文件夹链接
931 var toRemoves []string
932 for asset := range assetsPathMap {
933 for _, linkDestFolder := range linkDestFolderPaths {
934 if strings.HasPrefix(asset, linkDestFolder) {
935 toRemoves = append(toRemoves, asset)
936 }
937 }
938 for _, linkDestPath := range linkDestFilePaths {
939 if strings.HasPrefix(linkDestPath, asset) {
940 toRemoves = append(toRemoves, asset)
941 }
942 }
943 }
944 for _, toRemove := range toRemoves {
945 delete(assetsPathMap, toRemove)
946 }
947
948 for _, dest := range linkDestFilePaths {
949 linkDestMap[dest] = true
950
951 if strings.HasSuffix(dest, ".pdf") {
952 linkDestMap[dest+".sya"] = true
953 }
954 }
955 }
956
957 var toRemoves []string
958 for asset := range assetsPathMap {
959 if strings.HasSuffix(asset, "ocr-texts.json") {
960 // 排除 OCR 结果文本
961 toRemoves = append(toRemoves, asset)
962 continue
963 }
964
965 if strings.HasSuffix(asset, "android-notification-texts.txt") {
966 // 排除 Android 通知文本
967 toRemoves = append(toRemoves, asset)
968 continue
969 }
970 }
971
972 // 排除数据库中引用的资源文件
973 storageAvDir := filepath.Join(util.DataDir, "storage", "av")
974 if gulu.File.IsDir(storageAvDir) {
975 entries, readErr := os.ReadDir(storageAvDir)
976 if nil != readErr {
977 logging.LogErrorf("read dir [%s] failed: %s", storageAvDir, readErr)
978 err = readErr
979 return
980 }
981
982 for _, entry := range entries {
983 if !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(entry.Name(), ".json")) {
984 continue
985 }
986
987 data, readDataErr := filelock.ReadFile(filepath.Join(util.DataDir, "storage", "av", entry.Name()))
988 if nil != readDataErr {
989 logging.LogErrorf("read file [%s] failed: %s", entry.Name(), readDataErr)
990 err = readDataErr
991 return
992 }
993
994 for asset := range assetsPathMap {
995 if bytes.Contains(data, []byte(asset)) {
996 toRemoves = append(toRemoves, asset)
997 }
998 }
999 }
1000 }
1001
1002 for _, toRemove := range toRemoves {
1003 delete(assetsPathMap, toRemove)
1004 }
1005
1006 dataAssetsAbsPath := util.GetDataAssetsAbsPath()
1007 for dest, assetAbsPath := range assetsPathMap {
1008 if _, ok := linkDestMap[dest]; ok {
1009 continue
1010 }
1011
1012 var p string
1013 if strings.HasPrefix(dataAssetsAbsPath, assetAbsPath) {
1014 p = assetAbsPath[strings.Index(assetAbsPath, "assets"):]
1015 } else {
1016 p = strings.TrimPrefix(assetAbsPath, filepath.Dir(dataAssetsAbsPath))
1017 }
1018 p = filepath.ToSlash(p)
1019 if strings.HasPrefix(p, "/") {
1020 p = p[1:]
1021 }
1022 ret = append(ret, p)
1023 }
1024 sort.Strings(ret)
1025 return
1026}
1027
1028func MissingAssets() (ret []string) {
1029 defer logging.Recover()
1030 ret = []string{}
1031
1032 assetsPathMap, err := allAssetAbsPaths()
1033 if err != nil {
1034 return
1035 }
1036 notebooks, err := ListNotebooks()
1037 if err != nil {
1038 return
1039 }
1040 luteEngine := util.NewLute()
1041 for _, notebook := range notebooks {
1042 if notebook.Closed {
1043 continue
1044 }
1045
1046 dests := map[string]bool{}
1047 pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32)
1048 for _, paths := range pages {
1049 var trees []*parse.Tree
1050 for _, localPath := range paths {
1051 tree, loadTreeErr := loadTree(localPath, luteEngine)
1052 if nil != loadTreeErr {
1053 continue
1054 }
1055 trees = append(trees, tree)
1056 }
1057 for _, tree := range trees {
1058 for _, d := range getAssetsLinkDests(tree.Root) {
1059 dests[d] = true
1060 }
1061
1062 if titleImgPath := treenode.GetDocTitleImgPath(tree.Root); "" != titleImgPath {
1063 // 题头图计入
1064 if !util.IsAssetLinkDest([]byte(titleImgPath)) {
1065 continue
1066 }
1067 dests[titleImgPath] = true
1068 }
1069 }
1070 }
1071
1072 for dest := range dests {
1073 if !strings.HasPrefix(dest, "assets/") {
1074 continue
1075 }
1076
1077 if idx := strings.Index(dest, "?"); 0 < idx {
1078 dest = dest[:idx]
1079 }
1080
1081 if strings.HasSuffix(dest, "/") {
1082 continue
1083 }
1084
1085 if strings.Contains(strings.ToLower(dest), ".pdf/") {
1086 if idx := strings.LastIndex(dest, "/"); -1 < idx {
1087 if ast.IsNodeIDPattern(dest[idx+1:]) {
1088 // PDF 标注不计入 https://github.com/siyuan-note/siyuan/issues/13891
1089 continue
1090 }
1091 }
1092 }
1093
1094 if "" == assetsPathMap[dest] {
1095 if strings.HasPrefix(dest, "assets/.") {
1096 // Assets starting with `.` should not be considered missing assets https://github.com/siyuan-note/siyuan/issues/8821
1097 if !filelock.IsExist(filepath.Join(util.DataDir, dest)) {
1098 ret = append(ret, dest)
1099 }
1100 } else {
1101 ret = append(ret, dest)
1102 }
1103 continue
1104 }
1105 }
1106 }
1107
1108 sort.Strings(ret)
1109 return
1110}
1111
1112func emojisInTree(tree *parse.Tree) (ret []string) {
1113 if icon := tree.Root.IALAttr("icon"); "" != icon {
1114 if !strings.Contains(icon, "://") && !strings.HasPrefix(icon, "api/icon/") && !util.NativeEmojiChars[icon] {
1115 ret = append(ret, "/emojis/"+icon)
1116 }
1117 }
1118
1119 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
1120 if !entering {
1121 return ast.WalkContinue
1122 }
1123 if ast.NodeEmojiImg == n.Type {
1124 tokens := n.Tokens
1125 idx := bytes.Index(tokens, []byte("src=\""))
1126 if -1 == idx {
1127 return ast.WalkContinue
1128 }
1129 src := tokens[idx+len("src=\""):]
1130 src = src[:bytes.Index(src, []byte("\""))]
1131 ret = append(ret, string(src))
1132 }
1133 return ast.WalkContinue
1134 })
1135 ret = gulu.Str.RemoveDuplicatedElem(ret)
1136 return
1137}
1138
1139func getQueryEmbedNodesAssetsLinkDests(node *ast.Node) (ret []string) {
1140 // The images in the embed blocks are not uploaded to the community hosting https://github.com/siyuan-note/siyuan/issues/10042
1141
1142 ret = []string{}
1143 ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus {
1144 if !entering || ast.NodeBlockQueryEmbedScript != n.Type {
1145 return ast.WalkContinue
1146 }
1147
1148 stmt := n.TokensStr()
1149 stmt = html.UnescapeString(stmt)
1150 stmt = strings.ReplaceAll(stmt, editor.IALValEscNewLine, "\n")
1151 sqlBlocks := sql.SelectBlocksRawStmt(stmt, 1, Conf.Search.Limit)
1152 for _, sqlBlock := range sqlBlocks {
1153 subtree, _ := LoadTreeByBlockID(sqlBlock.ID)
1154 if nil == subtree {
1155 continue
1156 }
1157 embedNode := treenode.GetNodeInTree(subtree, sqlBlock.ID)
1158 if nil == embedNode {
1159 continue
1160 }
1161
1162 ret = append(ret, getAssetsLinkDests(embedNode)...)
1163 }
1164 return ast.WalkContinue
1165 })
1166 ret = gulu.Str.RemoveDuplicatedElem(ret)
1167 return
1168}
1169
1170func getAssetsLinkDests(node *ast.Node) (ret []string) {
1171 ret = []string{}
1172 ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus {
1173 if n.IsBlock() {
1174 // 以 custom-data-assets 开头的块属性值可能是多个资源文件链接,需要计入
1175 // Ignore assets associated with the `custom-data-assets` block attribute when cleaning unreferenced assets https://github.com/siyuan-note/siyuan/issues/12574
1176 for _, kv := range n.KramdownIAL {
1177 k := kv[0]
1178 if strings.HasPrefix(k, "custom-data-assets") {
1179 dest := kv[1]
1180 if "" == dest || !util.IsAssetLinkDest([]byte(dest)) {
1181 continue
1182 }
1183 ret = append(ret, dest)
1184 }
1185 }
1186 }
1187
1188 // 修改以下代码时需要同时修改 database 构造行级元素实现,增加必要的类型
1189 if !entering || (ast.NodeLinkDest != n.Type && ast.NodeHTMLBlock != n.Type && ast.NodeInlineHTML != n.Type &&
1190 ast.NodeIFrame != n.Type && ast.NodeWidget != n.Type && ast.NodeAudio != n.Type && ast.NodeVideo != n.Type &&
1191 ast.NodeAttributeView != n.Type && !n.IsTextMarkType("a") && !n.IsTextMarkType("file-annotation-ref")) {
1192 return ast.WalkContinue
1193 }
1194
1195 if ast.NodeLinkDest == n.Type {
1196 if !util.IsAssetLinkDest(n.Tokens) {
1197 return ast.WalkContinue
1198 }
1199
1200 dest := strings.TrimSpace(string(n.Tokens))
1201 ret = append(ret, dest)
1202 } else if n.IsTextMarkType("a") {
1203 if !util.IsAssetLinkDest(gulu.Str.ToBytes(n.TextMarkAHref)) {
1204 return ast.WalkContinue
1205 }
1206
1207 dest := strings.TrimSpace(n.TextMarkAHref)
1208 ret = append(ret, dest)
1209 } else if n.IsTextMarkType("file-annotation-ref") {
1210 if !util.IsAssetLinkDest(gulu.Str.ToBytes(n.TextMarkFileAnnotationRefID)) {
1211 return ast.WalkContinue
1212 }
1213
1214 if !strings.Contains(n.TextMarkFileAnnotationRefID, "/") {
1215 return ast.WalkContinue
1216 }
1217
1218 dest := n.TextMarkFileAnnotationRefID[:strings.LastIndexByte(n.TextMarkFileAnnotationRefID, '/')]
1219 dest = strings.TrimSpace(dest)
1220 ret = append(ret, dest)
1221 } else if ast.NodeAttributeView == n.Type {
1222 attrView, _ := av.ParseAttributeView(n.AttributeViewID)
1223 if nil == attrView {
1224 return ast.WalkContinue
1225 }
1226
1227 for _, keyValues := range attrView.KeyValues {
1228 if av.KeyTypeMAsset == keyValues.Key.Type {
1229 for _, value := range keyValues.Values {
1230 if 1 > len(value.MAsset) {
1231 continue
1232 }
1233
1234 for _, asset := range value.MAsset {
1235 dest := asset.Content
1236 if !util.IsAssetLinkDest([]byte(dest)) {
1237 continue
1238 }
1239 ret = append(ret, strings.TrimSpace(dest))
1240 }
1241 }
1242 } else if av.KeyTypeURL == keyValues.Key.Type {
1243 for _, value := range keyValues.Values {
1244 if nil != value.URL {
1245 dest := value.URL.Content
1246 if !util.IsAssetLinkDest([]byte(dest)) {
1247 continue
1248 }
1249 ret = append(ret, strings.TrimSpace(dest))
1250 }
1251 }
1252 }
1253 }
1254 } else {
1255 if ast.NodeWidget == n.Type {
1256 dataAssets := n.IALAttr("custom-data-assets")
1257 if "" == dataAssets {
1258 // 兼容两种属性名 custom-data-assets 和 data-assets https://github.com/siyuan-note/siyuan/issues/4122#issuecomment-1154796568
1259 dataAssets = n.IALAttr("data-assets")
1260 }
1261 if !util.IsAssetLinkDest([]byte(dataAssets)) {
1262 return ast.WalkContinue
1263 }
1264 ret = append(ret, dataAssets)
1265 } else { // HTMLBlock/InlineHTML/IFrame/Audio/Video
1266 dest := treenode.GetNodeSrcTokens(n)
1267 if !util.IsAssetLinkDest([]byte(dest)) {
1268 return ast.WalkContinue
1269 }
1270 ret = append(ret, dest)
1271 }
1272 }
1273 return ast.WalkContinue
1274 })
1275 ret = gulu.Str.RemoveDuplicatedElem(ret)
1276 for i, dest := range ret {
1277 // 对于 macOS 的 rtfd 文件夹格式需要特殊处理,为其加上结尾 /
1278 if strings.HasSuffix(dest, ".rtfd") {
1279 ret[i] = dest + "/"
1280 }
1281 }
1282 return
1283}
1284
1285func setAssetsLinkDest(node *ast.Node, oldDest, dest string) {
1286 if ast.NodeLinkDest == node.Type {
1287 if bytes.HasPrefix(node.Tokens, []byte("//")) {
1288 node.Tokens = append([]byte("https:"), node.Tokens...)
1289 }
1290 node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest))
1291 } else if node.IsTextMarkType("a") {
1292 if strings.HasPrefix(node.TextMarkAHref, "//") {
1293 node.TextMarkAHref = "https:" + node.TextMarkAHref
1294 }
1295 node.TextMarkAHref = strings.ReplaceAll(node.TextMarkAHref, oldDest, dest)
1296 } else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type {
1297 if strings.HasPrefix(node.TextMarkAHref, "//") {
1298 node.TextMarkAHref = "https:" + node.TextMarkAHref
1299 }
1300 node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest))
1301 } else if ast.NodeAttributeView == node.Type {
1302 needWrite := false
1303 attrView, _ := av.ParseAttributeView(node.AttributeViewID)
1304 if nil == attrView {
1305 return
1306 }
1307
1308 for _, keyValues := range attrView.KeyValues {
1309 if av.KeyTypeMAsset != keyValues.Key.Type {
1310 continue
1311 }
1312
1313 for _, value := range keyValues.Values {
1314 if 1 > len(value.MAsset) {
1315 continue
1316 }
1317
1318 for _, asset := range value.MAsset {
1319 if oldDest == asset.Content && oldDest != dest {
1320 asset.Content = dest
1321 needWrite = true
1322 }
1323 }
1324 }
1325 }
1326
1327 if needWrite {
1328 av.SaveAttributeView(attrView)
1329 }
1330 }
1331}
1332
1333func getRemoteAssetsLinkDests(node *ast.Node, onlyImg bool) (ret []string) {
1334 if onlyImg {
1335 if ast.NodeLinkDest == node.Type {
1336 if node.ParentIs(ast.NodeImage) {
1337 if !util.IsAssetLinkDest(node.Tokens) {
1338 ret = append(ret, string(node.Tokens))
1339 }
1340
1341 }
1342 } else if ast.NodeAttributeView == node.Type {
1343 attrView, _ := av.ParseAttributeView(node.AttributeViewID)
1344 if nil == attrView {
1345 return
1346 }
1347
1348 for _, keyValues := range attrView.KeyValues {
1349 if av.KeyTypeMAsset != keyValues.Key.Type {
1350 continue
1351 }
1352
1353 for _, value := range keyValues.Values {
1354 if 1 > len(value.MAsset) {
1355 continue
1356 }
1357
1358 for _, asset := range value.MAsset {
1359 if av.AssetTypeImage != asset.Type {
1360 continue
1361 }
1362
1363 dest := asset.Content
1364 if !util.IsAssetLinkDest([]byte(dest)) {
1365 ret = append(ret, strings.TrimSpace(dest))
1366 }
1367 }
1368 }
1369 }
1370 }
1371 } else {
1372 if ast.NodeLinkDest == node.Type {
1373 if !util.IsAssetLinkDest(node.Tokens) {
1374 ret = append(ret, string(node.Tokens))
1375 }
1376 } else if node.IsTextMarkType("a") {
1377 if !util.IsAssetLinkDest([]byte(node.TextMarkAHref)) {
1378 ret = append(ret, node.TextMarkAHref)
1379 }
1380 } else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type {
1381 src := treenode.GetNodeSrcTokens(node)
1382 if !util.IsAssetLinkDest([]byte(src)) {
1383 ret = append(ret, src)
1384 }
1385 } else if ast.NodeAttributeView == node.Type {
1386 attrView, _ := av.ParseAttributeView(node.AttributeViewID)
1387 if nil == attrView {
1388 return
1389 }
1390
1391 for _, keyValues := range attrView.KeyValues {
1392 if av.KeyTypeMAsset != keyValues.Key.Type {
1393 continue
1394 }
1395
1396 for _, value := range keyValues.Values {
1397 if 1 > len(value.MAsset) {
1398 continue
1399 }
1400
1401 for _, asset := range value.MAsset {
1402 dest := asset.Content
1403 if !util.IsAssetLinkDest([]byte(dest)) {
1404 ret = append(ret, strings.TrimSpace(dest))
1405 }
1406 }
1407 }
1408 }
1409 }
1410 }
1411 return
1412}
1413
1414func getRemoteAssetsLinkDestsInTree(tree *parse.Tree, onlyImg bool) (nodes []*ast.Node) {
1415 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus {
1416 if !entering {
1417 return ast.WalkContinue
1418 }
1419
1420 dests := getRemoteAssetsLinkDests(n, onlyImg)
1421 if 1 > len(dests) {
1422 return ast.WalkContinue
1423 }
1424
1425 nodes = append(nodes, n)
1426 return ast.WalkContinue
1427 })
1428 return
1429}
1430
1431// allAssetAbsPaths 返回 asset 相对路径(assets/xxx)到绝对路径(F:\SiYuan\data\assets\xxx)的映射。
1432func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) {
1433 notebooks, err := ListNotebooks()
1434 if err != nil {
1435 return
1436 }
1437
1438 assetsAbsPathMap = map[string]string{}
1439 // 笔记本 assets
1440 for _, notebook := range notebooks {
1441 notebookAbsPath := filepath.Join(util.DataDir, notebook.ID)
1442 filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error {
1443 if notebookAbsPath == path {
1444 return nil
1445 }
1446 if isSkipFile(d.Name()) {
1447 if d.IsDir() {
1448 return filepath.SkipDir
1449 }
1450 return nil
1451 }
1452
1453 if filelock.IsHidden(path) {
1454 // 清理资源文件时忽略隐藏文件 Ignore hidden files when cleaning unused assets https://github.com/siyuan-note/siyuan/issues/12172
1455 return nil
1456 }
1457
1458 if d.IsDir() && "assets" == d.Name() {
1459 filelock.Walk(path, func(assetPath string, d fs.DirEntry, err error) error {
1460 if path == assetPath {
1461 return nil
1462 }
1463 if isSkipFile(d.Name()) {
1464 if d.IsDir() {
1465 return filepath.SkipDir
1466 }
1467 return nil
1468 }
1469 relPath := filepath.ToSlash(assetPath)
1470 relPath = relPath[strings.Index(relPath, "assets/"):]
1471 if d.IsDir() {
1472 relPath += "/"
1473 }
1474 assetsAbsPathMap[relPath] = assetPath
1475 return nil
1476 })
1477 return filepath.SkipDir
1478 }
1479 return nil
1480 })
1481 }
1482
1483 // 全局 assets
1484 dataAssetsAbsPath := util.GetDataAssetsAbsPath()
1485 filelock.Walk(dataAssetsAbsPath, func(assetPath string, d fs.DirEntry, err error) error {
1486 if dataAssetsAbsPath == assetPath {
1487 return nil
1488 }
1489
1490 if isSkipFile(d.Name()) {
1491 if d.IsDir() {
1492 return filepath.SkipDir
1493 }
1494 return nil
1495 }
1496
1497 if filelock.IsHidden(assetPath) {
1498 // 清理资源文件时忽略隐藏文件 Ignore hidden files when cleaning unused assets https://github.com/siyuan-note/siyuan/issues/12172
1499 return nil
1500 }
1501
1502 relPath := filepath.ToSlash(assetPath)
1503 relPath = relPath[strings.Index(relPath, "assets/"):]
1504 if d.IsDir() {
1505 relPath += "/"
1506 }
1507 assetsAbsPathMap[relPath] = assetPath
1508 return nil
1509 })
1510 return
1511}
1512
1513// copyBoxAssetsToDataAssets 将笔记本路径下所有(包括子文档)的 assets 复制一份到 data/assets 中。
1514func copyBoxAssetsToDataAssets(boxID string) {
1515 boxLocalPath := filepath.Join(util.DataDir, boxID)
1516 copyAssetsToDataAssets(boxLocalPath)
1517}
1518
1519// copyDocAssetsToDataAssets 将文档路径下所有(包括子文档)的 assets 复制一份到 data/assets 中。
1520func copyDocAssetsToDataAssets(boxID, parentDocPath string) {
1521 boxLocalPath := filepath.Join(util.DataDir, boxID)
1522 parentDocDirAbsPath := filepath.Dir(filepath.Join(boxLocalPath, parentDocPath))
1523 copyAssetsToDataAssets(parentDocDirAbsPath)
1524}
1525
1526func copyAssetsToDataAssets(rootPath string) {
1527 var assetsDirPaths []string
1528 filelock.Walk(rootPath, func(path string, d fs.DirEntry, err error) error {
1529 if nil != err || rootPath == path || nil == d {
1530 return nil
1531 }
1532
1533 isDir, name := d.IsDir(), d.Name()
1534 if isSkipFile(name) {
1535 if isDir {
1536 return filepath.SkipDir
1537 }
1538 return nil
1539 }
1540
1541 if "assets" == name && isDir {
1542 assetsDirPaths = append(assetsDirPaths, path)
1543 }
1544 return nil
1545 })
1546
1547 dataAssetsPath := filepath.Join(util.DataDir, "assets")
1548 for _, assetsDirPath := range assetsDirPaths {
1549 if err := filelock.Copy(assetsDirPath, dataAssetsPath); err != nil {
1550 logging.LogErrorf("copy tree assets from [%s] to [%s] failed: %s", assetsDirPaths, dataAssetsPath, err)
1551 }
1552 }
1553}