A privacy-first, self-hosted, fully open source personal knowledge management software, written in typescript and golang. (PERSONAL FORK)
at lambda-fork/main 1553 lines 43 kB view raw
1// SiYuan - Refactor your thinking 2// Copyright (c) 2020-present, b3log.org 3// 4// This program is free software: you can redistribute it and/or modify 5// it under the terms of the GNU Affero General Public License as published by 6// the Free Software Foundation, either version 3 of the License, or 7// (at your option) any later version. 8// 9// This program is distributed in the hope that it will be useful, 10// but WITHOUT ANY WARRANTY; without even the implied warranty of 11// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 12// GNU Affero General Public License for more details. 13// 14// You should have received a copy of the GNU Affero General Public License 15// along with this program. If not, see <https://www.gnu.org/licenses/>. 16 17package model 18 19import ( 20 "bytes" 21 "errors" 22 "fmt" 23 "io/fs" 24 "mime" 25 "net/http" 26 "net/url" 27 "os" 28 "path" 29 "path/filepath" 30 "sort" 31 "strings" 32 "time" 33 34 "github.com/88250/go-humanize" 35 "github.com/88250/gulu" 36 "github.com/88250/lute/ast" 37 "github.com/88250/lute/editor" 38 "github.com/88250/lute/html" 39 "github.com/88250/lute/parse" 40 "github.com/disintegration/imaging" 41 "github.com/gabriel-vasile/mimetype" 42 "github.com/siyuan-note/filelock" 43 "github.com/siyuan-note/httpclient" 44 "github.com/siyuan-note/logging" 45 "github.com/siyuan-note/siyuan/kernel/av" 46 "github.com/siyuan-note/siyuan/kernel/cache" 47 "github.com/siyuan-note/siyuan/kernel/filesys" 48 "github.com/siyuan-note/siyuan/kernel/search" 49 "github.com/siyuan-note/siyuan/kernel/sql" 50 "github.com/siyuan-note/siyuan/kernel/treenode" 51 "github.com/siyuan-note/siyuan/kernel/util" 52) 53 54func GetAssetPathByHash(hash string) string { 55 assetHash := cache.GetAssetHash(hash) 56 if nil == assetHash { 57 sqlAsset := sql.QueryAssetByHash(hash) 58 if nil == sqlAsset { 59 return "" 60 } 61 cache.SetAssetHash(sqlAsset.Hash, sqlAsset.Path) 62 return sqlAsset.Path 63 } 64 return assetHash.Path 65} 66 67func HandleAssetsRemoveEvent(assetAbsPath string) { 68 removeIndexAssetContent(assetAbsPath) 69 removeAssetThumbnail(assetAbsPath) 70} 71 72func HandleAssetsChangeEvent(assetAbsPath string) { 73 indexAssetContent(assetAbsPath) 74 removeAssetThumbnail(assetAbsPath) 75} 76 77func removeAssetThumbnail(assetAbsPath string) { 78 if util.IsCompressibleAssetImage(assetAbsPath) { 79 p := filepath.ToSlash(assetAbsPath) 80 idx := strings.Index(p, "assets/") 81 if -1 == idx { 82 return 83 } 84 thumbnailPath := filepath.Join(util.TempDir, "thumbnails", "assets", p[idx+7:]) 85 os.RemoveAll(thumbnailPath) 86 } 87} 88 89func NeedGenerateAssetsThumbnail(sourceImgPath string) bool { 90 info, err := os.Stat(sourceImgPath) 91 if err != nil { 92 return false 93 } 94 if info.IsDir() { 95 return false 96 } 97 return info.Size() > 1024*1024 98} 99 100func GenerateAssetsThumbnail(sourceImgPath, resizedImgPath string) (err error) { 101 start := time.Now() 102 img, err := imaging.Open(sourceImgPath) 103 if err != nil { 104 return 105 } 106 107 // 获取原图宽高 108 originalWidth := img.Bounds().Dx() 109 originalHeight := img.Bounds().Dy() 110 111 // 固定最大宽度为 520,计算缩放比例 112 maxWidth := 520 113 scale := float64(maxWidth) / float64(originalWidth) 114 115 // 按比例计算新的宽高 116 newWidth := maxWidth 117 newHeight := int(float64(originalHeight) * scale) 118 119 // 缩放图片 120 resizedImg := imaging.Resize(img, newWidth, newHeight, imaging.Lanczos) 121 122 // 保存缩放后的图片 123 err = os.MkdirAll(filepath.Dir(resizedImgPath), 0755) 124 if err != nil { 125 return 126 } 127 err = imaging.Save(resizedImg, resizedImgPath) 128 if err != nil { 129 return 130 } 131 logging.LogDebugf("generated thumbnail image [%s] to [%s], cost [%d]ms", sourceImgPath, resizedImgPath, time.Since(start).Milliseconds()) 132 return 133} 134 135func DocImageAssets(rootID string) (ret []string, err error) { 136 tree, err := LoadTreeByBlockID(rootID) 137 if err != nil { 138 return 139 } 140 141 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { 142 if !entering { 143 return ast.WalkContinue 144 } 145 if ast.NodeImage == n.Type { 146 linkDest := n.ChildByType(ast.NodeLinkDest) 147 dest := linkDest.Tokens 148 if 1 > len(dest) { // 双击打开图片不对 https://github.com/siyuan-note/siyuan/issues/5876 149 return ast.WalkContinue 150 } 151 ret = append(ret, gulu.Str.FromBytes(dest)) 152 } 153 return ast.WalkContinue 154 }) 155 return 156} 157 158func DocAssets(rootID string) (ret []string, err error) { 159 tree, err := LoadTreeByBlockID(rootID) 160 if err != nil { 161 return 162 } 163 164 ret = getAssetsLinkDests(tree.Root) 165 return 166} 167 168func NetAssets2LocalAssets(rootID string, onlyImg bool, originalURL string) (err error) { 169 tree, err := LoadTreeByBlockID(rootID) 170 if err != nil { 171 return 172 } 173 174 var files int 175 msgId := gulu.Rand.String(7) 176 177 docDirLocalPath := filepath.Join(util.DataDir, tree.Box, path.Dir(tree.Path)) 178 assetsDirPath := getAssetsDir(filepath.Join(util.DataDir, tree.Box), docDirLocalPath) 179 if !gulu.File.IsExist(assetsDirPath) { 180 if err = os.MkdirAll(assetsDirPath, 0755); err != nil { 181 return 182 } 183 } 184 185 browserClient := util.NewCustomReqClient() // 自定义了 TLS 指纹,增加下载成功率 186 187 forbiddenCount := 0 188 destNodes := getRemoteAssetsLinkDestsInTree(tree, onlyImg) 189 for _, destNode := range destNodes { 190 dests := getRemoteAssetsLinkDests(destNode, onlyImg) 191 if 1 > len(dests) { 192 continue 193 } 194 195 for _, dest := range dests { 196 if strings.HasPrefix(strings.ToLower(dest), "file://") { // 处理本地文件链接 197 u := dest[7:] 198 unescaped, _ := url.PathUnescape(u) 199 if unescaped != u { 200 // `Convert network images/assets to local` supports URL-encoded local file names https://github.com/siyuan-note/siyuan/issues/9929 201 u = unescaped 202 } 203 if strings.Contains(u, ":") { 204 u = strings.TrimPrefix(u, "/") 205 } 206 if strings.Contains(u, "?") { 207 u = u[:strings.Index(u, "?")] 208 } 209 210 if !gulu.File.IsExist(u) || gulu.File.IsDir(u) { 211 continue 212 } 213 214 name := filepath.Base(u) 215 name = util.FilterUploadFileName(name) 216 name = "network-asset-" + name 217 name = util.AssetName(name, ast.NewNodeID()) 218 writePath := filepath.Join(assetsDirPath, name) 219 if err = filelock.Copy(u, writePath); err != nil { 220 logging.LogErrorf("copy [%s] to [%s] failed: %s", u, writePath, err) 221 continue 222 } 223 224 setAssetsLinkDest(destNode, dest, "assets/"+name) 225 files++ 226 continue 227 } 228 229 if strings.HasPrefix(strings.ToLower(dest), "https://") || strings.HasPrefix(strings.ToLower(dest), "http://") || strings.HasPrefix(dest, "//") { 230 if strings.HasPrefix(dest, "//") { 231 // `Convert network images to local` supports `//` https://github.com/siyuan-note/siyuan/issues/10598 232 dest = "https:" + dest 233 } 234 235 u := dest 236 if strings.Contains(u, "qpic.cn") { 237 // 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/5052 238 if strings.Contains(u, "http://") { 239 u = strings.Replace(u, "http://", "https://", 1) 240 } 241 242 // 改进 `网络图片转换为本地图片` 微信图片拉取 https://github.com/siyuan-note/siyuan/issues/6431 243 // 下面这部分需要注释掉,否则会导致响应 400 244 //if strings.HasSuffix(u, "/0") { 245 // u = strings.Replace(u, "/0", "/640", 1) 246 //} else if strings.Contains(u, "/0?") { 247 // u = strings.Replace(u, "/0?", "/640?", 1) 248 //} 249 } 250 251 displayU := u 252 if 64 < len(displayU) { 253 displayU = displayU[:64] + "..." 254 } 255 util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(119), displayU), 15000) 256 request := browserClient.R() 257 request.SetRetryCount(1).SetRetryFixedInterval(3 * time.Second) 258 if "" != originalURL { 259 request.SetHeader("Referer", originalURL) // 改进浏览器剪藏扩展转换本地图片成功率 https://github.com/siyuan-note/siyuan/issues/7464 260 } 261 resp, reqErr := request.Get(u) 262 if nil != reqErr { 263 logging.LogErrorf("download network asset [%s] failed: %s", u, reqErr) 264 continue 265 } 266 if http.StatusForbidden == resp.StatusCode || http.StatusUnauthorized == resp.StatusCode { 267 forbiddenCount++ 268 } 269 if strings.Contains(strings.ToLower(resp.GetContentType()), "text/html") { 270 // 忽略超链接网页 `Convert network assets to local` no longer process webpage https://github.com/siyuan-note/siyuan/issues/9965 271 continue 272 } 273 if 200 != resp.StatusCode { 274 logging.LogErrorf("download network asset [%s] failed: %d", u, resp.StatusCode) 275 continue 276 } 277 278 if 1024*1024*96 < resp.ContentLength { 279 logging.LogWarnf("network asset [%s]' size [%s] is large then [96 MB], ignore it", u, humanize.IBytes(uint64(resp.ContentLength))) 280 continue 281 } 282 283 data, repErr := resp.ToBytes() 284 if nil != repErr { 285 logging.LogErrorf("download network asset [%s] failed: %s", u, repErr) 286 continue 287 } 288 var name string 289 if strings.Contains(u, "?") { 290 name = u[:strings.Index(u, "?")] 291 name = path.Base(name) 292 } else { 293 name = path.Base(u) 294 } 295 if strings.Contains(name, "#") { 296 name = name[:strings.Index(name, "#")] 297 } 298 name, _ = url.PathUnescape(name) 299 name = util.FilterUploadFileName(name) 300 ext := util.Ext(name) 301 if !util.IsCommonExt(ext) { 302 if mtype := mimetype.Detect(data); nil != mtype { 303 ext = mtype.Extension() 304 name += ext 305 } 306 } 307 if "" == ext && bytes.HasPrefix(data, []byte("<svg ")) && bytes.HasSuffix(data, []byte("</svg>")) { 308 ext = ".svg" 309 name += ext 310 } 311 if "" == ext { 312 contentType := resp.Header.Get("Content-Type") 313 exts, _ := mime.ExtensionsByType(contentType) 314 if 0 < len(exts) { 315 ext = exts[0] 316 name += ext 317 } 318 } 319 name = util.AssetName(name, ast.NewNodeID()) 320 name = "network-asset-" + name 321 writePath := filepath.Join(assetsDirPath, name) 322 if err = filelock.WriteFile(writePath, data); err != nil { 323 logging.LogErrorf("write downloaded network asset [%s] to local asset [%s] failed: %s", u, writePath, err) 324 continue 325 } 326 327 setAssetsLinkDest(destNode, dest, "assets/"+name) 328 files++ 329 continue 330 } 331 } 332 } 333 334 util.PushClearMsg(msgId) 335 if 0 < files { 336 msgId = util.PushMsg(Conf.Language(113), 7000) 337 if err = writeTreeUpsertQueue(tree); err != nil { 338 return 339 } 340 util.PushUpdateMsg(msgId, fmt.Sprintf(Conf.Language(120), files), 5000) 341 342 if 0 < forbiddenCount { 343 util.PushErrMsg(fmt.Sprintf(Conf.Language(255), forbiddenCount), 5000) 344 } 345 } else { 346 if 0 < forbiddenCount { 347 util.PushErrMsg(fmt.Sprintf(Conf.Language(255), forbiddenCount), 5000) 348 } else { 349 util.PushMsg(Conf.Language(121), 3000) 350 } 351 } 352 return 353} 354 355func SearchAssetsByName(keyword string, exts []string) (ret []*cache.Asset) { 356 ret = []*cache.Asset{} 357 var keywords []string 358 keywords = append(keywords, keyword) 359 if "" != keyword { 360 keywords = append(keywords, strings.Split(keyword, " ")...) 361 } 362 pathHitCount := map[string]int{} 363 filterByExt := 0 < len(exts) 364 for _, asset := range cache.GetAssets() { 365 if filterByExt { 366 ext := filepath.Ext(asset.HName) 367 includeExt := false 368 for _, e := range exts { 369 if strings.ToLower(ext) == strings.ToLower(e) { 370 includeExt = true 371 break 372 } 373 } 374 if !includeExt { 375 continue 376 } 377 } 378 379 lowerHName := strings.ToLower(asset.HName) 380 lowerPath := strings.ToLower(asset.Path) 381 var hitNameCount, hitPathCount int 382 for i, k := range keywords { 383 lowerKeyword := strings.ToLower(k) 384 if 0 == i { 385 // 第一个是完全匹配,权重最高 386 if strings.Contains(lowerHName, lowerKeyword) { 387 hitNameCount += 64 388 } 389 if strings.Contains(lowerPath, lowerKeyword) { 390 hitPathCount += 64 391 } 392 } 393 394 hitNameCount += strings.Count(lowerHName, lowerKeyword) 395 hitPathCount += strings.Count(lowerPath, lowerKeyword) 396 if 1 > hitNameCount && 1 > hitPathCount { 397 continue 398 } 399 } 400 401 if 1 > hitNameCount+hitPathCount { 402 continue 403 } 404 pathHitCount[asset.Path] += hitNameCount + hitPathCount 405 406 hName := asset.HName 407 if 0 < hitNameCount { 408 _, hName = search.MarkText(asset.HName, strings.Join(keywords, search.TermSep), 64, Conf.Search.CaseSensitive) 409 } 410 ret = append(ret, &cache.Asset{ 411 HName: hName, 412 Path: asset.Path, 413 Updated: asset.Updated, 414 }) 415 } 416 417 if 0 < len(pathHitCount) { 418 sort.Slice(ret, func(i, j int) bool { 419 return pathHitCount[ret[i].Path] > pathHitCount[ret[j].Path] 420 }) 421 } else { 422 sort.Slice(ret, func(i, j int) bool { 423 return ret[i].Updated > ret[j].Updated 424 }) 425 } 426 427 if Conf.Search.Limit <= len(ret) { 428 ret = ret[:Conf.Search.Limit] 429 } 430 return 431} 432 433func GetAssetAbsPath(relativePath string) (ret string, err error) { 434 relativePath = strings.TrimSpace(relativePath) 435 if strings.Contains(relativePath, "?") { 436 relativePath = relativePath[:strings.Index(relativePath, "?")] 437 } 438 439 // 在全局 assets 路径下搜索 440 p := filepath.Join(util.DataDir, relativePath) 441 if gulu.File.IsExist(p) { 442 ret = p 443 if !util.IsSubPath(util.WorkspaceDir, ret) { 444 err = fmt.Errorf("[%s] is not sub path of workspace", ret) 445 return 446 } 447 return 448 } 449 450 // 在笔记本下搜索 451 notebooks, err := ListNotebooks() 452 if err != nil { 453 err = errors.New(Conf.Language(0)) 454 return 455 } 456 for _, notebook := range notebooks { 457 notebookAbsPath := filepath.Join(util.DataDir, notebook.ID) 458 filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error { 459 if isSkipFile(d.Name()) { 460 if d.IsDir() { 461 return filepath.SkipDir 462 } 463 return nil 464 } 465 if p := filepath.ToSlash(path); strings.HasSuffix(p, relativePath) { 466 if gulu.File.IsExist(path) { 467 ret = path 468 return fs.SkipAll 469 } 470 } 471 return nil 472 }) 473 474 if "" != ret { 475 if !util.IsSubPath(util.WorkspaceDir, ret) { 476 err = fmt.Errorf("[%s] is not sub path of workspace", ret) 477 return 478 } 479 return 480 } 481 } 482 return "", errors.New(fmt.Sprintf(Conf.Language(12), relativePath)) 483} 484 485func UploadAssets2Cloud(id string) (count int, err error) { 486 if !IsSubscriber() { 487 return 488 } 489 490 tree, err := LoadTreeByBlockID(id) 491 if err != nil { 492 return 493 } 494 495 node := treenode.GetNodeInTree(tree, id) 496 if nil == node { 497 err = ErrBlockNotFound 498 return 499 } 500 501 nodes := []*ast.Node{node} 502 if ast.NodeHeading == node.Type { 503 nodes = append(nodes, treenode.HeadingChildren(node)...) 504 } 505 506 var assets []string 507 for _, n := range nodes { 508 assets = append(assets, getAssetsLinkDests(n)...) 509 assets = append(assets, getQueryEmbedNodesAssetsLinkDests(n)...) 510 } 511 assets = gulu.Str.RemoveDuplicatedElem(assets) 512 count, err = uploadAssets2Cloud(assets, bizTypeUploadAssets) 513 if err != nil { 514 return 515 } 516 return 517} 518 519const ( 520 bizTypeUploadAssets = "upload-assets" 521 bizTypeExport2Liandi = "export-liandi" 522) 523 524// uploadAssets2Cloud 将资源文件上传到云端图床。 525func uploadAssets2Cloud(assetPaths []string, bizType string) (count int, err error) { 526 var uploadAbsAssets []string 527 for _, assetPath := range assetPaths { 528 var absPath string 529 absPath, err = GetAssetAbsPath(assetPath) 530 if err != nil { 531 logging.LogWarnf("get asset [%s] abs path failed: %s", assetPath, err) 532 return 533 } 534 if "" == absPath { 535 logging.LogErrorf("not found asset [%s]", assetPath) 536 continue 537 } 538 539 uploadAbsAssets = append(uploadAbsAssets, absPath) 540 } 541 542 uploadAbsAssets = gulu.Str.RemoveDuplicatedElem(uploadAbsAssets) 543 if 1 > len(uploadAbsAssets) { 544 return 545 } 546 547 logging.LogInfof("uploading [%d] assets", len(uploadAbsAssets)) 548 msgId := util.PushMsg(fmt.Sprintf(Conf.Language(27), len(uploadAbsAssets)), 3000) 549 if loadErr := LoadUploadToken(); nil != loadErr { 550 util.PushMsg(loadErr.Error(), 5000) 551 return 552 } 553 554 limitSize := uint64(3 * 1024 * 1024) // 3MB 555 if IsSubscriber() { 556 limitSize = 10 * 1024 * 1024 // 10MB 557 } 558 559 // metaType 为服务端 Filemeta.FILEMETA_TYPE,这里只有两个值: 560 // 561 // 5: SiYuan,表示为 SiYuan 上传图床 562 // 4: Client,表示作为客户端分享发布帖子时上传的文件 563 var metaType = "5" 564 if bizTypeUploadAssets == bizType { 565 metaType = "5" 566 } else if bizTypeExport2Liandi == bizType { 567 metaType = "4" 568 } 569 570 pushErrMsgCount := 0 571 var completedUploadAssets []string 572 for _, absAsset := range uploadAbsAssets { 573 fi, statErr := os.Stat(absAsset) 574 if nil != statErr { 575 logging.LogErrorf("stat file [%s] failed: %s", absAsset, statErr) 576 return count, statErr 577 } 578 579 if limitSize < uint64(fi.Size()) { 580 logging.LogWarnf("file [%s] larger than limit size [%s], ignore uploading it", absAsset, humanize.IBytes(limitSize)) 581 if 3 > pushErrMsgCount { 582 msg := fmt.Sprintf(Conf.Language(247), filepath.Base(absAsset), humanize.IBytes(limitSize)) 583 util.PushErrMsg(msg, 30000) 584 } 585 pushErrMsgCount++ 586 continue 587 } 588 589 msg := fmt.Sprintf(Conf.Language(27), html.EscapeString(absAsset)) 590 util.PushStatusBar(msg) 591 util.PushUpdateMsg(msgId, msg, 3000) 592 593 requestResult := gulu.Ret.NewResult() 594 request := httpclient.NewCloudFileRequest2m() 595 resp, reqErr := request. 596 SetSuccessResult(requestResult). 597 SetFile("file[]", absAsset). 598 SetCookies(&http.Cookie{Name: "symphony", Value: uploadToken}). 599 SetHeader("meta-type", metaType). 600 SetHeader("biz-type", bizType). 601 Post(util.GetCloudServer() + "/apis/siyuan/upload?ver=" + util.Ver) 602 if nil != reqErr { 603 logging.LogErrorf("upload assets failed: %s", reqErr) 604 return count, ErrFailedToConnectCloudServer 605 } 606 607 if 401 == resp.StatusCode { 608 err = errors.New(Conf.Language(31)) 609 return 610 } 611 612 if 0 != requestResult.Code { 613 logging.LogErrorf("upload assets failed: %s", requestResult.Msg) 614 err = errors.New(fmt.Sprintf(Conf.Language(94), requestResult.Msg)) 615 return 616 } 617 618 absAsset = filepath.ToSlash(absAsset) 619 relAsset := absAsset[strings.Index(absAsset, "assets/"):] 620 completedUploadAssets = append(completedUploadAssets, relAsset) 621 logging.LogInfof("uploaded asset [%s]", relAsset) 622 count++ 623 } 624 util.PushClearMsg(msgId) 625 626 if 0 < len(completedUploadAssets) { 627 logging.LogInfof("uploaded [%d] assets", len(completedUploadAssets)) 628 } 629 return 630} 631 632func RemoveUnusedAssets() (ret []string) { 633 ret = []string{} 634 var size int64 635 636 msgId := util.PushMsg(Conf.Language(100), 30*1000) 637 defer func() { 638 msg := fmt.Sprintf(Conf.Language(91), len(ret), humanize.BytesCustomCeil(uint64(size), 2)) 639 util.PushUpdateMsg(msgId, msg, 7000) 640 }() 641 642 unusedAssets := UnusedAssets() 643 644 historyDir, err := GetHistoryDir(HistoryOpClean) 645 if err != nil { 646 logging.LogErrorf("get history dir failed: %s", err) 647 return 648 } 649 650 var hashes []string 651 for _, p := range unusedAssets { 652 historyPath := filepath.Join(historyDir, p) 653 if p = filepath.Join(util.DataDir, p); filelock.IsExist(p) { 654 if filelock.IsHidden(p) { 655 continue 656 } 657 658 if err = filelock.Copy(p, historyPath); err != nil { 659 return 660 } 661 662 hash, _ := util.GetEtag(p) 663 hashes = append(hashes, hash) 664 cache.RemoveAssetHash(hash) 665 } 666 } 667 668 sql.BatchRemoveAssetsQueue(hashes) 669 670 for _, unusedAsset := range unusedAssets { 671 absPath := filepath.Join(util.DataDir, unusedAsset) 672 if filelock.IsExist(absPath) { 673 info, statErr := os.Stat(absPath) 674 if statErr == nil { 675 if info.IsDir() { 676 dirSize, _ := util.SizeOfDirectory(absPath) 677 size += dirSize 678 } else { 679 size += info.Size() 680 } 681 } 682 683 if err := filelock.Remove(absPath); err != nil { 684 logging.LogErrorf("remove unused asset [%s] failed: %s", absPath, err) 685 } 686 util.RemoveAssetText(unusedAsset) 687 } 688 ret = append(ret, absPath) 689 } 690 if 0 < len(ret) { 691 IncSync() 692 } 693 694 indexHistoryDir(filepath.Base(historyDir), util.NewLute()) 695 cache.LoadAssets() 696 return 697} 698 699func RemoveUnusedAsset(p string) (ret string) { 700 absPath := filepath.Join(util.DataDir, p) 701 if !filelock.IsExist(absPath) { 702 return absPath 703 } 704 705 historyDir, err := GetHistoryDir(HistoryOpClean) 706 if err != nil { 707 logging.LogErrorf("get history dir failed: %s", err) 708 return 709 } 710 711 newP := strings.TrimPrefix(absPath, util.DataDir) 712 historyPath := filepath.Join(historyDir, newP) 713 if filelock.IsExist(absPath) { 714 if err = filelock.Copy(absPath, historyPath); err != nil { 715 return 716 } 717 718 hash, _ := util.GetEtag(absPath) 719 sql.BatchRemoveAssetsQueue([]string{hash}) 720 cache.RemoveAssetHash(hash) 721 } 722 723 if err = filelock.Remove(absPath); err != nil { 724 logging.LogErrorf("remove unused asset [%s] failed: %s", absPath, err) 725 } 726 ret = absPath 727 728 util.RemoveAssetText(p) 729 730 IncSync() 731 732 indexHistoryDir(filepath.Base(historyDir), util.NewLute()) 733 cache.RemoveAsset(p) 734 return 735} 736 737func RenameAsset(oldPath, newName string) (newPath string, err error) { 738 util.PushEndlessProgress(Conf.Language(110)) 739 defer util.PushClearProgress() 740 741 newName = strings.TrimSpace(newName) 742 newName = util.FilterUploadFileName(newName) 743 if path.Base(oldPath) == newName { 744 return 745 } 746 if "" == newName { 747 return 748 } 749 750 if !gulu.File.IsValidFilename(newName) { 751 err = errors.New(Conf.Language(151)) 752 return 753 } 754 755 newName = util.AssetName(newName+filepath.Ext(oldPath), ast.NewNodeID()) 756 parentDir := path.Dir(oldPath) 757 newPath = path.Join(parentDir, newName) 758 oldAbsPath, getErr := GetAssetAbsPath(oldPath) 759 if getErr != nil { 760 logging.LogErrorf("get asset [%s] abs path failed: %s", oldPath, getErr) 761 return 762 } 763 newAbsPath := filepath.Join(filepath.Dir(oldAbsPath), newName) 764 if err = filelock.Copy(oldAbsPath, newAbsPath); err != nil { 765 logging.LogErrorf("copy asset [%s] failed: %s", oldAbsPath, err) 766 return 767 } 768 769 if filelock.IsExist(filepath.Join(util.DataDir, oldPath+".sya")) { 770 // Rename the .sya annotation file when renaming a PDF asset https://github.com/siyuan-note/siyuan/issues/9390 771 if err = filelock.Copy(filepath.Join(util.DataDir, oldPath+".sya"), filepath.Join(util.DataDir, newPath+".sya")); err != nil { 772 logging.LogErrorf("copy PDF annotation [%s] failed: %s", oldPath+".sya", err) 773 return 774 } 775 } 776 777 oldName := path.Base(oldPath) 778 779 notebooks, err := ListNotebooks() 780 if err != nil { 781 return 782 } 783 784 luteEngine := util.NewLute() 785 for _, notebook := range notebooks { 786 pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32) 787 788 for _, paths := range pages { 789 for _, treeAbsPath := range paths { 790 data, readErr := filelock.ReadFile(treeAbsPath) 791 if nil != readErr { 792 logging.LogErrorf("get data [path=%s] failed: %s", treeAbsPath, readErr) 793 err = readErr 794 return 795 } 796 797 if !bytes.Contains(data, []byte(oldName)) { 798 continue 799 } 800 801 data = bytes.Replace(data, []byte(oldName), []byte(newName), -1) 802 if writeErr := filelock.WriteFile(treeAbsPath, data); nil != writeErr { 803 logging.LogErrorf("write data [path=%s] failed: %s", treeAbsPath, writeErr) 804 err = writeErr 805 return 806 } 807 808 p := filepath.ToSlash(strings.TrimPrefix(treeAbsPath, filepath.Join(util.DataDir, notebook.ID))) 809 tree, parseErr := filesys.LoadTreeByData(data, notebook.ID, p, luteEngine) 810 if nil != parseErr { 811 logging.LogWarnf("parse json to tree [%s] failed: %s", treeAbsPath, parseErr) 812 continue 813 } 814 815 treenode.UpsertBlockTree(tree) 816 sql.UpsertTreeQueue(tree) 817 818 util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(tree.Root.IALAttr("title")))) 819 } 820 } 821 } 822 823 storageAvDir := filepath.Join(util.DataDir, "storage", "av") 824 if gulu.File.IsDir(storageAvDir) { 825 entries, readErr := os.ReadDir(storageAvDir) 826 if nil != readErr { 827 logging.LogErrorf("read dir [%s] failed: %s", storageAvDir, readErr) 828 err = readErr 829 return 830 } 831 832 for _, entry := range entries { 833 if !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(entry.Name(), ".json")) { 834 continue 835 } 836 837 data, readDataErr := filelock.ReadFile(filepath.Join(util.DataDir, "storage", "av", entry.Name())) 838 if nil != readDataErr { 839 logging.LogErrorf("read file [%s] failed: %s", entry.Name(), readDataErr) 840 err = readDataErr 841 return 842 } 843 844 if bytes.Contains(data, []byte(oldPath)) { 845 data = bytes.ReplaceAll(data, []byte(oldPath), []byte(newPath)) 846 if writeDataErr := filelock.WriteFile(filepath.Join(util.DataDir, "storage", "av", entry.Name()), data); nil != writeDataErr { 847 logging.LogErrorf("write file [%s] failed: %s", entry.Name(), writeDataErr) 848 err = writeDataErr 849 return 850 } 851 } 852 853 util.PushEndlessProgress(fmt.Sprintf(Conf.Language(111), util.EscapeHTML(entry.Name()))) 854 } 855 } 856 857 if ocrText := util.GetAssetText(oldPath); "" != ocrText { 858 // 图片重命名后 ocr-texts.json 需要更新 https://github.com/siyuan-note/siyuan/issues/12974 859 util.SetAssetText(newPath, ocrText) 860 } 861 862 IncSync() 863 return 864} 865 866func UnusedAssets() (ret []string) { 867 defer logging.Recover() 868 ret = []string{} 869 870 assetsPathMap, err := allAssetAbsPaths() 871 if err != nil { 872 return 873 } 874 linkDestMap := map[string]bool{} 875 notebooks, err := ListNotebooks() 876 if err != nil { 877 return 878 } 879 luteEngine := util.NewLute() 880 for _, notebook := range notebooks { 881 dests := map[string]bool{} 882 883 // 分页加载,优化清理未引用资源内存占用 https://github.com/siyuan-note/siyuan/issues/5200 884 pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32) 885 for _, paths := range pages { 886 var trees []*parse.Tree 887 for _, localPath := range paths { 888 tree, loadTreeErr := loadTree(localPath, luteEngine) 889 if nil != loadTreeErr { 890 continue 891 } 892 trees = append(trees, tree) 893 } 894 for _, tree := range trees { 895 for _, d := range getAssetsLinkDests(tree.Root) { 896 dests[d] = true 897 } 898 899 if titleImgPath := treenode.GetDocTitleImgPath(tree.Root); "" != titleImgPath { 900 // 题头图计入 901 if !util.IsAssetLinkDest([]byte(titleImgPath)) { 902 continue 903 } 904 dests[titleImgPath] = true 905 } 906 } 907 } 908 909 var linkDestFolderPaths, linkDestFilePaths []string 910 for dest := range dests { 911 if !strings.HasPrefix(dest, "assets/") { 912 continue 913 } 914 915 if idx := strings.Index(dest, "?"); 0 < idx { 916 // `pdf?page` 资源文件链接会被判定为未引用资源 https://github.com/siyuan-note/siyuan/issues/5649 917 dest = dest[:idx] 918 } 919 920 if "" == assetsPathMap[dest] { 921 continue 922 } 923 if strings.HasSuffix(dest, "/") { 924 linkDestFolderPaths = append(linkDestFolderPaths, dest) 925 } else { 926 linkDestFilePaths = append(linkDestFilePaths, dest) 927 } 928 } 929 930 // 排除文件夹链接 931 var toRemoves []string 932 for asset := range assetsPathMap { 933 for _, linkDestFolder := range linkDestFolderPaths { 934 if strings.HasPrefix(asset, linkDestFolder) { 935 toRemoves = append(toRemoves, asset) 936 } 937 } 938 for _, linkDestPath := range linkDestFilePaths { 939 if strings.HasPrefix(linkDestPath, asset) { 940 toRemoves = append(toRemoves, asset) 941 } 942 } 943 } 944 for _, toRemove := range toRemoves { 945 delete(assetsPathMap, toRemove) 946 } 947 948 for _, dest := range linkDestFilePaths { 949 linkDestMap[dest] = true 950 951 if strings.HasSuffix(dest, ".pdf") { 952 linkDestMap[dest+".sya"] = true 953 } 954 } 955 } 956 957 var toRemoves []string 958 for asset := range assetsPathMap { 959 if strings.HasSuffix(asset, "ocr-texts.json") { 960 // 排除 OCR 结果文本 961 toRemoves = append(toRemoves, asset) 962 continue 963 } 964 965 if strings.HasSuffix(asset, "android-notification-texts.txt") { 966 // 排除 Android 通知文本 967 toRemoves = append(toRemoves, asset) 968 continue 969 } 970 } 971 972 // 排除数据库中引用的资源文件 973 storageAvDir := filepath.Join(util.DataDir, "storage", "av") 974 if gulu.File.IsDir(storageAvDir) { 975 entries, readErr := os.ReadDir(storageAvDir) 976 if nil != readErr { 977 logging.LogErrorf("read dir [%s] failed: %s", storageAvDir, readErr) 978 err = readErr 979 return 980 } 981 982 for _, entry := range entries { 983 if !strings.HasSuffix(entry.Name(), ".json") || !ast.IsNodeIDPattern(strings.TrimSuffix(entry.Name(), ".json")) { 984 continue 985 } 986 987 data, readDataErr := filelock.ReadFile(filepath.Join(util.DataDir, "storage", "av", entry.Name())) 988 if nil != readDataErr { 989 logging.LogErrorf("read file [%s] failed: %s", entry.Name(), readDataErr) 990 err = readDataErr 991 return 992 } 993 994 for asset := range assetsPathMap { 995 if bytes.Contains(data, []byte(asset)) { 996 toRemoves = append(toRemoves, asset) 997 } 998 } 999 } 1000 } 1001 1002 for _, toRemove := range toRemoves { 1003 delete(assetsPathMap, toRemove) 1004 } 1005 1006 dataAssetsAbsPath := util.GetDataAssetsAbsPath() 1007 for dest, assetAbsPath := range assetsPathMap { 1008 if _, ok := linkDestMap[dest]; ok { 1009 continue 1010 } 1011 1012 var p string 1013 if strings.HasPrefix(dataAssetsAbsPath, assetAbsPath) { 1014 p = assetAbsPath[strings.Index(assetAbsPath, "assets"):] 1015 } else { 1016 p = strings.TrimPrefix(assetAbsPath, filepath.Dir(dataAssetsAbsPath)) 1017 } 1018 p = filepath.ToSlash(p) 1019 if strings.HasPrefix(p, "/") { 1020 p = p[1:] 1021 } 1022 ret = append(ret, p) 1023 } 1024 sort.Strings(ret) 1025 return 1026} 1027 1028func MissingAssets() (ret []string) { 1029 defer logging.Recover() 1030 ret = []string{} 1031 1032 assetsPathMap, err := allAssetAbsPaths() 1033 if err != nil { 1034 return 1035 } 1036 notebooks, err := ListNotebooks() 1037 if err != nil { 1038 return 1039 } 1040 luteEngine := util.NewLute() 1041 for _, notebook := range notebooks { 1042 if notebook.Closed { 1043 continue 1044 } 1045 1046 dests := map[string]bool{} 1047 pages := pagedPaths(filepath.Join(util.DataDir, notebook.ID), 32) 1048 for _, paths := range pages { 1049 var trees []*parse.Tree 1050 for _, localPath := range paths { 1051 tree, loadTreeErr := loadTree(localPath, luteEngine) 1052 if nil != loadTreeErr { 1053 continue 1054 } 1055 trees = append(trees, tree) 1056 } 1057 for _, tree := range trees { 1058 for _, d := range getAssetsLinkDests(tree.Root) { 1059 dests[d] = true 1060 } 1061 1062 if titleImgPath := treenode.GetDocTitleImgPath(tree.Root); "" != titleImgPath { 1063 // 题头图计入 1064 if !util.IsAssetLinkDest([]byte(titleImgPath)) { 1065 continue 1066 } 1067 dests[titleImgPath] = true 1068 } 1069 } 1070 } 1071 1072 for dest := range dests { 1073 if !strings.HasPrefix(dest, "assets/") { 1074 continue 1075 } 1076 1077 if idx := strings.Index(dest, "?"); 0 < idx { 1078 dest = dest[:idx] 1079 } 1080 1081 if strings.HasSuffix(dest, "/") { 1082 continue 1083 } 1084 1085 if strings.Contains(strings.ToLower(dest), ".pdf/") { 1086 if idx := strings.LastIndex(dest, "/"); -1 < idx { 1087 if ast.IsNodeIDPattern(dest[idx+1:]) { 1088 // PDF 标注不计入 https://github.com/siyuan-note/siyuan/issues/13891 1089 continue 1090 } 1091 } 1092 } 1093 1094 if "" == assetsPathMap[dest] { 1095 if strings.HasPrefix(dest, "assets/.") { 1096 // Assets starting with `.` should not be considered missing assets https://github.com/siyuan-note/siyuan/issues/8821 1097 if !filelock.IsExist(filepath.Join(util.DataDir, dest)) { 1098 ret = append(ret, dest) 1099 } 1100 } else { 1101 ret = append(ret, dest) 1102 } 1103 continue 1104 } 1105 } 1106 } 1107 1108 sort.Strings(ret) 1109 return 1110} 1111 1112func emojisInTree(tree *parse.Tree) (ret []string) { 1113 if icon := tree.Root.IALAttr("icon"); "" != icon { 1114 if !strings.Contains(icon, "://") && !strings.HasPrefix(icon, "api/icon/") && !util.NativeEmojiChars[icon] { 1115 ret = append(ret, "/emojis/"+icon) 1116 } 1117 } 1118 1119 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { 1120 if !entering { 1121 return ast.WalkContinue 1122 } 1123 if ast.NodeEmojiImg == n.Type { 1124 tokens := n.Tokens 1125 idx := bytes.Index(tokens, []byte("src=\"")) 1126 if -1 == idx { 1127 return ast.WalkContinue 1128 } 1129 src := tokens[idx+len("src=\""):] 1130 src = src[:bytes.Index(src, []byte("\""))] 1131 ret = append(ret, string(src)) 1132 } 1133 return ast.WalkContinue 1134 }) 1135 ret = gulu.Str.RemoveDuplicatedElem(ret) 1136 return 1137} 1138 1139func getQueryEmbedNodesAssetsLinkDests(node *ast.Node) (ret []string) { 1140 // The images in the embed blocks are not uploaded to the community hosting https://github.com/siyuan-note/siyuan/issues/10042 1141 1142 ret = []string{} 1143 ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus { 1144 if !entering || ast.NodeBlockQueryEmbedScript != n.Type { 1145 return ast.WalkContinue 1146 } 1147 1148 stmt := n.TokensStr() 1149 stmt = html.UnescapeString(stmt) 1150 stmt = strings.ReplaceAll(stmt, editor.IALValEscNewLine, "\n") 1151 sqlBlocks := sql.SelectBlocksRawStmt(stmt, 1, Conf.Search.Limit) 1152 for _, sqlBlock := range sqlBlocks { 1153 subtree, _ := LoadTreeByBlockID(sqlBlock.ID) 1154 if nil == subtree { 1155 continue 1156 } 1157 embedNode := treenode.GetNodeInTree(subtree, sqlBlock.ID) 1158 if nil == embedNode { 1159 continue 1160 } 1161 1162 ret = append(ret, getAssetsLinkDests(embedNode)...) 1163 } 1164 return ast.WalkContinue 1165 }) 1166 ret = gulu.Str.RemoveDuplicatedElem(ret) 1167 return 1168} 1169 1170func getAssetsLinkDests(node *ast.Node) (ret []string) { 1171 ret = []string{} 1172 ast.Walk(node, func(n *ast.Node, entering bool) ast.WalkStatus { 1173 if n.IsBlock() { 1174 // 以 custom-data-assets 开头的块属性值可能是多个资源文件链接,需要计入 1175 // Ignore assets associated with the `custom-data-assets` block attribute when cleaning unreferenced assets https://github.com/siyuan-note/siyuan/issues/12574 1176 for _, kv := range n.KramdownIAL { 1177 k := kv[0] 1178 if strings.HasPrefix(k, "custom-data-assets") { 1179 dest := kv[1] 1180 if "" == dest || !util.IsAssetLinkDest([]byte(dest)) { 1181 continue 1182 } 1183 ret = append(ret, dest) 1184 } 1185 } 1186 } 1187 1188 // 修改以下代码时需要同时修改 database 构造行级元素实现,增加必要的类型 1189 if !entering || (ast.NodeLinkDest != n.Type && ast.NodeHTMLBlock != n.Type && ast.NodeInlineHTML != n.Type && 1190 ast.NodeIFrame != n.Type && ast.NodeWidget != n.Type && ast.NodeAudio != n.Type && ast.NodeVideo != n.Type && 1191 ast.NodeAttributeView != n.Type && !n.IsTextMarkType("a") && !n.IsTextMarkType("file-annotation-ref")) { 1192 return ast.WalkContinue 1193 } 1194 1195 if ast.NodeLinkDest == n.Type { 1196 if !util.IsAssetLinkDest(n.Tokens) { 1197 return ast.WalkContinue 1198 } 1199 1200 dest := strings.TrimSpace(string(n.Tokens)) 1201 ret = append(ret, dest) 1202 } else if n.IsTextMarkType("a") { 1203 if !util.IsAssetLinkDest(gulu.Str.ToBytes(n.TextMarkAHref)) { 1204 return ast.WalkContinue 1205 } 1206 1207 dest := strings.TrimSpace(n.TextMarkAHref) 1208 ret = append(ret, dest) 1209 } else if n.IsTextMarkType("file-annotation-ref") { 1210 if !util.IsAssetLinkDest(gulu.Str.ToBytes(n.TextMarkFileAnnotationRefID)) { 1211 return ast.WalkContinue 1212 } 1213 1214 if !strings.Contains(n.TextMarkFileAnnotationRefID, "/") { 1215 return ast.WalkContinue 1216 } 1217 1218 dest := n.TextMarkFileAnnotationRefID[:strings.LastIndexByte(n.TextMarkFileAnnotationRefID, '/')] 1219 dest = strings.TrimSpace(dest) 1220 ret = append(ret, dest) 1221 } else if ast.NodeAttributeView == n.Type { 1222 attrView, _ := av.ParseAttributeView(n.AttributeViewID) 1223 if nil == attrView { 1224 return ast.WalkContinue 1225 } 1226 1227 for _, keyValues := range attrView.KeyValues { 1228 if av.KeyTypeMAsset == keyValues.Key.Type { 1229 for _, value := range keyValues.Values { 1230 if 1 > len(value.MAsset) { 1231 continue 1232 } 1233 1234 for _, asset := range value.MAsset { 1235 dest := asset.Content 1236 if !util.IsAssetLinkDest([]byte(dest)) { 1237 continue 1238 } 1239 ret = append(ret, strings.TrimSpace(dest)) 1240 } 1241 } 1242 } else if av.KeyTypeURL == keyValues.Key.Type { 1243 for _, value := range keyValues.Values { 1244 if nil != value.URL { 1245 dest := value.URL.Content 1246 if !util.IsAssetLinkDest([]byte(dest)) { 1247 continue 1248 } 1249 ret = append(ret, strings.TrimSpace(dest)) 1250 } 1251 } 1252 } 1253 } 1254 } else { 1255 if ast.NodeWidget == n.Type { 1256 dataAssets := n.IALAttr("custom-data-assets") 1257 if "" == dataAssets { 1258 // 兼容两种属性名 custom-data-assets 和 data-assets https://github.com/siyuan-note/siyuan/issues/4122#issuecomment-1154796568 1259 dataAssets = n.IALAttr("data-assets") 1260 } 1261 if !util.IsAssetLinkDest([]byte(dataAssets)) { 1262 return ast.WalkContinue 1263 } 1264 ret = append(ret, dataAssets) 1265 } else { // HTMLBlock/InlineHTML/IFrame/Audio/Video 1266 dest := treenode.GetNodeSrcTokens(n) 1267 if !util.IsAssetLinkDest([]byte(dest)) { 1268 return ast.WalkContinue 1269 } 1270 ret = append(ret, dest) 1271 } 1272 } 1273 return ast.WalkContinue 1274 }) 1275 ret = gulu.Str.RemoveDuplicatedElem(ret) 1276 for i, dest := range ret { 1277 // 对于 macOS 的 rtfd 文件夹格式需要特殊处理,为其加上结尾 / 1278 if strings.HasSuffix(dest, ".rtfd") { 1279 ret[i] = dest + "/" 1280 } 1281 } 1282 return 1283} 1284 1285func setAssetsLinkDest(node *ast.Node, oldDest, dest string) { 1286 if ast.NodeLinkDest == node.Type { 1287 if bytes.HasPrefix(node.Tokens, []byte("//")) { 1288 node.Tokens = append([]byte("https:"), node.Tokens...) 1289 } 1290 node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest)) 1291 } else if node.IsTextMarkType("a") { 1292 if strings.HasPrefix(node.TextMarkAHref, "//") { 1293 node.TextMarkAHref = "https:" + node.TextMarkAHref 1294 } 1295 node.TextMarkAHref = strings.ReplaceAll(node.TextMarkAHref, oldDest, dest) 1296 } else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type { 1297 if strings.HasPrefix(node.TextMarkAHref, "//") { 1298 node.TextMarkAHref = "https:" + node.TextMarkAHref 1299 } 1300 node.Tokens = bytes.ReplaceAll(node.Tokens, []byte(oldDest), []byte(dest)) 1301 } else if ast.NodeAttributeView == node.Type { 1302 needWrite := false 1303 attrView, _ := av.ParseAttributeView(node.AttributeViewID) 1304 if nil == attrView { 1305 return 1306 } 1307 1308 for _, keyValues := range attrView.KeyValues { 1309 if av.KeyTypeMAsset != keyValues.Key.Type { 1310 continue 1311 } 1312 1313 for _, value := range keyValues.Values { 1314 if 1 > len(value.MAsset) { 1315 continue 1316 } 1317 1318 for _, asset := range value.MAsset { 1319 if oldDest == asset.Content && oldDest != dest { 1320 asset.Content = dest 1321 needWrite = true 1322 } 1323 } 1324 } 1325 } 1326 1327 if needWrite { 1328 av.SaveAttributeView(attrView) 1329 } 1330 } 1331} 1332 1333func getRemoteAssetsLinkDests(node *ast.Node, onlyImg bool) (ret []string) { 1334 if onlyImg { 1335 if ast.NodeLinkDest == node.Type { 1336 if node.ParentIs(ast.NodeImage) { 1337 if !util.IsAssetLinkDest(node.Tokens) { 1338 ret = append(ret, string(node.Tokens)) 1339 } 1340 1341 } 1342 } else if ast.NodeAttributeView == node.Type { 1343 attrView, _ := av.ParseAttributeView(node.AttributeViewID) 1344 if nil == attrView { 1345 return 1346 } 1347 1348 for _, keyValues := range attrView.KeyValues { 1349 if av.KeyTypeMAsset != keyValues.Key.Type { 1350 continue 1351 } 1352 1353 for _, value := range keyValues.Values { 1354 if 1 > len(value.MAsset) { 1355 continue 1356 } 1357 1358 for _, asset := range value.MAsset { 1359 if av.AssetTypeImage != asset.Type { 1360 continue 1361 } 1362 1363 dest := asset.Content 1364 if !util.IsAssetLinkDest([]byte(dest)) { 1365 ret = append(ret, strings.TrimSpace(dest)) 1366 } 1367 } 1368 } 1369 } 1370 } 1371 } else { 1372 if ast.NodeLinkDest == node.Type { 1373 if !util.IsAssetLinkDest(node.Tokens) { 1374 ret = append(ret, string(node.Tokens)) 1375 } 1376 } else if node.IsTextMarkType("a") { 1377 if !util.IsAssetLinkDest([]byte(node.TextMarkAHref)) { 1378 ret = append(ret, node.TextMarkAHref) 1379 } 1380 } else if ast.NodeAudio == node.Type || ast.NodeVideo == node.Type { 1381 src := treenode.GetNodeSrcTokens(node) 1382 if !util.IsAssetLinkDest([]byte(src)) { 1383 ret = append(ret, src) 1384 } 1385 } else if ast.NodeAttributeView == node.Type { 1386 attrView, _ := av.ParseAttributeView(node.AttributeViewID) 1387 if nil == attrView { 1388 return 1389 } 1390 1391 for _, keyValues := range attrView.KeyValues { 1392 if av.KeyTypeMAsset != keyValues.Key.Type { 1393 continue 1394 } 1395 1396 for _, value := range keyValues.Values { 1397 if 1 > len(value.MAsset) { 1398 continue 1399 } 1400 1401 for _, asset := range value.MAsset { 1402 dest := asset.Content 1403 if !util.IsAssetLinkDest([]byte(dest)) { 1404 ret = append(ret, strings.TrimSpace(dest)) 1405 } 1406 } 1407 } 1408 } 1409 } 1410 } 1411 return 1412} 1413 1414func getRemoteAssetsLinkDestsInTree(tree *parse.Tree, onlyImg bool) (nodes []*ast.Node) { 1415 ast.Walk(tree.Root, func(n *ast.Node, entering bool) ast.WalkStatus { 1416 if !entering { 1417 return ast.WalkContinue 1418 } 1419 1420 dests := getRemoteAssetsLinkDests(n, onlyImg) 1421 if 1 > len(dests) { 1422 return ast.WalkContinue 1423 } 1424 1425 nodes = append(nodes, n) 1426 return ast.WalkContinue 1427 }) 1428 return 1429} 1430 1431// allAssetAbsPaths 返回 asset 相对路径(assets/xxx)到绝对路径(F:\SiYuan\data\assets\xxx)的映射。 1432func allAssetAbsPaths() (assetsAbsPathMap map[string]string, err error) { 1433 notebooks, err := ListNotebooks() 1434 if err != nil { 1435 return 1436 } 1437 1438 assetsAbsPathMap = map[string]string{} 1439 // 笔记本 assets 1440 for _, notebook := range notebooks { 1441 notebookAbsPath := filepath.Join(util.DataDir, notebook.ID) 1442 filelock.Walk(notebookAbsPath, func(path string, d fs.DirEntry, err error) error { 1443 if notebookAbsPath == path { 1444 return nil 1445 } 1446 if isSkipFile(d.Name()) { 1447 if d.IsDir() { 1448 return filepath.SkipDir 1449 } 1450 return nil 1451 } 1452 1453 if filelock.IsHidden(path) { 1454 // 清理资源文件时忽略隐藏文件 Ignore hidden files when cleaning unused assets https://github.com/siyuan-note/siyuan/issues/12172 1455 return nil 1456 } 1457 1458 if d.IsDir() && "assets" == d.Name() { 1459 filelock.Walk(path, func(assetPath string, d fs.DirEntry, err error) error { 1460 if path == assetPath { 1461 return nil 1462 } 1463 if isSkipFile(d.Name()) { 1464 if d.IsDir() { 1465 return filepath.SkipDir 1466 } 1467 return nil 1468 } 1469 relPath := filepath.ToSlash(assetPath) 1470 relPath = relPath[strings.Index(relPath, "assets/"):] 1471 if d.IsDir() { 1472 relPath += "/" 1473 } 1474 assetsAbsPathMap[relPath] = assetPath 1475 return nil 1476 }) 1477 return filepath.SkipDir 1478 } 1479 return nil 1480 }) 1481 } 1482 1483 // 全局 assets 1484 dataAssetsAbsPath := util.GetDataAssetsAbsPath() 1485 filelock.Walk(dataAssetsAbsPath, func(assetPath string, d fs.DirEntry, err error) error { 1486 if dataAssetsAbsPath == assetPath { 1487 return nil 1488 } 1489 1490 if isSkipFile(d.Name()) { 1491 if d.IsDir() { 1492 return filepath.SkipDir 1493 } 1494 return nil 1495 } 1496 1497 if filelock.IsHidden(assetPath) { 1498 // 清理资源文件时忽略隐藏文件 Ignore hidden files when cleaning unused assets https://github.com/siyuan-note/siyuan/issues/12172 1499 return nil 1500 } 1501 1502 relPath := filepath.ToSlash(assetPath) 1503 relPath = relPath[strings.Index(relPath, "assets/"):] 1504 if d.IsDir() { 1505 relPath += "/" 1506 } 1507 assetsAbsPathMap[relPath] = assetPath 1508 return nil 1509 }) 1510 return 1511} 1512 1513// copyBoxAssetsToDataAssets 将笔记本路径下所有(包括子文档)的 assets 复制一份到 data/assets 中。 1514func copyBoxAssetsToDataAssets(boxID string) { 1515 boxLocalPath := filepath.Join(util.DataDir, boxID) 1516 copyAssetsToDataAssets(boxLocalPath) 1517} 1518 1519// copyDocAssetsToDataAssets 将文档路径下所有(包括子文档)的 assets 复制一份到 data/assets 中。 1520func copyDocAssetsToDataAssets(boxID, parentDocPath string) { 1521 boxLocalPath := filepath.Join(util.DataDir, boxID) 1522 parentDocDirAbsPath := filepath.Dir(filepath.Join(boxLocalPath, parentDocPath)) 1523 copyAssetsToDataAssets(parentDocDirAbsPath) 1524} 1525 1526func copyAssetsToDataAssets(rootPath string) { 1527 var assetsDirPaths []string 1528 filelock.Walk(rootPath, func(path string, d fs.DirEntry, err error) error { 1529 if nil != err || rootPath == path || nil == d { 1530 return nil 1531 } 1532 1533 isDir, name := d.IsDir(), d.Name() 1534 if isSkipFile(name) { 1535 if isDir { 1536 return filepath.SkipDir 1537 } 1538 return nil 1539 } 1540 1541 if "assets" == name && isDir { 1542 assetsDirPaths = append(assetsDirPaths, path) 1543 } 1544 return nil 1545 }) 1546 1547 dataAssetsPath := filepath.Join(util.DataDir, "assets") 1548 for _, assetsDirPath := range assetsDirPaths { 1549 if err := filelock.Copy(assetsDirPath, dataAssetsPath); err != nil { 1550 logging.LogErrorf("copy tree assets from [%s] to [%s] failed: %s", assetsDirPaths, dataAssetsPath, err) 1551 } 1552 } 1553}