bnewbold.net / cobalt
go scratch code for atproto
fork

Configure Feed

Select the types of activity you want to include in your feed.

refactor some duplicated glot code

bnewbold.net 4ba56c52 90f02e14

+356 -333
+3 -3
cmd/glot/breaking.go
··· 22 22 Flags: []cli.Flag{ 23 23 &cli.StringFlag{ 24 24 Name: "lexicons-dir", 25 - Value: "./lexicons/", 25 + Value: "lexicons/", 26 26 Usage: "base directory for project Lexicon files", 27 27 Sources: cli.EnvVars("LEXICONS_DIR"), 28 28 }, ··· 35 35 } 36 36 37 37 func runBreaking(ctx context.Context, cmd *cli.Command) error { 38 - return compareSchemas(ctx, cmd, breakingCompare) 38 + return runComparisons(ctx, cmd, compareBreaking) 39 39 } 40 40 41 - func breakingCompare(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error { 41 + func compareBreaking(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error { 42 42 43 43 // skip schemas which aren't in both locations 44 44 if localJSON == nil || remoteJSON == nil {
+3 -3
cmd/glot/diff.go
··· 21 21 Flags: []cli.Flag{ 22 22 &cli.StringFlag{ 23 23 Name: "lexicons-dir", 24 - Value: "./lexicons/", 24 + Value: "lexicons/", 25 25 Usage: "base directory for project Lexicon files", 26 26 Sources: cli.EnvVars("LEXICONS_DIR"), 27 27 }, ··· 30 30 } 31 31 32 32 func runDiff(ctx context.Context, cmd *cli.Command) error { 33 - return compareSchemas(ctx, cmd, diffCompare) 33 + return runComparisons(ctx, cmd, compareDiff) 34 34 } 35 35 36 - func diffCompare(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error { 36 + func compareDiff(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error { 37 37 38 38 // skip schemas which aren't in both locations 39 39 if localJSON == nil || remoteJSON == nil {
+4 -43
cmd/glot/dns.go cmd/glot/check_dns.go
··· 2 2 3 3 import ( 4 4 "context" 5 - "encoding/json" 6 5 "fmt" 7 - "io/fs" 8 - "os" 9 - "path" 10 - "path/filepath" 11 6 "sort" 12 7 13 8 "github.com/bluesky-social/indigo/atproto/identity" ··· 24 19 Flags: []cli.Flag{ 25 20 &cli.StringFlag{ 26 21 Name: "lexicons-dir", 27 - Value: "./lexicons/", 22 + Value: "lexicons/", 28 23 Usage: "base directory for project Lexicon files", 29 24 Sources: cli.EnvVars("LEXICONS_DIR"), 30 25 }, ··· 44 39 */ 45 40 func runCheckDNS(ctx context.Context, cmd *cli.Command) error { 46 41 47 - paths := cmd.Args().Slice() 48 - if !cmd.Args().Present() { 49 - paths = []string{cmd.String("lexicons-dir")} 50 - _, err := os.Stat(paths[0]) 51 - if err != nil { 52 - return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err) 53 - } 54 - } 55 - 56 42 // collect all NSID/path mappings 57 - localSchemas := map[syntax.NSID]json.RawMessage{} 58 - 59 - for _, p := range paths { 60 - finfo, err := os.Stat(p) 61 - if err != nil { 62 - return fmt.Errorf("failed loading %s: %w", p, err) 63 - } 64 - if finfo.IsDir() { 65 - if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error { 66 - if d.IsDir() || path.Ext(fp) != ".json" { 67 - return nil 68 - } 69 - nsid, rec, err := loadSchemaPath(fp) 70 - if err != nil { 71 - return err 72 - } 73 - localSchemas[nsid] = *rec 74 - return nil 75 - }); err != nil { 76 - return err 77 - } 78 - continue 79 - } 80 - nsid, rec, err := loadSchemaPath(p) 81 - if err != nil { 82 - return err 83 - } 84 - localSchemas[nsid] = *rec 43 + localSchemas, err := collectSchemaJSON(cmd) 44 + if err != nil { 45 + return err 85 46 } 86 47 87 48 localGroups := map[string]bool{}
+8 -32
cmd/glot/lint.go
··· 6 6 "encoding/json" 7 7 "errors" 8 8 "fmt" 9 - "io/fs" 10 9 "log/slog" 11 10 "os" 12 - "path" 13 - "path/filepath" 14 11 "regexp" 15 12 "slices" 16 13 ··· 33 30 Flags: []cli.Flag{ 34 31 &cli.StringFlag{ 35 32 Name: "lexicons-dir", 36 - Value: "./lexicons/", 33 + Value: "lexicons/", 37 34 Usage: "base directory for project Lexicon files", 38 35 Sources: cli.EnvVars("LEXICONS_DIR"), 39 36 }, ··· 46 43 } 47 44 48 45 func runLint(ctx context.Context, cmd *cli.Command) error { 49 - paths := cmd.Args().Slice() 50 - if !cmd.Args().Present() { 51 - paths = []string{cmd.String("lexicons-dir")} 52 - _, err := os.Stat(paths[0]) 53 - if err != nil { 54 - return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err) 55 - } 46 + 47 + // enumerate lexicon JSON file paths 48 + filePaths, err := collectPaths(cmd) 49 + if err != nil { 50 + return err 56 51 } 57 52 58 53 // TODO: load up entire directory in to a catalog? or have a "linter" struct? 59 54 60 55 slog.Debug("starting lint run") 61 56 anyFailures := false 62 - for _, p := range paths { 63 - finfo, err := os.Stat(p) 57 + for _, fp := range filePaths { 58 + err = lintFilePath(ctx, cmd, fp) 64 59 if err != nil { 65 - return fmt.Errorf("failed loading %s: %w", p, err) 66 - } 67 - if finfo.IsDir() { 68 - if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error { 69 - if d.IsDir() || path.Ext(fp) != ".json" { 70 - return nil 71 - } 72 - err = lintFilePath(ctx, cmd, fp) 73 - if err == ErrLintFailures { 74 - anyFailures = true 75 - return nil 76 - } 77 - return err 78 - }); err != nil { 79 - return err 80 - } 81 - continue 82 - } 83 - if err := lintFilePath(ctx, cmd, p); err != nil { 84 60 if err == ErrLintFailures { 85 61 anyFailures = true 86 62 } else {
+1 -1
cmd/glot/new.go
··· 41 41 Flags: []cli.Flag{ 42 42 &cli.StringFlag{ 43 43 Name: "lexicons-dir", 44 - Value: "./lexicons/", 44 + Value: "lexicons/", 45 45 Usage: "base directory for project Lexicon files", 46 46 Sources: cli.EnvVars("LEXICONS_DIR"), 47 47 },
+6 -44
cmd/glot/publish.go
··· 4 4 "context" 5 5 "encoding/json" 6 6 "fmt" 7 - "io/fs" 8 - "os" 9 - "path" 10 - "path/filepath" 11 7 "reflect" 12 8 "sort" 13 9 ··· 28 24 Flags: []cli.Flag{ 29 25 &cli.StringFlag{ 30 26 Name: "lexicons-dir", 31 - Value: "./lexicons/", 27 + Value: "lexicons/", 32 28 Usage: "base directory for project Lexicon files", 33 29 Sources: cli.EnvVars("LEXICONS_DIR"), 34 30 }, ··· 86 82 return fmt.Errorf("require API client to have DID configured") 87 83 } 88 84 89 - paths := cmd.Args().Slice() 90 - if !cmd.Args().Present() { 91 - paths = []string{cmd.String("lexicons-dir")} 92 - _, err := os.Stat(paths[0]) 93 - if err != nil { 94 - return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err) 95 - } 96 - } 97 - 98 85 // collect all NSID/path mappings 99 - localSchemas := map[syntax.NSID]json.RawMessage{} 100 - remoteSchemas := map[syntax.NSID]json.RawMessage{} 101 - 102 - for _, p := range paths { 103 - finfo, err := os.Stat(p) 104 - if err != nil { 105 - return fmt.Errorf("failed loading %s: %w", p, err) 106 - } 107 - if finfo.IsDir() { 108 - if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error { 109 - if d.IsDir() || path.Ext(fp) != ".json" { 110 - return nil 111 - } 112 - nsid, rec, err := loadSchemaPath(fp) 113 - if err != nil { 114 - return err 115 - } 116 - localSchemas[nsid] = *rec 117 - return nil 118 - }); err != nil { 119 - return err 120 - } 121 - continue 122 - } 123 - nsid, rec, err := loadSchemaPath(p) 124 - if err != nil { 125 - return err 126 - } 127 - localSchemas[nsid] = *rec 86 + localSchemas, err := collectSchemaJSON(cmd) 87 + if err != nil { 88 + return err 128 89 } 90 + remoteSchemas := map[syntax.NSID]json.RawMessage{} 129 91 130 92 localGroups := map[string]bool{} 131 93 allNSIDMap := map[syntax.NSID]bool{} ··· 136 98 } 137 99 138 100 for g := range localGroups { 139 - if err := fetchLexiconGroup(ctx, cmd, g, &remoteSchemas); err != nil { 101 + if err := resolveLexiconGroup(ctx, cmd, g, &remoteSchemas); err != nil { 140 102 return err 141 103 } 142 104 }
+1 -1
cmd/glot/pull.go
··· 27 27 Flags: []cli.Flag{ 28 28 &cli.StringFlag{ 29 29 Name: "lexicons-dir", 30 - Value: "./lexicons/", 30 + Value: "lexicons/", 31 31 Usage: "base directory for project Lexicon files", 32 32 Sources: cli.EnvVars("LEXICONS_DIR"), 33 33 },
+3 -191
cmd/glot/status.go
··· 4 4 "context" 5 5 "encoding/json" 6 6 "fmt" 7 - "io/fs" 8 - "log/slog" 9 - "os" 10 - "path" 11 - "path/filepath" 12 7 "reflect" 13 - "sort" 14 8 15 - "github.com/bluesky-social/indigo/api/agnostic" 16 - "github.com/bluesky-social/indigo/atproto/atclient" 17 9 "github.com/bluesky-social/indigo/atproto/atdata" 18 - "github.com/bluesky-social/indigo/atproto/identity" 19 - "github.com/bluesky-social/indigo/atproto/lexicon" 20 10 "github.com/bluesky-social/indigo/atproto/syntax" 21 11 22 12 "github.com/urfave/cli/v3" ··· 30 20 Flags: []cli.Flag{ 31 21 &cli.StringFlag{ 32 22 Name: "lexicons-dir", 33 - Value: "./lexicons/", 23 + Value: "lexicons/", 34 24 Usage: "base directory for project Lexicon files", 35 25 Sources: cli.EnvVars("LEXICONS_DIR"), 36 26 }, ··· 38 28 Action: runStatus, 39 29 } 40 30 41 - func loadSchemaPath(fpath string) (syntax.NSID, *json.RawMessage, error) { 42 - b, err := os.ReadFile(fpath) 43 - if err != nil { 44 - return "", nil, err 45 - } 46 - 47 - // parse file to check for errors 48 - // TODO: use json/v2 when available for case-sensitivity 49 - var sf lexicon.SchemaFile 50 - err = json.Unmarshal(b, &sf) 51 - if err == nil { 52 - err = sf.FinishParse() 53 - } 54 - if err == nil { 55 - err = sf.CheckSchema() 56 - } 57 - if err != nil { 58 - return "", nil, err 59 - } 60 - 61 - var rec json.RawMessage 62 - if err := json.Unmarshal(b, &rec); err != nil { 63 - return "", nil, err 64 - } 65 - return syntax.NSID(sf.ID), &rec, nil 66 - } 67 - 68 31 func runStatus(ctx context.Context, cmd *cli.Command) error { 69 - return compareSchemas(ctx, cmd, statusCompare) 32 + return runComparisons(ctx, cmd, compareStatus) 70 33 } 71 34 72 - func statusCompare(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error { 35 + func compareStatus(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error { 73 36 74 37 // new remote schema (missing local) 75 38 if localJSON == nil { ··· 100 63 } 101 64 return nil 102 65 } 103 - 104 - func compareSchemas(ctx context.Context, cmd *cli.Command, comp func(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error) error { 105 - paths := cmd.Args().Slice() 106 - if !cmd.Args().Present() { 107 - paths = []string{cmd.String("lexicons-dir")} 108 - _, err := os.Stat(paths[0]) 109 - if err != nil { 110 - return fmt.Errorf("no path arguments specified and default lexicon directory not found\n%w", err) 111 - } 112 - } 113 - 114 - // collect all NSID/path mappings 115 - localSchemas := map[syntax.NSID]json.RawMessage{} 116 - remoteSchemas := map[syntax.NSID]json.RawMessage{} 117 - 118 - for _, p := range paths { 119 - finfo, err := os.Stat(p) 120 - if err != nil { 121 - return fmt.Errorf("failed loading %s: %w", p, err) 122 - } 123 - if finfo.IsDir() { 124 - if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error { 125 - if d.IsDir() || path.Ext(fp) != ".json" { 126 - return nil 127 - } 128 - nsid, rec, err := loadSchemaPath(fp) 129 - if err != nil { 130 - return err 131 - } 132 - localSchemas[nsid] = *rec 133 - return nil 134 - }); err != nil { 135 - return err 136 - } 137 - continue 138 - } 139 - nsid, rec, err := loadSchemaPath(p) 140 - if err != nil { 141 - return err 142 - } 143 - localSchemas[nsid] = *rec 144 - } 145 - 146 - localGroups := map[string]bool{} 147 - allNSIDMap := map[syntax.NSID]bool{} 148 - for k := range localSchemas { 149 - g := nsidGroup(k) 150 - localGroups[g] = true 151 - allNSIDMap[k] = true 152 - } 153 - 154 - for g := range localGroups { 155 - if err := fetchLexiconGroup(ctx, cmd, g, &remoteSchemas); err != nil { 156 - return err 157 - } 158 - } 159 - 160 - for k := range remoteSchemas { 161 - allNSIDMap[k] = true 162 - } 163 - allNSID := []string{} 164 - for k := range allNSIDMap { 165 - allNSID = append(allNSID, string(k)) 166 - } 167 - sort.Strings(allNSID) 168 - 169 - anyFailures := false 170 - for _, k := range allNSID { 171 - nsid := syntax.NSID(k) 172 - if err := comp(ctx, cmd, nsid, localSchemas[nsid], remoteSchemas[nsid]); err != nil { 173 - if err != ErrLintFailures { 174 - return err 175 - } 176 - anyFailures = true 177 - } 178 - } 179 - 180 - if anyFailures { 181 - return ErrLintFailures 182 - } 183 - return nil 184 - } 185 - 186 - func fetchLexiconGroup(ctx context.Context, cmd *cli.Command, group string, remote *map[syntax.NSID]json.RawMessage) error { 187 - 188 - slog.Debug("trying to load NSID group", "group", group) 189 - 190 - // TODO: netclient support for listing records 191 - dir := identity.BaseDirectory{} 192 - did, err := dir.ResolveNSID(ctx, syntax.NSID(group+"name")) 193 - if err != nil { 194 - // if NSID isn't registered, just skip comparison 195 - slog.Debug("skipping NSID pattern which did not resolve", "group", group) 196 - return nil 197 - } 198 - ident, err := dir.LookupDID(ctx, did) 199 - if err != nil { 200 - return err 201 - } 202 - c := atclient.NewAPIClient(ident.PDSEndpoint()) 203 - 204 - cursor := "" 205 - for { 206 - // collection string, cursor string, limit int64, repo string, reverse bool 207 - resp, err := agnostic.RepoListRecords(ctx, c, schemaNSID.String(), cursor, 100, ident.DID.String(), false) 208 - if err != nil { 209 - return err 210 - } 211 - for _, rec := range resp.Records { 212 - aturi, err := syntax.ParseATURI(rec.Uri) 213 - if err != nil { 214 - return err 215 - } 216 - nsid, err := syntax.ParseNSID(aturi.RecordKey().String()) 217 - if err != nil { 218 - slog.Warn("ignoring invalid schema NSID", "did", ident.DID, "rkey", aturi.RecordKey()) 219 - continue 220 - } 221 - if nsidGroup(nsid) != group { 222 - // ignoring other NSIDs 223 - continue 224 - } 225 - if rec.Value == nil { 226 - return fmt.Errorf("missing record value: %s", nsid) 227 - } 228 - 229 - // parse file to check for errors 230 - // TODO: use json/v2 when available for case-sensitivity 231 - var sf lexicon.SchemaFile 232 - err = json.Unmarshal(*rec.Value, &sf) 233 - if err == nil { 234 - err = sf.FinishParse() 235 - } 236 - if err == nil { 237 - err = sf.CheckSchema() 238 - } 239 - if err != nil { 240 - return fmt.Errorf("invalid lexicon schema record (%s): %w", nsid, err) 241 - } 242 - 243 - (*remote)[nsid] = *rec.Value 244 - 245 - } 246 - if resp.Cursor != nil && *resp.Cursor != "" { 247 - cursor = *resp.Cursor 248 - } else { 249 - break 250 - } 251 - } 252 - return nil 253 - }
-15
cmd/glot/util.go
··· 2 2 3 3 import ( 4 4 "fmt" 5 - "path" 6 5 "strings" 7 6 8 7 "github.com/bluesky-social/indigo/atproto/syntax" 9 - 10 - "github.com/urfave/cli/v3" 11 8 ) 12 9 13 10 var ( ··· 34 31 } 35 32 return raw, nil 36 33 } 37 - 38 - func pathForNSID(cmd *cli.Command, nsid syntax.NSID) string { 39 - 40 - odir := cmd.String("output-dir") 41 - if odir != "" { 42 - return path.Join(odir, nsid.Name()+".json") 43 - } 44 - 45 - base := cmd.String("lexicons-dir") 46 - sub := strings.ReplaceAll(nsid.String(), ".", "/") 47 - return path.Join(base, sub+".json") 48 - }
+60
cmd/glot/util_compare.go
··· 1 + package main 2 + 3 + import ( 4 + "context" 5 + "encoding/json" 6 + "sort" 7 + 8 + "github.com/bluesky-social/indigo/atproto/syntax" 9 + 10 + "github.com/urfave/cli/v3" 11 + ) 12 + 13 + func runComparisons(ctx context.Context, cmd *cli.Command, comp func(ctx context.Context, cmd *cli.Command, nsid syntax.NSID, localJSON, remoteJSON json.RawMessage) error) error { 14 + 15 + // collect all NSID/path mappings 16 + localSchemas, err := collectSchemaJSON(cmd) 17 + if err != nil { 18 + return err 19 + } 20 + remoteSchemas := map[syntax.NSID]json.RawMessage{} 21 + 22 + localGroups := map[string]bool{} 23 + allNSIDMap := map[syntax.NSID]bool{} 24 + for k := range localSchemas { 25 + g := nsidGroup(k) 26 + localGroups[g] = true 27 + allNSIDMap[k] = true 28 + } 29 + 30 + for g := range localGroups { 31 + if err := resolveLexiconGroup(ctx, cmd, g, &remoteSchemas); err != nil { 32 + return err 33 + } 34 + } 35 + 36 + for k := range remoteSchemas { 37 + allNSIDMap[k] = true 38 + } 39 + allNSID := []string{} 40 + for k := range allNSIDMap { 41 + allNSID = append(allNSID, string(k)) 42 + } 43 + sort.Strings(allNSID) 44 + 45 + anyFailures := false 46 + for _, k := range allNSID { 47 + nsid := syntax.NSID(k) 48 + if err := comp(ctx, cmd, nsid, localSchemas[nsid], remoteSchemas[nsid]); err != nil { 49 + if err != ErrLintFailures { 50 + return err 51 + } 52 + anyFailures = true 53 + } 54 + } 55 + 56 + if anyFailures { 57 + return ErrLintFailures 58 + } 59 + return nil 60 + }
+86
cmd/glot/util_fetch.go
··· 1 + package main 2 + 3 + import ( 4 + "context" 5 + "encoding/json" 6 + "fmt" 7 + "log/slog" 8 + 9 + "github.com/bluesky-social/indigo/api/agnostic" 10 + "github.com/bluesky-social/indigo/atproto/atclient" 11 + "github.com/bluesky-social/indigo/atproto/identity" 12 + "github.com/bluesky-social/indigo/atproto/lexicon" 13 + "github.com/bluesky-social/indigo/atproto/syntax" 14 + 15 + "github.com/urfave/cli/v3" 16 + ) 17 + 18 + // helper which resolves and fetches all lexicon schemas (as JSON), storing them in provided map 19 + func resolveLexiconGroup(ctx context.Context, cmd *cli.Command, group string, remote *map[syntax.NSID]json.RawMessage) error { 20 + 21 + slog.Debug("resolving schemas for NSID group", "group", group) 22 + 23 + // TODO: netclient support for listing records 24 + dir := identity.BaseDirectory{} 25 + did, err := dir.ResolveNSID(ctx, syntax.NSID(group+"name")) 26 + if err != nil { 27 + // if NSID isn't registered, just skip comparison 28 + slog.Warn("skipping NSID pattern which did not resolve", "group", group) 29 + return nil 30 + } 31 + ident, err := dir.LookupDID(ctx, did) 32 + if err != nil { 33 + return err 34 + } 35 + c := atclient.NewAPIClient(ident.PDSEndpoint()) 36 + 37 + cursor := "" 38 + for { 39 + // collection string, cursor string, limit int64, repo string, reverse bool 40 + resp, err := agnostic.RepoListRecords(ctx, c, schemaNSID.String(), cursor, 100, ident.DID.String(), false) 41 + if err != nil { 42 + return err 43 + } 44 + for _, rec := range resp.Records { 45 + aturi, err := syntax.ParseATURI(rec.Uri) 46 + if err != nil { 47 + return err 48 + } 49 + nsid, err := syntax.ParseNSID(aturi.RecordKey().String()) 50 + if err != nil { 51 + slog.Warn("ignoring invalid schema NSID", "did", ident.DID, "rkey", aturi.RecordKey()) 52 + continue 53 + } 54 + if nsidGroup(nsid) != group { 55 + // ignoring other NSIDs 56 + continue 57 + } 58 + if rec.Value == nil { 59 + return fmt.Errorf("missing record value: %s", nsid) 60 + } 61 + 62 + // parse file to check for errors 63 + // TODO: use json/v2 when available for case-sensitivity 64 + var sf lexicon.SchemaFile 65 + err = json.Unmarshal(*rec.Value, &sf) 66 + if err == nil { 67 + err = sf.FinishParse() 68 + } 69 + if err == nil { 70 + err = sf.CheckSchema() 71 + } 72 + if err != nil { 73 + return fmt.Errorf("invalid lexicon schema record (%s): %w", nsid, err) 74 + } 75 + 76 + (*remote)[nsid] = *rec.Value 77 + 78 + } 79 + if resp.Cursor != nil && *resp.Cursor != "" { 80 + cursor = *resp.Cursor 81 + } else { 82 + break 83 + } 84 + } 85 + return nil 86 + }
+181
cmd/glot/util_files.go
··· 1 + package main 2 + 3 + import ( 4 + "encoding/json" 5 + "fmt" 6 + "io" 7 + "io/fs" 8 + "os" 9 + "path" 10 + "path/filepath" 11 + "sort" 12 + "strings" 13 + 14 + "github.com/bluesky-social/indigo/atproto/lexicon" 15 + "github.com/bluesky-social/indigo/atproto/syntax" 16 + 17 + "github.com/urfave/cli/v3" 18 + ) 19 + 20 + func pathForNSID(cmd *cli.Command, nsid syntax.NSID) string { 21 + 22 + odir := cmd.String("output-dir") 23 + if odir != "" { 24 + return path.Join(odir, nsid.Name()+".json") 25 + } 26 + 27 + base := cmd.String("lexicons-dir") 28 + sub := strings.ReplaceAll(nsid.String(), ".", "/") 29 + return path.Join(base, sub+".json") 30 + } 31 + 32 + // parses through directories and files provided as CLI args, and returns a list of recursively enumerated .json files 33 + func collectPaths(cmd *cli.Command) ([]string, error) { 34 + 35 + paths := cmd.Args().Slice() 36 + if !cmd.Args().Present() { 37 + paths = []string{cmd.String("lexicons-dir")} 38 + _, err := os.Stat(paths[0]) 39 + if err != nil { 40 + return nil, fmt.Errorf("no path arguments specified and default lexicon directory not found") 41 + } 42 + } 43 + 44 + filePaths := []string{} 45 + 46 + for _, p := range paths { 47 + finfo, err := os.Stat(p) 48 + if err != nil { 49 + return nil, fmt.Errorf("failed reading path %s: %w", p, err) 50 + } 51 + if finfo.IsDir() { 52 + if err := filepath.WalkDir(p, func(fp string, d fs.DirEntry, err error) error { 53 + if d.IsDir() || path.Ext(fp) != ".json" { 54 + return nil 55 + } 56 + filePaths = append(filePaths, fp) 57 + return nil 58 + }); err != nil { 59 + return nil, err 60 + } 61 + continue 62 + } 63 + filePaths = append(filePaths, p) 64 + } 65 + 66 + sort.Strings(filePaths) 67 + return filePaths, nil 68 + } 69 + 70 + // parses through directories and files provided as CLI args, and returns broadly inclusive lexicon catalog. 71 + // 72 + // includes 'lexicons-dir', which may be broader than collectPaths() would return 73 + func collectCatalog(cmd *cli.Command) (lexicon.Catalog, error) { 74 + 75 + cat := lexicon.NewBaseCatalog() 76 + 77 + lexDir := cmd.String("lexicons-dir") 78 + paths := cmd.Args().Slice() 79 + if !cmd.Args().Present() { 80 + _, err := os.Stat(lexDir) 81 + if err != nil { 82 + return nil, fmt.Errorf("no path arguments specified and default lexicon directory not found") 83 + } 84 + } 85 + 86 + // load lexicon dir (recursively) 87 + ldinfo, err := os.Stat(lexDir) 88 + if err == nil && ldinfo.IsDir() { 89 + if err := cat.LoadDirectory(lexDir); err != nil { 90 + return nil, err 91 + } 92 + } 93 + 94 + for _, p := range paths { 95 + 96 + if strings.HasPrefix(p, lexDir) { 97 + // if path is under lexdir, we have already loaded, so skip 98 + // NOTE: this isn't particularly reliable 99 + continue 100 + } 101 + 102 + finfo, err := os.Stat(p) 103 + if err != nil { 104 + return nil, fmt.Errorf("failed reading path %s: %w", p, err) 105 + } 106 + if finfo.IsDir() { 107 + if p != lexDir { 108 + if err := cat.LoadDirectory(p); err != nil { 109 + return nil, err 110 + } 111 + } 112 + continue 113 + } 114 + if !finfo.Mode().IsRegular() && path.Ext(p) == ".json" { 115 + // load schema file in to catalog 116 + f, err := os.Open(p) 117 + if err != nil { 118 + return nil, err 119 + } 120 + defer func() { _ = f.Close() }() 121 + 122 + b, err := io.ReadAll(f) 123 + if err != nil { 124 + return nil, err 125 + } 126 + var sf lexicon.SchemaFile 127 + if err := json.Unmarshal(b, &sf); err != nil { 128 + return nil, err 129 + } 130 + if err := cat.AddSchemaFile(sf); err != nil { 131 + return nil, err 132 + } 133 + } 134 + } 135 + return &cat, nil 136 + } 137 + 138 + func loadSchemaJSON(fpath string) (syntax.NSID, *json.RawMessage, error) { 139 + b, err := os.ReadFile(fpath) 140 + if err != nil { 141 + return "", nil, err 142 + } 143 + 144 + // parse file to check for errors 145 + // TODO: use json/v2 when available for case-sensitivity 146 + var sf lexicon.SchemaFile 147 + err = json.Unmarshal(b, &sf) 148 + if err == nil { 149 + err = sf.FinishParse() 150 + } 151 + if err == nil { 152 + err = sf.CheckSchema() 153 + } 154 + if err != nil { 155 + return "", nil, err 156 + } 157 + 158 + var rec json.RawMessage 159 + if err := json.Unmarshal(b, &rec); err != nil { 160 + return "", nil, err 161 + } 162 + return syntax.NSID(sf.ID), &rec, nil 163 + } 164 + 165 + func collectSchemaJSON(cmd *cli.Command) (map[syntax.NSID]json.RawMessage, error) { 166 + schemas := map[syntax.NSID]json.RawMessage{} 167 + 168 + filePaths, err := collectPaths(cmd) 169 + if err != nil { 170 + return nil, err 171 + } 172 + 173 + for _, fp := range filePaths { 174 + nsid, rec, err := loadSchemaJSON(fp) 175 + if err != nil { 176 + return nil, err 177 + } 178 + schemas[nsid] = *rec 179 + } 180 + return schemas, nil 181 + }