spindled/engine: store workflow logs in s3 #1

+14 -7

go.mod

··· 7 7 github.com/alecthomas/assert/v2 v2.11.0 8 8 github.com/alecthomas/chroma/v2 v2.23.1 9 9 github.com/avast/retry-go/v4 v4.6.1 10 10 - github.com/aws/aws-sdk-go-v2 v1.41.1 11 11 - github.com/aws/aws-sdk-go-v2/credentials v1.19.9 10 10 + github.com/aws/aws-sdk-go-v2 v1.41.4 11 11 + github.com/aws/aws-sdk-go-v2/config v1.32.12 12 12 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 12 13 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 13 14 github.com/blevesearch/bleve/v2 v2.5.3 14 15 github.com/bluekeyes/go-gitdiff v0.8.1 ··· 67 68 github.com/alecthomas/repr v0.5.2 // indirect 68 69 github.com/anmitsu/go-shlex v0.0.0-20200514113438-38f4b401e2be // indirect 69 70 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 // indirect 70 70 - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 // indirect 71 71 - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 // indirect 71 71 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 // indirect 72 72 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 // indirect 73 73 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 // indirect 74 74 + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 // indirect 72 75 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 // indirect 73 73 - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 // indirect 76 76 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 // indirect 74 77 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 // indirect 75 75 - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 // indirect 78 78 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 // indirect 76 79 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 // indirect 77 77 - github.com/aws/smithy-go v1.24.0 // indirect 80 80 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 // indirect 81 81 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 // indirect 82 82 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 // indirect 83 83 + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 // indirect 84 84 + github.com/aws/smithy-go v1.24.2 // indirect 78 85 github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect 79 86 github.com/aymerick/douceur v0.2.0 // indirect 80 87 github.com/beorn7/perks v1.0.1 // indirect

+28 -14

go.sum

··· 22 22 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= 23 23 github.com/avast/retry-go/v4 v4.6.1 h1:VkOLRubHdisGrHnTu89g08aQEWEgRU7LVEop3GbIcMk= 24 24 github.com/avast/retry-go/v4 v4.6.1/go.mod h1:V6oF8njAwxJ5gRo1Q7Cxab24xs5NCWZBeaHHBklR8mA= 25 25 - github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= 26 26 - github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= 25 25 + github.com/aws/aws-sdk-go-v2 v1.41.4 h1:10f50G7WyU02T56ox1wWXq+zTX9I1zxG46HYuG1hH/k= 26 26 + github.com/aws/aws-sdk-go-v2 v1.41.4/go.mod h1:mwsPRE8ceUUpiTgF7QmQIJ7lgsKUPQOUl3o72QBrE1o= 27 27 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4 h1:489krEF9xIGkOaaX3CE/Be2uWjiXrkCH6gUX+bZA/BU= 28 28 github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.4/go.mod h1:IOAPF6oT9KCsceNTvvYMNHy0+kMF8akOjeDvPENWxp4= 29 29 - github.com/aws/aws-sdk-go-v2/credentials v1.19.9 h1:sWvTKsyrMlJGEuj/WgrwilpoJ6Xa1+KhIpGdzw7mMU8= 30 30 - github.com/aws/aws-sdk-go-v2/credentials v1.19.9/go.mod h1:+J44MBhmfVY/lETFiKI+klz0Vym2aCmIjqgClMmW82w= 31 31 - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17 h1:xOLELNKGp2vsiteLsvLPwxC+mYmO6OZ8PYgiuPJzF8U= 32 32 - github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.17/go.mod h1:5M5CI3D12dNOtH3/mk6minaRwI2/37ifCURZISxA/IQ= 33 33 - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17 h1:WWLqlh79iO48yLkj1v3ISRNiv+3KdQoZ6JWyfcsyQik= 34 34 - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.17/go.mod h1:EhG22vHRrvF8oXSTYStZhJc1aUgKtnJe+aOiFEV90cM= 29 29 + github.com/aws/aws-sdk-go-v2/config v1.32.12 h1:O3csC7HUGn2895eNrLytOJQdoL2xyJy0iYXhoZ1OmP0= 30 30 + github.com/aws/aws-sdk-go-v2/config v1.32.12/go.mod h1:96zTvoOFR4FURjI+/5wY1vc1ABceROO4lWgWJuxgy0g= 31 31 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12 h1:oqtA6v+y5fZg//tcTWahyN9PEn5eDU/Wpvc2+kJ4aY8= 32 32 + github.com/aws/aws-sdk-go-v2/credentials v1.19.12/go.mod h1:U3R1RtSHx6NB0DvEQFGyf/0sbrpJrluENHdPy1j/3TE= 33 33 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20 h1:zOgq3uezl5nznfoK3ODuqbhVg1JzAGDUhXOsU0IDCAo= 34 34 + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.20/go.mod h1:z/MVwUARehy6GAg/yQ1GO2IMl0k++cu1ohP9zo887wE= 35 35 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20 h1:CNXO7mvgThFGqOFgbNAP2nol2qAWBOGfqR/7tQlvLmc= 36 36 + github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.20/go.mod h1:oydPDJKcfMhgfcgBUZaG+toBbwy8yPWubJXBVERtI4o= 37 37 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20 h1:tN6W/hg+pkM+tf9XDkWUbDEjGLb+raoBMFsTodcoYKw= 38 38 + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.20/go.mod h1:YJ898MhD067hSHA6xYCx5ts/jEd8BSOLtQDL3iZsvbc= 39 39 + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6 h1:qYQ4pzQ2Oz6WpQ8T3HvGHnZydA72MnLuFK9tJwmrbHw= 40 40 + github.com/aws/aws-sdk-go-v2/internal/ini v1.8.6/go.mod h1:O3h0IK87yXci+kg6flUKzJnWeziQUKciKrLjcatSNcY= 35 41 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17 h1:JqcdRG//czea7Ppjb+g/n4o8i/R50aTBHkA7vu0lK+k= 36 42 github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.17/go.mod h1:CO+WeGmIdj/MlPel2KwID9Gt7CNq4M65HUfBW97liM0= 37 37 - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4 h1:0ryTNEdJbzUCEWkVXEXoqlXV72J5keC1GvILMOuD00E= 38 38 - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.4/go.mod h1:HQ4qwNZh32C3CBeO6iJLQlgtMzqeG17ziAA/3KDJFow= 43 43 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7 h1:5EniKhLZe4xzL7a+fU3C2tfUN4nWIqlLesfrjkuPFTY= 44 44 + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.13.7/go.mod h1:x0nZssQ3qZSnIcePWLvcoFisRXJzcTVvYpAAdYX8+GI= 39 45 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8 h1:Z5EiPIzXKewUQK0QTMkutjiaPVeVYXX7KIqhXu/0fXs= 40 46 github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.9.8/go.mod h1:FsTpJtvC4U1fyDXk7c71XoDv3HlRm8V3NiYLeYLh5YE= 41 41 - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17 h1:RuNSMoozM8oXlgLG/n6WLaFGoea7/CddrCfIiSA+xdY= 42 42 - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.17/go.mod h1:F2xxQ9TZz5gDWsclCtPQscGpP0VUOc8RqgFM3vDENmU= 47 47 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20 h1:2HvVAIq+YqgGotK6EkMf+KIEqTISmTYh5zLpYyeTo1Y= 48 48 + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.20/go.mod h1:V4X406Y666khGa8ghKmphma/7C0DAtEQYhkq9z4vpbk= 43 49 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17 h1:bGeHBsGZx0Dvu/eJC0Lh9adJa3M1xREcndxLNZlve2U= 44 50 github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.17/go.mod h1:dcW24lbU0CzHusTE8LLHhRLI42ejmINN8Lcr22bwh/g= 45 51 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0 h1:oeu8VPlOre74lBA/PMhxa5vewaMIMmILM+RraSyB8KA= 46 52 github.com/aws/aws-sdk-go-v2/service/s3 v1.96.0/go.mod h1:5jggDlZ2CLQhwJBiZJb4vfk4f0GxWdEDruWKEJ1xOdo= 47 47 - github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= 48 48 - github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= 53 53 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8 h1:0GFOLzEbOyZABS3PhYfBIx2rNBACYcKty+XGkTgw1ow= 54 54 + github.com/aws/aws-sdk-go-v2/service/signin v1.0.8/go.mod h1:LXypKvk85AROkKhOG6/YEcHFPoX+prKTowKnVdcaIxE= 55 55 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13 h1:kiIDLZ005EcKomYYITtfsjn7dtOwHDOFy7IbPXKek2o= 56 56 + github.com/aws/aws-sdk-go-v2/service/sso v1.30.13/go.mod h1:2h/xGEowcW/g38g06g3KpRWDlT+OTfxxI0o1KqayAB8= 57 57 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17 h1:jzKAXIlhZhJbnYwHbvUQZEB8KfgAEuG0dc08Bkda7NU= 58 58 + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.35.17/go.mod h1:Al9fFsXjv4KfbzQHGe6V4NZSZQXecFcvaIF4e70FoRA= 59 59 + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9 h1:Cng+OOwCHmFljXIxpEVXAGMnBia8MSU6Ch5i9PgBkcU= 60 60 + github.com/aws/aws-sdk-go-v2/service/sts v1.41.9/go.mod h1:LrlIndBDdjA/EeXeyNBle+gyCwTlizzW5ycgWnvIxkk= 61 61 + github.com/aws/smithy-go v1.24.2 h1:FzA3bu/nt/vDvmnkg+R8Xl46gmzEDam6mZ1hzmwXFng= 62 62 + github.com/aws/smithy-go v1.24.2/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= 49 63 github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= 50 64 github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= 51 65 github.com/aymerick/douceur v0.2.0 h1:Mv+mAeH1Q+n9Fr+oyamOlAkUNPWPlA8PPGR0QAaYuPk=

+35 -14

nix/gomod2nix.toml

··· 33 33 version = "v4.6.1" 34 34 hash = "sha256-PeZc8k4rDV64+k8nZt/oy1YNVbLevltXP3ZD1jf6Z6k=" 35 35 [mod."github.com/aws/aws-sdk-go-v2"] 36 36 - version = "v1.41.1" 37 37 - hash = "sha256-umafTZB+cuy8+Kzpl2WrlygJO3tR3D2WkTC0eTY5G/g=" 36 36 + version = "v1.41.4" 37 37 + hash = "sha256-k9xv4f8YPSzZ1yR3/zuyNDGenZKk0DD4lceL713yXtc=" 38 38 [mod."github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream"] 39 39 version = "v1.7.4" 40 40 hash = "sha256-ZY/Jn1p0IgDe8MONhp0RFHZmRgTBZZ5ddqXlNWEo7Ys=" 41 41 + [mod."github.com/aws/aws-sdk-go-v2/config"] 42 42 + version = "v1.32.12" 43 43 + hash = "sha256-aTkdSRe8KPmVZdsunU8j/hZQLhGw1ckKpLN/ryRBZM0=" 41 44 [mod."github.com/aws/aws-sdk-go-v2/credentials"] 42 42 - version = "v1.19.9" 43 43 - hash = "sha256-eqM5BmetQ/MrxTwoUCqRpkm5frFAspuq+QWod+5wzrU=" 45 45 + version = "v1.19.12" 46 46 + hash = "sha256-xEIT1ARA9RYrQtLZIus71E6niNHIOVM1J7mUnA5AhJQ=" 47 47 + [mod."github.com/aws/aws-sdk-go-v2/feature/ec2/imds"] 48 48 + version = "v1.18.20" 49 49 + hash = "sha256-dCTpdKZheVCSt+R+NnFOnlS0bCt4gPavlDh15Kl/sMQ=" 44 50 [mod."github.com/aws/aws-sdk-go-v2/internal/configsources"] 45 45 - version = "v1.4.17" 46 46 - hash = "sha256-W+d0WDYBrVJxdfRSvoe3cvZYgIgUSVyXsVdQlMcIZvc=" 51 51 + version = "v1.4.20" 52 52 + hash = "sha256-aATIk4oLd7aaV66ereBdjINLMDwmIHxu+NNsgKWH1t4=" 47 53 [mod."github.com/aws/aws-sdk-go-v2/internal/endpoints/v2"] 48 48 - version = "v2.7.17" 49 49 - hash = "sha256-dKJx0+K1DKi2LJKQNwnTofZH4GWLDgY6/ZI2XR2oCGI=" 54 54 + version = "v2.7.20" 55 55 + hash = "sha256-G6266uj64sgfDTJ9V1UY1sQs3UmryB0CFgxzmbjjChY=" 56 56 + [mod."github.com/aws/aws-sdk-go-v2/internal/ini"] 57 57 + version = "v1.8.6" 58 58 + hash = "sha256-oIRPqu99vnGINAWKnCEytpv7N0gRWO7S72tb1r8oxvk=" 50 59 [mod."github.com/aws/aws-sdk-go-v2/internal/v4a"] 51 60 version = "v1.4.17" 52 61 hash = "sha256-PKwNn9nFf+7TqocgQprxP9r1Fs4IxwGLGd05MrsxmLg=" 53 62 [mod."github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding"] 54 54 - version = "v1.13.4" 55 55 - hash = "sha256-Rm6czqOnOULP080D97WQQSqkBhmN6ei1qZaTa51SRj8=" 63 63 + version = "v1.13.7" 64 64 + hash = "sha256-AfYJdpmnW01Bk/jfHATlNU6lddjqcigFkHw/zcT9WO4=" 56 65 [mod."github.com/aws/aws-sdk-go-v2/service/internal/checksum"] 57 66 version = "v1.9.8" 58 67 hash = "sha256-DZhR0aqHrgAFBGSlnsSQ6XeAJ/q504RG/LBWhtQqRVg=" 59 68 [mod."github.com/aws/aws-sdk-go-v2/service/internal/presigned-url"] 60 60 - version = "v1.13.17" 61 61 - hash = "sha256-qS7Db9S/KQ24kvJdL8qH4gnoN116J1ezwOnbovCiFEI=" 69 69 + version = "v1.13.20" 70 70 + hash = "sha256-a5TifKunIoqKd2uAceYh6F1LvMHMyEQcWvJf0sxKhPM=" 62 71 [mod."github.com/aws/aws-sdk-go-v2/service/internal/s3shared"] 63 72 version = "v1.19.17" 64 73 hash = "sha256-x/Cb4j3HFlg1+U21YCAIhBnfpf1yehJU2Ss/PxamEMI=" 65 74 [mod."github.com/aws/aws-sdk-go-v2/service/s3"] 66 75 version = "v1.96.0" 67 76 hash = "sha256-lzAn2KHIkd742mDwEZqGEaIXlEfvltL6HdP+sEQ/8YA=" 77 77 + [mod."github.com/aws/aws-sdk-go-v2/service/signin"] 78 78 + version = "v1.0.8" 79 79 + hash = "sha256-o4pWg3yMZHxdI94x5Z6qbiRg7gpmzbpJnJWsR1BOc44=" 80 80 + [mod."github.com/aws/aws-sdk-go-v2/service/sso"] 81 81 + version = "v1.30.13" 82 82 + hash = "sha256-V277a0ikm/H0paIeDLtPGEyav2a69Kdb9d5bh+JLAeY=" 83 83 + [mod."github.com/aws/aws-sdk-go-v2/service/ssooidc"] 84 84 + version = "v1.35.17" 85 85 + hash = "sha256-r5V5DoCIR4yzN1Ttg+dIA85GVkWMPgeD6Zu0rWGqNJE=" 86 86 + [mod."github.com/aws/aws-sdk-go-v2/service/sts"] 87 87 + version = "v1.41.9" 88 88 + hash = "sha256-I15uxeoKxDURsZrEVDzCRtVIu/HE756M1Rt7PPpdZ7c=" 68 89 [mod."github.com/aws/smithy-go"] 69 69 - version = "v1.24.0" 70 70 - hash = "sha256-ZPFhf2Yv3BQpUn3cN4wSnoO7uBki8oCisZxL6F09nnE=" 90 90 + version = "v1.24.2" 91 91 + hash = "sha256-v0y+Lir61fgdCwdVoca5mK+FcGh9OD3cTEwHIfLytcI=" 71 92 [mod."github.com/aymanbagabas/go-osc52/v2"] 72 93 version = "v2.0.1" 73 94 hash = "sha256-6Bp0jBZ6npvsYcKZGHHIUSVSTAMEyieweAX2YAKDjjg="

+17

nix/modules/spindle.nix

··· 109 109 default = "5m"; 110 110 description = "Timeout for each step of a pipeline"; 111 111 }; 112 112 + 113 113 + logBucket = mkOption { 114 114 + type = types.str; 115 115 + default = "tangled-logs"; 116 116 + description = "S3 bucket for workflow logs"; 117 117 + }; 118 118 + 119 119 + uploadWorkflowLogs = mkOption { 120 120 + type = types.bool; 121 121 + default = false; 122 122 + description = "Enables uploading workflow logs to s3"; 123 123 + }; 112 124 }; 113 125 }; 114 126 }; ··· 138 150 "SPINDLE_SERVER_SECRETS_OPENBAO_MOUNT=${cfg.server.secrets.openbao.mount}" 139 151 "SPINDLE_NIXERY_PIPELINES_NIXERY=${cfg.pipelines.nixery}" 140 152 "SPINDLE_NIXERY_PIPELINES_WORKFLOW_TIMEOUT=${cfg.pipelines.workflowTimeout}" 153 153 + "SPINDLE_NIXERY_PIPELINES_LOG_BUCKET=${cfg.pipelines.logBucket}" 154 154 + "SPINDLE_NIXERY_PIPELINES_UPLOAD_WORKFLOW_LOGS=${lib.boolToString cfg.pipelines.uploadWorkflowLogs}" 155 155 + "AWS_ACCESS_KEY_ID=${builtins.getEnv "AWS_ACCESS_KEY_ID"}" 156 156 + "AWS_SECRET_ACCESS_KEY=${builtins.getEnv "AWS_SECRET_ACCESS_KEY"}" 157 157 + "AWS_REGION=${builtins.getEnv "AWS_REGION"}" 141 158 ]; 142 159 ExecStart = "${cfg.package}/bin/spindle"; 143 160 Restart = "always";

+4 -2

spindle/config/config.go

··· 37 37 } 38 38 39 39 type NixeryPipelines struct { 40 40 - Nixery string `env:"NIXERY, default=nixery.tangled.sh"` 41 41 - WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` 40 40 + Nixery string `env:"NIXERY, default=nixery.tangled.sh"` 41 41 + WorkflowTimeout string `env:"WORKFLOW_TIMEOUT, default=5m"` 42 42 + UploadWorkflowLogs bool `env:"UPLOAD_WORKFLOW_LOGS, default=false"` 43 43 + LogBucket string `env:"LOG_BUCKET, default=tangled-logs"` 42 44 } 43 45 44 46 type Config struct {

+19

spindle/engines/nixery/engine.go

··· 8 8 "io" 9 9 "log/slog" 10 10 "path" 11 11 + "path/filepath" 11 12 "runtime" 12 13 "sync" 13 14 "time" ··· 25 26 "tangled.org/core/spindle/engine" 26 27 "tangled.org/core/spindle/models" 27 28 "tangled.org/core/spindle/secrets" 29 29 + "tangled.org/core/spindle/utils" 28 30 ) 29 31 30 32 const ( ··· 116 118 117 119 // append setup steps in order to the start of workflow steps 118 120 swf.Steps = append(*setup, swf.Steps...) 121 121 + // append tear down steps to clean up after the workflow is done 119 122 swf.Data = addl 120 123 121 124 return swf, nil ··· 284 287 return nil 285 288 }) 286 289 290 290 + if e.cfg.NixeryPipelines.UploadWorkflowLogs { 291 291 + e.registerCleanup(wid, func(ctx context.Context) error { 292 292 + s3, err := utils.NewS3(e.cfg.NixeryPipelines.LogBucket) 293 293 + if err != nil { 294 294 + return fmt.Errorf("error creating s3 client: %w", err) 295 295 + } 296 296 + 297 297 + name := filepath.Join(e.cfg.Server.LogDir, fmt.Sprintf("%s.log", wid.String())) 298 298 + if err := s3.WriteFile(ctx, name); err != nil { 299 299 + return fmt.Errorf("error saving logs: %w", err) 300 300 + } 301 301 + 302 302 + return nil 303 303 + }) 304 304 + } 305 305 + 287 306 /// -------------------------CONTAINER START---------------------------------------- 288 307 wfLogger.DataWriter(setupStepIdx, "stdout").Write([]byte("starting container...")) 289 308 if err := e.docker.ContainerStart(ctx, resp.ID, container.StartOptions{}); err != nil {

+71

spindle/utils/s3.go

··· 1 1 + package utils 2 2 + 3 3 + import ( 4 4 + "context" 5 5 + "fmt" 6 6 + "io" 7 7 + "os" 8 8 + "path/filepath" 9 9 + 10 10 + "github.com/aws/aws-sdk-go-v2/aws" 11 11 + "github.com/aws/aws-sdk-go-v2/config" 12 12 + "github.com/aws/aws-sdk-go-v2/service/s3" 13 13 + ) 14 14 + 15 15 + type S3 struct { 16 16 + bucket string 17 17 + client *s3.Client 18 18 + } 19 19 + 20 20 + const BASE_S3_PATH = "spindle/workflows" 21 21 + 22 22 + func NewS3(bucket string) (*S3, error) { 23 23 + ctx := context.Background() 24 24 + sdkConfig, err := config.LoadDefaultConfig(ctx) 25 25 + 26 26 + if err != nil { 27 27 + return nil, fmt.Errorf("error loading s3 config: %w", err) 28 28 + } 29 29 + s3Client := s3.NewFromConfig(sdkConfig) 30 30 + 31 31 + return &S3{ 32 32 + bucket: bucket, 33 33 + client: s3Client, 34 34 + }, nil 35 35 + } 36 36 + 37 37 + func (s *S3) WriteFile(ctx context.Context, path string) error { 38 38 + s3_key := fmt.Sprintf("%s/%s", BASE_S3_PATH, filepath.Base(path)) 39 39 + 40 40 + file, err := os.Open(path) 41 41 + if err != nil { 42 42 + return fmt.Errorf("error opening file %s: %w", path, err) 43 43 + } 44 44 + defer file.Close() 45 45 + 46 46 + _, err = s.client.PutObject(ctx, &s3.PutObjectInput{ 47 47 + Bucket: &s.bucket, 48 48 + Key: &s3_key, 49 49 + Body: file, 50 50 + }) 51 51 + 52 52 + if err != nil { 53 53 + return fmt.Errorf("error writing to s3: %w", err) 54 54 + } 55 55 + 56 56 + return nil 57 57 + } 58 58 + 59 59 + func (s *S3) ReadFile(ctx context.Context, name string) ([]byte, error) { 60 60 + res, err := s.client.GetObject(ctx, &s3.GetObjectInput{ 61 61 + Bucket: &s.bucket, 62 62 + Key: aws.String(name), 63 63 + }) 64 64 + 65 65 + if err != nil { 66 66 + return nil, fmt.Errorf("error reading file %s: %w", name, err) 67 67 + } 68 68 + defer res.Body.Close() 69 69 + 70 70 + return io.ReadAll(res.Body) 71 71 + }