Simple S3-like server for development purposes, written in Go
juanlu.space
1package main
2
3import (
4 "crypto/md5"
5 "crypto/rand"
6 "encoding/hex"
7 "encoding/xml"
8 "errors"
9 "fmt"
10 "io"
11 "io/fs"
12 "log"
13 "math/big"
14 "net/http"
15 "os"
16 "path/filepath"
17 "strconv"
18 "strings"
19 "time"
20)
21
22type server struct {
23 rootDir string
24 logger *log.Logger
25}
26
27func listBuckets(root *os.Root) (ListAllMyBucketsResult, error) {
28 files, err := fs.ReadDir(root.FS(), ".")
29 if err != nil {
30 return ListAllMyBucketsResult{}, err
31 }
32
33 var buckets []Bucket
34 for _, file := range files {
35 fileInfo, err := file.Info()
36 var lastModified string
37 if err != nil {
38 // Some error retrieving the dir info, setting LastModified to dummy value
39 lastModified = "2000-01-01T00:00:00Z"
40 } else {
41 // NOTE: For actual birth time, see https://github.com/djherbis/times
42 lastModified = fileInfo.ModTime().UTC().Format(time.RFC3339)
43 }
44 buckets = append(buckets, Bucket{
45 Name: file.Name(),
46 CreationDate: lastModified,
47 })
48 }
49 result := ListAllMyBucketsResult{
50 Buckets: Buckets{Buckets: buckets},
51 }
52 return result, nil
53}
54
55func listObjects(root *os.Root, bucketName string, prefix string) (ListBucketResult, error) {
56 subpath, filter := filepath.Split(prefix)
57 path := filepath.Join(bucketName, subpath)
58 files, err := fs.ReadDir(root.FS(), path)
59 if err != nil {
60 if errors.Is(err, fs.ErrNotExist) {
61 return ListBucketResult{KeyCount: 0, IsTruncated: false, Prefix: prefix}, nil
62 } else {
63 return ListBucketResult{}, err
64 }
65 }
66
67 var objects []Object
68 for _, file := range files {
69 if strings.HasPrefix(file.Name(), filter) {
70 fileInfo, err := file.Info()
71 var lastModified string
72 var objectSize int64
73 if err != nil {
74 // Some error retrieving the file info, setting LastModified to dummy value
75 lastModified = "2000-01-01T00:00:00Z"
76 objectSize = 0
77 } else {
78 lastModified = fileInfo.ModTime().UTC().Format(time.RFC3339)
79 objectSize = fileInfo.Size()
80 }
81 key, _ := filepath.Rel(bucketName, filepath.Join(path, file.Name()))
82 objects = append(objects, Object{
83 Key: key,
84 LastModified: lastModified,
85 Size: objectSize,
86 })
87 }
88 }
89
90 result := ListBucketResult{
91 KeyCount: int64(len(objects)),
92 IsTruncated: false,
93 Prefix: prefix,
94 Contents: objects,
95 }
96 return result, nil
97}
98
99func (s *server) cpGet(w http.ResponseWriter, r *http.Request) {
100 root, err := os.OpenRoot(s.rootDir)
101 if err != nil {
102 http.Error(w, err.Error(), http.StatusInternalServerError)
103 }
104 defer root.Close()
105
106 path := filepath.Join(r.PathValue("bucket"), r.PathValue("key"))
107 fileInfo, err := root.Stat(path)
108 if err != nil {
109 if errors.Is(err, fs.ErrNotExist) {
110 http.Error(w, fmt.Sprintf("Key %s does not exist", r.PathValue("key")), http.StatusNotFound)
111 } else {
112 http.Error(w, err.Error(), http.StatusInternalServerError)
113 }
114 return
115 }
116
117 // NOTE: If the file is huge, we will open it even in HEAD requests,
118 // which shouldn't be needed - but then using http.ServeContent gets trickier
119 file, err := root.Open(path)
120 if err != nil {
121 http.Error(w, err.Error(), http.StatusInternalServerError)
122 }
123
124 http.ServeContent(w, r, fileInfo.Name(), fileInfo.ModTime(), file)
125}
126
127func (s *server) ls(w http.ResponseWriter, r *http.Request) {
128 root, err := os.OpenRoot(s.rootDir)
129 if err != nil {
130 http.Error(w, err.Error(), http.StatusInternalServerError)
131 }
132 defer root.Close()
133
134 bucketName := r.PathValue("bucket")
135 if bucketName == "" {
136 // List available buckets
137 result, err := listBuckets(root)
138 if err != nil {
139 http.Error(w, err.Error(), http.StatusInternalServerError)
140 }
141
142 w.Header().Set("Content-Type", "application/xml")
143 xml.NewEncoder(w).Encode(result)
144 } else {
145 // List objects
146 // But first, check if bucket exists
147 if _, err := root.Stat(bucketName); err != nil {
148 if errors.Is(err, fs.ErrNotExist) {
149 http.Error(w, "The specified bucket does not exist", http.StatusNotFound)
150 return
151 }
152 }
153
154 result, err := listObjects(root, bucketName, r.URL.Query().Get("prefix"))
155 if err != nil {
156 http.Error(w, err.Error(), http.StatusInternalServerError)
157 return
158 }
159
160 w.Header().Set("Content-Type", "application/xml")
161 xml.NewEncoder(w).Encode(result)
162 }
163}
164
165func (s *server) mb(w http.ResponseWriter, r *http.Request) {
166 root, err := os.OpenRoot(s.rootDir)
167 if err != nil {
168 http.Error(w, err.Error(), http.StatusInternalServerError)
169 }
170 defer root.Close()
171
172 path := filepath.Join(r.PathValue("bucket"))
173
174 bucketName := strings.Split(path, "/")[0]
175 if err := root.Mkdir(bucketName, 0775); err != nil {
176 if errors.Is(err, fs.ErrExist) {
177 http.Error(w, "Your previous request to create the named bucket succeeded and you already own it.", http.StatusConflict)
178 } else {
179 http.Error(w, err.Error(), http.StatusInternalServerError)
180 }
181 return
182 }
183
184 w.WriteHeader(http.StatusOK)
185}
186
187func (s *server) rb(w http.ResponseWriter, r *http.Request) {
188 root, err := os.OpenRoot(s.rootDir)
189 if err != nil {
190 http.Error(w, err.Error(), http.StatusInternalServerError)
191 }
192 defer root.Close()
193
194 path := filepath.Join(r.PathValue("bucket"))
195
196 bucketName := strings.Split(path, "/")[0]
197 files, err := fs.ReadDir(root.FS(), bucketName)
198 if err != nil {
199 if errors.Is(err, fs.ErrNotExist) {
200 http.Error(w, "The specified bucket does not exist", http.StatusConflict)
201 } else {
202 http.Error(w, err.Error(), http.StatusInternalServerError)
203 }
204 return
205 }
206 if len(files) > 0 {
207 http.Error(w, "The bucket you tried to delete is not empty", http.StatusConflict)
208 return
209 }
210
211 if err := root.Remove(bucketName); err != nil {
212 http.Error(w, err.Error(), http.StatusInternalServerError)
213 return
214 }
215
216 w.WriteHeader(http.StatusOK)
217}
218
219func (s *server) cpPut(w http.ResponseWriter, r *http.Request) {
220 root, err := os.OpenRoot(s.rootDir)
221 if err != nil {
222 http.Error(w, err.Error(), http.StatusInternalServerError)
223 return
224 }
225 defer root.Close()
226
227 path := filepath.Join(r.PathValue("bucket"), r.PathValue("key"))
228
229 // Prevents `cp` from accidentally creating a new bucket if it doesn't exist
230 bucketName := strings.Split(path, "/")[0]
231 if _, err := root.Stat(bucketName); err != nil {
232 if errors.Is(err, fs.ErrNotExist) {
233 http.Error(w, "The specified bucket does not exist", http.StatusNotFound)
234 } else {
235 http.Error(w, err.Error(), http.StatusInternalServerError)
236 }
237 return
238 }
239
240 randomInt, _ := rand.Int(rand.Reader, big.NewInt(1000))
241 tempName := fmt.Sprintf(".tmp.%d.%s", randomInt, filepath.Base(path))
242 tempPath := filepath.Join(bucketName, tempName)
243
244 tempFile, err := root.Create(tempPath)
245 if err != nil {
246 http.Error(w, err.Error(), http.StatusInternalServerError)
247 return
248 }
249 defer root.Remove(tempPath)
250 defer tempFile.Close()
251
252 hasher := md5.New()
253 teeReader := io.TeeReader(r.Body, hasher)
254
255 bytesWritten, err := io.Copy(tempFile, teeReader)
256 if err != nil {
257 http.Error(w, err.Error(), http.StatusInternalServerError)
258 return
259 }
260 // We ignore errors here, if anything, the program will break when renaming
261 tempFile.Sync()
262 tempFile.Close()
263
264 expectedContentLength := r.Header.Get("Content-Length")
265 actualContentLength := strconv.FormatInt(bytesWritten, 10)
266 if expectedContentLength != "" && expectedContentLength != actualContentLength {
267 http.Error(
268 w,
269 fmt.Sprintf(
270 "Expected Content-Length was %s, gor %s",
271 expectedContentLength,
272 actualContentLength,
273 ),
274 http.StatusInternalServerError,
275 )
276 }
277
278 // Safely create nested directories if needed
279 // TODO: Remove dangling paths if anything fails
280 if err := root.MkdirAll(filepath.Dir(path), 0775); err != nil {
281 http.Error(w, err.Error(), http.StatusInternalServerError)
282 return
283 }
284
285 // Move temporary file to final destination
286 if err := root.Rename(tempPath, path); err != nil {
287 http.Error(w, err.Error(), http.StatusInternalServerError)
288 return
289 }
290
291 etag := fmt.Sprintf("\"%s\"", hex.EncodeToString(hasher.Sum(nil)))
292 w.Header().Set("ETag", etag)
293
294 w.WriteHeader(http.StatusOK)
295}
296
297func (s *server) rm(w http.ResponseWriter, r *http.Request) {
298 root, err := os.OpenRoot(s.rootDir)
299 if err != nil {
300 http.Error(w, err.Error(), http.StatusInternalServerError)
301 }
302 defer root.Close()
303
304 // TODO: Dangling empty directory
305 path := filepath.Join(r.PathValue("bucket"), r.PathValue("key"))
306 if err := root.Remove(path); err != nil {
307 http.Error(w, err.Error(), http.StatusInternalServerError)
308 return
309 }
310
311 w.WriteHeader(http.StatusOK)
312}