this repo has no description

Add scrapyard server (#249)

Implemented a new `server_command` function to serve `.scrap` files over
HTTP, allowing retrieval by path or hash. Added error handling for
missing files. Included unit tests for server functionality in
`scrapscript_tests.py` to ensure proper operation and error responses.

---------

Co-authored-by: Max Bernstein <tekknolagi@gmail.com>
Co-authored-by: Max Bernstein <max@bernsteinbear.com>

authored by

Taylor Troesh
Max Bernstein
Max Bernstein
and committed by
GitHub
467a577f dd1973e2

+127
+76
scrapscript.py
··· 2482 2482 sys.stdout.buffer.write(serializer.output) 2483 2483 2484 2484 2485 + def server_command(args: argparse.Namespace) -> None: 2486 + import http.server 2487 + import socketserver 2488 + import hashlib 2489 + 2490 + dir = os.path.abspath(args.directory) 2491 + if not os.path.isdir(dir): 2492 + print(f"Error: {dir} is not a valid directory") 2493 + sys.exit(1) 2494 + 2495 + scraps = {} 2496 + for root, _, files in os.walk(dir): 2497 + for file in files: 2498 + file_path = os.path.join(root, file) 2499 + rel_path = os.path.relpath(file_path, dir) 2500 + if file.startswith("$"): 2501 + logger.debug(f"Skipping {rel_path}") 2502 + continue 2503 + rel_path_without_ext = os.path.splitext(rel_path)[0] 2504 + with open(file_path, "r") as f: 2505 + try: 2506 + program = parse(tokenize(f.read())) 2507 + serializer = Serializer() 2508 + serializer.serialize(program) 2509 + serialized = bytes(serializer.output) 2510 + scraps[rel_path_without_ext] = serialized 2511 + logger.debug(f"Loaded {rel_path_without_ext}") 2512 + file_hash = hashlib.sha256(serialized).hexdigest() 2513 + scraps[f"${file_hash}"] = serialized 2514 + logger.debug(f"Loaded {rel_path_without_ext} as ${file_hash}") 2515 + except Exception as e: 2516 + logger.error(f"Error processing {file_path}: {e}") 2517 + 2518 + keep_serving = True 2519 + 2520 + class ScrapHTTPRequestHandler(http.server.SimpleHTTPRequestHandler): 2521 + def do_QUIT(self) -> None: 2522 + self.send_response(200) 2523 + self.end_headers() 2524 + self.wfile.write(b"Quitting") 2525 + nonlocal keep_serving 2526 + keep_serving = False 2527 + 2528 + def do_GET(self) -> None: 2529 + path = self.path.lstrip("/") 2530 + scrap = scraps.get(path) 2531 + if scrap is not None: 2532 + self.send_response(200) 2533 + self.send_header("Content-Type", "application/scrap; charset=binary") 2534 + self.send_header("Content-Disposition", f'attachment; filename={json.dumps(f"{path}.scrap")}') 2535 + self.send_header("Content-Length", str(len(scrap))) 2536 + self.end_headers() 2537 + self.wfile.write(scrap) 2538 + else: 2539 + self.send_response(404) 2540 + self.send_header("Content-Type", "text/plain") 2541 + self.end_headers() 2542 + self.wfile.write(b"File not found") 2543 + 2544 + handler = ScrapHTTPRequestHandler 2545 + with socketserver.TCPServer((args.host, args.port), handler) as httpd: 2546 + logger.info(f"Serving {dir} at http://{args.host}:{args.port}") 2547 + while keep_serving: 2548 + httpd.handle_request() 2549 + 2550 + 2485 2551 def main() -> None: 2486 2552 parser = argparse.ArgumentParser(prog="scrapscript") 2487 2553 subparsers = parser.add_subparsers(dest="command") ··· 2520 2586 2521 2587 flat = subparsers.add_parser("flat") 2522 2588 flat.set_defaults(func=flat_command) 2589 + 2590 + yard = subparsers.add_parser("yard") 2591 + yard.set_defaults(func=lambda _: yard.print_help()) 2592 + yard_subparsers = yard.add_subparsers(dest="yard_command") 2593 + 2594 + yard_server = yard_subparsers.add_parser("server") 2595 + yard_server.set_defaults(func=server_command) 2596 + yard_server.add_argument("directory", type=str, nargs="?", default=".", help="Directory to serve") 2597 + yard_server.add_argument("--host", type=str, default="127.0.0.1", help="Host to bind to") 2598 + yard_server.add_argument("--port", type=int, default=8080, help="Port to listen on") 2523 2599 2524 2600 args = parser.parse_args() 2525 2601 if not args.command:
+51
scrapscript_tests.py
··· 1 1 import unittest 2 2 import re 3 3 from typing import Optional 4 + import urllib.request 4 5 5 6 # ruff: noqa: F405 6 7 # ruff: noqa: F403 ··· 4049 4050 4050 4051 obj = Variant("x", Function(Var("a"), Var("b"))) 4051 4052 self.assertEqual(pretty(obj), "#x (a -> b)") 4053 + 4054 + 4055 + class ServerCommandTests(unittest.TestCase): 4056 + def setUp(self) -> None: 4057 + import threading 4058 + import time 4059 + import os 4060 + import socket 4061 + import argparse 4062 + from scrapscript import server_command 4063 + 4064 + # Find a random available port 4065 + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: 4066 + s.bind(("127.0.0.1", 0)) 4067 + self.host, self.port = s.getsockname() 4068 + 4069 + args = argparse.Namespace( 4070 + directory=os.path.join(os.path.dirname(__file__), "examples"), 4071 + host=self.host, 4072 + port=self.port, 4073 + ) 4074 + 4075 + self.server_thread = threading.Thread(target=server_command, args=(args,)) 4076 + self.server_thread.daemon = True 4077 + self.server_thread.start() 4078 + 4079 + # Wait for the server to start 4080 + while True: 4081 + try: 4082 + with socket.create_connection((self.host, self.port), timeout=0.1) as s: 4083 + break 4084 + except (ConnectionRefusedError, socket.timeout): 4085 + time.sleep(0.01) 4086 + 4087 + def tearDown(self) -> None: 4088 + quit_request = urllib.request.Request(f"http://{self.host}:{self.port}/", method="QUIT") 4089 + urllib.request.urlopen(quit_request) 4090 + 4091 + def test_server_serves_scrap_by_path(self) -> None: 4092 + response = urllib.request.urlopen(f"http://{self.host}:{self.port}/0_home/factorial") 4093 + self.assertEqual(response.status, 200) 4094 + 4095 + def test_server_serves_scrap_by_hash(self) -> None: 4096 + response = urllib.request.urlopen(f"http://{self.host}:{self.port}/$09242a8dfec0ed32eb9ddd5452f0082998712d35306fec2042bad8ac5b6e9580") 4097 + self.assertEqual(response.status, 200) 4098 + 4099 + def test_server_fails_missing_scrap(self) -> None: 4100 + with self.assertRaises(urllib.error.HTTPError) as cm: 4101 + urllib.request.urlopen(f"http://{self.host}:{self.port}/foo") 4102 + self.assertEqual(cm.exception.code, 404) 4052 4103 4053 4104 4054 4105 if __name__ == "__main__":