Something went wrong. Try again.
Forking what is left of ZeroNet and hopefully adding an AT Proto Frontend/Proxy
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449# Included modulesimport osimport timeimport jsonimport collectionsimport itertools
# Third party modulesimport gevent
from Debug import Debugfrom Config import configfrom util import RateLimitfrom util import Msgpackfrom util import helperfrom Plugin import PluginManagerfrom contextlib import closing
FILE_BUFF = 1024 * 512
class RequestError(Exception): pass
# Incoming requests@PluginManager.acceptPluginsclass FileRequest(object): __slots__ = ("server", "connection", "req_id", "sites", "log", "responded")
def __init__(self, server, connection): self.server = server self.connection = connection
self.req_id = None self.sites = self.server.sites self.log = server.log self.responded = False # Responded to the request
def send(self, msg, streaming=False): if not self.connection.closed: self.connection.send(msg, streaming)
def sendRawfile(self, file, read_bytes): if not self.connection.closed: self.connection.sendRawfile(file, read_bytes)
def response(self, msg, streaming=False): if self.responded: if config.verbose: self.log.debug("Req id %s already responded" % self.req_id) return if not isinstance(msg, dict): # If msg not a dict create a {"body": msg} msg = {"body": msg} msg["cmd"] = "response" msg["to"] = self.req_id self.responded = True self.send(msg, streaming=streaming)
# Route file requests def route(self, cmd, req_id, params): self.req_id = req_id # Don't allow other sites than locked if "site" in params and self.connection.target_onion: valid_sites = self.connection.getValidSites() if params["site"] not in valid_sites and valid_sites != ["global"]: self.response({"error": "Invalid site"}) self.connection.log( "Site lock violation: %s not in %s, target onion: %s" % (params["site"], valid_sites, self.connection.target_onion) ) self.connection.badAction(5) return False
if cmd == "update": event = "%s update %s %s" % (self.connection.id, params["site"], params["inner_path"]) # If called more than once within 15 sec only keep the last update RateLimit.callAsync(event, max(self.connection.bad_actions, 15), self.actionUpdate, params) else: func_name = "action" + cmd[0].upper() + cmd[1:] func = getattr(self, func_name, None) if cmd not in ["getFile", "streamFile"]: # Skip IO bound functions if self.connection.cpu_time > 0.5: self.log.debug( "Delay %s %s, cpu_time used by connection: %.3fs" % (self.connection.ip, cmd, self.connection.cpu_time) ) time.sleep(self.connection.cpu_time) if self.connection.cpu_time > 5: self.connection.close("Cpu time: %.3fs" % self.connection.cpu_time) s = time.time() if func: func(params) else: self.actionUnknown(cmd, params)
if cmd not in ["getFile", "streamFile"]: taken = time.time() - s taken_sent = self.connection.last_sent_time - self.connection.last_send_time self.connection.cpu_time += taken - taken_sent
# Update a site file request def actionUpdate(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(1) self.connection.badAction(5) return False
inner_path = params.get("inner_path", "") current_content_modified = site.content_manager.contents.get(inner_path, {}).get("modified", 0) body = params["body"]
if not inner_path.endswith("content.json"): self.response({"error": "Only content.json update allowed"}) self.connection.badAction(5) return
should_validate_content = True if "modified" in params and params["modified"] <= current_content_modified: should_validate_content = False valid = None # Same or earlier content as we have elif not body: # No body sent, we have to download it first site.log.debug("Missing body from update for file %s, downloading ..." % inner_path) peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer try: body = peer.getFile(site.address, inner_path).read() except Exception as err: site.log.debug("Can't download updated file %s: %s" % (inner_path, err)) self.response({"error": "File invalid update: Can't download updaed file"}) self.connection.badAction(5) return
if should_validate_content: try: content = json.loads(body.decode()) except Exception as err: site.log.debug("Update for %s is invalid JSON: %s" % (inner_path, err)) self.response({"error": "File invalid JSON"}) self.connection.badAction(5) return
file_uri = "%s/%s:%s" % (site.address, inner_path, content["modified"])
if self.server.files_parsing.get(file_uri): # Check if we already working on it valid = None # Same file else: try: valid = site.content_manager.verifyFile(inner_path, content) except Exception as err: site.log.debug("Update for %s is invalid: %s" % (inner_path, err)) error = err valid = False
if valid is True: # Valid and changed site.log.info("Update for %s looks valid, saving..." % inner_path) self.server.files_parsing[file_uri] = True site.storage.write(inner_path, body) del params["body"]
site.onFileDone(inner_path) # Trigger filedone
if inner_path.endswith("content.json"): # Download every changed file from peer peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update") # Add or get peer # On complete publish to other peers diffs = params.get("diffs", {}) site.onComplete.once(lambda: site.publish(inner_path=inner_path, diffs=diffs, limit=3), "publish_%s" % inner_path)
# Load new content file and download changed files in new thread def downloader(): site.downloadContent(inner_path, peer=peer, diffs=params.get("diffs", {})) del self.server.files_parsing[file_uri]
gevent.spawn(downloader) else: del self.server.files_parsing[file_uri]
self.response({"ok": "Thanks, file %s updated!" % inner_path}) self.connection.goodAction()
elif valid is None: # Not changed peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="update old") # Add or get peer if peer: if not peer.connection: peer.connect(self.connection) # Assign current connection to peer if inner_path in site.content_manager.contents: peer.last_content_json_update = site.content_manager.contents[inner_path]["modified"] if config.verbose: site.log.debug( "Same version, adding new peer for locked files: %s, tasks: %s" % (peer.key, len(site.worker_manager.tasks)) ) for task in site.worker_manager.tasks: # New peer add to every ongoing task if task["peers"] and not task["optional_hash_id"]: # Download file from this peer too if its peer locked site.needFile(task["inner_path"], peer=peer, update=True, blocking=False)
self.response({"ok": "File not changed"}) self.connection.badAction()
else: # Invalid sign or sha hash self.response({"error": "File %s invalid: %s" % (inner_path, error)}) self.connection.badAction(5)
def isReadable(self, site, inner_path, file, pos): return True
# Send file content request def handleGetFile(self, params, streaming=False): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False try: file_path = site.storage.getPath(params["inner_path"]) if streaming: file_obj = site.storage.open(params["inner_path"]) else: file_obj = Msgpack.FilePart(file_path, "rb")
with file_obj as file: file.seek(params["location"]) read_bytes = params.get("read_bytes", FILE_BUFF) file_size = os.fstat(file.fileno()).st_size
if file_size > read_bytes: # Check if file is readable at current position (for big files) if not self.isReadable(site, params["inner_path"], file, params["location"]): raise RequestError("File not readable at position: %s" % params["location"]) else: if params.get("file_size") and params["file_size"] != file_size: self.connection.badAction(2) raise RequestError("File size does not match: %sB != %sB" % (params["file_size"], file_size))
if not streaming: file.read_bytes = read_bytes
if params["location"] > file_size: self.connection.badAction(5) raise RequestError("Bad file location")
if streaming: back = { "size": file_size, "location": min(file.tell() + read_bytes, file_size), "stream_bytes": min(read_bytes, file_size - params["location"]) } self.response(back) self.sendRawfile(file, read_bytes=read_bytes) else: back = { "body": file, "size": file_size, "location": min(file.tell() + file.read_bytes, file_size) } self.response(back, streaming=True)
bytes_sent = min(read_bytes, file_size - params["location"]) # Number of bytes we going to send site.settings["bytes_sent"] = site.settings.get("bytes_sent", 0) + bytes_sent if config.debug_socket: self.log.debug("File %s at position %s sent %s bytes" % (file_path, params["location"], bytes_sent))
# Add peer to site if not added before connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") if connected_peer: # Just added connected_peer.connect(self.connection) # Assign current connection to peer
return {"bytes_sent": bytes_sent, "file_size": file_size, "location": params["location"]}
except RequestError as err: self.log.debug("GetFile %s %s %s request error: %s" % (self.connection, params["site"], params["inner_path"], Debug.formatException(err))) self.response({"error": "File read error: %s" % err}) except OSError as err: if config.verbose: self.log.debug("GetFile read error: %s" % Debug.formatException(err)) self.response({"error": "File read error"}) return False except Exception as err: self.log.error("GetFile exception: %s" % Debug.formatException(err)) self.response({"error": "File read exception"}) return False
def actionGetFile(self, params): return self.handleGetFile(params)
def actionStreamFile(self, params): return self.handleGetFile(params, streaming=True)
# Peer exchange request def actionPex(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False
got_peer_keys = [] added = 0
# Add requester peer to site connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request")
if connected_peer: # It was not registered before added += 1 connected_peer.connect(self.connection) # Assign current connection to peer
# Add sent peers to site for packed_address in itertools.chain(params.get("peers", []), params.get("peers_ipv6", [])): address = helper.unpackAddress(packed_address) got_peer_keys.append("%s:%s" % address) if site.addPeer(*address, source="pex"): added += 1
# Add sent onion peers to site for packed_address in params.get("peers_onion", []): address = helper.unpackOnionAddress(packed_address) got_peer_keys.append("%s:%s" % address) if site.addPeer(*address, source="pex"): added += 1
# Send back peers that is not in the sent list and connectable (not port 0) packed_peers = helper.packPeers(site.getConnectablePeers(params["need"], ignore=got_peer_keys, allow_private=False))
if added: site.worker_manager.onPeers() if config.verbose: self.log.debug( "Added %s peers to %s using pex, sending back %s" % (added, site, {key: len(val) for key, val in packed_peers.items()}) )
back = { "peers": packed_peers["ipv4"], "peers_ipv6": packed_peers["ipv6"], "peers_onion": packed_peers["onion"] }
self.response(back)
# Get modified content.json files since def actionListModified(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False modified_files = site.content_manager.listModified(params["since"])
# Add peer to site if not added before connected_peer = site.addPeer(self.connection.ip, self.connection.port, source="request") if connected_peer: # Just added connected_peer.connect(self.connection) # Assign current connection to peer
self.response({"modified_files": modified_files})
def actionGetHashfield(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False
# Add peer to site if not added before peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, source="request") if not peer.connection: # Just added peer.connect(self.connection) # Assign current connection to peer
peer.time_my_hashfield_sent = time.time() # Don't send again if not changed
self.response({"hashfield_raw": site.content_manager.hashfield.tobytes()})
def findHashIds(self, site, hash_ids, limit=100): back = collections.defaultdict(lambda: collections.defaultdict(list)) found = site.worker_manager.findOptionalHashIds(hash_ids, limit=limit)
for hash_id, peers in found.items(): for peer in peers: ip_type = helper.getIpType(peer.ip) if len(back[ip_type][hash_id]) < 20: back[ip_type][hash_id].append(peer.packMyAddress()) return back
def actionFindHashIds(self, params): site = self.sites.get(params["site"]) s = time.time() if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False
event_key = "%s_findHashIds_%s_%s" % (self.connection.ip, params["site"], len(params["hash_ids"])) if self.connection.cpu_time > 0.5 or not RateLimit.isAllowed(event_key, 60 * 5): time.sleep(0.1) back = self.findHashIds(site, params["hash_ids"], limit=10) else: back = self.findHashIds(site, params["hash_ids"]) RateLimit.called(event_key)
my_hashes = [] my_hashfield_set = set(site.content_manager.hashfield) for hash_id in params["hash_ids"]: if hash_id in my_hashfield_set: my_hashes.append(hash_id)
if config.verbose: self.log.debug( "Found: %s for %s hashids in %.3fs" % ({key: len(val) for key, val in back.items()}, len(params["hash_ids"]), time.time() - s) ) self.response({"peers": back["ipv4"], "peers_onion": back["onion"], "peers_ipv6": back["ipv6"], "my": my_hashes})
def actionSetHashfield(self, params): site = self.sites.get(params["site"]) if not site or not site.isServing(): # Site unknown or not serving self.response({"error": "Unknown site"}) self.connection.badAction(5) return False
# Add or get peer peer = site.addPeer(self.connection.ip, self.connection.port, return_peer=True, connection=self.connection, source="request") if not peer.connection: peer.connect(self.connection) peer.hashfield.replaceFromBytes(params["hashfield_raw"]) self.response({"ok": "Updated"})
# Send a simple Pong! answer def actionPing(self, params): self.response(b"Pong!")
# Check requested port of the other peer def actionCheckport(self, params): if helper.getIpType(self.connection.ip) == "ipv6": sock_address = (self.connection.ip, params["port"], 0, 0) else: sock_address = (self.connection.ip, params["port"])
with closing(helper.createSocket(self.connection.ip)) as sock: sock.settimeout(5) if sock.connect_ex(sock_address) == 0: self.response({"status": "open", "ip_external": self.connection.ip}) else: self.response({"status": "closed", "ip_external": self.connection.ip})
# Unknown command def actionUnknown(self, cmd, params): self.response({"error": "Unknown command: %s" % cmd}) self.connection.badAction(5)