strawmelonjuice.com
Forking what is left of ZeroNet and hopefully adding an AT Proto Frontend/Proxy
1# Included modules
2import os
3import sys
4import stat
5import time
6import logging
7
8startup_errors = []
9def startupError(msg):
10 startup_errors.append(msg)
11 print("Startup error: %s" % msg)
12
13# Third party modules
14import gevent
15if gevent.version_info.major <= 1: # Workaround for random crash when libuv used with threads
16 try:
17 if "libev" not in str(gevent.config.loop):
18 gevent.config.loop = "libev-cext"
19 except Exception as err:
20 startupError("Unable to switch gevent loop to libev: %s" % err)
21
22import gevent.monkey
23gevent.monkey.patch_all(thread=False, subprocess=False)
24
25update_after_shutdown = False # If set True then update and restart zeronet after main loop ended
26restart_after_shutdown = False # If set True then restart zeronet after main loop ended
27
28# Load config
29from Config import config
30config.parse(silent=True) # Plugins need to access the configuration
31if not config.arguments: # Config parse failed, show the help screen and exit
32 config.parse()
33
34if not os.path.isdir(config.data_dir):
35 os.mkdir(config.data_dir)
36 try:
37 os.chmod(config.data_dir, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR)
38 except Exception as err:
39 startupError("Can't change permission of %s: %s" % (config.data_dir, err))
40
41if not os.path.isfile("%s/sites.json" % config.data_dir):
42 open("%s/sites.json" % config.data_dir, "w").write("{}")
43if not os.path.isfile("%s/users.json" % config.data_dir):
44 open("%s/users.json" % config.data_dir, "w").write("{}")
45
46if config.action == "main":
47 from util import helper
48 try:
49 lock = helper.openLocked("%s/lock.pid" % config.data_dir, "w")
50 lock.write("%s" % os.getpid())
51 except BlockingIOError as err:
52 startupError("Can't open lock file, your ZeroNet client is probably already running, exiting... (%s)" % err)
53 if config.open_browser and config.open_browser != "False":
54 print("Opening browser: %s...", config.open_browser)
55 import webbrowser
56 try:
57 if config.open_browser == "default_browser":
58 browser = webbrowser.get()
59 else:
60 browser = webbrowser.get(config.open_browser)
61 browser.open("http://%s:%s/%s" % (
62 config.ui_ip if config.ui_ip != "*" else "127.0.0.1", config.ui_port, config.homepage
63 ), new=2)
64 except Exception as err:
65 startupError("Error starting browser: %s" % err)
66 sys.exit()
67
68config.initLogging()
69
70# Debug dependent configuration
71from Debug import DebugHook
72
73# Load plugins
74from Plugin import PluginManager
75PluginManager.plugin_manager.loadPlugins()
76config.loadPlugins()
77config.parse() # Parse again to add plugin configuration options
78
79# Log current config
80logging.debug("Config: %s" % config)
81
82# Modify stack size on special hardwares
83if config.stack_size:
84 import threading
85 threading.stack_size(config.stack_size)
86
87# Use pure-python implementation of msgpack to save CPU
88if config.msgpack_purepython:
89 os.environ["MSGPACK_PUREPYTHON"] = "True"
90
91# Fix console encoding on Windows
92if sys.platform.startswith("win"):
93 import subprocess
94 try:
95 chcp_res = subprocess.check_output("chcp 65001", shell=True).decode(errors="ignore").strip()
96 logging.debug("Changed console encoding to utf8: %s" % chcp_res)
97 except Exception as err:
98 logging.error("Error changing console encoding to utf8: %s" % err)
99
100# Socket monkey patch
101if config.proxy:
102 from util import SocksProxy
103 import urllib.request
104 logging.info("Patching sockets to socks proxy: %s" % config.proxy)
105 if config.fileserver_ip == "*":
106 config.fileserver_ip = '127.0.0.1' # Do not accept connections anywhere but localhost
107 config.disable_udp = True # UDP not supported currently with proxy
108 SocksProxy.monkeyPatch(*config.proxy.split(":"))
109elif config.tor == "always":
110 from util import SocksProxy
111 import urllib.request
112 logging.info("Patching sockets to tor socks proxy: %s" % config.tor_proxy)
113 if config.fileserver_ip == "*":
114 config.fileserver_ip = '127.0.0.1' # Do not accept connections anywhere but localhost
115 SocksProxy.monkeyPatch(*config.tor_proxy.split(":"))
116 config.disable_udp = True
117elif config.bind:
118 bind = config.bind
119 if ":" not in config.bind:
120 bind += ":0"
121 from util import helper
122 helper.socketBindMonkeyPatch(*bind.split(":"))
123
124# -- Actions --
125
126
127@PluginManager.acceptPlugins
128class Actions(object):
129 def call(self, function_name, kwargs):
130 logging.info("Version: %s r%s, Python %s, Gevent: %s" % (config.version, config.rev, sys.version, gevent.__version__))
131
132 func = getattr(self, function_name, None)
133 back = func(**kwargs)
134 if back:
135 print(back)
136
137 # Default action: Start serving UiServer and FileServer
138 def main(self):
139 global ui_server, file_server
140 from File import FileServer
141 from Ui import UiServer
142 logging.info("Creating FileServer....")
143 file_server = FileServer()
144 logging.info("Creating UiServer....")
145 ui_server = UiServer()
146 file_server.ui_server = ui_server
147
148 for startup_error in startup_errors:
149 logging.error("Startup error: %s" % startup_error)
150
151 logging.info("Removing old SSL certs...")
152 from Crypt import CryptConnection
153 CryptConnection.manager.removeCerts()
154
155 logging.info("Starting servers....")
156 gevent.joinall([gevent.spawn(ui_server.start), gevent.spawn(file_server.start)])
157 logging.info("All server stopped")
158
159 # Site commands
160
161 def siteCreate(self, use_master_seed=True):
162 logging.info("Generating new privatekey (use_master_seed: %s)..." % config.use_master_seed)
163 from Crypt import CryptBitcoin
164 if use_master_seed:
165 from User import UserManager
166 user = UserManager.user_manager.get()
167 if not user:
168 user = UserManager.user_manager.create()
169 address, address_index, site_data = user.getNewSiteData()
170 privatekey = site_data["privatekey"]
171 logging.info("Generated using master seed from users.json, site index: %s" % address_index)
172 else:
173 privatekey = CryptBitcoin.newPrivatekey()
174 address = CryptBitcoin.privatekeyToAddress(privatekey)
175 logging.info("----------------------------------------------------------------------")
176 logging.info("Site private key: %s" % privatekey)
177 logging.info(" !!! ^ Save it now, required to modify the site ^ !!!")
178 logging.info("Site address: %s" % address)
179 logging.info("----------------------------------------------------------------------")
180
181 while True and not config.batch and not use_master_seed:
182 if input("? Have you secured your private key? (yes, no) > ").lower() == "yes":
183 break
184 else:
185 logging.info("Please, secure it now, you going to need it to modify your site!")
186
187 logging.info("Creating directory structure...")
188 from Site.Site import Site
189 from Site import SiteManager
190 SiteManager.site_manager.load()
191
192 os.mkdir("%s/%s" % (config.data_dir, address))
193 open("%s/%s/index.html" % (config.data_dir, address), "w").write("Hello %s!" % address)
194
195 logging.info("Creating content.json...")
196 site = Site(address)
197 extend = {"postmessage_nonce_security": True}
198 if use_master_seed:
199 extend["address_index"] = address_index
200
201 site.content_manager.sign(privatekey=privatekey, extend=extend)
202 site.settings["own"] = True
203 site.saveSettings()
204
205 logging.info("Site created!")
206
207 def siteSign(self, address, privatekey=None, inner_path="content.json", publish=False, remove_missing_optional=False):
208 from Site.Site import Site
209 from Site import SiteManager
210 from Debug import Debug
211 SiteManager.site_manager.load()
212 logging.info("Signing site: %s..." % address)
213 site = Site(address, allow_create=False)
214
215 if not privatekey: # If no privatekey defined
216 from User import UserManager
217 user = UserManager.user_manager.get()
218 if user:
219 site_data = user.getSiteData(address)
220 privatekey = site_data.get("privatekey")
221 else:
222 privatekey = None
223 if not privatekey:
224 # Not found in users.json, ask from console
225 import getpass
226 privatekey = getpass.getpass("Private key (input hidden):")
227 try:
228 succ = site.content_manager.sign(
229 inner_path=inner_path, privatekey=privatekey,
230 update_changed_files=True, remove_missing_optional=remove_missing_optional
231 )
232 except Exception as err:
233 logging.error("Sign error: %s" % Debug.formatException(err))
234 succ = False
235 if succ and publish:
236 self.sitePublish(address, inner_path=inner_path)
237
238 def siteVerify(self, address):
239 import time
240 from Site.Site import Site
241 from Site import SiteManager
242 SiteManager.site_manager.load()
243
244 s = time.time()
245 logging.info("Verifing site: %s..." % address)
246 site = Site(address)
247 bad_files = []
248
249 for content_inner_path in site.content_manager.contents:
250 s = time.time()
251 logging.info("Verifing %s signature..." % content_inner_path)
252 err = None
253 try:
254 file_correct = site.content_manager.verifyFile(
255 content_inner_path, site.storage.open(content_inner_path, "rb"), ignore_same=False
256 )
257 except Exception as err:
258 file_correct = False
259
260 if file_correct is True:
261 logging.info("[OK] %s (Done in %.3fs)" % (content_inner_path, time.time() - s))
262 else:
263 logging.error("[ERROR] %s: invalid file: %s!" % (content_inner_path, err))
264 input("Continue?")
265 bad_files += content_inner_path
266
267 logging.info("Verifying site files...")
268 bad_files += site.storage.verifyFiles()["bad_files"]
269 if not bad_files:
270 logging.info("[OK] All file sha512sum matches! (%.3fs)" % (time.time() - s))
271 else:
272 logging.error("[ERROR] Error during verifying site files!")
273
274 def dbRebuild(self, address):
275 from Site.Site import Site
276 from Site import SiteManager
277 SiteManager.site_manager.load()
278
279 logging.info("Rebuilding site sql cache: %s..." % address)
280 site = SiteManager.site_manager.get(address)
281 s = time.time()
282 try:
283 site.storage.rebuildDb()
284 logging.info("Done in %.3fs" % (time.time() - s))
285 except Exception as err:
286 logging.error(err)
287
288 def dbQuery(self, address, query):
289 from Site.Site import Site
290 from Site import SiteManager
291 SiteManager.site_manager.load()
292
293 import json
294 site = Site(address)
295 result = []
296 for row in site.storage.query(query):
297 result.append(dict(row))
298 print(json.dumps(result, indent=4))
299
300 def siteAnnounce(self, address):
301 from Site.Site import Site
302 from Site import SiteManager
303 SiteManager.site_manager.load()
304
305 logging.info("Opening a simple connection server")
306 global file_server
307 from File import FileServer
308 file_server = FileServer("127.0.0.1", 1234)
309 file_server.start()
310
311 logging.info("Announcing site %s to tracker..." % address)
312 site = Site(address)
313
314 s = time.time()
315 site.announce()
316 print("Response time: %.3fs" % (time.time() - s))
317 print(site.peers)
318
319 def siteDownload(self, address):
320 from Site.Site import Site
321 from Site import SiteManager
322 SiteManager.site_manager.load()
323
324 logging.info("Opening a simple connection server")
325 global file_server
326 from File import FileServer
327 file_server = FileServer("127.0.0.1", 1234)
328 file_server_thread = gevent.spawn(file_server.start, check_sites=False)
329
330 site = Site(address)
331
332 on_completed = gevent.event.AsyncResult()
333
334 def onComplete(evt):
335 evt.set(True)
336
337 site.onComplete.once(lambda: onComplete(on_completed))
338 print("Announcing...")
339 site.announce()
340
341 s = time.time()
342 print("Downloading...")
343 site.downloadContent("content.json", check_modifications=True)
344
345 print("Downloaded in %.3fs" % (time.time()-s))
346
347 def siteNeedFile(self, address, inner_path):
348 from Site.Site import Site
349 from Site import SiteManager
350 SiteManager.site_manager.load()
351
352 def checker():
353 while 1:
354 s = time.time()
355 time.sleep(1)
356 print("Switch time:", time.time() - s)
357 gevent.spawn(checker)
358
359 logging.info("Opening a simple connection server")
360 global file_server
361 from File import FileServer
362 file_server = FileServer("127.0.0.1", 1234)
363 file_server_thread = gevent.spawn(file_server.start, check_sites=False)
364
365 site = Site(address)
366 site.announce()
367 print(site.needFile(inner_path, update=True))
368
369 def siteCmd(self, address, cmd, parameters):
370 import json
371 from Site import SiteManager
372
373 site = SiteManager.site_manager.get(address)
374
375 if not site:
376 logging.error("Site not found: %s" % address)
377 return None
378
379 ws = self.getWebsocket(site)
380
381 ws.send(json.dumps({"cmd": cmd, "params": parameters, "id": 1}))
382 res_raw = ws.recv()
383
384 try:
385 res = json.loads(res_raw)
386 except Exception as err:
387 return {"error": "Invalid result: %s" % err, "res_raw": res_raw}
388
389 if "result" in res:
390 return res["result"]
391 else:
392 return res
393
394 def getWebsocket(self, site):
395 import websocket
396
397 ws_address = "ws://%s:%s/Websocket?wrapper_key=%s" % (config.ui_ip, config.ui_port, site.settings["wrapper_key"])
398 logging.info("Connecting to %s" % ws_address)
399 ws = websocket.create_connection(ws_address)
400 return ws
401
402 def sitePublish(self, address, peer_ip=None, peer_port=15441, inner_path="content.json"):
403 global file_server
404 from Site.Site import Site
405 from Site import SiteManager
406 from File import FileServer # We need fileserver to handle incoming file requests
407 from Peer import Peer
408 file_server = FileServer()
409 site = SiteManager.site_manager.get(address)
410 logging.info("Loading site...")
411 site.settings["serving"] = True # Serving the site even if its disabled
412
413 try:
414 ws = self.getWebsocket(site)
415 logging.info("Sending siteReload")
416 self.siteCmd(address, "siteReload", inner_path)
417
418 logging.info("Sending sitePublish")
419 self.siteCmd(address, "sitePublish", {"inner_path": inner_path, "sign": False})
420 logging.info("Done.")
421
422 except Exception as err:
423 logging.info("Can't connect to local websocket client: %s" % err)
424 logging.info("Creating FileServer....")
425 file_server_thread = gevent.spawn(file_server.start, check_sites=False) # Dont check every site integrity
426 time.sleep(0.001)
427
428 # Started fileserver
429 file_server.portCheck()
430 if peer_ip: # Announce ip specificed
431 site.addPeer(peer_ip, peer_port)
432 else: # Just ask the tracker
433 logging.info("Gathering peers from tracker")
434 site.announce() # Gather peers
435 published = site.publish(5, inner_path) # Push to peers
436 if published > 0:
437 time.sleep(3)
438 logging.info("Serving files (max 60s)...")
439 gevent.joinall([file_server_thread], timeout=60)
440 logging.info("Done.")
441 else:
442 logging.info("No peers found, sitePublish command only works if you already have visitors serving your site")
443
444 # Crypto commands
445 def cryptPrivatekeyToAddress(self, privatekey=None):
446 from Crypt import CryptBitcoin
447 if not privatekey: # If no privatekey in args then ask it now
448 import getpass
449 privatekey = getpass.getpass("Private key (input hidden):")
450
451 print(CryptBitcoin.privatekeyToAddress(privatekey))
452
453 def cryptSign(self, message, privatekey):
454 from Crypt import CryptBitcoin
455 print(CryptBitcoin.sign(message, privatekey))
456
457 def cryptVerify(self, message, sign, address):
458 from Crypt import CryptBitcoin
459 print(CryptBitcoin.verify(message, address, sign))
460
461 def cryptGetPrivatekey(self, master_seed, site_address_index=None):
462 from Crypt import CryptBitcoin
463 if len(master_seed) != 64:
464 logging.error("Error: Invalid master seed length: %s (required: 64)" % len(master_seed))
465 return False
466 privatekey = CryptBitcoin.hdPrivatekey(master_seed, site_address_index)
467 print("Requested private key: %s" % privatekey)
468
469 # Peer
470 def peerPing(self, peer_ip, peer_port=None):
471 if not peer_port:
472 peer_port = 15441
473 logging.info("Opening a simple connection server")
474 global file_server
475 from Connection import ConnectionServer
476 file_server = ConnectionServer("127.0.0.1", 1234)
477 file_server.start(check_connections=False)
478 from Crypt import CryptConnection
479 CryptConnection.manager.loadCerts()
480
481 from Peer import Peer
482 logging.info("Pinging 5 times peer: %s:%s..." % (peer_ip, int(peer_port)))
483 s = time.time()
484 peer = Peer(peer_ip, peer_port)
485 peer.connect()
486
487 if not peer.connection:
488 print("Error: Can't connect to peer (connection error: %s)" % peer.connection_error)
489 return False
490 if "shared_ciphers" in dir(peer.connection.sock):
491 print("Shared ciphers:", peer.connection.sock.shared_ciphers())
492 if "cipher" in dir(peer.connection.sock):
493 print("Cipher:", peer.connection.sock.cipher()[0])
494 if "version" in dir(peer.connection.sock):
495 print("TLS version:", peer.connection.sock.version())
496 print("Connection time: %.3fs (connection error: %s)" % (time.time() - s, peer.connection_error))
497
498 for i in range(5):
499 ping_delay = peer.ping()
500 print("Response time: %.3fs" % ping_delay)
501 time.sleep(1)
502 peer.remove()
503 print("Reconnect test...")
504 peer = Peer(peer_ip, peer_port)
505 for i in range(5):
506 ping_delay = peer.ping()
507 print("Response time: %.3fs" % ping_delay)
508 time.sleep(1)
509
510 def peerGetFile(self, peer_ip, peer_port, site, filename, benchmark=False):
511 logging.info("Opening a simple connection server")
512 global file_server
513 from Connection import ConnectionServer
514 file_server = ConnectionServer("127.0.0.1", 1234)
515 file_server.start(check_connections=False)
516 from Crypt import CryptConnection
517 CryptConnection.manager.loadCerts()
518
519 from Peer import Peer
520 logging.info("Getting %s/%s from peer: %s:%s..." % (site, filename, peer_ip, peer_port))
521 peer = Peer(peer_ip, peer_port)
522 s = time.time()
523 if benchmark:
524 for i in range(10):
525 peer.getFile(site, filename),
526 print("Response time: %.3fs" % (time.time() - s))
527 input("Check memory")
528 else:
529 print(peer.getFile(site, filename).read())
530
531 def peerCmd(self, peer_ip, peer_port, cmd, parameters):
532 logging.info("Opening a simple connection server")
533 global file_server
534 from Connection import ConnectionServer
535 file_server = ConnectionServer()
536 file_server.start(check_connections=False)
537 from Crypt import CryptConnection
538 CryptConnection.manager.loadCerts()
539
540 from Peer import Peer
541 peer = Peer(peer_ip, peer_port)
542
543 import json
544 if parameters:
545 parameters = json.loads(parameters.replace("'", '"'))
546 else:
547 parameters = {}
548 try:
549 res = peer.request(cmd, parameters)
550 print(json.dumps(res, indent=2, ensure_ascii=False))
551 except Exception as err:
552 print("Unknown response (%s): %s" % (err, res))
553
554 def getConfig(self):
555 import json
556 print(json.dumps(config.getServerInfo(), indent=2, ensure_ascii=False))
557
558 def test(self, test_name, *args, **kwargs):
559 import types
560 def funcToName(func_name):
561 test_name = func_name.replace("test", "")
562 return test_name[0].lower() + test_name[1:]
563
564 test_names = [funcToName(name) for name in dir(self) if name.startswith("test") and name != "test"]
565 if not test_name:
566 # No test specificed, list tests
567 print("\nNo test specified, possible tests:")
568 for test_name in test_names:
569 func_name = "test" + test_name[0].upper() + test_name[1:]
570 func = getattr(self, func_name)
571 if func.__doc__:
572 print("- %s: %s" % (test_name, func.__doc__.strip()))
573 else:
574 print("- %s" % test_name)
575 return None
576
577 # Run tests
578 func_name = "test" + test_name[0].upper() + test_name[1:]
579 if hasattr(self, func_name):
580 func = getattr(self, func_name)
581 print("- Running test: %s" % test_name, end="")
582 s = time.time()
583 ret = func(*args, **kwargs)
584 if type(ret) is types.GeneratorType:
585 for progress in ret:
586 print(progress, end="")
587 sys.stdout.flush()
588 print("\n* Test %s done in %.3fs" % (test_name, time.time() - s))
589 else:
590 print("Unknown test: %r (choose from: %s)" % (
591 test_name, test_names
592 ))
593
594
595actions = Actions()
596# Starts here when running zeronet.py
597
598
599def start():
600 # Call function
601 action_kwargs = config.getActionArguments()
602 actions.call(config.action, action_kwargs)