j3s.sh
capsul.org webapp
1import logging
2from logging.config import dictConfig as logging_dict_config
3
4import atexit
5import os
6import hashlib
7import requests
8import sys
9import json
10
11import stripe
12from dotenv import load_dotenv, find_dotenv
13from flask import Flask
14from flask_mail import Mail, Message
15from flask import render_template
16from flask import url_for
17from flask import current_app
18from flask import flash
19from flask import session
20from apscheduler.schedulers.background import BackgroundScheduler
21
22
23from capsulflask.shared import my_exec_info_message
24from capsulflask import hub_model, spoke_model, cli
25from capsulflask.payment import try_reconnnect_btcpay
26from capsulflask.http_client import MyHTTPClient
27
28
29load_dotenv(find_dotenv())
30
31app = Flask(__name__)
32
33app.config.from_mapping(
34 BASE_URL=os.environ.get("BASE_URL", default="http://localhost:5000"),
35 SECRET_KEY=os.environ.get("SECRET_KEY", default="dev"),
36 HUB_MODE_ENABLED=os.environ.get("HUB_MODE_ENABLED", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
37 SPOKE_MODE_ENABLED=os.environ.get("SPOKE_MODE_ENABLED", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
38 INTERNAL_HTTP_TIMEOUT_SECONDS=os.environ.get("INTERNAL_HTTP_TIMEOUT_SECONDS", default="300"),
39 HUB_MODEL=os.environ.get("HUB_MODEL", default="capsul-flask"),
40 SPOKE_MODEL=os.environ.get("SPOKE_MODEL", default="mock"),
41 LOG_LEVEL=os.environ.get("LOG_LEVEL", default="INFO"),
42 SPOKE_HOST_ID=os.environ.get("SPOKE_HOST_ID", default="baikal"),
43 SPOKE_HOST_TOKEN=os.environ.get("SPOKE_HOST_TOKEN", default="changeme"),
44 HUB_TOKEN=os.environ.get("HUB_TOKEN", default="changeme"),
45 LIBVIRT_DNSMASQ_PATH=os.environ.get("LIBVIRT_DNSMASQ_PATH", default="/var/lib/libvirt/dnsmasq").rstrip("/"),
46
47
48
49 # https://www.postgresql.org/docs/9.1/libpq-ssl.html#LIBPQ-SSL-SSLMODE-STATEMENTS
50 # https://stackoverflow.com/questions/56332906/where-to-put-ssl-certificates-when-trying-to-connect-to-a-remote-database-using
51 # TLS example: sslmode=verify-full sslrootcert=letsencrypt-root-ca.crt host=db.example.com port=5432 user=postgres password=dev dbname=postgres
52 POSTGRES_CONNECTION_PARAMETERS=os.environ.get(
53 "POSTGRES_CONNECTION_PARAMETERS",
54 default="host=localhost port=5432 user=postgres password=dev dbname=postgres"
55 ),
56
57 DATABASE_SCHEMA=os.environ.get("DATABASE_SCHEMA", default="public"),
58
59 MAIL_SERVER=os.environ.get("MAIL_SERVER", default=""),
60 MAIL_PORT=os.environ.get("MAIL_PORT", default="465"),
61 MAIL_USE_TLS=os.environ.get("MAIL_USE_TLS", default="False").lower() in ['true', '1', 't', 'y', 'yes'],
62 MAIL_USE_SSL=os.environ.get("MAIL_USE_SSL", default="True").lower() in ['true', '1', 't', 'y', 'yes'],
63 MAIL_USERNAME=os.environ.get("MAIL_USERNAME", default=""),
64 MAIL_PASSWORD=os.environ.get("MAIL_PASSWORD", default=""),
65 MAIL_DEFAULT_SENDER=os.environ.get("MAIL_DEFAULT_SENDER", default="no-reply@capsul.org"),
66 ADMIN_NOTIFICATION_EMAIL_ADDRESSES=os.environ.get("ADMIN_NOTIFICATION_EMAIL_ADDRESSES", default="support@capsul.org"),
67 ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES=os.environ.get("ADMIN_PANEL_ALLOW_EMAIL_ADDRESSES", default="j3s@c3f.net,forest.n.johnson@gmail.com,capsul@cyberia.club"),
68 BANNED_USERS_EMAIL_ADDRESSES=os.environ.get("BANNED_USERS_EMAIL_ADDRESSES", default=""),
69 BANNED_USERS_IP_ADDRESSES=os.environ.get("BANNED_USERS_IP_ADDRESSES", default=""),
70
71 VM_STORAGE_DIR=os.environ.get("VM_STORAGE_DIR", default="/tank/vm"),
72 BACKUP_STORAGE_MOUNTS=os.environ.get("BACKUP_STORAGE_MOUNTS", default="/mnt/backup1"),
73 BACKUP_DEBUG_LOG=os.environ.get("BACKUP_DEBUG_LOG", default="false"),
74
75 MAIL_DELIVERY_MANAGER_URL=os.environ.get("MAIL_DELIVERY_MANAGER_URL", default=""),
76 MAIL_DELIVERY_MANAGER_SECRET=os.environ.get("MAIL_DELIVERY_MANAGER_SECRET", default=""),
77
78 PROMETHEUS_URL=os.environ.get("PROMETHEUS_URL", default="https://prometheus.cyberia.club"),
79
80 STRIPE_API_VERSION=os.environ.get("STRIPE_API_VERSION", default="2020-03-02"),
81 STRIPE_SECRET_KEY=os.environ.get("STRIPE_SECRET_KEY", default=""),
82 STRIPE_PUBLISHABLE_KEY=os.environ.get("STRIPE_PUBLISHABLE_KEY", default=""),
83 #STRIPE_WEBHOOK_SECRET=os.environ.get("STRIPE_WEBHOOK_SECRET", default="")
84
85 BTCPAY_PRIVATE_KEY=os.environ.get("BTCPAY_PRIVATE_KEY", default="").replace("\\n", "\n"),
86 BTCPAY_URL=os.environ.get("BTCPAY_URL", default="")
87)
88
89app.config['HUB_URL'] = os.environ.get("HUB_URL", default=app.config['BASE_URL'])
90
91class SetLogLevelToDebugForHeartbeatRelatedMessagesFilter(logging.Filter):
92 def is_often_logged_scheduled_task_message(self, thing):
93 # thing_string = "<error>"
94 is_in_string = False
95 try:
96 thing_string = "%s" % thing
97 is_in_string = 'heartbeat-task' in thing_string \
98 or 'hub/heartbeat' in thing_string \
99 or 'spoke/heartbeat' in thing_string
100 except:
101 pass
102 # self.warning("is_often_logged_scheduled_task_message(%s): %s", thing_string, is_in_string )
103 return is_in_string
104
105 def filter(self, record):
106 if app.config['LOG_LEVEL'] == "DEBUG":
107 return True
108
109 related_to_backup_maintenance = next(filter(lambda x: 'backup-maintenance' in ("%s" % x), record.args), None) is not None
110 is_not_error = 'scheduled at' in record.msg or 'executed successfully' in record.msg
111 if related_to_backup_maintenance and is_not_error:
112 return False
113
114 if self.is_often_logged_scheduled_task_message(record.msg):
115 return False
116 for arg in record.args:
117 if self.is_often_logged_scheduled_task_message(arg):
118 return False
119
120 return True
121
122logging_dict_config({
123 'version': 1,
124 'formatters': {'default': {
125 'format': '[%(asctime)s] %(levelname)s in %(module)s: %(message)s',
126 }},
127 'filters': {
128 'setLogLevelToDebugForHeartbeatRelatedMessages': {
129 '()': SetLogLevelToDebugForHeartbeatRelatedMessagesFilter,
130 }
131 },
132 'handlers': {'wsgi': {
133 'class': 'logging.StreamHandler',
134 'stream': 'ext://flask.logging.wsgi_errors_stream',
135 'formatter': 'default',
136 'filters': ['setLogLevelToDebugForHeartbeatRelatedMessages']
137 }},
138 'root': {
139 'level': app.config['LOG_LEVEL'],
140 'handlers': ['wsgi']
141 }
142})
143
144# app.logger.critical("critical")
145# app.logger.error("error")
146# app.logger.warning("warning")
147# app.logger.info("info")
148# app.logger.debug("debug")
149
150stripe.api_key = app.config['STRIPE_SECRET_KEY']
151stripe.api_version = app.config['STRIPE_API_VERSION']
152
153class StdoutMockFlaskMail:
154 def send(self, message: Message):
155 current_app.logger.info(f"Email would have been sent if configured:\n\nto: {','.join(message.recipients)}\nsubject: {message.subject}\nbody:\n\n{message.body}\n\n")
156
157class DeliveryManagerBasedFlaskMail:
158 def send(self, message: Message):
159 responses = []
160 send_to = message.recipients if message.recipients is not None else []
161 if message.bcc is not None:
162 send_to = send_to + message.bcc
163
164 for r in send_to:
165 delivery_manager_headers = {'Authorization': f"{app.config['MAIL_DELIVERY_MANAGER_SECRET']}"}
166 delivery_manager_url = f"{app.config['MAIL_DELIVERY_MANAGER_URL']}/send"
167 payload = json.dumps({
168 'To': r,
169 'Subject': message.subject,
170 'Body': message.body
171 })
172 response = requests.post(delivery_manager_url, headers=delivery_manager_headers, data=payload)
173 if response.ok:
174 responses.append(response.json())
175 else:
176 app.logger.error(f"failed to contact delivery_manager_url={delivery_manager_url} : HTTP {response.status_code}: {response.text}")
177 responses.append({
178 'Attempted': False,
179 'Delivered': False,
180 'Error': response.text,
181 'WaitUntilTimestampMs': 0
182 })
183
184 return responses
185
186if app.config['MAIL_DELIVERY_MANAGER_URL'] != "":
187 app.config['FLASK_MAIL_INSTANCE'] = DeliveryManagerBasedFlaskMail()
188elif app.config['MAIL_SERVER'] != "":
189 app.config['FLASK_MAIL_INSTANCE'] = Mail(app)
190else:
191 app.logger.warn("No MAIL_SERVER configured. capsul will simply print emails to stdout.")
192 app.config['FLASK_MAIL_INSTANCE'] = StdoutMockFlaskMail()
193
194app.config['HTTP_CLIENT'] = MyHTTPClient(timeout_seconds=int(app.config['INTERNAL_HTTP_TIMEOUT_SECONDS']))
195
196if app.config['BTCPAY_URL'] != "":
197 try_reconnnect_btcpay(app)
198
199# only start the scheduler and attempt to migrate the database if we are running the app.
200# otherwise we are running a CLI command.
201command_line = ' '.join(sys.argv)
202is_running_server = ('flask run' in command_line) or ('gunicorn' in command_line)
203
204app.logger.info(f"is_running_server: {is_running_server}")
205
206if app.config['HUB_MODE_ENABLED']:
207
208 if app.config['HUB_MODEL'] == "capsul-flask":
209 app.config['HUB_MODEL'] = hub_model.CapsulFlaskHub(app)
210
211 # debug mode (flask reloader) runs two copies of the app. When running in debug mode,
212 # we only want to start the scheduler one time.
213 if is_running_server and (not app.debug or os.environ.get('WERKZEUG_RUN_MAIN') == 'true'):
214 scheduler = BackgroundScheduler()
215 hub_token_headers = {'Authorization': f"Bearer {app.config['HUB_TOKEN']}"}
216
217 heartbeat_task_url = f"{app.config['HUB_URL']}/hub/heartbeat-task"
218 trigger_heartbeat_task = lambda: requests.post(heartbeat_task_url, headers=hub_token_headers)
219 scheduler.add_job(name="heartbeat-task", func=trigger_heartbeat_task, trigger="interval", seconds=5)
220
221 seconds_in_a_day=86400
222 payment_maintenance_task_url = f"{app.config['HUB_URL']}/hub/payment-maintenance"
223 trigger_payment_maintenance_task = lambda: requests.post(payment_maintenance_task_url, headers=hub_token_headers)
224 scheduler.add_job(name="payment-maintenance", func=trigger_payment_maintenance_task, trigger="interval", seconds=seconds_in_a_day)
225
226 backup_maintenance_task_url = f"{app.config['HUB_URL']}/hub/backup-maintenance"
227 trigger_backup_maintenance_task = lambda: requests.post(backup_maintenance_task_url, headers=hub_token_headers)
228 scheduler.add_job(name="backup-maintenance", func=trigger_backup_maintenance_task, trigger="interval", seconds=15)
229
230 scheduler.start()
231
232 atexit.register(lambda: scheduler.shutdown())
233
234 else:
235 app.config['HUB_MODEL'] = hub_model.MockHub()
236
237 from capsulflask import db
238 db.init_app(app, is_running_server)
239
240 from capsulflask import auth, landing, console, payment, metrics, cli, hub_api, admin
241
242 app.register_blueprint(landing.bp)
243 app.register_blueprint(auth.bp)
244 app.register_blueprint(console.bp)
245 app.register_blueprint(payment.bp)
246 app.register_blueprint(metrics.bp)
247 app.register_blueprint(cli.bp)
248 app.register_blueprint(hub_api.bp)
249 app.register_blueprint(admin.bp)
250
251 app.add_url_rule("/", endpoint="index")
252
253
254
255if app.config['SPOKE_MODE_ENABLED']:
256
257 if app.config['SPOKE_MODEL'] == "shell-scripts":
258 app.config['SPOKE_MODEL'] = spoke_model.ShellScriptSpoke(app)
259 else:
260 app.config['SPOKE_MODEL'] = spoke_model.MockSpoke()
261
262 from capsulflask import spoke_api
263
264 app.register_blueprint(spoke_api.bp)
265
266@app.after_request
267def security_headers(response):
268 response.headers['X-Frame-Options'] = 'SAMEORIGIN'
269 if 'Content-Security-Policy' not in response.headers:
270 response.headers['Content-Security-Policy'] = "default-src 'self'; script-src 'self' 'wasm-unsafe-eval'; object-src 'none'"
271 response.headers['X-Content-Type-Options'] = 'nosniff'
272
273 if current_app.config['BROADCAST_BANNER_MESSAGE'] is not None and current_app.config['BROADCAST_BANNER_MESSAGE'] != "":
274 for t in session.get("_flashes", []):
275 if t is not None and t[1] == current_app.config['BROADCAST_BANNER_MESSAGE']:
276 return response
277
278 flash(current_app.config['BROADCAST_BANNER_MESSAGE'])
279
280 return response
281
282
283@app.context_processor
284def override_url_for():
285 """
286 override the url_for function built into flask
287 with our own custom implementation that busts the cache correctly when files change
288 """
289 return dict(url_for=url_for_with_cache_bust)
290
291
292def url_for_with_cache_bust(endpoint, **values):
293 """
294 Add a query parameter based on the hash of the file, this acts as a cache bust
295 """
296
297 if endpoint == 'static':
298 filename = values.get('filename', None)
299 if filename:
300 if 'STATIC_FILE_HASH_CACHE' not in current_app.config:
301 current_app.config['STATIC_FILE_HASH_CACHE'] = dict()
302
303 if filename not in current_app.config['STATIC_FILE_HASH_CACHE']:
304 filepath = os.path.join(current_app.root_path, endpoint, filename)
305 #print(filepath)
306 if os.path.isfile(filepath) and os.access(filepath, os.R_OK):
307
308 with open(filepath, 'rb') as file:
309 hasher = hashlib.md5()
310 hasher.update(file.read())
311 current_app.config['STATIC_FILE_HASH_CACHE'][filename] = hasher.hexdigest()[-6:]
312
313 values['q'] = current_app.config['STATIC_FILE_HASH_CACHE'][filename]
314
315 return url_for(endpoint, **values)