···3import torch.nn as nn
4from torch.nn import functional as func
5from safetensors import safe_open as st_open
067class Attention(nn.Module):
8 def __init__(self):
···153 for key in sd.keys():
154 sd[key].copy_(file.get_tensor(key))
155000000000000000000000000000000
···3import torch.nn as nn
4from torch.nn import functional as func
5from safetensors import safe_open as st_open
6+from PIL import Image
78class Attention(nn.Module):
9 def __init__(self):
···154 for key in sd.keys():
155 sd[key].copy_(file.get_tensor(key))
156157+approximation_matrix = [
158+ [0.85, 0.85, 0.6], # seems to be mainly value
159+ [-0.35, 0.2, 0.5], # mainly blue? maybe a little green, def not red
160+ [0.15, 0.15, 0], # yellow. but mainly encoding texture not color, i think
161+ [0.15, -0.35, -0.35] # inverted value? but also red
162+]
163+164+def save_approx_decode(latents, path):
165+ lmin = latents.min()
166+ l = latents - lmin
167+ lmax = latents.max()
168+ l = latents / lmax
169+ l = l.float().mul_(0.5).add_(0.5)
170+ ims = []
171+ for lat in l:
172+ apx_mat = torch.tensor(approximation_matrix).to("cuda")
173+ approx_decode = torch.einsum("...lhw,lr -> ...rhw", lat, apx_mat).mul_(255).round()
174+ #lat -= lat.min()
175+ #lat /= lat.max()
176+ im_data = approx_decode.permute(1,2,0).detach().cpu().numpy().astype("uint8")
177+ #im_data = im_data.round().astype("uint8")
178+ im = Image.fromarray(im_data).resize(size=(im_data.shape[1]*8,im_data.shape[0]*8), resample=Image.NEAREST)
179+ ims += [im]
180+181+ #clear_output()
182+ for im in ims:
183+ #im.save(f"out/tmp_approx_decode/{index:06d}.bmp")
184+ im.save(path)
185+ #display(im)
186+
···1+import dis
2+3+class Errs(type):
4+ def __iter__(_class):
5+ return iter((k,v) for (k,v) in vars(_class).items() if not k.startswith("_"))
6+7+ def __repr__(_class):
8+ return str(list(k for (k,v) in _class))
9+10+def errs(errs):
11+ def _errs(fn):
12+ fn.errs = errs
13+ return fn
14+ return _errs
15+16+# stuff above this line i probs move to another file
17+18+class _Errs(metaclass=Errs):
19+ FOUND_RETURN = "encountered return instruction"
20+ IN_DEFINITION = "error in fn definition"
21+ IN_EXECUTION = "error in fn execution"
22+ NON_DICT_RETURN = "fn dumped a non-dict"
23+24+def _modify_to_dump_locals(fn, fn_source, deftime_globals, log):
25+ instructions = dis.get_instructions(fn)
26+ for instruction in instructions:
27+ if instruction.opcode == dis.opmap["RETURN_VALUE"]:
28+ return (False, _Errs.FOUND_RETURN)
29+ fn_source = fn_source
30+ fn_source_modified = f"{fn_source}\n return locals()"
31+ sandbox = {}
32+ try:
33+ exec(fn_source_modified, globals=deftime_globals, locals=sandbox)
34+ except KeyboardInterrupt:
35+ raise
36+ except:
37+ log.indented().trace(source=fn)
38+ return (False, _Errs.IN_DEFINITION)
39+ return (True, sandbox[fn.__name__])
40+41+@errs(_Errs)
42+def try_dump_locals(fn, fn_source, args, kwargs, deftime_globals, log):
43+ (success, fn_or_err) = _modify_to_dump_locals(fn, fn_source, deftime_globals, log)
44+ if not success:
45+ return (False, fn_or_err)
46+ try:
47+ res = fn_or_err(*args, **kwargs)
48+ except KeyboardInterrupt:
49+ raise
50+ except:
51+ log.indented().trace(source=fn)
52+ return (False, _Errs.IN_EXECUTION)
53+ if not isinstance(res, dict):
54+ return (False, _Errs.NON_DICT_RETURN)
55+ return (True, res)
+1
lib/iter.py
···2def first(l, p):
3 return next((idx,value) for idx,value in enumerate(l) if p(value))
405def pairs(l):
6 return zip(l, l[1:])
7
···2def first(l, p):
3 return next((idx,value) for idx,value in enumerate(l) if p(value))
45+# just use itertools.pairwise
6def pairs(l):
7 return zip(l, l[1:])
8
···1+# pytorch adaptation of the methods described here:
2+# https://www.dgp.toronto.edu/public_user/stam/reality/Research/pdf/GDC03.pdf (overall design)
3+# https://www.karlsims.com/fluid-flow.html (divergence clearing step)
4+5+# only dependencies are pillow and pytorch
6+7+import math
8+import time
9+from pathlib import Path
10+from PIL import Image
11+12+import torch
13+from torch.nn.functional import conv2d
14+15+# settings
16+17+result_directory = "out"
18+Path(result_directory).mkdir(parents=True, exist_ok=True)
19+Path(result_directory + "/velocities").mkdir(parents=True, exist_ok=True)
20+Path(result_directory + "/densities").mkdir(parents=True, exist_ok=True)
21+22+scale = 10
23+w = 2**scale
24+h = 2**scale
25+26+max_iterations = 100000
27+save_every = 1
28+29+velocity_diffusion_rate = 20
30+density_diffusion_rate = None
31+density_timescale = 0.005
32+timescale = 0.01
33+34+torch.set_default_device("cuda")
35+36+diffusion_solver_steps = 10
37+divergence_clearing_steps = 20
38+39+40+# save images
41+42+def monochrome_save(tensor, filename):
43+ """save a 2d tensor of shape (h,w) as a monochrome image"""
44+ scaled = torch.clone(tensor).clamp_(0,1).mul_(255).round().type(torch.uint8)
45+ # copy to make 3 color channels
46+ rearranged = scaled.unsqueeze(2).expand(-1, -1, 3).cpu().numpy()
47+ Image.fromarray(rearranged).save(f"{result_directory}/{filename}.png")
48+49+def save(tensor, filename):
50+ """save a 3d tensor with shape (3,h,w) as an rgb image"""
51+ scaled = torch.clone(tensor).clamp_(0, 1).mul_(255).round().type(torch.uint8)
52+ rearranged = scaled.permute(1, 2, 0).cpu().numpy()
53+ Image.fromarray(rearranged).save(f"{result_directory}/{filename}.png")
54+55+56+57+58+59+# convolution
60+61+# the backbone of this simulation is a handful of discrete convolutions
62+# detailed definition here: https://en.wikipedia.org/wiki/Convolution#Discrete_convolution
63+64+# roughly: we have a discrete field and a "kernel".
65+# we look at a small window of values in the field, multiply each value there by a number,
66+# and sum the results. the kernel describes what number to multiply each value in the window by.
67+# we do this for every possible window, centered on every point we can fit the window around,
68+# skipping points where the window would hang off the edge of the field. thus, we end up with
69+# a result that's slightly smaller than the field
70+71+def convolve(field, kernel):
72+ # conv2d works in batches & we only ever need to do a 1-element batch
73+ # "unsqueeze" is pytorch's absolutely bizarre term for wrapping a tensor with another dimension. like going from [1,2] to [[1,2]]
74+ return conv2d(field.unsqueeze(0), kernel, bias=None, padding=[0], stride=[1])[0]
75+76+# convolution kernels:
77+78+# total of values of immediate neighbors in all 4 cardinal directions
79+diffusion_kernel = torch.tensor([[[
80+ [0, 1, 0],
81+ [1, 0, 1],
82+ [0, 1, 0]
83+]]], dtype=torch.float)
84+85+# gradient of divergence in the x direction
86+x_div_kernel = torch.tensor([[[
87+ [ 1, 2, 1],
88+ [-2,-4,-2],
89+ [ 1, 2, 1]
90+]]], dtype=torch.float)
91+92+div_opp_kernel = torch.tensor([[[
93+ [ 1, 0,-1],
94+ [ 0, 0, 0],
95+ [-1, 0, 1]
96+]]], dtype=torch.float)
97+98+# gradient of divergence in the y direction
99+y_div_kernel = torch.tensor([[[
100+ [ 1,-2, 1],
101+ [ 2,-4, 2],
102+ [ 1,-2, 1]
103+]]], dtype=torch.float)
104+105+# various boundary conditions
106+107+def continuous_boundary(field):
108+ """set the border values of the provided discretized 2d scalar field (that is, a 2d array of numbers) to be equal to their inner neighbors (& average the corner points)"""
109+ field[0] = field[1]
110+ field[-1] = field[-2]
111+ field[:,0] = field[:,1]
112+ field[:,-1] = field[:,-2]
113+ # this is doing four indexing operations at once, getting all four corners in one go
114+ field[(0,0,-1,-1),(0,-1,0,-1)] = 0.5 * (field[(0,0,-2,-2),(1,-2,0,-1)] + field[(1,1,-1,-1),(0,-1,1,-2)])
115+116+def opposed_vertical_boundary(field):
117+ """set the border values of a discretized 2d field to be equal to their inner neighbors horizontally but opposite their neighbors vertically. average at corners"""
118+ field[0] = field[1]
119+ field[-1] = field[-2]
120+ field[:,0] = -field[:,1]
121+ field[:,-1] = -field[:,-2]
122+ field[(0,0,-1,-1),(0,-1,0,-1)] = 0.5 * (field[(0,0,-2,-2),(1,-2,0,-1)] + field[(1,1,-1,-1),(0,-1,1,-2)])
123+124+def opposed_horizontal_boundary(field):
125+ """set border values equal vertically, opposite horizontally, average at corners"""
126+ field[0] = -field[1]
127+ field[-1] = -field[-2]
128+ field[:,0] = field[:,1]
129+ field[:,-1] = field[:,-2]
130+ field[(0,0,-1,-1),(0,-1,0,-1)] = 0.5 * (field[(0,0,-2,-2),(1,-2,0,-1)] + field[(1,1,-1,-1),(0,-1,1,-2)])
131+132+133+134+def diffuse(field, rate, boundary_condition, timescale):
135+ """diffuse the scalar field. basically means repeatedly applying a blur to it"""
136+ a = timescale * rate
137+ result = torch.clone(field)
138+ for n in range(diffusion_solver_steps):
139+ # scaled sum of surrounding points
140+ convolution = a * convolve(result, diffusion_kernel)
141+ # convolution operation doesn't produce output for the border rows & columns, thus the "1:n-1" indexing
142+ result[1:h-1,1:w-1] = field[1:h-1,1:w-1] + convolution
143+ result /= 1 + 4 * a
144+ boundary_condition(field)
145+ return result
146+147+# generate indices outside the function for a tiny performance improvement
148+indices_x = torch.arange(1,w-1,dtype=torch.float).repeat(h-2,1)
149+indices_y = torch.arange(1,h-1,dtype=torch.float).repeat(w-2,1).t()
150+indices = torch.stack((indices_y, indices_x))
151+152+# defining these here to reuse the same gpu memory for each advect() call
153+# TODO do the same for the rest of the tensors
154+offsets = indices.clone()
155+inverse_offsets = indices.clone()
156+indices_int = indices.int()
157+next_indices = indices_int.clone()
158+159+160+# advection
161+# this is a point where i deviate from the original paper.
162+# i take the velocity at every point, and add a scaled version of that to the index at each point
163+# to find where that velocity will carry whatever is being advected along it in one timestep.
164+# that will be a point that i decompose into the integer component & fractional component. like (3.4, 1.2) -> (3,1) + (0.4, 0.2)
165+# the point itself will be somewhere between two indices on each axis. the integer component determines the first of these two
166+# indices, and the fractional component determines which of the points it's closer to
167+168+# the paper i based this on did this weird backward version of that, where they use the negative velocity at each point to
169+# select a point & interpolate the values of the field at the surrounding points
170+171+def advect(field, velocities, timescale):
172+ """given any field, and a velocity field of the same height & width, move the values in the field a small distance in the direction of the velocity field"""
173+ offsets[...] = indices
174+ offsets.add_(timescale * velocities[:,1:h-1,1:w-1])
175+ offsets[1].clamp_(1.5, w - 2.5)
176+ offsets[0].clamp_(1.5, h - 2.5)
177+ indices_int[...] = offsets.int()
178+ offsets.sub_(indices_int)
179+ inverse_offsets[...] = 1 - offsets
180+ next_indices[...] = indices_int + 1
181+ inds_x_all = torch.stack([indices_int[1], next_indices[1], indices_int[1], next_indices[1]])
182+ inds_y_all = torch.stack([indices_int[0], indices_int[0], next_indices[0], next_indices[0]])
183+ values = torch.stack([field[:,1:h-1,1:w-1] * inverse_offsets[1] * inverse_offsets[0],
184+ field[:,1:h-1,1:w-1] * offsets[1] * inverse_offsets[0],
185+ field[:,1:h-1,1:w-1] * inverse_offsets[1] * offsets[0],
186+ field[:,1:h-1,1:w-1] * offsets[1] * offsets[0]])
187+ res = torch.zeros_like(field)
188+ res[0].index_put_((inds_y_all, inds_x_all), values[:,0,:,:], accumulate=True)
189+ if field.shape[0] == 1:
190+ continuous_boundary(res[0])
191+ else:
192+ res[1].index_put_((inds_y_all, inds_x_all), values[:,1,:,:], accumulate=True)
193+ opposed_horizontal_boundary(res[1])
194+ opposed_vertical_boundary(res[0])
195+ return res
196+197+def clear_divergence(field):
198+ opposed_horizontal_boundary(field[0])
199+ opposed_vertical_boundary(field[1])
200+ for i in range(divergence_clearing_steps):
201+ x_op = convolve(field[0], div_opp_kernel)
202+ y_op = convolve(field[1], div_opp_kernel)
203+ field[1,1:h-1,1:w-1] += (convolve(field[1], y_div_kernel) + x_op) / 8
204+ field[0,1:h-1,1:w-1] += (convolve(field[0], x_div_kernel) + y_op) / 8
205+ opposed_horizontal_boundary(field[0])
206+ opposed_vertical_boundary(field[1])
207+208+209+210+211+# initialize a small field of random velocities, then upscale it interpolating between those velocities,
212+# so that the variation in velocity is somewhat smooth instead of pure per-pixel noise,
213+# which would lead to a less interesting simulation
214+velocities = torch.randn([2, h//32, w//32]) * 120
215+upsample = torch.nn.Upsample(size=[h, w], mode='bilinear')
216+velocities = upsample(velocities.unsqueeze(0))[0]
217+218+# initialize "dye" densities to a uniform field
219+densities = torch.ones([1, h, w]) * 0.3
220+# add lines
221+#for x in range(20):
222+# densities[0,:,x*w//20] += 0.8
223+#for y in range(20):
224+# densities[0,y*h//20,:] += 0.8
225+226+frame_index = 0
227+228+last_frame = time.perf_counter()
229+230+limit = 1 / (timescale * 2 * math.sqrt(2))
231+232+# core loop of the simulation
233+for iteration in range(max_iterations):
234+ # add a tiny bit of "dye" to the fluid at all points
235+ densities.add_(0.003)
236+ #if iteration % (30 * save_every) == 0:
237+ # for x in range(20):
238+ # densities[0,:,x*w//20] += 0.3
239+ # for y in range(20):
240+ # densities[0,y*h//20,:] += 0.3
241+242+243+ #velocities[1,h//12+h//3:h//12+2*h//3,w//8] += 50 + 50 * math.sin(iteration * 0.005)
244+ #velocities[1,h//3-h//12:2*h//3-h//12,7*w//8] -= 20 + 70 * math.sin(iteration * 0.01)
245+ #velocities[0,7*h//8,4*w//6:5*w//6] -= 60 + 50 * math.sin(iteration * 0.03)
246+247+ # diffuse the velocities in both directions, enforcing a boundary
248+ # condition that maintains a constant inward velocity matching the outward velocity along the edges. that is, a wall
249+ velocities[0] = diffuse(velocities[0], velocity_diffusion_rate, opposed_horizontal_boundary, timescale)
250+ velocities[1] = diffuse(velocities[1], velocity_diffusion_rate, opposed_vertical_boundary, timescale)
251+ clear_divergence(velocities)
252+ velocities.clamp_(-limit, limit)
253+254+ # let the velocity field flow along itself
255+ velocities = advect(velocities, velocities, timescale)
256+ clear_divergence(velocities)
257+258+ # diffuse the densities & let them flow along the velocity field
259+ if density_diffusion_rate is not None:
260+ densities = diffuse(densities[0], density_diffusion_rate, continuous_boundary, density_timescale).unsqueeze(0)
261+ densities = advect(densities, velocities, density_timescale)
262+263+ # remove a little density
264+ densities.sub_(0.003)
265+ densities.clamp_(0,1)
266+267+ if iteration % save_every == 0:
268+ frame_index += 1
269+ frame_time = time.perf_counter() - last_frame
270+ image = torch.cat((0.5 + 0.5 * velocities / torch.sqrt(velocities[0]**2 + velocities[1]**2), torch.zeros((1,h,w))), dim=0)
271+ save(image, f"velocities/{frame_index:06d}")
272+ monochrome_save(densities[0], f"densities/{frame_index:06d}")
273+274+ print(f"frame {frame_index:06d}: {frame_time:06f}")
275+ print(f"[{frame_time/save_every:06f}/it for {save_every} iterations]")
276+ last_frame = time.perf_counter()
277+
+30
sketch/old/pre_snakepyt/fluid_minimal.py
···000000000000000000000000000000
···1+import torch as t;from PIL import Image as I;w,h,iters,save_every,r,dt,conv=2**9,2**9,100000,1,10,0.005,lambda f,k:t.nn.functional.conv2d(f.unsqueeze(0),k,bias=None,padding=[0],stride=[1])[0]
2+t.set_default_device("cuda");kd,kph,kpv=t.tensor([[[[0,1.0,0],[1,0,1],[0,1,0]]]]),t.tensor([[[[0,0,0],[1.0,0,-1],[0,0,0]]]]),t.tensor([[[[0,1.0,0],[0,0,0],[0,-1,0]]]])
3+def bd(f,lt=lambda t:(t[0],t[1])):f[(0,-1)],f[:,(0,-1)]=lt((f[(1,-2)],f[:,(1,-2)]));f[(0,0,-1,-1),(0,-1,0,-1)]=0.5*(f[(0,0,-2,-2),(1,-2,0,-1)]+f[(1,1,-1,-1),(0,-1,1,-2)])
4+ind,lim=t.stack((t.arange(1,h-1,dtype=t.float).repeat(w-2,1).t(),t.arange(1,w-1,dtype=t.float).repeat(h-2,1))),1/(dt*1.4142)
5+def adv(f, v):
6+ off=ind.clone().add_(dt*v[:,1:h-1,1:w-1]);off[1].clamp_(1.5,w-2.5);off[0].clamp_(1.5,h-2.5);ind_int=off.int()
7+ inv_off,next_ind=1-off.sub_(ind_int),ind_int+1;i=(t.stack([ind_int[0],ind_int[0],next_ind[0],next_ind[0]]),t.stack([ind_int[1],next_ind[1],ind_int[1],next_ind[1]]))
8+ res,values=t.zeros_like(f),t.stack([f[:,1:h-1,1:w-1]*inv_off[1]*inv_off[0],f[:,1:h-1,1:w-1]*off[1]*inv_off[0],f[:,1:h-1,1:w-1]*inv_off[1]*off[0],f[:,1:h-1,1:w-1]*off[1]*off[0]])
9+ res[0].index_put_(i,values[:,0,:,:],accumulate=True)
10+ if f.shape[0]==1:bd(res[0])
11+ else:res[1].index_put_(i,values[:,1,:,:],accumulate=True);bd(res[1],lambda t:(-t[0],t[1]));bd(res[0],lambda t:(t[0],-t[1]))
12+ return res
13+def proj(f):
14+ div,p=(conv(f[1],kph)+conv(f[0],kpv))*0.5,t.zeros_like(f[0]);bd(div)
15+ for i in range(80):p[1:h-1,1:w-1]=(div+conv(p,kd))/4;bd(p)
16+ f[1,1:h-1,1:w-1]+=0.5*conv(p,kph);f[0,1:h-1,1:w-1]+=0.5*conv(p,kpv);bd(f[1],lambda t:(-t[0],t[1]));bd(f[0],lambda t:(t[0],-t[1]))
17+v,d,i=t.nn.Upsample(size=[h,w],mode='bilinear')((t.randn([2,h//128,w//128])*50).unsqueeze(0))[0],t.ones([1,h,w])*0.1,1
18+d[0,:,tuple(x*w//20 for x in range(20))]+=0.8;d[0,tuple(x*h//20 for x in range(20)),:]+=0.8
19+for it in range(iters):
20+ v.nan_to_num_(0);
21+ d.add_(0.003);v[1,h//3:2*h//3,w//8].add_(40).clamp_(-lim,lim);v[0,1:h-1,1:w-1]=(v[0,1:h-1,1:w-1]+dt*r*conv(v[0],kd))/(1+4*dt*r)
22+ v[1,1:h-1,1:w-1]=(v[1,1:h-1,1:w-1]+dt*r*conv(v[1],kd))/(1+4*dt*r);bd(v);proj(v);v=adv(v,v);proj(v);d=adv(d,v).sub_(0.003).clamp_(0,1)
23+ print(f"{it}: {t.count_nonzero(t.isnan(v.view(-1))).item()} v nans")
24+ if it%save_every == 0:
25+ i+=1
26+ I.fromarray(d[0].detach().clone().clamp_(0,1).mul_(255).round().type(t.uint8).unsqueeze(2).expand(-1,-1,3).cpu().numpy()).save(f"out/densities/{i:06d}.png")
27+ #I.fromarray(t.cat((v.isnan(),t.zeros_like(v[0]).unsqueeze(0))).detach().clone().clamp_(0,1).mul_(255).round().type(t.uint8).permute(1,2,0).cpu().numpy()).save(f"out/velocities/nan_{i:06d}.png")
28+ I.fromarray(t.cat(((0.5 + 0.5 * v / t.sqrt(v[0]**2 + v[1]**2)), t.zeros_like(v[0]).unsqueeze(0))).detach().clone().clamp_(0,1).mul_(255).round().type(t.uint8).permute(1,2,0).cpu().numpy()).save(f"out/velocities/dir_{i:06d}.png")
29+ #I.fromarray((0.5 + 0.5 * (v[0]**2+v[1]**2) / (v[0]**2 + v[1]**2).abs().max()).detach().clone().clamp_(0,1).mul_(255).round().type(t.uint8).unsqueeze(2).expand(-1,-1,3).cpu().numpy()).save(f"out/velocities/mag_{i:06d}.png")
30+
···1+old_fic_svelte contains the old .svelte files from the original ficscript UI
2+3+the styling and scripts are probably still a useful baseline / reference for a lot of what i want to do with this repo
4+5+but i don't want to use svelte, so a little bit of work needs to be done to strip it down to html/css/js