Timestretch

Local Vector Search with llama.cpp Embeddings and sqlite_vec

Mon, 26 May 2025 11:56:13 -0500

With “embeddings”, you can search for passages of text with a similar meaning to another passage. You can create an embedding for a passage of text using a model that converts the text to a vector of numbers. Two passages that are similar will have similar embedding vectors.

Local Llama.cpp Embedding Server

I already use llama.cpp to run large language models locally and there is a nice tutorial showing how to use llama.cpp to generate embeddings using Snowflake’s embedding models.

I found a quantized version of the snowflake-arctic-embed-m-v1.5 model, and launched a server hosting the model with this command:

./build/bin/llama-server -m ~/Models/snowflake-arctic-embed-m-v1.5-q8_0.gguf --embeddings -c 512 -ngl 99

This model generates a vector of 768 floating point numbers. You can test it with this command:

curl -X POST "http://localhost:8080/embedding" --data '{"content":"Star Wars is better than Star Trek"}'                                           

[{"index":0,"embedding":[[0.01127300038933754,-0.005094126798212528,0.08973372727632523,0.03633967041969299,0.04224010556936264,...

Sqlite Vec

Now that we can generate embeddings, we need to store embeddings in a vector database and search them. The general pipeline will take a query string, embed it, then match it against every vector in the database, and finally sort the results by distance. This sounds like it would be slow, but in practice, it is fairly fast to search.

There is an impressive open source project called sqlite_vec that I leveraged to create a database of vectors, and search them. I used the Ruby sql_vec gem to store vectors for a document.

Get an embedding from Ruby

First, we need to be able to call the llama.cpp embedding API from Ruby. I implemented a method to do this.

# libs/embed.rb

require 'net/http'
require 'uri'
require 'json'

def embedding(content)
	uri = URI.parse("http://localhost:8080/embedding")
	http = Net::HTTP.new(uri.host, uri.port)
	
	request = Net::HTTP::Post.new(uri.request_uri)
	request.body = {
		content: content
	}.to_json
	
	request["Content-Type"] = "application/json"
	
	response = http.request(request)
	
	if response.code.to_i >= 200 && response.code.to_i < 300
		json = JSON.parse(response.body)
		return json[0]["embedding"].first
	end
	
	nil
end

Store the Vector in Sqlite

First, initialize an empty database

# init.rb

require 'sqlite3'
require 'sqlite_vec'

`rm embeddings.db`

db = SQLite3::Database.new('embeddings.db')
db.enable_load_extension(true)
SqliteVec.load(db)
db.enable_load_extension(false)

sqlite_version, vec_version = db.execute("select sqlite_version(), vec_version()").first

puts "sqlite_version=#{sqlite_version}, vec_version=#{vec_version}"

db.execute("CREATE TABLE document (
	document_id integer primary key,
	document_path varchar(255),
	document_sha2 varchar(64)
)")

db.execute("CREATE TABLE document_chunk (
	document_chunk_id integer primary key,
	document_id int,
	document_chunk varchar(255)
)")

db.execute("CREATE VIRTUAL TABLE vec_items USING vec0(embedding float[768])")

I made a script to import documents in a folder named corpus into the database so we can query them later. I’m ignoring really short passages, but maybe this isn’t needed? I’m also hashing each document, so I can efficiently re-index files that have changed in the future. Warning: This script deletes the database and recreates it.

# import.rb

require "./libs/embed"

require 'digest'
require 'sqlite3'
require 'sqlite_vec'

def file_sha256(file_path)
	sha = Digest::SHA2.new
	File.open(file_path) do |f|
		while chunk = f.read(256) # only load 256 bytes at a time
			sha << chunk
		end
	end
	sha.hexdigest # returns what you want
end

# watch out!
`rm embeddings.db`

db = SQLite3::Database.new('embeddings.db')
db.enable_load_extension(true)
SqliteVec.load(db)
db.enable_load_extension(false)

sqlite_version, vec_version = db.execute("select sqlite_version(), vec_version()").first

puts "sqlite_version=#{sqlite_version}, vec_version=#{vec_version}"

document_id = 1
document_chunk_id = 1

data = []

# Look in folder specified by argument, or 'corpus'

corpus_path = ARGV[0]&.strip || "corpus"

Dir[corpus_path + "**/*"].each do |file_path|
	
	db.transaction do
		
		sha256 = file_sha256(file_path)
		
		db.execute("INSERT INTO 
			document(document_id, document_path, document_sha2) 
				VALUES (?, ?, ?)", [document_id, file_path, sha256])
		
		IO.foreach(file_path) do |line|
	
			db.execute("INSERT INTO document_chunk(document_chunk_id, document_id, document_chunk) 
				VALUES (?, ?, ?)", [document_chunk_id, document_id, line])

			begin
				raw = line.strip
				if raw != '' && raw.length > 16
							
					data[document_chunk_id] = raw
			
					float_array = embedding(raw)
					db.execute("INSERT INTO vec_items(rowid, embedding) VALUES (?, ?)", [document_chunk_id, float_array.pack("f*")])
			
				end
	
				rescue => ex
					puts
					puts file_path
					puts raw
					puts ex
			end

			document_chunk_id += 1
			
		end
		
		document_id += 1
	end

end

Finally, query the database with this script. You can try either KNN or cosine vector search methods to see which works better. I didn’t see much difference in searching natural language.

# query.rb
require "./libs/embed"

require 'sqlite3'
require 'sqlite_vec'

db = SQLite3::Database.new('embeddings.db')
db.enable_load_extension(true)
SqliteVec.load(db)
db.enable_load_extension(false)

sqlite_version, vec_version = db.execute("select sqlite_version(), vec_version()").first

puts "sqlite_version=#{sqlite_version}, vec_version=#{vec_version}"

input_string = ARGV[0]

float_array = embedding(input_string)

# KNN search

# query = %(SELECT
# 		rowid,
# 		distance
# 	FROM vec_items
# 	WHERE embedding MATCH ?
# 	ORDER BY distance
# 	LIMIT 10)

query = %(SELECT
		rowid,
		vec_distance_cosine(embedding, ?) as score
	FROM vec_items
	ORDER BY score
	LIMIT 10)

rows = db.execute(query, [float_array.pack("f*")])

rowids = rows.map do |rowid, distance|
	rowid
end.join(',')

result = db.execute(
"SELECT 
	document.document_path,
	document_chunk.document_chunk_id, 
	document_chunk.document_chunk 
FROM document_chunk
LEFT JOIN document ON document.document_id = document_chunk.document_id
WHERE
	 document_chunk_id IN (#{rowids})")

chunks = result.map do |document_path, document_chunk_id, document_chunk|
	[document_chunk_id, document_path + ": " + document_chunk]
end.to_h

rows.map do |rowid, distance|
	puts
	puts "#{rowid} #{distance} #{chunks[rowid]}"
end

You can search your corpus for related passages with this script above.

ruby ./query.rb "search string here"

Mandelbrot Desktop Pictures

Mon, 24 Feb 2025 10:02:28 -0600

I made some Mandelbrot images using smooth iterations and Pickover stalks. All images are 5120 × 2880.

I processed the images using the sips command:

sips -Z 5120 *.png --out resized

sips --setProperty format jpeg --setProperty formatOptions 50 -Z 1280 *.png --out thumb/

Disco Diffusion

Sun, 29 May 2022 23:18:23 -0500

On April 9th, I saw the images from openai’s DALL-E 2 marketing, showing that high quality text to image was possible. I was blown away! I quickly scoured reddit and signed up for the DALL-E 2 beta program. Then, I waited. After waiting for a couple of weeks, I started looking for alternatives that were available now. There was something called Mid Journey, but it was also closed beta. And then, I discovered an open source project called Disco Diffusion. It is a python notebook that you can run for free in Google Colab.

I watched a couple of YouTube videos by Quick Eyed Sky, and started my journey learning what I could about all the knobs you can turn in Disco Diffusion. The best part about it is that it is open source, and configurable. I can script it, switch the models it uses, turn on and off flags. But despite all of the configurations, the most important thing is to choose a good prompt.

Prompts

A prompt is a string of text that the model translates to an image. The model is trained on captions and alt tag text associated with images scraped off the internet. The model tries to predict which of thousands of captions is best associated with the image. Surprisingly, this means you can sometimes append click-baity text like “Look at that detail!” and get great detail in your image.

Text to image explorers who started before me discovered that putting “trending on artstation” on the end of the prompt drastically increased the quality of the resulting image. I’ve found this to be true. You’ll get more detail, better compositions, and better color schemes.

Here’s the first prompt I tried in Disco Diffusion: “A beautiful painting of a green glowing ghost gliding through a dark fantasy forest with mystical creatures, Trending on artstation.” It worked! A couple of hours later, using a free K80 GPU on Google Colab, and I had this image:

I wish I could end it there, but I actually spent the next month learning to use this software and sometimes, make interesting pictures with it. As you change the words you use, you can get closer to your original vision. There is a random aspect to it, where sometimes you write a good prompt, but got an unlucky starting seed, and the image develops poorly. The AI can be frustratingly literal at times too. A prompt for “rain forest” is likely to render umbrella shaped trees for example.

But there is nothing more rewarding than seeing a beautiful image take shape, exceeding your expectations in some ways, but always a way to improve it. A successful prompt will often always generate a pleasing image, rarely producing a dud.

I subscribed to Google Colab Pro which allows me to use a slightly faster GPU in the cloud and make pictures in approximately 20 minutes to an hour. I quickly settled on a resolution to use, 1152 x 832, a good resolution for printing them on cards at 300 DPI.

Links

Here are some resources to learn more about how to choose artists, and art styles, for your prompts.

3D Printers and Filament Brands

Sun, 07 Feb 2021 08:58:23 -0600

3D Printers

Monoprice Select Mini v1

I finally took the plunge into the 3D printing world after reading positive reviews of the Monoprice Select Mini v1, and bought one in early 2017. It was great! Mechanically, it is well constructed. There was no assembly required. You can take it out of the packaging, add blue painter’s tape to the build plate, thread in filament, and start printing. As I usually print fairly small items, the small build volume didn’t bother me. The print quality was excellent. Parts printed very precisely, with good tolerances, and very square. The only printing defect I noticed as a small amount of Z-banding. I passed this printer along to my Dad as I didn’t use it for awhile due to starting a new job.

Monoprice Select Mini Pro

The next printer I tried, was the Monoprice Select Mini Pro. Although this printer was very similar to the v1, in a few small ways, it was inferior to the first one. The tiny touchscreen was a bit harder for me to operate precisely compared to the click wheel, and the print quality, although similar to the v1, displayed a consistent z-banding through the full height of the printer. The removable build plate was nice, but it had no ability to flex to remove the parts. Due to the enclosed Z-axis design of the printer, I wasn’t sure how to fix the banding artifacts. Since it was nearly brand new, I decided to send it back. I bought an Ender 3 Pro instead.

Ender 3 Pro

The Ender 3 is a kit which requires a minimal amount of assembly. The printer went together quickly. I noticed some concerning quality issues during assembly. A plastic tab was required to be cut off to achieve proper placement of the Z axis limit switch.

After assembly, it was not all smooth sailing. Although I was able to print items fairly quickly after assembly, I noticed the top surface of printed items was rough, due to the hot nozzle dragging into previously printed layers. The nozzle dragging also caused issues printing thin towers, where the nozzle would bump into the model and sometimes tear it away from the build plate. I was able to compensate for this issue by enabling Z-hop when printing very tiny parts, but it was frustrating. I embarked on a mission to improve the print quality.

I upgraded the bowden tube and bought better couplers to attach the tube to the frame, as the couplers included didn’t grip well and soon began to slip during filament retractions. I continued to struggle with the print-quality. I made adjustments to eccentric nuts on rollers, adjusted the spacing between the frame and the Z-axis lead screw, swapped nozzles, and even completely disassembled the printer to attempt to loosen and retighten the screws holding together the frame to make things more square.

After all of the small tweaks I’ve made, the printer can print objects that look great superficially. It exhibits very few Z-banding artifacts compared the Monoprice printers, which was the main reason I decided to go with this printer. It prints statues and toys impressively well, and fairly reliably.

But I have never been able to get it to print completely square. I’ve printed boxes for origami paper that ended up like parallelograms. I had to increase the gap between the walls of the box quite a bit to get the paper to sit inside. I’ve also had trouble with the Z-axis leaning slightly to the right as well. I’m sometimes able to compensate by orienting the narrow part of a taller box along the Y axis.

In summary, I’ve really enjoyed the Ender 3’s larger print volume, and aside from issues with squaring the printer, I’ve enjoyed the open frame design, as it does allow for adjustments. I recently purchased a machinist’s square and will see if I can use that to square the frame assembly. It is a good printer for people willing to invest considerable time in learning to solve its deficiencies.

OctoPrint: Recommended

I attached my Raspberry Pi 3B+ with OctoPrint installed to my Ender 3, and it has made a world of difference. It is so much easier to print directly from Cura, using the OctoPrint plug-in, to the printer than using a MicroSD card. I now print more often, and am able to more quickly try changes and adjustments to improve print quality.

Good Filament

Hatchbox

Hatchbox Silver PLA — This is the first filament I purchased. It prints great, sticks well and was consistent. The silver color reveals defects well, but this is high quality PLA filament.
Hatchbox Glow in the Dark Blue PLA — Prints great, sticks great, with a nice glossy slightly translucent finish. It prints very precisely.
Hatchbox Orange PLA — Prints great, no issues.
Hatchbox Dark Blue PLA — Prints great, no issues.

CCTree PLA+

CCTree PLA+ White — A sample of this filament came with my Ender 3. I liked it so I picked up another Kilo of it. It prints great! It is a slightly translucent white. You'd only notice it is translucent when comparing it to other more opaque whites though. It sticks well to itself, but still allows for tiny margins in print-in place hinges. It prints with a nice smooth texture as well, and hides small layer lines and defects. It is tough filament, bending slightly before breaking, but it less strong than standard PLA.
CCTree PLA+ Gray — This filament is similar to the white, but reveals tiny defects much more. It prints and sticks well to the build plate, but I have seen tiny divot defects throughout and I'm not sure how to fix them, or what causes it. Maybe they are also in the white but just less noticeable?

AIO Robotics

AIO robotics makes nice 1/2 KG sizes, and I’ve had fairly good luck with them. The are easy to print with, but each color has had slightly different attributes.

AIO Robotics Red PLA — Prints well. Maybe a slightly crumbly brittle texture. No real issues though.
AIO Robotics Pink PLA — Tended to under-extrude a tiny bit. I had to increase the flow rate to get infill to make good contact with the walls. Good for making print in place hinges and parts because it doesn't stick as well to itself. This one has more of a slick smooth texture.
AIO Robotics Thermochromic Green / Yellow PLA — I've had good results with this one. The color change works well, and it has been fun to use it to print a Raspberry Pi case that turns yellow as the computer heats up. It prints well and sticks to the build plate well, but sometimes has a bit of a brittle crumbly texture. It can be used for print in place hinges, but needs a bit more care in freeing them than other filaments.

Amolen

Amolen Silk-Like Pearl PLA — Good printing quality, but tends to fuse together. It is not usable for print in place hinges, but has a great shiny appearance.

Middle of the Road

Overture

Overture White PLA — This is an OK filament, but not great. It sticks to the bed, but it suffers from some problems that I haven't seen with other filaments. Although it sticks, it also suffers from more severe warping / contraction issues than other filaments. As the printed part gets larger, it will pull away from the build plate. You end up with very rounded bottom surfaces unless you use a large brim. It also has a strange toothpasty, gummy consistency that sometimes sticks to the nozzle, leading to occasional dark burnt plastic artifacts where it transfers off the nozzle onto the print. I've used this filament more than I expected due to its nice solid opaque white color, but I probably wouldn't buy it again.

Brands I’ve had trouble with

Sunlu PLA+

Sunlu PLA+ Black — If you can get it to stick to the build plate, it prints great. Smooth, even, shiny black. But getting it to stick is a struggle. I've only had good luck when I switched to printing with blue painter's tape instead of using the Ender 3 flexible magnetic bed. Worse, after you attempt to print with this filament, it puts an oily residue on the build surface which even prevent other filaments from sticking to it. It suffers from heat creep, and melts higher up in the extruder, and I ended up with clogs. I don't recommend it.

No Name

Gold PLA — I found this on Amazon and took a chance. Unfortunately it is brittle. This may have been older filament when I got it, but it has been by far the most brittle filament I've bought. I've had it break while printing and cause jams. I've had more parts printed from it break than any other filament I've ever used. Stick to name brands.

Try Out Ruby 3.0 Type Checking with Steep

Fri, 25 Dec 2020 21:20:51 -0600

Merry Christmas! Today Ruby 3.0 was released today with type checking. Here’s how to try it out yourself. First, read about Ruby 3.0 type checking here in The State of Ruby 3 Typing.

Install Ruby 3.0 with rbenv

To follow along, you’ll need Ruby 3.0. I use rbenv. To see the new version 3.0, I updated ruby-version using home brew.

brew upgrade ruby-build

Next, I was able to install the new version using rbenv.

rbenv install 3.0.0

After that, I created a new directory to experiment in.

mkdir PizzaFactory
cd PizzaFactory

I created a .ruby-version file to make sure rbenv was running Ruby 3.0.

echo "3.0.0" > .ruby-version

Double-check you are running Ruby 3.0

ruby -v

You should see the following print out:

ruby 3.0.0p0 (2020-12-25 revision 95aff21468) [x86_64-darwin20]

Define a Pizza class

I created a pizza.rb ruby file.

class Pizza

	attr_accessor :toppings
	
	def initialize(toppings)
		@toppings = toppings
	end
	
	def to_s
		toppings.join(", ")
	end
	
end

Annotate the Pizza class with types using an RBS file

Next, I created a sig/pizza.rbs file to describe the types I’ll be using.

class Pizza

	attr_accessor toppings: Array[String]
	
	def initialize: (Array[String]) -> void
	
	def to_s: () -> String
	
end

Check the types with Steep

Install steep to do type checking. I found the steep readme on github helpful:

gem install steep

I ran steep init and it created a Steepfile file, which I modified as shown.

target :pizza do
	signature "sig"

	check "*.rb"
end

I was curious if steep could help me find errors. I created an app.rb file that deliberately included a type error.

require_relative "./pizza"

# Correct type should be: Array[String]
# pizza = Pizza.new(["Pepperoni", "Green Peppers"])

# Incorrect type is passed instead: String 
pizza = Pizza.new("Coffee")

puts pizza.to_s

Check for type errors:

steep check

Success! Well, success in that it found the error I deliberately put in

app.rb:4:18: ArgumentTypeMismatch: receiver=singleton(::Pizza), 
expected=::Array[::String], 
actual=::String ("Coffee")

Closing thoughts

I’ve tried Sorbet briefly too, and I’m not sure yet whether I prefer inline type annotations or external rbs files. Also, hiding the type annotations in a ‘sig’ folder seems a bit odd. I’m used to .h files being in the same directory as their counterpart, and rbs files seem similar to a header file to me. I do love the rbs syntax though.

Kudos to all the developers working on improving Ruby and making the ecosystem richer with type checking. Thank you!

Setting up microk8s

Sat, 16 May 2020 22:12:18 -0500

This documents how to set up microk8s with nginx-ingress. I wanted this set-up so I can self-host multiple websites on a single-node Kubernetes cluster. I will use NAT from my router to forward traffic to port 443 and port 80.

These are mostly notes for me to remember how I set everything up.

First, install microk8s on Ubuntu 20.04 LTS.

sudo snap install microk8s --classic

Enable kubectl to access it from another machine. Solution from:

https://github.com/ubuntu/microk8s/issues/421

Open insecure port 8080 to everyone. To do this you need to edit /var/snap/microk8s/current/args/kube-apiserver and set the --insecure-bind-address=127.0.0.1 to 0.0.0.0. Then restart MicroK8s with microk8s.stop and microk8s.start.

--insecure-port=8080
--insecure-bind-address=0.0.0.0

Restart microk8s

microk8s.stop
microk8s.start

Enable addons

microk8s enable dns
microk8s enable storage

On another machine, set up connection in ~/.kube/config

apiVersion: v1
clusters:
- cluster:
    server: http://kubernetes:8080
  name: microk8s-cluster
contexts:
- context:
    cluster: microk8s-cluster
    user: admin
  name: microk8s
current-context: microk8s
kind: Config
preferences: {}
users:
- name: admin
  user:
    username: admin

Run this command:

kubectl get all --all-namespaces

To display the current enabled addons:

NAMESPACE     NAME                                        READY   STATUS    RESTARTS   AGE
kube-system   pod/coredns-588fd544bf-lsj6j                1/1     Running   0          2m32s
kube-system   pod/hostpath-provisioner-75fdc8fccd-72hr7   1/1     Running   0          119s


NAMESPACE     NAME                 TYPE        CLUSTER-IP      EXTERNAL-IP   PORT(S)                  AGE
default       service/kubernetes   ClusterIP   10.152.183.1    <none>        443/TCP                  9m20s
kube-system   service/kube-dns     ClusterIP   10.152.183.10   <none>        53/UDP,53/TCP,9153/TCP   2m32s


NAMESPACE     NAME                                   READY   UP-TO-DATE   AVAILABLE   AGE
kube-system   deployment.apps/coredns                1/1     1            1           2m32s
kube-system   deployment.apps/hostpath-provisioner   1/1     1            1           2m

NAMESPACE     NAME                                              DESIRED   CURRENT   READY   AGE
kube-system   replicaset.apps/coredns-588fd544bf                1         1         1       2m32s
kube-system   replicaset.apps/hostpath-provisioner-75fdc8fccd   1         1         1       2m

I couldn’t get the included nginx-ingress to work. What worked for me after being stuck for hours, was to use the Bitnami helm chart. (Thanks Todd!)

Check your helm version

helm version --short
v3.2.1+gfe51cd1

Add the Bitnami repo

helm repo add bitnami https://charts.bitnami.com/bitnami

Specify a namespace

kubectl create namespace nginx-ingress

I will use some assigned high port numbers and map port 80 and 443 to 31180 and 31443.

helm install nginx-ingress bitnami/nginx-ingress-controller \
	--namespace nginx-ingress \
	--set service.nodePorts.http=31180 \
	--set service.nodePorts.https=31443

This will create a LoadBalancer listening on those ports.

nginx-ingress   service/nginx-ingress-nginx-ingress-controller                   LoadBalancer   10.152.183.213   <pending>     80:31180/TCP,443:31443/TCP   11m

Load a bunch of yaml in:

kubectl apply -f test-site.yml

test-site.yml

apiVersion: apps/v1
kind: Deployment
metadata:
  name: test-site
spec:
  selector:
    matchLabels:
      run: test-site
  replicas: 1
  template:
    metadata:
      labels:
        run: test-site
    spec:
      containers:
      - name: test-site
        image: nginx
        ports:
        - containerPort: 80
      imagePullSecrets:
      - name: gcr-json-key

---

apiVersion: v1
kind: Service
metadata:
  name: test-site
  labels:
    run: test-site
spec:
  ports:
  - port: 80
    protocol: TCP
    targetPort: 80
  selector:
    run: test-site

---

apiVersion: extensions/v1beta1
kind: Ingress
metadata:
  name: test-site-ingress
  annotations:
    kubernetes.io/ingress.class: nginx
spec:
  rules:
  - host: test.timestretch.com
    http:
      paths:
      - backend:
          serviceName: test-site
          servicePort: 80

---

Spirograph Guilloche Generator

Sun, 10 May 2020 09:49:53 -0500

I’ve been working on a back design for playing cards, and decided to make a tool to draw ornate spiral figures. To make things ~~frustrating~~ challenging, I’m using Reason / React to generate SVG.

I’m still learning how to organize React components in Reason React. I haven’t been able to figure out how to split up the sliders into a subcomponent yet, so the code quality is a bit disappointing.

I noticed some strange things along the way. The <input type="range"> slider has some interesting types, with min being typed as an int, and max being typed as a string.

I struggled with building an SVG path which wants an array of points as a long string. A method to join an array of strings isn’t present in the standard (non-Javascript) library. Initially, I converted the array(point) to a list and ran List.fold_left to flatten it down to a string. But in the end, I used Js.Array.joinWith(" ") which is probably faster.

SpirographGenerator.res

type action =
| SetAngleA(string)
| SetAngleB(string)
| SetAngleC(string)
| SetAngleD(string)
| SetScaleA(string)
| SetScaleB(string)
| SetScaleC(string)
| SetScaleD(string)
| SetOffset(string)
| SetRepeatCount(string)
| SetRepeatOffset(string)
| SetThickness(string)
| SetScale(string)
;


type state = {
	angleA: int,
	angleB: int,
	angleC: int,
	angleD: int,
	scaleA: int,
	scaleB: int,
	scaleC: int,
	scaleD: int,
	offset: int,
	repeatCount: int,
	repeatOffset: int,
	thickness: int,
	scale: int,
};

type point = {
	x: float,
	y: float
};

module SvgPath = {
	@react.component
	let make = (~state) => {
		let moveto = (p) => Js.Float.toString(p.x) ++ "," ++ Js.Float.toString(p.y);

		let points = (step) => {

			let count = 360 * 8;
		
			let offset = float_of_int(state.offset) /. 500.0 *. float_of_int(step);
			let repeatOffset = float_of_int(state.repeatOffset) /. 50.0 *. float_of_int(step);
			let angleA = float_of_int(state.angleA);
			let angleB = float_of_int(state.angleB);
			let angleC = float_of_int(state.angleC);
			let angleD = float_of_int(state.angleD);
			let scaleA = float_of_int(state.scaleA);
			let scaleB = float_of_int(state.scaleB);
			let scaleC = float_of_int(state.scaleC);
			let scaleD = float_of_int(state.scaleD);
			let scale = float_of_int(state.scale) /. 100.0;

			let points = [{x: 0.0, y: 0.0}]

			let radian_conv = 0.01745329251;

			for (i in 0 to count) {

				let degrees = float_of_int(i) /. float_of_int(count) *. 360.0;


				let x = sin(degrees *. radian_conv *. angleA) *. scaleA;
				let y = cos(degrees *. radian_conv *. angleA) *. scaleA;

				let xx = x +. sin(degrees *. radian_conv *. angleB +. offset) *. scaleB;
				let yy = y +. cos(degrees *. radian_conv *. angleB +. offset) *. scaleB;

				let xxx = xx +. sin(degrees *. radian_conv *. angleC) *. (scaleC);
				let yyy = yy +. cos(degrees *. radian_conv *. angleC) *. (scaleC);

				let xxxx = xxx +. sin(degrees *. radian_conv *. angleD) *. (scaleD +. repeatOffset);
				let yyyy = yyy +. cos(degrees *. radian_conv *. angleD) *. (scaleD +. repeatOffset);

				points[i] = {
					x: xxxx *. scale,
					y: yyyy *. scale
				}
			}

			let joined = Array.map(points, (p) => moveto(p)) 
				|> Js.Array.joinWith(" ");

			"M" ++ joined ++ "z"
			
		};

		let count = state.repeatCount;
		let repeatCounter = [];

		for (i in 0 to count) {
			repeatCounter[i] = i;
		};
		
		(React.array(
			Array.map(repeatCounter, step => {
					let thickness = state.thickness;
					
					let lineThickness = (thickness > 0) ? Js.Float.toString(float_of_int(step) *. float_of_int(state.thickness) /. float_of_int(count)) : "1.0";

					<path 
						transform={"translate(" ++ string_of_int(640/2) ++ " " ++ string_of_int(640/2) ++ ")"}  
						d=points(step)
						stroke="black" 
						strokeWidth=lineThickness
						fill="none" 
					/>
				}
			) 
		))

		

	}
}

module SvgRender = {


	@react.component
	let make = (~state) => {

		<svg width="640" height="640">
			<rect 
				width="640" 
				height="640" 
				style=ReactDOM.Style.make(
					~fill="#fff", 
					~border="1px solid #ccc", 
					~padding="20px", ()) 
			/>
			
			<SvgPath state />

		</svg>
	}
}


@react.component
let make = () => {

	let (state, dispatch) =
		React.useReducer(
			(state, action) =>
				switch (action) {
					| SetAngleA(n) => {...state, angleA: int_of_string(n)}
					| SetAngleB(n) => {...state, angleB: int_of_string(n)}
					| SetAngleC(n) => {...state, angleC: int_of_string(n)}
					| SetAngleD(n) => {...state, angleD: int_of_string(n)}
					| SetScaleA(n) => {...state, scaleA: int_of_string(n)}
					| SetScaleB(n) => {...state, scaleB: int_of_string(n)}
					| SetScaleC(n) => {...state, scaleC: int_of_string(n)}
					| SetScaleD(n) => {...state, scaleD: int_of_string(n)}
					| SetOffset(n) => {...state, offset: int_of_string(n)}
					| SetRepeatOffset(n) => {...state, repeatOffset: int_of_string(n)}
					| SetRepeatCount(n) => {...state, repeatCount: int_of_string(n)}
					| SetThickness(n) => {...state, thickness: int_of_string(n)}
					| SetScale(n) => {...state, scale: int_of_string(n)}
				},
			{
				angleA: -8, 
				angleB: 17, 
				angleC: 7, 
				angleD: 0, 
				scaleA: 82, 
				scaleB: 41, 
				scaleC: 144, 
				scaleD: 0, 
				offset: 0,
				repeatOffset: 45,
				repeatCount: 5,
				thickness: 1,
				scale: 100,
			},
		);

	<div> 
		<form style=ReactDOM.Style.make(~position="absolute", ~left="20px", ~width="160px", ~margin="10px", ())>

			<div>
				<label htmlFor="angleA">{React.string("Angle A")} 
					{React.string(": " ++ string_of_int(state.angleA))}</label><br />
				<input 
					id="angleA"
					type_="range"
					value=string_of_int(state.angleA)
					name="angleA"
					min="-72"
					max="72"
					onChange=(
						event => dispatch(SetAngleA(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="angleB">{React.string("Angle B")} 
					{React.string(": " ++ string_of_int(state.angleB))}</label><br />
				<input 
					id="angleB"
					type_="range"
					value=string_of_int(state.angleB)
					name="angleB"
					min="-72"
					max="72"
					onChange=(
						event => dispatch(SetAngleB(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="angleC">{React.string("Angle C")} 
					{React.string(": " ++ string_of_int(state.angleC))}</label><br />
				<input 
					id="angleC"
					type_="range"
					value=string_of_int(state.angleC)
					name="angleC"
					min="-72"
					max="72"
					onChange=(
						event => dispatch(SetAngleC(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>
			
			<div>
				<label htmlFor="angleD">{React.string("Angle D")} 
					{React.string(": " ++ string_of_int(state.angleD))}</label><br />
				<input 
					id="angleD"
					type_="range"
					value=string_of_int(state.angleD)
					name="angleD"
					min="-72"
					max="72"
					onChange=(
						event => dispatch(SetAngleD(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="scaleA">{React.string("Scale A")} 
					{React.string(": " ++ string_of_int(state.scaleA))}</label><br />
				<input 
					id="scaleA"
					type_="range"
					value=string_of_int(state.scaleA)
					name="scaleA"
					min="-360"
					max="360" 
					onChange=(
						event => dispatch(SetScaleA(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="scaleB">{React.string("Scale B")} 
					{React.string(": " ++ string_of_int(state.scaleB))}</label><br />
				<input 
					id="scaleB"
					type_="range"
					value=string_of_int(state.scaleB)
					name="scaleB"
					min="-360"
					max="360" 
					onChange=(
						event => dispatch(SetScaleB(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="scaleC">{React.string("Scale C")} 
					{React.string(": " ++ string_of_int(state.scaleC))}</label><br />
				<input 
					id="scaleC"
					type_="range"
					value=string_of_int(state.scaleC)
					name="scaleC"
					min="-360"
					max="360" 
					onChange=(
						event => dispatch(SetScaleC(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="scaleD">{React.string("Scale D")} 
					{React.string(": " ++ string_of_int(state.scaleD))}</label><br />
				<input 
					id="scaleD"
					type_="range"
					value=string_of_int(state.scaleD)
					name="scaleD"
					min="-360"
					max="360" 
					onChange=(
						event => dispatch(SetScaleD(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="offset">{React.string("Offset")} {React.string(": " ++ string_of_int(state.offset))}</label><br />
				<input 
					id="offset"
					type_="range"
					value=string_of_int(state.offset)
					name="offset"
					min="-360"
					max="360" 
					onChange=(
						event => dispatch(SetOffset(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="offset">{React.string("Repeat Offset")} {React.string(": " ++ string_of_int(state.repeatOffset))}</label><br />
				<input 
					id="repeatOffset"
					type_="range"
					value=string_of_int(state.repeatOffset)
					name="repeatOffset"
					min="-720"
					max="720" 
					onChange=(
						event => dispatch(SetRepeatOffset(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>

			<div>
				<label htmlFor="offset">{React.string("Repeat Count")} {React.string(": " ++ string_of_int(state.repeatCount))}</label><br />
				<input 
					id="repeatCount"
					type_="range"
					value=string_of_int(state.repeatCount)
					name="repeatCount"
					min="0"
					max="50" 
					onChange=(
						event => dispatch(SetRepeatCount(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>
			
			<div>
				<label htmlFor="thickness">{React.string("Line Thickness")} {React.string(": " ++ string_of_int(state.thickness))}</label><br />
				<input 
					id="thickness"
					type_="range"
					value=string_of_int(state.thickness)
					name="thickness"
					min="0"
					max="10" 
					onChange=(
						event => dispatch(SetThickness(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>
			<div>
				<label htmlFor="scale">{React.string("Scale")} {React.string(": " ++ string_of_int(state.scale))}</label><br />
				<input 
					id="scale"
					type_="range"
					value=string_of_int(state.scale)
					name="scale"
					min="0"
					max="400" 
					onChange=(
						event => dispatch(SetScale(ReactEvent.Form.target(event)["value"]))
					)
					style=ReactDOM.Style.make(~width="100%", ()) 					
				/>
			</div>
			
		</form>

		
		<SvgRender state />

	</div>;
};

Ubuntu Upgrade Day

Sun, 26 Apr 2020 18:56:21 -0500

I’ve been putting off some system administration tasks while waiting for Ubuntu 20.04 LTS to come out. The new Ubuntu came out last week, so I installed it in a few VMware virtual machines over this weekend. I’m still putting out fires.

Ubuntu 20.04 Gitlab Runner

The first thing I installed 20.04 on was a Gitlab runner. In the Ubuntu installer I noticed checkboxes to install docker, and google-cloud-sdk snap packages. I checked them and began the installation.

Unfortunately, much later and after much troubleshooting, I found the google-cloud-sdk package won’t allow any extra components to be installed with it. Since I use both the beta and the docker-credential-gcr components, I threw in the towel on the snap package and tried installing it via an apt package. That failed because apt packages don’t yet exist for this new distribution. I ended up manually installing the SDK and troubleshooting path problems.

The other mistake I made was enabling LVM. I noticed on my next install, that there is a screen that is shown right afterwards that lets you configure the root volume allocation, but by default it sets it to a paltry 4GB, even specifying you want a larger volume on the previous screen. After much Googling, I found a solution to extending it while running, but it would be nice if it wasn’t necessary.

Upgrading Web Server running Ubuntu 16.04 to Ubuntu 18.04

I made the mistake of updating my 16.04 web server. I host a couple of Ruby / Sinatra websites and was using the Passenger nginx plug-in and acmetool for lets encrypt. I felt that 20.04 was a bit too fresh so I decided to play it safe and install 18.04. It’s been out two years now. Surely it is stable. I ran into a few problems.

First, I’ve found acmetool is pretty much abandoned at this point. There is a acmetool beta implementation of the v2 acme protocol but it hasn’t been released as a package yet. I decided to switch to certbot, and so far it has turned out to be a complete drop in replacement. I even used the automatic nginx config tool, and it worked perfectly.

I tried to continue using passenger, but passenger requires an older version of nginx than comes with 18.04 now. I tried following the installation instructions, and copy and pasting solutions from random Github issues, but threw in the towel after several hours.

I switched the websites over to using puma web server, and that seemed to work fine when I started the server from the command line. I then followed the instructions on setting up puma to use upstart to restart it automatically, only to realize about half way into it that 18.04 no longer uses upstart. I changed gears and tried setting up systemd to start up my puma servers. I had no luck, and ended up on some Github issue pages, but unfortunately none of the work arounds were successful. I ended up using a @reboot line in /etc/crontab to start the servers, and I’m limping along for now.

Microk8s

I also installed Microk8s on a Ubuntu 20.04, with plans to use it to replace the services running on the webserver above. I’m at the stage where I need to decide where my Docker container registry will live. More on this will come later.

Jekyll Copy Files Plugin Hook

Sat, 04 Apr 2020 15:29:24 -0500

I ran into some issues trying to embed source code that was outside of the jekyll root. Copying the files from their original location to src/_includes at build time was the only solution left after Googling for better solutions.

After trying to use webpack to do this, I decided to fall back on Ruby, and wrote a small Jekyll plugin to do it. It creates a hook on :after_init that copies files from the webpack source folder so I can include live versions of the source code used to build this site.

_plugins/copy_files.rb

require 'fileutils'

def copy_files(source, destination)
	puts %(copy_files.rb: Copying files from "#{source}" to "#{destination}")
	FileUtils.cp_r source, destination
end

Jekyll::Hooks.register :site, :after_init do |jekyll|
	copy_files 'webpack', 'src/_includes/code'
end

Mandelbrot Set

Sun, 12 May 2019 17:44:57 -0500

This example is rendered with eight WebWorkers, and uses ArrayBuffers with transferred ownership to pass data from the worker back to the main script.

mandelbrot_set.ts

interface WorkerJob {
	width: number;
	height: number;
	y: number;
}

interface WorkerResponse {
	y: number;
	row: ArrayBuffer;
}

const TASK_COUNT = 8

function createCanvas() {
	const canvas = document.createElement('canvas');

	canvas.id = "mandelbrot_canvas";
	
	canvas.setAttribute("style", "width:100%;")
	
	canvas.style.border = "1px solid #ccc";

	const root = document.getElementById("mandelbrot");
	root.appendChild(canvas);
	canvas.style.height = Math.floor(canvas.offsetWidth) + 'px';

	canvas.width = canvas.offsetWidth;
	canvas.height = canvas.offsetWidth;

	return canvas;	
}

function showMessage(msg: string) {
	let hello = document.createElement('div');
	let textNode = document.createTextNode(msg)
	hello.appendChild(textNode)

	let entry = document.getElementById("mandelbrot")
	entry.appendChild(hello)
}

function spawnWorkers(mandelbrot_canvas: HTMLCanvasElement) {

	let startTime: number
	let endTime: number

	function start() {
		startTime = new Date().getTime();
	};

	function end(width: number, height: number) {
		endTime = new Date().getTime();
		let timeDiff = endTime - startTime;

		// get milliseconds 
		let milliseconds = Math.round(timeDiff);
		showMessage(width + " x " + height + " " + milliseconds + " ms")
	}

	start()
	
	const ctx = mandelbrot_canvas.getContext("2d");

	const width = mandelbrot_canvas.width * 4
	const height = mandelbrot_canvas.height * 4
	let remaining = height
	mandelbrot_canvas.width = width
	mandelbrot_canvas.height = height
	mandelbrot_canvas.style.width  = (width / 4) + 'px';
	mandelbrot_canvas.style.height = (height / 4) + 'px';
	
	let tasks: Array<Worker> = []
	
	for (let count = 0; count < TASK_COUNT; count++) {
		const worker = new Worker("/assets/js/mandelbrot_worker_renderer.bundle.js");
		worker.onmessage = function(msg: MessageEvent) {

			const image = ctx.getImageData(0, 0, 
				mandelbrot_canvas.width * 2, 1);
	
			const data = image.data
	
			const messageData: WorkerResponse = msg.data
			const y = messageData["y"]
			const rowBuffer = messageData["row"]
			const row = new Uint32Array(rowBuffer)
			
	
			for (let x = 0; x < width; x ++) {
				const offset = (x * 4);
				const xx = (x / width) * 4.0 - 2.0 - 0.5;

				let gray = 0;
						
				if (row[x] % 2 == 0) {
					gray = 255;
				}

				data[offset + 0] = gray;    //red
				data[offset + 1] = gray;    //green
				data[offset + 2] = gray;    //blue
				data[offset + 3] = 255;     //alpha
			}
		
			ctx.putImageData(image, 0, y);
			remaining -= 1
			if (remaining == 0) {
				end(width, height)
			}
		}
		tasks.push(worker)
	}
	
	let task_counter = 0
	
	for (let y = 0; y < height; y++) {
		
		const msg: WorkerJob = {
			width: width, 
			height: height, 
			y: y
		}
		let worker = tasks[task_counter]
		worker.postMessage(msg)	
		task_counter += 1
		if (task_counter >= tasks.length) {
			task_counter = 0
		}
	}
	
}

const mandelbrot_canvas = createCanvas()

spawnWorkers(mandelbrot_canvas)

mandelbrot_worker_renderer.ts

declare function postMessage(message: any, transfer?: Transferable[]):void

let Mandelbrot = {
	
	ITERATIONS: 1024,
	BAILOUT: 16,

	iterate: function (xx: number, yy: number) {
	
		const iterations = this.ITERATIONS
		const bailout = this.BAILOUT
		
		const cr = xx
		const ci = yy
		let zi = 0.0
		let zr = 0.0
	
		let i = 0
	
		while (i++ < iterations) {
			const temp = zr * zi
			const zr2 = zr * zr
			const zi2 = zi * zi
			zr = zr2 - zi2 + cr
			zi = 2.0 * temp + ci
			if (zi2 + zr2 > bailout) {
				return i
			}
		}
		return i
	},
}


function render(messageData: WorkerJob) {

	const width = messageData["width"]
	const height = messageData["height"]
	const y = messageData["y"]

	const arrayBuffer = new ArrayBuffer(width * 32)
	const rowIterations = new Uint32Array(arrayBuffer)
	
	const yy = ((y / height) * 4.0 - 2.0)

	for (let x = 0; x < width; x++) {
	
		const xx = (x / width) * 4.0 - 2.0 - 0.5
		rowIterations[x] = Mandelbrot.iterate(xx, yy)
	}
	
	const result: WorkerResponse = {
		y: y,
		row: arrayBuffer
	}

	postMessage(result, [result.row])
}

self.onmessage = function (msg: MessageEvent) {
	const messageData:WorkerJob = msg.data
	render(messageData)
}