From c92edfbae2ec9ec06473590b268cbc8fec2961ff Mon Sep 17 00:00:00 2001 From: nat Date: Thu, 12 Jun 2025 21:39:43 -0700 Subject: [PATCH] blog: add python bash builtins post --- .../data/thoughts/python-bash-builtins/bar.c | 4 + .../data/thoughts/python-bash-builtins/bash.h | 62 +++++++ .../python-bash-builtins/benchmark.html | 4 + .../thoughts/python-bash-builtins/build.sh | 39 ++++ .../expensive_calculation.py | 16 ++ .../data/thoughts/python-bash-builtins/foo.py | 3 + .../thoughts/python-bash-builtins/math.py | 16 ++ .../thoughts/python-bash-builtins/syntax-hl | 7 + .../html/thoughts/python-bash-builtins.hy | 170 ++++++++++++++++++ 9 files changed, 321 insertions(+) create mode 100644 www/src/data/thoughts/python-bash-builtins/bar.c create mode 100644 www/src/data/thoughts/python-bash-builtins/bash.h create mode 100644 www/src/data/thoughts/python-bash-builtins/benchmark.html create mode 100755 www/src/data/thoughts/python-bash-builtins/build.sh create mode 100644 www/src/data/thoughts/python-bash-builtins/expensive_calculation.py create mode 100644 www/src/data/thoughts/python-bash-builtins/foo.py create mode 100644 www/src/data/thoughts/python-bash-builtins/math.py create mode 100755 www/src/data/thoughts/python-bash-builtins/syntax-hl create mode 100644 www/src/pages/html/thoughts/python-bash-builtins.hy diff --git a/www/src/data/thoughts/python-bash-builtins/bar.c b/www/src/data/thoughts/python-bash-builtins/bar.c new file mode 100644 index 0000000..3eb864b --- /dev/null +++ b/www/src/data/thoughts/python-bash-builtins/bar.c @@ -0,0 +1,4 @@ +#include "bash.h" +#include "foo.c" + +PY_FUNC(foo, py_test, _foo); diff --git a/www/src/data/thoughts/python-bash-builtins/bash.h b/www/src/data/thoughts/python-bash-builtins/bash.h new file mode 100644 index 0000000..db9a719 --- /dev/null +++ b/www/src/data/thoughts/python-bash-builtins/bash.h @@ -0,0 +1,62 @@ +#include + +#include +#include +#include +#include + +extern char **make_builtin_argv(WORD_LIST*, int*); + +#define DEFINE_BUILTIN(name) \ + struct builtin name##_struct = { \ + #name, \ + name##_builtin_func, \ + BUILTIN_ENABLED, \ + NULL, \ + NULL, \ + 0 \ + } + +#define WRAP_FUNC_WITH_BUILTIN(name)\ + int name##_builtin_func(WORD_LIST *list) { \ + int argc, ret; \ + char **argv; \ + argv = make_builtin_argv(list, &argc); \ + ret = name(argc, argv); \ + free(argv); \ + return ret;\ + } + +#define PY_FUNC(bash_name, modname, function) \ + int bash_name(int argc, char **argv) { \ + if (!Py_IsInitialized()) { \ + dlopen("libpython3.13.so", RTLD_NOW | RTLD_GLOBAL);\ + PyImport_AppendInittab(#modname, PyInit_##modname); \ + Py_Initialize(); \ + } \ + int ret = 1; \ + PyObject *mod = NULL, *func = NULL, *result; \ + \ + mod = PyImport_ImportModule(#modname); \ + func = PyObject_GetAttrString(mod, #function); \ + \ + PyObject *py_argv = PyTuple_New(argc - 1); \ + \ + for (int i = 1; i < argc; i++) { \ + PyObject *arg_as_str = PyUnicode_FromString(argv[i]); \ + PyTuple_SetItem(py_argv, i-1, arg_as_str); \ + } \ + \ + result = PyObject_CallObject(func, py_argv); \ + if (!result) { PyErr_Print(); goto finally; } \ + ret = (int) PyFloat_AsDouble(result); \ + \ + finally: \ + Py_XDECREF(result); \ + Py_XDECREF(py_argv); \ + Py_XDECREF(func); \ + Py_XDECREF(mod); \ + return ret; \ + } \ + WRAP_FUNC_WITH_BUILTIN(bash_name); \ + DEFINE_BUILTIN(bash_name); diff --git a/www/src/data/thoughts/python-bash-builtins/benchmark.html b/www/src/data/thoughts/python-bash-builtins/benchmark.html new file mode 100644 index 0000000..f145722 --- /dev/null +++ b/www/src/data/thoughts/python-bash-builtins/benchmark.html @@ -0,0 +1,4 @@ +
Benchmark 1: curl http://test-of.v2.natalieee.net/html/view-thought.html?thought=dollcode
+  Time (mean ± σ):     310.4 ms ±   7.3 ms    [User: 22.5 ms, System: 10.8 ms]
+  Range (min … max):   300.2 ms … 348.1 ms    100 runs
+
diff --git a/www/src/data/thoughts/python-bash-builtins/build.sh b/www/src/data/thoughts/python-bash-builtins/build.sh new file mode 100755 index 0000000..6414abd --- /dev/null +++ b/www/src/data/thoughts/python-bash-builtins/build.sh @@ -0,0 +1,39 @@ +#!/bin/bash +# maybe uses non-posix regex and thus may be non-portable. oops. +# the bash regex implementation is actually platform dependent, lol. + +shopt -s extglob + +PY_VERSION=3.13 +PYFLAGS="$(/usr/bin/python${PY_VERSION}-config --cflags --ldflags)" +BASH_INCLUDE_DIR="/usr/include/bash" + +function translate_function_names() { + # cython generates garbled names like __pyx_pf_7py_test_2foo for a python function named foo + # this function generates a list of #defines that translate these names to what they are in python + + file="${1}" + + grep 'static PyObject \*__pyx_pf_' "${file}" | grep -v 'proto' | while read -r line; do + desired_name=$"${line#*$"${file/.c/}"_}" + desired_name="${desired_name/#+([0-9])/}" + real_name="${line#*\*}" + echo "#define ${desired_name%(*} ${real_name%(*}" +done +} + +py2c() { + rm "${1/.py/.c}" + cython3 -3 "${1}" -o "${1/.py/.c}" + translate_function_names $(basename ${1/.py/.c}) >> "${1/.py/.c}" +} + +compile() { + gcc -I$BASH_INCLUDE_DIR{/include,/builtins,}\ + $PYFLAGS $1 -o ${1/.c/} -Wall\ + -lpython3.13 \ + -fPIC -shared -O3 +} + +py2c foo.py +compile example_builtin.c diff --git a/www/src/data/thoughts/python-bash-builtins/expensive_calculation.py b/www/src/data/thoughts/python-bash-builtins/expensive_calculation.py new file mode 100644 index 0000000..15920d3 --- /dev/null +++ b/www/src/data/thoughts/python-bash-builtins/expensive_calculation.py @@ -0,0 +1,16 @@ +def _math(*args): + import numpy as np + matrix_size, iterations, *_ = args + matrix_size, iterations = int(matrix_size), int(iterations) + + for _ in range(iterations): + a = np.random.rand(matrix_size, matrix_size) + b = np.random.rand(matrix_size, matrix_size) + + result = np.dot(a, b) + + print(result.shape) + return 0 + +import sys +_math(*sys.argv[1:]) diff --git a/www/src/data/thoughts/python-bash-builtins/foo.py b/www/src/data/thoughts/python-bash-builtins/foo.py new file mode 100644 index 0000000..3fc05f4 --- /dev/null +++ b/www/src/data/thoughts/python-bash-builtins/foo.py @@ -0,0 +1,3 @@ +def _foo(*_): + print("foo") + return 0 diff --git a/www/src/data/thoughts/python-bash-builtins/math.py b/www/src/data/thoughts/python-bash-builtins/math.py new file mode 100644 index 0000000..f79d967 --- /dev/null +++ b/www/src/data/thoughts/python-bash-builtins/math.py @@ -0,0 +1,16 @@ +def _math(*args): + import numpy as np + matrix_size, iterations, *_ = args + matrix_size, iterations = int(matrix_size), int(iterations) + + for _ in range(iterations): + a = np.random.rand(matrix_size, matrix_size) + b = np.random.rand(matrix_size, matrix_size) + + result = np.dot(a, b) + + print(result) + return 0 + +import sys +print(sys.argv) diff --git a/www/src/data/thoughts/python-bash-builtins/syntax-hl b/www/src/data/thoughts/python-bash-builtins/syntax-hl new file mode 100755 index 0000000..0066d6b --- /dev/null +++ b/www/src/data/thoughts/python-bash-builtins/syntax-hl @@ -0,0 +1,7 @@ +#!/bin/bash +tmpfile=$(mktemp) + +nvim --headless $1 "+set noswapfile | set nonu | set nornu | set conceallevel=0" "+%s/^$/ /g" "+Lazy! load nvim-treesitter | TSEnable highlight | TOhtml | w! ${tmpfile}" '+qa!' 2>/dev/null + +cat "${tmpfile}" | pup --pre 'style, pre' | sed '/^$/d' + diff --git a/www/src/pages/html/thoughts/python-bash-builtins.hy b/www/src/pages/html/thoughts/python-bash-builtins.hy new file mode 100644 index 0000000..296e96a --- /dev/null +++ b/www/src/pages/html/thoughts/python-bash-builtins.hy @@ -0,0 +1,170 @@ +(thought + :title "bash builtins with python" + :date "2025-06-01 05:11:23" + :tags ["programming" "shell" "hacks" "python"] + :description "fact: doing this decreased the amount of time it took to serve this page by ~200ms" + :content `( + ~(run "syntax-hl data/thoughts/python-bash-builtins/bash.h | pup 'style' --pre") + (p ~f"this post was inspired by {(form->html (link #[[https://web.archive.org/web/20160303032434/http://cfajohnson.com/shell/articles/dynamically-loadable/]] #[[this article]]))}, + {(form->html (link #[[https://git.sakamoto.pl/servfail/ruszt]] #[[this project]]))}, and vaguely just the entirety section 4 of the bash manual.") + (p ~f"some example code can be found {(form->html (link #[[//git.natalieee.net/nat/python-bash-builtins]] 'here))}.") + (br) + (h2 "what are bash builtins?") + (hr) + (p "until recently, this one was unaware of the concept of loading new bash builtins during a shell session. everyone it discussed such things with were also unaware of this, thus + it seems appropriate to provide some explanation. in case the reader has not read the bash manual, or is just unfamiliar with the topic, it will also + provide a summary of what a builtin itself is.") + + (p "a bash builtin is a means of running code within bash without creating a new process. many commands one interacts with while using bash are + in fact builtins loaded at shell initialization: examples of such commands include `read`, `test`, and `[`. in addition to those built in to + the shell, new builtins may be loaded from shared object files via the `enable` builtin. the reason one would want to do this instead of simply + calling an external program is that, as previously stated, builtins are executed within the shell, without creating a new process. + because while executing a builtin, a new process does not need to be spawned—and then streams for that process read from and written to—there + are cases in which bash builtins are vastly more efficient than calling external programs." ~(run #[f[make-footnote 'as is touched on {(form->html (link + "https://web.archive.org/web/20160303032434/http://cfajohnson.com/shell/articles/dynamically-loadable/" "here"))}']f])) + + (br) + (h2 "why?") + (hr) + (p "because it is natalie, its first thought upon discovering that one could load new builtins at runtime was something along the lines of "huh, neat, how can python be involved?" + we all have our flaws. additionally, at the time of discovering this, it was still writing the previous post, and as such was thinking quite a bit about optimizing the response time for + this page. this will be relevant later.") + + (br) + (h2 "how?") + (hr) + (p ~f"one may wonder: python is not a compiled language, so how can we produce a shared object containing python code? thankfully, {(form->html (link #[[//cython.org]] 'cython))} exists. + this will allow us to generate c code from python code such that we can then wrap this generated code with other code to create a bash builtin. in this implementation, the bash side of this wrapping process is achieved + via the following two macros:" ~(run #[[make-footnote "please note that it is not a c programmer, would not consider itself to know c, and has almost no experience with it. its code will be bad."]])) + + (figure + (figcaption "(bash.h) macros to create a shell builtin from a function of type int (int, char**)") + (code (pre ~(run "syntax-hl data/thoughts/python-bash-builtins/bash.h | pup 'pre' --pre | tail -n57 | head -n22")))) + + (p "the first macro, `DEFINE_BUILTIN`, takes `name` and creates a builtin struct with `name_struct`. it defines the name of the builtin (as it will appear to `enable`) as `name`, + and sets the associated function to `name_builtin_func`, which it assumes will be defined.") + + (p "the second macro, `WRAP_FUNC_WITH_BUILTIN`, takes `name` and creates a function `name_builtin_func`, which wraps the function `name` with logic that parses the arguments + passed to the builtin in to an int argc and char** argv. as one will see later, arguments to python functions must be `PyObject`s. having this wrapper function allows arguments + from bash to be parsed in to an intermediary form that does not rely on bash's WORD_LIST, and then to be parsed in to `PyObject`s later. this intermediary step exists solely to + make the process of calling python functions simpler.") + + (p "the process of obtaining a c function derived from python code that can reasonably be used is a bit more complicated. because the way that cython is being utilized + here is not really the intended means of doing so (shockingly), this process becomes rather hacky. in the shell script that builds this code (which is included below), we first invoke + cython on a .py file, and then run a function `translate_function_names` on the output.") + + (figure + (figcaption "the afforementioned build script") + (code ~(run "syntax-hl data/thoughts/python-bash-builtins/build.sh"))) + + (p "as is detailed in this build script, the cython output contains mangled function names. to unmangle them, we generate a series of define statements that get concatonated to the end of the c output file.") + (p "as we have now established, a function `foo` in a .py file will become accessible under the name `foo` in a c file, so long as that c file includes the file in which the `foo` function is defined—though + with the .py suffix substituted with a .c suffix. understanding this, we may examine the final macro which transposes our newly c-ified python functions in to a form that can be passed to + `WRAP_FUNC_WITH_BUILTIN` and `DEFINE_BUILTIN`:") + + (figure + (figcaption "(bash.h) the `PY_FUNC` macro") + (code (pre ~(run "syntax-hl data/thoughts/python-bash-builtins/bash.h | pup --pre 'pre' | tail -n34")))) + + (p "this macro takes a `bash_name`, `modname`, and `function` and expands to a function definition for a function named the value of `bash_name`, which returns an int while taking an int argc and char** argv. + this `bash_name` function is then passed to `WRAP_FUNC_WITH_BUILTIN` (expanding to a definition of a builtin wrapper function that calls `bash_name`), and then to `DEFINE_BUILTIN` (expanding to + the actual `builtin` struct), thusly defining a builting named `bash_name` to call a series of wrapper functions (first the `WORD_LIST` to (int, char**) wrapper, and then the `bash_name` function) which + arrive at a call to the relevant python code.") + + (p "inside of this `bash_name` function, we firstly check to see if python has been initialized and should it not be, initialize it after appending the module defined by `modname` to the builtin module table. + following this, we `dlopen()` the python shared object. this is necessary to load symbols defined within libpython.so, which are expected to be defined by c extensions loaded by python.") + + (p "after checking for the initialization of—and possibly initializing—python, we import `modname`. within `modname`, we find the attribute associated with the name passed in the `function` + argument of the `PY_FUNC` macro, and save this to the variable `func`. next, an argument tuple is built such that its elements are each string in the `argv` argument passed to `bash_name`. + this is then used to call the `func` variable, after which we check for errors. in the presence of errors, we jump to the cleanup section and return the default value of `ret`, which is one. otherwise, + we set the value of `ret` to the return value of `func`, and proceed with cleaning up. notably `Py_Finalize` is never called, thus leaving the interpreter running. it is necessary to do this because + of some strangeness with how the python interpreter handles state.") + + (p "this allows python functions of the signature ([str]) -> int to be called from bash such that the argument of the function is a list of all arguments passed to its associated builtin, and its return value + is the exit code of the builtin. additionally, because `Py_Finalize` is never called, after the first call to a builtin the python interpreter remains initialized. this creates significant savings in performance when + executing a large number of calls to a builtin, as compared to initializing a python interpreter for each call.") + + (p "in practice, this looks like the following:") + (figure + (figcaption "foo.py") + (code ~(run "syntax-hl data/thoughts/python-bash-builtins/foo.py"))) + + (figure + (figcaption "bar.c") + (code ~(run "syntax-hl data/thoughts/python-bash-builtins/bar.c"))) + + (br) + (h2 "why on earth would anyone want this?") + (hr) + (p "say there exists some python script that does something, and we want to call it many times in a bash script. for this example, lets say the function simply prints \"foo\" to stdout. + normally, to do this one would call `python3 foo.py` on each iteration of the loop:") + + (figure + (figcaption "the afforementioned method:") + (code (pre + "time { for i in {0..100}; do python3 foo.py >/dev/null; done }\n\n" + "real 0m1.260s\n" + "user 0m0.826s\n" + "sys 0m0.435s"))) + + (p "for just printing \"foo\" 100 times, this takes an absurd amount of time. compiling this code to a bash builtin, we see:") + (figure + (figcaption "shell builtin method:") + (code (pre + "enable -f ./foo foo\n" + "time { for i in {0..100}; do foo >/dev/null; done }\n\n" + "real 0m0.001s\n" + "user 0m0.001s\n" + "sys 0m0.000s"))) + + (p "this difference becomes even more pronounced when the python scripts being called import large libraries like numpy. because, when compiled to a bash builtin, the python interpreter is + only initialized once, any libraries only need to be imported once.") + + (figure + (figcaption "a useless python program that imports a large library") + (code ~(run "syntax-hl data/thoughts/python-bash-builtins/math.py"))) + + (p "pretend this program actually does something useful, and that we want to process a large amount of data with it from the shell. to repeat the previous demonstration, doing this the traditional + way goes as follows:") + + (figure + (figcaption "calling the python interpreter in a loop") + (code (pre + "time { for i in {0..100}; do python3 math.py 2 2 >/dev/null; done }\n\n" + "real 0m11.261s\n" + "user 1m49.283s\n" + "sys 0m2.261s\n"))) + + (figure + (figcaption "calling a shell builtin in a loop") + (code (pre + "time { for i in {0..100}; do math 2 2 >/dev/null; done }\n\n" + "real 0m0.002s\n" + "user 0m0.000s\n" + "sys 0m0.002s\n"))) + + (p "the difference speaks for itself.") + + (br) + (h2 "using this to improve site performance") + (hr) + (p ~f"as was explained in the {(form->html (link "?thought=natalieee.net-v2" "the previous thought"))}, these posts are stored on the server as fragments of html documents and rendered at request time. + this is done via a hy program that runs the same code that the server uses to run bash snippets for server side rendering. executing this program on posts is the lengthiest part of the post rendering + process, which is why rendering a specific post takes ~330ms longer than rendering the list of posts.") + + (p "using `hy2py`, hy code can be converted in to python code. accordingly, this allows us to compile said code in to a bash builtin. it follows that we can then replace the code to execute bash with a + call to the resulting builtin. this results in a significant decrease in server rendering time:") + + (figure + (figcaption "benchmark comparing the old rendering code to the new code") + (code ~(run "cat data/thoughts/python-bash-builtins/benchmark.html"))) + + (p "this is a significant improvement, as the benchmarks in the previous post established the time taken to render a post and subsequently respond to the associated request to be around 500ms.") + + (br) + (h2 "problems/limitations with this implementation") + (p "it created this almost entirely without reading documentation, and thus there are several glaring flaws. foremost among these is the fact that one cannot include multiple .c cython output files + in a .c file that is to be compiled to a builtin, due to redefinition errors. additionally, the build process is sort of clunky and probably requires manual modification on a per system basis. + finally, there isn't a good way to compile a python program consisting of multiple .py files in to a builtin, without installing it as a library on python's import path. none of these, however, + are particularly relevant to a toy proof-of-concept."))) +