Răsfoiți Sursa

Persist browser cookies and localStorage in /data

When the installed disk's /data volume is mounted, the browser keeps the
helpers' cookie jar in /data/BCOOKIES.BIN and every origin's localStorage
in one /data/BSTORE.DAT, so logins survive a reboot.  /share stays the
working copy: the jar is restored into /share at start, a store missing
from /share is read from disk on first use, and the main loop writes back
what changed at most every 3 s.  A cleared store is kept as "{}" so the
disk copy cannot resurrect it.  sessionStorage stays in RAM.
Johannes Zellner 1 săptămână în urmă
părinte
comite
a6d2afec15
4 a modificat fișierele cu 305 adăugiri și 12 ștergeri
  1. 12 2
      AGENTS.md
  2. 292 9
      browser.c
  3. 1 1
      docs/network.html
  4. 0 0
      docs/userland.html

+ 12 - 2
AGENTS.md

@@ -1368,8 +1368,18 @@ it to `0x7E00`.  The kernel region itself only has one wall (`_end` below
   `js_host_http` submits `X <method> <len> <ctype> <url>` to a helper and
   returns the status and body whatever the HTTP status (2 MB response, 16 KB
   request body).  Web Storage is stored in `/share/bls<origin-hash>`
-  (local, per boot) and `/share/bss<pid>-<origin-hash>` (session, per
-  window), outside the page heap that navigation resets.  `console.*` print
+  (local) and `/share/bss<pid>-<origin-hash>` (session, per window),
+  outside the page heap that navigation resets.  When `/data` is mounted
+  the browser also persists the helpers' cookie jar (`/share/bcookies` ->
+  `/data/BCOOKIES.BIN`) and localStorage (every origin in one
+  `/data/BSTORE.DAT`: `NBLS1\n` then `<origin>\n<len>\n<json>\n` records,
+  192 KB cap) across reboots: `persist_restore()` seeds `/share` from disk
+  at start (a store missing from `/share` is looked up on disk on first
+  use, and a cleared store is kept as `{}` so it cannot reappear), and
+  `persist_tick()` in the main loop writes back what changed at most every
+  3 s (`PERSIST_MS`).  The helpers only touch `/share`.  Both files are
+  plaintext bearer credentials, like `CHAT.CFG`; sessionStorage stays in
+  RAM.  Without `/data` the disk accesses just fail.  `console.*` print
   Error stacks.  `location.assign`/`replace`/`reload` navigate.  The
   Cloudron dashboard (`my.<domain>`: a Vite/Vue 3 app, 12 modules / 3.5 MB)
   boots, builds its PKCE request, and its OIDC login form renders, submits

+ 292 - 9
browser.c

@@ -20,6 +20,7 @@
 #include "browser_html.h"
 #include "js_browser.h"
 #include "uhttp.h"
+#include "uhttp_cookie.h"
 #include "img.h"
 #include "inflate.h"
 #include "ufont.h"
@@ -1283,19 +1284,41 @@ static char *js_host_load_script(const char *url, unsigned int *len)
 /* Web Storage lives in /share, outside the page heap that every navigation
  * resets: localStorage per origin for the boot, sessionStorage per origin
  * and browser window. */
-static void js_storage_path(int kind, char *out, unsigned int cap)
+#define JS_ORIGIN_MAX 160
+
+/* scheme://host[:port] of the page (a local page's whole path). */
+static void js_page_origin(char *origin, unsigned int cap)
 {
-	char origin[160];
-	unsigned int n = 0, i = 0, slashes = 0;
+	unsigned int i = 0, slashes = 0;
 
-	while (page_url[i] && i + 1 < sizeof(origin)) {
+	while (page_url[i] && i + 1 < cap) {
 		if (page_url[i] == '/' && ++slashes == 3)
 			break;
 		origin[i] = page_url[i];
 		i++;
 	}
 	origin[i] = '\0';
-	s_append(out, &n, kind ? "/share/bss" : "/share/bls", cap);
+}
+
+static void js_origin_path(const char *origin, char *out, unsigned int cap)
+{
+	unsigned int n = 0;
+
+	s_append(out, &n, "/share/bls", cap);
+	s_append_dec(out, &n, script_cache_hash(origin), cap);
+}
+
+static void js_storage_path(int kind, char *out, unsigned int cap)
+{
+	char origin[JS_ORIGIN_MAX];
+	unsigned int n = 0;
+
+	js_page_origin(origin, sizeof(origin));
+	if (!kind) {
+		js_origin_path(origin, out, cap);
+		return;
+	}
+	s_append(out, &n, "/share/bss", cap);
 	if (kind) {
 		s_append_dec(out, &n, (unsigned int)u_getpid(), cap);
 		s_append(out, &n, "-", cap);
@@ -1303,27 +1326,285 @@ static void js_storage_path(int kind, char *out, unsigned int cap)
 	s_append_dec(out, &n, script_cache_hash(origin), cap);
 }
 
+static void persist_mark_origin(const char *origin);
+static int persist_store_lookup(const char *origin, char *out, int cap);
+
 static int js_host_storage(int kind, const char *set, char *out, int cap)
 {
-	char path[48];
+	char path[48], origin[JS_ORIGIN_MAX];
 	int rv;
 
 	js_storage_path(kind, path, sizeof(path));
 	if (set) {
-		if (!set[0] || !strcmp(set, "{}")) {
-			u_syscall1(SYS_UNLINK, (int)path);
-			return 0;
+		if (kind) {
+			if (!set[0] || !strcmp(set, "{}")) {
+				u_syscall1(SYS_UNLINK, (int)path);
+				return 0;
+			}
+		} else {
+			/* localStorage keeps an explicit "{}" rather than no
+			 * file: a missing /share file means "not loaded from
+			 * /data yet", and a cleared store must not come back
+			 * from the disk copy before it is flushed. */
+			if (!set[0])
+				set = "{}";
+			js_page_origin(origin, sizeof(origin));
+			persist_mark_origin(origin);
 		}
 		return u_syscall3(SYS_WRITE_FILE, (int)path, (int)set,
 				  (int)strlen(set)) < 0 ? -1 : 0;
 	}
 	rv = u_syscall3(SYS_READ_FILE, (int)path, (int)out, cap - 1);
+	if (rv < 0 && !kind) {
+		/* First use this boot: the persistent copy on /data. */
+		js_page_origin(origin, sizeof(origin));
+		rv = persist_store_lookup(origin, out, cap);
+		if (rv >= 0)
+			u_syscall3(SYS_WRITE_FILE, (int)path, (int)out, rv);
+	}
 	if (rv < 0)
 		return -1;
 	out[rv] = '\0';
 	return rv;
 }
 
+/* ---- persistence on /data ----
+ *
+ * /share is the working copy (RAM, shared by every helper and window); the
+ * installed disk's /data volume keeps the cookie jar and localStorage
+ * across reboots when it is mounted.  The browser owns the disk copies: at
+ * start it restores the jar into /share, and its main loop writes back
+ * whatever changed at most every PERSIST_MS, so a burst of cookie or
+ * setItem updates costs one FAT write.  The helpers only touch /share (they
+ * are killed on focus loss, so they cannot be trusted to flush).  Without a
+ * /data mount every disk access simply fails.  Both files are plaintext
+ * bearer credentials, like /data/CHAT.CFG.  sessionStorage stays in RAM. */
+#define PERSIST_MS        3000u
+#define SHARE_COOKIES     "/share/bcookies"
+#define DATA_COOKIES      "/data/BCOOKIES.BIN"
+#define DATA_STORE        "/data/BSTORE.DAT"
+#define DATA_STORE_MAGIC  "NBLS1\n"
+#define DATA_STORE_MAX    (192u * 1024u)
+#define PERSIST_DIRTY_MAX 8
+
+static unsigned int persist_due;
+static unsigned int cookie_sum_saved;
+static int cookie_sum_valid;
+static char persist_dirty[PERSIST_DIRTY_MAX][JS_ORIGIN_MAX];
+static int persist_ndirty;
+
+static unsigned int persist_sum(const unsigned char *p, unsigned int n)
+{
+	unsigned int h = 2166136261u, i;
+
+	for (i = 0; i < n; i++)
+		h = (h ^ p[i]) * 16777619u;
+	return h;
+}
+
+/* Called once at startup, before any helper runs. */
+static void persist_restore(void)
+{
+	unsigned char *jar = (unsigned char *)malloc(sizeof(struct uhttp_jar));
+	int n;
+
+	if (!jar)
+		return;
+	n = u_syscall3(SYS_READ_FILE, (int)SHARE_COOKIES, (int)jar,
+		       (int)sizeof(struct uhttp_jar));
+	if (n != (int)sizeof(struct uhttp_jar)) {
+		n = u_syscall3(SYS_READ_FILE, (int)DATA_COOKIES, (int)jar,
+			       (int)sizeof(struct uhttp_jar));
+		if (n == (int)sizeof(struct uhttp_jar)) {
+			u_syscall3(SYS_WRITE_FILE, (int)SHARE_COOKIES,
+				   (int)jar, n);
+			cookie_sum_saved = persist_sum(jar, (unsigned int)n);
+			cookie_sum_valid = 1;
+		}
+	}
+	free(jar);
+	persist_due = u_ticks_ms() + PERSIST_MS;
+}
+
+static void persist_mark_origin(const char *origin)
+{
+	int i;
+
+	for (i = 0; i < persist_ndirty; i++)
+		if (s_eq(persist_dirty[i], origin))
+			return;
+	if (persist_ndirty < PERSIST_DIRTY_MAX)
+		s_copy(persist_dirty[persist_ndirty++], origin, JS_ORIGIN_MAX);
+}
+
+/* BSTORE.DAT: "NBLS1\n", then per origin "<origin>\n<length>\n<json>\n".
+ * Walk it; returns 1 with the record's fields, 0 at the end. */
+static int store_next(const char *buf, unsigned int len, unsigned int *pos,
+		      unsigned int *o_off, unsigned int *o_len,
+		      unsigned int *d_off, unsigned int *d_len,
+		      unsigned int *rec_end)
+{
+	unsigned int p = *pos, v = 0;
+
+	if (p >= len)
+		return 0;
+	*o_off = p;
+	while (p < len && buf[p] != '\n')
+		p++;
+	if (p >= len)
+		return 0;
+	*o_len = p - *o_off;
+	p++;
+	while (p < len && buf[p] >= '0' && buf[p] <= '9')
+		v = v * 10 + (unsigned int)(buf[p++] - '0');
+	if (p >= len || buf[p] != '\n')
+		return 0;
+	p++;
+	if (v > len - p)
+		return 0;
+	*d_off = p;
+	*d_len = v;
+	p += v;
+	if (p < len && buf[p] == '\n')
+		p++;
+	*rec_end = p;
+	*pos = p;
+	return 1;
+}
+
+static int store_load(char **out)
+{
+	char *buf = (char *)malloc(DATA_STORE_MAX + 1);
+	int n;
+
+	*out = buf;
+	if (!buf)
+		return -1;
+	n = u_syscall3(SYS_READ_FILE, (int)DATA_STORE, (int)buf,
+		       (int)DATA_STORE_MAX);
+	if (n < (int)s_len(DATA_STORE_MAGIC) ||
+	    !starts_with(buf, DATA_STORE_MAGIC))
+		return 0;
+	return n;
+}
+
+static int persist_store_lookup(const char *origin, char *out, int cap)
+{
+	char *buf;
+	unsigned int pos = s_len(DATA_STORE_MAGIC), oo, ol, d, dl, e;
+	unsigned int olen = s_len(origin);
+	int n = store_load(&buf), rv = -1;
+
+	while (n > 0 && store_next(buf, (unsigned int)n, &pos, &oo, &ol, &d,
+				   &dl, &e)) {
+		if (ol == olen && !memcmp(buf + oo, origin, olen)) {
+			if ((int)dl < cap) {
+				memcpy(out, buf + d, dl);
+				out[dl] = '\0';
+				rv = (int)dl;
+			}
+			break;
+		}
+	}
+	free(buf);
+	return rv;
+}
+
+/* Rewrite BSTORE.DAT with the dirty origins' current /share copies. */
+static void persist_store_flush(void)
+{
+	char *old, *nb, path[48];
+	unsigned int pos = s_len(DATA_STORE_MAGIC), oo, ol, d, dl, e, nn = 0;
+	int n = store_load(&old), i, w;
+
+	nb = (char *)malloc(DATA_STORE_MAX + 1);
+	if (n < 0 || !nb) {
+		free(old);
+		free(nb);
+		return;
+	}
+	s_append(nb, &nn, DATA_STORE_MAGIC, DATA_STORE_MAX);
+	/* Untouched origins first, in their old order. */
+	while (n > 0 && store_next(old, (unsigned int)n, &pos, &oo, &ol, &d,
+				   &dl, &e)) {
+		int dirty = 0;
+
+		for (i = 0; i < persist_ndirty; i++)
+			if (s_len(persist_dirty[i]) == ol &&
+			    !memcmp(persist_dirty[i], old + oo, ol))
+				dirty = 1;
+		if (dirty || nn + (e - oo) > DATA_STORE_MAX)
+			continue;
+		memcpy(nb + nn, old + oo, e - oo);
+		nn += e - oo;
+	}
+	/* Then the changed ones (an emptied store is dropped). */
+	for (i = 0; i < persist_ndirty; i++) {
+		char num[12];
+		unsigned int k = 0;
+
+		js_origin_path(persist_dirty[i], path, sizeof(path));
+		w = u_syscall3(SYS_READ_FILE, (int)path, (int)old,
+			       (int)DATA_STORE_MAX);
+		if (w <= 2)     /* missing or "{}" */
+			continue;
+		s_append_dec(num, &k, (unsigned int)w, sizeof(num));
+		if (nn + s_len(persist_dirty[i]) + k + (unsigned int)w + 3 >
+		    DATA_STORE_MAX) {
+			set_status("Saved site data is full; a site's "
+				   "localStorage was not saved", 1, 6000);
+			continue;
+		}
+		s_append(nb, &nn, persist_dirty[i], DATA_STORE_MAX);
+		nb[nn++] = '\n';
+		s_append(nb, &nn, num, DATA_STORE_MAX);
+		nb[nn++] = '\n';
+		memcpy(nb + nn, old, (unsigned int)w);
+		nn += (unsigned int)w;
+		nb[nn++] = '\n';
+	}
+	u_syscall3(SYS_WRITE_FILE, (int)DATA_STORE, (int)nb, (int)nn);
+	persist_ndirty = 0;
+	free(old);
+	free(nb);
+}
+
+static void persist_cookies_flush(void)
+{
+	unsigned char *jar = (unsigned char *)malloc(sizeof(struct uhttp_jar));
+	unsigned int sum;
+	int n;
+
+	if (!jar)
+		return;
+	n = u_syscall3(SYS_READ_FILE, (int)SHARE_COOKIES, (int)jar,
+		       (int)sizeof(struct uhttp_jar));
+	if (n == (int)sizeof(struct uhttp_jar)) {
+		sum = persist_sum(jar, (unsigned int)n);
+		if (!cookie_sum_valid || sum != cookie_sum_saved) {
+			if (u_syscall3(SYS_WRITE_FILE, (int)DATA_COOKIES,
+				       (int)jar, n) >= 0) {
+				cookie_sum_saved = sum;
+				cookie_sum_valid = 1;
+			}
+		}
+	}
+	free(jar);
+}
+
+/* Main-loop hook: write back to /data at most every PERSIST_MS. */
+static void persist_tick(void)
+{
+	unsigned int now = u_ticks_ms();
+
+	if ((int)(now - persist_due) < 0)
+		return;
+	persist_due = now + PERSIST_MS;
+	persist_cookies_flush();
+	if (persist_ndirty)
+		persist_store_flush();
+}
+
 /* <link rel="modulepreload">: download the page's module graph on all
  * helpers at once into the boot script cache, before any script runs.  The
  * engine's module loader is sequential (one blocking fetch per import), so
@@ -10053,6 +10334,7 @@ void _start(const char *arg)
 	origin_cache_load();
 	css_cache_init();
 	script_cache_init();
+	persist_restore();
 	page_scroll.offset = 0;
 	page_scroll.content_h = 0;
 
@@ -10093,6 +10375,7 @@ void _start(const char *arg)
 			preload_active = 1;
 		}
 		js_browser_tick();
+		persist_tick();
 		if (pending_action)
 			continue;
 		now = u_ticks_ms();

+ 1 - 1
docs/network.html

@@ -410,7 +410,7 @@
 			<li><strong>curl</strong> — <code>http://</code> or <code>https://</code> GET with DNS, ARP, redirects (up to five hops), chunked decoding, status/headers/body dump and a TLS warning.</li>
 			<li><strong>uhttp</strong> — shared ring-3 HTTP/1.1 client used by curl, the browser and chat: URL parsing (spaces, controls, quotes and non-ASCII bytes in the path are percent-encoded, and <code>//</code>/<code>.</code>/<code>..</code> normalization stops at the query, which may carry a URL such as <code>redirect_uri=https://…</code>), redirects (up to eight hops; 1 KB paths and 1.5 KB header lines, enough for OIDC URLs), chunked decoding, timeouts and TLS.  Requests and bodies of any size are split into pieces the transport accepts (1024-byte TLS records, one peer MSS for plain TCP).  Optional per-request fields set the <code>Accept</code> header, extra header lines and a cookie jar; 1xx/204/304 responses are treated as bodyless so keep-alive does not wait for an idle timeout.</li>
 			<li><strong>uhttp_cookie</strong> — the cookie jar (<span class="file">uhttp_cookie.c</span>, 24 cookies of up to 768 bytes).  Every <code>Set-Cookie</code>, including those on redirect responses, is stored; matching cookies are sent on every request and redirect hop.  Cookies are host-only (<code>Domain</code> is ignored), <code>Path</code> is a prefix match, <code>Secure</code> cookies only go over https, and <code>Max-Age&lt;=0</code> or a 1970 <code>Expires</code> deletes a cookie.  The jar outlives the connection, so a cross-origin redirect that reconnects keeps the session.</li>
-			<li><strong>browser</strong> — runs network fetches in up to four headless <code>curl __serve</code> workers, each with its own request/body/meta files under <code>/share/bf&lt;session&gt;_&lt;slot&gt;*</code>.  A request line is <code>&lt;seq&gt; &lt;cap&gt; [POST &lt;len&gt;] &lt;url&gt;</code> for page/form loads or <code>&lt;seq&gt; &lt;cap&gt; X &lt;method&gt; &lt;len&gt; &lt;content-type|-&gt; &lt;url&gt;</code> for script <code>fetch()</code>/XHR and empty-body form POSTs.  All helpers share one cookie jar in <code>/share/bcookies</code> (read before and written after every request that changed it), so a login session survives helper restarts and the parallel workers; it is RAM only and cleared at reboot.  The session id comes from the browser pid, so multiple browser windows do not share queue files.  Stylesheet and external-script bodies are downloaded in parallel but parsed/executed in document order; one image starts before the script phase, and the remaining images are prefetched in parallel and decoded as jobs complete, so the UI keeps drawing and scrolling.  Each worker keeps its own TCP/TLS connection; idle helpers stay warm across navigations (a cross-origin job just drops the old connection, and only busy helpers are killed), and helpers also exit after ten idle seconds or when the window loses focus.  A DNS failure is retried once after tearing the helpers down.  The helpers' DNS/TLS log lines go to a private tty that the browser drains and discards from every poll loop; a full tty ring blocks its writers, so an undrained one used to hang every helper after about sixteen TLS connections.  The status bar reports the page's total, header and first-byte times.</li>
+			<li><strong>browser</strong> — runs network fetches in up to four headless <code>curl __serve</code> workers, each with its own request/body/meta files under <code>/share/bf&lt;session&gt;_&lt;slot&gt;*</code>.  A request line is <code>&lt;seq&gt; &lt;cap&gt; [POST &lt;len&gt;] &lt;url&gt;</code> for page/form loads or <code>&lt;seq&gt; &lt;cap&gt; X &lt;method&gt; &lt;len&gt; &lt;content-type|-&gt; &lt;url&gt;</code> for script <code>fetch()</code>/XHR and empty-body form POSTs.  All helpers share one cookie jar in <code>/share/bcookies</code> (read before and written after every request that changed it), so a login session survives helper restarts and the parallel workers.  When <code>/data</code> is mounted the browser copies the jar to <code>/data/BCOOKIES.BIN</code> (at most every 3 s, when it changed) and restores it at start, so sessions survive a reboot; the file is a plaintext bearer credential, like <code>/data/CHAT.CFG</code>.  The session id comes from the browser pid, so multiple browser windows do not share queue files.  Stylesheet and external-script bodies are downloaded in parallel but parsed/executed in document order; one image starts before the script phase, and the remaining images are prefetched in parallel and decoded as jobs complete, so the UI keeps drawing and scrolling.  Each worker keeps its own TCP/TLS connection; idle helpers stay warm across navigations (a cross-origin job just drops the old connection, and only busy helpers are killed), and helpers also exit after ten idle seconds or when the window loses focus.  A DNS failure is retried once after tearing the helpers down.  The helpers' DNS/TLS log lines go to a private tty that the browser drains and discards from every poll loop; a full tty ring blocks its writers, so an undrained one used to hang every helper after about sixteen TLS connections.  The status bar reports the page's total, header and first-byte times.</li>
 			<li><strong>websrv</strong> — the server side, speaking HTTP/1.1 + WebDAV over the TCP listener; see <a href="filesystems.html#webdav">Filesystems → WebDAV</a>.</li>
 		</ul>
 		<p>

Fișier diff suprimat deoarece este prea mare
+ 0 - 0
docs/userland.html


Unele fișiere nu au fost afișate deoarece prea multe fișiere au fost modificate în acest diff