Просмотр исходного кода

Write VRAM through the stolen-VRAM alias with streaming stores

The boot log on the AT3IONT-I showed why the WC MTRR never helped: the BIOS
leaves a 512 MB uncacheable MTRR at index 0 over the PCIe apertures, and the
lowest-numbered matching entry wins -- our WC entry at index 2 is ignored, so
the VBE LFB (a BAR3 window into that aperture) stays UC at ~33 MB/s.

The ION's VRAM is stolen DRAM (E820 reserved at 0xD0000000), outside that UC
range:

- nvidia_lfb_in_vram() finds the framebuffer inside the stolen VRAM by
  writing a marker through the LFB (8 MB past the frame) and scanning the
  VRAM at page granularity through a cached mapping.
- vga_shadow_init() switches vga_fb_lfb to that alias only after a WC MTRR
  covers the stolen range; a cached framebuffer would stop updating the
  screen, so no WC entry means staying on the UC aperture.  Both paths log
  their choice.
- The flush copies with streaming stores (vga_fb_copy_row_nt, movntdq with
  an aligned head and one sfence per rect): rep movsl does not combine on
  this path.  QEMU: 204 -> 769 MB/s.  ION: 32 -> 75 MB/s.
- vga_fb_lfb_wc() reports the path actually in use instead of the page-type
  decision the BIOS MTRR overrides; gfxbench prints it.
- paging_map_ram() maps a physical window cacheable (4 MB pages).

The ION result (75 MB/s) shows the chipset treats the stolen range as
uncached for the CPU anyway, so the remaining levers are fewer bytes (16 bpp,
smaller drag damage) or GPU-assisted copies, not the CPU write path.

Also shrinks the non-hot drivers/filesystem/shell objects with -Os: the
kernel had 368 bytes of headroom below the 0x7F000 wall before.
Johannes Zellner 2 недель назад
Родитель
Сommit
c1b8f16b27
10 измененных файлов с 225 добавлено и 27 удалено
  1. 12 5
      AGENTS.md
  2. 26 18
      Makefile
  3. 4 3
      gfxbench.c
  4. 59 0
      nvidia.c
  5. 7 0
      nvidia.h
  6. 12 0
      paging.c
  7. 5 0
      paging.h
  8. 51 1
      vga.c
  9. 6 0
      vga.h
  10. 43 0
      vga_copy.S

+ 12 - 5
AGENTS.md

@@ -125,11 +125,18 @@ below the run area), so adding code no longer moves any address.
   fixed frame rate.  A rect that fits the budget is copied whole, so a window
   repaint never tears; only a bigger one (a full-screen redraw) is chunked, and
   its remaining rows continue after the newer damage.  ATI Mach64 CRTC
-  fixup; NVIDIA is VBE-only (never touch the legacy VGA CRTC).  The LFB is
-  mapped UC unless the WC MTRR covered the range (`mtrr_set_wc()` returns the
-  covered range and `paging.c` makes only those pages cacheable -- a PCD=1
-  page is UC no matter what the MTRRs say, which held VRAM writes at ~33 MB/s
-  on the ION until it was fixed). `nvidia.c`
+  fixup; NVIDIA is VBE-only (never touch the legacy VGA CRTC).  On the ION the
+  framebuffer is stolen DRAM and the VBE LFB is a BAR3 window into it; the
+  BIOS leaves a 512 MB uncacheable MTRR over the PCIe apertures, and since the
+  lowest-numbered matching entry wins, the aperture can never be
+  write-combining (~33 MB/s of separate PCIe writes).  `nvidia_lfb_in_vram()`
+  therefore locates the framebuffer in the stolen VRAM (marker through the
+  LFB, page-granular scan of a cached mapping) and `vga_shadow_init()` writes
+  through that alias once a WC MTRR covers the stolen range (only then -- a
+  cached framebuffer would stop updating the screen).  Everything else keeps
+  the UC LFB: `paging.c` maps LFB pages cacheable only when `mtrr_set_wc()`
+  says it covered them.  `vga_fb_lfb_wc()` reports the effective path and
+  `gfxbench` prints it. `nvidia.c`
   identifies the card, maps BAR0/BAR1/BAR3 and reads the MCP7x stolen-VRAM
   geometry; `nv50.c` probes the nv50-family display engine at boot (read-only,
   serial/netmon). The ION/AT3IONT-I hardware cursor is deliberately not driven:

+ 26 - 18
Makefile

@@ -36,7 +36,7 @@ fbtext.o: fbtext.c fbtext.h font_8x8.h vga.h
 	$(CC) $(CFLAGS) -o $@ $<
 
 terminal.o: terminal.c terminal.h screen.h vfs.h fat.h blkdev.h io.h dbg.h kexec.h process.h timer.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 wm.o: wm.c wm.h terminal.h screen.h fbtext.h idle.h io.h mouse.h process.h rtc.h timer.h version.h vga.h
 	$(CC) $(CFLAGS) -o $@ $<
@@ -51,10 +51,10 @@ poweroff.o: poweroff.c
 	$(CC) $(CFLAGS) -o $@ $<
 
 disk.o: disk.c disk.h io.h timer.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 fat.o: fat.c fat.h blkdev.h io.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 run.o: run.c fat.h io.h fbtext.h run_api.h timer.h vga.h vfs.h
 	$(CC) $(CFLAGS) -o $@ $<
@@ -123,34 +123,34 @@ display.o: display.c io.h vga.h gpu.h ati.h nvidia.h radeon.h mouse.h fbtext.h
 	$(CC) $(CFLAGS) -o $@ $<
 
 uhci.o: uhci.c uhci.h usb_hcd.h pci.h serial.h timer.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 ohci.o: ohci.c ohci.h usb_hcd.h pci.h paging.h mm.h serial.h timer.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 usb.o: usb.c usb.h usb_hcd.h io.h serial.h timer.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 usbhid.o: usbhid.c usbhid.h io.h usb.h usb_hcd.h serial.h timer.h
 	$(CC) $(CFLAGS) -o $@ $<
 
 lsusb.o: lsusb.c io.h usb.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 kdbg.o: kdbg.c kdbg.h font_8x8.h timer.h vga.h
 	$(CC) $(CFLAGS) -o $@ $<
 
 lspci.o: lspci.c io.h pci.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 peek.o: peek.c io.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 ne2000.o: ne2000.c ne2000.h pci.h timer.h io.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 r8168.o: r8168.c r8168.h netdev.h io.h kdbg.h mm.h paging.h pci.h process.h serial.h timer.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 netkexec.o: netkexec.c netkexec.h io.h kexec.h net.h netdev.h timer.h
 	$(CC) $(CFLAGS) -o $@ $<
@@ -174,7 +174,7 @@ net.o: net.c net.h io.h netdev.h
 	$(CC) $(CFLAGS) -o $@ $<
 
 blkdev.o: blkdev.c blkdev.h ata.h ahci.h disk.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 ata.o: ata.c ata.h pci.h serial.h timer.h
 	$(CC) $(CFLAGS) -o $@ $<
@@ -183,13 +183,13 @@ ahci.o: ahci.c ahci.h ata.h mm.h paging.h pci.h process.h serial.h timer.h
 	$(CC) $(CFLAGS) -o $@ $<
 
 fatdetect.o: fatdetect.c fatdetect.h fat.h blkdev.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 fatformat.o: fatformat.c fatformat.h blkdev.h mm.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 format.o: format.c blkdev.h disk.h fatformat.h io.h vga.h vfs.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 mouse.o: mouse.c mouse.h cirrus.h radeon.h timer.h usbhid.h vga.h
 	$(CC) $(CFLAGS) -o $@ $<
@@ -204,10 +204,10 @@ kexec_tramp.o: kexec_tramp.S
 	$(AS) $(ASFLAGS) -o $@ $<
 
 dbg.o: dbg.c dbg.h serial.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 vfs.o: vfs.c vfs.h fat.h fatdetect.h blkdev.h io.h
-	$(CC) $(CFLAGS) -o $@ $<
+	$(CC) $(CFLAGS) -Os -o $@ $<
 
 boot.elf: boot.o
 	$(LD) $(BOOT_LDFLAGS) -o $@ $<
@@ -343,7 +343,7 @@ prog: prog.bin
 # ucmd_<name>.c to UCMDS; the pack is rebuilt automatically.
 UCMDS = ucmd_cpuinfo ucmd_ls ucmd_cat ucmd_save ucmd_rm ucmd_mkdir ucmd_rmdir \
         ucmd_meminfo ucmd_mouseinfo ucmd_storage ucmd_top \
-        ucmd_clock ucmd_colors \
+        ucmd_clock ucmd_colors ucmd_edit \
         ucmd_netinfo ucmd_netip ucmd_netdns ucmd_host ucmd_dhcp \
         ucmd_netsend ucmd_netrecv ucmd_ping ucmd_pong ucmd_netlog \
         ucmd_netconsole ucmd_netregs ucmd_netloop ucmd_netreset ucmd_curl
@@ -359,6 +359,14 @@ ucmd_%.elf: ucmd_%.o ucmd.ld
 	$(LD) -T ucmd.ld -nostdlib -m elf_i386 -o $@ $<
 	$(OBJCOPY) --strip-all $@ $@
 
+# edit draws text itself, so it links the kernel's 8x8 font into its ELF.
+ucmd_edit.o: ucmd_edit.c ucmd_lib.h syscall.h font_8x8.h
+	$(CC) $(CFLAGS) -o $@ $<
+
+ucmd_edit.elf: ucmd_edit.o font_8x8.o ucmd.ld
+	$(LD) -T ucmd.ld -nostdlib -m elf_i386 -o $@ $(filter %.o,$^)
+	$(OBJCOPY) --strip-all $@ $@
+
 # TLS 1.3 client library (ring-3, linked into ucmd_curl; see tls.h).  The
 # crypto files are pure freestanding C, so tls_test can exercise them on
 # the host (make test-crypto).

+ 4 - 3
gfxbench.c

@@ -13,7 +13,6 @@
 #include "gfx.h"
 #include "io.h"
 #include "mm.h"
-#include "paging.h"
 #include "timer.h"
 #include "vga.h"
 
@@ -92,7 +91,7 @@ void program_gfxbench_main(void)
 	io_putstr("gfxbench (sse2: ");
 	io_putstr(cpu_has_sse2() ? "yes" : "no");
 	io_putstr(", lfb wc: ");
-	io_putstr(paging_lfb_wc() ? "yes" : "no");
+	io_putstr(vga_fb_lfb_wc() ? "yes" : "no");
 	io_putstr(")\n");
 
 	layer_px = alloc_pages(layer_bytes);
@@ -159,7 +158,9 @@ void program_gfxbench_main(void)
 
 	vga_set_target(0);
 
-	/* Flush: shadow -> VRAM (bus bound, not CPU bound).  Report the bytes
+	/* Flush: shadow -> VRAM.  This is the copy the WM runs; it uses
+	 * streaming stores because rep movsl does not combine on the VRAM
+	 * path.  Report the bytes
 	 * the presenter really copied: a long flush can be preempted, and a
 	 * flush started while another process is inside one is skipped. */
 	{

+ 59 - 0
nvidia.c

@@ -22,6 +22,8 @@
 #include "pci.h"
 #include "process.h"
 #include "serial.h"
+#include "paging.h"
+#include "vga.h"
 
 #define NV_VENDOR_ID     0x10DE
 #define NV_BAR0_SIZE     0x1000000   /* register aperture (16 MB on nv50+) */
@@ -249,3 +251,60 @@ unsigned int nvidia_vram_size(void)
 {
 	return nv_vram_size;
 }
+
+/* The VBE LFB is a BAR3 window into the stolen VRAM, which is ordinary DRAM:
+ * writing it through the PCIe aperture costs ~33 MB/s (separate uncached
+ * transactions, see the MTRR notes), while the DRAM alias runs at DRAM
+ * speed.  Find where the framebuffer lives in the stolen VRAM by writing a
+ * marker through the LFB and scanning the VRAM at page granularity (the LFB
+ * and the frame are page-aligned, so the marker can only start on a page). */
+unsigned int nvidia_lfb_in_vram(void)
+{
+	static const unsigned int mark[4] = { 0xA5C3F00Du, 0x5A3C0FF0u,
+					      0xDEADBEEFu, 0x12345678u };
+	unsigned int vram = nv_vram_base, vsize = nv_vram_size;
+	unsigned int lfb = (unsigned int)vga_fb_lfb_addr();
+	unsigned int off, i, hit = 0;
+
+	if (!vram || vsize < 0x01000000u || !nv_bar3 || !nv_bar3_size)
+		return 0;
+	if (lfb < nv_bar3 || lfb > nv_bar3 + nv_bar3_size - 0x1000u)
+		return 0;          /* the VBE LFB is not in the VRAM aperture */
+	if (nv_bar3_size < 0x00801000u)
+		return 0;          /* no room for a marker past the window */
+
+	paging_map_ram(vram, vsize);
+
+	/* Marker at LFB + 8 MB: past any 1280x1024x32 frame. */
+	{
+		volatile unsigned int *p =
+			(volatile unsigned int *)(lfb + 0x00800000u);
+
+		for (i = 0; i < 4; i++)
+			p[i] = mark[i];
+	}
+
+	for (off = 0; off + 0x00800010u <= vsize; off += 0x1000u) {
+		const unsigned int *q = (const unsigned int *)(vram + off);
+
+		if (q[0] == mark[0] && q[1] == mark[1] &&
+		    q[2] == mark[2] && q[3] == mark[3]) {
+			hit = off;
+			break;
+		}
+	}
+
+	/* Clear the marker again (off-screen VRAM). */
+	{
+		volatile unsigned int *p =
+			(volatile unsigned int *)(lfb + 0x00800000u);
+
+		for (i = 0; i < 4; i++)
+			p[i] = 0;
+	}
+
+	if (!hit)
+		return 0;
+	/* The marker was at frame + 8 MB. */
+	return vram + (hit - 0x00800000u);
+}

+ 7 - 0
nvidia.h

@@ -36,4 +36,11 @@ unsigned int nvidia_bar3_size(void);
 unsigned int nvidia_vram_base(void);
 unsigned int nvidia_vram_size(void);
 
+/* The VBE LFB is a window into the stolen VRAM, which is ordinary DRAM and
+ * therefore much faster to write than the PCIe aperture.  Locates the
+ * framebuffer in the stolen VRAM by writing a marker through the LFB and
+ * scanning the (now cacheably mapped) VRAM at page granularity.  Returns the
+ * framebuffer's address in the stolen VRAM, or 0 when not applicable. */
+unsigned int nvidia_lfb_in_vram(void);
+
 #endif

+ 12 - 0
paging.c

@@ -72,6 +72,18 @@ int paging_lfb_wc(void)
 	return lfb_wc_ok;
 }
 
+void paging_map_ram(unsigned int base, unsigned int size)
+{
+	unsigned int p0 = base >> 22;
+	unsigned int p1 = (base + size + 0x3FFFFFu) >> 22;
+	unsigned int i;
+
+	if (!page_dir)
+		return;
+	for (i = p0; i < p1 && i < 1024; i++)
+		page_dir[i] = (i << 22) | PDE_PS | PDE_WRITABLE | PDE_PRESENT;
+}
+
 void paging_init(void)
 {
 	unsigned int i;

+ 5 - 0
paging.h

@@ -8,6 +8,11 @@ void paging_lfb_refresh(void);
 /* 1 when the LFB is mapped for write-combining. */
 int paging_lfb_wc(void);
 
+/* Map a physical window identity-mapped with caching enabled (4 MB pages).
+ * Used for the stolen-VRAM alias, which is ordinary DRAM; the effective type
+ * is whatever the MTRRs say (write-combining when we added an entry). */
+void paging_map_ram(unsigned int base, unsigned int size);
+
 /* Physical address of the bootstrap (current) page directory. */
 unsigned int paging_current_cr3(void);
 

+ 51 - 1
vga.c

@@ -5,6 +5,9 @@
 #include "pci.h"
 #include "gpu.h"
 #include "ati.h"
+#include "cpu.h"
+#include "mtrr.h"
+#include "nvidia.h"
 #include "paging.h"
 #include "process.h"
 #include "serial.h"
@@ -294,6 +297,9 @@ int vga_has_vesa(void)
  * Kept in sync by vga_shadow_init() invalidating it. */
 static struct surface screen_surf;
 
+/* 1 when the flush writes VRAM through a write-combining alias. */
+static int lfb_is_wc;
+
 struct surface *vga_screen_surface(void)
 {
 	vga_cache_boot_fb();
@@ -484,6 +490,36 @@ void vga_shadow_init(void)
 	 * framebuffer mapping is re-applied once everything is up. */
 	paging_lfb_refresh();
 
+	/* On ION-class hardware the framebuffer is stolen DRAM and the VBE LFB
+	 * is just a BAR3 window into it.  The BIOS leaves a 512 MB uncacheable
+	 * MTRR over the PCIe apertures (and the lowest-numbered matching entry
+	 * wins), so the aperture can never be write-combining: every dword is a
+	 * separate transaction (~33 MB/s).  Write through the DRAM alias
+	 * instead, with a WC MTRR over the stolen range -- and only switch once
+	 * that entry is in place, otherwise the alias would be cached and the
+	 * screen would not update. */
+	if (gpu_vendor() == GPU_VENDOR_NVIDIA) {
+		unsigned int alias = nvidia_lfb_in_vram();
+
+		if (alias) {
+			unsigned int wb, ws;
+
+			if (mtrr_set_wc(nvidia_vram_base(),
+					nvidia_vram_size(), &wb, &ws)) {
+				vga_fb_lfb = (volatile unsigned char *)alias;
+				lfb_is_wc = 1;
+				serial_puts(SERIAL_COM1,
+					"vga: LFB via stolen VRAM (write-combining)\n");
+			} else {
+				serial_puts(SERIAL_COM1,
+					"vga: LFB alias found, no WC entry; staying on BAR3\n");
+			}
+		} else {
+			serial_puts(SERIAL_COM1,
+				"vga: LFB has no stolen-VRAM alias; using BAR3\n");
+		}
+	}
+
 	vga_shadow_buf = (unsigned char *)VGA_SHADOW_BASE;
 	vga_shadow_pitch = vga_fb_pitch_val;
 	vga_shadow_size = vga_fb_h * vga_shadow_pitch;
@@ -504,6 +540,7 @@ void vga_shadow_init(void)
 static unsigned int shadow_copy_damage(unsigned int max_bytes)
 {
 	unsigned int i = 0, bytes = 0;
+	int use_sse2 = cpu_has_sse2();
 	unsigned int pb = vga_pixel_bytes();
 	unsigned int align = (pb == 2) ? 2 : (pb == 3) ? 4 : 1;
 	unsigned char *src = vga_shadow_buf;
@@ -535,7 +572,13 @@ static unsigned int shadow_copy_damage(unsigned int max_bytes)
 			unsigned char *s = src + (unsigned long)y * vga_shadow_pitch + rx0 * pb;
 			unsigned char *t = dst + (unsigned long)y * vga_fb_pitch_val + rx0 * pb;
 
-			vga_fb_copy_row(t, s, dwords);
+			/* Streaming stores: VRAM is WC (or UC on machines where the
+			 * MTRR cannot cover it) and rep movsl does not combine
+			 * there, while movntdq does. */
+			if (use_sse2)
+				vga_fb_copy_row_nt(t, s, dwords);
+			else
+				vga_fb_copy_row(t, s, dwords);
 			if (tail) {
 				unsigned int k;
 				for (k = dwords * 4; k < row_bytes; k++)
@@ -556,6 +599,8 @@ static unsigned int shadow_copy_damage(unsigned int max_bytes)
 			return bytes;
 		}
 		damage_rects[i] = damage_rects[--damage_count];
+		if (use_sse2)
+			__asm__ volatile ("sfence" ::: "memory");
 		if (max_bytes && bytes >= max_bytes)
 			break;
 	}
@@ -625,6 +670,11 @@ void vga_fb_flush(void)
 	vga_fb_flush_budget(0);
 }
 
+int vga_fb_lfb_wc(void)
+{
+	return lfb_is_wc;
+}
+
 int vga_shadow_is_active(void)
 {
 	return vga_shadow_buf != 0;

+ 6 - 0
vga.h

@@ -68,6 +68,11 @@ void vga_target_damage(unsigned int x, unsigned int y, unsigned int w,
  * the pending damage rects from the shadow buffer to VRAM. */
 void vga_present_shadow(unsigned int max_bytes);
 
+/* 1 when VRAM writes go through a write-combining path (the stolen-VRAM
+ * alias on ION-class hardware, where the PCIe aperture is UC no matter what
+ * the MTRR says). */
+int vga_fb_lfb_wc(void);
+
 /* Bytes the presenter has copied in total (lets a benchmark measure the real
  * work even when the flush is skipped because another process is in one). */
 extern unsigned int vga_present_copied;
@@ -97,6 +102,7 @@ void vga_fb_flush_budget(unsigned int max_bytes);
 
 /* Low-level row helpers from vga_copy.S (the blit backend uses the fill). */
 void vga_fb_copy_row(void *dst, const void *src, unsigned int dwords);
+void vga_fb_copy_row_nt(void *dst, const void *src, unsigned int dwords);
 void vga_fb_fill_row_sse2(void *dst, unsigned int value, unsigned int dwords);
 void vga_fb_fill_row_nt(void *dst, unsigned int value, unsigned int dwords);
 

+ 43 - 0
vga_copy.S

@@ -87,3 +87,46 @@ vga_fb_fill_row_nt:
 	jnz   5b
 9:	popl  %edi
 	ret
+
+/* void vga_fb_copy_row_nt(void *dst, const void *src, unsigned int dwords);
+ *
+ * Same as vga_fb_copy_row, but with streaming (non-temporal) stores: WC
+ * destinations combine them into bursts, which rep movsl does not do here.
+ * The destination is aligned with normal stores first; the caller sfences. */
+.globl vga_fb_copy_row_nt
+vga_fb_copy_row_nt:
+	pushl %esi
+	pushl %edi
+	movl  12(%esp), %edi	/* dst */
+	movl  16(%esp), %esi	/* src */
+	movl  20(%esp), %ecx	/* dword count */
+1:	testl $15, %edi
+	jz    2f
+	testl %ecx, %ecx
+	jz    9f
+	movl  (%esi), %eax
+	movl  %eax, (%edi)
+	addl  $4, %esi
+	addl  $4, %edi
+	decl  %ecx
+	jmp   1b
+2:	movl  %ecx, %edx
+	shrl  $2, %edx
+	jz    4f
+3:	movdqu (%esi), %xmm0
+	movntdq %xmm0, (%edi)
+	addl  $16, %esi
+	addl  $16, %edi
+	decl  %edx
+	jnz   3b
+4:	andl  $3, %ecx
+	jz    9f
+5:	movl  (%esi), %eax
+	movl  %eax, (%edi)
+	addl  $4, %esi
+	addl  $4, %edi
+	decl  %ecx
+	jnz   5b
+9:	popl  %edi
+	popl  %esi
+	ret