[Vm-dev] [commit][3017] Rewrite Spur memory allocation on win32 similarly to unix.

commits at squeakvm.org commits at squeakvm.org
Wed Jun 25 20:21:14 UTC 2014


Revision: 3017
Author:   eliot
Date:     2014-06-25 13:21:12 -0700 (Wed, 25 Jun 2014)
Log Message:
-----------
Rewrite Spur memory allocation on win32 similarly to unix.  Can now allocate
up to 1.8Gb on Windows XP (which has a 2Gb address space limit).  Add a flag
to indicate if the win32 exe is running as a console app and don't write to
the in-window console if so.

Fix a slip in sqWin32VMProfile.c.
Fix access to revisionAsString in sqUnixHeartbeat.c (it should include
sqSCCSVersion.h) and revert revisionAsString to static.

Modified Paths:
--------------
    branches/Cog/platforms/Cross/vm/sqSCCSVersion.h
    branches/Cog/platforms/unix/vm/sqUnixHeartbeat.c
    branches/Cog/platforms/win32/vm/sqWin32.h
    branches/Cog/platforms/win32/vm/sqWin32Alloc.c
    branches/Cog/platforms/win32/vm/sqWin32Intel.c
    branches/Cog/platforms/win32/vm/sqWin32VMProfile.c

Added Paths:
-----------
    branches/Cog/platforms/win32/vm/sqWin32SpurAlloc.c

Property Changed:
----------------
    branches/Cog/platforms/Cross/vm/sqSCCSVersion.h

Modified: branches/Cog/platforms/Cross/vm/sqSCCSVersion.h
===================================================================
--- branches/Cog/platforms/Cross/vm/sqSCCSVersion.h	2014-06-24 18:38:36 UTC (rev 3016)
+++ branches/Cog/platforms/Cross/vm/sqSCCSVersion.h	2014-06-25 20:21:12 UTC (rev 3017)
@@ -33,7 +33,7 @@
 static char SvnRawRepositoryURL[] = "$URL$";
 # define URL_START (SvnRawRepositoryURL + 6)
 
-char *
+static char *
 revisionAsString()
 {
 	char *maybe_space = strchr(REV_START,' ');


Property changes on: branches/Cog/platforms/Cross/vm/sqSCCSVersion.h
___________________________________________________________________
Modified: checkindate
   - Tue Jun 24 11:38:03 PDT 2014
   + Wed Jun 25 13:16:16 PDT 2014

Modified: branches/Cog/platforms/unix/vm/sqUnixHeartbeat.c
===================================================================
--- branches/Cog/platforms/unix/vm/sqUnixHeartbeat.c	2014-06-24 18:38:36 UTC (rev 3016)
+++ branches/Cog/platforms/unix/vm/sqUnixHeartbeat.c	2014-06-25 20:21:12 UTC (rev 3017)
@@ -26,6 +26,7 @@
 #include "sq.h"
 #include "sqAssert.h"
 #include "sqMemoryFence.h"
+#include "sqSCCSVersion.h"
 #include <errno.h>
 #include <pthread.h>
 #include <stdio.h> /* for fprintf */

Modified: branches/Cog/platforms/win32/vm/sqWin32.h
===================================================================
--- branches/Cog/platforms/win32/vm/sqWin32.h	2014-06-24 18:38:36 UTC (rev 3016)
+++ branches/Cog/platforms/win32/vm/sqWin32.h	2014-06-25 20:21:12 UTC (rev 3017)
@@ -283,6 +283,7 @@
 extern BITMAPINFO *bmi16;	     /*	16 bit depth bitmap info */
 extern BITMAPINFO *bmi32;	     /*	32 bit depth bitmap info */
 extern BOOL fWindows95;          /* Are we running on Win95 or NT? */
+extern BOOL fIsConsole;          /* Are we running as a console app? */
 
 /* Startup options */
 extern BOOL  fHeadlessImage; /* Do we run headless? */

Modified: branches/Cog/platforms/win32/vm/sqWin32Alloc.c
===================================================================
--- branches/Cog/platforms/win32/vm/sqWin32Alloc.c	2014-06-24 18:38:36 UTC (rev 3016)
+++ branches/Cog/platforms/win32/vm/sqWin32Alloc.c	2014-06-25 20:21:12 UTC (rev 3017)
@@ -13,7 +13,7 @@
 #include <windows.h>
 #include "sq.h"
 
-#ifndef NO_VIRTUAL_MEMORY
+#if !defined(NO_VIRTUAL_MEMORY) && !SPURVM /* Spur uses sqWin32SpurAlloc.c */
 
 /* For Qwaq Forums: Disallow memory shrinking to avoid crashes
    due to GC/OpenGL relocation problems within glDrawElements.
@@ -176,12 +176,11 @@
   }
   return bytesLeft;
 }
-#endif /* NO_VIRTUAL_MEMORY */
 
 #define roundDownToPage(v) ((v)&pageMask)
 #define roundUpToPage(v) (((v)+pageSize-1)&pageMask)
 
-#if COGVM
+# if COGVM
 void
 sqMakeMemoryExecutableFromTo(unsigned long startAddr, unsigned long endAddr)
 {
@@ -205,70 +204,5 @@
 						&previous))
 		perror("VirtualProtect(x,y,PAGE_EXECUTE_READWRITE)");
 }
-#endif /* COGVM */
-
-#if SPURVM
-/* Allocate a region of memory of at least size bytes, at or above minAddress.
- *  If the attempt fails, answer null.  If the attempt succeeds, answer the
- * start of the region and assign its size through allocatedSizePointer.
- *
- * This from the VirtualFree doc is rather scary:
-	dwSize [in]
-
-		The size of the region of memory to be freed, in bytes.
-
-		If the dwFreeType parameter is MEM_RELEASE, this parameter must be 0
-		(zero). The function frees the entire region that is reserved in the
-		initial allocation call to VirtualAlloc.
- *
- * So we rely on the SpurMemoryManager to free exactly the segments that were
- * allocated.
- */
-#define SizeForRelease(bytes) 0
-void *
-sqAllocateMemorySegmentOfSizeAboveAllocatedSizeInto(sqInt size, void *minAddress, sqInt *allocatedSizePointer)
-{
-	void *alloc;
-	long bytes = roundUpToPage(size);
-
-	*allocatedSizePointer = bytes;
-#if 0 /* It appears VirtualAlloc answers low memory by default. */
-	alloc = VirtualAlloc(0, bytes, MEM_COMMIT, PAGE_READWRITE);
-	if (!alloc) {
-		sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Error:"),
-					"Unable to VirtualAlloc committed memory (%d bytes requested), Error: %ul",
-					bytes, GetLastError());
-		return NULL;
-	}
-	if ((unsigned long)alloc >= (unsigned long)minAddress)
-		return alloc;
-	if (!VirtualFree(alloc, SizeForRelease(bytes), MEM_RELEASE))
-		sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Warning:"),
-					"Unable to VirtualFree committed memory (%d bytes requested), Error: %ul",
-					bytes, GetLastError());
-#endif /* 0 */
-	alloc = VirtualAlloc(0, bytes, MEM_COMMIT+MEM_TOP_DOWN, PAGE_READWRITE);
-	if ((unsigned long)alloc >= (unsigned long)minAddress)
-		return alloc;
-	if (alloc && !VirtualFree(alloc, SizeForRelease(bytes), MEM_RELEASE))
-		sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Warning:"),
-					"Unable to VirtualFree committed memory (%d bytes requested), Error: %ul",
-					bytes, GetLastError());
-	sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Error:"),
-				"Unable to VirtualAlloc committed memory at desired address (%d bytes requested at or above %p), Error: %ul",
-				bytes, minAddress, GetLastError());
-	return NULL;
-}
-
-/* Deallocate a region of memory previously allocated by
- * sqAllocateMemorySegmentOfSizeAboveAllocatedSizeInto.  Cannot fail.
- */
-void
-sqDeallocateMemorySegmentAtOfSize(void *addr, sqInt sz)
-{
-	if (!VirtualFree(addr, SizeForRelease(sz), MEM_RELEASE))
-		sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Warning:"),
-					"Unable to VirtualFree committed memory (%d bytes requested), Error: %ul",
-					sz, GetLastError());
-}
-#endif /* SPURVM */
+# endif /* COGVM */
+#endif /* !defined(NO_VIRTUAL_MEMORY) && !SPURVM */

Modified: branches/Cog/platforms/win32/vm/sqWin32Intel.c
===================================================================
--- branches/Cog/platforms/win32/vm/sqWin32Intel.c	2014-06-24 18:38:36 UTC (rev 3016)
+++ branches/Cog/platforms/win32/vm/sqWin32Intel.c	2014-06-25 20:21:12 UTC (rev 3017)
@@ -63,6 +63,7 @@
 static char **clargv;
 
 /* console buffer */
+BOOL fIsConsole = 0;
 TCHAR consoleBuffer[4096];
 
 /* stderr and stdout names */
@@ -231,8 +232,10 @@
 { va_list al;
 
   va_start(al, fmt);
-  wvsprintf(consoleBuffer, fmt, al);
-  OutputDebugString(consoleBuffer);
+  if (!fIsConsole) {
+	wvsprintf(consoleBuffer, fmt, al);
+	OutputDebugString(consoleBuffer);
+  }
   vfprintf(stdout, fmt, al);
   va_end(al);
   return 1;
@@ -248,10 +251,12 @@
   int result;
 
   va_start(al, fmt);
-  wvsprintf(consoleBuffer, fmt, al);
-  OutputLogMessage(consoleBuffer);
-  if(IsWindow(stWindow)) /* not running as service? */
-    OutputConsoleString(consoleBuffer);
+  if (!fIsConsole) {
+	wvsprintf(consoleBuffer, fmt, al);
+	OutputLogMessage(consoleBuffer);
+	if(IsWindow(stWindow)) /* not running as service? */
+	  OutputConsoleString(consoleBuffer);
+  }
   result = vfprintf(stdout, fmt, al);
   va_end(al);
   return result;
@@ -263,7 +268,7 @@
   int result;
 
   va_start(al, fmt);
-  if(fp == stdout || fp == stderr)
+  if(!fIsConsole && (fp == stdout || fp == stderr))
     {
       wvsprintf(consoleBuffer, fmt, al);
       OutputLogMessage(consoleBuffer);
@@ -277,10 +282,7 @@
 
 
 int __cdecl
-putchar(int c)
-{
-  return printf("%c",c);
-}
+putchar(int c) { return printf("%c",c); }
 
 #endif /* !defined(_MSC_VER) && !defined(NODBGPRINT) */
 
@@ -1510,11 +1512,18 @@
 int WINAPI
 WinMain(HINSTANCE hInst, HINSTANCE hPrevInstance, LPSTR lpCmdLine, int nCmdShow)
 {
+  DWORD mode;
+
   /* a few things which need to be done first */
   gatherSystemInfo();
 
   /* check if we're running NT or 95 */
   fWindows95 = (GetVersion() & 0x80000000) != 0;
+  /* Determine if we're running as a console application  We can't report
+   * allocation failures unless running as a console app because doing so
+   * via a MessageBox will make the system unusable.
+   */
+  fIsConsole = GetConsoleMode(GetStdHandle(STD_OUTPUT_HANDLE), &mode);
 
   /* fetch us a copy of the command line */
   initialCmdLine = _strdup(lpCmdLine);

Added: branches/Cog/platforms/win32/vm/sqWin32SpurAlloc.c
===================================================================
--- branches/Cog/platforms/win32/vm/sqWin32SpurAlloc.c	                        (rev 0)
+++ branches/Cog/platforms/win32/vm/sqWin32SpurAlloc.c	2014-06-25 20:21:12 UTC (rev 3017)
@@ -0,0 +1,286 @@
+/****************************************************************************
+*   PROJECT: Squeak port for Win32 (NT / Win95)
+*   FILE:    sqWin32SpurAlloc.c
+*   CONTENT: Virtual Memory Management For Spur
+*
+*   AUTHOR:  Eliot Miranda
+*   EMAIL:   eliot.miranda at gmail.com
+*
+*****************************************************************************/
+
+#include <windows.h>
+#include "sq.h"
+
+#if SPURVM /* Non-spur uses sqWin32Alloc.c */
+
+/* Why does this have to be *here*?? eem 6/24/2014 */
+#if !defined(NDEBUG)
+/* in debug mode, let the system crash so that we can see where it happened */
+#define EXCEPTION_WRONG_ACCESS EXCEPTION_CONTINUE_SEARCH
+#else
+/* in release mode, execute the exception handler notifying the user what happened */
+#define EXCEPTION_WRONG_ACCESS EXCEPTION_EXECUTE_HANDLER
+#endif
+
+LONG CALLBACK sqExceptionFilter(LPEXCEPTION_POINTERS exp)
+{
+  /* always wrong access - we handle memory differently now */
+  return EXCEPTION_WRONG_ACCESS;
+}
+
+static DWORD  pageMask;     /* bit mask for the start of a memory page */
+static DWORD  pageSize;     /* size of a memory page */
+static char  *minAppAddr;	/* SYSTEM_INFO lpMinimumApplicationAddress */
+static char  *maxAppAddr;	/* SYSTEM_INFO lpMaximumApplicationAddress */
+
+# define roundDownToPage(v) ((v)&pageMask)
+# define roundUpToPage(v) (((v)+pageSize-1)&pageMask)
+
+/************************************************************************/
+/* sqAllocateMemory: Initialize virtual memory                          */
+/************************************************************************/
+void *
+sqAllocateMemory(usqInt minHeapSize, usqInt desiredHeapSize)
+{
+	char *hint, *address, *alloc;
+	unsigned long alignment;
+	sqInt allocBytes;
+	SYSTEM_INFO sysInfo;
+
+	if (pageSize) {
+		sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Error:"),
+					 "sqAllocateMemory already called");
+		exit(1);
+	}
+
+	/* determine page boundaries & available address space */
+	GetSystemInfo(&sysInfo);
+	pageSize = sysInfo.dwPageSize;
+	pageMask = ~(pageSize - 1);
+	minAppAddr = sysInfo.lpMinimumApplicationAddress;
+	maxAppAddr = sysInfo.lpMaximumApplicationAddress;
+
+	/* choose a suitable starting point. In MinGW the malloc heap is below the
+	 * program, so take the max of a malloc and something form uninitialized
+	 * data.
+	 */
+	hint = malloc(1);
+	free(hint);
+	hint = max(hint,(char *)&fIsConsole);
+
+	alignment = max(pageSize,1024*1024);
+	address = (char *)(((usqInt)hint + alignment - 1) & ~(alignment - 1));
+
+	alloc = sqAllocateMemorySegmentOfSizeAboveAllocatedSizeInto
+				(roundUpToPage(desiredHeapSize), address, &allocBytes);
+	if (!alloc) {
+		exit(errno);
+		sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Error:"),
+					 "sqAllocateMemory: initial alloc failed!\n");
+		exit(1);
+	}
+	return alloc;
+}
+
+#define roundDownToPage(v) ((v)&pageMask)
+#define roundUpToPage(v) (((v)+pageSize-1)&pageMask)
+
+/* Allocate a region of memory of at least size bytes, at or above minAddress.
+ *  If the attempt fails, answer null.  If the attempt succeeds, answer the
+ * start of the region and assign its size through allocatedSizePointer.
+ *
+ * This from the VirtualFree doc is rather scary:
+	dwSize [in]
+
+		The size of the region of memory to be freed, in bytes.
+
+		If the dwFreeType parameter is MEM_RELEASE, this parameter must be 0
+		(zero). The function frees the entire region that is reserved in the
+		initial allocation call to VirtualAlloc.
+ *
+ * So we rely on the SpurMemoryManager to free exactly the segments that were
+ * allocated.
+ */
+#define SizeForRelease(bytes) 0
+
+static int
+address_space_used(char *address, unsigned long bytes)
+{
+	MEMORY_BASIC_INFORMATION info;
+	int addressSpaceUnused;
+
+	if (address < minAppAddr || address > maxAppAddr)
+		return 1;
+	if (!VirtualQuery(address, &info, sizeof(info)))
+		sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Error:"),
+					"Unable to VirtualQuery range [%p, %p), Error: %u",
+					address, (char *)address + bytes, GetLastError());
+
+	addressSpaceUnused = info.BaseAddress == address
+						&& info.RegionSize >= bytes
+						&& info.State == MEM_FREE;
+
+	return !addressSpaceUnused;
+}
+
+void *
+sqAllocateMemorySegmentOfSizeAboveAllocatedSizeInto(sqInt size, void *minAddress, sqInt *allocatedSizePointer)
+{
+	char *address, *alloc;
+	long bytes, delta;
+
+	address = (char *)roundUpToPage((unsigned long)minAddress);
+	bytes = roundUpToPage(size);
+	delta = max(pageSize,1024*1024);
+
+# define printProbes 0
+# define printMaps 0
+	while ((unsigned long)(address + bytes) > (unsigned long)address) {
+		if (printProbes && fIsConsole)
+			printf("probing [%p,%p)\n", address, address + bytes);
+		if (address_space_used(address, bytes)) {
+			address += delta;
+			continue;
+		}
+		alloc = VirtualAlloc(address, bytes, MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
+		/* For some reason (large page support?) we can ask for a page-aligned
+		 * address such as 0xNNNNf000 but VirtualAlloc will answer 0xNNNN0000.
+		 * So accept allocs above minAddress rather than allocs above address
+		 */
+		if (alloc >= minAddress && alloc <= address + delta) {
+			if (printMaps && fIsConsole)
+				fprintf(stderr,
+						"VirtualAlloc [%p,%p) above %p)\n",
+						address, address+bytes, minAddress);
+			*allocatedSizePointer = bytes;
+			return alloc;
+		}
+		if (!alloc) {
+			DWORD lastError = GetLastError();
+#if 0 /* Can't report this without making the system unusable... */
+			sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Error:"),
+						"Unable to VirtualAlloc committed memory at desired address (%d bytes requested at %p, above %p), Error: %ul",
+						bytes, address, minAddress, lastError);
+#else
+			if (fIsConsole)
+				fprintf(stderr,
+						"Unable to VirtualAlloc committed memory at desired address (%d bytes requested at %p, above %p), Error: %ul\n",
+						bytes, address, minAddress, lastError);
+#endif
+			return 0;
+		}
+		/* VirtualAlloc answered a mapping well away from where Spur prefers.
+		 * Discard the mapping and try again delta higher.
+		 */
+		if (alloc && !VirtualFree(alloc, SizeForRelease(bytes), MEM_RELEASE))
+			sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Warning:"),
+						"Unable to VirtualFree committed memory (%d bytes requested), Error: %ul",
+						bytes, GetLastError());
+		address += delta;
+	}
+	return 0;
+}
+
+/* Deallocate a region of memory previously allocated by
+ * sqAllocateMemorySegmentOfSizeAboveAllocatedSizeInto.  Cannot fail.
+ */
+void
+sqDeallocateMemorySegmentAtOfSize(void *addr, sqInt sz)
+{
+	if (!VirtualFree(addr, SizeForRelease(sz), MEM_RELEASE))
+		sqMessageBox(MB_OK | MB_ICONSTOP, TEXT("VM Warning:"),
+					"Unable to VirtualFree committed memory (%d bytes requested), Error: %ul",
+					sz, GetLastError());
+}
+
+# if COGVM
+void
+sqMakeMemoryExecutableFromTo(unsigned long startAddr, unsigned long endAddr)
+{
+	DWORD previous;
+
+	if (!VirtualProtect((void *)startAddr,
+						endAddr - startAddr + 1,
+						PAGE_EXECUTE_READWRITE,
+						&previous))
+		perror("VirtualProtect(x,y,PAGE_EXECUTE_READWRITE)");
+}
+
+void
+sqMakeMemoryNotExecutableFromTo(unsigned long startAddr, unsigned long endAddr)
+{
+	DWORD previous;
+
+	if (!VirtualProtect((void *)startAddr,
+						endAddr - startAddr + 1,
+						PAGE_READWRITE,
+						&previous))
+		perror("VirtualProtect(x,y,PAGE_EXECUTE_READWRITE)");
+}
+# endif /* COGVM */
+
+# if TEST_MEMORY
+
+#	define MBytes	*1024UL*1024UL
+
+BOOL fIsConsole = 1;
+
+int
+main()
+{
+	char *mem;
+	usqInt i, t = 16 MBytes;
+
+	mem= (char *)sqAllocateMemory(t, t);
+	printf("memory allocated at %p\n", mem);
+	*mem = 1;
+	/* create some roadbumps */
+	for (i = 80 MBytes; i < 2048UL MBytes; i += 80 MBytes) {
+		void *alloc = VirtualAlloc(mem + i, pageSize, MEM_COMMIT|MEM_RESERVE, PAGE_READWRITE);
+		printf("roadbump created at %p (%p)\n", mem + i, alloc);
+		*(char *)alloc = 1;
+	}
+	for (;;) {
+		sqInt segsz = 0;
+		char *seg = sqAllocateMemorySegmentOfSizeAboveAllocatedSizeInto(32 MBytes, mem + 16 MBytes, &segsz);
+		if (!seg)
+			return 0;
+		*seg = 1;
+		t += segsz;
+		printf("memory extended at %p (total %ld Mb)\n", seg, t / (1 MBytes));
+	}
+	return 0;
+}
+int __cdecl
+sqMessageBox(DWORD dwFlags, const TCHAR *titleString, const char* fmt, ...)
+{
+	va_list args;
+	int result;
+	char buf[1024];
+
+	strcpy(buf, titleString);
+	strcat(buf, fmt);
+	strcat(buf, "\n");
+	va_start(args, fmt);
+#if 0
+	result = vfprintf(stderr, buf, args);
+#else
+	result = vprintf(buf, args);
+#endif
+	va_end(args);
+	printLastError((char *)titleString);
+	return result;
+}
+void printLastError(TCHAR *prefix)
+{ LPVOID lpMsgBuf;
+  DWORD lastError;
+
+  lastError = GetLastError();
+  FormatMessage(FORMAT_MESSAGE_ALLOCATE_BUFFER |  FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_IGNORE_INSERTS,
+                NULL, GetLastError(), MAKELANGID(LANG_NEUTRAL, SUBLANG_DEFAULT),
+                (LPTSTR) &lpMsgBuf, 0, NULL );
+  fprintf(stderr,TEXT("%s (%d) -- %s\n"), prefix, lastError, lpMsgBuf);
+  LocalFree( lpMsgBuf );
+}
+# endif /* TEST_MEMORY */
+#endif /* SPURVM */

Modified: branches/Cog/platforms/win32/vm/sqWin32VMProfile.c
===================================================================
--- branches/Cog/platforms/win32/vm/sqWin32VMProfile.c	2014-06-24 18:38:36 UTC (rev 3016)
+++ branches/Cog/platforms/win32/vm/sqWin32VMProfile.c	2014-06-25 20:21:12 UTC (rev 3017)
@@ -54,8 +54,8 @@
  * entire address space.
  */
 
-extern  void btext(void);
-extern  void etext(void);
+extern  void *btext(void);
+extern  void *etext(void);
 
 void
 ioProfileTextRange(void **startpc, void **endpc)



More information about the Vm-dev mailing list