Add complete multidraw

api/src/main/java/com/seibel/distanthorizons/api/interfaces/config/client/IDhApiGpuBuffersConfig.java

@@ -36,13 +36,4 @@ public interface IDhApiGpuBuffersConfig extends IDhApiConfigGroup
 	/** Defines how geometry data is uploaded to the GPU. */
 	IDhApiConfigValue<EDhApiGpuUploadMethod> gpuUploadMethod();
 	
-	/**
-	 * Defines how long we should wait after uploading one
-	 * Megabyte of geometry data to the GPU before uploading
-	 * the next Megabyte of data. <br>
-	 * This can be set to a non-zero number to reduce stuttering caused by
-	 * uploading buffers to the GPU.
-	 */
-	IDhApiConfigValue<Integer> gpuUploadPerMegabyteInMilliseconds();
-	
 }

api/src/main/java/com/seibel/distanthorizons/coreapi/util/StringUtil.java

@@ -19,7 +19,8 @@
 
 package com.seibel.distanthorizons.coreapi.util;
 
-import java.util.ArrayList;
+import java.text.CharacterIterator;
+import java.text.StringCharacterIterator;
 import java.util.Arrays;
 
 /**
@@ -85,4 +86,25 @@ public class StringUtil
 		return new String(hexChars);
 	}
 	
+	/**
+	 * Source:
+	 * https://stackoverflow.com/questions/3758606/how-can-i-convert-byte-size-into-a-human-readable-format-in-java#3758880
+	 */
+	public static String convertByteCountToHumanReadableSI(long bytes)
+	{
+		if (-1000 < bytes && bytes < 1000)
+		{
+			return bytes + " B";
+		}
+		
+		CharacterIterator ci = new StringCharacterIterator("kMGTPE");
+		while (bytes <= -999_950 || bytes >= 999_950)
+		{
+			bytes /= 1000;
+			ci.next();
+		}
+		
+		return String.format("%.1f %cB", bytes / 1000.0, ci.current());
+	}
+	
 }

core/src/main/java/com/seibel/distanthorizons/core/api/external/methods/config/client/DhApiGpuBuffersConfig.java

@@ -36,7 +36,4 @@ public class DhApiGpuBuffersConfig implements IDhApiGpuBuffersConfig
 	public IDhApiConfigValue<EDhApiGpuUploadMethod> gpuUploadMethod()
 	{ return new DhApiConfigValue<>(Config.Client.Advanced.GpuBuffers.gpuUploadMethod); }
 	
-	public IDhApiConfigValue<Integer> gpuUploadPerMegabyteInMilliseconds()
-	{ return new DhApiConfigValue<>(Config.Client.Advanced.GpuBuffers.gpuUploadPerMegabyteInMilliseconds); }
-	
 }

core/src/main/java/com/seibel/distanthorizons/core/config/Config.java

@@ -1054,20 +1054,6 @@ public class Config
 								+ "")
 						.build();
 				
-				public static ConfigEntry<Integer> gpuUploadPerMegabyteInMilliseconds = new ConfigEntry.Builder<Integer>()
-						.setMinDefaultMax(0, 0, 50)
-						.comment(""
-								+ "How long should a buffer wait per Megabyte of data uploaded? \n"
-								+ "Helpful resource for frame times: https://fpstoms.com \n"
-								+ "\n"
-								+ "Longer times may reduce stuttering but will make LODs \n"
-								+ "transition and load slower. Change this to [0] for no timeout. \n"
-								+ "\n"
-								+ "NOTE:\n"
-								+ "Before changing this config, try changing the \"GPU Upload method\" first. \n"
-								+ "")
-						.build();
-				
 			}
 			
 			public static class AutoUpdater

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBuffer.java

@@ -20,15 +20,11 @@
 package com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding;
 
 import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam;
-import com.seibel.distanthorizons.core.config.Config;
 import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.pos.DhBlockPos;
-import com.seibel.distanthorizons.core.render.glObject.GLProxy;
-import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer;
 import com.seibel.distanthorizons.core.render.renderer.LodRenderer;
 import com.seibel.distanthorizons.core.util.LodUtil;
-import com.seibel.distanthorizons.core.util.objects.StatsMap;
 import com.seibel.distanthorizons.api.enums.config.EDhApiGpuUploadMethod;
 import com.seibel.distanthorizons.core.util.*;
 import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
@@ -59,13 +55,12 @@ public class ColumnRenderBuffer implements AutoCloseable
 	
 	
 	
-	
 	public final DhBlockPos pos;
 	
 	public boolean buffersUploaded = false;
 	
-	private GLVertexBuffer[] vbos;
+	public SharedVbo.BufferSlice[] opaqueBufferSlices;
-	private GLVertexBuffer[] vbosTransparent;
+	public SharedVbo.BufferSlice[] transparentBufferSlices;
 	
 	
 	
@@ -76,14 +71,12 @@ public class ColumnRenderBuffer implements AutoCloseable
 	public ColumnRenderBuffer(DhBlockPos pos)
 	{
 		this.pos = pos;
-		this.vbos = new GLVertexBuffer[0];
+		this.opaqueBufferSlices = new SharedVbo.BufferSlice[0];
-		this.vbosTransparent = new GLVertexBuffer[0];
+		this.transparentBufferSlices = new SharedVbo.BufferSlice[0];
 	}
 	
 	
 	
-	
-	
 	//==================//
 	// buffer uploading //
 	//==================//
@@ -128,193 +121,132 @@ public class ColumnRenderBuffer implements AutoCloseable
 	}
 	private void uploadBuffersUsingUploadMethod(LodQuadBuilder builder, EDhApiGpuUploadMethod gpuUploadMethod) throws InterruptedException
 	{
-		if (gpuUploadMethod.useEarlyMapping)
+		//if (gpuUploadMethod.useEarlyMapping)
-		{
+		//{
-			this.uploadBuffersMapped(builder, gpuUploadMethod);
+		//	this.uploadBuffersMapped(builder, gpuUploadMethod);
-		}
+		//}
-		else
+		//else
-		{
+		//{
-			this.uploadBuffersDirect(builder, gpuUploadMethod);
+			this.uploadBuffersDirect(builder);
-		}
+		//}
 		
 		this.buffersUploaded = true;
 	}
 	
 	
 	
-	private void uploadBuffersMapped(LodQuadBuilder builder, EDhApiGpuUploadMethod method)
+	//private void uploadBuffersMapped(LodQuadBuilder builder, EDhApiGpuUploadMethod method)
+	//{
+	//	// opaque vbos //
+	//	
+	//	this.vbos = ColumnRenderBufferBuilder.resizeBuffer(this.vbos, builder.getCurrentNeededOpaqueVertexBufferCount());
+	//	for (int i = 0; i < this.vbos.length; i++)
+	//	{
+	//		if (this.vbos[i] == null)
+	//		{
+	//			this.vbos[i] = new GLVertexBuffer(method.useBufferStorage);
+	//		}
+	//	}
+	//	LodQuadBuilder.BufferFiller func = builder.makeOpaqueBufferFiller(method);
+	//	for (GLVertexBuffer vbo : this.vbos)
+	//	{
+	//		func.fill(vbo);
+	//	}
+	//	
+	//	
+	//	// transparent vbos //
+	//	
+	//	this.vbosTransparent = ColumnRenderBufferBuilder.resizeBuffer(this.vbosTransparent, builder.getCurrentNeededTransparentVertexBufferCount());
+	//	for (int i = 0; i < this.vbosTransparent.length; i++)
+	//	{
+	//		if (this.vbosTransparent[i] == null)
+	//		{
+	//			this.vbosTransparent[i] = new GLVertexBuffer(method.useBufferStorage);
+	//		}
+	//	}
+	//	LodQuadBuilder.BufferFiller transparentFillerFunc = builder.makeTransparentBufferFiller(method);
+	//	for (GLVertexBuffer vbo : this.vbosTransparent)
+	//	{
+	//		transparentFillerFunc.fill(vbo);
+	//	}
+	//}
+	
+	private void uploadBuffersDirect(LodQuadBuilder builder)
 	{
-		// opaque vbos //
+		int opaqueSliceCount = builder.getCurrentNeededOpaqueVertexBufferCount();
-		
+		if (this.opaqueBufferSlices.length != opaqueSliceCount)
-		this.vbos = ColumnRenderBufferBuilder.resizeBuffer(this.vbos, builder.getCurrentNeededOpaqueVertexBufferCount());
-		for (int i = 0; i < this.vbos.length; i++)
 		{
-			if (this.vbos[i] == null)
+			SharedVbo.OPAQUE.deallocate(this.opaqueBufferSlices);
-			{
+			this.opaqueBufferSlices = new SharedVbo.BufferSlice[opaqueSliceCount];
-				this.vbos[i] = new GLVertexBuffer(method.useBufferStorage);
-			}
-		}
-		LodQuadBuilder.BufferFiller func = builder.makeOpaqueBufferFiller(method);
-		for (GLVertexBuffer vbo : this.vbos)
-		{
-			func.fill(vbo);
 		}
+		uploadBuffersDirect(SharedVbo.OPAQUE, this.opaqueBufferSlices, builder.makeOpaqueVertexBuffers());
 		
 		
-		// transparent vbos //
+		int transparentSliceCount = builder.getCurrentNeededTransparentVertexBufferCount();
-		
+		if (this.transparentBufferSlices.length != transparentSliceCount)
-		this.vbosTransparent = ColumnRenderBufferBuilder.resizeBuffer(this.vbosTransparent, builder.getCurrentNeededTransparentVertexBufferCount());
-		for (int i = 0; i < this.vbosTransparent.length; i++)
 		{
-			if (this.vbosTransparent[i] == null)
+			SharedVbo.TRANSPARENT.deallocate(this.transparentBufferSlices);
-			{
+			this.transparentBufferSlices = new SharedVbo.BufferSlice[transparentSliceCount];
-				this.vbosTransparent[i] = new GLVertexBuffer(method.useBufferStorage);
-			}
-		}
-		LodQuadBuilder.BufferFiller transparentFillerFunc = builder.makeTransparentBufferFiller(method);
-		for (GLVertexBuffer vbo : this.vbosTransparent)
-		{
-			transparentFillerFunc.fill(vbo);
 		}
+		uploadBuffersDirect(SharedVbo.TRANSPARENT, this.transparentBufferSlices, builder.makeTransparentVertexBuffers());
 	}
 	
-	private void uploadBuffersDirect(LodQuadBuilder builder, EDhApiGpuUploadMethod method) throws InterruptedException
+	private static void uploadBuffersDirect(SharedVbo handler, SharedVbo.BufferSlice[] bufferSlices, Iterator<ByteBuffer> iter)
 	{
-		this.vbos = ColumnRenderBufferBuilder.resizeBuffer(this.vbos, builder.getCurrentNeededOpaqueVertexBufferCount());
+		int i = 0;
-		uploadBuffersDirect(this.vbos, builder.makeOpaqueVertexBuffers(), method);
-		
-		this.vbosTransparent = ColumnRenderBufferBuilder.resizeBuffer(this.vbosTransparent, builder.getCurrentNeededTransparentVertexBufferCount());
-		uploadBuffersDirect(this.vbosTransparent, builder.makeTransparentVertexBuffers(), method);
-	}
-	private static void uploadBuffersDirect(GLVertexBuffer[] vbos, Iterator<ByteBuffer> iter, EDhApiGpuUploadMethod method) throws InterruptedException
-	{
-		long remainingMS = 0;
-		long MBPerMS = Config.Client.Advanced.GpuBuffers.gpuUploadPerMegabyteInMilliseconds.get();
-		int vboIndex = 0;
 		while (iter.hasNext())
 		{
-			if (vboIndex >= vbos.length)
+			if (i >= bufferSlices.length)
 			{
 				throw new RuntimeException("Too many vertex buffers!!");
 			}
 			
-			
+			// get or create the buffer
-			// get or create the VBO
+			if (bufferSlices[i] == null)
-			if (vbos[vboIndex] == null)
 			{
-				vbos[vboIndex] = new GLVertexBuffer(method.useBufferStorage);
+				bufferSlices[i] = handler.allocateSlice();
 			}
-			GLVertexBuffer vbo = vbos[vboIndex];
+			SharedVbo.BufferSlice slice = bufferSlices[i];
 			
 			
-			ByteBuffer bb = iter.next();
+			ByteBuffer buffer = iter.next();
-			int size = bb.limit() - bb.position();
-			
 			try
 			{
-				vbo.bind();
+				handler.updateBuffer(slice, buffer);
-				vbo.uploadBuffer(bb, size / LodUtil.LOD_VERTEX_FORMAT.getByteSize(), method, FULL_SIZED_BUFFER);
 			}
 			catch (Exception e)
 			{
-				vbos[vboIndex] = null;
+				bufferSlices[i] = null;
-				vbo.close();
+				handler.deallocate(slice);
-				LOGGER.error("Failed to upload buffer: ", e);
+				LOGGER.error("Failed to upload buffer. Error: ["+e.getMessage()+"].", e);
 			}
 			
-			
+			i++;
-			if (MBPerMS > 0)
-			{
-				// upload buffers over an extended period of time
-				// to hopefully prevent stuttering.
-				remainingMS += size * MBPerMS;
-				if (remainingMS >= TimeUnit.NANOSECONDS.convert(1000 / 60, TimeUnit.MILLISECONDS))
-				{
-					if (remainingMS > MAX_BUFFER_UPLOAD_TIMEOUT_NANOSECONDS)
-					{
-						remainingMS = MAX_BUFFER_UPLOAD_TIMEOUT_NANOSECONDS;
-					}
-					
-					Thread.sleep(remainingMS / 1000000, (int) (remainingMS % 1000000));
-					remainingMS = 0;
-				}
-			}
-			
-			vboIndex++;
 		}
 		
-		if (vboIndex < vbos.length)
+		if (i < bufferSlices.length)
 		{
-			throw new RuntimeException("Too few vertex buffers!!");
+			throw new RuntimeException("Too few buffer chunks!");
 		}
 	}
 	
 	
 	
-	
-	
 	//========//
 	// render //
 	//========//
 	
-	/** @return true if something was rendered, false otherwise */
+	//public void renderOpaque(LodRenderer renderContext, DhApiRenderParam renderEventParam)
-	public boolean renderOpaque(LodRenderer renderContext, DhApiRenderParam renderEventParam)
+	//{
-	{
+	//	renderContext.setModelViewMatrixOffset(this.pos, renderEventParam);
-		boolean hasRendered = false;
+	//	renderContext.drawSharedVbo(SharedVbo.OPAQUE, this.opaqueBufferSlices);
-		renderContext.setModelViewMatrixOffset(this.pos, renderEventParam);
+	//}
-		for (GLVertexBuffer vbo : this.vbos)
+	//
-		{
+	//public void renderTransparent(LodRenderer renderContext, DhApiRenderParam renderEventParam)
-			if (vbo == null)
+	//{
-			{
+	//	renderContext.setModelViewMatrixOffset(this.pos, renderEventParam);
-				continue;
+	//	renderContext.drawSharedVbo(SharedVbo.TRANSPARENT, this.transparentBufferSlices);
-			}
+	//}
-			
-			if (vbo.getVertexCount() == 0)
-			{
-				continue;
-			}
-			
-			hasRendered = true;
-			renderContext.drawVbo(vbo);
-			//LodRenderer.tickLogger.info("Vertex buffer: {}", vbo);
-		}
-		return hasRendered;
-	}
-	
-	/** @return true if something was rendered, false otherwise */
-	public boolean renderTransparent(LodRenderer renderContext, DhApiRenderParam renderEventParam)
-	{
-		boolean hasRendered = false;
-		
-		try
-		{
-			// can throw an IllegalStateException if the GL program was freed before it should've been
-			renderContext.setModelViewMatrixOffset(this.pos, renderEventParam);
-			
-			for (GLVertexBuffer vbo : this.vbosTransparent)
-			{
-				if (vbo == null)
-				{
-					continue;
-				}
-				
-				if (vbo.getVertexCount() == 0)
-				{
-					continue;
-				}
-				
-				hasRendered = true;
-				renderContext.drawVbo(vbo);
-				//LodRenderer.tickLogger.info("Vertex buffer: {}", vbo);
-			}
-		}
-		catch (IllegalStateException e)
-		{
-			LOGGER.error("renderContext program doesn't exist for pos: "+this.pos, e);
-		}
-		
-		return hasRendered;
-	}
 	
 	
 	
@@ -322,50 +254,28 @@ public class ColumnRenderBuffer implements AutoCloseable
 	// misc methods //
 	//==============//
 	
+	// TODO
 	/** can be used when debugging */
-	public boolean hasNonNullVbos() { return this.vbos != null || this.vbosTransparent != null; }
+	public boolean hasNonNullVbos() { return false; } //this.vbos != null || this.vbosTransparent != null; }
 	
 	/** can be used when debugging */
-	public int vboBufferCount() 
+	public int bufferSliceCount() 
 	{
 		int count = 0;
 		
-		if (this.vbos != null)
+		if (this.opaqueBufferSlices != null)
 		{
-			count += this.vbos.length;
+			count += this.opaqueBufferSlices.length;
 		}
-		
+
-		if (this.vbosTransparent != null)
+		if (this.transparentBufferSlices != null)
 		{
-			count += this.vbosTransparent.length;
+			count += this.transparentBufferSlices.length;
 		}
 		
 		return count;
 	}
 	
-	public void debugDumpStats(StatsMap statsMap)
-	{
-		statsMap.incStat("RenderBuffers");
-		statsMap.incStat("SimpleRenderBuffers");
-		for (GLVertexBuffer vertexBuffer : vbos)
-		{
-			if (vertexBuffer != null)
-			{
-				statsMap.incStat("VBOs");
-				if (vertexBuffer.getSize() == FULL_SIZED_BUFFER)
-				{
-					statsMap.incStat("FullsizedVBOs");
-				}
-				
-				if (vertexBuffer.getSize() == 0)
-				{
-					GLProxy.GL_LOGGER.warn("VBO with size 0");
-				}
-				statsMap.incBytesStat("TotalUsage", vertexBuffer.getSize());
-			}
-		}
-	}
-	
 	/**
 	 * This method is called when object is no longer in use.
 	 * Called either after uploadBuffers() returned false (On buffer Upload
@@ -377,24 +287,8 @@ public class ColumnRenderBuffer implements AutoCloseable
 	{
 		this.buffersUploaded = false;
 		
-		GLProxy.getInstance().queueRunningOnRenderThread(() ->
+		SharedVbo.OPAQUE.deallocate(this.opaqueBufferSlices);
-		{
+		SharedVbo.TRANSPARENT.deallocate(this.transparentBufferSlices);
-			for (GLVertexBuffer buffer : this.vbos)
-			{
-				if (buffer != null)
-				{
-					buffer.destroyAsync();
-				}
-			}
-			
-			for (GLVertexBuffer buffer : this.vbosTransparent)
-			{
-				if (buffer != null)
-				{
-					buffer.destroyAsync();
-				}
-			}
-		});
 	}
 	
 }

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/ColumnRenderBufferBuilder.java

@@ -333,32 +333,4 @@ public class ColumnRenderBufferBuilder
 		quadBuilder.finalizeData();
 	}
 	
-	
-	
-	//=================//
-	// vbo interaction //
-	//=================//
-	
-	public static GLVertexBuffer[] resizeBuffer(GLVertexBuffer[] vbos, int newSize)
-	{
-		if (vbos.length == newSize)
-		{
-			return vbos;
-		}
-		
-		GLVertexBuffer[] newVbos = new GLVertexBuffer[newSize];
-		System.arraycopy(vbos, 0, newVbos, 0, Math.min(vbos.length, newSize));
-		if (newSize < vbos.length)
-		{
-			for (int i = newSize; i < vbos.length; i++)
-			{
-				if (vbos[i] != null)
-				{
-					vbos[i].close();
-				}
-			}
-		}
-		return newVbos;
-	}
-	
 }

core/src/main/java/com/seibel/distanthorizons/core/dataObjects/render/bufferBuilding/SharedVbo.java

@@ -0,0 +1,188 @@
+package com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding;
+
+import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
+import com.seibel.distanthorizons.core.logging.f3.F3Screen;
+import com.seibel.distanthorizons.core.render.glObject.GLProxy;
+import com.seibel.distanthorizons.core.util.LodUtil;
+import com.seibel.distanthorizons.coreapi.util.StringUtil;
+import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.Nullable;
+import org.lwjgl.opengl.GL32;
+
+import java.nio.ByteBuffer;
+import java.util.ArrayDeque;
+import java.util.Queue;
+import java.util.concurrent.ConcurrentHashMap;
+
+/** Used to allow multiple {@link ColumnRenderBuffer}'s to be put in a single VBO. */
+public class SharedVbo 
+{
+	private static final Logger LOGGER = DhLoggerBuilder.getLogger();
+	
+	public static final SharedVbo OPAQUE = new SharedVbo(2_000_000_000L/*1GB*/, 1024 * 1024/*1MB*/);
+	public static final SharedVbo TRANSPARENT = new SharedVbo(2_000_000_000L/*1GB*/, 1024 * 1024/*1MB*/);
+	
+	
+	public final int vboId;
+	
+	
+	private final long bufferTotalByteSize;
+	/** the length of a single chunk of this VBO in bytes. */
+	private final int chunkByteSize;
+	
+	private final ConcurrentHashMap<Long, BufferSlice> bufferSliceByStartingIndex = new ConcurrentHashMap<>();
+	private long nextMemoryAddress = 0L;
+	private final Queue<BufferSlice> availableSlices = new ArrayDeque<>();
+	
+	
+	
+	//=============//
+	// constructor //
+	//=============//
+	
+	public SharedVbo(long totalSize, int chunkByteSize) 
+	{
+		LodUtil.assertTrue(GLProxy.getInstance().runningOnRenderThread(), "Buffer Handler has to be created on the render thread.");
+		
+		this.bufferTotalByteSize = totalSize;
+		this.chunkByteSize = chunkByteSize;
+		
+		// Generate and bind the VBO
+		this.vboId = GL32.glGenBuffers();
+		GL32.glBindBuffer(GL32.GL_ARRAY_BUFFER, this.vboId);
+		GL32.glBufferData(GL32.GL_ARRAY_BUFFER, this.bufferTotalByteSize, GL32.GL_DYNAMIC_DRAW);
+		GL32.glBindBuffer(GL32.GL_ARRAY_BUFFER, 0);
+	}
+	
+	
+	
+	//============//
+	// allocation //
+	//============//
+	
+	@Nullable
+	public SharedVbo.BufferSlice allocateSlice()
+	{
+		BufferSlice availableSlice = this.availableSlices.poll();
+		if (availableSlice != null)
+		{
+			return availableSlice;
+		}
+		
+		// Find the first free chunk
+		for (long startingIndex = this.nextMemoryAddress; startingIndex < this.bufferTotalByteSize; startingIndex += this.chunkByteSize)
+		{
+			// check if this section is free
+			BufferSlice newSlice = new BufferSlice(startingIndex, this.chunkByteSize);
+			if (this.bufferSliceByStartingIndex.putIfAbsent(startingIndex, newSlice) == null)
+			{
+				this.nextMemoryAddress = startingIndex;
+				return newSlice;
+			}
+		}
+		
+		return null; // No free chunk found
+	}
+	public void deallocate(BufferSlice[] slices)
+	{
+		if (slices != null)
+		{
+			for (BufferSlice slice : slices)
+			{
+				if (slice != null)
+				{
+					this.bufferSliceByStartingIndex.remove(slice.startIndex);
+					this.availableSlices.add(slice);
+				}
+			}
+		}
+	}
+	public void deallocate(BufferSlice slice) 
+	{
+		if (slice != null)
+		{
+			this.bufferSliceByStartingIndex.remove(slice.startIndex);
+			this.availableSlices.add(slice);
+		}
+	}
+	public void clear() 
+	{
+		this.bufferSliceByStartingIndex.clear(); 
+	}
+	
+	
+	
+	//=================//
+	// buffer handling //
+	//=================//
+	
+	public void updateBuffer(BufferSlice chunk, ByteBuffer buffer) 
+	{
+		int size = buffer.limit() - buffer.position();
+		if (size > chunk.length)
+		{
+			// if this was fired that means we didn't split up the buffer into the right size
+			// if this isn't stopped the buffer will overwrite an adjacent section and cause incorrect rendering
+			throw new RuntimeException("Programmer error: Uploaded buffer bigger than the allocated area. Allocated: ["+chunk.length+"], buffer: ["+size+"]");
+		}
+		
+		GL32.glBindBuffer(GL32.GL_ARRAY_BUFFER, this.vboId);
+		GL32.glBufferSubData(GL32.GL_ARRAY_BUFFER, chunk.startIndex, buffer);
+		GL32.glBindBuffer(GL32.GL_ARRAY_BUFFER, 0);
+		chunk.vertexCount = size / LodUtil.LOD_VERTEX_FORMAT.getByteSize();
+	}
+	
+	public void bind() { GL32.glBindBuffer(GL32.GL_ARRAY_BUFFER, this.vboId); }
+	public void unbind() { GL32.glBindBuffer(GL32.GL_ARRAY_BUFFER, 0); }
+	
+	
+	
+	//=======//
+	// debug //
+	//=======//
+	
+	public String getDebugMenuString()
+	{
+		long maxChunkCount = (this.bufferTotalByteSize / this.chunkByteSize);
+		long chunkCount = 0;
+		long allocatedBytes = 0;
+		
+		for (BufferSlice slice : this.bufferSliceByStartingIndex.values())
+		{
+			allocatedBytes += slice.length;
+			chunkCount++;
+		}
+		
+		return "Slices: ["+F3Screen.NUMBER_FORMAT.format(chunkCount)+"/"+F3Screen.NUMBER_FORMAT.format(maxChunkCount)+"], " +
+				"Mem: ["+StringUtil.convertByteCountToHumanReadableSI(allocatedBytes)+"/"+StringUtil.convertByteCountToHumanReadableSI(this.bufferTotalByteSize)+"]";
+	}
+	
+	
+	
+	//================//
+	// helper classes //
+	//================//
+	
+	/** represents a single allocated slice of the parent VBO */
+	public static class BufferSlice 
+	{
+		public final long startIndex;
+		public final int length;
+		
+		public int vertexCount;
+		
+		
+		
+		//=============//
+		// constructor //
+		//=============//
+		
+		public BufferSlice(long startIndex, int length) 
+		{
+			this.startIndex = startIndex;
+			this.length = length;
+		}
+		
+	}
+	
+}

core/src/main/java/com/seibel/distanthorizons/core/logging/f3/F3Screen.java

@@ -20,6 +20,7 @@
 package com.seibel.distanthorizons.core.logging.f3;
 
 import com.seibel.distanthorizons.core.api.internal.SharedApi;
+import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.SharedVbo;
 import com.seibel.distanthorizons.core.level.IDhLevel;
 import com.seibel.distanthorizons.core.render.RenderBufferHandler;
 import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer;
@@ -111,6 +112,11 @@ public class F3Screen
 				messageList.add(genericRenderer.getVboRenderDebugMenuString());
 			}
 		}
+		messageList.add("");
+		// GPU memory
+		messageList.add("GPU Opaque " + SharedVbo.OPAQUE.getDebugMenuString());
+		messageList.add("GPU Transp " + SharedVbo.TRANSPARENT.getDebugMenuString());
+		messageList.add("");
 	}
 	
 	

core/src/main/java/com/seibel/distanthorizons/core/render/LodQuadTree.java

@@ -31,7 +31,6 @@ import com.seibel.distanthorizons.core.pos.DhBlockPos2D;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.render.renderer.DebugRenderer;
 import com.seibel.distanthorizons.core.render.renderer.IDebugRenderable;
-import com.seibel.distanthorizons.core.sql.repo.BeaconBeamRepo;
 import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.util.ThreadUtil;
 import com.seibel.distanthorizons.core.util.objects.quadTree.QuadNode;
@@ -647,7 +646,7 @@ public class LodQuadTree extends QuadTree<LodRenderSection> implements IDebugRen
 				}
 				else if (renderSection.renderBuffer.hasNonNullVbos())
 				{
-					if (renderSection.renderBuffer.vboBufferCount() != 0)
+					if (renderSection.renderBuffer.bufferSliceCount() != 0)
 					{
 						color = Color.GREEN;
 					}

core/src/main/java/com/seibel/distanthorizons/core/render/RenderBufferHandler.java

@@ -25,11 +25,13 @@ import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiShadow
 import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam;
 import com.seibel.distanthorizons.core.config.Config;
 import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer;
+import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.SharedVbo;
 import com.seibel.distanthorizons.core.dependencyInjection.ModAccessorInjector;
 import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
 import com.seibel.distanthorizons.core.enums.EDhDirection;
 import com.seibel.distanthorizons.core.logging.DhLoggerBuilder;
 import com.seibel.distanthorizons.core.logging.f3.F3Screen;
+import com.seibel.distanthorizons.core.pos.DhBlockPos;
 import com.seibel.distanthorizons.core.pos.DhLodPos;
 import com.seibel.distanthorizons.core.pos.DhSectionPos;
 import com.seibel.distanthorizons.core.pos.Pos2D;
@@ -48,9 +50,7 @@ import org.apache.logging.log4j.Logger;
 import org.joml.Matrix4f;
 import org.joml.Matrix4fc;
 
-import java.util.Comparator;
+import java.util.*;
-import java.util.Iterator;
-import java.util.ListIterator;
 import java.util.concurrent.atomic.AtomicBoolean;
 
 /**
@@ -348,7 +348,11 @@ public class RenderBufferHandler implements AutoCloseable
 		// TODO why can these sometimes be null when teleporting between multiverses
 		if (this.loadedNearToFarBuffers != null)
 		{
-			this.loadedNearToFarBuffers.forEach(loadedBuffer -> loadedBuffer.buffer.renderOpaque(renderContext, renderEventParam));
+			renderContext.setModelViewMatrixOffset(DhBlockPos.ZERO, renderEventParam);
+			
+			ArrayList<SharedVbo.BufferSlice> sliceList = new ArrayList<>();
+			this.loadedNearToFarBuffers.forEach(loadedBuffer -> sliceList.addAll(Arrays.asList(loadedBuffer.buffer.opaqueBufferSlices)));
+			renderContext.drawSharedVbo(SharedVbo.OPAQUE, sliceList.toArray(new SharedVbo.BufferSlice[0]));
 		}
 	}
 	public void renderTransparent(LodRenderer renderContext, DhApiRenderParam renderEventParam)
@@ -356,12 +360,11 @@ public class RenderBufferHandler implements AutoCloseable
 		// TODO why can these sometimes be null when teleporting between multiverses
 		if (this.loadedNearToFarBuffers != null)
 		{
-			ListIterator<LoadedRenderBuffer> iter = this.loadedNearToFarBuffers.listIterator(this.loadedNearToFarBuffers.size());
+			renderContext.setModelViewMatrixOffset(DhBlockPos.ZERO, renderEventParam);
-			while (iter.hasPrevious())
+			
-			{
+			ArrayList<SharedVbo.BufferSlice> sliceList = new ArrayList<>();
-				LoadedRenderBuffer loadedBuffer = iter.previous();
+			this.loadedNearToFarBuffers.forEach(loadedBuffer -> sliceList.addAll(Arrays.asList(loadedBuffer.buffer.transparentBufferSlices)));
-				loadedBuffer.buffer.renderTransparent(renderContext, renderEventParam);
+			renderContext.drawSharedVbo(SharedVbo.TRANSPARENT, sliceList.toArray(new SharedVbo.BufferSlice[0]));
-			}
 		}
 	}
 	

core/src/main/java/com/seibel/distanthorizons/core/render/renderer/LodRenderer.java

@@ -25,6 +25,7 @@ import com.seibel.distanthorizons.api.interfaces.override.rendering.IDhApiShader
 import com.seibel.distanthorizons.api.methods.events.abstractEvents.*;
 import com.seibel.distanthorizons.api.methods.events.sharedParameterObjects.DhApiRenderParam;
 import com.seibel.distanthorizons.core.config.Config;
+import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.SharedVbo;
 import com.seibel.distanthorizons.core.dataObjects.render.bufferBuilding.ColumnRenderBuffer;
 import com.seibel.distanthorizons.core.dependencyInjection.ModAccessorInjector;
 import com.seibel.distanthorizons.core.dependencyInjection.SingletonInjector;
@@ -35,11 +36,11 @@ import com.seibel.distanthorizons.core.render.DhApiRenderProxy;
 import com.seibel.distanthorizons.core.render.RenderBufferHandler;
 import com.seibel.distanthorizons.core.render.glObject.GLProxy;
 import com.seibel.distanthorizons.core.render.glObject.GLState;
-import com.seibel.distanthorizons.core.render.glObject.buffer.GLVertexBuffer;
 import com.seibel.distanthorizons.core.render.glObject.buffer.QuadElementBuffer;
 import com.seibel.distanthorizons.core.render.glObject.texture.*;
 import com.seibel.distanthorizons.core.render.renderer.generic.GenericObjectRenderer;
 import com.seibel.distanthorizons.core.render.renderer.shaders.*;
+import com.seibel.distanthorizons.core.util.LodUtil;
 import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftClientWrapper;
 import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IMinecraftRenderWrapper;
 import com.seibel.distanthorizons.core.wrapperInterfaces.minecraft.IProfilerWrapper;
@@ -54,10 +55,13 @@ import com.seibel.distanthorizons.coreapi.DependencyInjection.OverrideInjector;
 import com.seibel.distanthorizons.core.util.math.Mat4f;
 import com.seibel.distanthorizons.core.util.math.Vec3d;
 import com.seibel.distanthorizons.core.util.math.Vec3f;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
 import org.apache.logging.log4j.LogManager;
+import org.lwjgl.PointerBuffer;
 import org.lwjgl.opengl.GL32;
 
 import java.awt.*;
+import java.nio.IntBuffer;
 import java.time.Duration;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.locks.ReentrantLock;
@@ -301,8 +305,11 @@ public class LodRenderer
 				// terrain
 				profiler.popPush("LOD Opaque");
 				ApiEventInjector.INSTANCE.fireAllEvents(DhApiBeforeRenderPassEvent.class, renderEventParam);
+				
 				this.bufferHandler.renderOpaque(this, renderEventParam);
 				
+				
+				
 				// custom objects with SSAO
 				if (Config.Client.Advanced.Graphics.GenericRendering.enableRendering.get())
 				{
@@ -466,6 +473,7 @@ public class LodRenderer
 		GL32.glBlendEquation(GL32.GL_FUNC_ADD);
 		GL32.glBlendFuncSeparate(GL32.GL_SRC_ALPHA, GL32.GL_ONE_MINUS_SRC_ALPHA, GL32.GL_ONE, GL32.GL_ONE_MINUS_SRC_ALPHA);
 		ApiEventInjector.INSTANCE.fireAllEvents(DhApiBeforeRenderPassEvent.class, renderEventParam);
+		
 		this.bufferHandler.renderTransparent(this, renderEventParam);
 		GL32.glDepthMask(true); // Apparently the depth mask state is stored in the FBO, so glState fails to restore it...
 		
@@ -496,31 +504,47 @@ public class LodRenderer
 		ApiEventInjector.INSTANCE.fireAllEvents(DhApiBeforeBufferRenderEvent.class, new DhApiBeforeBufferRenderEvent.EventParam(renderEventParam, modelPos));
 	}
 	
-	public void drawVbo(GLVertexBuffer vbo)
+	public void drawSharedVbo(SharedVbo sharedVbo, SharedVbo.BufferSlice[] slices)
 	{
-		//// can be uncommented to add additional debug validation to prevent crashes if invalid buffers are being created
+		IDhApiShaderProgram shaderProgram = this.lodRenderProgram;
-		//// shouldn't be used in production due to the performance hit
+		IDhApiShaderProgram shaderProgramOverride = OverrideInjector.INSTANCE.get(IDhApiShaderProgram.class);
-		//if (GL32.glIsBuffer(vbo.getId()))
+		if (shaderProgramOverride != null && shaderProgram.overrideThisFrame())
 		{
-			IDhApiShaderProgram shaderProgram = this.lodRenderProgram;
+			shaderProgram = shaderProgramOverride;
-			IDhApiShaderProgram shaderProgramOverride = OverrideInjector.INSTANCE.get(IDhApiShaderProgram.class);
-			if (shaderProgramOverride != null && shaderProgram.overrideThisFrame())
-			{
-				shaderProgram = shaderProgramOverride;
-			}
-			
-			
-			vbo.bind();
-			shaderProgram.bindVertexBuffer(vbo.getId());
-			GL32.glDrawElements(GL32.GL_TRIANGLES, (vbo.getVertexCount() / 4) * 6, // TODO what does the 4 and 6 here represent?
-					this.quadIBO.getType(), 0);
-			vbo.unbind();
 		}
-		//else
+		
-		//{
+		
-		//	// will spam the log if uncommented, but helpful for validation
+		sharedVbo.bind();
-		//	//LOGGER.warn("Unable to draw VBO: "+vbo.getId());
+		shaderProgram.bindVertexBuffer(sharedVbo.vboId);
-		//}
+		
+		IntArrayList countList = new IntArrayList();
+		IntArrayList baseVertexList = new IntArrayList();
+		for (int i = 0; i < slices.length; i++)
+		{
+			SharedVbo.BufferSlice slice = slices[i];
+			if (slice != null && slice.vertexCount != 0)
+			{
+				countList.add((slice.vertexCount / 4) * 6); // 4 vertices per quad, 6 indices per quad
+				baseVertexList.add((int)slice.startIndex / LodUtil.LOD_VERTEX_FORMAT.getByteSize());
+			}
+		}
+		
+		int[] counts = countList.toIntArray();
+		PointerBuffer iboIndicies = PointerBuffer.allocateDirect(counts.length);
+		for (int i = 0; i < counts.length; i++)
+		{
+			iboIndicies.put(0L);
+		}
+		iboIndicies.rewind();
+		int[] baseVertecies = baseVertexList.toIntArray();
+		
+		
+		if (counts.length != 0)
+		{
+			GL32.glMultiDrawElementsBaseVertex(GL32.GL_TRIANGLES, counts, this.quadIBO.getType(), iboIndicies, baseVertecies);
+		}
+		
+		sharedVbo.unbind();
 	}
 	
 	

core/src/test/java/tests/CompressionTest.java

@@ -23,12 +23,11 @@ import com.seibel.distanthorizons.api.enums.config.EDhApiDataCompressionMode;
 import com.seibel.distanthorizons.core.dataObjects.fullData.sources.FullDataSourceV2;
 import com.seibel.distanthorizons.core.sql.dto.FullDataSourceV2DTO;
 import com.seibel.distanthorizons.core.sql.repo.FullDataSourceV2Repo;
+import com.seibel.distanthorizons.coreapi.util.StringUtil;
 import it.unimi.dsi.fastutil.longs.LongArrayList;
 import org.junit.Assert;
 
 import java.io.*;
-import java.text.CharacterIterator;
-import java.text.StringCharacterIterator;
 
 /**
  * <strong>Note:</strong>
@@ -356,10 +355,10 @@ public class CompressionTest
 			System.out.println("\n");
 			System.out.println("Results: " + compressorName);
 			System.out.println();
-			System.out.println("Total uncompressed data: [" + humanReadableByteCountSI(totalUncompressedFileSizeInBytes) + "] Total compressed data: [" + humanReadableByteCountSI(totalCompressedFileSizeInBytes) + "]. Compression ratio: [" + compressionRatioString + "].");
+			System.out.println("Total uncompressed data: [" + StringUtil.convertByteCountToHumanReadableSI(totalUncompressedFileSizeInBytes) + "] Total compressed data: [" + StringUtil.convertByteCountToHumanReadableSI(totalCompressedFileSizeInBytes) + "]. Compression ratio: [" + compressionRatioString + "].");
-			System.out.println("Min uncompressed data: [" + humanReadableByteCountSI(minUncompressedDtoSizeInBytes) + "] Min compressed data: [" + humanReadableByteCountSI(minCompressedDtoSizeInBytes) + "].");
+			System.out.println("Min uncompressed data: [" + StringUtil.convertByteCountToHumanReadableSI(minUncompressedDtoSizeInBytes) + "] Min compressed data: [" + StringUtil.convertByteCountToHumanReadableSI(minCompressedDtoSizeInBytes) + "].");
-			System.out.println("Max uncompressed data: [" + humanReadableByteCountSI(maxUncompressedDtoSizeInBytes) + "] Max compressed data: [" + humanReadableByteCountSI(maxCompressedDtoSizeInBytes) + "].");
+			System.out.println("Max uncompressed data: [" + StringUtil.convertByteCountToHumanReadableSI(maxUncompressedDtoSizeInBytes) + "] Max compressed data: [" + StringUtil.convertByteCountToHumanReadableSI(maxCompressedDtoSizeInBytes) + "].");
-			System.out.println("Avg uncompressed data: [" + humanReadableByteCountSI(avgUncompressedDtoSizeInBytes) + "] Avg compressed data: [" + humanReadableByteCountSI(avgCompressedDtoSizeInBytes) + "].");
+			System.out.println("Avg uncompressed data: [" + StringUtil.convertByteCountToHumanReadableSI(avgUncompressedDtoSizeInBytes) + "] Avg compressed data: [" + StringUtil.convertByteCountToHumanReadableSI(avgCompressedDtoSizeInBytes) + "].");
 			System.out.println();
 			System.out.println("Total read time in MS: [" + totalReadTimeInNano / 1_000_000.0 + "] Average read time per dto: [" + (totalReadTimeInNano / processedDtoCount) / 1_000_000.0 + "]");
 			System.out.println("Total write time in MS: [" + totalWriteTimeInNano / 1_000_000.0 + "] Average write time per dto: [" + (totalWriteTimeInNano / processedDtoCount) / 1_000_000.0 + "]");
@@ -373,23 +372,4 @@ public class CompressionTest
 	}
 	
 	
-	/**
-	 * Source:
-	 * https://stackoverflow.com/questions/3758606/how-can-i-convert-byte-size-into-a-human-readable-format-in-java#3758880
-	 */
-	public static String humanReadableByteCountSI(long bytes)
-	{
-		if (-1000 < bytes && bytes < 1000)
-		{
-			return bytes + " B";
-		}
-		CharacterIterator ci = new StringCharacterIterator("kMGTPE");
-		while (bytes <= -999_950 || bytes >= 999_950)
-		{
-			bytes /= 1000;
-			ci.next();
-		}
-		return String.format("%.1f %cB", bytes / 1000.0, ci.current());
-	}
-	
 }